@query-farm/vgi 0.37.1 → 0.38.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +74 -0
- package/dist/catalog/composite.d.ts +23 -1
- package/dist/catalog/contents-digest.d.ts +25 -0
- package/dist/catalog/interface.d.ts +107 -2
- package/dist/catalog/read-only.d.ts +45 -1
- package/dist/client/client.d.ts +19 -1
- package/dist/client-entry.js +141 -4
- package/dist/client-entry.js.map +9 -8
- package/dist/codec/asd.d.ts +8 -0
- package/dist/generated/vgi-client.d.ts +19 -0
- package/dist/generated/vgi-protocol-schemas.d.ts +3 -0
- package/dist/generated/vgi-protocol-types.d.ts +26 -0
- package/dist/http/fetch.d.ts +2 -1
- package/dist/index.core.d.ts +5 -2
- package/dist/index.js +574 -36
- package/dist/index.js.map +20 -16
- package/dist/protocol/handlers/catalog/contents.d.ts +27 -0
- package/dist/rpc-server.d.ts +97 -0
- package/dist/serve-entry.d.ts +2 -1
- package/dist/serve-entry.js +405 -25
- package/dist/serve-entry.js.map +18 -14
- package/dist/worker-cf-entry.d.ts +3 -0
- package/dist/worker-cf-entry.js +528 -31
- package/dist/worker-cf-entry.js.map +20 -16
- package/dist/worker.d.ts +9 -1
- package/package.json +3 -3
- package/src/catalog/composite.ts +81 -0
- package/src/catalog/contents-digest.ts +78 -0
- package/src/catalog/interface.ts +167 -5
- package/src/catalog/read-only.ts +68 -6
- package/src/client/client.ts +53 -0
- package/src/codec/asd.ts +96 -1
- package/src/generated/vgi-client.ts +22 -1
- package/src/generated/vgi-protocol-schemas.ts +30 -1
- package/src/generated/vgi-protocol-types.ts +126 -0
- package/src/http/fetch.ts +10 -2
- package/src/index.core.ts +21 -2
- package/src/protocol/dispatch.ts +1 -1
- package/src/protocol/handlers/catalog/admin.ts +35 -26
- package/src/protocol/handlers/catalog/contents.ts +162 -0
- package/src/protocol/handlers/catalog/macro.ts +2 -1
- package/src/protocol/handlers/catalog/shared.ts +12 -1
- package/src/rpc-server.ts +273 -0
- package/src/serve-entry.ts +7 -1
- package/src/worker-cf-entry.ts +8 -0
- package/src/worker.ts +24 -6
package/src/codec/asd.ts
CHANGED
|
@@ -40,7 +40,102 @@ export function encodeASD(
|
|
|
40
40
|
schema: VgiSchema,
|
|
41
41
|
obj: Record<string, any>,
|
|
42
42
|
): Uint8Array {
|
|
43
|
-
return serializeBatch(batchFromRows([obj], schema));
|
|
43
|
+
return serializeBatch(batchFromRows([sortMapsForSchema(schema, obj)], schema));
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
// --------------------------------------------------------------------------- //
|
|
47
|
+
// Encode side: deterministic map columns
|
|
48
|
+
// --------------------------------------------------------------------------- //
|
|
49
|
+
//
|
|
50
|
+
// A map value arrives as a JS object (or Map / pair list), whose iteration
|
|
51
|
+
// order is insertion order -- except that integer-like keys ("1", "2") always
|
|
52
|
+
// come first, ascending. Two builds of the same record could therefore encode
|
|
53
|
+
// differently (and a catalog_contents content-hash etag would change with no
|
|
54
|
+
// catalog change). Every map is written in sorted key order instead, matching
|
|
55
|
+
// the Go / Java / C# / Rust SDKs: keys compared by Unicode code point (the
|
|
56
|
+
// same order as comparing their UTF-8 bytes), not by UTF-16 code unit.
|
|
57
|
+
|
|
58
|
+
/** Compare two strings by Unicode code point (= UTF-8 byte order). */
|
|
59
|
+
export function compareCodePoints(a: string, b: string): number {
|
|
60
|
+
const n = Math.min(a.length, b.length);
|
|
61
|
+
for (let i = 0; i < n; i++) {
|
|
62
|
+
const x = a.charCodeAt(i);
|
|
63
|
+
const y = b.charCodeAt(i);
|
|
64
|
+
if (x === y) continue;
|
|
65
|
+
// A surrogate (a code point above U+FFFF) sorts after every BMP code unit,
|
|
66
|
+
// though as a code unit (0xD800-0xDFFF) it sorts below U+E000-U+FFFF.
|
|
67
|
+
const xs = x >= 0xd800 && x <= 0xdfff;
|
|
68
|
+
const ys = y >= 0xd800 && y <= 0xdfff;
|
|
69
|
+
if (xs !== ys) return xs ? 1 : -1;
|
|
70
|
+
return x - y;
|
|
71
|
+
}
|
|
72
|
+
return a.length - b.length;
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
function mapPairs(raw: any): Array<[unknown, unknown]> {
|
|
76
|
+
if (Array.isArray(raw)) return raw.map((e: any) => (Array.isArray(e) ? [e[0], e[1]] : [e?.key, e?.value]));
|
|
77
|
+
if (raw instanceof Map) return Array.from(raw.entries());
|
|
78
|
+
if (typeof raw[Symbol.iterator] === "function") return Array.from(raw as Iterable<[unknown, unknown]>);
|
|
79
|
+
return Object.entries(raw);
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
function compareKeys(a: unknown, b: unknown): number {
|
|
83
|
+
if (typeof a === "string" && typeof b === "string") return compareCodePoints(a, b);
|
|
84
|
+
if ((typeof a === "number" || typeof a === "bigint") && (typeof b === "number" || typeof b === "bigint")) {
|
|
85
|
+
return a < b ? -1 : a > b ? 1 : 0;
|
|
86
|
+
}
|
|
87
|
+
return compareCodePoints(String(a), String(b));
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
/** Whether a type contains a map anywhere (memoized per type object). */
|
|
91
|
+
const HAS_MAP = new WeakMap<object, boolean>();
|
|
92
|
+
function typeHasMap(type: VgiDataType): boolean {
|
|
93
|
+
const cached = HAS_MAP.get(type as object);
|
|
94
|
+
if (cached !== undefined) return cached;
|
|
95
|
+
let has = isMap(type);
|
|
96
|
+
if (!has && (isList(type) || isStruct(type))) {
|
|
97
|
+
has = ((type as any).children as VgiField[]).some((c) => typeHasMap(c.type));
|
|
98
|
+
}
|
|
99
|
+
HAS_MAP.set(type as object, has);
|
|
100
|
+
return has;
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
function sortMapsForType(type: VgiDataType, value: any): any {
|
|
104
|
+
if (value == null || !typeHasMap(type)) return value;
|
|
105
|
+
if (isMap(type)) {
|
|
106
|
+
const entries = (type as any).children[0].type as VgiDataType;
|
|
107
|
+
const valueType = ((entries as any).children as VgiField[])[1].type;
|
|
108
|
+
return mapPairs(value)
|
|
109
|
+
.map(([k, v]): [unknown, unknown] => [k, sortMapsForType(valueType, v)])
|
|
110
|
+
.sort((x, y) => compareKeys(x[0], y[0]));
|
|
111
|
+
}
|
|
112
|
+
if (isList(type)) {
|
|
113
|
+
const child = (type as any).children[0].type as VgiDataType;
|
|
114
|
+
return Array.from(value as Iterable<unknown>, (v) => sortMapsForType(child, v));
|
|
115
|
+
}
|
|
116
|
+
if (isStruct(type)) {
|
|
117
|
+
const out: Record<string, any> = { ...value };
|
|
118
|
+
for (const cf of (type as any).children as VgiField[]) out[cf.name] = sortMapsForType(cf.type, value[cf.name]);
|
|
119
|
+
return out;
|
|
120
|
+
}
|
|
121
|
+
return value;
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
/**
|
|
125
|
+
* `obj` with every map-typed field (at any depth) as a key-sorted pair list.
|
|
126
|
+
* Returns `obj` itself when the schema has no map column. Never mutates `obj`
|
|
127
|
+
* (catalog items may be frozen and shared).
|
|
128
|
+
*/
|
|
129
|
+
export function sortMapsForSchema(schema: VgiSchema, obj: Record<string, any>): Record<string, any> {
|
|
130
|
+
let out: Record<string, any> | null = null;
|
|
131
|
+
for (const f of schema.fields) {
|
|
132
|
+
if (!typeHasMap(f.type)) continue;
|
|
133
|
+
const v = obj[f.name];
|
|
134
|
+
if (v == null) continue;
|
|
135
|
+
out ??= { ...obj };
|
|
136
|
+
out[f.name] = sortMapsForType(f.type, v);
|
|
137
|
+
}
|
|
138
|
+
return out ?? obj;
|
|
44
139
|
}
|
|
45
140
|
|
|
46
141
|
/**
|
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
// GENERATED by vgi.codegen.ts_client. DO NOT EDIT BY HAND.
|
|
3
3
|
//
|
|
4
4
|
// Generator: vgi-gen-ts-client v1
|
|
5
|
-
// Content hash:
|
|
5
|
+
// Content hash: 5cf87195498c
|
|
6
6
|
//
|
|
7
7
|
// To regenerate:
|
|
8
8
|
// uv run --project ~/Development/vgi-python vgi-gen-ts-client \
|
|
@@ -423,12 +423,33 @@ export interface CatalogAttachResult {
|
|
|
423
423
|
global_function_prefix: string;
|
|
424
424
|
resolved_data_version?: string | null;
|
|
425
425
|
resolved_implementation_version?: string | null;
|
|
426
|
+
supports_catalog_contents: boolean;
|
|
426
427
|
}
|
|
427
428
|
|
|
428
429
|
export interface CatalogsResponse {
|
|
429
430
|
items: CatalogInfo[];
|
|
430
431
|
}
|
|
431
432
|
|
|
433
|
+
export interface SchemaContents {
|
|
434
|
+
path: string[];
|
|
435
|
+
schema: Uint8Array;
|
|
436
|
+
tables: Uint8Array[];
|
|
437
|
+
views: Uint8Array[];
|
|
438
|
+
scalar_functions: Uint8Array[];
|
|
439
|
+
aggregate_functions: Uint8Array[];
|
|
440
|
+
table_functions: Uint8Array[];
|
|
441
|
+
scalar_macros: Uint8Array[];
|
|
442
|
+
table_macros: Uint8Array[];
|
|
443
|
+
indexes: Uint8Array[];
|
|
444
|
+
}
|
|
445
|
+
|
|
446
|
+
export interface CatalogContentsResponse {
|
|
447
|
+
catalog_version: number;
|
|
448
|
+
etag?: string | null;
|
|
449
|
+
not_modified: boolean;
|
|
450
|
+
schemas: SchemaContents[];
|
|
451
|
+
}
|
|
452
|
+
|
|
432
453
|
export interface CopyFromFormatsResponse {
|
|
433
454
|
items: Uint8Array[];
|
|
434
455
|
}
|
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
// GENERATED by vgi.codegen.ts_schemas. DO NOT EDIT BY HAND.
|
|
3
3
|
//
|
|
4
4
|
// Generator: vgi-gen-ts-schemas v2
|
|
5
|
-
// Content hash:
|
|
5
|
+
// Content hash: 36a9087555c9
|
|
6
6
|
//
|
|
7
7
|
// To regenerate:
|
|
8
8
|
// uv run --project ~/Development/vgi-python vgi-gen-ts-schemas \
|
|
@@ -228,6 +228,20 @@ export const ForeignKeyInfoSchema = schema([
|
|
|
228
228
|
field("referenced_schema_path", list(field("item", utf8(), true)), false),
|
|
229
229
|
]);
|
|
230
230
|
|
|
231
|
+
// Origin: SchemaContents
|
|
232
|
+
export const SchemaContentsSchema = schema([
|
|
233
|
+
field("path", list(field("item", utf8(), true)), false),
|
|
234
|
+
field("schema", binary(), false),
|
|
235
|
+
field("tables", list(field("item", binary(), true)), false),
|
|
236
|
+
field("views", list(field("item", binary(), true)), false),
|
|
237
|
+
field("scalar_functions", list(field("item", binary(), true)), false),
|
|
238
|
+
field("aggregate_functions", list(field("item", binary(), true)), false),
|
|
239
|
+
field("table_functions", list(field("item", binary(), true)), false),
|
|
240
|
+
field("scalar_macros", list(field("item", binary(), true)), false),
|
|
241
|
+
field("table_macros", list(field("item", binary(), true)), false),
|
|
242
|
+
field("indexes", list(field("item", binary(), true)), false),
|
|
243
|
+
]);
|
|
244
|
+
|
|
231
245
|
// Origin: AggregateBindRequest
|
|
232
246
|
export const AggregateBindRequestSchema = schema([
|
|
233
247
|
field("function_name", utf8(), false),
|
|
@@ -638,6 +652,7 @@ export const CatalogAttachResultSchema = schema([
|
|
|
638
652
|
field("global_function_prefix", utf8(), false),
|
|
639
653
|
field("resolved_data_version", utf8(), true),
|
|
640
654
|
field("resolved_implementation_version", utf8(), true),
|
|
655
|
+
field("supports_catalog_contents", bool(), false),
|
|
641
656
|
]);
|
|
642
657
|
|
|
643
658
|
// Origin: method 'catalog_catalogs' result
|
|
@@ -645,6 +660,14 @@ export const CatalogCatalogsResultSchema = schema([
|
|
|
645
660
|
field("items", list(field("item", binary(), true)), false),
|
|
646
661
|
]);
|
|
647
662
|
|
|
663
|
+
// Origin: method 'catalog_contents' result
|
|
664
|
+
export const CatalogContentsResultSchema = schema([
|
|
665
|
+
field("catalog_version", int64(), false),
|
|
666
|
+
field("etag", utf8(), true),
|
|
667
|
+
field("not_modified", bool(), false),
|
|
668
|
+
field("schemas", list(field("item", struct([field("path", list(field("item", utf8(), true)), false), field("schema", binary(), false), field("tables", list(field("item", binary(), true)), false), field("views", list(field("item", binary(), true)), false), field("scalar_functions", list(field("item", binary(), true)), false), field("aggregate_functions", list(field("item", binary(), true)), false), field("table_functions", list(field("item", binary(), true)), false), field("scalar_macros", list(field("item", binary(), true)), false), field("table_macros", list(field("item", binary(), true)), false), field("indexes", list(field("item", binary(), true)), false)]), true)), false),
|
|
669
|
+
]);
|
|
670
|
+
|
|
648
671
|
// Origin: method 'catalog_copy_from_formats' result
|
|
649
672
|
export const CatalogCopyFromFormatsResultSchema = schema([
|
|
650
673
|
field("items", list(field("item", binary(), true)), false),
|
|
@@ -833,6 +856,12 @@ export const CatalogAttachParamsSchema = schema([
|
|
|
833
856
|
// Origin: method 'catalog_catalogs' params
|
|
834
857
|
export const CatalogCatalogsParamsSchema = schema([]);
|
|
835
858
|
|
|
859
|
+
// Origin: method 'catalog_contents' params
|
|
860
|
+
export const CatalogContentsParamsSchema = schema([
|
|
861
|
+
field("attach_opaque_data", binary(), false),
|
|
862
|
+
field("if_none_match", utf8(), true),
|
|
863
|
+
]);
|
|
864
|
+
|
|
836
865
|
// Origin: method 'catalog_copy_from_formats' params
|
|
837
866
|
export const CatalogCopyFromFormatsParamsSchema = schema([
|
|
838
867
|
field("attach_opaque_data", binary(), false),
|
|
@@ -0,0 +1,126 @@
|
|
|
1
|
+
// ============================================================================
|
|
2
|
+
// GENERATED by vgi.codegen.ts_types. DO NOT EDIT BY HAND.
|
|
3
|
+
//
|
|
4
|
+
// Generator: python -m vgi.codegen.ts_types v1
|
|
5
|
+
// Content hash: d240510f013b
|
|
6
|
+
//
|
|
7
|
+
// To regenerate:
|
|
8
|
+
// uv run --project ~/Development/vgi-python python scripts/regen_generated.py
|
|
9
|
+
// ============================================================================
|
|
10
|
+
|
|
11
|
+
import { encodeASD, decodeASD } from "../codec/asd.js";
|
|
12
|
+
import {
|
|
13
|
+
CatalogAttachResultSchema,
|
|
14
|
+
CatalogContentsResultSchema,
|
|
15
|
+
SchemaContentsSchema,
|
|
16
|
+
} from "./vgi-protocol-schemas.js";
|
|
17
|
+
import type { CatalogAttachResult, CatalogContentsResponse, SchemaContents } from "./vgi-client.js";
|
|
18
|
+
export type { CatalogAttachResult, CatalogContentsResponse, SchemaContents };
|
|
19
|
+
|
|
20
|
+
// ----------------------------------------------------------------------------
|
|
21
|
+
// CatalogAttachResult (schema: CatalogAttachResultSchema)
|
|
22
|
+
// ----------------------------------------------------------------------------
|
|
23
|
+
|
|
24
|
+
/** Fields of `CatalogAttachResult` that have a protocol default. */
|
|
25
|
+
export type CatalogAttachResultDefaultedField =
|
|
26
|
+
| "attach_opaque_data_required"
|
|
27
|
+
| "default_schema"
|
|
28
|
+
| "settings"
|
|
29
|
+
| "secret_types"
|
|
30
|
+
| "attach_catalogs"
|
|
31
|
+
| "comment"
|
|
32
|
+
| "tags"
|
|
33
|
+
| "supports_column_statistics"
|
|
34
|
+
| "global_functions"
|
|
35
|
+
| "global_function_prefix"
|
|
36
|
+
| "supports_catalog_contents";
|
|
37
|
+
/** `CatalogAttachResult` with every field that has a protocol default optional. */
|
|
38
|
+
export type CatalogAttachResultInit = Omit<CatalogAttachResult, CatalogAttachResultDefaultedField> & Partial<Pick<CatalogAttachResult, CatalogAttachResultDefaultedField>>;
|
|
39
|
+
|
|
40
|
+
/** Build a complete `CatalogAttachResult`, filling the protocol defaults, in wire order. */
|
|
41
|
+
export function buildCatalogAttachResult(init: CatalogAttachResultInit): CatalogAttachResult {
|
|
42
|
+
return {
|
|
43
|
+
attach_opaque_data: init.attach_opaque_data,
|
|
44
|
+
supports_transactions: init.supports_transactions,
|
|
45
|
+
supports_time_travel: init.supports_time_travel,
|
|
46
|
+
catalog_version_frozen: init.catalog_version_frozen,
|
|
47
|
+
catalog_version: init.catalog_version,
|
|
48
|
+
attach_opaque_data_required: init.attach_opaque_data_required ?? true,
|
|
49
|
+
default_schema: init.default_schema ?? "main",
|
|
50
|
+
settings: init.settings ?? [],
|
|
51
|
+
secret_types: init.secret_types ?? [],
|
|
52
|
+
attach_catalogs: init.attach_catalogs ?? [],
|
|
53
|
+
comment: init.comment ?? null,
|
|
54
|
+
tags: init.tags ?? {},
|
|
55
|
+
supports_column_statistics: init.supports_column_statistics ?? false,
|
|
56
|
+
global_functions: init.global_functions ?? [],
|
|
57
|
+
global_function_prefix: init.global_function_prefix ?? "",
|
|
58
|
+
resolved_data_version: init.resolved_data_version ?? null,
|
|
59
|
+
resolved_implementation_version: init.resolved_implementation_version ?? null,
|
|
60
|
+
supports_catalog_contents: init.supports_catalog_contents ?? false,
|
|
61
|
+
};
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
export const encodeCatalogAttachResult = (v: CatalogAttachResult): Uint8Array => encodeASD(CatalogAttachResultSchema, v);
|
|
65
|
+
export const decodeCatalogAttachResult = (b: Uint8Array): CatalogAttachResult => decodeASD<CatalogAttachResult>(CatalogAttachResultSchema, b);
|
|
66
|
+
|
|
67
|
+
// ----------------------------------------------------------------------------
|
|
68
|
+
// SchemaContents (schema: SchemaContentsSchema)
|
|
69
|
+
// ----------------------------------------------------------------------------
|
|
70
|
+
|
|
71
|
+
/** Fields of `SchemaContents` that have a protocol default. */
|
|
72
|
+
export type SchemaContentsDefaultedField =
|
|
73
|
+
| "tables"
|
|
74
|
+
| "views"
|
|
75
|
+
| "scalar_functions"
|
|
76
|
+
| "aggregate_functions"
|
|
77
|
+
| "table_functions"
|
|
78
|
+
| "scalar_macros"
|
|
79
|
+
| "table_macros"
|
|
80
|
+
| "indexes";
|
|
81
|
+
/** `SchemaContents` with every field that has a protocol default optional. */
|
|
82
|
+
export type SchemaContentsInit = Omit<SchemaContents, SchemaContentsDefaultedField> & Partial<Pick<SchemaContents, SchemaContentsDefaultedField>>;
|
|
83
|
+
|
|
84
|
+
/** Build a complete `SchemaContents`, filling the protocol defaults, in wire order. */
|
|
85
|
+
export function buildSchemaContents(init: SchemaContentsInit): SchemaContents {
|
|
86
|
+
return {
|
|
87
|
+
path: init.path,
|
|
88
|
+
schema: init.schema,
|
|
89
|
+
tables: init.tables ?? [],
|
|
90
|
+
views: init.views ?? [],
|
|
91
|
+
scalar_functions: init.scalar_functions ?? [],
|
|
92
|
+
aggregate_functions: init.aggregate_functions ?? [],
|
|
93
|
+
table_functions: init.table_functions ?? [],
|
|
94
|
+
scalar_macros: init.scalar_macros ?? [],
|
|
95
|
+
table_macros: init.table_macros ?? [],
|
|
96
|
+
indexes: init.indexes ?? [],
|
|
97
|
+
};
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
export const encodeSchemaContents = (v: SchemaContents): Uint8Array => encodeASD(SchemaContentsSchema, v);
|
|
101
|
+
export const decodeSchemaContents = (b: Uint8Array): SchemaContents => decodeASD<SchemaContents>(SchemaContentsSchema, b);
|
|
102
|
+
|
|
103
|
+
// ----------------------------------------------------------------------------
|
|
104
|
+
// CatalogContentsResponse (schema: CatalogContentsResultSchema)
|
|
105
|
+
// ----------------------------------------------------------------------------
|
|
106
|
+
|
|
107
|
+
/** Fields of `CatalogContentsResponse` that have a protocol default. */
|
|
108
|
+
export type CatalogContentsResponseDefaultedField =
|
|
109
|
+
| "etag"
|
|
110
|
+
| "not_modified"
|
|
111
|
+
| "schemas";
|
|
112
|
+
/** `CatalogContentsResponse` with every field that has a protocol default optional. */
|
|
113
|
+
export type CatalogContentsResponseInit = Omit<CatalogContentsResponse, CatalogContentsResponseDefaultedField> & Partial<Pick<CatalogContentsResponse, CatalogContentsResponseDefaultedField>>;
|
|
114
|
+
|
|
115
|
+
/** Build a complete `CatalogContentsResponse`, filling the protocol defaults, in wire order. */
|
|
116
|
+
export function buildCatalogContentsResponse(init: CatalogContentsResponseInit): CatalogContentsResponse {
|
|
117
|
+
return {
|
|
118
|
+
catalog_version: init.catalog_version,
|
|
119
|
+
etag: init.etag ?? null,
|
|
120
|
+
not_modified: init.not_modified ?? false,
|
|
121
|
+
schemas: init.schemas ?? [],
|
|
122
|
+
};
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
export const encodeCatalogContentsResponse = (v: CatalogContentsResponse): Uint8Array => encodeASD(CatalogContentsResultSchema, v);
|
|
126
|
+
export const decodeCatalogContentsResponse = (b: Uint8Array): CatalogContentsResponse => decodeASD<CatalogContentsResponse>(CatalogContentsResultSchema, b);
|
package/src/http/fetch.ts
CHANGED
|
@@ -25,9 +25,10 @@ import {
|
|
|
25
25
|
import { arrowStateSerializer } from "../protocol/state-serializer.js";
|
|
26
26
|
import { buildVgiProtocol, type ProtocolConfig } from "../protocol/dispatch.js";
|
|
27
27
|
import { createLandingRoutes, type LandingInfo } from "./landing.js";
|
|
28
|
+
import { buildRpcServer, type HostingOptions } from "../rpc-server.js";
|
|
28
29
|
import { setRequestAuthScope, type RequestAuthHolder } from "../request-auth.js";
|
|
29
30
|
|
|
30
|
-
export interface VgiFetchOptions {
|
|
31
|
+
export interface VgiFetchOptions extends HostingOptions {
|
|
31
32
|
/** Wire-protocol config (registry + catalogInterface). */
|
|
32
33
|
protocol: ProtocolConfig;
|
|
33
34
|
/** HMAC key for state-token signing. Pass a stable, secret 32-byte key
|
|
@@ -197,7 +198,14 @@ export function createVgiFetch(opts: VgiFetchOptions): (req: Request) => Promise
|
|
|
197
198
|
...opts.protocol,
|
|
198
199
|
});
|
|
199
200
|
|
|
200
|
-
|
|
201
|
+
// The same builder every other transport uses: `vgi.v2`, the worker's
|
|
202
|
+
// hostedProtocols(), reflection, and -- HTTP being the transport that
|
|
203
|
+
// authenticates callers -- vgi_rpc.Identity.v1 when the worker opts in.
|
|
204
|
+
// Built at construction, so a worker that opts into introspection without an
|
|
205
|
+
// allowlist fails here rather than serving traffic.
|
|
206
|
+
const server = buildRpcServer(protocol, opts, { transport: "http", serverId });
|
|
207
|
+
|
|
208
|
+
const handler = createHttpHandler(server, {
|
|
201
209
|
prefix,
|
|
202
210
|
serverId,
|
|
203
211
|
tokenKey: opts.signingKey,
|
package/src/index.core.ts
CHANGED
|
@@ -77,9 +77,9 @@ export {
|
|
|
77
77
|
export type { VgiFunction, FunctionMeta, StreamHandlers, FunctionExample, HandlerState } from "./functions/types.js";
|
|
78
78
|
|
|
79
79
|
// Catalog
|
|
80
|
-
export { CatalogInterface, type CatalogAttachResult, type SchemaInfo, encodeSchemaInfo, decodeSchemaInfo, type TableInfo, encodeTableInfo, decodeTableInfo, type ViewInfo, encodeViewInfo, decodeViewInfo, type CatalogInfo, encodeCatalogInfo, decodeCatalogInfo, type FunctionInfo, encodeFunctionInfo, decodeFunctionInfo, type FunctionInfoOptions, type MacroInfo, encodeMacroInfo, decodeMacroInfo, type MacroType, type CopyFromFormatInfo, encodeCopyFromFormatInfo, type AttachOpaqueData, type TransactionOpaqueData, buildScanBranchesResult, encodeFormatOptions, type ScanBranchInput } from "./catalog/interface.js";
|
|
80
|
+
export { CatalogInterface, type CatalogAttachResult, type CatalogAttachResultInit, buildCatalogAttachResult, type SchemaContentsInfo, type CatalogContentsResult, type SchemaInfo, encodeSchemaInfo, decodeSchemaInfo, type TableInfo, encodeTableInfo, decodeTableInfo, type ViewInfo, encodeViewInfo, decodeViewInfo, type CatalogInfo, encodeCatalogInfo, decodeCatalogInfo, type FunctionInfo, encodeFunctionInfo, decodeFunctionInfo, type FunctionInfoOptions, type MacroInfo, encodeMacroInfo, decodeMacroInfo, type MacroType, type CopyFromFormatInfo, encodeCopyFromFormatInfo, type AttachOpaqueData, type TransactionOpaqueData, buildScanBranchesResult, encodeFormatOptions, type ScanBranchInput } from "./catalog/interface.js";
|
|
81
81
|
export type { CatalogDescriptor, SchemaDescriptor, TableDescriptor, ViewDescriptor, MacroDescriptor, SettingDescriptor, SecretTypeDescriptor, ForeignKeyDef, DefaultValue } from "./catalog/descriptors.js";
|
|
82
|
-
export { ReadOnlyCatalogInterface } from "./catalog/read-only.js";
|
|
82
|
+
export { ReadOnlyCatalogInterface, READ_ONLY_ITEM_ATTACH_ID } from "./catalog/read-only.js";
|
|
83
83
|
export { CompositeCatalogInterface } from "./catalog/composite.js";
|
|
84
84
|
export { FunctionRegistry } from "./functions/registry.js";
|
|
85
85
|
|
|
@@ -309,4 +309,23 @@ export type {
|
|
|
309
309
|
export { str, bytes, int, int32, float, float32, bool, toSchema, OutputCollector, AuthContext } from "@query-farm/vgi-rpc";
|
|
310
310
|
export { subprocessConnect, httpConnect, tcpConnect } from "@query-farm/vgi-rpc";
|
|
311
311
|
export type { RpcClient, StreamSession, LogMessage } from "@query-farm/vgi-rpc";
|
|
312
|
+
|
|
313
|
+
// Hosting additional protocols and opting into vgi_rpc.Identity.v1. The errors
|
|
314
|
+
// are re-exported so a worker's resolveToken/mintGrant can throw the right one:
|
|
315
|
+
// AuthUnavailableError (or IdentityUnavailableError) for "could not find out",
|
|
316
|
+
// GrantRefusedError to decline a grant.
|
|
317
|
+
export {
|
|
318
|
+
buildRpcServer,
|
|
319
|
+
INTROSPECT_PRINCIPALS_ENV,
|
|
320
|
+
type BuildRpcServerOptions,
|
|
321
|
+
type HostingOptions,
|
|
322
|
+
type ServerTransport,
|
|
323
|
+
} from "./rpc-server.js";
|
|
324
|
+
export {
|
|
325
|
+
AuthUnavailableError,
|
|
326
|
+
GrantRefusedError,
|
|
327
|
+
IdentityUnavailableError,
|
|
328
|
+
StatusError,
|
|
329
|
+
} from "@query-farm/vgi-rpc";
|
|
330
|
+
export type { GrantMinter, IssuedGrant, TokenIdentity, TokenResolver } from "@query-farm/vgi-rpc";
|
|
312
331
|
export { normalizeSchemaPath, schemaPathDisplay, schemaPathKey, schemaPathsEqual, type SchemaPath } from "./schema-path.js";
|
package/src/protocol/dispatch.ts
CHANGED
|
@@ -57,7 +57,7 @@ export interface ProtocolConfig {
|
|
|
57
57
|
export const VGI_PROTOCOL_NAME = "vgi.v2";
|
|
58
58
|
|
|
59
59
|
export function buildVgiProtocol(config: ProtocolConfig): Protocol {
|
|
60
|
-
const protocol = new Protocol(VGI_PROTOCOL_NAME, { protocolVersion: "2.
|
|
60
|
+
const protocol = new Protocol(VGI_PROTOCOL_NAME, { protocolVersion: "2.1.0" });
|
|
61
61
|
|
|
62
62
|
registerFunctionMethods(protocol, {
|
|
63
63
|
registry: config.registry,
|
|
@@ -4,12 +4,20 @@
|
|
|
4
4
|
|
|
5
5
|
import { type VgiSchema, schema, type VgiField, field, type VgiDataType, binary, utf8, bool } from "../../../arrow/index.js";
|
|
6
6
|
import { Protocol, type CallContext } from "@query-farm/vgi-rpc";
|
|
7
|
-
import {
|
|
7
|
+
import {
|
|
8
|
+
encodeSchemaInfo,
|
|
9
|
+
encodeTableInfo,
|
|
10
|
+
encodeViewInfo,
|
|
11
|
+
encodeCatalogInfo,
|
|
12
|
+
} from "../../../generated/vgi-client.js";
|
|
13
|
+
import { buildCatalogAttachResult } from "../../../generated/vgi-protocol-types.js";
|
|
14
|
+
import { serveCatalogContents } from "./contents.js";
|
|
8
15
|
import { encodeFunctionInfoOnce } from "../../../catalog/item-encoding.js";
|
|
9
16
|
import { encodeCopyFromFormatInfo } from "../../../catalog/interface.js";
|
|
10
17
|
import {
|
|
11
18
|
CatalogAttachParamsSchema,
|
|
12
19
|
CatalogAttachResultSchema,
|
|
20
|
+
CatalogContentsParamsSchema,
|
|
13
21
|
CatalogCatalogsParamsSchema,
|
|
14
22
|
CatalogCatalogsResultSchema,
|
|
15
23
|
CatalogCopyFromFormatsParamsSchema,
|
|
@@ -105,33 +113,19 @@ export function registerCatalogAdminMethods(protocol: Protocol, getCatalog: GetC
|
|
|
105
113
|
(ctx as CallContext | undefined)?.auth,
|
|
106
114
|
signingKey,
|
|
107
115
|
);
|
|
108
|
-
|
|
116
|
+
// The generated builder emits every column of the pinned
|
|
117
|
+
// CatalogAttachResult schema, filling vgi-python's defaults for any a
|
|
118
|
+
// catalog left out (supports_catalog_contents=false: the client then
|
|
119
|
+
// keeps to the per-schema RPCs).
|
|
120
|
+
return wrapResult(buildCatalogAttachResult({
|
|
121
|
+
...result,
|
|
109
122
|
attach_opaque_data: sealedAttach,
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
attach_opaque_data_required: result.attach_opaque_data_required ?? true,
|
|
115
|
-
default_schema: result.default_schema ?? "main",
|
|
116
|
-
settings: result.settings ?? [],
|
|
117
|
-
secret_types: result.secret_types ?? [],
|
|
118
|
-
attach_catalogs: result.attach_catalogs ?? [],
|
|
119
|
-
comment: result.comment ?? null,
|
|
120
|
-
tags: result.tags ?? {},
|
|
121
|
-
// True so DuckDB will route catalog_table_column_statistics_get RPCs
|
|
122
|
-
// to our handler for tables whose TableInfo.supports_column_statistics
|
|
123
|
-
// is also true. Catalogs that never serve column stats can override
|
|
124
|
-
// this in attach() via CatalogAttachResult.supports_column_statistics.
|
|
123
|
+
// SDK default, unlike vgi-python's false: true so DuckDB will route
|
|
124
|
+
// catalog_table_column_statistics_get RPCs to our handler for tables
|
|
125
|
+
// whose TableInfo.supports_column_statistics is also true. Catalogs
|
|
126
|
+
// that never serve column stats set it false in attach().
|
|
125
127
|
supports_column_statistics: result.supports_column_statistics ?? true,
|
|
126
|
-
|
|
127
|
-
// its global namespace. Empty unless the catalog advertises them —
|
|
128
|
-
// but the columns must still be present, since the extension pins the
|
|
129
|
-
// full CatalogAttachResult field list on the response schema.
|
|
130
|
-
global_functions: result.global_functions ?? [],
|
|
131
|
-
global_function_prefix: result.global_function_prefix ?? "",
|
|
132
|
-
resolved_data_version: result.resolved_data_version ?? null,
|
|
133
|
-
resolved_implementation_version: result.resolved_implementation_version ?? null,
|
|
134
|
-
}, CatalogAttachResultSchema);
|
|
128
|
+
}), CatalogAttachResultSchema);
|
|
135
129
|
},
|
|
136
130
|
});
|
|
137
131
|
|
|
@@ -247,6 +241,21 @@ export function registerCatalogAdminMethods(protocol: Protocol, getCatalog: GetC
|
|
|
247
241
|
},
|
|
248
242
|
});
|
|
249
243
|
|
|
244
|
+
// catalog_contents — every schema and all of its contents in one result,
|
|
245
|
+
// for a client whose attach result set supports_catalog_contents. Takes no
|
|
246
|
+
// transaction: the client caches the answer for the whole attach, so it is
|
|
247
|
+
// the committed catalog at catalog_version. `if_none_match` revalidates a
|
|
248
|
+
// snapshot the client holds (etag / not_modified); see ./contents.ts for the
|
|
249
|
+
// rules, the content-hash etag and the frozen-catalog cache.
|
|
250
|
+
catalogUnary(protocol, signingKey, "catalog_contents", {
|
|
251
|
+
params: CatalogContentsParamsSchema,
|
|
252
|
+
result: RESULT_BINARY_SCHEMA,
|
|
253
|
+
handler: async (params) => {
|
|
254
|
+
const ifNoneMatch = params.if_none_match == null ? null : String(params.if_none_match);
|
|
255
|
+
return serveCatalogContents(getCatalog(), toUint8Array(params.attach_opaque_data), ifNoneMatch);
|
|
256
|
+
},
|
|
257
|
+
});
|
|
258
|
+
|
|
250
259
|
// catalog_copy_from_formats — catalog-level (not schema-scoped). Lists the
|
|
251
260
|
// custom COPY ... FROM formats this catalog advertises; empty list when none.
|
|
252
261
|
catalogUnary(protocol, signingKey, "catalog_copy_from_formats", {
|
|
@@ -0,0 +1,162 @@
|
|
|
1
|
+
// Copyright 2025, 2026 Query Farm LLC - https://query.farm
|
|
2
|
+
// SPDX-License-Identifier: LicenseRef-QueryFarm-Source-Available-1.0
|
|
3
|
+
|
|
4
|
+
// catalog_contents: the worker side of the protocol 2.1.0 bulk enumeration,
|
|
5
|
+
// with etag revalidation and the frozen-catalog response cache. Mirrors
|
|
6
|
+
// vgi-python's Worker.catalog_contents / _catalog_contents_response.
|
|
7
|
+
|
|
8
|
+
import {
|
|
9
|
+
encodeSchemaInfo,
|
|
10
|
+
encodeTableInfo,
|
|
11
|
+
encodeViewInfo,
|
|
12
|
+
encodeMacroInfo,
|
|
13
|
+
encodeIndexInfo,
|
|
14
|
+
} from "../../../generated/vgi-client.js";
|
|
15
|
+
import {
|
|
16
|
+
buildCatalogContentsResponse,
|
|
17
|
+
buildSchemaContents,
|
|
18
|
+
type CatalogContentsResponse,
|
|
19
|
+
type SchemaContents,
|
|
20
|
+
} from "../../../generated/vgi-protocol-types.js";
|
|
21
|
+
import { CatalogContentsResultSchema } from "../../../generated/vgi-protocol-schemas.js";
|
|
22
|
+
import { encodeFunctionInfoOnce } from "../../../catalog/item-encoding.js";
|
|
23
|
+
import type { AttachOpaqueData, CatalogInterface } from "../../../catalog/interface.js";
|
|
24
|
+
import { catalogContentsDigest } from "../../../catalog/contents-digest.js";
|
|
25
|
+
import { schemaPathKey } from "../../../schema-path.js";
|
|
26
|
+
import { wrapResult } from "../shared.js";
|
|
27
|
+
|
|
28
|
+
type Wire = { result: Uint8Array };
|
|
29
|
+
|
|
30
|
+
/** One cached answer per catalog instance, valid for one catalog version. */
|
|
31
|
+
interface ContentsSnapshot {
|
|
32
|
+
version: number;
|
|
33
|
+
etag: string | null;
|
|
34
|
+
wire: Wire;
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
/**
|
|
38
|
+
* Cached catalog_contents responses, keyed by the owning catalog instance
|
|
39
|
+
* (weak, so a dropped catalog takes its snapshot with it). Holds the build
|
|
40
|
+
* promise, so concurrent first calls build once.
|
|
41
|
+
*/
|
|
42
|
+
const CACHE = new WeakMap<CatalogInterface, Promise<ContentsSnapshot>>();
|
|
43
|
+
|
|
44
|
+
/**
|
|
45
|
+
* Ask the catalog for its contents and shape the typed wire response.
|
|
46
|
+
*
|
|
47
|
+
* Enforces the revalidation rules: `not_modified` needs an etag equal to
|
|
48
|
+
* `ifNoneMatch` and no schemas; a catalog with no etag never yields
|
|
49
|
+
* `not_modified` (and so ignores `ifNoneMatch`). Validates the schema paths
|
|
50
|
+
* (unique, parent present) and orders them parent-first. Every item is
|
|
51
|
+
* encoded with the SAME encoder its per-schema RPC uses, so items are
|
|
52
|
+
* byte-identical to catalog_schemas / catalog_schema_contents_* items.
|
|
53
|
+
*/
|
|
54
|
+
export async function buildCatalogContents(
|
|
55
|
+
cat: CatalogInterface,
|
|
56
|
+
owner: CatalogInterface,
|
|
57
|
+
attach: AttachOpaqueData,
|
|
58
|
+
version: number,
|
|
59
|
+
ifNoneMatch: string | null,
|
|
60
|
+
): Promise<CatalogContentsResponse> {
|
|
61
|
+
const result = await cat.catalogContents(attach, ifNoneMatch);
|
|
62
|
+
const etag0 = result.etag ?? null;
|
|
63
|
+
if (result.not_modified) {
|
|
64
|
+
if (etag0 === null || ifNoneMatch === null || etag0 !== ifNoneMatch) {
|
|
65
|
+
throw new Error(
|
|
66
|
+
"catalog_contents returned not_modified, but only a catalog whose etag equals " +
|
|
67
|
+
"if_none_match may (and it must return that etag)",
|
|
68
|
+
);
|
|
69
|
+
}
|
|
70
|
+
if (result.schemas && result.schemas.length > 0) {
|
|
71
|
+
throw new Error("catalog_contents returned not_modified with schemas; it must return none");
|
|
72
|
+
}
|
|
73
|
+
return buildCatalogContentsResponse({ catalog_version: version, etag: etag0, not_modified: true });
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
const infos = [...(result.schemas ?? [])];
|
|
77
|
+
const keys = infos.map((c) => schemaPathKey(c.schema.path));
|
|
78
|
+
const keySet = new Set(keys);
|
|
79
|
+
if (keySet.size !== keys.length) throw new Error("catalog_contents returned duplicate schema paths");
|
|
80
|
+
for (const c of infos) {
|
|
81
|
+
if (c.schema.path.length > 1 && !keySet.has(schemaPathKey(c.schema.path.slice(0, -1)))) {
|
|
82
|
+
throw new Error(`catalog_contents returned schema path ${JSON.stringify(c.schema.path)} without its parent`);
|
|
83
|
+
}
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
// Same parent-before-child order catalog_schemas guarantees (stable sort).
|
|
87
|
+
const schemas: SchemaContents[] = infos
|
|
88
|
+
.sort((a, b) => a.schema.path.length - b.schema.path.length)
|
|
89
|
+
.map((c) =>
|
|
90
|
+
buildSchemaContents({
|
|
91
|
+
path: [...c.schema.path],
|
|
92
|
+
schema: encodeSchemaInfo(c.schema),
|
|
93
|
+
tables: c.tables.map((t) => encodeTableInfo(t)),
|
|
94
|
+
views: c.views.map((v) => encodeViewInfo(v)),
|
|
95
|
+
scalar_functions: c.scalar_functions.map((f) => encodeFunctionInfoOnce(f)),
|
|
96
|
+
aggregate_functions: c.aggregate_functions.map((f) => encodeFunctionInfoOnce(f)),
|
|
97
|
+
table_functions: c.table_functions.map((f) => encodeFunctionInfoOnce(f)),
|
|
98
|
+
scalar_macros: c.scalar_macros.map((m) => encodeMacroInfo(m)),
|
|
99
|
+
table_macros: c.table_macros.map((m) => encodeMacroInfo(m)),
|
|
100
|
+
indexes: c.indexes.map((i) => encodeIndexInfo(i)),
|
|
101
|
+
}),
|
|
102
|
+
);
|
|
103
|
+
let etag = etag0;
|
|
104
|
+
if (etag === null && owner.catalogContentsEtag === "content-hash") {
|
|
105
|
+
etag = await catalogContentsDigest(schemas);
|
|
106
|
+
}
|
|
107
|
+
if (etag !== null && ifNoneMatch !== null && etag === ifNoneMatch) {
|
|
108
|
+
return buildCatalogContentsResponse({ catalog_version: version, etag, not_modified: true });
|
|
109
|
+
}
|
|
110
|
+
return buildCatalogContentsResponse({ catalog_version: version, etag, schemas });
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
/**
|
|
114
|
+
* Serve one catalog_contents call (the attach value is already opened).
|
|
115
|
+
*
|
|
116
|
+
* Caching: a catalog whose `catalogContentsCacheable()` is true (by default:
|
|
117
|
+
* version frozen and contents attach-independent; ReadOnlyCatalogInterface) is
|
|
118
|
+
* built once per (catalog instance, version); every later call reuses the same
|
|
119
|
+
* serialized response bytes. A conditional call matching the cached etag is
|
|
120
|
+
* answered `not_modified` without touching the snapshot.
|
|
121
|
+
*/
|
|
122
|
+
export async function serveCatalogContents(
|
|
123
|
+
cat: CatalogInterface,
|
|
124
|
+
attach: AttachOpaqueData,
|
|
125
|
+
ifNoneMatch: string | null,
|
|
126
|
+
): Promise<Wire> {
|
|
127
|
+
const owner = cat.catalogContentsOwner(attach);
|
|
128
|
+
const version = Number(await cat.version(attach));
|
|
129
|
+
if (owner.catalogContentsCacheable()) {
|
|
130
|
+
let pending = CACHE.get(owner);
|
|
131
|
+
let snapshot = pending ? await pending.catch(() => null) : null;
|
|
132
|
+
if (!snapshot || snapshot.version !== version) {
|
|
133
|
+
pending = (async () => {
|
|
134
|
+
const built = await buildCatalogContents(cat, owner, attach, version, null);
|
|
135
|
+
return {
|
|
136
|
+
version,
|
|
137
|
+
etag: built.etag ?? null,
|
|
138
|
+
wire: wrapResult(built, CatalogContentsResultSchema),
|
|
139
|
+
};
|
|
140
|
+
})();
|
|
141
|
+
CACHE.set(owner, pending);
|
|
142
|
+
try {
|
|
143
|
+
snapshot = await pending;
|
|
144
|
+
} catch (err) {
|
|
145
|
+
// A failed build is not cached: the next call tries again.
|
|
146
|
+
if (CACHE.get(owner) === pending) CACHE.delete(owner);
|
|
147
|
+
throw err;
|
|
148
|
+
}
|
|
149
|
+
}
|
|
150
|
+
if (ifNoneMatch !== null && snapshot.etag !== null && ifNoneMatch === snapshot.etag) {
|
|
151
|
+
return wrapResult(
|
|
152
|
+
buildCatalogContentsResponse({ catalog_version: version, etag: snapshot.etag, not_modified: true }),
|
|
153
|
+
CatalogContentsResultSchema,
|
|
154
|
+
);
|
|
155
|
+
}
|
|
156
|
+
return snapshot.wire;
|
|
157
|
+
}
|
|
158
|
+
return wrapResult(
|
|
159
|
+
await buildCatalogContents(cat, owner, attach, version, ifNoneMatch),
|
|
160
|
+
CatalogContentsResultSchema,
|
|
161
|
+
);
|
|
162
|
+
}
|