@query-farm/vgi 0.37.0 → 0.38.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +74 -0
- package/dist/catalog/attach-option.d.ts +11 -8
- package/dist/catalog/composite.d.ts +23 -1
- package/dist/catalog/contents-digest.d.ts +25 -0
- package/dist/catalog/interface.d.ts +107 -2
- package/dist/catalog/read-only.d.ts +45 -1
- package/dist/client/client.d.ts +19 -1
- package/dist/client-entry.js +141 -4
- package/dist/client-entry.js.map +10 -9
- package/dist/codec/asd.d.ts +8 -0
- package/dist/generated/vgi-client.d.ts +19 -0
- package/dist/generated/vgi-protocol-schemas.d.ts +3 -0
- package/dist/generated/vgi-protocol-types.d.ts +26 -0
- package/dist/http/fetch.d.ts +2 -1
- package/dist/index.core.d.ts +5 -2
- package/dist/index.js +574 -36
- package/dist/index.js.map +21 -17
- package/dist/protocol/handlers/catalog/contents.d.ts +27 -0
- package/dist/rpc-server.d.ts +97 -0
- package/dist/serve-entry.d.ts +2 -1
- package/dist/serve-entry.js +405 -25
- package/dist/serve-entry.js.map +18 -14
- package/dist/worker-cf-entry.d.ts +3 -0
- package/dist/worker-cf-entry.js +528 -31
- package/dist/worker-cf-entry.js.map +21 -17
- package/dist/worker.d.ts +9 -1
- package/package.json +3 -3
- package/src/catalog/attach-option.ts +11 -8
- package/src/catalog/composite.ts +81 -0
- package/src/catalog/contents-digest.ts +78 -0
- package/src/catalog/interface.ts +167 -5
- package/src/catalog/read-only.ts +68 -6
- package/src/client/client.ts +53 -0
- package/src/codec/asd.ts +96 -1
- package/src/generated/vgi-client.ts +22 -1
- package/src/generated/vgi-protocol-schemas.ts +30 -1
- package/src/generated/vgi-protocol-types.ts +126 -0
- package/src/http/fetch.ts +10 -2
- package/src/index.core.ts +21 -2
- package/src/protocol/dispatch.ts +1 -1
- package/src/protocol/handlers/catalog/admin.ts +35 -26
- package/src/protocol/handlers/catalog/contents.ts +162 -0
- package/src/protocol/handlers/catalog/macro.ts +2 -1
- package/src/protocol/handlers/catalog/shared.ts +12 -1
- package/src/rpc-server.ts +273 -0
- package/src/serve-entry.ts +7 -1
- package/src/worker-cf-entry.ts +8 -0
- package/src/worker.ts +24 -6
package/dist/worker.d.ts
CHANGED
|
@@ -1,9 +1,10 @@
|
|
|
1
|
+
import { type HostingOptions } from "./rpc-server.js";
|
|
1
2
|
import { FunctionRegistry } from "./functions/registry.js";
|
|
2
3
|
import type { VgiFunction } from "./functions/types.js";
|
|
3
4
|
import { type CatalogDescriptor } from "./catalog/descriptors.js";
|
|
4
5
|
import type { CatalogInterface } from "./catalog/interface.js";
|
|
5
6
|
import { ReadOnlyCatalogInterface } from "./catalog/read-only.js";
|
|
6
|
-
export interface WorkerConfig {
|
|
7
|
+
export interface WorkerConfig extends HostingOptions {
|
|
7
8
|
functions?: VgiFunction[];
|
|
8
9
|
catalog?: CatalogDescriptor;
|
|
9
10
|
catalogInterface?: CatalogInterface;
|
|
@@ -21,10 +22,17 @@ export interface WorkerConfig {
|
|
|
21
22
|
}
|
|
22
23
|
export declare class Worker {
|
|
23
24
|
private _registry;
|
|
25
|
+
private _hosting;
|
|
24
26
|
private _catalogInterface?;
|
|
25
27
|
private _catalogName?;
|
|
26
28
|
constructor(config: WorkerConfig);
|
|
27
29
|
run(argv?: readonly string[]): void;
|
|
30
|
+
/**
|
|
31
|
+
* Build the vgi-rpc server for one transport: `vgi.v2`, the worker's
|
|
32
|
+
* `hostedProtocols()` and reflection everywhere; identity on HTTP only. The
|
|
33
|
+
* one construction path every transport uses (see `rpc-server.ts`).
|
|
34
|
+
*/
|
|
35
|
+
private buildServer;
|
|
28
36
|
/** Build the VGI protocol from this worker's registry + catalog interface. */
|
|
29
37
|
private buildProtocol;
|
|
30
38
|
/**
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@query-farm/vgi",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.38.0",
|
|
4
4
|
"description": "TypeScript framework for extending DuckDB with user-defined functions via the Vector Gateway Interface (VGI) over Arrow IPC.",
|
|
5
5
|
"license": "SEE LICENSE IN LICENSE",
|
|
6
6
|
"author": "Query Farm LLC (https://query.farm)",
|
|
@@ -91,12 +91,12 @@
|
|
|
91
91
|
},
|
|
92
92
|
"peerDependencies": {
|
|
93
93
|
"@query-farm/apache-arrow": "^21.1.1",
|
|
94
|
-
"@query-farm/vgi-rpc": "
|
|
94
|
+
"@query-farm/vgi-rpc": "^0.26.0"
|
|
95
95
|
},
|
|
96
96
|
"devDependencies": {
|
|
97
97
|
"@duckdb/node-api": "^1.5.5-r.4",
|
|
98
98
|
"@query-farm/apache-arrow": "^21.1.1",
|
|
99
|
-
"@query-farm/vgi-rpc": "^0.
|
|
99
|
+
"@query-farm/vgi-rpc": "^0.26.0",
|
|
100
100
|
"@types/bun": "latest",
|
|
101
101
|
"typescript": "^7.0.2"
|
|
102
102
|
},
|
|
@@ -82,16 +82,19 @@ export interface AttachOptionSpec {
|
|
|
82
82
|
* The option carries a credential: an API key, a token, a password, or
|
|
83
83
|
* anything else that must not be shown, stored or logged in plain text.
|
|
84
84
|
*
|
|
85
|
-
* Credential options **MUST** be declared `secret: true`.
|
|
86
|
-
*
|
|
87
|
-
*
|
|
88
|
-
*
|
|
89
|
-
*
|
|
90
|
-
*
|
|
85
|
+
* Credential options **MUST** be declared `secret: true`. The value is still
|
|
86
|
+
* passed inline as an ordinary ATTACH option; the flag tells the DuckDB
|
|
87
|
+
* extension to redact it from `duckdb_databases()`, keep only a salted hash
|
|
88
|
+
* of it in its result-cache key, and never log it. Clients use the flag to
|
|
89
|
+
* mask the field and keep the value out of shared links and exported
|
|
90
|
+
* configuration.
|
|
91
|
+
*
|
|
92
|
+
* To keep the credential out of the SQL text itself, pass it as an
|
|
93
|
+
* expression:
|
|
91
94
|
*
|
|
92
95
|
* ```sql
|
|
93
|
-
*
|
|
94
|
-
*
|
|
96
|
+
* ATTACH 'sales' (TYPE vgi, LOCATION 'https://worker.example.com',
|
|
97
|
+
* api_key getenv('SALES_API_KEY'));
|
|
95
98
|
* ```
|
|
96
99
|
*
|
|
97
100
|
* Combines with `required` (a credential the catalog cannot attach without,
|
package/src/catalog/composite.ts
CHANGED
|
@@ -17,6 +17,7 @@ import type {
|
|
|
17
17
|
IndexInfo,
|
|
18
18
|
MacroInfo,
|
|
19
19
|
MacroType,
|
|
20
|
+
CatalogContentsResult,
|
|
20
21
|
SchemaInfo,
|
|
21
22
|
TableInfo,
|
|
22
23
|
TransactionOpaqueData,
|
|
@@ -171,6 +172,86 @@ export class CompositeCatalogInterface extends CatalogInterface {
|
|
|
171
172
|
return await this._route(attachOpaqueData).macroGet(attachOpaqueData, schemaPath, name, transactionOpaqueData);
|
|
172
173
|
}
|
|
173
174
|
|
|
175
|
+
// DDL is routed like every read: a writable backend (e.g. the
|
|
176
|
+
// contents_memory fixture) behind a composite would otherwise inherit
|
|
177
|
+
// CatalogInterface's read-only refusals. Every one takes attachOpaqueData
|
|
178
|
+
// first.
|
|
179
|
+
override async schemaCreate(...args: Parameters<CatalogInterface["schemaCreate"]>): Promise<void> {
|
|
180
|
+
await (this._route(args[0]).schemaCreate as (...a: typeof args) => unknown)(...args);
|
|
181
|
+
}
|
|
182
|
+
override async schemaDrop(...args: Parameters<CatalogInterface["schemaDrop"]>): Promise<void> {
|
|
183
|
+
await (this._route(args[0]).schemaDrop as (...a: typeof args) => unknown)(...args);
|
|
184
|
+
}
|
|
185
|
+
override async tableCreate(...args: Parameters<CatalogInterface["tableCreate"]>): Promise<void> {
|
|
186
|
+
await (this._route(args[0]).tableCreate as (...a: typeof args) => unknown)(...args);
|
|
187
|
+
}
|
|
188
|
+
override async tableDrop(...args: Parameters<CatalogInterface["tableDrop"]>): Promise<void> {
|
|
189
|
+
await (this._route(args[0]).tableDrop as (...a: typeof args) => unknown)(...args);
|
|
190
|
+
}
|
|
191
|
+
override async tableCommentSet(...args: Parameters<CatalogInterface["tableCommentSet"]>): Promise<void> {
|
|
192
|
+
await (this._route(args[0]).tableCommentSet as (...a: typeof args) => unknown)(...args);
|
|
193
|
+
}
|
|
194
|
+
override async tableRename(...args: Parameters<CatalogInterface["tableRename"]>): Promise<void> {
|
|
195
|
+
await (this._route(args[0]).tableRename as (...a: typeof args) => unknown)(...args);
|
|
196
|
+
}
|
|
197
|
+
override async tableColumnAdd(...args: Parameters<CatalogInterface["tableColumnAdd"]>): Promise<void> {
|
|
198
|
+
await (this._route(args[0]).tableColumnAdd as (...a: typeof args) => unknown)(...args);
|
|
199
|
+
}
|
|
200
|
+
override async tableColumnDrop(...args: Parameters<CatalogInterface["tableColumnDrop"]>): Promise<void> {
|
|
201
|
+
await (this._route(args[0]).tableColumnDrop as (...a: typeof args) => unknown)(...args);
|
|
202
|
+
}
|
|
203
|
+
override async tableColumnRename(...args: Parameters<CatalogInterface["tableColumnRename"]>): Promise<void> {
|
|
204
|
+
await (this._route(args[0]).tableColumnRename as (...a: typeof args) => unknown)(...args);
|
|
205
|
+
}
|
|
206
|
+
override async tableColumnDefaultSet(...args: Parameters<CatalogInterface["tableColumnDefaultSet"]>): Promise<void> {
|
|
207
|
+
await (this._route(args[0]).tableColumnDefaultSet as (...a: typeof args) => unknown)(...args);
|
|
208
|
+
}
|
|
209
|
+
override async tableColumnDefaultDrop(...args: Parameters<CatalogInterface["tableColumnDefaultDrop"]>): Promise<void> {
|
|
210
|
+
await (this._route(args[0]).tableColumnDefaultDrop as (...a: typeof args) => unknown)(...args);
|
|
211
|
+
}
|
|
212
|
+
override async tableColumnTypeChange(...args: Parameters<CatalogInterface["tableColumnTypeChange"]>): Promise<void> {
|
|
213
|
+
await (this._route(args[0]).tableColumnTypeChange as (...a: typeof args) => unknown)(...args);
|
|
214
|
+
}
|
|
215
|
+
override async tableNotNullSet(...args: Parameters<CatalogInterface["tableNotNullSet"]>): Promise<void> {
|
|
216
|
+
await (this._route(args[0]).tableNotNullSet as (...a: typeof args) => unknown)(...args);
|
|
217
|
+
}
|
|
218
|
+
override async tableNotNullDrop(...args: Parameters<CatalogInterface["tableNotNullDrop"]>): Promise<void> {
|
|
219
|
+
await (this._route(args[0]).tableNotNullDrop as (...a: typeof args) => unknown)(...args);
|
|
220
|
+
}
|
|
221
|
+
override async viewCreate(...args: Parameters<CatalogInterface["viewCreate"]>): Promise<void> {
|
|
222
|
+
await (this._route(args[0]).viewCreate as (...a: typeof args) => unknown)(...args);
|
|
223
|
+
}
|
|
224
|
+
override async viewDrop(...args: Parameters<CatalogInterface["viewDrop"]>): Promise<void> {
|
|
225
|
+
await (this._route(args[0]).viewDrop as (...a: typeof args) => unknown)(...args);
|
|
226
|
+
}
|
|
227
|
+
override async viewRename(...args: Parameters<CatalogInterface["viewRename"]>): Promise<void> {
|
|
228
|
+
await (this._route(args[0]).viewRename as (...a: typeof args) => unknown)(...args);
|
|
229
|
+
}
|
|
230
|
+
override async viewCommentSet(...args: Parameters<CatalogInterface["viewCommentSet"]>): Promise<void> {
|
|
231
|
+
await (this._route(args[0]).viewCommentSet as (...a: typeof args) => unknown)(...args);
|
|
232
|
+
}
|
|
233
|
+
override async macroCreate(...args: Parameters<CatalogInterface["macroCreate"]>): Promise<void> {
|
|
234
|
+
await (this._route(args[0]).macroCreate as (...a: typeof args) => unknown)(...args);
|
|
235
|
+
}
|
|
236
|
+
override async macroDrop(...args: Parameters<CatalogInterface["macroDrop"]>): Promise<void> {
|
|
237
|
+
await (this._route(args[0]).macroDrop as (...a: typeof args) => unknown)(...args);
|
|
238
|
+
}
|
|
239
|
+
|
|
240
|
+
// Routed whole, so a backend that overrides catalogContents (or keeps the
|
|
241
|
+
// default) answers for its own catalog.
|
|
242
|
+
override async catalogContents(
|
|
243
|
+
attachOpaqueData: AttachOpaqueData,
|
|
244
|
+
ifNoneMatch?: string | null,
|
|
245
|
+
): Promise<CatalogContentsResult> {
|
|
246
|
+
return await this._route(attachOpaqueData).catalogContents(attachOpaqueData, ifNoneMatch);
|
|
247
|
+
}
|
|
248
|
+
|
|
249
|
+
// The backend's etag / cache settings apply, and its instance keys the
|
|
250
|
+
// worker's catalog_contents cache.
|
|
251
|
+
override catalogContentsOwner(attachOpaqueData: AttachOpaqueData): CatalogInterface {
|
|
252
|
+
return this._route(attachOpaqueData).catalogContentsOwner(attachOpaqueData);
|
|
253
|
+
}
|
|
254
|
+
|
|
174
255
|
override async copyFromFormats(attachOpaqueData: AttachOpaqueData, transactionOpaqueData?: TransactionOpaqueData): Promise<CopyFromFormatInfo[]> {
|
|
175
256
|
return await this._route(attachOpaqueData).copyFromFormats(attachOpaqueData, transactionOpaqueData);
|
|
176
257
|
}
|
|
@@ -0,0 +1,78 @@
|
|
|
1
|
+
// Copyright 2025, 2026 Query Farm LLC - https://query.farm
|
|
2
|
+
// SPDX-License-Identifier: LicenseRef-QueryFarm-Source-Available-1.0
|
|
3
|
+
|
|
4
|
+
/**
|
|
5
|
+
* The `content-hash` etag for `catalog_contents`.
|
|
6
|
+
*
|
|
7
|
+
* Mirrors vgi-python's `catalog_contents_digest()` byte for byte, so the same
|
|
8
|
+
* snapshot (the same item bytes) gives the same etag in every SDK.
|
|
9
|
+
*/
|
|
10
|
+
|
|
11
|
+
import type { SchemaContents } from "../generated/vgi-protocol-types.js";
|
|
12
|
+
|
|
13
|
+
const KINDS = [
|
|
14
|
+
"tables",
|
|
15
|
+
"views",
|
|
16
|
+
"scalar_functions",
|
|
17
|
+
"aggregate_functions",
|
|
18
|
+
"table_functions",
|
|
19
|
+
"scalar_macros",
|
|
20
|
+
"table_macros",
|
|
21
|
+
"indexes",
|
|
22
|
+
] as const;
|
|
23
|
+
|
|
24
|
+
/**
|
|
25
|
+
* Hex SHA-256 over a `catalog_contents` snapshot: the `content-hash` etag.
|
|
26
|
+
*
|
|
27
|
+
* Covers every schema's path and the exact item bytes of every kind, in
|
|
28
|
+
* order, each length-prefixed (8-byte little-endian) so no two different
|
|
29
|
+
* snapshots share an input:
|
|
30
|
+
*
|
|
31
|
+
* u64(#schemas) { u64(#path) {u64(len) utf8(part)}* u64(len) schema
|
|
32
|
+
* ( u64(#items) {u64(len) item}* ) x 8 kinds }*
|
|
33
|
+
*
|
|
34
|
+
* Deterministic because the items' encoding is (map columns are written in
|
|
35
|
+
* sorted key order, see `encodeASD`), so two builds of the same catalog hash
|
|
36
|
+
* alike.
|
|
37
|
+
*
|
|
38
|
+
* @param schemas The snapshot, in wire order.
|
|
39
|
+
* @returns The lowercase hex digest.
|
|
40
|
+
*/
|
|
41
|
+
export async function catalogContentsDigest(schemas: readonly SchemaContents[]): Promise<string> {
|
|
42
|
+
const utf8 = new TextEncoder();
|
|
43
|
+
const parts: Uint8Array[] = [];
|
|
44
|
+
let total = 0;
|
|
45
|
+
const count = (n: number) => {
|
|
46
|
+
const b = new Uint8Array(8);
|
|
47
|
+
new DataView(b.buffer).setBigUint64(0, BigInt(n), true);
|
|
48
|
+
parts.push(b);
|
|
49
|
+
total += 8;
|
|
50
|
+
};
|
|
51
|
+
const chunk = (data: Uint8Array) => {
|
|
52
|
+
count(data.byteLength);
|
|
53
|
+
parts.push(data);
|
|
54
|
+
total += data.byteLength;
|
|
55
|
+
};
|
|
56
|
+
const chunks = (values: readonly Uint8Array[]) => {
|
|
57
|
+
count(values.length);
|
|
58
|
+
for (const v of values) chunk(v);
|
|
59
|
+
};
|
|
60
|
+
|
|
61
|
+
count(schemas.length);
|
|
62
|
+
for (const entry of schemas) {
|
|
63
|
+
chunks(entry.path.map((p) => utf8.encode(p)));
|
|
64
|
+
chunk(entry.schema);
|
|
65
|
+
for (const kind of KINDS) chunks(entry[kind]);
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
const buf = new Uint8Array(total);
|
|
69
|
+
let off = 0;
|
|
70
|
+
for (const p of parts) {
|
|
71
|
+
buf.set(p, off);
|
|
72
|
+
off += p.byteLength;
|
|
73
|
+
}
|
|
74
|
+
const digest = new Uint8Array(await crypto.subtle.digest("SHA-256", buf as BufferSource));
|
|
75
|
+
let hex = "";
|
|
76
|
+
for (const b of digest) hex += b.toString(16).padStart(2, "0");
|
|
77
|
+
return hex;
|
|
78
|
+
}
|
package/src/catalog/interface.ts
CHANGED
|
@@ -14,11 +14,16 @@ export type { MacroType };
|
|
|
14
14
|
|
|
15
15
|
// CatalogAttachResult is the generated snake_case wire shape.
|
|
16
16
|
// The `supports_column_statistics` field is a catalog-level opt-in for the
|
|
17
|
-
// catalog_table_column_statistics_get RPC —
|
|
18
|
-
// individual tables still need to set
|
|
19
|
-
// before DuckDB will actually call it.
|
|
20
|
-
|
|
21
|
-
|
|
17
|
+
// catalog_table_column_statistics_get RPC — the catalog_attach handler defaults
|
|
18
|
+
// it to true when unspecified; individual tables still need to set
|
|
19
|
+
// TableInfo.supports_column_statistics before DuckDB will actually call it.
|
|
20
|
+
//
|
|
21
|
+
// `buildCatalogAttachResult` (generated, src/generated/vgi-protocol-types.ts)
|
|
22
|
+
// fills every other field the protocol defaults, exactly as vgi-python's
|
|
23
|
+
// dataclass does — so an appended column needs no hand-written default here.
|
|
24
|
+
import type { CatalogAttachResult, CatalogAttachResultInit } from "../generated/vgi-protocol-types.js";
|
|
25
|
+
export type { CatalogAttachResult, CatalogAttachResultInit };
|
|
26
|
+
export { buildCatalogAttachResult } from "../generated/vgi-protocol-types.js";
|
|
22
27
|
|
|
23
28
|
// ============================================================================
|
|
24
29
|
// Serializable info types
|
|
@@ -495,6 +500,51 @@ export function buildScanBranchesResult(
|
|
|
495
500
|
};
|
|
496
501
|
}
|
|
497
502
|
|
|
503
|
+
// ============================================================================
|
|
504
|
+
// catalog_contents
|
|
505
|
+
// ============================================================================
|
|
506
|
+
|
|
507
|
+
/**
|
|
508
|
+
* One schema and everything in it, as returned by
|
|
509
|
+
* `CatalogInterface.catalogContents` (vgi-python's `SchemaContentsInfo`).
|
|
510
|
+
*
|
|
511
|
+
* Each kind is complete: an empty list means the schema has none of that kind.
|
|
512
|
+
* The `catalog_contents` handler encodes every item with the same encoder the
|
|
513
|
+
* matching per-schema RPC uses, so the items are byte-for-byte what
|
|
514
|
+
* `catalog_schemas` / `catalog_schema_contents_*` would return.
|
|
515
|
+
*/
|
|
516
|
+
export interface SchemaContentsInfo {
|
|
517
|
+
schema: SchemaInfo;
|
|
518
|
+
tables: TableInfo[];
|
|
519
|
+
views: ViewInfo[];
|
|
520
|
+
scalar_functions: FunctionInfo[];
|
|
521
|
+
aggregate_functions: FunctionInfo[];
|
|
522
|
+
table_functions: FunctionInfo[];
|
|
523
|
+
scalar_macros: MacroInfo[];
|
|
524
|
+
table_macros: MacroInfo[];
|
|
525
|
+
indexes: IndexInfo[];
|
|
526
|
+
}
|
|
527
|
+
|
|
528
|
+
/**
|
|
529
|
+
* What `CatalogInterface.catalogContents` returns: a snapshot, or "not
|
|
530
|
+
* modified" (vgi-python's `CatalogContentsResult`).
|
|
531
|
+
*
|
|
532
|
+
* - `schemas`: one `SchemaContentsInfo` per schema. Must be empty (or absent)
|
|
533
|
+
* when `not_modified` is set.
|
|
534
|
+
* - `etag`: opaque validator for this snapshot (a generation counter, schema
|
|
535
|
+
* version, git sha, ...). The client sends it back as `if_none_match`.
|
|
536
|
+
* Null/absent means the catalog does not revalidate -- unless it opts in to
|
|
537
|
+
* the framework's content hash with `catalogContentsEtag = "content-hash"`.
|
|
538
|
+
* - `not_modified`: true when the request's `if_none_match` equals the
|
|
539
|
+
* current etag, so the catalog skipped building the snapshot. Requires
|
|
540
|
+
* `etag` (the matching validator).
|
|
541
|
+
*/
|
|
542
|
+
export interface CatalogContentsResult {
|
|
543
|
+
schemas?: SchemaContentsInfo[];
|
|
544
|
+
etag?: string | null;
|
|
545
|
+
not_modified?: boolean;
|
|
546
|
+
}
|
|
547
|
+
|
|
498
548
|
// ============================================================================
|
|
499
549
|
// CatalogInterface abstract class
|
|
500
550
|
// ============================================================================
|
|
@@ -511,6 +561,39 @@ function isPromise<T>(v: Awaitable<T>): v is Promise<T> {
|
|
|
511
561
|
}
|
|
512
562
|
|
|
513
563
|
export abstract class CatalogInterface {
|
|
564
|
+
/**
|
|
565
|
+
* Opt-in framework etag for `catalog_contents`. `"content-hash"`: when
|
|
566
|
+
* `catalogContents` returns no etag of its own, the worker uses the hex
|
|
567
|
+
* SHA-256 of the serialized snapshot (`catalogContentsDigest`) as the etag
|
|
568
|
+
* and turns a matching `if_none_match` into `not_modified` (it still builds
|
|
569
|
+
* the snapshot, but saves the transfer and the client's decode). `null`
|
|
570
|
+
* (default): no etag unless the catalog returns one. Off by default because
|
|
571
|
+
* for a catalog whose version is not frozen the client would revalidate with
|
|
572
|
+
* a full build on every transaction instead of a cheap `catalog_version`
|
|
573
|
+
* poll. Mirrors vgi-python's `catalog_contents_etag`.
|
|
574
|
+
*/
|
|
575
|
+
catalogContentsEtag: "content-hash" | null = null;
|
|
576
|
+
|
|
577
|
+
/**
|
|
578
|
+
* Whether `catalogContents` depends only on `version()`: not on the attach
|
|
579
|
+
* (`attach_opaque_data`, attach options), the caller's identity, or anything
|
|
580
|
+
* else. Together with `catalogVersionFrozen`, it lets the worker build the
|
|
581
|
+
* `catalog_contents` response once per (catalog, version) and serve the same
|
|
582
|
+
* serialized bytes to every caller. Default false.
|
|
583
|
+
*
|
|
584
|
+
* `ReadOnlyCatalogInterface` sets it (its items carry a fixed attach id,
|
|
585
|
+
* like vgi-python's), unless its `perAttachItemIds` opt-in is on. A
|
|
586
|
+
* subclass whose contents vary per caller or attach option sets it false.
|
|
587
|
+
*/
|
|
588
|
+
catalogContentsAttachIndependent = false;
|
|
589
|
+
|
|
590
|
+
/**
|
|
591
|
+
* Declares that `version()` never changes for this catalog instance (the
|
|
592
|
+
* attach result's `catalog_version_frozen`). Only the `catalog_contents`
|
|
593
|
+
* worker cache reads it. `ReadOnlyCatalogInterface` sets it.
|
|
594
|
+
*/
|
|
595
|
+
catalogVersionFrozen = false;
|
|
596
|
+
|
|
514
597
|
/**
|
|
515
598
|
* Which catalog an `attach_opaque_data` belongs to, or null if unknown.
|
|
516
599
|
*
|
|
@@ -921,6 +1004,85 @@ export abstract class CatalogInterface {
|
|
|
921
1004
|
): Awaitable<CopyFromFormatInfo[]> {
|
|
922
1005
|
return [];
|
|
923
1006
|
}
|
|
1007
|
+
/**
|
|
1008
|
+
* Every schema and all of its contents, for the `catalog_contents` RPC.
|
|
1009
|
+
*
|
|
1010
|
+
* Served only to a client whose attach result set
|
|
1011
|
+
* `supports_catalog_contents` (ReadOnlyCatalogInterface does). The default
|
|
1012
|
+
* composes `schemas()` with the per-kind `schemaContents*()` calls, skipping
|
|
1013
|
+
* a kind whose `estimated_object_count` is exactly 0 (a hard guarantee), and
|
|
1014
|
+
* returns no etag -- so any catalog can serve it without new code. It runs
|
|
1015
|
+
* with no transaction: the client caches the answer for the whole attach.
|
|
1016
|
+
*
|
|
1017
|
+
* Override it to build the snapshot more cheaply, or to revalidate: return
|
|
1018
|
+
* an `etag` with the contents, and when `ifNoneMatch` equals the current
|
|
1019
|
+
* etag return `{ etag, not_modified: true }` *before* building anything.
|
|
1020
|
+
* Mirrors vgi-python's `CatalogInterface.catalog_contents`.
|
|
1021
|
+
*
|
|
1022
|
+
* @param ifNoneMatch The etag of the snapshot the client already holds, or
|
|
1023
|
+
* null.
|
|
1024
|
+
* @returns One `SchemaContentsInfo` per schema, in `schemas()` order (the
|
|
1025
|
+
* handler sorts parents before children), plus its etag -- or
|
|
1026
|
+
* `not_modified`.
|
|
1027
|
+
*/
|
|
1028
|
+
async catalogContents(
|
|
1029
|
+
attachOpaqueData: AttachOpaqueData,
|
|
1030
|
+
_ifNoneMatch?: string | null,
|
|
1031
|
+
): Promise<CatalogContentsResult> {
|
|
1032
|
+
// The default has no etag, so it never revalidates and ignores ifNoneMatch.
|
|
1033
|
+
const schemas = await this.schemas(attachOpaqueData, undefined);
|
|
1034
|
+
const contents = await Promise.all(
|
|
1035
|
+
schemas.map(async (schema): Promise<SchemaContentsInfo> => {
|
|
1036
|
+
const counts = schema.estimated_object_count ?? {};
|
|
1037
|
+
const path = schema.path;
|
|
1038
|
+
// A missing count is unknown, never zero: only an explicit 0 skips.
|
|
1039
|
+
const kind = async <T>(countKey: string, list: () => Awaitable<T[]>): Promise<T[]> =>
|
|
1040
|
+
(counts[countKey] ?? 1) === 0 ? [] : [...(await list())];
|
|
1041
|
+
const [tables, views, scalarFunctions, aggregateFunctions, tableFunctions, scalarMacros, tableMacros, indexes] =
|
|
1042
|
+
await Promise.all([
|
|
1043
|
+
kind("table", () => this.schemaContentsTables(attachOpaqueData, path)),
|
|
1044
|
+
kind("view", () => this.schemaContentsViews(attachOpaqueData, path)),
|
|
1045
|
+
kind("scalar_function", () => this.schemaContentsFunctions(attachOpaqueData, path, "scalar_function")),
|
|
1046
|
+
kind("aggregate_function", () => this.schemaContentsFunctions(attachOpaqueData, path, "aggregate_function")),
|
|
1047
|
+
kind("table_function", () => this.schemaContentsFunctions(attachOpaqueData, path, "table_function")),
|
|
1048
|
+
kind("macro", () => this.schemaContentsMacros(attachOpaqueData, path, "scalar_macro")),
|
|
1049
|
+
kind("macro", () => this.schemaContentsMacros(attachOpaqueData, path, "table_macro")),
|
|
1050
|
+
kind("index", () => this.schemaContentsIndexes(attachOpaqueData, path)),
|
|
1051
|
+
]);
|
|
1052
|
+
return {
|
|
1053
|
+
schema,
|
|
1054
|
+
tables: tables as TableInfo[],
|
|
1055
|
+
views: views as ViewInfo[],
|
|
1056
|
+
scalar_functions: scalarFunctions as FunctionInfo[],
|
|
1057
|
+
aggregate_functions: aggregateFunctions as FunctionInfo[],
|
|
1058
|
+
table_functions: tableFunctions as FunctionInfo[],
|
|
1059
|
+
scalar_macros: scalarMacros as MacroInfo[],
|
|
1060
|
+
table_macros: tableMacros as MacroInfo[],
|
|
1061
|
+
indexes: indexes as IndexInfo[],
|
|
1062
|
+
};
|
|
1063
|
+
}),
|
|
1064
|
+
);
|
|
1065
|
+
return { schemas: contents };
|
|
1066
|
+
}
|
|
1067
|
+
|
|
1068
|
+
/**
|
|
1069
|
+
* Whether the worker may build `catalog_contents` once per (catalog
|
|
1070
|
+
* instance, version) and reuse the serialized response for every caller:
|
|
1071
|
+
* `catalogContentsAttachIndependent && catalogVersionFrozen` by default.
|
|
1072
|
+
*/
|
|
1073
|
+
catalogContentsCacheable(): boolean {
|
|
1074
|
+
return this.catalogContentsAttachIndependent && this.catalogVersionFrozen;
|
|
1075
|
+
}
|
|
1076
|
+
|
|
1077
|
+
/**
|
|
1078
|
+
* The catalog that answers `catalog_contents` for this attach: whose
|
|
1079
|
+
* `catalogContentsEtag` / `catalogContentsAttachIndependent` /
|
|
1080
|
+
* `catalogVersionFrozen` the handler honours and whose instance keys the
|
|
1081
|
+
* worker cache. `this`, except for a composite, which routes to its backend.
|
|
1082
|
+
*/
|
|
1083
|
+
catalogContentsOwner(_attachOpaqueData: AttachOpaqueData): CatalogInterface {
|
|
1084
|
+
return this;
|
|
1085
|
+
}
|
|
924
1086
|
transactionBegin(attachOpaqueData: AttachOpaqueData): Awaitable<Uint8Array | null> {
|
|
925
1087
|
return null;
|
|
926
1088
|
}
|
package/src/catalog/read-only.ts
CHANGED
|
@@ -18,6 +18,7 @@ import {
|
|
|
18
18
|
type CopyFromFormatInfo,
|
|
19
19
|
encodeAttachCatalogInfo,
|
|
20
20
|
encodeScanFunctionResult,
|
|
21
|
+
buildCatalogAttachResult,
|
|
21
22
|
} from "./interface.js";
|
|
22
23
|
import { schemaDescriptorPath, type CatalogDescriptor, type SchemaDescriptor, type TableDescriptor, type ViewDescriptor, type MacroDescriptor, type SettingDescriptor, type SecretTypeDescriptor, type ForeignKeyDef, type DefaultValue } from "./descriptors.js";
|
|
23
24
|
import { serializeColumnStatistics } from "../util/statistics.js";
|
|
@@ -33,7 +34,48 @@ import { normalizeSchemaPath, schemaPathDisplay, schemaPathsEqual } from "../sch
|
|
|
33
34
|
import { ForeignKeyInfoSchema } from "../generated/vgi-protocol-schemas.js";
|
|
34
35
|
import { encodeFunctionInfoOnce, freezeCatalogItem } from "./item-encoding.js";
|
|
35
36
|
|
|
37
|
+
/**
|
|
38
|
+
* The fixed attach id ReadOnlyCatalogInterface writes into its items (same
|
|
39
|
+
* bytes as vgi-python's `ReadOnlyCatalogInterface._FIXED_ATTACH_ID`). Shared:
|
|
40
|
+
* treat it as read-only (a typed array cannot be frozen).
|
|
41
|
+
*/
|
|
42
|
+
export const READ_ONLY_ITEM_ATTACH_ID: AttachOpaqueData = new TextEncoder().encode("readonly-catalog-");
|
|
43
|
+
|
|
36
44
|
export class ReadOnlyCatalogInterface extends CatalogInterface {
|
|
45
|
+
/**
|
|
46
|
+
* Advertise `supports_catalog_contents` on attach, letting the client load
|
|
47
|
+
* the whole catalog with one `catalog_contents` call instead of
|
|
48
|
+
* `catalog_schemas` plus a `catalog_schema_contents_*` call per schema and
|
|
49
|
+
* kind. True by default — a descriptor catalog is static. Set it to false
|
|
50
|
+
* (before attach) to keep the client on the per-schema RPCs.
|
|
51
|
+
*/
|
|
52
|
+
supportsCatalogContents = true;
|
|
53
|
+
|
|
54
|
+
/** A descriptor catalog's version never changes (`catalog_version_frozen`). */
|
|
55
|
+
override catalogVersionFrozen = true;
|
|
56
|
+
|
|
57
|
+
/**
|
|
58
|
+
* The catalog items (`SchemaInfo.attach_opaque_data`, and the attach value
|
|
59
|
+
* handed to a table function's bind when resolving a table's columns) carry
|
|
60
|
+
* one fixed id, `READ_ONLY_ITEM_ATTACH_ID`, not the per-ATTACH id -- as
|
|
61
|
+
* vgi-python's ReadOnlyCatalogInterface does. So the items are the same for
|
|
62
|
+
* every attach and caller, and the worker caches the `catalog_contents`
|
|
63
|
+
* response (see `catalogContentsCacheable`).
|
|
64
|
+
*
|
|
65
|
+
* `attach()` still mints a random per-ATTACH `attach_opaque_data` (sealed
|
|
66
|
+
* into the per-attach envelope the client holds), and function calls still
|
|
67
|
+
* receive it, so per-attach scoping in functions (the accumulate example) is
|
|
68
|
+
* unchanged.
|
|
69
|
+
*
|
|
70
|
+
* Set true (or pass `{ perAttachItemIds: true }` to the constructor) for a
|
|
71
|
+
* catalog whose items must echo the per-attach id. Its contents then depend
|
|
72
|
+
* on the attach, so the catalog_contents cache is off for it.
|
|
73
|
+
*/
|
|
74
|
+
perAttachItemIds = false;
|
|
75
|
+
|
|
76
|
+
/** Static contents: attach-independent unless `perAttachItemIds` is set. */
|
|
77
|
+
override catalogContentsAttachIndependent = true;
|
|
78
|
+
|
|
37
79
|
private _descriptor: CatalogDescriptor;
|
|
38
80
|
private _registry: FunctionRegistry;
|
|
39
81
|
private _attachments = new Map<string, AttachOpaqueData>();
|
|
@@ -50,10 +92,15 @@ export class ReadOnlyCatalogInterface extends CatalogInterface {
|
|
|
50
92
|
/** One frozen FunctionInfo per (declaring schema, function), shared by listings and `global_functions`. */
|
|
51
93
|
private _functionInfos = new Map<SchemaDescriptor, Map<VgiFunction, FunctionInfo>>();
|
|
52
94
|
|
|
53
|
-
constructor(
|
|
95
|
+
constructor(
|
|
96
|
+
descriptor: CatalogDescriptor,
|
|
97
|
+
registry: FunctionRegistry,
|
|
98
|
+
options: { perAttachItemIds?: boolean } = {},
|
|
99
|
+
) {
|
|
54
100
|
super();
|
|
55
101
|
this._descriptor = descriptor;
|
|
56
102
|
this._registry = registry;
|
|
103
|
+
if (options.perAttachItemIds) this.perAttachItemIds = true;
|
|
57
104
|
// Index every function this catalog declares by its owning schema, so a
|
|
58
105
|
// schema-qualified bind resolves to the right implementation when one name
|
|
59
106
|
// is declared in more than one schema. Table-backing functions are indexed
|
|
@@ -161,7 +208,7 @@ export class ReadOnlyCatalogInterface extends CatalogInterface {
|
|
|
161
208
|
(s) => s.tables?.some((t) => t.supportsTimeTravel) ?? false
|
|
162
209
|
);
|
|
163
210
|
|
|
164
|
-
return {
|
|
211
|
+
return buildCatalogAttachResult({
|
|
165
212
|
attach_opaque_data: attachOpaqueData,
|
|
166
213
|
supports_transactions: false,
|
|
167
214
|
supports_time_travel: hasTimeTravel,
|
|
@@ -184,7 +231,10 @@ export class ReadOnlyCatalogInterface extends CatalogInterface {
|
|
|
184
231
|
global_function_prefix: this._descriptor.globalFunctionPrefix ?? "",
|
|
185
232
|
resolved_data_version: null,
|
|
186
233
|
resolved_implementation_version: null,
|
|
187
|
-
|
|
234
|
+
// Static, version-frozen catalog: the whole thing can be served in one
|
|
235
|
+
// catalog_contents call (as vgi-python's ReadOnlyCatalogInterface does).
|
|
236
|
+
supports_catalog_contents: this.supportsCatalogContents,
|
|
237
|
+
});
|
|
188
238
|
}
|
|
189
239
|
|
|
190
240
|
detach(attachOpaqueData: AttachOpaqueData): void {
|
|
@@ -201,9 +251,21 @@ export class ReadOnlyCatalogInterface extends CatalogInterface {
|
|
|
201
251
|
return this._version;
|
|
202
252
|
}
|
|
203
253
|
|
|
254
|
+
/**
|
|
255
|
+
* The attach value written into this catalog's items: the fixed
|
|
256
|
+
* `READ_ONLY_ITEM_ATTACH_ID`, or the caller's attach with `perAttachItemIds`.
|
|
257
|
+
*/
|
|
258
|
+
protected itemAttachId(attachOpaqueData: AttachOpaqueData): AttachOpaqueData {
|
|
259
|
+
return this.perAttachItemIds ? attachOpaqueData : READ_ONLY_ITEM_ATTACH_ID;
|
|
260
|
+
}
|
|
261
|
+
|
|
262
|
+
override catalogContentsCacheable(): boolean {
|
|
263
|
+
return super.catalogContentsCacheable() && !this.perAttachItemIds;
|
|
264
|
+
}
|
|
265
|
+
|
|
204
266
|
schemas(attachOpaqueData: AttachOpaqueData, transactionOpaqueData?: TransactionOpaqueData): SchemaInfo[] {
|
|
205
267
|
return this._descriptor.schemas.map((s) => ({
|
|
206
|
-
attach_opaque_data: attachOpaqueData,
|
|
268
|
+
attach_opaque_data: this.itemAttachId(attachOpaqueData),
|
|
207
269
|
path: schemaDescriptorPath(s),
|
|
208
270
|
comment: s.comment ?? null,
|
|
209
271
|
tags: s.tags ?? {},
|
|
@@ -241,7 +303,7 @@ export class ReadOnlyCatalogInterface extends CatalogInterface {
|
|
|
241
303
|
const desc = this._descriptor.schemas.find((s) => schemaPathsEqual(schemaDescriptorPath(s), path));
|
|
242
304
|
if (!desc) return null;
|
|
243
305
|
return {
|
|
244
|
-
attach_opaque_data: attachOpaqueData,
|
|
306
|
+
attach_opaque_data: this.itemAttachId(attachOpaqueData),
|
|
245
307
|
path: schemaDescriptorPath(desc),
|
|
246
308
|
comment: desc.comment ?? null,
|
|
247
309
|
tags: desc.tags ?? {},
|
|
@@ -273,7 +335,7 @@ export class ReadOnlyCatalogInterface extends CatalogInterface {
|
|
|
273
335
|
input_schema: null,
|
|
274
336
|
settings: null,
|
|
275
337
|
secrets: null,
|
|
276
|
-
attach_opaque_data: attachOpaqueData,
|
|
338
|
+
attach_opaque_data: this.itemAttachId(attachOpaqueData),
|
|
277
339
|
transaction_opaque_data: null,
|
|
278
340
|
resolved_secrets_provided: false,
|
|
279
341
|
};
|
package/src/client/client.ts
CHANGED
|
@@ -40,6 +40,8 @@ import {
|
|
|
40
40
|
decodeMacroInfo,
|
|
41
41
|
type MacroType,
|
|
42
42
|
type CatalogAttachResult,
|
|
43
|
+
type SchemaContentsInfo,
|
|
44
|
+
decodeIndexInfo,
|
|
43
45
|
type AttachOpaqueData,
|
|
44
46
|
type TransactionOpaqueData,
|
|
45
47
|
decodeCatalogInfo,
|
|
@@ -54,6 +56,7 @@ import { VgiClientError, wrapRpcWithErrorEnrichment } from "./errors.js";
|
|
|
54
56
|
import { normalizeSchemaPath } from "../schema-path.js";
|
|
55
57
|
import { encodeASD } from "../codec/asd.js";
|
|
56
58
|
import { ClientCapabilitiesSchema, MacroCreateRequestSchema } from "../generated/vgi-protocol-schemas.js";
|
|
59
|
+
import { decodeCatalogContentsResponse } from "../generated/vgi-protocol-types.js";
|
|
57
60
|
import { deserializeInfoList, deserializeTags, toAsyncIterator } from "./helpers.js";
|
|
58
61
|
|
|
59
62
|
export { VgiClientError };
|
|
@@ -678,6 +681,56 @@ export class VgiClient {
|
|
|
678
681
|
global_function_prefix: inner.global_function_prefix ?? "",
|
|
679
682
|
resolved_data_version: inner.resolved_data_version ?? null,
|
|
680
683
|
resolved_implementation_version: inner.resolved_implementation_version ?? null,
|
|
684
|
+
supports_catalog_contents: inner.supports_catalog_contents ?? false,
|
|
685
|
+
};
|
|
686
|
+
}
|
|
687
|
+
|
|
688
|
+
/**
|
|
689
|
+
* Load the whole catalog in one `catalog_contents` call: every schema (parents
|
|
690
|
+
* before children) with all of its tables, views, functions, macros and
|
|
691
|
+
* indexes. Only call it when the attach result set
|
|
692
|
+
* `supports_catalog_contents`; it is the committed catalog at
|
|
693
|
+
* `catalog_version` (no transaction).
|
|
694
|
+
*
|
|
695
|
+
* Pass the `etag` of a snapshot you hold as `ifNoneMatch`: when it is still
|
|
696
|
+
* current the worker answers `not_modified` with no schemas. `etag` null
|
|
697
|
+
* means the worker does not revalidate. Each schema's wire `path` is checked
|
|
698
|
+
* against its decoded `SchemaInfo.path`.
|
|
699
|
+
*/
|
|
700
|
+
async catalogContents(
|
|
701
|
+
attachOpaqueData: AttachOpaqueData,
|
|
702
|
+
ifNoneMatch: string | null = null,
|
|
703
|
+
): Promise<{ catalog_version: number; etag: string | null; not_modified: boolean; schemas: SchemaContentsInfo[] }> {
|
|
704
|
+
const result = await this.rpc.call("catalog_contents", {
|
|
705
|
+
attach_opaque_data: attachOpaqueData,
|
|
706
|
+
if_none_match: ifNoneMatch,
|
|
707
|
+
});
|
|
708
|
+
if (!result) throw new VgiClientError("catalog_contents returned null");
|
|
709
|
+
const response = decodeCatalogContentsResponse(toUint8Array(result.result));
|
|
710
|
+
const schemas = response.schemas.map((c) => {
|
|
711
|
+
const schema = decodeSchemaInfo(c.schema);
|
|
712
|
+
if (JSON.stringify(c.path) !== JSON.stringify(schema.path)) {
|
|
713
|
+
throw new VgiClientError(
|
|
714
|
+
`catalog_contents: SchemaContents.path ${JSON.stringify(c.path)} != SchemaInfo.path ${JSON.stringify(schema.path)}`,
|
|
715
|
+
);
|
|
716
|
+
}
|
|
717
|
+
return {
|
|
718
|
+
schema,
|
|
719
|
+
tables: c.tables.map(decodeTableInfo),
|
|
720
|
+
views: c.views.map(decodeViewInfo),
|
|
721
|
+
scalar_functions: c.scalar_functions.map(decodeFunctionInfo),
|
|
722
|
+
aggregate_functions: c.aggregate_functions.map(decodeFunctionInfo),
|
|
723
|
+
table_functions: c.table_functions.map(decodeFunctionInfo),
|
|
724
|
+
scalar_macros: c.scalar_macros.map(decodeMacroInfo),
|
|
725
|
+
table_macros: c.table_macros.map(decodeMacroInfo),
|
|
726
|
+
indexes: c.indexes.map(decodeIndexInfo),
|
|
727
|
+
};
|
|
728
|
+
});
|
|
729
|
+
return {
|
|
730
|
+
catalog_version: Number(response.catalog_version ?? 0),
|
|
731
|
+
etag: response.etag ?? null,
|
|
732
|
+
not_modified: Boolean(response.not_modified),
|
|
733
|
+
schemas,
|
|
681
734
|
};
|
|
682
735
|
}
|
|
683
736
|
|