@query-farm/vgi 0.21.0 → 0.22.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cache-control.d.ts +26 -0
- package/dist/catalog/composite.d.ts +2 -0
- package/dist/catalog/interface.d.ts +11 -0
- package/dist/client-entry.js +5 -3
- package/dist/client-entry.js.map +5 -5
- package/dist/functions/registry.d.ts +31 -0
- package/dist/functions/scalar.d.ts +7 -3
- package/dist/generated/vgi-client.d.ts +16 -0
- package/dist/index.core.d.ts +1 -1
- package/dist/index.js +120 -16
- package/dist/index.js.map +15 -15
- package/dist/protocol/handlers/function.d.ts +8 -0
- package/dist/protocol/handlers/shared.d.ts +4 -0
- package/dist/protocol/types.d.ts +8 -0
- package/dist/serve-entry.js +35 -13
- package/dist/serve-entry.js.map +8 -8
- package/dist/worker-cf-entry.js +115 -14
- package/dist/worker-cf-entry.js.map +13 -13
- package/dist/worker.d.ts +9 -0
- package/package.json +3 -3
- package/src/cache-control.ts +27 -0
- package/src/catalog/composite.ts +9 -0
- package/src/catalog/interface.ts +15 -0
- package/src/catalog/read-only.ts +13 -0
- package/src/functions/registry.ts +106 -1
- package/src/functions/scalar.ts +10 -5
- package/src/generated/vgi-client.ts +17 -1
- package/src/index.core.ts +1 -0
- package/src/protocol/dispatch.ts +1 -0
- package/src/protocol/handlers/function.ts +29 -8
- package/src/protocol/handlers/shared.ts +25 -1
- package/src/protocol/serializers/bind.ts +7 -0
- package/src/protocol/types.ts +8 -0
- package/src/worker.ts +15 -1
package/dist/worker.d.ts
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { FunctionRegistry } from "./functions/registry.js";
|
|
1
2
|
import type { VgiFunction } from "./functions/types.js";
|
|
2
3
|
import type { CatalogDescriptor } from "./catalog/descriptors.js";
|
|
3
4
|
import type { CatalogInterface } from "./catalog/interface.js";
|
|
@@ -9,6 +10,14 @@ export interface WorkerConfig {
|
|
|
9
10
|
/** Factory that receives the built ReadOnlyCatalogInterface and returns a custom catalog. */
|
|
10
11
|
catalogInterfaceFactory?: (base: ReadOnlyCatalogInterface) => CatalogInterface;
|
|
11
12
|
catalogName?: string;
|
|
13
|
+
/**
|
|
14
|
+
* Registry to dispatch through. Pass the same instance used to build a
|
|
15
|
+
* pre-constructed `catalogInterface` — catalogs record which schema (and
|
|
16
|
+
* catalog) declares each function on the registry they are given, and
|
|
17
|
+
* schema-qualified resolution only works if that is the registry the worker
|
|
18
|
+
* dispatches on. Omit to have the worker create its own.
|
|
19
|
+
*/
|
|
20
|
+
registry?: FunctionRegistry;
|
|
12
21
|
}
|
|
13
22
|
export declare class Worker {
|
|
14
23
|
private _registry;
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@query-farm/vgi",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.22.0",
|
|
4
4
|
"description": "TypeScript framework for extending DuckDB with user-defined functions via the Vector Gateway Interface (VGI) over Arrow IPC.",
|
|
5
5
|
"license": "SEE LICENSE IN LICENSE",
|
|
6
6
|
"author": "Query Farm LLC (https://query.farm)",
|
|
@@ -87,12 +87,12 @@
|
|
|
87
87
|
},
|
|
88
88
|
"peerDependencies": {
|
|
89
89
|
"@query-farm/apache-arrow": "^21.1.1",
|
|
90
|
-
"@query-farm/vgi-rpc": "^0.15.
|
|
90
|
+
"@query-farm/vgi-rpc": "^0.15.4"
|
|
91
91
|
},
|
|
92
92
|
"devDependencies": {
|
|
93
93
|
"@duckdb/node-api": "^1.5.2-r.1",
|
|
94
94
|
"@query-farm/apache-arrow": "^21.1.1",
|
|
95
|
-
"@query-farm/vgi-rpc": "^0.15.
|
|
95
|
+
"@query-farm/vgi-rpc": "^0.15.4",
|
|
96
96
|
"@types/bun": "latest",
|
|
97
97
|
"typescript": "^6.0.3"
|
|
98
98
|
},
|
package/src/cache-control.ts
CHANGED
|
@@ -42,6 +42,7 @@ export const CACHE_STALE_WHILE_REVALIDATE_KEY = "vgi.cache.stale_while_revalidat
|
|
|
42
42
|
export const CACHE_STALE_IF_ERROR_KEY = "vgi.cache.stale_if_error";
|
|
43
43
|
export const CACHE_NOT_MODIFIED_KEY = "vgi.cache.not_modified";
|
|
44
44
|
export const CACHE_PARTITION_SCOPE_KEY = "vgi.cache.partition_scope";
|
|
45
|
+
export const CACHE_PER_VALUE_KEY = "vgi.cache.per_value";
|
|
45
46
|
|
|
46
47
|
// --- Request-side metadata keys (client -> worker) --------------------------
|
|
47
48
|
// Ride on the first producer tick; surfaced as `TableProcessParams.ifNoneMatch`
|
|
@@ -98,6 +99,31 @@ export interface CacheControl {
|
|
|
98
99
|
* partitions without calling the worker. Only meaningful for a
|
|
99
100
|
* `SINGLE_VALUE_PARTITIONS` table function; purely additive. */
|
|
100
101
|
partitionScope?: boolean;
|
|
102
|
+
/**
|
|
103
|
+
* Opt in to per-VALUE memoization: in addition to the whole-result entry, the
|
|
104
|
+
* client memoizes each distinct worker-input tuple's output keyed on that
|
|
105
|
+
* tuple, so the same value is served without calling the worker on a later
|
|
106
|
+
* chunk or a later query. Only meaningful for an exchange-mode MAP — a
|
|
107
|
+
* scalar, or a blended table-in-out invoked through a correlated `LATERAL`.
|
|
108
|
+
*
|
|
109
|
+
* **Default off, and it should stay off unless one call is genuinely
|
|
110
|
+
* expensive.** This is not a free win: serving a value from the memo costs a
|
|
111
|
+
* key probe, a decode and a per-value assembly step, and that only pays back
|
|
112
|
+
* when it is cheaper than asking you for the answer. For a cheap map —
|
|
113
|
+
* arithmetic, a string tweak, a lookup in a table the worker already has in
|
|
114
|
+
* memory — it is a large net loss (the engine measures a per-value serve at
|
|
115
|
+
* roughly 50x the cost of just calling the worker for a simple arithmetic
|
|
116
|
+
* map). Only the function author knows which side of that line a call falls
|
|
117
|
+
* on, which is why the engine will not guess: turn it on for model
|
|
118
|
+
* inference, geocoding, an external API call, or anything rate-limited or
|
|
119
|
+
* billed per request.
|
|
120
|
+
*
|
|
121
|
+
* Independent of freshness: `ttl`/`expires` make a result cacheable at all,
|
|
122
|
+
* while this decides whether the per-value tier is additionally populated.
|
|
123
|
+
* Requires a deterministic, side-effect-free function, exactly like the rest
|
|
124
|
+
* of the result cache.
|
|
125
|
+
*/
|
|
126
|
+
perValue?: boolean;
|
|
101
127
|
}
|
|
102
128
|
|
|
103
129
|
const VALID_SCOPES: readonly string[] = [CACHE_SCOPE_CATALOG, CACHE_SCOPE_TRANSACTION];
|
|
@@ -137,5 +163,6 @@ export function cacheControlMetadata(cc: CacheControl, extra?: Map<string, strin
|
|
|
137
163
|
if (cc.staleIfError !== undefined) md.set(CACHE_STALE_IF_ERROR_KEY, String(cc.staleIfError));
|
|
138
164
|
if (cc.notModified) md.set(CACHE_NOT_MODIFIED_KEY, "1");
|
|
139
165
|
if (cc.partitionScope) md.set(CACHE_PARTITION_SCOPE_KEY, "1");
|
|
166
|
+
if (cc.perValue) md.set(CACHE_PER_VALUE_KEY, "1");
|
|
140
167
|
return md;
|
|
141
168
|
}
|
package/src/catalog/composite.ts
CHANGED
|
@@ -56,6 +56,15 @@ export class CompositeCatalogInterface extends CatalogInterface {
|
|
|
56
56
|
return route;
|
|
57
57
|
}
|
|
58
58
|
|
|
59
|
+
/** Route to the owning backend and ask it, so the answer is per-attachment. */
|
|
60
|
+
override catalogNameForAttach(attachOpaqueData: Uint8Array): string | null {
|
|
61
|
+
try {
|
|
62
|
+
return this._route(attachOpaqueData).catalogNameForAttach(attachOpaqueData);
|
|
63
|
+
} catch {
|
|
64
|
+
return null; // Unroutable attach — fall back to unscoped resolution.
|
|
65
|
+
}
|
|
66
|
+
}
|
|
67
|
+
|
|
59
68
|
catalogs(): string[] {
|
|
60
69
|
const all: string[] = [];
|
|
61
70
|
for (const b of this._backends) all.push(...b.catalogs());
|
package/src/catalog/interface.ts
CHANGED
|
@@ -365,6 +365,21 @@ function isPromise<T>(v: Awaitable<T>): v is Promise<T> {
|
|
|
365
365
|
}
|
|
366
366
|
|
|
367
367
|
export abstract class CatalogInterface {
|
|
368
|
+
/**
|
|
369
|
+
* Which catalog an `attach_opaque_data` belongs to, or null if unknown.
|
|
370
|
+
*
|
|
371
|
+
* Two catalogs served by one worker may declare the same schema *and*
|
|
372
|
+
* function name, in which case the attachment is the only thing that tells
|
|
373
|
+
* them apart — the bind path uses this to scope function resolution. The
|
|
374
|
+
* default answers from `catalogs()` when this interface owns exactly one,
|
|
375
|
+
* which is right for every single-catalog implementation; the composite
|
|
376
|
+
* overrides it to route by its backend index byte.
|
|
377
|
+
*/
|
|
378
|
+
catalogNameForAttach(_attachOpaqueData: Uint8Array): string | null {
|
|
379
|
+
const names = this.catalogs();
|
|
380
|
+
return names.length === 1 ? names[0] : null;
|
|
381
|
+
}
|
|
382
|
+
|
|
368
383
|
// Required overrides
|
|
369
384
|
abstract catalogs(): string[];
|
|
370
385
|
/**
|
package/src/catalog/read-only.ts
CHANGED
|
@@ -38,6 +38,19 @@ export class ReadOnlyCatalogInterface extends CatalogInterface {
|
|
|
38
38
|
super();
|
|
39
39
|
this._descriptor = descriptor;
|
|
40
40
|
this._registry = registry;
|
|
41
|
+
// Index every function this catalog declares by its owning schema, so a
|
|
42
|
+
// schema-qualified bind resolves to the right implementation when one name
|
|
43
|
+
// is declared in more than one schema. Table-backing functions are indexed
|
|
44
|
+
// under their table's schema for the same reason.
|
|
45
|
+
for (const sch of descriptor.schemas ?? []) {
|
|
46
|
+
for (const fn of sch.functions ?? []) registry.registerInSchema(fn, sch.name, descriptor.name);
|
|
47
|
+
for (const tbl of sch.tables ?? []) {
|
|
48
|
+
for (const key of ["function", "insertFunction", "updateFunction", "deleteFunction"] as const) {
|
|
49
|
+
const fn = (tbl as any)[key];
|
|
50
|
+
if (fn) registry.registerInSchema(fn, sch.name, descriptor.name);
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
}
|
|
41
54
|
}
|
|
42
55
|
|
|
43
56
|
catalogs(): string[] {
|
|
@@ -16,6 +16,20 @@ export interface OverloadContext {
|
|
|
16
16
|
arguments?: Arguments;
|
|
17
17
|
inputSchema?: VgiSchema | null;
|
|
18
18
|
isScalar?: boolean;
|
|
19
|
+
/**
|
|
20
|
+
* Catalog schema that declares the function, from `BindRequest.schema_name`.
|
|
21
|
+
* A worker may register the same name in more than one schema, so the bare
|
|
22
|
+
* name is not a unique key. When set, resolution is scoped to that schema and
|
|
23
|
+
* never falls back to another one; when absent, every schema is searched.
|
|
24
|
+
*/
|
|
25
|
+
schemaName?: string | null;
|
|
26
|
+
/**
|
|
27
|
+
* Catalog that owns the function, resolved from the bind's
|
|
28
|
+
* `attach_opaque_data`. Two catalogs served by one worker may each declare
|
|
29
|
+
* the same schema *and* function name, in which case only the catalog tells
|
|
30
|
+
* them apart. Absent for calls with no attachment.
|
|
31
|
+
*/
|
|
32
|
+
catalogName?: string | null;
|
|
19
33
|
}
|
|
20
34
|
|
|
21
35
|
const EXACT_MATCH_SCORE = 2;
|
|
@@ -190,8 +204,26 @@ function filterByArgumentTypes(
|
|
|
190
204
|
return scored.filter(s => s.score === maxScore).map(s => s.func);
|
|
191
205
|
}
|
|
192
206
|
|
|
207
|
+
/** Key for the schema-scoped index: lowercased schema, then function name. */
|
|
208
|
+
function schemaKey(schemaName: string, functionName: string): string {
|
|
209
|
+
return `${schemaName.toLowerCase()}\u0000${functionName}`;
|
|
210
|
+
}
|
|
211
|
+
|
|
212
|
+
/** Key for the catalog-scoped index: lowercased catalog, then the schema key. */
|
|
213
|
+
function catalogKey(catalogName: string, schemaName: string, functionName: string): string {
|
|
214
|
+
return `${catalogName.toLowerCase()}\u0000${schemaKey(schemaName, functionName)}`;
|
|
215
|
+
}
|
|
216
|
+
|
|
193
217
|
export class FunctionRegistry {
|
|
194
218
|
private _functions: Map<string, VgiFunction[]> = new Map();
|
|
219
|
+
// (lowercased schema, function name) -> functions declared in that schema.
|
|
220
|
+
// Populated by registerInSchema(), which catalog construction calls for every
|
|
221
|
+
// function a SchemaDescriptor lists. Lets a schema-qualified bind pick the
|
|
222
|
+
// right implementation when one name is registered in several schemas.
|
|
223
|
+
private _bySchema: Map<string, VgiFunction[]> = new Map();
|
|
224
|
+
// (lowercased catalog, lowercased schema, function name) -> functions. Needed
|
|
225
|
+
// because two catalogs in one worker may declare the same schema AND name.
|
|
226
|
+
private _byCatalog: Map<string, VgiFunction[]> = new Map();
|
|
195
227
|
|
|
196
228
|
register(func: VgiFunction): void {
|
|
197
229
|
const name = func.meta.name;
|
|
@@ -201,12 +233,85 @@ export class FunctionRegistry {
|
|
|
201
233
|
this._functions.get(name)!.push(func);
|
|
202
234
|
}
|
|
203
235
|
|
|
236
|
+
/**
|
|
237
|
+
* Record that `func` is declared in `schemaName`, in addition to the flat
|
|
238
|
+
* by-name index. Idempotent, and independent of `register()` so a function
|
|
239
|
+
* reachable from several schemas (e.g. a scan function referenced by tables in
|
|
240
|
+
* more than one schema) resolves from any of them.
|
|
241
|
+
*/
|
|
242
|
+
registerInSchema(func: VgiFunction, schemaName: string, catalogName?: string): void {
|
|
243
|
+
const add = (map: Map<string, VgiFunction[]>, key: string) => {
|
|
244
|
+
const bucket = map.get(key);
|
|
245
|
+
if (!bucket) {
|
|
246
|
+
map.set(key, [func]);
|
|
247
|
+
return;
|
|
248
|
+
}
|
|
249
|
+
if (!bucket.includes(func)) bucket.push(func);
|
|
250
|
+
};
|
|
251
|
+
add(this._bySchema, schemaKey(schemaName, func.meta.name));
|
|
252
|
+
if (catalogName) {
|
|
253
|
+
add(this._byCatalog, catalogKey(catalogName, schemaName, func.meta.name));
|
|
254
|
+
}
|
|
255
|
+
}
|
|
256
|
+
|
|
257
|
+
/** Schemas that declare `functionName`, sorted — for error messages. */
|
|
258
|
+
schemasFor(functionName: string): string[] {
|
|
259
|
+
const out: string[] = [];
|
|
260
|
+
for (const key of this._bySchema.keys()) {
|
|
261
|
+
const [schema, name] = key.split("\u0000");
|
|
262
|
+
if (name === functionName) out.push(schema);
|
|
263
|
+
}
|
|
264
|
+
return out.sort();
|
|
265
|
+
}
|
|
266
|
+
|
|
204
267
|
get(name: string, context?: OverloadContext): VgiFunction {
|
|
205
|
-
|
|
268
|
+
let candidates = this._functions.get(name);
|
|
269
|
+
|
|
270
|
+
// A schema-qualified lookup is exact: only functions declared in that schema
|
|
271
|
+
// are candidates, so a name registered in two schemas dispatches to the one
|
|
272
|
+
// the caller named rather than colliding as an overload.
|
|
273
|
+
// Catalog first: it is the only key that separates two catalogs declaring
|
|
274
|
+
// the same schema and name.
|
|
275
|
+
const catalogName = context?.catalogName;
|
|
276
|
+
const schemaName = context?.schemaName;
|
|
277
|
+
if (catalogName && schemaName) {
|
|
278
|
+
const scoped = this._byCatalog.get(catalogKey(catalogName, schemaName, name));
|
|
279
|
+
if (scoped && scoped.length > 0) {
|
|
280
|
+
return this._disambiguate(name, scoped, context);
|
|
281
|
+
}
|
|
282
|
+
}
|
|
283
|
+
if (schemaName) {
|
|
284
|
+
const scoped = this._bySchema.get(schemaKey(schemaName, name));
|
|
285
|
+
if (scoped && scoped.length > 0) {
|
|
286
|
+
candidates = scoped;
|
|
287
|
+
} else if (candidates && candidates.length > 0) {
|
|
288
|
+
const schemas = this.schemasFor(name);
|
|
289
|
+
if (schemas.length > 0) {
|
|
290
|
+
throw new Error(
|
|
291
|
+
`Function '${name}' is not registered in schema '${schemaName}'. ` +
|
|
292
|
+
`It is available in: [${schemas.join(", ")}]`,
|
|
293
|
+
);
|
|
294
|
+
}
|
|
295
|
+
}
|
|
296
|
+
}
|
|
297
|
+
|
|
206
298
|
if (!candidates || candidates.length === 0) {
|
|
207
299
|
throw new FunctionNotFoundError(name, this.names());
|
|
208
300
|
}
|
|
209
301
|
|
|
302
|
+
return this._disambiguate(name, candidates, context);
|
|
303
|
+
}
|
|
304
|
+
|
|
305
|
+
/**
|
|
306
|
+
* Pick one function from an already-scoped candidate list, by argument shape
|
|
307
|
+
* and types. Scoping (catalog / schema) happens in `get()`; by the time this
|
|
308
|
+
* runs, every candidate is a legitimate overload of the same name.
|
|
309
|
+
*/
|
|
310
|
+
private _disambiguate(
|
|
311
|
+
name: string,
|
|
312
|
+
candidates: VgiFunction[],
|
|
313
|
+
context?: OverloadContext,
|
|
314
|
+
): VgiFunction {
|
|
210
315
|
// Fast path: single candidate
|
|
211
316
|
if (candidates.length === 1) {
|
|
212
317
|
return candidates[0];
|
package/src/functions/scalar.ts
CHANGED
|
@@ -185,10 +185,14 @@ export interface ScalarFunctionConfig<
|
|
|
185
185
|
/**
|
|
186
186
|
* Result-cache opt-in: when set, this CacheControl's `vgi.cache.*` metadata
|
|
187
187
|
* rides every output batch's custom metadata (per-batch — NOT the schema,
|
|
188
|
-
* which the IPC stream fixes at open)
|
|
189
|
-
*
|
|
190
|
-
* only — advertising this on a non-pure scalar serves stale rows. Mirrors
|
|
188
|
+
* which the IPC stream fixes at open). A pure, deterministic scalar only —
|
|
189
|
+
* advertising this on a non-pure scalar serves stale rows. Mirrors
|
|
191
190
|
* vgi-python's `ScalarFunction.CACHE_CONTROL`.
|
|
191
|
+
*
|
|
192
|
+
* Per-VALUE memoization (the extension remembering one output per distinct
|
|
193
|
+
* input value) additionally requires `perValue: true`, which is off by
|
|
194
|
+
* default because a memo serve only beats a worker call when the call is
|
|
195
|
+
* expensive — see {@link CacheControl.perValue}.
|
|
192
196
|
*/
|
|
193
197
|
cacheControl?: CacheControl;
|
|
194
198
|
}
|
|
@@ -405,8 +409,9 @@ export function defineScalarFunction<
|
|
|
405
409
|
}
|
|
406
410
|
|
|
407
411
|
// Result-cache opt-in: a scalar declaring cacheControl rides its
|
|
408
|
-
// vgi.cache.* keys on the emit path's per-batch custom metadata
|
|
409
|
-
//
|
|
412
|
+
// vgi.cache.* keys on the emit path's per-batch custom metadata. The
|
|
413
|
+
// per-value memo tier is only populated when that CacheControl also
|
|
414
|
+
// sets `perValue` (default off — it costs more than a cheap call).
|
|
410
415
|
out.emit(
|
|
411
416
|
outputBatch,
|
|
412
417
|
config.cacheControl ? cacheControlMetadata(config.cacheControl) : undefined,
|
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
// GENERATED by vgi.codegen.ts_client. DO NOT EDIT BY HAND.
|
|
3
3
|
//
|
|
4
4
|
// Generator: vgi-gen-ts-client v1
|
|
5
|
-
// Content hash:
|
|
5
|
+
// Content hash: 80ec56060e88
|
|
6
6
|
//
|
|
7
7
|
// To regenerate:
|
|
8
8
|
// uv run --project ~/Development/vgi-python vgi-gen-ts-client \
|
|
@@ -174,6 +174,7 @@ export interface AggregateBindRequest {
|
|
|
174
174
|
settings?: /** @record-batch */ Uint8Array | null;
|
|
175
175
|
secrets?: /** @record-batch */ Uint8Array | null;
|
|
176
176
|
attach_opaque_data?: Uint8Array | null;
|
|
177
|
+
schema_name?: string | null;
|
|
177
178
|
}
|
|
178
179
|
|
|
179
180
|
export interface AggregateBindResponse {
|
|
@@ -186,6 +187,7 @@ export interface AggregateCombineRequest {
|
|
|
186
187
|
execution_id: Uint8Array;
|
|
187
188
|
merge_batch: Uint8Array;
|
|
188
189
|
attach_opaque_data?: Uint8Array | null;
|
|
190
|
+
schema_name?: string | null;
|
|
189
191
|
}
|
|
190
192
|
|
|
191
193
|
export interface AggregateCombineResponse {
|
|
@@ -196,6 +198,7 @@ export interface AggregateDestructorRequest {
|
|
|
196
198
|
execution_id: Uint8Array;
|
|
197
199
|
group_ids_batch: Uint8Array;
|
|
198
200
|
attach_opaque_data?: Uint8Array | null;
|
|
201
|
+
schema_name?: string | null;
|
|
199
202
|
}
|
|
200
203
|
|
|
201
204
|
export interface AggregateDestructorResponse {
|
|
@@ -207,6 +210,7 @@ export interface AggregateFinalizeRequest {
|
|
|
207
210
|
group_ids_batch: Uint8Array;
|
|
208
211
|
output_schema: /** @arrow-schema */ Uint8Array;
|
|
209
212
|
attach_opaque_data?: Uint8Array | null;
|
|
213
|
+
schema_name?: string | null;
|
|
210
214
|
}
|
|
211
215
|
|
|
212
216
|
export interface AggregateFinalizeResponse {
|
|
@@ -218,6 +222,7 @@ export interface AggregateStreamingChunkRequest {
|
|
|
218
222
|
execution_id: Uint8Array;
|
|
219
223
|
input_batch: Uint8Array;
|
|
220
224
|
attach_opaque_data?: Uint8Array | null;
|
|
225
|
+
schema_name?: string | null;
|
|
221
226
|
}
|
|
222
227
|
|
|
223
228
|
export interface AggregateStreamingChunkResponse {
|
|
@@ -228,6 +233,7 @@ export interface AggregateStreamingCloseRequest {
|
|
|
228
233
|
function_name: string;
|
|
229
234
|
execution_id: Uint8Array;
|
|
230
235
|
attach_opaque_data?: Uint8Array | null;
|
|
236
|
+
schema_name?: string | null;
|
|
231
237
|
}
|
|
232
238
|
|
|
233
239
|
export interface AggregateStreamingCloseResponse {
|
|
@@ -243,6 +249,7 @@ export interface AggregateStreamingOpenRequest {
|
|
|
243
249
|
settings?: /** @record-batch */ Uint8Array | null;
|
|
244
250
|
secrets?: /** @record-batch */ Uint8Array | null;
|
|
245
251
|
attach_opaque_data?: Uint8Array | null;
|
|
252
|
+
schema_name?: string | null;
|
|
246
253
|
}
|
|
247
254
|
|
|
248
255
|
export interface AggregateStreamingOpenResponse {
|
|
@@ -254,6 +261,7 @@ export interface AggregateUpdateRequest {
|
|
|
254
261
|
execution_id: Uint8Array;
|
|
255
262
|
input_batch: Uint8Array;
|
|
256
263
|
attach_opaque_data?: Uint8Array | null;
|
|
264
|
+
schema_name?: string | null;
|
|
257
265
|
}
|
|
258
266
|
|
|
259
267
|
export interface AggregateUpdateResponse {
|
|
@@ -267,6 +275,7 @@ export interface AggregateWindowRequest {
|
|
|
267
275
|
frame_starts: number[];
|
|
268
276
|
frame_ends: number[];
|
|
269
277
|
attach_opaque_data?: Uint8Array | null;
|
|
278
|
+
schema_name?: string | null;
|
|
270
279
|
}
|
|
271
280
|
|
|
272
281
|
export interface AggregateWindowResponse {
|
|
@@ -283,6 +292,7 @@ export interface AggregateWindowBatchRequest {
|
|
|
283
292
|
frame_starts: number[];
|
|
284
293
|
frame_ends: number[];
|
|
285
294
|
attach_opaque_data?: Uint8Array | null;
|
|
295
|
+
schema_name?: string | null;
|
|
286
296
|
}
|
|
287
297
|
|
|
288
298
|
export interface AggregateWindowBatchResponse {
|
|
@@ -294,6 +304,7 @@ export interface AggregateWindowDestructorRequest {
|
|
|
294
304
|
execution_id: Uint8Array;
|
|
295
305
|
partition_id: number;
|
|
296
306
|
attach_opaque_data?: Uint8Array | null;
|
|
307
|
+
schema_name?: string | null;
|
|
297
308
|
}
|
|
298
309
|
|
|
299
310
|
export interface AggregateWindowDestructorResponse {
|
|
@@ -310,6 +321,7 @@ export interface AggregateWindowInitRequest {
|
|
|
310
321
|
frame_stats: Uint8Array;
|
|
311
322
|
all_valid: Uint8Array;
|
|
312
323
|
attach_opaque_data?: Uint8Array | null;
|
|
324
|
+
schema_name?: string | null;
|
|
313
325
|
}
|
|
314
326
|
|
|
315
327
|
export interface AggregateWindowInitResponse {
|
|
@@ -340,6 +352,7 @@ export interface BindRequest {
|
|
|
340
352
|
at_value?: string | null;
|
|
341
353
|
copy_from?: CopyFromContext | null;
|
|
342
354
|
copy_to?: CopyToContext | null;
|
|
355
|
+
schema_name?: string | null;
|
|
343
356
|
}
|
|
344
357
|
|
|
345
358
|
export interface BindResponse {
|
|
@@ -487,6 +500,7 @@ export interface TableBufferingCombineRequest {
|
|
|
487
500
|
state_ids: Uint8Array[];
|
|
488
501
|
attach_opaque_data?: Uint8Array | null;
|
|
489
502
|
transaction_id?: Uint8Array | null;
|
|
503
|
+
schema_name?: string | null;
|
|
490
504
|
}
|
|
491
505
|
|
|
492
506
|
export interface TableBufferingCombineResponse {
|
|
@@ -498,6 +512,7 @@ export interface TableBufferingDestructorRequest {
|
|
|
498
512
|
execution_id: Uint8Array;
|
|
499
513
|
attach_opaque_data?: Uint8Array | null;
|
|
500
514
|
transaction_id?: Uint8Array | null;
|
|
515
|
+
schema_name?: string | null;
|
|
501
516
|
}
|
|
502
517
|
|
|
503
518
|
export interface TableBufferingDestructorResponse {
|
|
@@ -510,6 +525,7 @@ export interface TableBufferingProcessRequest {
|
|
|
510
525
|
attach_opaque_data?: Uint8Array | null;
|
|
511
526
|
transaction_id?: Uint8Array | null;
|
|
512
527
|
batch_index?: number | null;
|
|
528
|
+
schema_name?: string | null;
|
|
513
529
|
}
|
|
514
530
|
|
|
515
531
|
export interface TableBufferingProcessResponse {
|
package/src/index.core.ts
CHANGED
package/src/protocol/dispatch.ts
CHANGED
|
@@ -43,6 +43,7 @@ export function buildVgiProtocol(config: ProtocolConfig): Protocol {
|
|
|
43
43
|
registry: config.registry,
|
|
44
44
|
recoverExchangeState: config.recoverExchangeState,
|
|
45
45
|
signingKey: config.signingKey,
|
|
46
|
+
catalogInterface: config.catalogInterface,
|
|
46
47
|
});
|
|
47
48
|
registerAggregateMethods(protocol, config.registry);
|
|
48
49
|
registerTableBufferingMethods(protocol, config.registry, config.signingKey);
|
|
@@ -35,10 +35,16 @@ export interface FunctionHandlerConfig {
|
|
|
35
35
|
registry: FunctionRegistry;
|
|
36
36
|
recoverExchangeState?: (opaqueData: Uint8Array) => any;
|
|
37
37
|
signingKey?: Uint8Array;
|
|
38
|
+
/**
|
|
39
|
+
* Used to map a bind's `attach_opaque_data` to its catalog, so resolution can
|
|
40
|
+
* be scoped when two catalogs declare the same schema and function name.
|
|
41
|
+
* Optional — without it, resolution falls back to (schema, name).
|
|
42
|
+
*/
|
|
43
|
+
catalogInterface?: { catalogNameForAttach(a: Uint8Array): string | null };
|
|
38
44
|
}
|
|
39
45
|
|
|
40
46
|
export function registerFunctionMethods(protocol: Protocol, config: FunctionHandlerConfig): void {
|
|
41
|
-
const { registry, signingKey } = config;
|
|
47
|
+
const { registry, signingKey, catalogInterface } = config;
|
|
42
48
|
|
|
43
49
|
// The framework mints every attach as uuid(16) || catalog_bytes (sealed on
|
|
44
50
|
// HTTP, plaintext on subprocess). Function bodies — like catalog bodies —
|
|
@@ -51,6 +57,21 @@ export function registerFunctionMethods(protocol: Protocol, config: FunctionHand
|
|
|
51
57
|
return openAttach(env, ctx?.auth, signingKey);
|
|
52
58
|
}
|
|
53
59
|
|
|
60
|
+
// Build an overload context whose attach has been unsealed, so catalog
|
|
61
|
+
// resolution sees the real route byte. The bind/init handlers strip their
|
|
62
|
+
// request in place and can call overloadContext directly; every other site
|
|
63
|
+
// still holds the sealed envelope (73 bytes on HTTP vs 16 plaintext), whose
|
|
64
|
+
// leading byte would otherwise route to an arbitrary catalog.
|
|
65
|
+
async function strippedContext(bindCall: any, ctx?: any) {
|
|
66
|
+
let attach: Uint8Array | null = null;
|
|
67
|
+
try {
|
|
68
|
+
attach = await stripAttach(bindCall.attach_opaque_data, ctx);
|
|
69
|
+
} catch {
|
|
70
|
+
attach = null; // Unopenable (wrong key / not sealed) — resolve unscoped.
|
|
71
|
+
}
|
|
72
|
+
return overloadContext({ ...bindCall, attach_opaque_data: attach }, catalogInterface);
|
|
73
|
+
}
|
|
74
|
+
|
|
54
75
|
const initHeaderSchema = schema([
|
|
55
76
|
field("execution_id", binary(), false),
|
|
56
77
|
field("opaque_data", binary(), true),
|
|
@@ -77,7 +98,7 @@ export function registerFunctionMethods(protocol: Protocol, config: FunctionHand
|
|
|
77
98
|
const innerParams = unwrapRequest(params.request);
|
|
78
99
|
const request = deserializeBindRequest(innerParams);
|
|
79
100
|
request.attach_opaque_data = await stripAttach(request.attach_opaque_data, ctx);
|
|
80
|
-
const func = registry.get(request.function_name, overloadContext(request));
|
|
101
|
+
const func = registry.get(request.function_name, overloadContext(request, catalogInterface));
|
|
81
102
|
const response = await func.bind(request);
|
|
82
103
|
const serialized = serializeBindResponse(response);
|
|
83
104
|
return wrapResult(serialized, BindResultSchema);
|
|
@@ -99,7 +120,7 @@ export function registerFunctionMethods(protocol: Protocol, config: FunctionHand
|
|
|
99
120
|
// Exchange init has no ctx; on subprocess (no signing key) openAttach is a
|
|
100
121
|
// pass-through strip that needs no auth. HTTP carries auth via the bind path.
|
|
101
122
|
request.bind_call.attach_opaque_data = await stripAttach(request.bind_call.attach_opaque_data, undefined);
|
|
102
|
-
const func = registry.get(request.bind_call.function_name, overloadContext(request.bind_call));
|
|
123
|
+
const func = registry.get(request.bind_call.function_name, overloadContext(request.bind_call, catalogInterface));
|
|
103
124
|
|
|
104
125
|
// globalInit is async — table function onInit may touch HTTP-backed
|
|
105
126
|
// FunctionStorage (e.g. Cloudflare DO).
|
|
@@ -178,7 +199,7 @@ export function registerFunctionMethods(protocol: Protocol, config: FunctionHand
|
|
|
178
199
|
const initRequestBatch = deserializeBatch(state.initRequestIpc);
|
|
179
200
|
const initRequestDict = batchToScalarDict(initRequestBatch);
|
|
180
201
|
const request = deserializeInitRequest(initRequestDict);
|
|
181
|
-
const func = registry.get(state.functionName,
|
|
202
|
+
const func = registry.get(state.functionName, await strippedContext(request.bind_call));
|
|
182
203
|
const executionId = state.executionId;
|
|
183
204
|
const opaqueData = state.opaqueData ?? null;
|
|
184
205
|
const initResponse: GlobalInitResponse = {
|
|
@@ -243,7 +264,7 @@ export function registerFunctionMethods(protocol: Protocol, config: FunctionHand
|
|
|
243
264
|
handler: async (params) => {
|
|
244
265
|
const innerParams = unwrapRequest(params.request);
|
|
245
266
|
const request = deserializeCardinalityRequest(innerParams);
|
|
246
|
-
const func = registry.get(request.bind_call.function_name,
|
|
267
|
+
const func = registry.get(request.bind_call.function_name, await strippedContext(request.bind_call));
|
|
247
268
|
let cardResult: Record<string, any>;
|
|
248
269
|
if (func.cardinality) {
|
|
249
270
|
cardResult = serializeTableCardinality(await func.cardinality(request));
|
|
@@ -264,10 +285,10 @@ export function registerFunctionMethods(protocol: Protocol, config: FunctionHand
|
|
|
264
285
|
protocol.unary("table_function_statistics", {
|
|
265
286
|
params: REQUEST_PARAMS_SCHEMA,
|
|
266
287
|
result: RESULT_BINARY_NULLABLE_SCHEMA,
|
|
267
|
-
handler: (params) => {
|
|
288
|
+
handler: async (params) => {
|
|
268
289
|
const innerParams = unwrapRequest(params.request);
|
|
269
290
|
const request = deserializeCardinalityRequest(innerParams);
|
|
270
|
-
const func = registry.get(request.bind_call.function_name,
|
|
291
|
+
const func = registry.get(request.bind_call.function_name, await strippedContext(request.bind_call));
|
|
271
292
|
if (!func.statistics) return { result: null };
|
|
272
293
|
const stats = func.statistics(request);
|
|
273
294
|
if (!stats || stats.length === 0) return { result: null };
|
|
@@ -298,7 +319,7 @@ export function registerFunctionMethods(protocol: Protocol, config: FunctionHand
|
|
|
298
319
|
const bindOpaqueData = innerParams.bind_opaque_data
|
|
299
320
|
? toUint8Array(innerParams.bind_opaque_data)
|
|
300
321
|
: null;
|
|
301
|
-
const func = registry.get(bindCall.function_name,
|
|
322
|
+
const func = registry.get(bindCall.function_name, await strippedContext(bindCall));
|
|
302
323
|
const map = func.dynamicToString
|
|
303
324
|
? await func.dynamicToString({ bindCall, bindOpaqueData, globalExecutionId })
|
|
304
325
|
: {};
|
|
@@ -51,11 +51,35 @@ export function wrapResult(
|
|
|
51
51
|
return { result: serializeBatch(batch) };
|
|
52
52
|
}
|
|
53
53
|
|
|
54
|
-
export function overloadContext(
|
|
54
|
+
export function overloadContext(
|
|
55
|
+
req: {
|
|
56
|
+
function_name: string;
|
|
57
|
+
arguments: any;
|
|
58
|
+
input_schema: any;
|
|
59
|
+
function_type: any;
|
|
60
|
+
schema_name?: string | null;
|
|
61
|
+
attach_opaque_data?: Uint8Array | null;
|
|
62
|
+
},
|
|
63
|
+
catalogInterface?: { catalogNameForAttach(a: Uint8Array): string | null },
|
|
64
|
+
): OverloadContext {
|
|
65
|
+
// The attachment identifies the catalog, which is the only key separating two
|
|
66
|
+
// catalogs that declare the same schema AND function name.
|
|
67
|
+
let catalogName: string | null = null;
|
|
68
|
+
if (catalogInterface && req.attach_opaque_data) {
|
|
69
|
+
try {
|
|
70
|
+
catalogName = catalogInterface.catalogNameForAttach(req.attach_opaque_data);
|
|
71
|
+
} catch {
|
|
72
|
+
catalogName = null;
|
|
73
|
+
}
|
|
74
|
+
}
|
|
55
75
|
return {
|
|
56
76
|
arguments: req.arguments,
|
|
57
77
|
inputSchema: req.input_schema,
|
|
58
78
|
isScalar: String(req.function_type).toLowerCase() === "scalar",
|
|
79
|
+
// Scopes resolution when the caller named a schema; a name registered in
|
|
80
|
+
// several schemas is otherwise ambiguous.
|
|
81
|
+
schemaName: req.schema_name ?? null,
|
|
82
|
+
catalogName,
|
|
59
83
|
};
|
|
60
84
|
}
|
|
61
85
|
|
|
@@ -30,6 +30,10 @@ const BIND_REQUEST_SCHEMA = schema([
|
|
|
30
30
|
// Both null when the scan has no AT clause. See BindRequest.at_unit.
|
|
31
31
|
field("at_unit", utf8(), true),
|
|
32
32
|
field("at_value", utf8(), true),
|
|
33
|
+
// Catalog schema owning the function. A worker may register one name in
|
|
34
|
+
// several schemas, so the bare name is not a unique key — resolution is by
|
|
35
|
+
// (schema_name, function_name). Null for callers with no catalog context.
|
|
36
|
+
field("schema_name", utf8(), true),
|
|
33
37
|
]);
|
|
34
38
|
|
|
35
39
|
// COPY ... FROM context — a nullable nested struct<format, file_path,
|
|
@@ -76,6 +80,7 @@ export function serializeBindRequest(req: BindRequest): VgiBatch {
|
|
|
76
80
|
resolved_secrets_provided: req.resolved_secrets_provided ?? false,
|
|
77
81
|
at_unit: req.at_unit ?? null,
|
|
78
82
|
at_value: req.at_value ?? null,
|
|
83
|
+
schema_name: req.schema_name ?? null,
|
|
79
84
|
};
|
|
80
85
|
// Append the copy_from / copy_to struct columns only for COPY scans/sinks, so
|
|
81
86
|
// non-COPY binds serialize to the exact legacy shape (fields matched by name
|
|
@@ -172,6 +177,8 @@ export function deserializeBindRequest(
|
|
|
172
177
|
// Empty string -> null (matches the C++ BuildBindRequest convention).
|
|
173
178
|
at_unit: params.at_unit ? String(params.at_unit) : null,
|
|
174
179
|
at_value: params.at_value ? String(params.at_value) : null,
|
|
180
|
+
// Empty string -> null, same convention as at_unit/at_value.
|
|
181
|
+
schema_name: params.schema_name ? String(params.schema_name) : null,
|
|
175
182
|
// COPY ... FROM context — absent for ordinary scans (params.copy_from
|
|
176
183
|
// undefined -> null). The struct column decodes to a plain object.
|
|
177
184
|
copy_from: parseCopyFromContext(params.copy_from),
|
package/src/protocol/types.ts
CHANGED
|
@@ -88,6 +88,14 @@ export interface BindRequest {
|
|
|
88
88
|
*/
|
|
89
89
|
at_unit?: string | null;
|
|
90
90
|
at_value?: string | null;
|
|
91
|
+
/**
|
|
92
|
+
* Catalog schema that declares the function being bound. A worker may
|
|
93
|
+
* register the same name in more than one schema, so the bare name does not
|
|
94
|
+
* identify an implementation — resolution is by `(schema_name,
|
|
95
|
+
* function_name)`. Null for callers with no catalog context (COPY handler
|
|
96
|
+
* binds). Mirrors vgi-python's `BindRequest.schema_name`.
|
|
97
|
+
*/
|
|
98
|
+
schema_name?: string | null;
|
|
91
99
|
}
|
|
92
100
|
|
|
93
101
|
export interface BindResponse {
|