@query-farm/vgi 0.20.0 → 0.22.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/arrow/impl-arrowjs/index.d.ts +7 -0
- package/dist/arrow/impl-flechette/arrowjs-shape.d.ts +16 -0
- package/dist/arrow/impl-flechette/compat.d.ts +22 -0
- package/dist/arrow/impl-flechette/index.d.ts +2 -0
- package/dist/arrow/impl-flechette/ipc.d.ts +1 -1
- package/dist/arrow/index.d.ts +2 -2
- package/dist/arrow/predicates.d.ts +16 -0
- package/dist/cache-control.d.ts +33 -0
- package/dist/catalog/composite.d.ts +2 -0
- package/dist/catalog/interface.d.ts +11 -0
- package/dist/client-entry.js +9 -3
- package/dist/client-entry.js.map +7 -7
- package/dist/functions/registry.d.ts +31 -0
- package/dist/functions/scalar.d.ts +7 -3
- package/dist/generated/vgi-client.d.ts +16 -0
- package/dist/index.core.d.ts +2 -2
- package/dist/index.js +175 -19
- package/dist/index.js.map +19 -19
- package/dist/protocol/handlers/function.d.ts +8 -0
- package/dist/protocol/handlers/shared.d.ts +4 -0
- package/dist/protocol/types.d.ts +8 -0
- package/dist/serve-entry.js +40 -13
- package/dist/serve-entry.js.map +10 -10
- package/dist/util/arrow/index.d.ts +1 -1
- package/dist/worker-cf-entry.js +167 -17
- package/dist/worker-cf-entry.js.map +17 -17
- package/dist/worker.d.ts +9 -0
- package/package.json +5 -5
- package/src/arrow/impl-arrowjs/index.ts +10 -0
- package/src/arrow/impl-flechette/arrowjs-shape.ts +78 -0
- package/src/arrow/impl-flechette/canonical.ts +19 -0
- package/src/arrow/impl-flechette/compat.ts +254 -0
- package/src/arrow/impl-flechette/empty.ts +5 -1
- package/src/arrow/impl-flechette/index.ts +8 -0
- package/src/arrow/impl-flechette/ipc.ts +72 -6
- package/src/arrow/impl-flechette/normalize-type.ts +10 -2
- package/src/arrow/impl-flechette/schema.ts +13 -9
- package/src/arrow/index.ts +3 -0
- package/src/arrow/predicates.ts +59 -0
- package/src/cache-control.ts +35 -0
- package/src/catalog/composite.ts +9 -0
- package/src/catalog/interface.ts +15 -0
- package/src/catalog/read-only.ts +13 -0
- package/src/functions/registry.ts +127 -5
- package/src/functions/scalar.ts +10 -5
- package/src/generated/vgi-client.ts +17 -1
- package/src/index.core.ts +4 -0
- package/src/protocol/dispatch.ts +1 -0
- package/src/protocol/handlers/function.ts +38 -9
- package/src/protocol/handlers/shared.ts +25 -1
- package/src/protocol/serializers/bind.ts +7 -0
- package/src/protocol/types.ts +8 -0
- package/src/util/arrow/index.ts +1 -0
- package/src/worker.ts +15 -1
|
@@ -52,6 +52,7 @@ import {
|
|
|
52
52
|
} from "@query-farm/flechette";
|
|
53
53
|
|
|
54
54
|
import type { VgiField, VgiSchema, VgiDataType } from "../types.js";
|
|
55
|
+
import { aliasIntSigned } from "./arrowjs-shape.js";
|
|
55
56
|
|
|
56
57
|
// ----- Unit / mode constants -----------------------------------------------
|
|
57
58
|
|
|
@@ -91,16 +92,19 @@ export function field(
|
|
|
91
92
|
export const nullType = (): VgiDataType => f_nullType() as unknown as VgiDataType;
|
|
92
93
|
export const bool = (): VgiDataType => f_bool() as unknown as VgiDataType;
|
|
93
94
|
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
export const
|
|
98
|
-
export const
|
|
99
|
-
export const
|
|
100
|
-
export const
|
|
101
|
-
export const
|
|
95
|
+
// Int types carry the `isSigned` alias (arrow-js's spelling of flechette's
|
|
96
|
+
// `signed`) so worker code that branches on signedness reads the same property
|
|
97
|
+
// on both backends — see ./compat.ts.
|
|
98
|
+
export const int8 = (): VgiDataType => aliasIntSigned(f_int8()) as unknown as VgiDataType;
|
|
99
|
+
export const int16 = (): VgiDataType => aliasIntSigned(f_int16()) as unknown as VgiDataType;
|
|
100
|
+
export const int32 = (): VgiDataType => aliasIntSigned(f_int32()) as unknown as VgiDataType;
|
|
101
|
+
export const int64 = (): VgiDataType => aliasIntSigned(f_int64()) as unknown as VgiDataType;
|
|
102
|
+
export const uint8 = (): VgiDataType => aliasIntSigned(f_uint8()) as unknown as VgiDataType;
|
|
103
|
+
export const uint16 = (): VgiDataType => aliasIntSigned(f_uint16()) as unknown as VgiDataType;
|
|
104
|
+
export const uint32 = (): VgiDataType => aliasIntSigned(f_uint32()) as unknown as VgiDataType;
|
|
105
|
+
export const uint64 = (): VgiDataType => aliasIntSigned(f_uint64()) as unknown as VgiDataType;
|
|
102
106
|
export const int = (bitWidth: 8 | 16 | 32 | 64 = 32, signed = true): VgiDataType =>
|
|
103
|
-
f_int(bitWidth, signed) as unknown as VgiDataType;
|
|
107
|
+
aliasIntSigned(f_int(bitWidth, signed)) as unknown as VgiDataType;
|
|
104
108
|
|
|
105
109
|
export const float16 = (): VgiDataType => f_float16() as unknown as VgiDataType;
|
|
106
110
|
export const float32 = (): VgiDataType => f_float32() as unknown as VgiDataType;
|
package/src/arrow/index.ts
CHANGED
|
@@ -44,6 +44,7 @@ export {
|
|
|
44
44
|
isDuration,
|
|
45
45
|
isDictionary,
|
|
46
46
|
isBatch,
|
|
47
|
+
typeSignature,
|
|
47
48
|
} from "./predicates.js";
|
|
48
49
|
|
|
49
50
|
// Active backend's surface (IPC, build, iterate, utilities).
|
|
@@ -55,6 +56,8 @@ export {
|
|
|
55
56
|
filterBatch, projectSchema, projectBatch,
|
|
56
57
|
// per-record-batch metadata attachment
|
|
57
58
|
withBatchMetadata,
|
|
59
|
+
// arrow-js method shape for batches decoded outside this module's backend copy
|
|
60
|
+
adoptArrowJsShape,
|
|
58
61
|
// canonical single-value read (backend-agnostic, lossless)
|
|
59
62
|
readCanonicalValue,
|
|
60
63
|
// IPC
|
package/src/arrow/predicates.ts
CHANGED
|
@@ -53,6 +53,65 @@ export function isMap(t: VgiDataType): boolean { return t.typeId === TypeId.Map;
|
|
|
53
53
|
export function isDuration(t: VgiDataType): boolean { return t.typeId === TypeId.Duration; }
|
|
54
54
|
export function isDictionary(t: VgiDataType): boolean { return t.typeId === TypeId.Dictionary; }
|
|
55
55
|
|
|
56
|
+
/**
|
|
57
|
+
* Stable structural identity for an Arrow type, usable for equality.
|
|
58
|
+
*
|
|
59
|
+
* `String(type)` is NOT usable for this: arrow-js DataTypes are class
|
|
60
|
+
* instances with a meaningful `toString()` ("Int64", "Utf8", …), while
|
|
61
|
+
* flechette types are plain object literals that stringify to
|
|
62
|
+
* "[object Object]". Comparing with `toString()` therefore reports *every*
|
|
63
|
+
* flechette type as equal to every other, and never equal to a declared
|
|
64
|
+
* arrow-js type — which collapsed overload resolution to a first-match tie
|
|
65
|
+
* (`type_info(42::BIGINT)` picked the INTEGER overload).
|
|
66
|
+
*
|
|
67
|
+
* Both libraries agree on `typeId` and on the parameter *values*; they differ
|
|
68
|
+
* only on a few property names, so those are read under both spellings. The
|
|
69
|
+
* output is an opaque key — compare it, don't display it.
|
|
70
|
+
*/
|
|
71
|
+
export function typeSignature(t: VgiDataType | null | undefined): string {
|
|
72
|
+
if (t == null) return "null";
|
|
73
|
+
const a = t as any;
|
|
74
|
+
const kids = (): string =>
|
|
75
|
+
Array.isArray(a.children)
|
|
76
|
+
? a.children.map((c: any) => `${c?.name ?? ""}:${typeSignature(c?.type ?? c)}`).join(",")
|
|
77
|
+
: "";
|
|
78
|
+
switch (t.typeId) {
|
|
79
|
+
case TypeId.Int:
|
|
80
|
+
return `int${a.bitWidth}${(a.isSigned ?? a.signed ?? true) ? "" : "u"}`;
|
|
81
|
+
case TypeId.Float:
|
|
82
|
+
return `float${a.precision}`;
|
|
83
|
+
case TypeId.Decimal:
|
|
84
|
+
return `decimal${a.bitWidth ?? 128}(${a.precision},${a.scale})`;
|
|
85
|
+
case TypeId.Date:
|
|
86
|
+
case TypeId.Time:
|
|
87
|
+
case TypeId.Duration:
|
|
88
|
+
return `${t.typeId}[${a.unit}]`;
|
|
89
|
+
case TypeId.Timestamp:
|
|
90
|
+
return `ts[${a.unit},${a.timezone ?? ""}]`;
|
|
91
|
+
case TypeId.Interval:
|
|
92
|
+
return `iv[${a.unit}]`;
|
|
93
|
+
case TypeId.FixedSizeBinary:
|
|
94
|
+
return `fsb(${a.byteWidth ?? a.stride})`;
|
|
95
|
+
case TypeId.FixedSizeList:
|
|
96
|
+
return `fsl(${a.listSize ?? a.stride},${kids()})`;
|
|
97
|
+
case TypeId.List:
|
|
98
|
+
case TypeId.Struct:
|
|
99
|
+
case TypeId.Map:
|
|
100
|
+
return `${t.typeId}<${kids()}>`;
|
|
101
|
+
case TypeId.Union:
|
|
102
|
+
return `union(${a.mode},${kids()})`;
|
|
103
|
+
case TypeId.Dictionary:
|
|
104
|
+
return `dict(${typeSignature(a.dictionary)},${typeSignature(a.indices)},${
|
|
105
|
+
a.ordered ?? a.isOrdered ?? false
|
|
106
|
+
})`;
|
|
107
|
+
default:
|
|
108
|
+
// Covers the parameterless types (Null, Bool, Utf8, Binary and their
|
|
109
|
+
// Large variants) plus anything not enumerated above; children are
|
|
110
|
+
// folded in when present so a nested type still differentiates.
|
|
111
|
+
return Array.isArray(a.children) ? `${t.typeId}<${kids()}>` : String(t.typeId);
|
|
112
|
+
}
|
|
113
|
+
}
|
|
114
|
+
|
|
56
115
|
/** Runtime check: does `x` quack like a VgiBatch? Used at API boundaries
|
|
57
116
|
* where callers may pass either a batch or a plain column dict. Duck-typed
|
|
58
117
|
* because arrow-js and flechette use unrelated classes. */
|
package/src/cache-control.ts
CHANGED
|
@@ -41,6 +41,8 @@ export const CACHE_REVALIDATABLE_KEY = "vgi.cache.revalidatable";
|
|
|
41
41
|
export const CACHE_STALE_WHILE_REVALIDATE_KEY = "vgi.cache.stale_while_revalidate";
|
|
42
42
|
export const CACHE_STALE_IF_ERROR_KEY = "vgi.cache.stale_if_error";
|
|
43
43
|
export const CACHE_NOT_MODIFIED_KEY = "vgi.cache.not_modified";
|
|
44
|
+
export const CACHE_PARTITION_SCOPE_KEY = "vgi.cache.partition_scope";
|
|
45
|
+
export const CACHE_PER_VALUE_KEY = "vgi.cache.per_value";
|
|
44
46
|
|
|
45
47
|
// --- Request-side metadata keys (client -> worker) --------------------------
|
|
46
48
|
// Ride on the first producer tick; surfaced as `TableProcessParams.ifNoneMatch`
|
|
@@ -91,6 +93,37 @@ export interface CacheControl {
|
|
|
91
93
|
* assert the client's stored payload is still fresh (the client reuses it
|
|
92
94
|
* instead of re-streaming). */
|
|
93
95
|
notModified?: boolean;
|
|
96
|
+
/** Opt in to per-PARTITION caching: in addition to the whole-scan entry, the
|
|
97
|
+
* client stores the result split by distinct partition tuple, so a later
|
|
98
|
+
* `=`/`IN`-filtered scan on the partition column(s) can serve the requested
|
|
99
|
+
* partitions without calling the worker. Only meaningful for a
|
|
100
|
+
* `SINGLE_VALUE_PARTITIONS` table function; purely additive. */
|
|
101
|
+
partitionScope?: boolean;
|
|
102
|
+
/**
|
|
103
|
+
* Opt in to per-VALUE memoization: in addition to the whole-result entry, the
|
|
104
|
+
* client memoizes each distinct worker-input tuple's output keyed on that
|
|
105
|
+
* tuple, so the same value is served without calling the worker on a later
|
|
106
|
+
* chunk or a later query. Only meaningful for an exchange-mode MAP — a
|
|
107
|
+
* scalar, or a blended table-in-out invoked through a correlated `LATERAL`.
|
|
108
|
+
*
|
|
109
|
+
* **Default off, and it should stay off unless one call is genuinely
|
|
110
|
+
* expensive.** This is not a free win: serving a value from the memo costs a
|
|
111
|
+
* key probe, a decode and a per-value assembly step, and that only pays back
|
|
112
|
+
* when it is cheaper than asking you for the answer. For a cheap map —
|
|
113
|
+
* arithmetic, a string tweak, a lookup in a table the worker already has in
|
|
114
|
+
* memory — it is a large net loss (the engine measures a per-value serve at
|
|
115
|
+
* roughly 50x the cost of just calling the worker for a simple arithmetic
|
|
116
|
+
* map). Only the function author knows which side of that line a call falls
|
|
117
|
+
* on, which is why the engine will not guess: turn it on for model
|
|
118
|
+
* inference, geocoding, an external API call, or anything rate-limited or
|
|
119
|
+
* billed per request.
|
|
120
|
+
*
|
|
121
|
+
* Independent of freshness: `ttl`/`expires` make a result cacheable at all,
|
|
122
|
+
* while this decides whether the per-value tier is additionally populated.
|
|
123
|
+
* Requires a deterministic, side-effect-free function, exactly like the rest
|
|
124
|
+
* of the result cache.
|
|
125
|
+
*/
|
|
126
|
+
perValue?: boolean;
|
|
94
127
|
}
|
|
95
128
|
|
|
96
129
|
const VALID_SCOPES: readonly string[] = [CACHE_SCOPE_CATALOG, CACHE_SCOPE_TRANSACTION];
|
|
@@ -129,5 +162,7 @@ export function cacheControlMetadata(cc: CacheControl, extra?: Map<string, strin
|
|
|
129
162
|
if (cc.staleWhileRevalidate !== undefined) md.set(CACHE_STALE_WHILE_REVALIDATE_KEY, String(cc.staleWhileRevalidate));
|
|
130
163
|
if (cc.staleIfError !== undefined) md.set(CACHE_STALE_IF_ERROR_KEY, String(cc.staleIfError));
|
|
131
164
|
if (cc.notModified) md.set(CACHE_NOT_MODIFIED_KEY, "1");
|
|
165
|
+
if (cc.partitionScope) md.set(CACHE_PARTITION_SCOPE_KEY, "1");
|
|
166
|
+
if (cc.perValue) md.set(CACHE_PER_VALUE_KEY, "1");
|
|
132
167
|
return md;
|
|
133
168
|
}
|
package/src/catalog/composite.ts
CHANGED
|
@@ -56,6 +56,15 @@ export class CompositeCatalogInterface extends CatalogInterface {
|
|
|
56
56
|
return route;
|
|
57
57
|
}
|
|
58
58
|
|
|
59
|
+
/** Route to the owning backend and ask it, so the answer is per-attachment. */
|
|
60
|
+
override catalogNameForAttach(attachOpaqueData: Uint8Array): string | null {
|
|
61
|
+
try {
|
|
62
|
+
return this._route(attachOpaqueData).catalogNameForAttach(attachOpaqueData);
|
|
63
|
+
} catch {
|
|
64
|
+
return null; // Unroutable attach — fall back to unscoped resolution.
|
|
65
|
+
}
|
|
66
|
+
}
|
|
67
|
+
|
|
59
68
|
catalogs(): string[] {
|
|
60
69
|
const all: string[] = [];
|
|
61
70
|
for (const b of this._backends) all.push(...b.catalogs());
|
package/src/catalog/interface.ts
CHANGED
|
@@ -365,6 +365,21 @@ function isPromise<T>(v: Awaitable<T>): v is Promise<T> {
|
|
|
365
365
|
}
|
|
366
366
|
|
|
367
367
|
export abstract class CatalogInterface {
|
|
368
|
+
/**
|
|
369
|
+
* Which catalog an `attach_opaque_data` belongs to, or null if unknown.
|
|
370
|
+
*
|
|
371
|
+
* Two catalogs served by one worker may declare the same schema *and*
|
|
372
|
+
* function name, in which case the attachment is the only thing that tells
|
|
373
|
+
* them apart — the bind path uses this to scope function resolution. The
|
|
374
|
+
* default answers from `catalogs()` when this interface owns exactly one,
|
|
375
|
+
* which is right for every single-catalog implementation; the composite
|
|
376
|
+
* overrides it to route by its backend index byte.
|
|
377
|
+
*/
|
|
378
|
+
catalogNameForAttach(_attachOpaqueData: Uint8Array): string | null {
|
|
379
|
+
const names = this.catalogs();
|
|
380
|
+
return names.length === 1 ? names[0] : null;
|
|
381
|
+
}
|
|
382
|
+
|
|
368
383
|
// Required overrides
|
|
369
384
|
abstract catalogs(): string[];
|
|
370
385
|
/**
|
package/src/catalog/read-only.ts
CHANGED
|
@@ -38,6 +38,19 @@ export class ReadOnlyCatalogInterface extends CatalogInterface {
|
|
|
38
38
|
super();
|
|
39
39
|
this._descriptor = descriptor;
|
|
40
40
|
this._registry = registry;
|
|
41
|
+
// Index every function this catalog declares by its owning schema, so a
|
|
42
|
+
// schema-qualified bind resolves to the right implementation when one name
|
|
43
|
+
// is declared in more than one schema. Table-backing functions are indexed
|
|
44
|
+
// under their table's schema for the same reason.
|
|
45
|
+
for (const sch of descriptor.schemas ?? []) {
|
|
46
|
+
for (const fn of sch.functions ?? []) registry.registerInSchema(fn, sch.name, descriptor.name);
|
|
47
|
+
for (const tbl of sch.tables ?? []) {
|
|
48
|
+
for (const key of ["function", "insertFunction", "updateFunction", "deleteFunction"] as const) {
|
|
49
|
+
const fn = (tbl as any)[key];
|
|
50
|
+
if (fn) registry.registerInSchema(fn, sch.name, descriptor.name);
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
}
|
|
41
54
|
}
|
|
42
55
|
|
|
43
56
|
catalogs(): string[] {
|
|
@@ -1,7 +1,11 @@
|
|
|
1
1
|
// Copyright 2025, 2026 Query Farm LLC - https://query.farm
|
|
2
2
|
// Function registry: name -> VgiFunction lookup with overload resolution.
|
|
3
3
|
|
|
4
|
-
import {
|
|
4
|
+
import {
|
|
5
|
+
type VgiDataType,
|
|
6
|
+
isBinary, isBool, isDecimal, isFloat, isInt, isNull, isUtf8,
|
|
7
|
+
typeSignature,
|
|
8
|
+
} from "../arrow/index.js";
|
|
5
9
|
import type { VgiSchema } from "../arrow/index.js";
|
|
6
10
|
import type { VgiFunction } from "./types.js";
|
|
7
11
|
import type { Arguments } from "../arguments/arguments.js";
|
|
@@ -12,13 +16,40 @@ export interface OverloadContext {
|
|
|
12
16
|
arguments?: Arguments;
|
|
13
17
|
inputSchema?: VgiSchema | null;
|
|
14
18
|
isScalar?: boolean;
|
|
19
|
+
/**
|
|
20
|
+
* Catalog schema that declares the function, from `BindRequest.schema_name`.
|
|
21
|
+
* A worker may register the same name in more than one schema, so the bare
|
|
22
|
+
* name is not a unique key. When set, resolution is scoped to that schema and
|
|
23
|
+
* never falls back to another one; when absent, every schema is searched.
|
|
24
|
+
*/
|
|
25
|
+
schemaName?: string | null;
|
|
26
|
+
/**
|
|
27
|
+
* Catalog that owns the function, resolved from the bind's
|
|
28
|
+
* `attach_opaque_data`. Two catalogs served by one worker may each declare
|
|
29
|
+
* the same schema *and* function name, in which case only the catalog tells
|
|
30
|
+
* them apart. Absent for calls with no attachment.
|
|
31
|
+
*/
|
|
32
|
+
catalogName?: string | null;
|
|
15
33
|
}
|
|
16
34
|
|
|
17
35
|
const EXACT_MATCH_SCORE = 2;
|
|
18
36
|
const FAMILY_MATCH_SCORE = 1;
|
|
19
37
|
|
|
38
|
+
/**
|
|
39
|
+
* Exact structural type identity.
|
|
40
|
+
*
|
|
41
|
+
* `toString()` cannot be used: arrow-js DataTypes stringify to a meaningful
|
|
42
|
+
* name, flechette's plain-object types all stringify to "[object Object]".
|
|
43
|
+
* Under flechette that made every exact-match test fail, every overload score
|
|
44
|
+
* collapse to the family tie, and `type_info(42::BIGINT)` resolve to the
|
|
45
|
+
* INTEGER overload.
|
|
46
|
+
*/
|
|
47
|
+
function typesIdentical(a: VgiDataType, b: VgiDataType): boolean {
|
|
48
|
+
return typeSignature(a) === typeSignature(b);
|
|
49
|
+
}
|
|
50
|
+
|
|
20
51
|
function typesCompatible(actual: VgiDataType, declared: VgiDataType): boolean {
|
|
21
|
-
if (actual
|
|
52
|
+
if (typesIdentical(actual, declared)) return true;
|
|
22
53
|
// Integer family
|
|
23
54
|
if (isInt(actual) && isInt(declared)) return true;
|
|
24
55
|
// Float/decimal family
|
|
@@ -47,7 +78,7 @@ function scoreTypes(
|
|
|
47
78
|
if (spec.isAnyType || isNull(spec.arrowType)) continue;
|
|
48
79
|
const actual = actualTypes[i];
|
|
49
80
|
if (actual === null) continue;
|
|
50
|
-
if (actual
|
|
81
|
+
if (typesIdentical(actual, spec.arrowType)) {
|
|
51
82
|
score += EXACT_MATCH_SCORE;
|
|
52
83
|
} else if (typesCompatible(actual, spec.arrowType)) {
|
|
53
84
|
score += FAMILY_MATCH_SCORE;
|
|
@@ -61,7 +92,7 @@ function scoreTypes(
|
|
|
61
92
|
for (let i = specs.length; i < actualTypes.length; i++) {
|
|
62
93
|
const actual = actualTypes[i];
|
|
63
94
|
if (actual === null) continue;
|
|
64
|
-
if (actual
|
|
95
|
+
if (typesIdentical(actual, varArgsSpec.arrowType)) {
|
|
65
96
|
score += EXACT_MATCH_SCORE;
|
|
66
97
|
} else if (typesCompatible(actual, varArgsSpec.arrowType)) {
|
|
67
98
|
score += FAMILY_MATCH_SCORE;
|
|
@@ -173,8 +204,26 @@ function filterByArgumentTypes(
|
|
|
173
204
|
return scored.filter(s => s.score === maxScore).map(s => s.func);
|
|
174
205
|
}
|
|
175
206
|
|
|
207
|
+
/** Key for the schema-scoped index: lowercased schema, then function name. */
|
|
208
|
+
function schemaKey(schemaName: string, functionName: string): string {
|
|
209
|
+
return `${schemaName.toLowerCase()}\u0000${functionName}`;
|
|
210
|
+
}
|
|
211
|
+
|
|
212
|
+
/** Key for the catalog-scoped index: lowercased catalog, then the schema key. */
|
|
213
|
+
function catalogKey(catalogName: string, schemaName: string, functionName: string): string {
|
|
214
|
+
return `${catalogName.toLowerCase()}\u0000${schemaKey(schemaName, functionName)}`;
|
|
215
|
+
}
|
|
216
|
+
|
|
176
217
|
export class FunctionRegistry {
|
|
177
218
|
private _functions: Map<string, VgiFunction[]> = new Map();
|
|
219
|
+
// (lowercased schema, function name) -> functions declared in that schema.
|
|
220
|
+
// Populated by registerInSchema(), which catalog construction calls for every
|
|
221
|
+
// function a SchemaDescriptor lists. Lets a schema-qualified bind pick the
|
|
222
|
+
// right implementation when one name is registered in several schemas.
|
|
223
|
+
private _bySchema: Map<string, VgiFunction[]> = new Map();
|
|
224
|
+
// (lowercased catalog, lowercased schema, function name) -> functions. Needed
|
|
225
|
+
// because two catalogs in one worker may declare the same schema AND name.
|
|
226
|
+
private _byCatalog: Map<string, VgiFunction[]> = new Map();
|
|
178
227
|
|
|
179
228
|
register(func: VgiFunction): void {
|
|
180
229
|
const name = func.meta.name;
|
|
@@ -184,12 +233,85 @@ export class FunctionRegistry {
|
|
|
184
233
|
this._functions.get(name)!.push(func);
|
|
185
234
|
}
|
|
186
235
|
|
|
236
|
+
/**
|
|
237
|
+
* Record that `func` is declared in `schemaName`, in addition to the flat
|
|
238
|
+
* by-name index. Idempotent, and independent of `register()` so a function
|
|
239
|
+
* reachable from several schemas (e.g. a scan function referenced by tables in
|
|
240
|
+
* more than one schema) resolves from any of them.
|
|
241
|
+
*/
|
|
242
|
+
registerInSchema(func: VgiFunction, schemaName: string, catalogName?: string): void {
|
|
243
|
+
const add = (map: Map<string, VgiFunction[]>, key: string) => {
|
|
244
|
+
const bucket = map.get(key);
|
|
245
|
+
if (!bucket) {
|
|
246
|
+
map.set(key, [func]);
|
|
247
|
+
return;
|
|
248
|
+
}
|
|
249
|
+
if (!bucket.includes(func)) bucket.push(func);
|
|
250
|
+
};
|
|
251
|
+
add(this._bySchema, schemaKey(schemaName, func.meta.name));
|
|
252
|
+
if (catalogName) {
|
|
253
|
+
add(this._byCatalog, catalogKey(catalogName, schemaName, func.meta.name));
|
|
254
|
+
}
|
|
255
|
+
}
|
|
256
|
+
|
|
257
|
+
/** Schemas that declare `functionName`, sorted — for error messages. */
|
|
258
|
+
schemasFor(functionName: string): string[] {
|
|
259
|
+
const out: string[] = [];
|
|
260
|
+
for (const key of this._bySchema.keys()) {
|
|
261
|
+
const [schema, name] = key.split("\u0000");
|
|
262
|
+
if (name === functionName) out.push(schema);
|
|
263
|
+
}
|
|
264
|
+
return out.sort();
|
|
265
|
+
}
|
|
266
|
+
|
|
187
267
|
get(name: string, context?: OverloadContext): VgiFunction {
|
|
188
|
-
|
|
268
|
+
let candidates = this._functions.get(name);
|
|
269
|
+
|
|
270
|
+
// A schema-qualified lookup is exact: only functions declared in that schema
|
|
271
|
+
// are candidates, so a name registered in two schemas dispatches to the one
|
|
272
|
+
// the caller named rather than colliding as an overload.
|
|
273
|
+
// Catalog first: it is the only key that separates two catalogs declaring
|
|
274
|
+
// the same schema and name.
|
|
275
|
+
const catalogName = context?.catalogName;
|
|
276
|
+
const schemaName = context?.schemaName;
|
|
277
|
+
if (catalogName && schemaName) {
|
|
278
|
+
const scoped = this._byCatalog.get(catalogKey(catalogName, schemaName, name));
|
|
279
|
+
if (scoped && scoped.length > 0) {
|
|
280
|
+
return this._disambiguate(name, scoped, context);
|
|
281
|
+
}
|
|
282
|
+
}
|
|
283
|
+
if (schemaName) {
|
|
284
|
+
const scoped = this._bySchema.get(schemaKey(schemaName, name));
|
|
285
|
+
if (scoped && scoped.length > 0) {
|
|
286
|
+
candidates = scoped;
|
|
287
|
+
} else if (candidates && candidates.length > 0) {
|
|
288
|
+
const schemas = this.schemasFor(name);
|
|
289
|
+
if (schemas.length > 0) {
|
|
290
|
+
throw new Error(
|
|
291
|
+
`Function '${name}' is not registered in schema '${schemaName}'. ` +
|
|
292
|
+
`It is available in: [${schemas.join(", ")}]`,
|
|
293
|
+
);
|
|
294
|
+
}
|
|
295
|
+
}
|
|
296
|
+
}
|
|
297
|
+
|
|
189
298
|
if (!candidates || candidates.length === 0) {
|
|
190
299
|
throw new FunctionNotFoundError(name, this.names());
|
|
191
300
|
}
|
|
192
301
|
|
|
302
|
+
return this._disambiguate(name, candidates, context);
|
|
303
|
+
}
|
|
304
|
+
|
|
305
|
+
/**
|
|
306
|
+
* Pick one function from an already-scoped candidate list, by argument shape
|
|
307
|
+
* and types. Scoping (catalog / schema) happens in `get()`; by the time this
|
|
308
|
+
* runs, every candidate is a legitimate overload of the same name.
|
|
309
|
+
*/
|
|
310
|
+
private _disambiguate(
|
|
311
|
+
name: string,
|
|
312
|
+
candidates: VgiFunction[],
|
|
313
|
+
context?: OverloadContext,
|
|
314
|
+
): VgiFunction {
|
|
193
315
|
// Fast path: single candidate
|
|
194
316
|
if (candidates.length === 1) {
|
|
195
317
|
return candidates[0];
|
package/src/functions/scalar.ts
CHANGED
|
@@ -185,10 +185,14 @@ export interface ScalarFunctionConfig<
|
|
|
185
185
|
/**
|
|
186
186
|
* Result-cache opt-in: when set, this CacheControl's `vgi.cache.*` metadata
|
|
187
187
|
* rides every output batch's custom metadata (per-batch — NOT the schema,
|
|
188
|
-
* which the IPC stream fixes at open)
|
|
189
|
-
*
|
|
190
|
-
* only — advertising this on a non-pure scalar serves stale rows. Mirrors
|
|
188
|
+
* which the IPC stream fixes at open). A pure, deterministic scalar only —
|
|
189
|
+
* advertising this on a non-pure scalar serves stale rows. Mirrors
|
|
191
190
|
* vgi-python's `ScalarFunction.CACHE_CONTROL`.
|
|
191
|
+
*
|
|
192
|
+
* Per-VALUE memoization (the extension remembering one output per distinct
|
|
193
|
+
* input value) additionally requires `perValue: true`, which is off by
|
|
194
|
+
* default because a memo serve only beats a worker call when the call is
|
|
195
|
+
* expensive — see {@link CacheControl.perValue}.
|
|
192
196
|
*/
|
|
193
197
|
cacheControl?: CacheControl;
|
|
194
198
|
}
|
|
@@ -405,8 +409,9 @@ export function defineScalarFunction<
|
|
|
405
409
|
}
|
|
406
410
|
|
|
407
411
|
// Result-cache opt-in: a scalar declaring cacheControl rides its
|
|
408
|
-
// vgi.cache.* keys on the emit path's per-batch custom metadata
|
|
409
|
-
//
|
|
412
|
+
// vgi.cache.* keys on the emit path's per-batch custom metadata. The
|
|
413
|
+
// per-value memo tier is only populated when that CacheControl also
|
|
414
|
+
// sets `perValue` (default off — it costs more than a cheap call).
|
|
410
415
|
out.emit(
|
|
411
416
|
outputBatch,
|
|
412
417
|
config.cacheControl ? cacheControlMetadata(config.cacheControl) : undefined,
|
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
// GENERATED by vgi.codegen.ts_client. DO NOT EDIT BY HAND.
|
|
3
3
|
//
|
|
4
4
|
// Generator: vgi-gen-ts-client v1
|
|
5
|
-
// Content hash:
|
|
5
|
+
// Content hash: 80ec56060e88
|
|
6
6
|
//
|
|
7
7
|
// To regenerate:
|
|
8
8
|
// uv run --project ~/Development/vgi-python vgi-gen-ts-client \
|
|
@@ -174,6 +174,7 @@ export interface AggregateBindRequest {
|
|
|
174
174
|
settings?: /** @record-batch */ Uint8Array | null;
|
|
175
175
|
secrets?: /** @record-batch */ Uint8Array | null;
|
|
176
176
|
attach_opaque_data?: Uint8Array | null;
|
|
177
|
+
schema_name?: string | null;
|
|
177
178
|
}
|
|
178
179
|
|
|
179
180
|
export interface AggregateBindResponse {
|
|
@@ -186,6 +187,7 @@ export interface AggregateCombineRequest {
|
|
|
186
187
|
execution_id: Uint8Array;
|
|
187
188
|
merge_batch: Uint8Array;
|
|
188
189
|
attach_opaque_data?: Uint8Array | null;
|
|
190
|
+
schema_name?: string | null;
|
|
189
191
|
}
|
|
190
192
|
|
|
191
193
|
export interface AggregateCombineResponse {
|
|
@@ -196,6 +198,7 @@ export interface AggregateDestructorRequest {
|
|
|
196
198
|
execution_id: Uint8Array;
|
|
197
199
|
group_ids_batch: Uint8Array;
|
|
198
200
|
attach_opaque_data?: Uint8Array | null;
|
|
201
|
+
schema_name?: string | null;
|
|
199
202
|
}
|
|
200
203
|
|
|
201
204
|
export interface AggregateDestructorResponse {
|
|
@@ -207,6 +210,7 @@ export interface AggregateFinalizeRequest {
|
|
|
207
210
|
group_ids_batch: Uint8Array;
|
|
208
211
|
output_schema: /** @arrow-schema */ Uint8Array;
|
|
209
212
|
attach_opaque_data?: Uint8Array | null;
|
|
213
|
+
schema_name?: string | null;
|
|
210
214
|
}
|
|
211
215
|
|
|
212
216
|
export interface AggregateFinalizeResponse {
|
|
@@ -218,6 +222,7 @@ export interface AggregateStreamingChunkRequest {
|
|
|
218
222
|
execution_id: Uint8Array;
|
|
219
223
|
input_batch: Uint8Array;
|
|
220
224
|
attach_opaque_data?: Uint8Array | null;
|
|
225
|
+
schema_name?: string | null;
|
|
221
226
|
}
|
|
222
227
|
|
|
223
228
|
export interface AggregateStreamingChunkResponse {
|
|
@@ -228,6 +233,7 @@ export interface AggregateStreamingCloseRequest {
|
|
|
228
233
|
function_name: string;
|
|
229
234
|
execution_id: Uint8Array;
|
|
230
235
|
attach_opaque_data?: Uint8Array | null;
|
|
236
|
+
schema_name?: string | null;
|
|
231
237
|
}
|
|
232
238
|
|
|
233
239
|
export interface AggregateStreamingCloseResponse {
|
|
@@ -243,6 +249,7 @@ export interface AggregateStreamingOpenRequest {
|
|
|
243
249
|
settings?: /** @record-batch */ Uint8Array | null;
|
|
244
250
|
secrets?: /** @record-batch */ Uint8Array | null;
|
|
245
251
|
attach_opaque_data?: Uint8Array | null;
|
|
252
|
+
schema_name?: string | null;
|
|
246
253
|
}
|
|
247
254
|
|
|
248
255
|
export interface AggregateStreamingOpenResponse {
|
|
@@ -254,6 +261,7 @@ export interface AggregateUpdateRequest {
|
|
|
254
261
|
execution_id: Uint8Array;
|
|
255
262
|
input_batch: Uint8Array;
|
|
256
263
|
attach_opaque_data?: Uint8Array | null;
|
|
264
|
+
schema_name?: string | null;
|
|
257
265
|
}
|
|
258
266
|
|
|
259
267
|
export interface AggregateUpdateResponse {
|
|
@@ -267,6 +275,7 @@ export interface AggregateWindowRequest {
|
|
|
267
275
|
frame_starts: number[];
|
|
268
276
|
frame_ends: number[];
|
|
269
277
|
attach_opaque_data?: Uint8Array | null;
|
|
278
|
+
schema_name?: string | null;
|
|
270
279
|
}
|
|
271
280
|
|
|
272
281
|
export interface AggregateWindowResponse {
|
|
@@ -283,6 +292,7 @@ export interface AggregateWindowBatchRequest {
|
|
|
283
292
|
frame_starts: number[];
|
|
284
293
|
frame_ends: number[];
|
|
285
294
|
attach_opaque_data?: Uint8Array | null;
|
|
295
|
+
schema_name?: string | null;
|
|
286
296
|
}
|
|
287
297
|
|
|
288
298
|
export interface AggregateWindowBatchResponse {
|
|
@@ -294,6 +304,7 @@ export interface AggregateWindowDestructorRequest {
|
|
|
294
304
|
execution_id: Uint8Array;
|
|
295
305
|
partition_id: number;
|
|
296
306
|
attach_opaque_data?: Uint8Array | null;
|
|
307
|
+
schema_name?: string | null;
|
|
297
308
|
}
|
|
298
309
|
|
|
299
310
|
export interface AggregateWindowDestructorResponse {
|
|
@@ -310,6 +321,7 @@ export interface AggregateWindowInitRequest {
|
|
|
310
321
|
frame_stats: Uint8Array;
|
|
311
322
|
all_valid: Uint8Array;
|
|
312
323
|
attach_opaque_data?: Uint8Array | null;
|
|
324
|
+
schema_name?: string | null;
|
|
313
325
|
}
|
|
314
326
|
|
|
315
327
|
export interface AggregateWindowInitResponse {
|
|
@@ -340,6 +352,7 @@ export interface BindRequest {
|
|
|
340
352
|
at_value?: string | null;
|
|
341
353
|
copy_from?: CopyFromContext | null;
|
|
342
354
|
copy_to?: CopyToContext | null;
|
|
355
|
+
schema_name?: string | null;
|
|
343
356
|
}
|
|
344
357
|
|
|
345
358
|
export interface BindResponse {
|
|
@@ -487,6 +500,7 @@ export interface TableBufferingCombineRequest {
|
|
|
487
500
|
state_ids: Uint8Array[];
|
|
488
501
|
attach_opaque_data?: Uint8Array | null;
|
|
489
502
|
transaction_id?: Uint8Array | null;
|
|
503
|
+
schema_name?: string | null;
|
|
490
504
|
}
|
|
491
505
|
|
|
492
506
|
export interface TableBufferingCombineResponse {
|
|
@@ -498,6 +512,7 @@ export interface TableBufferingDestructorRequest {
|
|
|
498
512
|
execution_id: Uint8Array;
|
|
499
513
|
attach_opaque_data?: Uint8Array | null;
|
|
500
514
|
transaction_id?: Uint8Array | null;
|
|
515
|
+
schema_name?: string | null;
|
|
501
516
|
}
|
|
502
517
|
|
|
503
518
|
export interface TableBufferingDestructorResponse {
|
|
@@ -510,6 +525,7 @@ export interface TableBufferingProcessRequest {
|
|
|
510
525
|
attach_opaque_data?: Uint8Array | null;
|
|
511
526
|
transaction_id?: Uint8Array | null;
|
|
512
527
|
batch_index?: number | null;
|
|
528
|
+
schema_name?: string | null;
|
|
513
529
|
}
|
|
514
530
|
|
|
515
531
|
export interface TableBufferingProcessResponse {
|
package/src/index.core.ts
CHANGED
|
@@ -16,6 +16,8 @@ export {
|
|
|
16
16
|
CACHE_LAST_MODIFIED_KEY,
|
|
17
17
|
CACHE_NO_STORE_KEY,
|
|
18
18
|
CACHE_NOT_MODIFIED_KEY,
|
|
19
|
+
CACHE_PARTITION_SCOPE_KEY,
|
|
20
|
+
CACHE_PER_VALUE_KEY,
|
|
19
21
|
CACHE_REVALIDATABLE_KEY,
|
|
20
22
|
CACHE_SCOPE_CATALOG,
|
|
21
23
|
CACHE_SCOPE_KEY,
|
|
@@ -247,7 +249,9 @@ export {
|
|
|
247
249
|
isNull, isInt, isFloat, isBinary, isUtf8, isBool, isDecimal, isDate,
|
|
248
250
|
isTime, isTimestamp, isInterval, isList, isStruct, isUnion,
|
|
249
251
|
isFixedSizeBinary, isFixedSizeList, isMap, isDuration, isDictionary, isBatch,
|
|
252
|
+
typeSignature,
|
|
250
253
|
readCanonicalValue, columnFromArray, decodeDictValue, backend,
|
|
254
|
+
adoptArrowJsShape,
|
|
251
255
|
} from "./arrow/index.js";
|
|
252
256
|
|
|
253
257
|
export type {
|
package/src/protocol/dispatch.ts
CHANGED
|
@@ -43,6 +43,7 @@ export function buildVgiProtocol(config: ProtocolConfig): Protocol {
|
|
|
43
43
|
registry: config.registry,
|
|
44
44
|
recoverExchangeState: config.recoverExchangeState,
|
|
45
45
|
signingKey: config.signingKey,
|
|
46
|
+
catalogInterface: config.catalogInterface,
|
|
46
47
|
});
|
|
47
48
|
registerAggregateMethods(protocol, config.registry);
|
|
48
49
|
registerTableBufferingMethods(protocol, config.registry, config.signingKey);
|