@query-farm/vgi 0.28.0 → 0.29.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +52 -8
- package/dist/arguments/argument-spec.d.ts +57 -2
- package/dist/arrow/impl-arrowjs/index.d.ts +1 -1
- package/dist/arrow/impl-arrowjs/iterate.d.ts +10 -1
- package/dist/arrow/impl-arrowjs/schema.d.ts +2 -0
- package/dist/arrow/impl-flechette/index.d.ts +1 -1
- package/dist/arrow/impl-flechette/iterate.d.ts +10 -1
- package/dist/arrow/impl-flechette/schema.d.ts +2 -0
- package/dist/arrow/index.d.ts +1 -1
- package/dist/arrow/schema-types.d.ts +3 -1
- package/dist/arrow/types.d.ts +39 -3
- package/dist/catalog/composite.d.ts +13 -13
- package/dist/catalog/descriptors.d.ts +7 -3
- package/dist/catalog/interface.d.ts +108 -38
- package/dist/catalog/read-only.d.ts +32 -15
- package/dist/client/client.d.ts +37 -30
- package/dist/client/types.d.ts +2 -0
- package/dist/client-entry.d.ts +1 -1
- package/dist/client-entry.js +791 -447
- package/dist/client-entry.js.map +18 -17
- package/dist/filter-pushdown/capabilities.d.ts +3 -0
- package/dist/filter-pushdown/deserialize.d.ts +14 -15
- package/dist/filter-pushdown/evaluate.d.ts +14 -44
- package/dist/filter-pushdown/index.d.ts +2 -2
- package/dist/filter-pushdown/types.d.ts +120 -68
- package/dist/functions/aggregate.d.ts +4 -0
- package/dist/functions/registry.d.ts +5 -5
- package/dist/functions/scalar.d.ts +6 -1
- package/dist/functions/table-buffering.d.ts +9 -0
- package/dist/functions/table-in-out.d.ts +13 -0
- package/dist/functions/table.d.ts +72 -2
- package/dist/functions/types.d.ts +60 -2
- package/dist/generated/vgi-client.d.ts +114 -26
- package/dist/generated/vgi-protocol-schemas.d.ts +34 -0
- package/dist/http/fetch.d.ts +16 -0
- package/dist/index.core.d.ts +4 -3
- package/dist/index.js +3396 -2064
- package/dist/index.js.map +55 -49
- package/dist/metadata/types.d.ts +15 -2
- package/dist/protocol/handlers/catalog/shared.d.ts +0 -8
- package/dist/protocol/handlers/function.d.ts +11 -0
- package/dist/protocol/handlers/shared.d.ts +3 -1
- package/dist/protocol/serializers/bind.d.ts +1 -2
- package/dist/protocol/serializers/cardinality.d.ts +1 -2
- package/dist/protocol/serializers/splits.d.ts +126 -0
- package/dist/protocol/types.d.ts +36 -3
- package/dist/request-auth.d.ts +28 -0
- package/dist/schema-path.d.ts +7 -0
- package/dist/serve-entry.d.ts +25 -0
- package/dist/serve-entry.js +1129 -569
- package/dist/serve-entry.js.map +36 -32
- package/dist/split-token.d.ts +88 -0
- package/dist/types.d.ts +8 -0
- package/dist/worker-cf-entry.js +2928 -1689
- package/dist/worker-cf-entry.js.map +51 -44
- package/dist/worker.d.ts +1 -1
- package/package.json +3 -3
- package/src/arguments/argument-spec.ts +93 -4
- package/src/arguments/arguments.ts +0 -1
- package/src/arrow/impl-arrowjs/index.ts +1 -1
- package/src/arrow/impl-arrowjs/iterate.ts +21 -0
- package/src/arrow/impl-arrowjs/schema.ts +4 -0
- package/src/arrow/impl-flechette/index.ts +1 -1
- package/src/arrow/impl-flechette/iterate.ts +24 -1
- package/src/arrow/impl-flechette/schema.ts +4 -0
- package/src/arrow/index.ts +1 -1
- package/src/arrow/schema-types.ts +4 -0
- package/src/arrow/types.ts +39 -3
- package/src/catalog/composite.ts +26 -26
- package/src/catalog/descriptors.ts +12 -3
- package/src/catalog/interface.ts +217 -72
- package/src/catalog/read-only.ts +141 -84
- package/src/client/client.ts +97 -73
- package/src/client/types.ts +2 -0
- package/src/client-entry.ts +2 -1
- package/src/filter-pushdown/capabilities.ts +13 -0
- package/src/filter-pushdown/collector.ts +22 -35
- package/src/filter-pushdown/deserialize.ts +657 -221
- package/src/filter-pushdown/evaluate.ts +410 -499
- package/src/filter-pushdown/index.ts +16 -2
- package/src/filter-pushdown/types.ts +88 -71
- package/src/functions/aggregate.ts +12 -1
- package/src/functions/copy-from.ts +2 -1
- package/src/functions/copy-to.ts +2 -1
- package/src/functions/registry.ts +24 -22
- package/src/functions/scalar.ts +37 -0
- package/src/functions/table-buffering.ts +48 -6
- package/src/functions/table-in-out.ts +194 -33
- package/src/functions/table.ts +187 -22
- package/src/functions/types.ts +64 -1
- package/src/generated/vgi-client.ts +122 -28
- package/src/generated/vgi-protocol-schemas.ts +455 -58
- package/src/http/fetch.ts +129 -2
- package/src/index.core.ts +4 -2
- package/src/metadata/resolve.ts +43 -1
- package/src/metadata/serialize.ts +43 -5
- package/src/metadata/types.ts +17 -1
- package/src/protocol/dispatch.ts +1 -1
- package/src/protocol/handlers/aggregate.ts +23 -12
- package/src/protocol/handlers/catalog/admin.ts +47 -54
- package/src/protocol/handlers/catalog/index_methods.ts +6 -13
- package/src/protocol/handlers/catalog/macro.ts +12 -20
- package/src/protocol/handlers/catalog/shared.ts +12 -57
- package/src/protocol/handlers/catalog/table.ts +49 -112
- package/src/protocol/handlers/catalog/view.ts +18 -22
- package/src/protocol/handlers/function.ts +311 -18
- package/src/protocol/handlers/shared.ts +12 -3
- package/src/protocol/handlers/table-buffering.ts +11 -8
- package/src/protocol/serializers/bind.ts +69 -67
- package/src/protocol/serializers/cardinality.ts +7 -7
- package/src/protocol/serializers/init.ts +52 -10
- package/src/protocol/serializers/splits.ts +221 -0
- package/src/protocol/state-serializer.ts +55 -1
- package/src/protocol/types.ts +36 -3
- package/src/request-auth.ts +73 -0
- package/src/schema-path.ts +25 -0
- package/src/serve-entry.ts +69 -2
- package/src/split-token.ts +297 -0
- package/src/types.ts +9 -0
- package/src/worker-cf-entry.ts +2 -1
- package/src/worker.ts +79 -14
package/src/catalog/read-only.ts
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
// Copyright 2025, 2026 Query Farm LLC - https://query.farm
|
|
2
2
|
// ReadOnlyCatalogInterface: derives catalog from registered functions + descriptors.
|
|
3
3
|
|
|
4
|
-
import { type VgiSchema, schema as schema_, type VgiField, field, type VgiDataType, int64, utf8, binary, bool, nullType, list, isInt } from "../arrow/index.js";
|
|
4
|
+
import { type VgiSchema, schema as schema_, type VgiField, field, type VgiDataType, int64, utf8, binary, bool, nullType, list, isInt, typeSignature } from "../arrow/index.js";
|
|
5
5
|
import {
|
|
6
6
|
CatalogInterface,
|
|
7
7
|
type AttachOpaqueData,
|
|
@@ -17,9 +17,10 @@ import {
|
|
|
17
17
|
type IndexConstraintType,
|
|
18
18
|
type CopyFromFormatInfo,
|
|
19
19
|
encodeAttachCatalogInfo,
|
|
20
|
+
encodeScanFunctionResult,
|
|
20
21
|
encodeFunctionInfo,
|
|
21
22
|
} from "./interface.js";
|
|
22
|
-
import type
|
|
23
|
+
import { schemaDescriptorPath, type CatalogDescriptor, type SchemaDescriptor, type TableDescriptor, type ViewDescriptor, type MacroDescriptor, type SettingDescriptor, type SecretTypeDescriptor, type ForeignKeyDef, type DefaultValue } from "./descriptors.js";
|
|
23
24
|
import { serializeColumnStatistics } from "../util/statistics.js";
|
|
24
25
|
import type { VgiFunction } from "../functions/types.js";
|
|
25
26
|
import type { FunctionRegistry } from "../functions/registry.js";
|
|
@@ -29,6 +30,8 @@ import { resolveMetadata } from "../metadata/resolve.js";
|
|
|
29
30
|
import { type AttachOptionSpec, validateRequiredAttachOptions } from "./attach-option.js";
|
|
30
31
|
import { FunctionStability, NullHandling, OrderPreservation, DEFAULT_MAX_WORKERS } from "../types.js";
|
|
31
32
|
import { Arguments } from "../arguments/arguments.js";
|
|
33
|
+
import { normalizeSchemaPath, schemaPathDisplay, schemaPathsEqual } from "../schema-path.js";
|
|
34
|
+
import { ForeignKeyInfoSchema } from "../generated/vgi-protocol-schemas.js";
|
|
32
35
|
|
|
33
36
|
export class ReadOnlyCatalogInterface extends CatalogInterface {
|
|
34
37
|
private _descriptor: CatalogDescriptor;
|
|
@@ -45,16 +48,39 @@ export class ReadOnlyCatalogInterface extends CatalogInterface {
|
|
|
45
48
|
// is declared in more than one schema. Table-backing functions are indexed
|
|
46
49
|
// under their table's schema for the same reason.
|
|
47
50
|
for (const sch of descriptor.schemas ?? []) {
|
|
48
|
-
for (const fn of sch.functions ?? []) registry.registerInSchema(fn, sch
|
|
51
|
+
for (const fn of sch.functions ?? []) registry.registerInSchema(fn, schemaDescriptorPath(sch), descriptor.name);
|
|
49
52
|
for (const tbl of sch.tables ?? []) {
|
|
50
53
|
for (const key of ["function", "insertFunction", "updateFunction", "deleteFunction"] as const) {
|
|
51
54
|
const fn = (tbl as any)[key];
|
|
52
|
-
if (fn) registry.registerInSchema(fn, sch
|
|
55
|
+
if (fn) registry.registerInSchema(fn, schemaDescriptorPath(sch), descriptor.name);
|
|
53
56
|
}
|
|
54
57
|
}
|
|
55
58
|
}
|
|
56
59
|
}
|
|
57
60
|
|
|
61
|
+
/**
|
|
62
|
+
* Which schema declares `fn`, by object identity, or null if no schema does.
|
|
63
|
+
*
|
|
64
|
+
* A table's backing function is NOT necessarily declared in the table's own
|
|
65
|
+
* schema — `data.large_sequence` scans via `sequence`, which the example
|
|
66
|
+
* catalog declares in `main`. Populating `ScanFunctionResult.schemaPath` /
|
|
67
|
+
* `ScanBranch.schemaPath` (protocol 2.0.0) correctly needs this reverse
|
|
68
|
+
* lookup, not the `schemaPath` parameter of the call being served.
|
|
69
|
+
*
|
|
70
|
+
* Identity, not name: the whole point of the field is to disambiguate one
|
|
71
|
+
* name declared in several schemas, so a name match would answer the very
|
|
72
|
+
* question being asked. Null when the function is deliberately unadvertised
|
|
73
|
+
* (`cache_multicol`) — it has no schema to report and the client falls back
|
|
74
|
+
* to its own heuristic, which is exactly the pre-1.5.0 behaviour.
|
|
75
|
+
* Mirrors vgi-python's `ReadOnlyCatalogInterface._schema_for_function`.
|
|
76
|
+
*/
|
|
77
|
+
private _schemaForFunction(fn: VgiFunction): string[] | null {
|
|
78
|
+
for (const sch of this._descriptor.schemas ?? []) {
|
|
79
|
+
if ((sch.functions ?? []).includes(fn)) return schemaDescriptorPath(sch);
|
|
80
|
+
}
|
|
81
|
+
return null;
|
|
82
|
+
}
|
|
83
|
+
|
|
58
84
|
catalogs(): string[] {
|
|
59
85
|
return [this._descriptor.name];
|
|
60
86
|
}
|
|
@@ -167,7 +193,7 @@ export class ReadOnlyCatalogInterface extends CatalogInterface {
|
|
|
167
193
|
schemas(attachOpaqueData: AttachOpaqueData, transactionOpaqueData?: TransactionOpaqueData): SchemaInfo[] {
|
|
168
194
|
return this._descriptor.schemas.map((s) => ({
|
|
169
195
|
attach_opaque_data: attachOpaqueData,
|
|
170
|
-
|
|
196
|
+
path: schemaDescriptorPath(s),
|
|
171
197
|
comment: s.comment ?? null,
|
|
172
198
|
tags: s.tags ?? {},
|
|
173
199
|
estimated_object_count: this._estimatedObjectCount(s),
|
|
@@ -198,14 +224,14 @@ export class ReadOnlyCatalogInterface extends CatalogInterface {
|
|
|
198
224
|
|
|
199
225
|
override schemaGet(
|
|
200
226
|
attachOpaqueData: AttachOpaqueData,
|
|
201
|
-
|
|
227
|
+
path: string[],
|
|
202
228
|
transactionOpaqueData?: TransactionOpaqueData
|
|
203
229
|
): SchemaInfo | null {
|
|
204
|
-
const desc = this._descriptor.schemas.find((s) => s
|
|
230
|
+
const desc = this._descriptor.schemas.find((s) => schemaPathsEqual(schemaDescriptorPath(s), path));
|
|
205
231
|
if (!desc) return null;
|
|
206
232
|
return {
|
|
207
233
|
attach_opaque_data: attachOpaqueData,
|
|
208
|
-
|
|
234
|
+
path: schemaDescriptorPath(desc),
|
|
209
235
|
comment: desc.comment ?? null,
|
|
210
236
|
tags: desc.tags ?? {},
|
|
211
237
|
estimated_object_count: this._estimatedObjectCount(desc),
|
|
@@ -214,10 +240,10 @@ export class ReadOnlyCatalogInterface extends CatalogInterface {
|
|
|
214
240
|
|
|
215
241
|
override async schemaContentsTables(
|
|
216
242
|
attachOpaqueData: AttachOpaqueData,
|
|
217
|
-
|
|
243
|
+
path: string[],
|
|
218
244
|
transactionOpaqueData?: TransactionOpaqueData
|
|
219
245
|
): Promise<TableInfo[]> {
|
|
220
|
-
const schema = this._descriptor.schemas.find((s) => s
|
|
246
|
+
const schema = this._descriptor.schemas.find((s) => schemaPathsEqual(schemaDescriptorPath(s), path));
|
|
221
247
|
if (!schema || !schema.tables) return [];
|
|
222
248
|
|
|
223
249
|
return await Promise.all(schema.tables.map(async (t) => {
|
|
@@ -272,7 +298,7 @@ export class ReadOnlyCatalogInterface extends CatalogInterface {
|
|
|
272
298
|
const notNullIndices = resolveIndices(colSchema, t.notNull);
|
|
273
299
|
const uniqueIndices = resolveIndexGroups(colSchema, t.unique);
|
|
274
300
|
const pkIndices = resolveIndexGroups(colSchema, t.primaryKey);
|
|
275
|
-
const fkBytes = serializeForeignKeys(t.foreignKey,
|
|
301
|
+
const fkBytes = serializeForeignKeys(t.foreignKey, path);
|
|
276
302
|
|
|
277
303
|
// Validate required_filters: an AND (outer list) of OR-groups (inner
|
|
278
304
|
// lists). Each OR-group must be non-empty, and the leading dotted segment
|
|
@@ -310,7 +336,7 @@ export class ReadOnlyCatalogInterface extends CatalogInterface {
|
|
|
310
336
|
comment: t.comment ?? null,
|
|
311
337
|
tags: t.tags ?? {},
|
|
312
338
|
name: t.name,
|
|
313
|
-
|
|
339
|
+
schema_path: normalizeSchemaPath(path),
|
|
314
340
|
columns,
|
|
315
341
|
not_null_constraints: notNullIndices,
|
|
316
342
|
unique_constraints: uniqueIndices,
|
|
@@ -328,7 +354,7 @@ export class ReadOnlyCatalogInterface extends CatalogInterface {
|
|
|
328
354
|
// shadow the per-call dispatch and prevent workers from varying
|
|
329
355
|
// the scan function (e.g. version-resolved time-travel).
|
|
330
356
|
scan_function: t.function && !t.columns
|
|
331
|
-
? inlineScanFunction(t.function, t.arguments)
|
|
357
|
+
? inlineScanFunction(t.function, t.arguments, this._schemaForFunction(t.function))
|
|
332
358
|
: null,
|
|
333
359
|
insert_function: null,
|
|
334
360
|
update_function: null,
|
|
@@ -346,11 +372,11 @@ export class ReadOnlyCatalogInterface extends CatalogInterface {
|
|
|
346
372
|
|
|
347
373
|
override tableColumnStatisticsGet(
|
|
348
374
|
attachOpaqueData: AttachOpaqueData,
|
|
349
|
-
|
|
375
|
+
schemaPath: string[],
|
|
350
376
|
name: string,
|
|
351
377
|
transactionOpaqueData?: TransactionOpaqueData,
|
|
352
378
|
): { bytes: Uint8Array; cacheMaxAgeSeconds: number | null } | null {
|
|
353
|
-
const schema = this._descriptor.schemas.find((s) => s
|
|
379
|
+
const schema = this._descriptor.schemas.find((s) => schemaPathsEqual(schemaDescriptorPath(s), schemaPath));
|
|
354
380
|
if (!schema || !schema.tables) return null;
|
|
355
381
|
const table = schema.tables.find((t) => t.name === name);
|
|
356
382
|
if (!table || !table.statistics) return null;
|
|
@@ -364,17 +390,17 @@ export class ReadOnlyCatalogInterface extends CatalogInterface {
|
|
|
364
390
|
|
|
365
391
|
override schemaContentsViews(
|
|
366
392
|
attachOpaqueData: AttachOpaqueData,
|
|
367
|
-
|
|
393
|
+
path: string[],
|
|
368
394
|
transactionOpaqueData?: TransactionOpaqueData
|
|
369
395
|
): ViewInfo[] {
|
|
370
|
-
const schema = this._descriptor.schemas.find((s) => s
|
|
396
|
+
const schema = this._descriptor.schemas.find((s) => schemaPathsEqual(schemaDescriptorPath(s), path));
|
|
371
397
|
if (!schema || !schema.views) return [];
|
|
372
398
|
|
|
373
399
|
return schema.views.map((v) => ({
|
|
374
400
|
comment: v.comment ?? null,
|
|
375
401
|
tags: v.tags ?? {},
|
|
376
402
|
name: v.name,
|
|
377
|
-
|
|
403
|
+
schema_path: normalizeSchemaPath(path),
|
|
378
404
|
definition: v.definition,
|
|
379
405
|
column_comments: v.columnComments ?? {},
|
|
380
406
|
}));
|
|
@@ -382,11 +408,11 @@ export class ReadOnlyCatalogInterface extends CatalogInterface {
|
|
|
382
408
|
|
|
383
409
|
override schemaContentsFunctions(
|
|
384
410
|
attachOpaqueData: AttachOpaqueData,
|
|
385
|
-
|
|
411
|
+
path: string[],
|
|
386
412
|
type: string,
|
|
387
413
|
transactionOpaqueData?: TransactionOpaqueData
|
|
388
414
|
): FunctionInfo[] {
|
|
389
|
-
const schema = this._descriptor.schemas.find((s) => s
|
|
415
|
+
const schema = this._descriptor.schemas.find((s) => schemaPathsEqual(schemaDescriptorPath(s), path));
|
|
390
416
|
if (!schema || !schema.functions) return [];
|
|
391
417
|
|
|
392
418
|
return schema.functions
|
|
@@ -400,21 +426,44 @@ export class ReadOnlyCatalogInterface extends CatalogInterface {
|
|
|
400
426
|
if (t === "AGGREGATE_FUNCTION") return (f.kind as string) === "aggregate";
|
|
401
427
|
return true;
|
|
402
428
|
})
|
|
403
|
-
.map((f) => this._functionToInfo(f,
|
|
429
|
+
.map((f) => this._functionToInfo(f, path));
|
|
404
430
|
}
|
|
405
431
|
|
|
406
432
|
/**
|
|
407
433
|
* Build the wire `FunctionInfo` for one declared function.
|
|
408
434
|
*
|
|
409
|
-
* `
|
|
435
|
+
* `schemaPath` is the schema the function actually lives in — it is the
|
|
410
436
|
* bind-dispatch key, so a function advertised through more than one route
|
|
411
437
|
* (schema contents *and* `global_functions`) must carry the same value in
|
|
412
438
|
* both. Shared by `schemaContentsFunctions` and `_globalFunctionInfos`.
|
|
413
439
|
*/
|
|
414
|
-
private _functionToInfo(f: VgiFunction,
|
|
440
|
+
private _functionToInfo(f: VgiFunction, path: string[]): FunctionInfo {
|
|
415
441
|
const meta = resolveMetadata(f);
|
|
416
442
|
const argSchema = argumentSpecsToSchema(f.argumentSpecs);
|
|
417
443
|
const argBytes = serializeSchema(argSchema);
|
|
444
|
+
const defaults = f.meta.parameterDefaultValues ?? null;
|
|
445
|
+
if (defaults) {
|
|
446
|
+
if (defaults.numRows !== 1) {
|
|
447
|
+
throw new Error(`parameterDefaultValues must contain exactly one row, got ${defaults.numRows}`);
|
|
448
|
+
}
|
|
449
|
+
let argumentIndex = 0;
|
|
450
|
+
for (const defaultField of defaults.schema.fields) {
|
|
451
|
+
while (argumentIndex < argSchema.fields.length && argSchema.fields[argumentIndex].name !== defaultField.name) {
|
|
452
|
+
argumentIndex++;
|
|
453
|
+
}
|
|
454
|
+
if (argumentIndex === argSchema.fields.length) {
|
|
455
|
+
throw new Error(`parameterDefaultValues field '${defaultField.name}' is not in argument signature order`);
|
|
456
|
+
}
|
|
457
|
+
const argumentField = argSchema.fields[argumentIndex];
|
|
458
|
+
if (typeSignature(argumentField.type) !== typeSignature(defaultField.type)) {
|
|
459
|
+
throw new Error(
|
|
460
|
+
`parameterDefaultValues field '${defaultField.name}' has type ${typeSignature(defaultField.type)}, ` +
|
|
461
|
+
`expected ${typeSignature(argumentField.type)}`,
|
|
462
|
+
);
|
|
463
|
+
}
|
|
464
|
+
argumentIndex++;
|
|
465
|
+
}
|
|
466
|
+
}
|
|
418
467
|
|
|
419
468
|
// Use the function's default output schema if available,
|
|
420
469
|
// otherwise empty schema (table functions determine output at bind time)
|
|
@@ -442,12 +491,16 @@ export class ReadOnlyCatalogInterface extends CatalogInterface {
|
|
|
442
491
|
comment: null,
|
|
443
492
|
tags,
|
|
444
493
|
name: f.meta.name,
|
|
445
|
-
|
|
494
|
+
schema_path: normalizeSchemaPath(path),
|
|
446
495
|
function_type: meta.functionType.toUpperCase() as "SCALAR" | "TABLE" | "TABLE_BUFFERING" | "AGGREGATE",
|
|
447
496
|
arguments: argBytes,
|
|
448
497
|
output_schema: outputSchemaBytes,
|
|
498
|
+
parameter_default_values: defaults
|
|
499
|
+
? serializeBatch(defaults)
|
|
500
|
+
: null,
|
|
449
501
|
stability: meta.stability as any,
|
|
450
502
|
null_handling: meta.nullHandling as any,
|
|
503
|
+
argument_monotonicity: meta.argumentMonotonicity,
|
|
451
504
|
description: meta.description,
|
|
452
505
|
examples: meta.examples.map((e) => ({
|
|
453
506
|
sql: e.sql,
|
|
@@ -462,7 +515,16 @@ export class ReadOnlyCatalogInterface extends CatalogInterface {
|
|
|
462
515
|
// The C++ extension only honours this for rowid-bearing tables.
|
|
463
516
|
late_materialization:
|
|
464
517
|
(f.kind as string) === "table" && f.meta.lateMaterialization === true ? true : null,
|
|
465
|
-
|
|
518
|
+
filter_semantic_profiles: meta.filterSemanticProfiles,
|
|
519
|
+
additional_filter_functions: meta.additionalFilterFunctions.map((value) => ({
|
|
520
|
+
namespace: value.namespace, name: value.name, version: BigInt(value.version) as any,
|
|
521
|
+
})),
|
|
522
|
+
runtime_filter_algorithms: meta.runtimeFilterAlgorithms.map((value) => ({
|
|
523
|
+
namespace: value.namespace, name: value.name, version: BigInt(value.version) as any,
|
|
524
|
+
})),
|
|
525
|
+
filter_evaluation_contexts: meta.filterEvaluationContexts.map((value) => ({
|
|
526
|
+
profile: value.profile, provider_fingerprint: value.providerFingerprint,
|
|
527
|
+
})),
|
|
466
528
|
order_preservation: meta.preservesOrder as any,
|
|
467
529
|
max_workers: meta.maxWorkers ?? DEFAULT_MAX_WORKERS,
|
|
468
530
|
// Opt-in features the read-only catalog's functions do not use; the
|
|
@@ -473,6 +535,22 @@ export class ReadOnlyCatalogInterface extends CatalogInterface {
|
|
|
473
535
|
supports_batch_index:
|
|
474
536
|
((f.kind as string) === "table_buffering" && f.meta.requiresInputBatchIndex === true) ||
|
|
475
537
|
((f.kind as string) === "table" && f.meta.supportsBatchIndex === true),
|
|
538
|
+
// Split-based scan planning (protocol 1.4.0). A function opts in by
|
|
539
|
+
// declaring `supportsSplits` AND implementing plan()/onSplit(): the
|
|
540
|
+
// declaration is what a distributed engine reads to decide it can
|
|
541
|
+
// retry a task, because a split NAMES its work and re-running one
|
|
542
|
+
// reads exactly the same rows.
|
|
543
|
+
supports_splits: (f.kind as string) === "table" && f.meta.supportsSplits === true,
|
|
544
|
+
// The worker applies pushed filters EXACTLY, so the engine may drop
|
|
545
|
+
// its own copy. Wrong answers if declared falsely.
|
|
546
|
+
filters_exactly_applied:
|
|
547
|
+
(f.kind as string) === "table" && f.meta.filtersExactlyApplied === true,
|
|
548
|
+
// Addressable positions in the data, for incremental/streaming reads.
|
|
549
|
+
supports_positions: (f.kind as string) === "table" && f.meta.supportsPositions === true,
|
|
550
|
+
// null means UNBOUNDED, not "expires immediately" — a client must not
|
|
551
|
+
// assume a TTL exists, or long-running streams are foreclosed.
|
|
552
|
+
split_token_ttl_seconds:
|
|
553
|
+
(f.kind as string) === "table" ? (f.meta.splitTokenTtlSeconds ?? null) : null,
|
|
476
554
|
partition_kind:
|
|
477
555
|
(f.kind as string) === "table" && f.meta.partitionKind
|
|
478
556
|
? f.meta.partitionKind
|
|
@@ -516,7 +594,7 @@ export class ReadOnlyCatalogInterface extends CatalogInterface {
|
|
|
516
594
|
/**
|
|
517
595
|
* Build `FunctionInfo` records for `CatalogDescriptor.globalFunctions`.
|
|
518
596
|
*
|
|
519
|
-
* Each entry's `
|
|
597
|
+
* Each entry's `schema_path` is the schema it actually lives in — the
|
|
520
598
|
* globally visible name is derived client-side from
|
|
521
599
|
* `global_function_prefix`, so `name` stays unprefixed here. An entry not
|
|
522
600
|
* declared in any schema is a descriptor bug, so it throws rather than
|
|
@@ -527,32 +605,32 @@ export class ReadOnlyCatalogInterface extends CatalogInterface {
|
|
|
527
605
|
const globals = this._descriptor.globalFunctions ?? [];
|
|
528
606
|
if (globals.length === 0) return [];
|
|
529
607
|
|
|
530
|
-
const schemaOf = new Map<VgiFunction, string>();
|
|
608
|
+
const schemaOf = new Map<VgiFunction, string[]>();
|
|
531
609
|
for (const sch of this._descriptor.schemas) {
|
|
532
610
|
for (const fn of sch.functions ?? []) {
|
|
533
|
-
if (!schemaOf.has(fn)) schemaOf.set(fn, sch
|
|
611
|
+
if (!schemaOf.has(fn)) schemaOf.set(fn, schemaDescriptorPath(sch));
|
|
534
612
|
}
|
|
535
613
|
}
|
|
536
614
|
|
|
537
615
|
return globals.map((fn) => {
|
|
538
|
-
const
|
|
539
|
-
if (
|
|
616
|
+
const schemaPath = schemaOf.get(fn);
|
|
617
|
+
if (schemaPath === undefined) {
|
|
540
618
|
throw new Error(
|
|
541
619
|
`Catalog '${this._descriptor.name}': '${fn.meta.name}' is listed in ` +
|
|
542
620
|
`globalFunctions but is not declared in any schema.`,
|
|
543
621
|
);
|
|
544
622
|
}
|
|
545
|
-
return this._functionToInfo(fn,
|
|
623
|
+
return this._functionToInfo(fn, schemaPath);
|
|
546
624
|
});
|
|
547
625
|
}
|
|
548
626
|
|
|
549
627
|
override schemaContentsMacros(
|
|
550
628
|
attachOpaqueData: AttachOpaqueData,
|
|
551
|
-
|
|
629
|
+
path: string[],
|
|
552
630
|
type: string,
|
|
553
631
|
transactionOpaqueData?: TransactionOpaqueData
|
|
554
632
|
): MacroInfo[] {
|
|
555
|
-
const schema = this._descriptor.schemas.find((s) => s
|
|
633
|
+
const schema = this._descriptor.schemas.find((s) => schemaPathsEqual(schemaDescriptorPath(s), path));
|
|
556
634
|
if (!schema || !schema.macros) return [];
|
|
557
635
|
|
|
558
636
|
return schema.macros
|
|
@@ -567,7 +645,7 @@ export class ReadOnlyCatalogInterface extends CatalogInterface {
|
|
|
567
645
|
comment: m.comment ?? null,
|
|
568
646
|
tags: m.tags ?? {},
|
|
569
647
|
name: m.name,
|
|
570
|
-
|
|
648
|
+
schema_path: normalizeSchemaPath(path),
|
|
571
649
|
macro_type: (m.macroType === "scalar" ? "SCALAR" : "TABLE") as "SCALAR" | "TABLE",
|
|
572
650
|
parameters: m.parameters,
|
|
573
651
|
parameter_default_values: m.parameterDefaultValues ?? new Uint8Array(0),
|
|
@@ -584,16 +662,16 @@ export class ReadOnlyCatalogInterface extends CatalogInterface {
|
|
|
584
662
|
|
|
585
663
|
override schemaContentsIndexes(
|
|
586
664
|
attachOpaqueData: AttachOpaqueData,
|
|
587
|
-
|
|
665
|
+
path: string[],
|
|
588
666
|
transactionOpaqueData?: TransactionOpaqueData
|
|
589
667
|
): IndexInfo[] {
|
|
590
|
-
const schema = this._descriptor.schemas.find((s) => s
|
|
668
|
+
const schema = this._descriptor.schemas.find((s) => schemaPathsEqual(schemaDescriptorPath(s), path));
|
|
591
669
|
if (!schema || !schema.indexes) return [];
|
|
592
670
|
return schema.indexes.map((idx) => ({
|
|
593
671
|
comment: idx.comment ?? null,
|
|
594
672
|
tags: idx.tags ?? {},
|
|
595
673
|
name: idx.name,
|
|
596
|
-
|
|
674
|
+
schema_path: normalizeSchemaPath(path),
|
|
597
675
|
table_name: idx.tableName,
|
|
598
676
|
index_type: idx.indexType ?? "ART",
|
|
599
677
|
constraint_type: (idx.constraintType ?? "NONE") as IndexConstraintType,
|
|
@@ -604,29 +682,29 @@ export class ReadOnlyCatalogInterface extends CatalogInterface {
|
|
|
604
682
|
|
|
605
683
|
override indexGet(
|
|
606
684
|
attachOpaqueData: AttachOpaqueData,
|
|
607
|
-
|
|
685
|
+
schemaPath: string[],
|
|
608
686
|
name: string,
|
|
609
687
|
transactionOpaqueData?: TransactionOpaqueData
|
|
610
688
|
): IndexInfo | null {
|
|
611
|
-
const all = this.schemaContentsIndexes(attachOpaqueData,
|
|
689
|
+
const all = this.schemaContentsIndexes(attachOpaqueData, schemaPath, transactionOpaqueData);
|
|
612
690
|
return all.find((i) => i.name === name) ?? null;
|
|
613
691
|
}
|
|
614
692
|
|
|
615
693
|
override macroGet(
|
|
616
694
|
attachOpaqueData: AttachOpaqueData,
|
|
617
|
-
|
|
695
|
+
schemaPath: string[],
|
|
618
696
|
name: string,
|
|
619
697
|
transactionOpaqueData?: TransactionOpaqueData
|
|
620
698
|
): MacroInfo | null {
|
|
621
|
-
const macros = this.schemaContentsMacros(attachOpaqueData,
|
|
622
|
-
const tableMacros = this.schemaContentsMacros(attachOpaqueData,
|
|
699
|
+
const macros = this.schemaContentsMacros(attachOpaqueData, schemaPath, "SCALAR_MACRO", transactionOpaqueData);
|
|
700
|
+
const tableMacros = this.schemaContentsMacros(attachOpaqueData, schemaPath, "TABLE_MACRO", transactionOpaqueData);
|
|
623
701
|
const all = [...macros, ...tableMacros];
|
|
624
702
|
return all.find((m) => m.name.toLowerCase() === name.toLowerCase()) ?? null;
|
|
625
703
|
}
|
|
626
704
|
|
|
627
705
|
override tableScanFunctionGet(
|
|
628
706
|
attachOpaqueData: AttachOpaqueData,
|
|
629
|
-
|
|
707
|
+
schemaPath: string[],
|
|
630
708
|
name: string,
|
|
631
709
|
atUnit?: string,
|
|
632
710
|
atValue?: string,
|
|
@@ -635,18 +713,18 @@ export class ReadOnlyCatalogInterface extends CatalogInterface {
|
|
|
635
713
|
validateAtParams(atUnit, atValue);
|
|
636
714
|
|
|
637
715
|
// Find the table descriptor
|
|
638
|
-
const schema = this._descriptor.schemas.find((s) => s
|
|
716
|
+
const schema = this._descriptor.schemas.find((s) => schemaPathsEqual(schemaDescriptorPath(s), schemaPath));
|
|
639
717
|
if (!schema || !schema.tables) {
|
|
640
|
-
throw new Error(`Table '${name}' not found in schema
|
|
718
|
+
throw new Error(`Table '${name}' not found in schema ${schemaPathDisplay(schemaPath)}`);
|
|
641
719
|
}
|
|
642
720
|
const table = schema.tables.find((t) => t.name === name);
|
|
643
721
|
if (!table) {
|
|
644
|
-
throw new Error(`Table '${name}' not found in schema
|
|
722
|
+
throw new Error(`Table '${name}' not found in schema ${schemaPathDisplay(schemaPath)}`);
|
|
645
723
|
}
|
|
646
724
|
|
|
647
725
|
// Reject AT clause on tables that don't support time travel
|
|
648
726
|
if (atUnit && !table.supportsTimeTravel) {
|
|
649
|
-
throw new Error(`Table
|
|
727
|
+
throw new Error(`Table ${schemaPathDisplay(schemaPath)}.${name} does not support time travel queries`);
|
|
650
728
|
}
|
|
651
729
|
|
|
652
730
|
if (table.function) {
|
|
@@ -658,6 +736,7 @@ export class ReadOnlyCatalogInterface extends CatalogInterface {
|
|
|
658
736
|
function_name: table.function.meta.name,
|
|
659
737
|
arguments: argBytes,
|
|
660
738
|
required_extensions: [],
|
|
739
|
+
schema_path: this._schemaForFunction(table.function),
|
|
661
740
|
};
|
|
662
741
|
}
|
|
663
742
|
|
|
@@ -666,7 +745,7 @@ export class ReadOnlyCatalogInterface extends CatalogInterface {
|
|
|
666
745
|
|
|
667
746
|
override async tableGet(
|
|
668
747
|
attachOpaqueData: AttachOpaqueData,
|
|
669
|
-
|
|
748
|
+
schemaPath: string[],
|
|
670
749
|
name: string,
|
|
671
750
|
atUnit?: string,
|
|
672
751
|
atValue?: string,
|
|
@@ -675,25 +754,25 @@ export class ReadOnlyCatalogInterface extends CatalogInterface {
|
|
|
675
754
|
validateAtParams(atUnit, atValue);
|
|
676
755
|
|
|
677
756
|
// Find the table descriptor to check supports_time_travel
|
|
678
|
-
const schema = this._descriptor.schemas.find((s) => s
|
|
757
|
+
const schema = this._descriptor.schemas.find((s) => schemaPathsEqual(schemaDescriptorPath(s), schemaPath));
|
|
679
758
|
if (schema?.tables) {
|
|
680
759
|
const tableDesc = schema.tables.find((t) => t.name === name);
|
|
681
760
|
if (tableDesc && atUnit && !tableDesc.supportsTimeTravel) {
|
|
682
|
-
throw new Error(`Table
|
|
761
|
+
throw new Error(`Table ${schemaPathDisplay(schemaPath)}.${name} does not support time travel queries`);
|
|
683
762
|
}
|
|
684
763
|
}
|
|
685
764
|
|
|
686
|
-
const tables = await this.schemaContentsTables(attachOpaqueData,
|
|
765
|
+
const tables = await this.schemaContentsTables(attachOpaqueData, schemaPath, transactionOpaqueData);
|
|
687
766
|
return tables.find((t) => t.name === name) ?? null;
|
|
688
767
|
}
|
|
689
768
|
|
|
690
769
|
override viewGet(
|
|
691
770
|
attachOpaqueData: AttachOpaqueData,
|
|
692
|
-
|
|
771
|
+
schemaPath: string[],
|
|
693
772
|
name: string,
|
|
694
773
|
transactionOpaqueData?: TransactionOpaqueData
|
|
695
774
|
): ViewInfo | null {
|
|
696
|
-
const views = this.schemaContentsViews(attachOpaqueData,
|
|
775
|
+
const views = this.schemaContentsViews(attachOpaqueData, schemaPath, transactionOpaqueData);
|
|
697
776
|
return views.find((v) => v.name === name) ?? null;
|
|
698
777
|
}
|
|
699
778
|
|
|
@@ -769,14 +848,7 @@ function resolveIndexGroups(schema: VgiSchema, groups?: string[][]): number[][]
|
|
|
769
848
|
return groups.map((group) => resolveIndices(schema, group));
|
|
770
849
|
}
|
|
771
850
|
|
|
772
|
-
|
|
773
|
-
field("fk_columns", list(field("item", utf8(), true)), false),
|
|
774
|
-
field("pk_columns", list(field("item", utf8(), true)), false),
|
|
775
|
-
field("referenced_table", utf8(), false),
|
|
776
|
-
field("referenced_schema", utf8(), false),
|
|
777
|
-
]);
|
|
778
|
-
|
|
779
|
-
function serializeForeignKeys(fks?: ForeignKeyDef[], schemaName?: string): Uint8Array[] {
|
|
851
|
+
function serializeForeignKeys(fks?: ForeignKeyDef[], schemaPath?: string[]): Uint8Array[] {
|
|
780
852
|
if (!fks || fks.length === 0) return [];
|
|
781
853
|
return fks.map((fk) =>
|
|
782
854
|
serializeBatch(
|
|
@@ -785,9 +857,9 @@ function serializeForeignKeys(fks?: ForeignKeyDef[], schemaName?: string): Uint8
|
|
|
785
857
|
fk_columns: [fk.columns],
|
|
786
858
|
pk_columns: [fk.referencedColumns],
|
|
787
859
|
referenced_table: [fk.referencedTable],
|
|
788
|
-
|
|
860
|
+
referenced_schema_path: [fk.referencedSchemaPath ?? schemaPath ?? ["main"]],
|
|
789
861
|
},
|
|
790
|
-
|
|
862
|
+
ForeignKeyInfoSchema
|
|
791
863
|
)
|
|
792
864
|
)
|
|
793
865
|
);
|
|
@@ -984,31 +1056,16 @@ function inferScalarType(val: any): VgiDataType {
|
|
|
984
1056
|
|
|
985
1057
|
// Build the inlined `scan_function` payload for a function-backed table.
|
|
986
1058
|
// Mirrors python's _inline_function_result: a serialized one-row batch of
|
|
987
|
-
// ScanFunctionResultSchema (function_name, arguments, required_extensions
|
|
988
|
-
// The C++ extension uses these bytes verbatim and skips the
|
|
989
|
-
// catalog_table_scan_function_get RPC
|
|
1059
|
+
// ScanFunctionResultSchema (function_name, arguments, required_extensions,
|
|
1060
|
+
// schema_path). The C++ extension uses these bytes verbatim and skips the
|
|
1061
|
+
// catalog_table_scan_function_get RPC, so the schema this inlines has to agree
|
|
1062
|
+
// with what that RPC would have answered.
|
|
990
1063
|
function inlineScanFunction(
|
|
991
1064
|
func: VgiFunction,
|
|
992
1065
|
args: Arguments | undefined,
|
|
1066
|
+
schemaPath: string[] | null,
|
|
993
1067
|
): Uint8Array {
|
|
994
1068
|
const argSchema = argumentSpecsToSchema(func.argumentSpecs);
|
|
995
1069
|
const argBytes = serializeArgsBatch(args ?? new Arguments(), argSchema);
|
|
996
|
-
|
|
997
|
-
field("function_name", utf8(), false),
|
|
998
|
-
field("arguments", binary(), false),
|
|
999
|
-
field(
|
|
1000
|
-
"required_extensions",
|
|
1001
|
-
list(field("item", utf8(), true)),
|
|
1002
|
-
false,
|
|
1003
|
-
),
|
|
1004
|
-
]);
|
|
1005
|
-
const batch = batchFromColumns(
|
|
1006
|
-
{
|
|
1007
|
-
function_name: [func.meta.name],
|
|
1008
|
-
arguments: [argBytes],
|
|
1009
|
-
required_extensions: [[] as string[]],
|
|
1010
|
-
},
|
|
1011
|
-
scanSchema,
|
|
1012
|
-
);
|
|
1013
|
-
return serializeBatch(batch);
|
|
1070
|
+
return encodeScanFunctionResult(func.meta.name, argBytes, [], schemaPath);
|
|
1014
1071
|
}
|