@query-farm/vgi 0.28.0 → 0.29.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (121) hide show
  1. package/README.md +52 -8
  2. package/dist/arguments/argument-spec.d.ts +57 -2
  3. package/dist/arrow/impl-arrowjs/index.d.ts +1 -1
  4. package/dist/arrow/impl-arrowjs/iterate.d.ts +10 -1
  5. package/dist/arrow/impl-arrowjs/schema.d.ts +2 -0
  6. package/dist/arrow/impl-flechette/index.d.ts +1 -1
  7. package/dist/arrow/impl-flechette/iterate.d.ts +10 -1
  8. package/dist/arrow/impl-flechette/schema.d.ts +2 -0
  9. package/dist/arrow/index.d.ts +1 -1
  10. package/dist/arrow/schema-types.d.ts +3 -1
  11. package/dist/arrow/types.d.ts +39 -3
  12. package/dist/catalog/composite.d.ts +13 -13
  13. package/dist/catalog/descriptors.d.ts +7 -3
  14. package/dist/catalog/interface.d.ts +108 -38
  15. package/dist/catalog/read-only.d.ts +32 -15
  16. package/dist/client/client.d.ts +37 -30
  17. package/dist/client/types.d.ts +2 -0
  18. package/dist/client-entry.d.ts +1 -1
  19. package/dist/client-entry.js +791 -447
  20. package/dist/client-entry.js.map +18 -17
  21. package/dist/filter-pushdown/capabilities.d.ts +3 -0
  22. package/dist/filter-pushdown/deserialize.d.ts +14 -15
  23. package/dist/filter-pushdown/evaluate.d.ts +14 -44
  24. package/dist/filter-pushdown/index.d.ts +2 -2
  25. package/dist/filter-pushdown/types.d.ts +120 -68
  26. package/dist/functions/aggregate.d.ts +4 -0
  27. package/dist/functions/registry.d.ts +5 -5
  28. package/dist/functions/scalar.d.ts +6 -1
  29. package/dist/functions/table-buffering.d.ts +9 -0
  30. package/dist/functions/table-in-out.d.ts +13 -0
  31. package/dist/functions/table.d.ts +72 -2
  32. package/dist/functions/types.d.ts +60 -2
  33. package/dist/generated/vgi-client.d.ts +114 -26
  34. package/dist/generated/vgi-protocol-schemas.d.ts +34 -0
  35. package/dist/http/fetch.d.ts +16 -0
  36. package/dist/index.core.d.ts +4 -3
  37. package/dist/index.js +3396 -2064
  38. package/dist/index.js.map +55 -49
  39. package/dist/metadata/types.d.ts +15 -2
  40. package/dist/protocol/handlers/catalog/shared.d.ts +0 -8
  41. package/dist/protocol/handlers/function.d.ts +11 -0
  42. package/dist/protocol/handlers/shared.d.ts +3 -1
  43. package/dist/protocol/serializers/bind.d.ts +1 -2
  44. package/dist/protocol/serializers/cardinality.d.ts +1 -2
  45. package/dist/protocol/serializers/splits.d.ts +126 -0
  46. package/dist/protocol/types.d.ts +36 -3
  47. package/dist/request-auth.d.ts +28 -0
  48. package/dist/schema-path.d.ts +7 -0
  49. package/dist/serve-entry.d.ts +25 -0
  50. package/dist/serve-entry.js +1129 -569
  51. package/dist/serve-entry.js.map +36 -32
  52. package/dist/split-token.d.ts +88 -0
  53. package/dist/types.d.ts +8 -0
  54. package/dist/worker-cf-entry.js +2928 -1689
  55. package/dist/worker-cf-entry.js.map +51 -44
  56. package/dist/worker.d.ts +1 -1
  57. package/package.json +3 -3
  58. package/src/arguments/argument-spec.ts +93 -4
  59. package/src/arguments/arguments.ts +0 -1
  60. package/src/arrow/impl-arrowjs/index.ts +1 -1
  61. package/src/arrow/impl-arrowjs/iterate.ts +21 -0
  62. package/src/arrow/impl-arrowjs/schema.ts +4 -0
  63. package/src/arrow/impl-flechette/index.ts +1 -1
  64. package/src/arrow/impl-flechette/iterate.ts +24 -1
  65. package/src/arrow/impl-flechette/schema.ts +4 -0
  66. package/src/arrow/index.ts +1 -1
  67. package/src/arrow/schema-types.ts +4 -0
  68. package/src/arrow/types.ts +39 -3
  69. package/src/catalog/composite.ts +26 -26
  70. package/src/catalog/descriptors.ts +12 -3
  71. package/src/catalog/interface.ts +217 -72
  72. package/src/catalog/read-only.ts +141 -84
  73. package/src/client/client.ts +97 -73
  74. package/src/client/types.ts +2 -0
  75. package/src/client-entry.ts +2 -1
  76. package/src/filter-pushdown/capabilities.ts +13 -0
  77. package/src/filter-pushdown/collector.ts +22 -35
  78. package/src/filter-pushdown/deserialize.ts +657 -221
  79. package/src/filter-pushdown/evaluate.ts +410 -499
  80. package/src/filter-pushdown/index.ts +16 -2
  81. package/src/filter-pushdown/types.ts +88 -71
  82. package/src/functions/aggregate.ts +12 -1
  83. package/src/functions/copy-from.ts +2 -1
  84. package/src/functions/copy-to.ts +2 -1
  85. package/src/functions/registry.ts +24 -22
  86. package/src/functions/scalar.ts +37 -0
  87. package/src/functions/table-buffering.ts +48 -6
  88. package/src/functions/table-in-out.ts +194 -33
  89. package/src/functions/table.ts +187 -22
  90. package/src/functions/types.ts +64 -1
  91. package/src/generated/vgi-client.ts +122 -28
  92. package/src/generated/vgi-protocol-schemas.ts +455 -58
  93. package/src/http/fetch.ts +129 -2
  94. package/src/index.core.ts +4 -2
  95. package/src/metadata/resolve.ts +43 -1
  96. package/src/metadata/serialize.ts +43 -5
  97. package/src/metadata/types.ts +17 -1
  98. package/src/protocol/dispatch.ts +1 -1
  99. package/src/protocol/handlers/aggregate.ts +23 -12
  100. package/src/protocol/handlers/catalog/admin.ts +47 -54
  101. package/src/protocol/handlers/catalog/index_methods.ts +6 -13
  102. package/src/protocol/handlers/catalog/macro.ts +12 -20
  103. package/src/protocol/handlers/catalog/shared.ts +12 -57
  104. package/src/protocol/handlers/catalog/table.ts +49 -112
  105. package/src/protocol/handlers/catalog/view.ts +18 -22
  106. package/src/protocol/handlers/function.ts +311 -18
  107. package/src/protocol/handlers/shared.ts +12 -3
  108. package/src/protocol/handlers/table-buffering.ts +11 -8
  109. package/src/protocol/serializers/bind.ts +69 -67
  110. package/src/protocol/serializers/cardinality.ts +7 -7
  111. package/src/protocol/serializers/init.ts +52 -10
  112. package/src/protocol/serializers/splits.ts +221 -0
  113. package/src/protocol/state-serializer.ts +55 -1
  114. package/src/protocol/types.ts +36 -3
  115. package/src/request-auth.ts +73 -0
  116. package/src/schema-path.ts +25 -0
  117. package/src/serve-entry.ts +69 -2
  118. package/src/split-token.ts +297 -0
  119. package/src/types.ts +9 -0
  120. package/src/worker-cf-entry.ts +2 -1
  121. package/src/worker.ts +79 -14
@@ -4,7 +4,7 @@
4
4
  // implements VgiFunction.bind/globalInit/createStreamHandlers.
5
5
 
6
6
  import { type VgiSchema, schema, type VgiField, field, type VgiDataType, binary, int64 } from "../../arrow/index.js";
7
- import { Protocol } from "@query-farm/vgi-rpc";
7
+ import { Protocol, type AuthContext } from "@query-farm/vgi-rpc";
8
8
  import type { FunctionRegistry } from "../../functions/registry.js";
9
9
  import type { StreamHandlers, HandlerState } from "../../functions/types.js";
10
10
  import {
@@ -19,7 +19,17 @@ import type { GlobalInitResponse } from "../types.js";
19
19
  import { batchToScalarDict, deserializeBatch, adoptArrowJsShape } from "../../util/arrow/index.js";
20
20
  import { toUint8Array } from "../../util/bytes.js";
21
21
  import { serializeColumnStatistics } from "../../util/statistics.js";
22
- import { BindResultSchema, TableFunctionCardinalityResultSchema, TableFunctionDynamicToStringResultSchema } from "../../generated/vgi-protocol-schemas.js";
22
+ import {
23
+ BindParamsSchema,
24
+ BindResultSchema,
25
+ InitParamsSchema,
26
+ TableFunctionCardinalityParamsSchema,
27
+ TableFunctionCardinalityResultSchema,
28
+ TableFunctionDynamicToStringParamsSchema,
29
+ TableFunctionDynamicToStringResultSchema,
30
+ TableFunctionPlanParamsSchema,
31
+ TableFunctionStatisticsParamsSchema,
32
+ } from "../../generated/vgi-protocol-schemas.js";
23
33
  import {
24
34
  REQUEST_PARAMS_SCHEMA,
25
35
  RESULT_BINARY_SCHEMA,
@@ -29,7 +39,25 @@ import {
29
39
  overloadContext,
30
40
  recoverFinalizeState,
31
41
  } from "./shared.js";
42
+ import { GLOBAL_INIT_RESPONSE_SCHEMA } from "../serializers/init.js";
32
43
  import { openAttach } from "./catalog/shared.js";
44
+ import { currentRequestAuth } from "../../request-auth.js";
45
+ import { batchFromColumns, serializeBatch } from "../../util/arrow/index.js";
46
+ import {
47
+ deserializePlanRequest,
48
+ fingerprintInputs,
49
+ planResponseRow,
50
+ scanSplitRow,
51
+ PLAN_RESPONSE_SCHEMA,
52
+ SCAN_SPLIT_SCHEMA,
53
+ type PlanResult,
54
+ } from "../serializers/splits.js";
55
+ import {
56
+ bindFingerprint,
57
+ buildSplitToken,
58
+ openSplitToken,
59
+ splitAnchor,
60
+ } from "../../split-token.js";
33
61
 
34
62
  export interface FunctionHandlerConfig {
35
63
  registry: FunctionRegistry;
@@ -40,7 +68,134 @@ export interface FunctionHandlerConfig {
40
68
  * be scoped when two catalogs declare the same schema and function name.
41
69
  * Optional — without it, resolution falls back to (schema, name).
42
70
  */
43
- catalogInterface?: { catalogNameForAttach(a: Uint8Array): string | null };
71
+ catalogInterface?: {
72
+ catalogNameForAttach(a: Uint8Array): string | null;
73
+ /**
74
+ * The catalog's current version, which is the consistency anchor every split
75
+ * token is stamped with AND checked against.
76
+ *
77
+ * Both sides read it from here on purpose. Minting from a different value
78
+ * than redemption compares against is not a subtle bug: it refuses every
79
+ * token, and the documented response to SPLIT_SNAPSHOT_EXPIRED is "re-run
80
+ * the query", which re-plans, mints the same mismatch and fails again — a
81
+ * livelock returning no rows, blaming the data for moving when it has not.
82
+ */
83
+ version?(attach: Uint8Array, txn?: Uint8Array): number | Promise<number>;
84
+ };
85
+ }
86
+
87
+ /**
88
+ * The live anchor a split token must still name, or undefined when this worker's
89
+ * catalog reports no version.
90
+ *
91
+ * Undefined means "skip the staleness check" rather than "fail" — a check that
92
+ * cannot be made must not become a check that always fails, which would refuse
93
+ * every split token on every unversioned catalog.
94
+ */
95
+ async function liveSplitAnchor(
96
+ catalogInterface: FunctionHandlerConfig["catalogInterface"],
97
+ bindCall: { attach_opaque_data?: Uint8Array | null; transaction_opaque_data?: Uint8Array | null },
98
+ auth: AuthContext | undefined,
99
+ signingKey: Uint8Array | undefined,
100
+ ): Promise<Uint8Array | undefined> {
101
+ if (!catalogInterface?.version || bindCall.attach_opaque_data == null) return undefined;
102
+ const raw = toUint8Array(bindCall.attach_opaque_data);
103
+ const txn = bindCall.transaction_opaque_data
104
+ ? toUint8Array(bindCall.transaction_opaque_data)
105
+ : undefined;
106
+
107
+ // The two call sites see the attach envelope at DIFFERENT processing stages —
108
+ // `plan` receives it with the framework's 16-byte UUID prefix still attached,
109
+ // `init` receives it already stripped — so neither form alone works for both.
110
+ // Try each. Getting this wrong is not a degraded check but a total one: if
111
+ // mint resolves a version and redemption does not (or vice versa) then every
112
+ // token fails SPLIT_SNAPSHOT_EXPIRED, and the documented response to that is
113
+ // "re-run the query", which re-plans and reproduces the same mismatch.
114
+ // Each form is produced LAZILY, inside the try. Built as an array literal —
115
+ // `[raw, await openAttach(...)]` — the unseal is evaluated before the loop
116
+ // begins, so it throws from OUTSIDE the try and escapes the function instead
117
+ // of falling through to the next candidate. `init` hands this function
118
+ // already-stripped plaintext, which openAttach rejects, so the first form
119
+ // never got a turn: every split scan died with OpaqueDataRejectedError, on
120
+ // HTTP only (sealing is a pass-through when there is no signing key, which is
121
+ // every non-HTTP transport).
122
+ const forms: Array<() => Promise<Uint8Array>> = [
123
+ async () => raw,
124
+ async () => openAttach(raw, auth, signingKey),
125
+ ];
126
+ for (const form of forms) {
127
+ try {
128
+ return splitAnchor(await catalogInterface.version(await form(), txn));
129
+ } catch {
130
+ // try the next form
131
+ }
132
+ }
133
+ // A catalog that cannot report a version means "skip the staleness check",
134
+ // never "fail it" — a check that cannot be made must not become a check that
135
+ // always fails, which would refuse every split token on every unversioned
136
+ // catalog.
137
+ return undefined;
138
+ }
139
+
140
+ /**
141
+ * Open this request's split envelopes into `request.split_payloads`.
142
+ *
143
+ * `split_payloads` is DERIVED, not wire state: the tokens ride the request, the
144
+ * payloads are what falls out of opening them. Only this layer holds the
145
+ * signing key — a function does not — so redemption happens here and only the
146
+ * payloads are handed down.
147
+ *
148
+ * It must run on continuation turns too. The HTTP exchange path rebuilds the
149
+ * *wire* request from the cursor, which carries `split_tokens` but of course no
150
+ * derived field, so a split-capable function used to see `splitPayloads:
151
+ * undefined` on every turn after the first and could not tell that from "the
152
+ * planner gave me no splits". Over HTTP that made every split scan fail on its
153
+ * second turn; subprocess never noticed because it holds the handlers in memory
154
+ * and never rebuilds anything.
155
+ *
156
+ * `checkAnchor` is false on a continuation, deliberately. The consistency
157
+ * anchor answers "is this plan still redeemable", which is a question about
158
+ * REDEMPTION — asked once, when the split is admitted. Re-asking it mid-stream
159
+ * would let a catalog version bump kill an in-flight scan that had already been
160
+ * accepted, turning a successful query into SPLIT_SNAPSHOT_EXPIRED halfway
161
+ * through its rows.
162
+ */
163
+ async function resolveSplitPayloads(
164
+ request: any,
165
+ innerParams: any,
166
+ opts: {
167
+ auth: any;
168
+ signingKey: Uint8Array | undefined;
169
+ catalogInterface: any;
170
+ checkAnchor: boolean;
171
+ },
172
+ ): Promise<void> {
173
+ if (!request.split_tokens) return;
174
+ const fp = fingerprintInputs(toUint8Array(innerParams.bind_call));
175
+ const expected = await bindFingerprint(
176
+ fp.schemaPath,
177
+ fp.functionName,
178
+ fp.args,
179
+ fp.settings,
180
+ new Uint8Array(0),
181
+ );
182
+ const currentAnchor = opts.checkAnchor
183
+ ? await liveSplitAnchor(opts.catalogInterface, request.bind_call as any, opts.auth, opts.signingKey)
184
+ : undefined;
185
+ request.split_payloads = [];
186
+ for (const token of request.split_tokens) {
187
+ request.split_payloads.push(
188
+ await openSplitToken(token, {
189
+ signingKey: opts.signingKey,
190
+ // Same principal `table_function_plan` sealed under. Omitting it opens
191
+ // under the anonymous tail, so an authenticated caller's own tokens
192
+ // fail authentication.
193
+ auth: opts.auth,
194
+ expectedFingerprint: expected,
195
+ currentAnchor,
196
+ }),
197
+ );
198
+ }
44
199
  }
45
200
 
46
201
  export function registerFunctionMethods(protocol: Protocol, config: FunctionHandlerConfig): void {
@@ -50,11 +205,17 @@ export function registerFunctionMethods(protocol: Protocol, config: FunctionHand
50
205
  // HTTP, plaintext on subprocess). Function bodies — like catalog bodies —
51
206
  // must see the catalog's own bytes, so unseal (when keyed) and strip the
52
207
  // framework UUID prefix before the user function reads attach_opaque_data.
208
+ // `ctx` is the CallContext when the dispatcher supplies one (every unary
209
+ // method) and absent otherwise (stream init); `currentRequestAuth()` is the
210
+ // ambient per-request fallback the HTTP entry point publishes, so both reach
211
+ // the same principal. Getting an authenticated caller's identity wrong here
212
+ // is not a degraded lookup but a hard `OpaqueDataRejectedError`: the envelope
213
+ // is sealed under `attachAad(auth)`.
53
214
  async function stripAttach(attach: any, ctx: any): Promise<Uint8Array | null> {
54
215
  if (attach == null) return null;
55
216
  const env = toUint8Array(attach);
56
217
  if (env.length === 0) return env;
57
- return openAttach(env, ctx?.auth, signingKey);
218
+ return openAttach(env, ctx?.auth ?? currentRequestAuth(), signingKey);
58
219
  }
59
220
 
60
221
  // Build an overload context whose attach has been unsealed, so catalog
@@ -72,11 +233,9 @@ export function registerFunctionMethods(protocol: Protocol, config: FunctionHand
72
233
  return overloadContext({ ...bindCall, attach_opaque_data: attach }, catalogInterface);
73
234
  }
74
235
 
75
- const initHeaderSchema = schema([
76
- field("execution_id", binary(), false),
77
- field("opaque_data", binary(), true),
78
- field("max_workers", int64(), false),
79
- ]);
236
+ // Shared with the serializer that actually emits the header, so the
237
+ // advertised shape and the emitted shape cannot drift apart.
238
+ const initHeaderSchema = GLOBAL_INIT_RESPONSE_SCHEMA;
80
239
 
81
240
  const emptySchema = schema([]);
82
241
 
@@ -92,7 +251,7 @@ export function registerFunctionMethods(protocol: Protocol, config: FunctionHand
92
251
  // bind (unary)
93
252
  // --------------------------------------------------------------------------
94
253
  protocol.unary("bind", {
95
- params: REQUEST_PARAMS_SCHEMA,
254
+ params: BindParamsSchema,
96
255
  result: RESULT_BINARY_SCHEMA,
97
256
  handler: async (params, ctx) => {
98
257
  const innerParams = unwrapRequest(params.request);
@@ -109,17 +268,46 @@ export function registerFunctionMethods(protocol: Protocol, config: FunctionHand
109
268
  // init (streaming) - dynamically produces either producer or exchange streams
110
269
  // --------------------------------------------------------------------------
111
270
  protocol.exchange("init", {
112
- params: REQUEST_PARAMS_SCHEMA,
271
+ params: InitParamsSchema,
113
272
  inputSchema: dummyInputSchema,
114
273
  outputSchema: emptySchema,
115
- init: async (params) => {
274
+ init: async (params, ctx?: any) => {
116
275
  // Preserve raw request IPC bytes for exchange reconstruction
117
276
  const requestIpcBytes = toUint8Array(params.request);
118
277
  const innerParams = unwrapRequest(params.request);
119
278
  const request = deserializeInitRequest(innerParams);
120
- // Exchange init has no ctx; on subprocess (no signing key) openAttach is a
121
- // pass-through strip that needs no auth. HTTP carries auth via the bind path.
122
- request.bind_call.attach_opaque_data = await stripAttach(request.bind_call.attach_opaque_data, undefined);
279
+ // The caller's identity, and the load-bearing line of this handler.
280
+ //
281
+ // vgi-rpc types stream init as `(params) => state`, so unlike every unary
282
+ // method there is no CallContext to read `auth` off — `ctx` is only ever
283
+ // populated by a caller that has one (the in-process client). The HTTP
284
+ // entry point therefore publishes the dispatching principal on an ambient
285
+ // per-request scope, the same shape vgi-python's `current_auth()`
286
+ // ContextVar and vgi-go's `handleInit(callCtx)` give those ports.
287
+ //
288
+ // It has to be the REAL principal, not `undefined`: `attach_opaque_data`
289
+ // is sealed by `catalog_attach` under `attachAad(auth)` and every split
290
+ // token by `table_function_plan` under `splitTokenAad(body, auth)`, both
291
+ // of which are unary and so seal under the authenticated caller. Opening
292
+ // them here under the anonymous tail matches only for an anonymous
293
+ // caller; an authenticated one got `OpaqueDataRejectedError:
294
+ // attach_opaque_data not recognized` on its first scan, i.e. this worker
295
+ // could not serve authenticated traffic at all.
296
+ const auth = ctx?.auth ?? currentRequestAuth();
297
+ request.bind_call.attach_opaque_data = await stripAttach(
298
+ request.bind_call.attach_opaque_data,
299
+ ctx,
300
+ );
301
+
302
+ // Verify and strip the split envelopes BEFORE any user code runs, so an
303
+ // unverified token can never be acted on.
304
+ await resolveSplitPayloads(request, innerParams, {
305
+ auth,
306
+ signingKey,
307
+ catalogInterface,
308
+ checkAnchor: true,
309
+ });
310
+
123
311
  const func = registry.get(request.bind_call.function_name, overloadContext(request.bind_call, catalogInterface));
124
312
 
125
313
  // globalInit is async — table function onInit may touch HTTP-backed
@@ -199,6 +387,28 @@ export function registerFunctionMethods(protocol: Protocol, config: FunctionHand
199
387
  const initRequestBatch = deserializeBatch(state.initRequestIpc);
200
388
  const initRequestDict = batchToScalarDict(initRequestBatch);
201
389
  const request = deserializeInitRequest(initRequestDict);
390
+ // Re-derive the split payloads. The cursor carries the wire request,
391
+ // which has `split_tokens` but not the opened `split_payloads` — that
392
+ // field only ever existed in memory on the init turn. Without this a
393
+ // split-capable function sees `splitPayloads: undefined` from the
394
+ // second turn onward, indistinguishable from "the planner gave me no
395
+ // splits at all". checkAnchor is false: redemption was already decided
396
+ // on the init turn, and re-checking here would let a mid-scan catalog
397
+ // bump fail a query that had already been admitted.
398
+ // fingerprintInputs hashes the RAW bind_call bytes, so pass the
399
+ // pre-deserialization blob from the cursor — handing it the decoded
400
+ // object hashes a different shape and every token fails as
401
+ // SPLIT_TOKEN_INVALID "minted for a different bind".
402
+ await resolveSplitPayloads(request, { bind_call: initRequestDict.bind_call }, {
403
+ // Same principal the tokens were sealed under. The exchange dispatch
404
+ // gets no ctx (see stripAttach above), so the request-scoped identity
405
+ // is the only source — opening under the anonymous tail would fail
406
+ // authentication for every authenticated caller's own tokens.
407
+ auth: currentRequestAuth(),
408
+ signingKey,
409
+ catalogInterface,
410
+ checkAnchor: false,
411
+ });
202
412
  const func = registry.get(state.functionName, await strippedContext(request.bind_call));
203
413
  const executionId = state.executionId;
204
414
  const opaqueData = state.opaqueData ?? null;
@@ -259,7 +469,7 @@ export function registerFunctionMethods(protocol: Protocol, config: FunctionHand
259
469
  // table_function_cardinality (unary)
260
470
  // --------------------------------------------------------------------------
261
471
  protocol.unary("table_function_cardinality", {
262
- params: REQUEST_PARAMS_SCHEMA,
472
+ params: TableFunctionCardinalityParamsSchema,
263
473
  result: RESULT_BINARY_SCHEMA,
264
474
  handler: async (params) => {
265
475
  const innerParams = unwrapRequest(params.request);
@@ -275,6 +485,89 @@ export function registerFunctionMethods(protocol: Protocol, config: FunctionHand
275
485
  },
276
486
  });
277
487
 
488
+ // --------------------------------------------------------------------------
489
+ // table_function_plan (unary)
490
+ // --------------------------------------------------------------------------
491
+ // Divide a scan into named, independently redeemable splits, so a distributed
492
+ // engine can retry a task without re-reading or skipping rows.
493
+ //
494
+ // A function that declares no `plan` hook gets the framework default: a SINGLE
495
+ // empty-payload split, which is what "not split-capable" means — the whole scan
496
+ // is one unit of work. That keeps every existing worker serving unchanged under
497
+ // protocol 1.4.0, so splits stay opt-in rather than something every worker must
498
+ // now implement.
499
+ protocol.unary("table_function_plan", {
500
+ params: TableFunctionPlanParamsSchema,
501
+ result: RESULT_BINARY_SCHEMA,
502
+ handler: async (params, ctx?: any) => {
503
+ const innerParams = unwrapRequest(params.request);
504
+ const request = deserializePlanRequest(innerParams);
505
+ const func = registry.get(
506
+ request.bind_call.function_name,
507
+ await strippedContext(request.bind_call),
508
+ );
509
+
510
+ const plan: PlanResult = func.plan
511
+ ? await func.plan(request)
512
+ : { splits: [{ payload: new Uint8Array(0) }] };
513
+
514
+ // The framework stamps every token: an author cannot forget the
515
+ // consistency anchor, cannot mis-bind the fingerprint, and never writes
516
+ // crypto — and the envelope stays a private implementation detail whose
517
+ // layout can change without touching worker code in five languages.
518
+ const fp = fingerprintInputs(toUint8Array(innerParams.bind_call));
519
+ const fingerprint = await bindFingerprint(
520
+ fp.schemaPath,
521
+ fp.functionName,
522
+ fp.args,
523
+ fp.settings,
524
+ // projection_ids is not a bind-call field (it rides the init request),
525
+ // so it feeds in empty — matching the reference implementation, which
526
+ // reads it off the bind call and likewise finds nothing.
527
+ new Uint8Array(0),
528
+ );
529
+ // A worker that names its version is taken at its word — it knows which
530
+ // snapshot it planned against. One that leaves it unset gets the LIVE
531
+ // version, never 0: minting 0 while redemption compares against the live
532
+ // counter refuses every token, and is invisible on a catalog whose version
533
+ // happens to be 0, which is most fixtures.
534
+ const anchor =
535
+ plan.catalogVersion != null
536
+ ? splitAnchor(plan.catalogVersion)
537
+ : ((await liveSplitAnchor(catalogInterface, request.bind_call as any, ctx?.auth, signingKey)) ??
538
+ splitAnchor(0));
539
+
540
+ const blobs: Uint8Array[] = [];
541
+ for (const split of plan.splits) {
542
+ const token = await buildSplitToken({
543
+ payload: split.payload,
544
+ fingerprint,
545
+ anchor,
546
+ signingKey,
547
+ // Bind the caller. Without it every token seals under the anonymous
548
+ // identity, so one tenant's splits are replayable by another — the
549
+ // exact replay the AAD exists to stop.
550
+ auth: ctx?.auth,
551
+ });
552
+ blobs.push(
553
+ serializeBatch(
554
+ batchFromColumns(
555
+ Object.fromEntries(
556
+ SCAN_SPLIT_SCHEMA.fields.map((f) => [
557
+ f.name,
558
+ [scanSplitRow(split, token)[f.name] ?? null],
559
+ ]),
560
+ ),
561
+ SCAN_SPLIT_SCHEMA,
562
+ ),
563
+ ),
564
+ );
565
+ }
566
+
567
+ return wrapResult(planResponseRow(plan, blobs), PLAN_RESPONSE_SCHEMA);
568
+ },
569
+ });
570
+
278
571
  // --------------------------------------------------------------------------
279
572
  // table_function_statistics (unary)
280
573
  // --------------------------------------------------------------------------
@@ -283,7 +576,7 @@ export function registerFunctionMethods(protocol: Protocol, config: FunctionHand
283
576
  // list, else null. DuckDB uses the bounds for plan-time filter elimination
284
577
  // (folds impossible filters to EMPTY_RESULT).
285
578
  protocol.unary("table_function_statistics", {
286
- params: REQUEST_PARAMS_SCHEMA,
579
+ params: TableFunctionStatisticsParamsSchema,
287
580
  result: RESULT_BINARY_NULLABLE_SCHEMA,
288
581
  handler: async (params) => {
289
582
  const innerParams = unwrapRequest(params.request);
@@ -306,7 +599,7 @@ export function registerFunctionMethods(protocol: Protocol, config: FunctionHand
306
599
  // declare dynamicToString, return empty maps so the C++ side falls back
307
600
  // to intrinsics only.
308
601
  protocol.unary("table_function_dynamic_to_string", {
309
- params: REQUEST_PARAMS_SCHEMA,
602
+ params: TableFunctionDynamicToStringParamsSchema,
310
603
  result: RESULT_BINARY_SCHEMA,
311
604
  handler: async (params) => {
312
605
  const innerParams = unwrapRequest(params.request);
@@ -7,6 +7,7 @@ import { batchToScalarDict, deserializeBatch, serializeBatch, batchFromColumns }
7
7
  import { toUint8Array } from "../../util/bytes.js";
8
8
  import { TableInOutPhase } from "../../types.js";
9
9
  import type { InitRequest } from "../types.js";
10
+ import { normalizeSchemaPath } from "../../schema-path.js";
10
11
 
11
12
  // The Python vgi-rpc framework wraps ALL non-void unary results in a single
12
13
  // "result" column. For ArrowSerializableDataclass types, the result is serialized
@@ -31,7 +32,15 @@ export const RESULT_BINARY_NULLABLE_SCHEMA = schema([
31
32
  export function unwrapRequest(requestBytes: any): Record<string, any> {
32
33
  const bytes = toUint8Array(requestBytes);
33
34
  const innerBatch = deserializeBatch(bytes);
34
- return batchToScalarDict(innerBatch);
35
+ const params = batchToScalarDict(innerBatch);
36
+ normalizeRequestSchemaPaths(params);
37
+ return params;
38
+ }
39
+
40
+ /** Convert Arrow list scalars at RPC boundaries into ordinary string arrays. */
41
+ export function normalizeRequestSchemaPaths(params: Record<string, any>): void {
42
+ if (params.path != null) params.path = normalizeSchemaPath(params.path);
43
+ if (params.schema_path != null) params.schema_path = normalizeSchemaPath(params.schema_path);
35
44
  }
36
45
 
37
46
  /**
@@ -57,7 +66,7 @@ export function overloadContext(
57
66
  arguments: any;
58
67
  input_schema: any;
59
68
  function_type: any;
60
- schema_name?: string | null;
69
+ schema_path?: string[] | null;
61
70
  attach_opaque_data?: Uint8Array | null;
62
71
  },
63
72
  catalogInterface?: { catalogNameForAttach(a: Uint8Array): string | null },
@@ -78,7 +87,7 @@ export function overloadContext(
78
87
  isScalar: String(req.function_type).toLowerCase() === "scalar",
79
88
  // Scopes resolution when the caller named a schema; a name registered in
80
89
  // several schemas is otherwise ambiguous.
81
- schemaName: req.schema_name ?? null,
90
+ schemaPath: req.schema_path ?? null,
82
91
  catalogName,
83
92
  };
84
93
  }
@@ -15,9 +15,12 @@ import type { FunctionRegistry } from "../../functions/registry.js";
15
15
  import { toUint8Array } from "../../util/bytes.js";
16
16
  import { batchToScalarDict, batchToSecretDict } from "../../util/arrow/index.js";
17
17
  import {
18
- TableBufferingProcessResultSchema,
18
+ TableBufferingCombineParamsSchema,
19
19
  TableBufferingCombineResultSchema,
20
+ TableBufferingDestructorParamsSchema,
20
21
  TableBufferingDestructorResultSchema,
22
+ TableBufferingProcessParamsSchema,
23
+ TableBufferingProcessResultSchema,
21
24
  } from "../../generated/vgi-protocol-schemas.js";
22
25
  import {
23
26
  REQUEST_PARAMS_SCHEMA,
@@ -43,19 +46,19 @@ export function registerTableBufferingMethods(
43
46
  registry: FunctionRegistry,
44
47
  signingKey?: Uint8Array,
45
48
  ): void {
46
- // `schemaName` scopes resolution to the schema the caller named. The
49
+ // `schemaPath` scopes resolution to the schema the caller named. The
47
50
  // buffering process/combine RPCs carry it (protocol 1.2.0), so a name
48
51
  // declared in two schemas resolves to the one the request names rather than
49
52
  // colliding on the flat by-name index. `null` keeps the cross-schema lookup.
50
53
  function resolveBuffering(
51
54
  name: string,
52
- schemaName?: string | null,
55
+ schemaPath?: string[] | null,
53
56
  ): TableBufferingVgiFunction {
54
57
  const func = registry.get(name, {
55
58
  arguments: new Arguments(),
56
59
  inputSchema: null,
57
60
  isScalar: false,
58
- schemaName: schemaName ?? null,
61
+ schemaPath: schemaPath ?? null,
59
62
  }) as any;
60
63
  if (!func || func.kind !== "table_buffering") {
61
64
  throw new Error(`Function '${name}' is not a table_buffering function`);
@@ -83,7 +86,7 @@ export function registerTableBufferingMethods(
83
86
  : rawAttach;
84
87
  const transactionId = inner.transaction_id ? toUint8Array(inner.transaction_id) : null;
85
88
 
86
- const func = resolveBuffering(functionName, inner.schema_name ?? null);
89
+ const func = resolveBuffering(functionName, inner.schema_path ?? null);
87
90
  const bound = new BoundStorage(defaultStorage, executionId);
88
91
  const payload = await bound.stateGet(
89
92
  FrameworkNS.BUFFERING_INIT,
@@ -121,7 +124,7 @@ export function registerTableBufferingMethods(
121
124
  // table_buffering_process — sink one batch, return state_id (unary)
122
125
  // ------------------------------------------------------------------------
123
126
  protocol.unary("table_buffering_process", {
124
- params: REQUEST_PARAMS_SCHEMA,
127
+ params: TableBufferingProcessParamsSchema,
125
128
  result: RESULT_BINARY_SCHEMA,
126
129
  handler: async (rpcParams, ctx?: any) => {
127
130
  const inner = unwrapRequest(rpcParams.request);
@@ -145,7 +148,7 @@ export function registerTableBufferingMethods(
145
148
  // table_buffering_combine — group/merge state_ids (unary)
146
149
  // ------------------------------------------------------------------------
147
150
  protocol.unary("table_buffering_combine", {
148
- params: REQUEST_PARAMS_SCHEMA,
151
+ params: TableBufferingCombineParamsSchema,
149
152
  result: RESULT_BINARY_SCHEMA,
150
153
  handler: async (rpcParams, ctx?: any) => {
151
154
  const inner = unwrapRequest(rpcParams.request);
@@ -172,7 +175,7 @@ export function registerTableBufferingMethods(
172
175
  // table_buffering_destructor — best-effort end-of-query cleanup (unary)
173
176
  // ------------------------------------------------------------------------
174
177
  protocol.unary("table_buffering_destructor", {
175
- params: REQUEST_PARAMS_SCHEMA,
178
+ params: TableBufferingDestructorParamsSchema,
176
179
  result: RESULT_BINARY_SCHEMA,
177
180
  handler: async (rpcParams) => {
178
181
  try {