@query-farm/vgi 0.28.0 → 0.29.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +52 -8
- package/dist/arguments/argument-spec.d.ts +57 -2
- package/dist/arrow/impl-arrowjs/index.d.ts +1 -1
- package/dist/arrow/impl-arrowjs/iterate.d.ts +10 -1
- package/dist/arrow/impl-arrowjs/schema.d.ts +2 -0
- package/dist/arrow/impl-flechette/index.d.ts +1 -1
- package/dist/arrow/impl-flechette/iterate.d.ts +10 -1
- package/dist/arrow/impl-flechette/schema.d.ts +2 -0
- package/dist/arrow/index.d.ts +1 -1
- package/dist/arrow/schema-types.d.ts +3 -1
- package/dist/arrow/types.d.ts +39 -3
- package/dist/catalog/composite.d.ts +13 -13
- package/dist/catalog/descriptors.d.ts +7 -3
- package/dist/catalog/interface.d.ts +108 -38
- package/dist/catalog/read-only.d.ts +32 -15
- package/dist/client/client.d.ts +37 -30
- package/dist/client/types.d.ts +2 -0
- package/dist/client-entry.d.ts +1 -1
- package/dist/client-entry.js +791 -447
- package/dist/client-entry.js.map +18 -17
- package/dist/filter-pushdown/capabilities.d.ts +3 -0
- package/dist/filter-pushdown/deserialize.d.ts +14 -15
- package/dist/filter-pushdown/evaluate.d.ts +14 -44
- package/dist/filter-pushdown/index.d.ts +2 -2
- package/dist/filter-pushdown/types.d.ts +120 -68
- package/dist/functions/aggregate.d.ts +4 -0
- package/dist/functions/registry.d.ts +5 -5
- package/dist/functions/scalar.d.ts +6 -1
- package/dist/functions/table-buffering.d.ts +9 -0
- package/dist/functions/table-in-out.d.ts +13 -0
- package/dist/functions/table.d.ts +72 -2
- package/dist/functions/types.d.ts +60 -2
- package/dist/generated/vgi-client.d.ts +114 -26
- package/dist/generated/vgi-protocol-schemas.d.ts +34 -0
- package/dist/http/fetch.d.ts +16 -0
- package/dist/index.core.d.ts +4 -3
- package/dist/index.js +3396 -2064
- package/dist/index.js.map +55 -49
- package/dist/metadata/types.d.ts +15 -2
- package/dist/protocol/handlers/catalog/shared.d.ts +0 -8
- package/dist/protocol/handlers/function.d.ts +11 -0
- package/dist/protocol/handlers/shared.d.ts +3 -1
- package/dist/protocol/serializers/bind.d.ts +1 -2
- package/dist/protocol/serializers/cardinality.d.ts +1 -2
- package/dist/protocol/serializers/splits.d.ts +126 -0
- package/dist/protocol/types.d.ts +36 -3
- package/dist/request-auth.d.ts +28 -0
- package/dist/schema-path.d.ts +7 -0
- package/dist/serve-entry.d.ts +25 -0
- package/dist/serve-entry.js +1129 -569
- package/dist/serve-entry.js.map +36 -32
- package/dist/split-token.d.ts +88 -0
- package/dist/types.d.ts +8 -0
- package/dist/worker-cf-entry.js +2928 -1689
- package/dist/worker-cf-entry.js.map +51 -44
- package/dist/worker.d.ts +1 -1
- package/package.json +3 -3
- package/src/arguments/argument-spec.ts +93 -4
- package/src/arguments/arguments.ts +0 -1
- package/src/arrow/impl-arrowjs/index.ts +1 -1
- package/src/arrow/impl-arrowjs/iterate.ts +21 -0
- package/src/arrow/impl-arrowjs/schema.ts +4 -0
- package/src/arrow/impl-flechette/index.ts +1 -1
- package/src/arrow/impl-flechette/iterate.ts +24 -1
- package/src/arrow/impl-flechette/schema.ts +4 -0
- package/src/arrow/index.ts +1 -1
- package/src/arrow/schema-types.ts +4 -0
- package/src/arrow/types.ts +39 -3
- package/src/catalog/composite.ts +26 -26
- package/src/catalog/descriptors.ts +12 -3
- package/src/catalog/interface.ts +217 -72
- package/src/catalog/read-only.ts +141 -84
- package/src/client/client.ts +97 -73
- package/src/client/types.ts +2 -0
- package/src/client-entry.ts +2 -1
- package/src/filter-pushdown/capabilities.ts +13 -0
- package/src/filter-pushdown/collector.ts +22 -35
- package/src/filter-pushdown/deserialize.ts +657 -221
- package/src/filter-pushdown/evaluate.ts +410 -499
- package/src/filter-pushdown/index.ts +16 -2
- package/src/filter-pushdown/types.ts +88 -71
- package/src/functions/aggregate.ts +12 -1
- package/src/functions/copy-from.ts +2 -1
- package/src/functions/copy-to.ts +2 -1
- package/src/functions/registry.ts +24 -22
- package/src/functions/scalar.ts +37 -0
- package/src/functions/table-buffering.ts +48 -6
- package/src/functions/table-in-out.ts +194 -33
- package/src/functions/table.ts +187 -22
- package/src/functions/types.ts +64 -1
- package/src/generated/vgi-client.ts +122 -28
- package/src/generated/vgi-protocol-schemas.ts +455 -58
- package/src/http/fetch.ts +129 -2
- package/src/index.core.ts +4 -2
- package/src/metadata/resolve.ts +43 -1
- package/src/metadata/serialize.ts +43 -5
- package/src/metadata/types.ts +17 -1
- package/src/protocol/dispatch.ts +1 -1
- package/src/protocol/handlers/aggregate.ts +23 -12
- package/src/protocol/handlers/catalog/admin.ts +47 -54
- package/src/protocol/handlers/catalog/index_methods.ts +6 -13
- package/src/protocol/handlers/catalog/macro.ts +12 -20
- package/src/protocol/handlers/catalog/shared.ts +12 -57
- package/src/protocol/handlers/catalog/table.ts +49 -112
- package/src/protocol/handlers/catalog/view.ts +18 -22
- package/src/protocol/handlers/function.ts +311 -18
- package/src/protocol/handlers/shared.ts +12 -3
- package/src/protocol/handlers/table-buffering.ts +11 -8
- package/src/protocol/serializers/bind.ts +69 -67
- package/src/protocol/serializers/cardinality.ts +7 -7
- package/src/protocol/serializers/init.ts +52 -10
- package/src/protocol/serializers/splits.ts +221 -0
- package/src/protocol/state-serializer.ts +55 -1
- package/src/protocol/types.ts +36 -3
- package/src/request-auth.ts +73 -0
- package/src/schema-path.ts +25 -0
- package/src/serve-entry.ts +69 -2
- package/src/split-token.ts +297 -0
- package/src/types.ts +9 -0
- package/src/worker-cf-entry.ts +2 -1
- package/src/worker.ts +79 -14
|
@@ -4,7 +4,7 @@
|
|
|
4
4
|
// implements VgiFunction.bind/globalInit/createStreamHandlers.
|
|
5
5
|
|
|
6
6
|
import { type VgiSchema, schema, type VgiField, field, type VgiDataType, binary, int64 } from "../../arrow/index.js";
|
|
7
|
-
import { Protocol } from "@query-farm/vgi-rpc";
|
|
7
|
+
import { Protocol, type AuthContext } from "@query-farm/vgi-rpc";
|
|
8
8
|
import type { FunctionRegistry } from "../../functions/registry.js";
|
|
9
9
|
import type { StreamHandlers, HandlerState } from "../../functions/types.js";
|
|
10
10
|
import {
|
|
@@ -19,7 +19,17 @@ import type { GlobalInitResponse } from "../types.js";
|
|
|
19
19
|
import { batchToScalarDict, deserializeBatch, adoptArrowJsShape } from "../../util/arrow/index.js";
|
|
20
20
|
import { toUint8Array } from "../../util/bytes.js";
|
|
21
21
|
import { serializeColumnStatistics } from "../../util/statistics.js";
|
|
22
|
-
import {
|
|
22
|
+
import {
|
|
23
|
+
BindParamsSchema,
|
|
24
|
+
BindResultSchema,
|
|
25
|
+
InitParamsSchema,
|
|
26
|
+
TableFunctionCardinalityParamsSchema,
|
|
27
|
+
TableFunctionCardinalityResultSchema,
|
|
28
|
+
TableFunctionDynamicToStringParamsSchema,
|
|
29
|
+
TableFunctionDynamicToStringResultSchema,
|
|
30
|
+
TableFunctionPlanParamsSchema,
|
|
31
|
+
TableFunctionStatisticsParamsSchema,
|
|
32
|
+
} from "../../generated/vgi-protocol-schemas.js";
|
|
23
33
|
import {
|
|
24
34
|
REQUEST_PARAMS_SCHEMA,
|
|
25
35
|
RESULT_BINARY_SCHEMA,
|
|
@@ -29,7 +39,25 @@ import {
|
|
|
29
39
|
overloadContext,
|
|
30
40
|
recoverFinalizeState,
|
|
31
41
|
} from "./shared.js";
|
|
42
|
+
import { GLOBAL_INIT_RESPONSE_SCHEMA } from "../serializers/init.js";
|
|
32
43
|
import { openAttach } from "./catalog/shared.js";
|
|
44
|
+
import { currentRequestAuth } from "../../request-auth.js";
|
|
45
|
+
import { batchFromColumns, serializeBatch } from "../../util/arrow/index.js";
|
|
46
|
+
import {
|
|
47
|
+
deserializePlanRequest,
|
|
48
|
+
fingerprintInputs,
|
|
49
|
+
planResponseRow,
|
|
50
|
+
scanSplitRow,
|
|
51
|
+
PLAN_RESPONSE_SCHEMA,
|
|
52
|
+
SCAN_SPLIT_SCHEMA,
|
|
53
|
+
type PlanResult,
|
|
54
|
+
} from "../serializers/splits.js";
|
|
55
|
+
import {
|
|
56
|
+
bindFingerprint,
|
|
57
|
+
buildSplitToken,
|
|
58
|
+
openSplitToken,
|
|
59
|
+
splitAnchor,
|
|
60
|
+
} from "../../split-token.js";
|
|
33
61
|
|
|
34
62
|
export interface FunctionHandlerConfig {
|
|
35
63
|
registry: FunctionRegistry;
|
|
@@ -40,7 +68,134 @@ export interface FunctionHandlerConfig {
|
|
|
40
68
|
* be scoped when two catalogs declare the same schema and function name.
|
|
41
69
|
* Optional — without it, resolution falls back to (schema, name).
|
|
42
70
|
*/
|
|
43
|
-
catalogInterface?: {
|
|
71
|
+
catalogInterface?: {
|
|
72
|
+
catalogNameForAttach(a: Uint8Array): string | null;
|
|
73
|
+
/**
|
|
74
|
+
* The catalog's current version, which is the consistency anchor every split
|
|
75
|
+
* token is stamped with AND checked against.
|
|
76
|
+
*
|
|
77
|
+
* Both sides read it from here on purpose. Minting from a different value
|
|
78
|
+
* than redemption compares against is not a subtle bug: it refuses every
|
|
79
|
+
* token, and the documented response to SPLIT_SNAPSHOT_EXPIRED is "re-run
|
|
80
|
+
* the query", which re-plans, mints the same mismatch and fails again — a
|
|
81
|
+
* livelock returning no rows, blaming the data for moving when it has not.
|
|
82
|
+
*/
|
|
83
|
+
version?(attach: Uint8Array, txn?: Uint8Array): number | Promise<number>;
|
|
84
|
+
};
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
/**
|
|
88
|
+
* The live anchor a split token must still name, or undefined when this worker's
|
|
89
|
+
* catalog reports no version.
|
|
90
|
+
*
|
|
91
|
+
* Undefined means "skip the staleness check" rather than "fail" — a check that
|
|
92
|
+
* cannot be made must not become a check that always fails, which would refuse
|
|
93
|
+
* every split token on every unversioned catalog.
|
|
94
|
+
*/
|
|
95
|
+
async function liveSplitAnchor(
|
|
96
|
+
catalogInterface: FunctionHandlerConfig["catalogInterface"],
|
|
97
|
+
bindCall: { attach_opaque_data?: Uint8Array | null; transaction_opaque_data?: Uint8Array | null },
|
|
98
|
+
auth: AuthContext | undefined,
|
|
99
|
+
signingKey: Uint8Array | undefined,
|
|
100
|
+
): Promise<Uint8Array | undefined> {
|
|
101
|
+
if (!catalogInterface?.version || bindCall.attach_opaque_data == null) return undefined;
|
|
102
|
+
const raw = toUint8Array(bindCall.attach_opaque_data);
|
|
103
|
+
const txn = bindCall.transaction_opaque_data
|
|
104
|
+
? toUint8Array(bindCall.transaction_opaque_data)
|
|
105
|
+
: undefined;
|
|
106
|
+
|
|
107
|
+
// The two call sites see the attach envelope at DIFFERENT processing stages —
|
|
108
|
+
// `plan` receives it with the framework's 16-byte UUID prefix still attached,
|
|
109
|
+
// `init` receives it already stripped — so neither form alone works for both.
|
|
110
|
+
// Try each. Getting this wrong is not a degraded check but a total one: if
|
|
111
|
+
// mint resolves a version and redemption does not (or vice versa) then every
|
|
112
|
+
// token fails SPLIT_SNAPSHOT_EXPIRED, and the documented response to that is
|
|
113
|
+
// "re-run the query", which re-plans and reproduces the same mismatch.
|
|
114
|
+
// Each form is produced LAZILY, inside the try. Built as an array literal —
|
|
115
|
+
// `[raw, await openAttach(...)]` — the unseal is evaluated before the loop
|
|
116
|
+
// begins, so it throws from OUTSIDE the try and escapes the function instead
|
|
117
|
+
// of falling through to the next candidate. `init` hands this function
|
|
118
|
+
// already-stripped plaintext, which openAttach rejects, so the first form
|
|
119
|
+
// never got a turn: every split scan died with OpaqueDataRejectedError, on
|
|
120
|
+
// HTTP only (sealing is a pass-through when there is no signing key, which is
|
|
121
|
+
// every non-HTTP transport).
|
|
122
|
+
const forms: Array<() => Promise<Uint8Array>> = [
|
|
123
|
+
async () => raw,
|
|
124
|
+
async () => openAttach(raw, auth, signingKey),
|
|
125
|
+
];
|
|
126
|
+
for (const form of forms) {
|
|
127
|
+
try {
|
|
128
|
+
return splitAnchor(await catalogInterface.version(await form(), txn));
|
|
129
|
+
} catch {
|
|
130
|
+
// try the next form
|
|
131
|
+
}
|
|
132
|
+
}
|
|
133
|
+
// A catalog that cannot report a version means "skip the staleness check",
|
|
134
|
+
// never "fail it" — a check that cannot be made must not become a check that
|
|
135
|
+
// always fails, which would refuse every split token on every unversioned
|
|
136
|
+
// catalog.
|
|
137
|
+
return undefined;
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
/**
|
|
141
|
+
* Open this request's split envelopes into `request.split_payloads`.
|
|
142
|
+
*
|
|
143
|
+
* `split_payloads` is DERIVED, not wire state: the tokens ride the request, the
|
|
144
|
+
* payloads are what falls out of opening them. Only this layer holds the
|
|
145
|
+
* signing key — a function does not — so redemption happens here and only the
|
|
146
|
+
* payloads are handed down.
|
|
147
|
+
*
|
|
148
|
+
* It must run on continuation turns too. The HTTP exchange path rebuilds the
|
|
149
|
+
* *wire* request from the cursor, which carries `split_tokens` but of course no
|
|
150
|
+
* derived field, so a split-capable function used to see `splitPayloads:
|
|
151
|
+
* undefined` on every turn after the first and could not tell that from "the
|
|
152
|
+
* planner gave me no splits". Over HTTP that made every split scan fail on its
|
|
153
|
+
* second turn; subprocess never noticed because it holds the handlers in memory
|
|
154
|
+
* and never rebuilds anything.
|
|
155
|
+
*
|
|
156
|
+
* `checkAnchor` is false on a continuation, deliberately. The consistency
|
|
157
|
+
* anchor answers "is this plan still redeemable", which is a question about
|
|
158
|
+
* REDEMPTION — asked once, when the split is admitted. Re-asking it mid-stream
|
|
159
|
+
* would let a catalog version bump kill an in-flight scan that had already been
|
|
160
|
+
* accepted, turning a successful query into SPLIT_SNAPSHOT_EXPIRED halfway
|
|
161
|
+
* through its rows.
|
|
162
|
+
*/
|
|
163
|
+
async function resolveSplitPayloads(
|
|
164
|
+
request: any,
|
|
165
|
+
innerParams: any,
|
|
166
|
+
opts: {
|
|
167
|
+
auth: any;
|
|
168
|
+
signingKey: Uint8Array | undefined;
|
|
169
|
+
catalogInterface: any;
|
|
170
|
+
checkAnchor: boolean;
|
|
171
|
+
},
|
|
172
|
+
): Promise<void> {
|
|
173
|
+
if (!request.split_tokens) return;
|
|
174
|
+
const fp = fingerprintInputs(toUint8Array(innerParams.bind_call));
|
|
175
|
+
const expected = await bindFingerprint(
|
|
176
|
+
fp.schemaPath,
|
|
177
|
+
fp.functionName,
|
|
178
|
+
fp.args,
|
|
179
|
+
fp.settings,
|
|
180
|
+
new Uint8Array(0),
|
|
181
|
+
);
|
|
182
|
+
const currentAnchor = opts.checkAnchor
|
|
183
|
+
? await liveSplitAnchor(opts.catalogInterface, request.bind_call as any, opts.auth, opts.signingKey)
|
|
184
|
+
: undefined;
|
|
185
|
+
request.split_payloads = [];
|
|
186
|
+
for (const token of request.split_tokens) {
|
|
187
|
+
request.split_payloads.push(
|
|
188
|
+
await openSplitToken(token, {
|
|
189
|
+
signingKey: opts.signingKey,
|
|
190
|
+
// Same principal `table_function_plan` sealed under. Omitting it opens
|
|
191
|
+
// under the anonymous tail, so an authenticated caller's own tokens
|
|
192
|
+
// fail authentication.
|
|
193
|
+
auth: opts.auth,
|
|
194
|
+
expectedFingerprint: expected,
|
|
195
|
+
currentAnchor,
|
|
196
|
+
}),
|
|
197
|
+
);
|
|
198
|
+
}
|
|
44
199
|
}
|
|
45
200
|
|
|
46
201
|
export function registerFunctionMethods(protocol: Protocol, config: FunctionHandlerConfig): void {
|
|
@@ -50,11 +205,17 @@ export function registerFunctionMethods(protocol: Protocol, config: FunctionHand
|
|
|
50
205
|
// HTTP, plaintext on subprocess). Function bodies — like catalog bodies —
|
|
51
206
|
// must see the catalog's own bytes, so unseal (when keyed) and strip the
|
|
52
207
|
// framework UUID prefix before the user function reads attach_opaque_data.
|
|
208
|
+
// `ctx` is the CallContext when the dispatcher supplies one (every unary
|
|
209
|
+
// method) and absent otherwise (stream init); `currentRequestAuth()` is the
|
|
210
|
+
// ambient per-request fallback the HTTP entry point publishes, so both reach
|
|
211
|
+
// the same principal. Getting an authenticated caller's identity wrong here
|
|
212
|
+
// is not a degraded lookup but a hard `OpaqueDataRejectedError`: the envelope
|
|
213
|
+
// is sealed under `attachAad(auth)`.
|
|
53
214
|
async function stripAttach(attach: any, ctx: any): Promise<Uint8Array | null> {
|
|
54
215
|
if (attach == null) return null;
|
|
55
216
|
const env = toUint8Array(attach);
|
|
56
217
|
if (env.length === 0) return env;
|
|
57
|
-
return openAttach(env, ctx?.auth, signingKey);
|
|
218
|
+
return openAttach(env, ctx?.auth ?? currentRequestAuth(), signingKey);
|
|
58
219
|
}
|
|
59
220
|
|
|
60
221
|
// Build an overload context whose attach has been unsealed, so catalog
|
|
@@ -72,11 +233,9 @@ export function registerFunctionMethods(protocol: Protocol, config: FunctionHand
|
|
|
72
233
|
return overloadContext({ ...bindCall, attach_opaque_data: attach }, catalogInterface);
|
|
73
234
|
}
|
|
74
235
|
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
field("max_workers", int64(), false),
|
|
79
|
-
]);
|
|
236
|
+
// Shared with the serializer that actually emits the header, so the
|
|
237
|
+
// advertised shape and the emitted shape cannot drift apart.
|
|
238
|
+
const initHeaderSchema = GLOBAL_INIT_RESPONSE_SCHEMA;
|
|
80
239
|
|
|
81
240
|
const emptySchema = schema([]);
|
|
82
241
|
|
|
@@ -92,7 +251,7 @@ export function registerFunctionMethods(protocol: Protocol, config: FunctionHand
|
|
|
92
251
|
// bind (unary)
|
|
93
252
|
// --------------------------------------------------------------------------
|
|
94
253
|
protocol.unary("bind", {
|
|
95
|
-
params:
|
|
254
|
+
params: BindParamsSchema,
|
|
96
255
|
result: RESULT_BINARY_SCHEMA,
|
|
97
256
|
handler: async (params, ctx) => {
|
|
98
257
|
const innerParams = unwrapRequest(params.request);
|
|
@@ -109,17 +268,46 @@ export function registerFunctionMethods(protocol: Protocol, config: FunctionHand
|
|
|
109
268
|
// init (streaming) - dynamically produces either producer or exchange streams
|
|
110
269
|
// --------------------------------------------------------------------------
|
|
111
270
|
protocol.exchange("init", {
|
|
112
|
-
params:
|
|
271
|
+
params: InitParamsSchema,
|
|
113
272
|
inputSchema: dummyInputSchema,
|
|
114
273
|
outputSchema: emptySchema,
|
|
115
|
-
init: async (params) => {
|
|
274
|
+
init: async (params, ctx?: any) => {
|
|
116
275
|
// Preserve raw request IPC bytes for exchange reconstruction
|
|
117
276
|
const requestIpcBytes = toUint8Array(params.request);
|
|
118
277
|
const innerParams = unwrapRequest(params.request);
|
|
119
278
|
const request = deserializeInitRequest(innerParams);
|
|
120
|
-
//
|
|
121
|
-
//
|
|
122
|
-
|
|
279
|
+
// The caller's identity, and the load-bearing line of this handler.
|
|
280
|
+
//
|
|
281
|
+
// vgi-rpc types stream init as `(params) => state`, so unlike every unary
|
|
282
|
+
// method there is no CallContext to read `auth` off — `ctx` is only ever
|
|
283
|
+
// populated by a caller that has one (the in-process client). The HTTP
|
|
284
|
+
// entry point therefore publishes the dispatching principal on an ambient
|
|
285
|
+
// per-request scope, the same shape vgi-python's `current_auth()`
|
|
286
|
+
// ContextVar and vgi-go's `handleInit(callCtx)` give those ports.
|
|
287
|
+
//
|
|
288
|
+
// It has to be the REAL principal, not `undefined`: `attach_opaque_data`
|
|
289
|
+
// is sealed by `catalog_attach` under `attachAad(auth)` and every split
|
|
290
|
+
// token by `table_function_plan` under `splitTokenAad(body, auth)`, both
|
|
291
|
+
// of which are unary and so seal under the authenticated caller. Opening
|
|
292
|
+
// them here under the anonymous tail matches only for an anonymous
|
|
293
|
+
// caller; an authenticated one got `OpaqueDataRejectedError:
|
|
294
|
+
// attach_opaque_data not recognized` on its first scan, i.e. this worker
|
|
295
|
+
// could not serve authenticated traffic at all.
|
|
296
|
+
const auth = ctx?.auth ?? currentRequestAuth();
|
|
297
|
+
request.bind_call.attach_opaque_data = await stripAttach(
|
|
298
|
+
request.bind_call.attach_opaque_data,
|
|
299
|
+
ctx,
|
|
300
|
+
);
|
|
301
|
+
|
|
302
|
+
// Verify and strip the split envelopes BEFORE any user code runs, so an
|
|
303
|
+
// unverified token can never be acted on.
|
|
304
|
+
await resolveSplitPayloads(request, innerParams, {
|
|
305
|
+
auth,
|
|
306
|
+
signingKey,
|
|
307
|
+
catalogInterface,
|
|
308
|
+
checkAnchor: true,
|
|
309
|
+
});
|
|
310
|
+
|
|
123
311
|
const func = registry.get(request.bind_call.function_name, overloadContext(request.bind_call, catalogInterface));
|
|
124
312
|
|
|
125
313
|
// globalInit is async — table function onInit may touch HTTP-backed
|
|
@@ -199,6 +387,28 @@ export function registerFunctionMethods(protocol: Protocol, config: FunctionHand
|
|
|
199
387
|
const initRequestBatch = deserializeBatch(state.initRequestIpc);
|
|
200
388
|
const initRequestDict = batchToScalarDict(initRequestBatch);
|
|
201
389
|
const request = deserializeInitRequest(initRequestDict);
|
|
390
|
+
// Re-derive the split payloads. The cursor carries the wire request,
|
|
391
|
+
// which has `split_tokens` but not the opened `split_payloads` — that
|
|
392
|
+
// field only ever existed in memory on the init turn. Without this a
|
|
393
|
+
// split-capable function sees `splitPayloads: undefined` from the
|
|
394
|
+
// second turn onward, indistinguishable from "the planner gave me no
|
|
395
|
+
// splits at all". checkAnchor is false: redemption was already decided
|
|
396
|
+
// on the init turn, and re-checking here would let a mid-scan catalog
|
|
397
|
+
// bump fail a query that had already been admitted.
|
|
398
|
+
// fingerprintInputs hashes the RAW bind_call bytes, so pass the
|
|
399
|
+
// pre-deserialization blob from the cursor — handing it the decoded
|
|
400
|
+
// object hashes a different shape and every token fails as
|
|
401
|
+
// SPLIT_TOKEN_INVALID "minted for a different bind".
|
|
402
|
+
await resolveSplitPayloads(request, { bind_call: initRequestDict.bind_call }, {
|
|
403
|
+
// Same principal the tokens were sealed under. The exchange dispatch
|
|
404
|
+
// gets no ctx (see stripAttach above), so the request-scoped identity
|
|
405
|
+
// is the only source — opening under the anonymous tail would fail
|
|
406
|
+
// authentication for every authenticated caller's own tokens.
|
|
407
|
+
auth: currentRequestAuth(),
|
|
408
|
+
signingKey,
|
|
409
|
+
catalogInterface,
|
|
410
|
+
checkAnchor: false,
|
|
411
|
+
});
|
|
202
412
|
const func = registry.get(state.functionName, await strippedContext(request.bind_call));
|
|
203
413
|
const executionId = state.executionId;
|
|
204
414
|
const opaqueData = state.opaqueData ?? null;
|
|
@@ -259,7 +469,7 @@ export function registerFunctionMethods(protocol: Protocol, config: FunctionHand
|
|
|
259
469
|
// table_function_cardinality (unary)
|
|
260
470
|
// --------------------------------------------------------------------------
|
|
261
471
|
protocol.unary("table_function_cardinality", {
|
|
262
|
-
params:
|
|
472
|
+
params: TableFunctionCardinalityParamsSchema,
|
|
263
473
|
result: RESULT_BINARY_SCHEMA,
|
|
264
474
|
handler: async (params) => {
|
|
265
475
|
const innerParams = unwrapRequest(params.request);
|
|
@@ -275,6 +485,89 @@ export function registerFunctionMethods(protocol: Protocol, config: FunctionHand
|
|
|
275
485
|
},
|
|
276
486
|
});
|
|
277
487
|
|
|
488
|
+
// --------------------------------------------------------------------------
|
|
489
|
+
// table_function_plan (unary)
|
|
490
|
+
// --------------------------------------------------------------------------
|
|
491
|
+
// Divide a scan into named, independently redeemable splits, so a distributed
|
|
492
|
+
// engine can retry a task without re-reading or skipping rows.
|
|
493
|
+
//
|
|
494
|
+
// A function that declares no `plan` hook gets the framework default: a SINGLE
|
|
495
|
+
// empty-payload split, which is what "not split-capable" means — the whole scan
|
|
496
|
+
// is one unit of work. That keeps every existing worker serving unchanged under
|
|
497
|
+
// protocol 1.4.0, so splits stay opt-in rather than something every worker must
|
|
498
|
+
// now implement.
|
|
499
|
+
protocol.unary("table_function_plan", {
|
|
500
|
+
params: TableFunctionPlanParamsSchema,
|
|
501
|
+
result: RESULT_BINARY_SCHEMA,
|
|
502
|
+
handler: async (params, ctx?: any) => {
|
|
503
|
+
const innerParams = unwrapRequest(params.request);
|
|
504
|
+
const request = deserializePlanRequest(innerParams);
|
|
505
|
+
const func = registry.get(
|
|
506
|
+
request.bind_call.function_name,
|
|
507
|
+
await strippedContext(request.bind_call),
|
|
508
|
+
);
|
|
509
|
+
|
|
510
|
+
const plan: PlanResult = func.plan
|
|
511
|
+
? await func.plan(request)
|
|
512
|
+
: { splits: [{ payload: new Uint8Array(0) }] };
|
|
513
|
+
|
|
514
|
+
// The framework stamps every token: an author cannot forget the
|
|
515
|
+
// consistency anchor, cannot mis-bind the fingerprint, and never writes
|
|
516
|
+
// crypto — and the envelope stays a private implementation detail whose
|
|
517
|
+
// layout can change without touching worker code in five languages.
|
|
518
|
+
const fp = fingerprintInputs(toUint8Array(innerParams.bind_call));
|
|
519
|
+
const fingerprint = await bindFingerprint(
|
|
520
|
+
fp.schemaPath,
|
|
521
|
+
fp.functionName,
|
|
522
|
+
fp.args,
|
|
523
|
+
fp.settings,
|
|
524
|
+
// projection_ids is not a bind-call field (it rides the init request),
|
|
525
|
+
// so it feeds in empty — matching the reference implementation, which
|
|
526
|
+
// reads it off the bind call and likewise finds nothing.
|
|
527
|
+
new Uint8Array(0),
|
|
528
|
+
);
|
|
529
|
+
// A worker that names its version is taken at its word — it knows which
|
|
530
|
+
// snapshot it planned against. One that leaves it unset gets the LIVE
|
|
531
|
+
// version, never 0: minting 0 while redemption compares against the live
|
|
532
|
+
// counter refuses every token, and is invisible on a catalog whose version
|
|
533
|
+
// happens to be 0, which is most fixtures.
|
|
534
|
+
const anchor =
|
|
535
|
+
plan.catalogVersion != null
|
|
536
|
+
? splitAnchor(plan.catalogVersion)
|
|
537
|
+
: ((await liveSplitAnchor(catalogInterface, request.bind_call as any, ctx?.auth, signingKey)) ??
|
|
538
|
+
splitAnchor(0));
|
|
539
|
+
|
|
540
|
+
const blobs: Uint8Array[] = [];
|
|
541
|
+
for (const split of plan.splits) {
|
|
542
|
+
const token = await buildSplitToken({
|
|
543
|
+
payload: split.payload,
|
|
544
|
+
fingerprint,
|
|
545
|
+
anchor,
|
|
546
|
+
signingKey,
|
|
547
|
+
// Bind the caller. Without it every token seals under the anonymous
|
|
548
|
+
// identity, so one tenant's splits are replayable by another — the
|
|
549
|
+
// exact replay the AAD exists to stop.
|
|
550
|
+
auth: ctx?.auth,
|
|
551
|
+
});
|
|
552
|
+
blobs.push(
|
|
553
|
+
serializeBatch(
|
|
554
|
+
batchFromColumns(
|
|
555
|
+
Object.fromEntries(
|
|
556
|
+
SCAN_SPLIT_SCHEMA.fields.map((f) => [
|
|
557
|
+
f.name,
|
|
558
|
+
[scanSplitRow(split, token)[f.name] ?? null],
|
|
559
|
+
]),
|
|
560
|
+
),
|
|
561
|
+
SCAN_SPLIT_SCHEMA,
|
|
562
|
+
),
|
|
563
|
+
),
|
|
564
|
+
);
|
|
565
|
+
}
|
|
566
|
+
|
|
567
|
+
return wrapResult(planResponseRow(plan, blobs), PLAN_RESPONSE_SCHEMA);
|
|
568
|
+
},
|
|
569
|
+
});
|
|
570
|
+
|
|
278
571
|
// --------------------------------------------------------------------------
|
|
279
572
|
// table_function_statistics (unary)
|
|
280
573
|
// --------------------------------------------------------------------------
|
|
@@ -283,7 +576,7 @@ export function registerFunctionMethods(protocol: Protocol, config: FunctionHand
|
|
|
283
576
|
// list, else null. DuckDB uses the bounds for plan-time filter elimination
|
|
284
577
|
// (folds impossible filters to EMPTY_RESULT).
|
|
285
578
|
protocol.unary("table_function_statistics", {
|
|
286
|
-
params:
|
|
579
|
+
params: TableFunctionStatisticsParamsSchema,
|
|
287
580
|
result: RESULT_BINARY_NULLABLE_SCHEMA,
|
|
288
581
|
handler: async (params) => {
|
|
289
582
|
const innerParams = unwrapRequest(params.request);
|
|
@@ -306,7 +599,7 @@ export function registerFunctionMethods(protocol: Protocol, config: FunctionHand
|
|
|
306
599
|
// declare dynamicToString, return empty maps so the C++ side falls back
|
|
307
600
|
// to intrinsics only.
|
|
308
601
|
protocol.unary("table_function_dynamic_to_string", {
|
|
309
|
-
params:
|
|
602
|
+
params: TableFunctionDynamicToStringParamsSchema,
|
|
310
603
|
result: RESULT_BINARY_SCHEMA,
|
|
311
604
|
handler: async (params) => {
|
|
312
605
|
const innerParams = unwrapRequest(params.request);
|
|
@@ -7,6 +7,7 @@ import { batchToScalarDict, deserializeBatch, serializeBatch, batchFromColumns }
|
|
|
7
7
|
import { toUint8Array } from "../../util/bytes.js";
|
|
8
8
|
import { TableInOutPhase } from "../../types.js";
|
|
9
9
|
import type { InitRequest } from "../types.js";
|
|
10
|
+
import { normalizeSchemaPath } from "../../schema-path.js";
|
|
10
11
|
|
|
11
12
|
// The Python vgi-rpc framework wraps ALL non-void unary results in a single
|
|
12
13
|
// "result" column. For ArrowSerializableDataclass types, the result is serialized
|
|
@@ -31,7 +32,15 @@ export const RESULT_BINARY_NULLABLE_SCHEMA = schema([
|
|
|
31
32
|
export function unwrapRequest(requestBytes: any): Record<string, any> {
|
|
32
33
|
const bytes = toUint8Array(requestBytes);
|
|
33
34
|
const innerBatch = deserializeBatch(bytes);
|
|
34
|
-
|
|
35
|
+
const params = batchToScalarDict(innerBatch);
|
|
36
|
+
normalizeRequestSchemaPaths(params);
|
|
37
|
+
return params;
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
/** Convert Arrow list scalars at RPC boundaries into ordinary string arrays. */
|
|
41
|
+
export function normalizeRequestSchemaPaths(params: Record<string, any>): void {
|
|
42
|
+
if (params.path != null) params.path = normalizeSchemaPath(params.path);
|
|
43
|
+
if (params.schema_path != null) params.schema_path = normalizeSchemaPath(params.schema_path);
|
|
35
44
|
}
|
|
36
45
|
|
|
37
46
|
/**
|
|
@@ -57,7 +66,7 @@ export function overloadContext(
|
|
|
57
66
|
arguments: any;
|
|
58
67
|
input_schema: any;
|
|
59
68
|
function_type: any;
|
|
60
|
-
|
|
69
|
+
schema_path?: string[] | null;
|
|
61
70
|
attach_opaque_data?: Uint8Array | null;
|
|
62
71
|
},
|
|
63
72
|
catalogInterface?: { catalogNameForAttach(a: Uint8Array): string | null },
|
|
@@ -78,7 +87,7 @@ export function overloadContext(
|
|
|
78
87
|
isScalar: String(req.function_type).toLowerCase() === "scalar",
|
|
79
88
|
// Scopes resolution when the caller named a schema; a name registered in
|
|
80
89
|
// several schemas is otherwise ambiguous.
|
|
81
|
-
|
|
90
|
+
schemaPath: req.schema_path ?? null,
|
|
82
91
|
catalogName,
|
|
83
92
|
};
|
|
84
93
|
}
|
|
@@ -15,9 +15,12 @@ import type { FunctionRegistry } from "../../functions/registry.js";
|
|
|
15
15
|
import { toUint8Array } from "../../util/bytes.js";
|
|
16
16
|
import { batchToScalarDict, batchToSecretDict } from "../../util/arrow/index.js";
|
|
17
17
|
import {
|
|
18
|
-
|
|
18
|
+
TableBufferingCombineParamsSchema,
|
|
19
19
|
TableBufferingCombineResultSchema,
|
|
20
|
+
TableBufferingDestructorParamsSchema,
|
|
20
21
|
TableBufferingDestructorResultSchema,
|
|
22
|
+
TableBufferingProcessParamsSchema,
|
|
23
|
+
TableBufferingProcessResultSchema,
|
|
21
24
|
} from "../../generated/vgi-protocol-schemas.js";
|
|
22
25
|
import {
|
|
23
26
|
REQUEST_PARAMS_SCHEMA,
|
|
@@ -43,19 +46,19 @@ export function registerTableBufferingMethods(
|
|
|
43
46
|
registry: FunctionRegistry,
|
|
44
47
|
signingKey?: Uint8Array,
|
|
45
48
|
): void {
|
|
46
|
-
// `
|
|
49
|
+
// `schemaPath` scopes resolution to the schema the caller named. The
|
|
47
50
|
// buffering process/combine RPCs carry it (protocol 1.2.0), so a name
|
|
48
51
|
// declared in two schemas resolves to the one the request names rather than
|
|
49
52
|
// colliding on the flat by-name index. `null` keeps the cross-schema lookup.
|
|
50
53
|
function resolveBuffering(
|
|
51
54
|
name: string,
|
|
52
|
-
|
|
55
|
+
schemaPath?: string[] | null,
|
|
53
56
|
): TableBufferingVgiFunction {
|
|
54
57
|
const func = registry.get(name, {
|
|
55
58
|
arguments: new Arguments(),
|
|
56
59
|
inputSchema: null,
|
|
57
60
|
isScalar: false,
|
|
58
|
-
|
|
61
|
+
schemaPath: schemaPath ?? null,
|
|
59
62
|
}) as any;
|
|
60
63
|
if (!func || func.kind !== "table_buffering") {
|
|
61
64
|
throw new Error(`Function '${name}' is not a table_buffering function`);
|
|
@@ -83,7 +86,7 @@ export function registerTableBufferingMethods(
|
|
|
83
86
|
: rawAttach;
|
|
84
87
|
const transactionId = inner.transaction_id ? toUint8Array(inner.transaction_id) : null;
|
|
85
88
|
|
|
86
|
-
const func = resolveBuffering(functionName, inner.
|
|
89
|
+
const func = resolveBuffering(functionName, inner.schema_path ?? null);
|
|
87
90
|
const bound = new BoundStorage(defaultStorage, executionId);
|
|
88
91
|
const payload = await bound.stateGet(
|
|
89
92
|
FrameworkNS.BUFFERING_INIT,
|
|
@@ -121,7 +124,7 @@ export function registerTableBufferingMethods(
|
|
|
121
124
|
// table_buffering_process — sink one batch, return state_id (unary)
|
|
122
125
|
// ------------------------------------------------------------------------
|
|
123
126
|
protocol.unary("table_buffering_process", {
|
|
124
|
-
params:
|
|
127
|
+
params: TableBufferingProcessParamsSchema,
|
|
125
128
|
result: RESULT_BINARY_SCHEMA,
|
|
126
129
|
handler: async (rpcParams, ctx?: any) => {
|
|
127
130
|
const inner = unwrapRequest(rpcParams.request);
|
|
@@ -145,7 +148,7 @@ export function registerTableBufferingMethods(
|
|
|
145
148
|
// table_buffering_combine — group/merge state_ids (unary)
|
|
146
149
|
// ------------------------------------------------------------------------
|
|
147
150
|
protocol.unary("table_buffering_combine", {
|
|
148
|
-
params:
|
|
151
|
+
params: TableBufferingCombineParamsSchema,
|
|
149
152
|
result: RESULT_BINARY_SCHEMA,
|
|
150
153
|
handler: async (rpcParams, ctx?: any) => {
|
|
151
154
|
const inner = unwrapRequest(rpcParams.request);
|
|
@@ -172,7 +175,7 @@ export function registerTableBufferingMethods(
|
|
|
172
175
|
// table_buffering_destructor — best-effort end-of-query cleanup (unary)
|
|
173
176
|
// ------------------------------------------------------------------------
|
|
174
177
|
protocol.unary("table_buffering_destructor", {
|
|
175
|
-
params:
|
|
178
|
+
params: TableBufferingDestructorParamsSchema,
|
|
176
179
|
result: RESULT_BINARY_SCHEMA,
|
|
177
180
|
handler: async (rpcParams) => {
|
|
178
181
|
try {
|