@query-farm/vgi 0.42.0 → 0.44.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (59) hide show
  1. package/README.md +5 -2
  2. package/dist/attach-ticket.d.ts +18 -7
  3. package/dist/client-entry.js +215 -91
  4. package/dist/client-entry.js.map +10 -9
  5. package/dist/codec/asd.d.ts +3 -1
  6. package/dist/codec/conform.d.ts +51 -0
  7. package/dist/errors.d.ts +5 -1
  8. package/dist/generated/vgi-service.d.ts +1315 -0
  9. package/dist/index.core.d.ts +1 -1
  10. package/dist/index.js +2143 -932
  11. package/dist/index.js.map +29 -28
  12. package/dist/protocol/dispatch.d.ts +14 -5
  13. package/dist/protocol/handlers/aggregate.d.ts +3 -2
  14. package/dist/protocol/handlers/catalog/admin.d.ts +2 -2
  15. package/dist/protocol/handlers/catalog/index.d.ts +3 -2
  16. package/dist/protocol/handlers/catalog/index_methods.d.ts +2 -2
  17. package/dist/protocol/handlers/catalog/macro.d.ts +2 -2
  18. package/dist/protocol/handlers/catalog/shared.d.ts +7 -14
  19. package/dist/protocol/handlers/catalog/table.d.ts +2 -2
  20. package/dist/protocol/handlers/catalog/view.d.ts +2 -2
  21. package/dist/protocol/handlers/function.d.ts +3 -2
  22. package/dist/protocol/handlers/shared.d.ts +2 -2
  23. package/dist/protocol/handlers/table-buffering.d.ts +3 -2
  24. package/dist/protocol/handlers/unimplemented.d.ts +4 -17
  25. package/dist/protocol/serializers/shared.d.ts +2 -1
  26. package/dist/rpc-server.d.ts +2 -2
  27. package/dist/serve-entry.js +2104 -903
  28. package/dist/serve-entry.js.map +25 -24
  29. package/dist/worker-cf-entry.js +2045 -842
  30. package/dist/worker-cf-entry.js.map +29 -28
  31. package/package.json +3 -3
  32. package/src/attach-ticket.ts +41 -13
  33. package/src/catalog/composite.ts +2 -1
  34. package/src/catalog/interface.ts +5 -4
  35. package/src/catalog/read-only.ts +8 -6
  36. package/src/client/client.ts +13 -5
  37. package/src/codec/asd.ts +5 -2
  38. package/src/codec/conform.ts +193 -0
  39. package/src/errors.ts +27 -7
  40. package/src/functions/registry.ts +3 -2
  41. package/src/generated/vgi-service.ts +2678 -0
  42. package/src/http/fetch.ts +1 -1
  43. package/src/index.core.ts +3 -1
  44. package/src/protocol/dispatch.ts +40 -29
  45. package/src/protocol/handlers/aggregate.ts +205 -232
  46. package/src/protocol/handlers/catalog/admin.ts +228 -299
  47. package/src/protocol/handlers/catalog/index.ts +17 -16
  48. package/src/protocol/handlers/catalog/index_methods.ts +26 -36
  49. package/src/protocol/handlers/catalog/macro.ts +54 -77
  50. package/src/protocol/handlers/catalog/shared.ts +30 -47
  51. package/src/protocol/handlers/catalog/table.ts +205 -270
  52. package/src/protocol/handlers/catalog/view.ts +59 -85
  53. package/src/protocol/handlers/function.ts +127 -156
  54. package/src/protocol/handlers/shared.ts +8 -6
  55. package/src/protocol/handlers/table-buffering.ts +53 -71
  56. package/src/protocol/handlers/unimplemented.ts +18 -62
  57. package/src/protocol/serializers/bind.ts +3 -1
  58. package/src/protocol/serializers/shared.ts +5 -2
  59. package/src/rpc-server.ts +7 -7
package/src/http/fetch.ts CHANGED
@@ -207,7 +207,7 @@ export function createVgiFetch(opts: VgiFetchOptions): (req: Request) => Promise
207
207
 
208
208
  // The same builder every other transport uses: `vgi.v2`, the worker's
209
209
  // hostedProtocols(), reflection, and -- HTTP being the transport that
210
- // authenticates callers -- vgi_rpc.Identity.v1 when the worker opts in.
210
+ // authenticates callers -- vgi_rpc.Identity.v2 when the worker opts in.
211
211
  // Built at construction, so a worker that opts into introspection without an
212
212
  // allowlist fails here rather than serving traffic.
213
213
  const server = buildRpcServer(protocol, opts, {
package/src/index.core.ts CHANGED
@@ -186,8 +186,10 @@ export {
186
186
  AttachTicketInvalidError,
187
187
  AttachTicketRequestError,
188
188
  SealAttachDeniedError,
189
+ type AttachTicket,
189
190
  type AttachTicketClaims,
190
191
  type RestoredAttach,
192
+ encodeAttachTicket,
191
193
  attachTicketAad,
192
194
  mintAttachTicket,
193
195
  openAttachTicket,
@@ -338,7 +340,7 @@ export { str, bytes, int, int32, float, float32, bool, toSchema, OutputCollector
338
340
  export { subprocessConnect, httpConnect, tcpConnect } from "@query-farm/vgi-rpc";
339
341
  export type { RpcClient, StreamSession, LogMessage } from "@query-farm/vgi-rpc";
340
342
 
341
- // Hosting additional protocols and opting into vgi_rpc.Identity.v1. The errors
343
+ // Hosting additional protocols and opting into vgi_rpc.Identity.v2. The errors
342
344
  // are re-exported so a worker's resolveToken/mintGrant can throw the right one:
343
345
  // AuthUnavailableError (or IdentityUnavailableError) for "could not find out",
344
346
  // GrantRefusedError to decline a grant.
@@ -1,21 +1,23 @@
1
1
  // Copyright 2025, 2026 Query Farm LLC - https://query.farm
2
2
  // Build VGI protocol from worker implementation.
3
- // Orchestrator that registers function, aggregate, and catalog handlers
4
- // against vgi-rpc Protocol. The handler implementations live in handlers/.
3
+ // The method table, its schemas and the UNIMPLEMENTED defaults are generated
4
+ // (src/generated/vgi-service.ts, from vgi-python's VgiProtocol); this module
5
+ // composes the handler groups in handlers/ over those defaults.
5
6
 
6
- import { Protocol } from "@query-farm/vgi-rpc";
7
+ import type { Protocol } from "@query-farm/vgi-rpc";
7
8
 
8
- // NOTE: We must NOT use vgi-rpc's str/bytes/int/etc. singletons in Schema objects
9
- // because Bun loads apache-arrow as separate module instances for our code vs vgi-rpc's
10
- // compiled dist. Instead, we pre-build Schema objects and pass them directly to Protocol
11
- // methods (toSchema() passes Schema instances through without instanceof checks).
12
9
  import type { FunctionRegistry } from "../functions/registry.js";
13
10
  import type { CatalogInterface } from "../catalog/interface.js";
14
- import { registerFunctionMethods } from "./handlers/function.js";
15
- import { registerAggregateMethods } from "./handlers/aggregate.js";
16
- import { registerTableBufferingMethods } from "./handlers/table-buffering.js";
17
- import { registerCatalogMethods } from "./handlers/catalog/index.js";
18
- import { registerUnimplementedMethods } from "./handlers/unimplemented.js";
11
+ import {
12
+ createVgiProtocol,
13
+ UNIMPLEMENTED_VGI_SERVICE,
14
+ VGI_PROTOCOL_NAME,
15
+ type VgiService,
16
+ } from "../generated/vgi-service.js";
17
+ import { functionHandlers } from "./handlers/function.js";
18
+ import { aggregateHandlers } from "./handlers/aggregate.js";
19
+ import { tableBufferingHandlers } from "./handlers/table-buffering.js";
20
+ import { catalogHandlers } from "./handlers/catalog/index.js";
19
21
 
20
22
  export interface ProtocolConfig {
21
23
  registry: FunctionRegistry;
@@ -41,7 +43,8 @@ export interface ProtocolConfig {
41
43
  * That matters for this consumer specifically: the DuckDB extension ships to
42
44
  * users and cannot be flag-dayed.
43
45
  *
44
- * Declared rather than derived. Until the transports made the routing key
46
+ * Declared rather than derived, and now generated with the method table
47
+ * (src/generated/vgi-service.ts). Until the transports made the routing key
45
48
  * required, each implementation's wire name defaulted to whatever its local
46
49
  * type was called, which left the six ports disagreeing four ways — Python
47
50
  * `VgiProtocol`, Java and C# `VgiService`, Go the framework default `Service`,
@@ -55,15 +58,16 @@ export interface ProtocolConfig {
55
58
  * so the HTTP entry points build that scope from the same constant the
56
59
  * protocol is registered under rather than a second copy of the literal.
57
60
  */
58
- export const VGI_PROTOCOL_NAME = "vgi.v2";
61
+ export { VGI_PROTOCOL_NAME };
59
62
 
60
63
  /**
61
64
  * The `vgi.v2` protocol hash `vgi_rpc.Reflection.v1` must report for this
62
65
  * worker: the reference's (vgi-python 0.43.0, 72 methods). Every SDK hosts
63
66
  * every `vgi.v2` method with the reference's schemas, so the hash is one value
64
67
  * across the ports; a method this SDK does not implement is still registered
65
- * (handlers/unimplemented.ts). The hash covers method names, types and the
66
- * params/result/header schemas only (WIRE_PROTOCOL.md §14).
68
+ * with the generated UNIMPLEMENTED default (src/generated/vgi-service.ts). The
69
+ * hash covers method names, types and the params/result/header schemas only
70
+ * (WIRE_PROTOCOL.md §14).
67
71
  *
68
72
  * It changes only with vgi.v2's protocol version (currently 2.1.0). Asserted
69
73
  * by src/protocol/__tests__/vgi-v2-hash.test.ts; if that test fails, the
@@ -71,18 +75,25 @@ export const VGI_PROTOCOL_NAME = "vgi.v2";
71
75
  */
72
76
  export const VGI_V2_PROTOCOL_HASH = "774cb80090d71ea76d09aa311b9cda4ca4c33c3bf72c43242eb6dc87b6f79ce5";
73
77
 
74
- export function buildVgiProtocol(config: ProtocolConfig): Protocol {
75
- const protocol = new Protocol(VGI_PROTOCOL_NAME, { protocolVersion: "2.1.0" });
76
-
77
- registerFunctionMethods(protocol, {
78
- registry: config.registry,
79
- signingKey: config.signingKey,
80
- catalogInterface: config.catalogInterface,
81
- });
82
- registerAggregateMethods(protocol, config.registry);
83
- registerTableBufferingMethods(protocol, config.registry, config.signingKey);
84
- registerCatalogMethods(protocol, config.catalogInterface, config.catalogName, config.signingKey);
85
- registerUnimplementedMethods(protocol);
78
+ /**
79
+ * This worker's `vgi.v2` implementation: the generated UNIMPLEMENTED default
80
+ * for every method, overridden by the handler groups for what this SDK serves.
81
+ */
82
+ export function buildVgiService(config: ProtocolConfig): VgiService {
83
+ return {
84
+ ...UNIMPLEMENTED_VGI_SERVICE,
85
+ ...functionHandlers({
86
+ registry: config.registry,
87
+ signingKey: config.signingKey,
88
+ catalogInterface: config.catalogInterface,
89
+ }),
90
+ ...aggregateHandlers(config.registry),
91
+ ...tableBufferingHandlers(config.registry, config.signingKey),
92
+ ...catalogHandlers(config.catalogInterface, config.signingKey),
93
+ };
94
+ }
86
95
 
87
- return protocol;
96
+ /** The `vgi.v2` protocol, registered from the generated table with {@link buildVgiService}. */
97
+ export function buildVgiProtocol(config: ProtocolConfig): Protocol {
98
+ return createVgiProtocol(buildVgiService(config));
88
99
  }
@@ -14,19 +14,14 @@ import {
14
14
  serializeBatch,
15
15
  deserializeBatch,
16
16
  } from "../../arrow/index.js";
17
- import { Protocol } from "@query-farm/vgi-rpc";
17
+ import type { VgiService } from "../../generated/vgi-service.js";
18
18
  import type { FunctionRegistry } from "../../functions/registry.js";
19
19
  import { deserializeArguments } from "../serialize.js";
20
20
  import { deserializeSchema, serializeSchema, batchToSecretDict } from "../../util/arrow/index.js";
21
21
  import { toUint8Array } from "../../util/bytes.js";
22
22
  import {
23
- AggregateBindParamsSchema,
24
23
  AggregateBindResultSchema,
25
- AggregateCombineParamsSchema,
26
- AggregateDestructorParamsSchema,
27
- AggregateFinalizeParamsSchema,
28
24
  AggregateFinalizeResultSchema,
29
- AggregateUpdateParamsSchema,
30
25
  } from "../../generated/vgi-protocol-schemas.js";
31
26
  import {
32
27
  getExecutionState,
@@ -38,14 +33,10 @@ import {
38
33
  } from "../../functions/aggregate.js";
39
34
  import { Arguments } from "../../arguments/arguments.js";
40
35
  import { validateConstConstraints } from "../../arguments/argument-spec.js";
41
- import {
42
- REQUEST_PARAMS_SCHEMA,
43
- RESULT_BINARY_SCHEMA,
44
- unwrapRequest,
45
- wrapResult,
46
- } from "./shared.js";
36
+ import { unwrapRequest, wrapResult } from "./shared.js";
47
37
 
48
- export function registerAggregateMethods(protocol: Protocol, registry: FunctionRegistry): void {
38
+ /** The aggregate lifecycle's `vgi.v2` handlers. */
39
+ export function aggregateHandlers(registry: FunctionRegistry): Partial<VgiService> {
49
40
  // `schemaPath` scopes resolution to the schema the caller named. Every
50
41
  // aggregate RPC carries it (protocol 1.2.0), so an aggregate name declared in
51
42
  // two schemas dispatches to the one the request names rather than colliding
@@ -69,218 +60,202 @@ export function registerAggregateMethods(protocol: Protocol, registry: FunctionR
69
60
  // ------------------------------------------------------------------------
70
61
  // aggregate_bind — allocate execution_id, return output schema
71
62
  // ------------------------------------------------------------------------
72
- protocol.unary("aggregate_bind", {
73
- params: AggregateBindParamsSchema,
74
- result: RESULT_BINARY_SCHEMA,
75
- handler: async (params) => {
76
- const innerParams = unwrapRequest(params.request);
77
- const functionName: string = innerParams.function_name;
78
- const cfg = resolveAggregate(functionName, innerParams.schema_path ?? null);
79
- const argBytes = toUint8Array(innerParams.arguments);
80
- const args = deserializeArguments(argBytes);
81
- const inputSchema = innerParams.input_schema
82
- ? deserializeSchema(toUint8Array(innerParams.input_schema))
83
- : null;
84
- // Decode the resolved-secrets batch the extension pre-resolves for
85
- // aggregates that declared `requiredSecrets`. Mirrors the table-buffering
86
- // handler (batchToSecretDict). The secret VALUES are delivered only here,
87
- // at bind time — update/combine/finalize see no secrets by design.
88
- const secretsBytes = innerParams.secrets ? toUint8Array(innerParams.secrets) : null;
89
- const secretsBatch =
90
- secretsBytes && secretsBytes.length > 0 ? deserializeBatch(secretsBytes) : null;
91
- const secrets = secretsBatch ? batchToSecretDict(secretsBatch) : {};
92
- const bindParams: AggregateBindParams<any> = {
93
- args: extractArgMap(cfg, args),
94
- arguments: args,
95
- inputSchema,
96
- settings: {},
97
- secrets,
98
- argumentNames: innerParams.argument_names
99
- ? (Array.isArray(innerParams.argument_names)
100
- ? innerParams.argument_names
101
- : [...innerParams.argument_names])
102
- .map((value: unknown) => value == null ? null : String(value))
103
- : null,
104
- };
105
- const executionId = new Uint8Array(16);
106
- crypto.getRandomValues(executionId);
107
- setExecutionState(executionId, {
108
- states: new Map(),
109
- bindArgs: args,
110
- bindParams,
111
- });
112
- // Output schema is a single "result" column. Type comes from the
113
- // optional onBind hook (for aggregates whose return type depends on
114
- // input — e.g. vgi_generic_sum takes ANY and returns the same type);
115
- // otherwise the declared cfg.outputType.
116
- const outType = cfg.onBind ? await cfg.onBind(bindParams) : cfg.outputType;
117
- const outSchema = makeSchema([field("result", outType, true)]);
118
- return wrapResult(
119
- { output_schema: serializeSchema(outSchema), execution_id: executionId },
120
- AggregateBindResultSchema,
121
- );
122
- },
123
- });
63
+ const aggregateBind: VgiService["aggregateBind"] = async (params) => {
64
+ const innerParams = unwrapRequest(params.request);
65
+ const functionName: string = innerParams.function_name;
66
+ const cfg = resolveAggregate(functionName, innerParams.schema_path ?? null);
67
+ const argBytes = toUint8Array(innerParams.arguments);
68
+ const args = deserializeArguments(argBytes);
69
+ const inputSchema = innerParams.input_schema
70
+ ? deserializeSchema(toUint8Array(innerParams.input_schema))
71
+ : null;
72
+ // Decode the resolved-secrets batch the extension pre-resolves for
73
+ // aggregates that declared `requiredSecrets`. Mirrors the table-buffering
74
+ // handler (batchToSecretDict). The secret VALUES are delivered only here,
75
+ // at bind time — update/combine/finalize see no secrets by design.
76
+ const secretsBytes = innerParams.secrets ? toUint8Array(innerParams.secrets) : null;
77
+ const secretsBatch =
78
+ secretsBytes && secretsBytes.length > 0 ? deserializeBatch(secretsBytes) : null;
79
+ const secrets = secretsBatch ? batchToSecretDict(secretsBatch) : {};
80
+ const bindParams: AggregateBindParams<any> = {
81
+ args: extractArgMap(cfg, args),
82
+ arguments: args,
83
+ inputSchema,
84
+ settings: {},
85
+ secrets,
86
+ argumentNames: innerParams.argument_names
87
+ ? (Array.isArray(innerParams.argument_names)
88
+ ? innerParams.argument_names
89
+ : [...innerParams.argument_names])
90
+ .map((value: unknown) => value == null ? null : String(value))
91
+ : null,
92
+ };
93
+ const executionId = new Uint8Array(16);
94
+ crypto.getRandomValues(executionId);
95
+ setExecutionState(executionId, {
96
+ states: new Map(),
97
+ bindArgs: args,
98
+ bindParams,
99
+ });
100
+ // Output schema is a single "result" column. Type comes from the
101
+ // optional onBind hook (for aggregates whose return type depends on
102
+ // input — e.g. vgi_generic_sum takes ANY and returns the same type);
103
+ // otherwise the declared cfg.outputType.
104
+ const outType = cfg.onBind ? await cfg.onBind(bindParams) : cfg.outputType;
105
+ const outSchema = makeSchema([field("result", outType, true)]);
106
+ return wrapResult(
107
+ { output_schema: serializeSchema(outSchema), execution_id: executionId },
108
+ AggregateBindResultSchema,
109
+ );
110
+ };
124
111
 
125
112
  // ------------------------------------------------------------------------
126
113
  // aggregate_update — fold the incoming batch into per-group state
127
114
  // ------------------------------------------------------------------------
128
- protocol.unary("aggregate_update", {
129
- params: AggregateUpdateParamsSchema,
130
- result: RESULT_BINARY_SCHEMA,
131
- handler: (params) => {
132
- const innerParams = unwrapRequest(params.request);
133
- const functionName: string = innerParams.function_name;
134
- const executionId = toUint8Array(innerParams.execution_id);
135
- const cfg = resolveAggregate(functionName, innerParams.schema_path ?? null);
136
- const batch = readSingleBatch(toUint8Array(innerParams.input_batch));
137
- if (!batch || batch.numRows === 0) return wrapResult({}, makeSchema([]));
115
+ const aggregateUpdate: VgiService["aggregateUpdate"] = (params) => {
116
+ const innerParams = unwrapRequest(params.request);
117
+ const functionName: string = innerParams.function_name;
118
+ const executionId = toUint8Array(innerParams.execution_id);
119
+ const cfg = resolveAggregate(functionName, innerParams.schema_path ?? null);
120
+ const batch = readSingleBatch(toUint8Array(innerParams.input_batch));
121
+ if (!batch || batch.numRows === 0) return wrapResult({}, makeSchema([]));
138
122
 
139
- const gidIdx = batch.schema.fields.findIndex((f) => f.name === GROUP_COLUMN_NAME);
140
- if (gidIdx < 0) {
141
- throw new Error(`aggregate_update: input batch missing ${GROUP_COLUMN_NAME} column`);
142
- }
143
- const gidCol = batch.getChildAt(gidIdx)!;
144
- const groupIds: bigint[] = [];
145
- const uniqueGids = new Set<bigint>();
146
- for (let i = 0; i < batch.numRows; i++) {
147
- const v = gidCol.get(i);
148
- const g = typeof v === "bigint" ? v : BigInt(v as any);
149
- groupIds.push(g);
150
- uniqueGids.add(g);
151
- }
152
- const exec = getExecutionState(executionId, uniqueGids);
153
- if (!exec) {
154
- throw new Error(`aggregate_update: unknown execution_id for '${functionName}' (bind missing or state dropped)`);
155
- }
123
+ const gidIdx = batch.schema.fields.findIndex((f) => f.name === GROUP_COLUMN_NAME);
124
+ if (gidIdx < 0) {
125
+ throw new Error(`aggregate_update: input batch missing ${GROUP_COLUMN_NAME} column`);
126
+ }
127
+ const gidCol = batch.getChildAt(gidIdx)!;
128
+ const groupIds: bigint[] = [];
129
+ const uniqueGids = new Set<bigint>();
130
+ for (let i = 0; i < batch.numRows; i++) {
131
+ const v = gidCol.get(i);
132
+ const g = typeof v === "bigint" ? v : BigInt(v as any);
133
+ groupIds.push(g);
134
+ uniqueGids.add(g);
135
+ }
136
+ const exec = getExecutionState(executionId, uniqueGids);
137
+ if (!exec) {
138
+ throw new Error(`aggregate_update: unknown execution_id for '${functionName}' (bind missing or state dropped)`);
139
+ }
156
140
 
157
- // Build column-arg arrays — every non-__vgi_group_id column, in
158
- // schema order — and pass to user's update(). We deliberately do NOT
159
- // pre-allocate state for every group_id seen — the user decides when
160
- // to call ensureState() so that groups with only-NULL input (skipped
161
- // rows) never get an entry, which lets finalize() return SQL NULL.
162
- const columns: any[] = [];
163
- for (let i = 0; i < batch.schema.fields.length; i++) {
164
- if (i === gidIdx) continue;
165
- columns.push(batch.getChildAt(i));
141
+ // Build column-arg arrays — every non-__vgi_group_id column, in
142
+ // schema order — and pass to user's update(). We deliberately do NOT
143
+ // pre-allocate state for every group_id seen — the user decides when
144
+ // to call ensureState() so that groups with only-NULL input (skipped
145
+ // rows) never get an entry, which lets finalize() return SQL NULL.
146
+ const columns: any[] = [];
147
+ for (let i = 0; i < batch.schema.fields.length; i++) {
148
+ if (i === gidIdx) continue;
149
+ columns.push(batch.getChildAt(i));
150
+ }
151
+ const ensureState = (gid: bigint) => {
152
+ let s = exec.states.get(gid);
153
+ if (s === undefined) {
154
+ s = cfg.initialState(exec.bindParams);
155
+ exec.states.set(gid, s);
166
156
  }
167
- const ensureState = (gid: bigint) => {
168
- let s = exec.states.get(gid);
169
- if (s === undefined) {
170
- s = cfg.initialState(exec.bindParams);
171
- exec.states.set(gid, s);
172
- }
173
- return s;
174
- };
175
- cfg.update({
176
- states: exec.states,
177
- groupIds,
178
- columns,
179
- args: exec.bindParams.args,
180
- ensureState,
181
- });
182
- // Flush mutated states to disk so sibling workers see them. We persist
183
- // every group that was referenced this call — some may be unchanged
184
- // (skipped by NULL handling), in which case the file simply reflects
185
- // the pre-existing initial state; the idempotent rewrite is cheap
186
- // relative to the RPC overhead.
187
- persistGroupStates(executionId, uniqueGids);
188
- return wrapResult({}, makeSchema([]));
189
- },
190
- });
157
+ return s;
158
+ };
159
+ cfg.update({
160
+ states: exec.states,
161
+ groupIds,
162
+ columns,
163
+ args: exec.bindParams.args,
164
+ ensureState,
165
+ });
166
+ // Flush mutated states to disk so sibling workers see them. We persist
167
+ // every group that was referenced this call — some may be unchanged
168
+ // (skipped by NULL handling), in which case the file simply reflects
169
+ // the pre-existing initial state; the idempotent rewrite is cheap
170
+ // relative to the RPC overhead.
171
+ persistGroupStates(executionId, uniqueGids);
172
+ return wrapResult({}, makeSchema([]));
173
+ };
191
174
 
192
175
  // ------------------------------------------------------------------------
193
176
  // aggregate_combine — merge (source_group_id, target_group_id) pairs
194
177
  // ------------------------------------------------------------------------
195
- protocol.unary("aggregate_combine", {
196
- params: AggregateCombineParamsSchema,
197
- result: RESULT_BINARY_SCHEMA,
198
- handler: (params) => {
199
- const innerParams = unwrapRequest(params.request);
200
- const functionName: string = innerParams.function_name;
201
- const executionId = toUint8Array(innerParams.execution_id);
202
- const cfg = resolveAggregate(functionName, innerParams.schema_path ?? null);
203
- const batch = readSingleBatch(toUint8Array(innerParams.merge_batch));
204
- if (!batch || batch.numRows === 0) return wrapResult({}, makeSchema([]));
178
+ const aggregateCombine: VgiService["aggregateCombine"] = (params) => {
179
+ const innerParams = unwrapRequest(params.request);
180
+ const functionName: string = innerParams.function_name;
181
+ const executionId = toUint8Array(innerParams.execution_id);
182
+ const cfg = resolveAggregate(functionName, innerParams.schema_path ?? null);
183
+ const batch = readSingleBatch(toUint8Array(innerParams.merge_batch));
184
+ if (!batch || batch.numRows === 0) return wrapResult({}, makeSchema([]));
205
185
 
206
- const srcCol = batch.getChild("source_group_id")!;
207
- const tgtCol = batch.getChild("target_group_id")!;
208
- const referencedGids = new Set<bigint>();
209
- const srcIds: bigint[] = [];
210
- const tgtIds: bigint[] = [];
211
- for (let i = 0; i < batch.numRows; i++) {
212
- const src = BigInt(srcCol.get(i) as any);
213
- const tgt = BigInt(tgtCol.get(i) as any);
214
- srcIds.push(src); tgtIds.push(tgt);
215
- referencedGids.add(src); referencedGids.add(tgt);
216
- }
217
- const exec = getExecutionState(executionId, referencedGids);
218
- if (!exec) {
219
- throw new Error(
220
- `aggregate_combine: unknown execution_id for '${functionName}' ` +
221
- `(context missing on disk — bind may have failed or been cleaned up)`,
222
- );
223
- }
224
- const touchedTargets = new Set<bigint>();
225
- for (let i = 0; i < batch.numRows; i++) {
226
- const src = srcIds[i];
227
- const tgt = tgtIds[i];
228
- const srcHas = exec.states.has(src);
229
- const tgtHas = exec.states.has(tgt);
230
- if (!srcHas && !tgtHas) continue;
231
- const srcState = srcHas ? exec.states.get(src)! : cfg.initialState(exec.bindParams);
232
- const tgtState = tgtHas ? exec.states.get(tgt)! : cfg.initialState(exec.bindParams);
233
- exec.states.set(tgt, cfg.combine(srcState, tgtState, exec.bindParams));
234
- touchedTargets.add(tgt);
235
- }
236
- persistGroupStates(executionId, touchedTargets);
237
- return wrapResult({}, makeSchema([]));
238
- },
239
- });
186
+ const srcCol = batch.getChild("source_group_id")!;
187
+ const tgtCol = batch.getChild("target_group_id")!;
188
+ const referencedGids = new Set<bigint>();
189
+ const srcIds: bigint[] = [];
190
+ const tgtIds: bigint[] = [];
191
+ for (let i = 0; i < batch.numRows; i++) {
192
+ const src = BigInt(srcCol.get(i) as any);
193
+ const tgt = BigInt(tgtCol.get(i) as any);
194
+ srcIds.push(src); tgtIds.push(tgt);
195
+ referencedGids.add(src); referencedGids.add(tgt);
196
+ }
197
+ const exec = getExecutionState(executionId, referencedGids);
198
+ if (!exec) {
199
+ throw new Error(
200
+ `aggregate_combine: unknown execution_id for '${functionName}' ` +
201
+ `(context missing on disk — bind may have failed or been cleaned up)`,
202
+ );
203
+ }
204
+ const touchedTargets = new Set<bigint>();
205
+ for (let i = 0; i < batch.numRows; i++) {
206
+ const src = srcIds[i];
207
+ const tgt = tgtIds[i];
208
+ const srcHas = exec.states.has(src);
209
+ const tgtHas = exec.states.has(tgt);
210
+ if (!srcHas && !tgtHas) continue;
211
+ const srcState = srcHas ? exec.states.get(src)! : cfg.initialState(exec.bindParams);
212
+ const tgtState = tgtHas ? exec.states.get(tgt)! : cfg.initialState(exec.bindParams);
213
+ exec.states.set(tgt, cfg.combine(srcState, tgtState, exec.bindParams));
214
+ touchedTargets.add(tgt);
215
+ }
216
+ persistGroupStates(executionId, touchedTargets);
217
+ return wrapResult({}, makeSchema([]));
218
+ };
240
219
 
241
220
  // ------------------------------------------------------------------------
242
221
  // aggregate_finalize — emit one result row per requested group_id
243
222
  // ------------------------------------------------------------------------
244
- protocol.unary("aggregate_finalize", {
245
- params: AggregateFinalizeParamsSchema,
246
- result: RESULT_BINARY_SCHEMA,
247
- handler: (params) => {
248
- const innerParams = unwrapRequest(params.request);
249
- const functionName: string = innerParams.function_name;
250
- const executionId = toUint8Array(innerParams.execution_id);
251
- const cfg = resolveAggregate(functionName, innerParams.schema_path ?? null);
252
- const outputSchema = deserializeSchema(toUint8Array(innerParams.output_schema));
253
- const gidBatch = readSingleBatch(toUint8Array(innerParams.group_ids_batch));
254
- const groupIds: bigint[] = [];
255
- const uniqueGids = new Set<bigint>();
256
- if (gidBatch) {
257
- const col = gidBatch.getChild("group_id")!;
258
- for (let i = 0; i < gidBatch.numRows; i++) {
259
- const v = col.get(i);
260
- const g = typeof v === "bigint" ? v : BigInt(v as any);
261
- groupIds.push(g);
262
- uniqueGids.add(g);
263
- }
264
- }
265
- const exec = getExecutionState(executionId, uniqueGids);
266
- if (!exec) {
267
- throw new Error(`aggregate_finalize: unknown execution_id for '${functionName}'`);
268
- }
269
- // Groups never updated pass state = null so finalize() can emit SQL NULL
270
- // (matches Python semantics for SUM/AVG/MIN/MAX over zero rows).
271
- const states = new Map<bigint, any | null>();
272
- for (const gid of groupIds) {
273
- states.set(gid, exec.states.has(gid) ? exec.states.get(gid) : null);
223
+ const aggregateFinalize: VgiService["aggregateFinalize"] = (params) => {
224
+ const innerParams = unwrapRequest(params.request);
225
+ const functionName: string = innerParams.function_name;
226
+ const executionId = toUint8Array(innerParams.execution_id);
227
+ const cfg = resolveAggregate(functionName, innerParams.schema_path ?? null);
228
+ const outputSchema = deserializeSchema(toUint8Array(innerParams.output_schema));
229
+ const gidBatch = readSingleBatch(toUint8Array(innerParams.group_ids_batch));
230
+ const groupIds: bigint[] = [];
231
+ const uniqueGids = new Set<bigint>();
232
+ if (gidBatch) {
233
+ const col = gidBatch.getChild("group_id")!;
234
+ for (let i = 0; i < gidBatch.numRows; i++) {
235
+ const v = col.get(i);
236
+ const g = typeof v === "bigint" ? v : BigInt(v as any);
237
+ groupIds.push(g);
238
+ uniqueGids.add(g);
274
239
  }
275
- const resultBatch = cfg.finalize({
276
- groupIds, states, outputSchema, args: exec.bindParams.args,
277
- });
278
- return wrapResult(
279
- { result_batch: serializeBatchWithSchema(resultBatch) },
280
- AggregateFinalizeResultSchema,
281
- );
282
- },
283
- });
240
+ }
241
+ const exec = getExecutionState(executionId, uniqueGids);
242
+ if (!exec) {
243
+ throw new Error(`aggregate_finalize: unknown execution_id for '${functionName}'`);
244
+ }
245
+ // Groups never updated pass state = null so finalize() can emit SQL NULL
246
+ // (matches Python semantics for SUM/AVG/MIN/MAX over zero rows).
247
+ const states = new Map<bigint, any | null>();
248
+ for (const gid of groupIds) {
249
+ states.set(gid, exec.states.has(gid) ? exec.states.get(gid) : null);
250
+ }
251
+ const resultBatch = cfg.finalize({
252
+ groupIds, states, outputSchema, args: exec.bindParams.args,
253
+ });
254
+ return wrapResult(
255
+ { result_batch: serializeBatchWithSchema(resultBatch) },
256
+ AggregateFinalizeResultSchema,
257
+ );
258
+ };
284
259
 
285
260
  // ------------------------------------------------------------------------
286
261
  // aggregate_destructor — drop state for finished groups (best-effort)
@@ -291,30 +266,28 @@ export function registerAggregateMethods(protocol: Protocol, registry: FunctionR
291
266
  // (matches Python's conservative behavior; the execution entry is cleaned
292
267
  // up when the worker process exits).
293
268
  // ------------------------------------------------------------------------
294
- protocol.unary("aggregate_destructor", {
295
- params: AggregateDestructorParamsSchema,
296
- result: RESULT_BINARY_SCHEMA,
297
- handler: (params) => {
298
- const innerParams = unwrapRequest(params.request);
299
- const executionId = toUint8Array(innerParams.execution_id);
300
- const exec = getExecutionState(executionId);
301
- if (!exec) return wrapResult({}, makeSchema([]));
302
- const gidBatch = readSingleBatch(toUint8Array(innerParams.group_ids_batch));
303
- if (!gidBatch || gidBatch.numRows === 0) return wrapResult({}, makeSchema([]));
304
- const col = gidBatch.getChild("group_id")!;
305
- const toDrop = new Set<bigint>();
306
- for (let i = 0; i < gidBatch.numRows; i++) {
307
- const v = col.get(i);
308
- const g = typeof v === "bigint" ? v : BigInt(v as any);
309
- exec.states.delete(g);
310
- toDrop.add(g);
311
- }
312
- // Persist deletions (unlinks the per-group file so sibling workers
313
- // don't revive the state via getExecutionState's lazy disk load).
314
- persistGroupStates(executionId, toDrop);
315
- return wrapResult({}, makeSchema([]));
316
- },
317
- });
269
+ const aggregateDestructor: VgiService["aggregateDestructor"] = (params) => {
270
+ const innerParams = unwrapRequest(params.request);
271
+ const executionId = toUint8Array(innerParams.execution_id);
272
+ const exec = getExecutionState(executionId);
273
+ if (!exec) return wrapResult({}, makeSchema([]));
274
+ const gidBatch = readSingleBatch(toUint8Array(innerParams.group_ids_batch));
275
+ if (!gidBatch || gidBatch.numRows === 0) return wrapResult({}, makeSchema([]));
276
+ const col = gidBatch.getChild("group_id")!;
277
+ const toDrop = new Set<bigint>();
278
+ for (let i = 0; i < gidBatch.numRows; i++) {
279
+ const v = col.get(i);
280
+ const g = typeof v === "bigint" ? v : BigInt(v as any);
281
+ exec.states.delete(g);
282
+ toDrop.add(g);
283
+ }
284
+ // Persist deletions (unlinks the per-group file so sibling workers
285
+ // don't revive the state via getExecutionState's lazy disk load).
286
+ persistGroupStates(executionId, toDrop);
287
+ return wrapResult({}, makeSchema([]));
288
+ };
289
+
290
+ return { aggregateBind, aggregateUpdate, aggregateCombine, aggregateFinalize, aggregateDestructor };
318
291
  }
319
292
 
320
293
  function extractArgMap(