@query-farm/vgi 0.42.0 → 0.44.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +5 -2
- package/dist/attach-ticket.d.ts +18 -7
- package/dist/client-entry.js +215 -91
- package/dist/client-entry.js.map +10 -9
- package/dist/codec/asd.d.ts +3 -1
- package/dist/codec/conform.d.ts +51 -0
- package/dist/errors.d.ts +5 -1
- package/dist/generated/vgi-service.d.ts +1315 -0
- package/dist/index.core.d.ts +1 -1
- package/dist/index.js +2143 -932
- package/dist/index.js.map +29 -28
- package/dist/protocol/dispatch.d.ts +14 -5
- package/dist/protocol/handlers/aggregate.d.ts +3 -2
- package/dist/protocol/handlers/catalog/admin.d.ts +2 -2
- package/dist/protocol/handlers/catalog/index.d.ts +3 -2
- package/dist/protocol/handlers/catalog/index_methods.d.ts +2 -2
- package/dist/protocol/handlers/catalog/macro.d.ts +2 -2
- package/dist/protocol/handlers/catalog/shared.d.ts +7 -14
- package/dist/protocol/handlers/catalog/table.d.ts +2 -2
- package/dist/protocol/handlers/catalog/view.d.ts +2 -2
- package/dist/protocol/handlers/function.d.ts +3 -2
- package/dist/protocol/handlers/shared.d.ts +2 -2
- package/dist/protocol/handlers/table-buffering.d.ts +3 -2
- package/dist/protocol/handlers/unimplemented.d.ts +4 -17
- package/dist/protocol/serializers/shared.d.ts +2 -1
- package/dist/rpc-server.d.ts +2 -2
- package/dist/serve-entry.js +2104 -903
- package/dist/serve-entry.js.map +25 -24
- package/dist/worker-cf-entry.js +2045 -842
- package/dist/worker-cf-entry.js.map +29 -28
- package/package.json +3 -3
- package/src/attach-ticket.ts +41 -13
- package/src/catalog/composite.ts +2 -1
- package/src/catalog/interface.ts +5 -4
- package/src/catalog/read-only.ts +8 -6
- package/src/client/client.ts +13 -5
- package/src/codec/asd.ts +5 -2
- package/src/codec/conform.ts +193 -0
- package/src/errors.ts +27 -7
- package/src/functions/registry.ts +3 -2
- package/src/generated/vgi-service.ts +2678 -0
- package/src/http/fetch.ts +1 -1
- package/src/index.core.ts +3 -1
- package/src/protocol/dispatch.ts +40 -29
- package/src/protocol/handlers/aggregate.ts +205 -232
- package/src/protocol/handlers/catalog/admin.ts +228 -299
- package/src/protocol/handlers/catalog/index.ts +17 -16
- package/src/protocol/handlers/catalog/index_methods.ts +26 -36
- package/src/protocol/handlers/catalog/macro.ts +54 -77
- package/src/protocol/handlers/catalog/shared.ts +30 -47
- package/src/protocol/handlers/catalog/table.ts +205 -270
- package/src/protocol/handlers/catalog/view.ts +59 -85
- package/src/protocol/handlers/function.ts +127 -156
- package/src/protocol/handlers/shared.ts +8 -6
- package/src/protocol/handlers/table-buffering.ts +53 -71
- package/src/protocol/handlers/unimplemented.ts +18 -62
- package/src/protocol/serializers/bind.ts +3 -1
- package/src/protocol/serializers/shared.ts +5 -2
- package/src/rpc-server.ts +7 -7
package/src/http/fetch.ts
CHANGED
|
@@ -207,7 +207,7 @@ export function createVgiFetch(opts: VgiFetchOptions): (req: Request) => Promise
|
|
|
207
207
|
|
|
208
208
|
// The same builder every other transport uses: `vgi.v2`, the worker's
|
|
209
209
|
// hostedProtocols(), reflection, and -- HTTP being the transport that
|
|
210
|
-
// authenticates callers -- vgi_rpc.Identity.
|
|
210
|
+
// authenticates callers -- vgi_rpc.Identity.v2 when the worker opts in.
|
|
211
211
|
// Built at construction, so a worker that opts into introspection without an
|
|
212
212
|
// allowlist fails here rather than serving traffic.
|
|
213
213
|
const server = buildRpcServer(protocol, opts, {
|
package/src/index.core.ts
CHANGED
|
@@ -186,8 +186,10 @@ export {
|
|
|
186
186
|
AttachTicketInvalidError,
|
|
187
187
|
AttachTicketRequestError,
|
|
188
188
|
SealAttachDeniedError,
|
|
189
|
+
type AttachTicket,
|
|
189
190
|
type AttachTicketClaims,
|
|
190
191
|
type RestoredAttach,
|
|
192
|
+
encodeAttachTicket,
|
|
191
193
|
attachTicketAad,
|
|
192
194
|
mintAttachTicket,
|
|
193
195
|
openAttachTicket,
|
|
@@ -338,7 +340,7 @@ export { str, bytes, int, int32, float, float32, bool, toSchema, OutputCollector
|
|
|
338
340
|
export { subprocessConnect, httpConnect, tcpConnect } from "@query-farm/vgi-rpc";
|
|
339
341
|
export type { RpcClient, StreamSession, LogMessage } from "@query-farm/vgi-rpc";
|
|
340
342
|
|
|
341
|
-
// Hosting additional protocols and opting into vgi_rpc.Identity.
|
|
343
|
+
// Hosting additional protocols and opting into vgi_rpc.Identity.v2. The errors
|
|
342
344
|
// are re-exported so a worker's resolveToken/mintGrant can throw the right one:
|
|
343
345
|
// AuthUnavailableError (or IdentityUnavailableError) for "could not find out",
|
|
344
346
|
// GrantRefusedError to decline a grant.
|
package/src/protocol/dispatch.ts
CHANGED
|
@@ -1,21 +1,23 @@
|
|
|
1
1
|
// Copyright 2025, 2026 Query Farm LLC - https://query.farm
|
|
2
2
|
// Build VGI protocol from worker implementation.
|
|
3
|
-
//
|
|
4
|
-
//
|
|
3
|
+
// The method table, its schemas and the UNIMPLEMENTED defaults are generated
|
|
4
|
+
// (src/generated/vgi-service.ts, from vgi-python's VgiProtocol); this module
|
|
5
|
+
// composes the handler groups in handlers/ over those defaults.
|
|
5
6
|
|
|
6
|
-
import { Protocol } from "@query-farm/vgi-rpc";
|
|
7
|
+
import type { Protocol } from "@query-farm/vgi-rpc";
|
|
7
8
|
|
|
8
|
-
// NOTE: We must NOT use vgi-rpc's str/bytes/int/etc. singletons in Schema objects
|
|
9
|
-
// because Bun loads apache-arrow as separate module instances for our code vs vgi-rpc's
|
|
10
|
-
// compiled dist. Instead, we pre-build Schema objects and pass them directly to Protocol
|
|
11
|
-
// methods (toSchema() passes Schema instances through without instanceof checks).
|
|
12
9
|
import type { FunctionRegistry } from "../functions/registry.js";
|
|
13
10
|
import type { CatalogInterface } from "../catalog/interface.js";
|
|
14
|
-
import {
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
11
|
+
import {
|
|
12
|
+
createVgiProtocol,
|
|
13
|
+
UNIMPLEMENTED_VGI_SERVICE,
|
|
14
|
+
VGI_PROTOCOL_NAME,
|
|
15
|
+
type VgiService,
|
|
16
|
+
} from "../generated/vgi-service.js";
|
|
17
|
+
import { functionHandlers } from "./handlers/function.js";
|
|
18
|
+
import { aggregateHandlers } from "./handlers/aggregate.js";
|
|
19
|
+
import { tableBufferingHandlers } from "./handlers/table-buffering.js";
|
|
20
|
+
import { catalogHandlers } from "./handlers/catalog/index.js";
|
|
19
21
|
|
|
20
22
|
export interface ProtocolConfig {
|
|
21
23
|
registry: FunctionRegistry;
|
|
@@ -41,7 +43,8 @@ export interface ProtocolConfig {
|
|
|
41
43
|
* That matters for this consumer specifically: the DuckDB extension ships to
|
|
42
44
|
* users and cannot be flag-dayed.
|
|
43
45
|
*
|
|
44
|
-
* Declared rather than derived
|
|
46
|
+
* Declared rather than derived, and now generated with the method table
|
|
47
|
+
* (src/generated/vgi-service.ts). Until the transports made the routing key
|
|
45
48
|
* required, each implementation's wire name defaulted to whatever its local
|
|
46
49
|
* type was called, which left the six ports disagreeing four ways — Python
|
|
47
50
|
* `VgiProtocol`, Java and C# `VgiService`, Go the framework default `Service`,
|
|
@@ -55,15 +58,16 @@ export interface ProtocolConfig {
|
|
|
55
58
|
* so the HTTP entry points build that scope from the same constant the
|
|
56
59
|
* protocol is registered under rather than a second copy of the literal.
|
|
57
60
|
*/
|
|
58
|
-
export
|
|
61
|
+
export { VGI_PROTOCOL_NAME };
|
|
59
62
|
|
|
60
63
|
/**
|
|
61
64
|
* The `vgi.v2` protocol hash `vgi_rpc.Reflection.v1` must report for this
|
|
62
65
|
* worker: the reference's (vgi-python 0.43.0, 72 methods). Every SDK hosts
|
|
63
66
|
* every `vgi.v2` method with the reference's schemas, so the hash is one value
|
|
64
67
|
* across the ports; a method this SDK does not implement is still registered
|
|
65
|
-
* (
|
|
66
|
-
* params/result/header schemas only
|
|
68
|
+
* with the generated UNIMPLEMENTED default (src/generated/vgi-service.ts). The
|
|
69
|
+
* hash covers method names, types and the params/result/header schemas only
|
|
70
|
+
* (WIRE_PROTOCOL.md §14).
|
|
67
71
|
*
|
|
68
72
|
* It changes only with vgi.v2's protocol version (currently 2.1.0). Asserted
|
|
69
73
|
* by src/protocol/__tests__/vgi-v2-hash.test.ts; if that test fails, the
|
|
@@ -71,18 +75,25 @@ export const VGI_PROTOCOL_NAME = "vgi.v2";
|
|
|
71
75
|
*/
|
|
72
76
|
export const VGI_V2_PROTOCOL_HASH = "774cb80090d71ea76d09aa311b9cda4ca4c33c3bf72c43242eb6dc87b6f79ce5";
|
|
73
77
|
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
78
|
+
/**
|
|
79
|
+
* This worker's `vgi.v2` implementation: the generated UNIMPLEMENTED default
|
|
80
|
+
* for every method, overridden by the handler groups for what this SDK serves.
|
|
81
|
+
*/
|
|
82
|
+
export function buildVgiService(config: ProtocolConfig): VgiService {
|
|
83
|
+
return {
|
|
84
|
+
...UNIMPLEMENTED_VGI_SERVICE,
|
|
85
|
+
...functionHandlers({
|
|
86
|
+
registry: config.registry,
|
|
87
|
+
signingKey: config.signingKey,
|
|
88
|
+
catalogInterface: config.catalogInterface,
|
|
89
|
+
}),
|
|
90
|
+
...aggregateHandlers(config.registry),
|
|
91
|
+
...tableBufferingHandlers(config.registry, config.signingKey),
|
|
92
|
+
...catalogHandlers(config.catalogInterface, config.signingKey),
|
|
93
|
+
};
|
|
94
|
+
}
|
|
86
95
|
|
|
87
|
-
|
|
96
|
+
/** The `vgi.v2` protocol, registered from the generated table with {@link buildVgiService}. */
|
|
97
|
+
export function buildVgiProtocol(config: ProtocolConfig): Protocol {
|
|
98
|
+
return createVgiProtocol(buildVgiService(config));
|
|
88
99
|
}
|
|
@@ -14,19 +14,14 @@ import {
|
|
|
14
14
|
serializeBatch,
|
|
15
15
|
deserializeBatch,
|
|
16
16
|
} from "../../arrow/index.js";
|
|
17
|
-
import {
|
|
17
|
+
import type { VgiService } from "../../generated/vgi-service.js";
|
|
18
18
|
import type { FunctionRegistry } from "../../functions/registry.js";
|
|
19
19
|
import { deserializeArguments } from "../serialize.js";
|
|
20
20
|
import { deserializeSchema, serializeSchema, batchToSecretDict } from "../../util/arrow/index.js";
|
|
21
21
|
import { toUint8Array } from "../../util/bytes.js";
|
|
22
22
|
import {
|
|
23
|
-
AggregateBindParamsSchema,
|
|
24
23
|
AggregateBindResultSchema,
|
|
25
|
-
AggregateCombineParamsSchema,
|
|
26
|
-
AggregateDestructorParamsSchema,
|
|
27
|
-
AggregateFinalizeParamsSchema,
|
|
28
24
|
AggregateFinalizeResultSchema,
|
|
29
|
-
AggregateUpdateParamsSchema,
|
|
30
25
|
} from "../../generated/vgi-protocol-schemas.js";
|
|
31
26
|
import {
|
|
32
27
|
getExecutionState,
|
|
@@ -38,14 +33,10 @@ import {
|
|
|
38
33
|
} from "../../functions/aggregate.js";
|
|
39
34
|
import { Arguments } from "../../arguments/arguments.js";
|
|
40
35
|
import { validateConstConstraints } from "../../arguments/argument-spec.js";
|
|
41
|
-
import {
|
|
42
|
-
REQUEST_PARAMS_SCHEMA,
|
|
43
|
-
RESULT_BINARY_SCHEMA,
|
|
44
|
-
unwrapRequest,
|
|
45
|
-
wrapResult,
|
|
46
|
-
} from "./shared.js";
|
|
36
|
+
import { unwrapRequest, wrapResult } from "./shared.js";
|
|
47
37
|
|
|
48
|
-
|
|
38
|
+
/** The aggregate lifecycle's `vgi.v2` handlers. */
|
|
39
|
+
export function aggregateHandlers(registry: FunctionRegistry): Partial<VgiService> {
|
|
49
40
|
// `schemaPath` scopes resolution to the schema the caller named. Every
|
|
50
41
|
// aggregate RPC carries it (protocol 1.2.0), so an aggregate name declared in
|
|
51
42
|
// two schemas dispatches to the one the request names rather than colliding
|
|
@@ -69,218 +60,202 @@ export function registerAggregateMethods(protocol: Protocol, registry: FunctionR
|
|
|
69
60
|
// ------------------------------------------------------------------------
|
|
70
61
|
// aggregate_bind — allocate execution_id, return output schema
|
|
71
62
|
// ------------------------------------------------------------------------
|
|
72
|
-
|
|
73
|
-
params
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
? (
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
AggregateBindResultSchema,
|
|
121
|
-
);
|
|
122
|
-
},
|
|
123
|
-
});
|
|
63
|
+
const aggregateBind: VgiService["aggregateBind"] = async (params) => {
|
|
64
|
+
const innerParams = unwrapRequest(params.request);
|
|
65
|
+
const functionName: string = innerParams.function_name;
|
|
66
|
+
const cfg = resolveAggregate(functionName, innerParams.schema_path ?? null);
|
|
67
|
+
const argBytes = toUint8Array(innerParams.arguments);
|
|
68
|
+
const args = deserializeArguments(argBytes);
|
|
69
|
+
const inputSchema = innerParams.input_schema
|
|
70
|
+
? deserializeSchema(toUint8Array(innerParams.input_schema))
|
|
71
|
+
: null;
|
|
72
|
+
// Decode the resolved-secrets batch the extension pre-resolves for
|
|
73
|
+
// aggregates that declared `requiredSecrets`. Mirrors the table-buffering
|
|
74
|
+
// handler (batchToSecretDict). The secret VALUES are delivered only here,
|
|
75
|
+
// at bind time — update/combine/finalize see no secrets by design.
|
|
76
|
+
const secretsBytes = innerParams.secrets ? toUint8Array(innerParams.secrets) : null;
|
|
77
|
+
const secretsBatch =
|
|
78
|
+
secretsBytes && secretsBytes.length > 0 ? deserializeBatch(secretsBytes) : null;
|
|
79
|
+
const secrets = secretsBatch ? batchToSecretDict(secretsBatch) : {};
|
|
80
|
+
const bindParams: AggregateBindParams<any> = {
|
|
81
|
+
args: extractArgMap(cfg, args),
|
|
82
|
+
arguments: args,
|
|
83
|
+
inputSchema,
|
|
84
|
+
settings: {},
|
|
85
|
+
secrets,
|
|
86
|
+
argumentNames: innerParams.argument_names
|
|
87
|
+
? (Array.isArray(innerParams.argument_names)
|
|
88
|
+
? innerParams.argument_names
|
|
89
|
+
: [...innerParams.argument_names])
|
|
90
|
+
.map((value: unknown) => value == null ? null : String(value))
|
|
91
|
+
: null,
|
|
92
|
+
};
|
|
93
|
+
const executionId = new Uint8Array(16);
|
|
94
|
+
crypto.getRandomValues(executionId);
|
|
95
|
+
setExecutionState(executionId, {
|
|
96
|
+
states: new Map(),
|
|
97
|
+
bindArgs: args,
|
|
98
|
+
bindParams,
|
|
99
|
+
});
|
|
100
|
+
// Output schema is a single "result" column. Type comes from the
|
|
101
|
+
// optional onBind hook (for aggregates whose return type depends on
|
|
102
|
+
// input — e.g. vgi_generic_sum takes ANY and returns the same type);
|
|
103
|
+
// otherwise the declared cfg.outputType.
|
|
104
|
+
const outType = cfg.onBind ? await cfg.onBind(bindParams) : cfg.outputType;
|
|
105
|
+
const outSchema = makeSchema([field("result", outType, true)]);
|
|
106
|
+
return wrapResult(
|
|
107
|
+
{ output_schema: serializeSchema(outSchema), execution_id: executionId },
|
|
108
|
+
AggregateBindResultSchema,
|
|
109
|
+
);
|
|
110
|
+
};
|
|
124
111
|
|
|
125
112
|
// ------------------------------------------------------------------------
|
|
126
113
|
// aggregate_update — fold the incoming batch into per-group state
|
|
127
114
|
// ------------------------------------------------------------------------
|
|
128
|
-
|
|
129
|
-
params
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
const cfg = resolveAggregate(functionName, innerParams.schema_path ?? null);
|
|
136
|
-
const batch = readSingleBatch(toUint8Array(innerParams.input_batch));
|
|
137
|
-
if (!batch || batch.numRows === 0) return wrapResult({}, makeSchema([]));
|
|
115
|
+
const aggregateUpdate: VgiService["aggregateUpdate"] = (params) => {
|
|
116
|
+
const innerParams = unwrapRequest(params.request);
|
|
117
|
+
const functionName: string = innerParams.function_name;
|
|
118
|
+
const executionId = toUint8Array(innerParams.execution_id);
|
|
119
|
+
const cfg = resolveAggregate(functionName, innerParams.schema_path ?? null);
|
|
120
|
+
const batch = readSingleBatch(toUint8Array(innerParams.input_batch));
|
|
121
|
+
if (!batch || batch.numRows === 0) return wrapResult({}, makeSchema([]));
|
|
138
122
|
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
123
|
+
const gidIdx = batch.schema.fields.findIndex((f) => f.name === GROUP_COLUMN_NAME);
|
|
124
|
+
if (gidIdx < 0) {
|
|
125
|
+
throw new Error(`aggregate_update: input batch missing ${GROUP_COLUMN_NAME} column`);
|
|
126
|
+
}
|
|
127
|
+
const gidCol = batch.getChildAt(gidIdx)!;
|
|
128
|
+
const groupIds: bigint[] = [];
|
|
129
|
+
const uniqueGids = new Set<bigint>();
|
|
130
|
+
for (let i = 0; i < batch.numRows; i++) {
|
|
131
|
+
const v = gidCol.get(i);
|
|
132
|
+
const g = typeof v === "bigint" ? v : BigInt(v as any);
|
|
133
|
+
groupIds.push(g);
|
|
134
|
+
uniqueGids.add(g);
|
|
135
|
+
}
|
|
136
|
+
const exec = getExecutionState(executionId, uniqueGids);
|
|
137
|
+
if (!exec) {
|
|
138
|
+
throw new Error(`aggregate_update: unknown execution_id for '${functionName}' (bind missing or state dropped)`);
|
|
139
|
+
}
|
|
156
140
|
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
141
|
+
// Build column-arg arrays — every non-__vgi_group_id column, in
|
|
142
|
+
// schema order — and pass to user's update(). We deliberately do NOT
|
|
143
|
+
// pre-allocate state for every group_id seen — the user decides when
|
|
144
|
+
// to call ensureState() so that groups with only-NULL input (skipped
|
|
145
|
+
// rows) never get an entry, which lets finalize() return SQL NULL.
|
|
146
|
+
const columns: any[] = [];
|
|
147
|
+
for (let i = 0; i < batch.schema.fields.length; i++) {
|
|
148
|
+
if (i === gidIdx) continue;
|
|
149
|
+
columns.push(batch.getChildAt(i));
|
|
150
|
+
}
|
|
151
|
+
const ensureState = (gid: bigint) => {
|
|
152
|
+
let s = exec.states.get(gid);
|
|
153
|
+
if (s === undefined) {
|
|
154
|
+
s = cfg.initialState(exec.bindParams);
|
|
155
|
+
exec.states.set(gid, s);
|
|
166
156
|
}
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
// (skipped by NULL handling), in which case the file simply reflects
|
|
185
|
-
// the pre-existing initial state; the idempotent rewrite is cheap
|
|
186
|
-
// relative to the RPC overhead.
|
|
187
|
-
persistGroupStates(executionId, uniqueGids);
|
|
188
|
-
return wrapResult({}, makeSchema([]));
|
|
189
|
-
},
|
|
190
|
-
});
|
|
157
|
+
return s;
|
|
158
|
+
};
|
|
159
|
+
cfg.update({
|
|
160
|
+
states: exec.states,
|
|
161
|
+
groupIds,
|
|
162
|
+
columns,
|
|
163
|
+
args: exec.bindParams.args,
|
|
164
|
+
ensureState,
|
|
165
|
+
});
|
|
166
|
+
// Flush mutated states to disk so sibling workers see them. We persist
|
|
167
|
+
// every group that was referenced this call — some may be unchanged
|
|
168
|
+
// (skipped by NULL handling), in which case the file simply reflects
|
|
169
|
+
// the pre-existing initial state; the idempotent rewrite is cheap
|
|
170
|
+
// relative to the RPC overhead.
|
|
171
|
+
persistGroupStates(executionId, uniqueGids);
|
|
172
|
+
return wrapResult({}, makeSchema([]));
|
|
173
|
+
};
|
|
191
174
|
|
|
192
175
|
// ------------------------------------------------------------------------
|
|
193
176
|
// aggregate_combine — merge (source_group_id, target_group_id) pairs
|
|
194
177
|
// ------------------------------------------------------------------------
|
|
195
|
-
|
|
196
|
-
params
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
const cfg = resolveAggregate(functionName, innerParams.schema_path ?? null);
|
|
203
|
-
const batch = readSingleBatch(toUint8Array(innerParams.merge_batch));
|
|
204
|
-
if (!batch || batch.numRows === 0) return wrapResult({}, makeSchema([]));
|
|
178
|
+
const aggregateCombine: VgiService["aggregateCombine"] = (params) => {
|
|
179
|
+
const innerParams = unwrapRequest(params.request);
|
|
180
|
+
const functionName: string = innerParams.function_name;
|
|
181
|
+
const executionId = toUint8Array(innerParams.execution_id);
|
|
182
|
+
const cfg = resolveAggregate(functionName, innerParams.schema_path ?? null);
|
|
183
|
+
const batch = readSingleBatch(toUint8Array(innerParams.merge_batch));
|
|
184
|
+
if (!batch || batch.numRows === 0) return wrapResult({}, makeSchema([]));
|
|
205
185
|
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
});
|
|
186
|
+
const srcCol = batch.getChild("source_group_id")!;
|
|
187
|
+
const tgtCol = batch.getChild("target_group_id")!;
|
|
188
|
+
const referencedGids = new Set<bigint>();
|
|
189
|
+
const srcIds: bigint[] = [];
|
|
190
|
+
const tgtIds: bigint[] = [];
|
|
191
|
+
for (let i = 0; i < batch.numRows; i++) {
|
|
192
|
+
const src = BigInt(srcCol.get(i) as any);
|
|
193
|
+
const tgt = BigInt(tgtCol.get(i) as any);
|
|
194
|
+
srcIds.push(src); tgtIds.push(tgt);
|
|
195
|
+
referencedGids.add(src); referencedGids.add(tgt);
|
|
196
|
+
}
|
|
197
|
+
const exec = getExecutionState(executionId, referencedGids);
|
|
198
|
+
if (!exec) {
|
|
199
|
+
throw new Error(
|
|
200
|
+
`aggregate_combine: unknown execution_id for '${functionName}' ` +
|
|
201
|
+
`(context missing on disk — bind may have failed or been cleaned up)`,
|
|
202
|
+
);
|
|
203
|
+
}
|
|
204
|
+
const touchedTargets = new Set<bigint>();
|
|
205
|
+
for (let i = 0; i < batch.numRows; i++) {
|
|
206
|
+
const src = srcIds[i];
|
|
207
|
+
const tgt = tgtIds[i];
|
|
208
|
+
const srcHas = exec.states.has(src);
|
|
209
|
+
const tgtHas = exec.states.has(tgt);
|
|
210
|
+
if (!srcHas && !tgtHas) continue;
|
|
211
|
+
const srcState = srcHas ? exec.states.get(src)! : cfg.initialState(exec.bindParams);
|
|
212
|
+
const tgtState = tgtHas ? exec.states.get(tgt)! : cfg.initialState(exec.bindParams);
|
|
213
|
+
exec.states.set(tgt, cfg.combine(srcState, tgtState, exec.bindParams));
|
|
214
|
+
touchedTargets.add(tgt);
|
|
215
|
+
}
|
|
216
|
+
persistGroupStates(executionId, touchedTargets);
|
|
217
|
+
return wrapResult({}, makeSchema([]));
|
|
218
|
+
};
|
|
240
219
|
|
|
241
220
|
// ------------------------------------------------------------------------
|
|
242
221
|
// aggregate_finalize — emit one result row per requested group_id
|
|
243
222
|
// ------------------------------------------------------------------------
|
|
244
|
-
|
|
245
|
-
params
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
const
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
const
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
const g = typeof v === "bigint" ? v : BigInt(v as any);
|
|
261
|
-
groupIds.push(g);
|
|
262
|
-
uniqueGids.add(g);
|
|
263
|
-
}
|
|
264
|
-
}
|
|
265
|
-
const exec = getExecutionState(executionId, uniqueGids);
|
|
266
|
-
if (!exec) {
|
|
267
|
-
throw new Error(`aggregate_finalize: unknown execution_id for '${functionName}'`);
|
|
268
|
-
}
|
|
269
|
-
// Groups never updated pass state = null so finalize() can emit SQL NULL
|
|
270
|
-
// (matches Python semantics for SUM/AVG/MIN/MAX over zero rows).
|
|
271
|
-
const states = new Map<bigint, any | null>();
|
|
272
|
-
for (const gid of groupIds) {
|
|
273
|
-
states.set(gid, exec.states.has(gid) ? exec.states.get(gid) : null);
|
|
223
|
+
const aggregateFinalize: VgiService["aggregateFinalize"] = (params) => {
|
|
224
|
+
const innerParams = unwrapRequest(params.request);
|
|
225
|
+
const functionName: string = innerParams.function_name;
|
|
226
|
+
const executionId = toUint8Array(innerParams.execution_id);
|
|
227
|
+
const cfg = resolveAggregate(functionName, innerParams.schema_path ?? null);
|
|
228
|
+
const outputSchema = deserializeSchema(toUint8Array(innerParams.output_schema));
|
|
229
|
+
const gidBatch = readSingleBatch(toUint8Array(innerParams.group_ids_batch));
|
|
230
|
+
const groupIds: bigint[] = [];
|
|
231
|
+
const uniqueGids = new Set<bigint>();
|
|
232
|
+
if (gidBatch) {
|
|
233
|
+
const col = gidBatch.getChild("group_id")!;
|
|
234
|
+
for (let i = 0; i < gidBatch.numRows; i++) {
|
|
235
|
+
const v = col.get(i);
|
|
236
|
+
const g = typeof v === "bigint" ? v : BigInt(v as any);
|
|
237
|
+
groupIds.push(g);
|
|
238
|
+
uniqueGids.add(g);
|
|
274
239
|
}
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
240
|
+
}
|
|
241
|
+
const exec = getExecutionState(executionId, uniqueGids);
|
|
242
|
+
if (!exec) {
|
|
243
|
+
throw new Error(`aggregate_finalize: unknown execution_id for '${functionName}'`);
|
|
244
|
+
}
|
|
245
|
+
// Groups never updated pass state = null so finalize() can emit SQL NULL
|
|
246
|
+
// (matches Python semantics for SUM/AVG/MIN/MAX over zero rows).
|
|
247
|
+
const states = new Map<bigint, any | null>();
|
|
248
|
+
for (const gid of groupIds) {
|
|
249
|
+
states.set(gid, exec.states.has(gid) ? exec.states.get(gid) : null);
|
|
250
|
+
}
|
|
251
|
+
const resultBatch = cfg.finalize({
|
|
252
|
+
groupIds, states, outputSchema, args: exec.bindParams.args,
|
|
253
|
+
});
|
|
254
|
+
return wrapResult(
|
|
255
|
+
{ result_batch: serializeBatchWithSchema(resultBatch) },
|
|
256
|
+
AggregateFinalizeResultSchema,
|
|
257
|
+
);
|
|
258
|
+
};
|
|
284
259
|
|
|
285
260
|
// ------------------------------------------------------------------------
|
|
286
261
|
// aggregate_destructor — drop state for finished groups (best-effort)
|
|
@@ -291,30 +266,28 @@ export function registerAggregateMethods(protocol: Protocol, registry: FunctionR
|
|
|
291
266
|
// (matches Python's conservative behavior; the execution entry is cleaned
|
|
292
267
|
// up when the worker process exits).
|
|
293
268
|
// ------------------------------------------------------------------------
|
|
294
|
-
|
|
295
|
-
params
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
const
|
|
305
|
-
const
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
},
|
|
317
|
-
});
|
|
269
|
+
const aggregateDestructor: VgiService["aggregateDestructor"] = (params) => {
|
|
270
|
+
const innerParams = unwrapRequest(params.request);
|
|
271
|
+
const executionId = toUint8Array(innerParams.execution_id);
|
|
272
|
+
const exec = getExecutionState(executionId);
|
|
273
|
+
if (!exec) return wrapResult({}, makeSchema([]));
|
|
274
|
+
const gidBatch = readSingleBatch(toUint8Array(innerParams.group_ids_batch));
|
|
275
|
+
if (!gidBatch || gidBatch.numRows === 0) return wrapResult({}, makeSchema([]));
|
|
276
|
+
const col = gidBatch.getChild("group_id")!;
|
|
277
|
+
const toDrop = new Set<bigint>();
|
|
278
|
+
for (let i = 0; i < gidBatch.numRows; i++) {
|
|
279
|
+
const v = col.get(i);
|
|
280
|
+
const g = typeof v === "bigint" ? v : BigInt(v as any);
|
|
281
|
+
exec.states.delete(g);
|
|
282
|
+
toDrop.add(g);
|
|
283
|
+
}
|
|
284
|
+
// Persist deletions (unlinks the per-group file so sibling workers
|
|
285
|
+
// don't revive the state via getExecutionState's lazy disk load).
|
|
286
|
+
persistGroupStates(executionId, toDrop);
|
|
287
|
+
return wrapResult({}, makeSchema([]));
|
|
288
|
+
};
|
|
289
|
+
|
|
290
|
+
return { aggregateBind, aggregateUpdate, aggregateCombine, aggregateFinalize, aggregateDestructor };
|
|
318
291
|
}
|
|
319
292
|
|
|
320
293
|
function extractArgMap(
|