@query-farm/vgi 0.28.0 → 0.29.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +52 -8
- package/dist/arguments/argument-spec.d.ts +57 -2
- package/dist/arrow/impl-arrowjs/index.d.ts +1 -1
- package/dist/arrow/impl-arrowjs/iterate.d.ts +10 -1
- package/dist/arrow/impl-arrowjs/schema.d.ts +2 -0
- package/dist/arrow/impl-flechette/index.d.ts +1 -1
- package/dist/arrow/impl-flechette/iterate.d.ts +10 -1
- package/dist/arrow/impl-flechette/schema.d.ts +2 -0
- package/dist/arrow/index.d.ts +1 -1
- package/dist/arrow/schema-types.d.ts +3 -1
- package/dist/arrow/types.d.ts +39 -3
- package/dist/catalog/composite.d.ts +13 -13
- package/dist/catalog/descriptors.d.ts +7 -3
- package/dist/catalog/interface.d.ts +108 -38
- package/dist/catalog/read-only.d.ts +32 -15
- package/dist/client/client.d.ts +37 -30
- package/dist/client/types.d.ts +2 -0
- package/dist/client-entry.d.ts +1 -1
- package/dist/client-entry.js +791 -447
- package/dist/client-entry.js.map +18 -17
- package/dist/filter-pushdown/capabilities.d.ts +3 -0
- package/dist/filter-pushdown/deserialize.d.ts +14 -15
- package/dist/filter-pushdown/evaluate.d.ts +14 -44
- package/dist/filter-pushdown/index.d.ts +2 -2
- package/dist/filter-pushdown/types.d.ts +120 -68
- package/dist/functions/aggregate.d.ts +4 -0
- package/dist/functions/registry.d.ts +5 -5
- package/dist/functions/scalar.d.ts +6 -1
- package/dist/functions/table-buffering.d.ts +9 -0
- package/dist/functions/table-in-out.d.ts +13 -0
- package/dist/functions/table.d.ts +72 -2
- package/dist/functions/types.d.ts +60 -2
- package/dist/generated/vgi-client.d.ts +114 -26
- package/dist/generated/vgi-protocol-schemas.d.ts +34 -0
- package/dist/http/fetch.d.ts +16 -0
- package/dist/index.core.d.ts +4 -3
- package/dist/index.js +3396 -2064
- package/dist/index.js.map +55 -49
- package/dist/metadata/types.d.ts +15 -2
- package/dist/protocol/handlers/catalog/shared.d.ts +0 -8
- package/dist/protocol/handlers/function.d.ts +11 -0
- package/dist/protocol/handlers/shared.d.ts +3 -1
- package/dist/protocol/serializers/bind.d.ts +1 -2
- package/dist/protocol/serializers/cardinality.d.ts +1 -2
- package/dist/protocol/serializers/splits.d.ts +126 -0
- package/dist/protocol/types.d.ts +36 -3
- package/dist/request-auth.d.ts +28 -0
- package/dist/schema-path.d.ts +7 -0
- package/dist/serve-entry.d.ts +25 -0
- package/dist/serve-entry.js +1129 -569
- package/dist/serve-entry.js.map +36 -32
- package/dist/split-token.d.ts +88 -0
- package/dist/types.d.ts +8 -0
- package/dist/worker-cf-entry.js +2928 -1689
- package/dist/worker-cf-entry.js.map +51 -44
- package/dist/worker.d.ts +1 -1
- package/package.json +3 -3
- package/src/arguments/argument-spec.ts +93 -4
- package/src/arguments/arguments.ts +0 -1
- package/src/arrow/impl-arrowjs/index.ts +1 -1
- package/src/arrow/impl-arrowjs/iterate.ts +21 -0
- package/src/arrow/impl-arrowjs/schema.ts +4 -0
- package/src/arrow/impl-flechette/index.ts +1 -1
- package/src/arrow/impl-flechette/iterate.ts +24 -1
- package/src/arrow/impl-flechette/schema.ts +4 -0
- package/src/arrow/index.ts +1 -1
- package/src/arrow/schema-types.ts +4 -0
- package/src/arrow/types.ts +39 -3
- package/src/catalog/composite.ts +26 -26
- package/src/catalog/descriptors.ts +12 -3
- package/src/catalog/interface.ts +217 -72
- package/src/catalog/read-only.ts +141 -84
- package/src/client/client.ts +97 -73
- package/src/client/types.ts +2 -0
- package/src/client-entry.ts +2 -1
- package/src/filter-pushdown/capabilities.ts +13 -0
- package/src/filter-pushdown/collector.ts +22 -35
- package/src/filter-pushdown/deserialize.ts +657 -221
- package/src/filter-pushdown/evaluate.ts +410 -499
- package/src/filter-pushdown/index.ts +16 -2
- package/src/filter-pushdown/types.ts +88 -71
- package/src/functions/aggregate.ts +12 -1
- package/src/functions/copy-from.ts +2 -1
- package/src/functions/copy-to.ts +2 -1
- package/src/functions/registry.ts +24 -22
- package/src/functions/scalar.ts +37 -0
- package/src/functions/table-buffering.ts +48 -6
- package/src/functions/table-in-out.ts +194 -33
- package/src/functions/table.ts +187 -22
- package/src/functions/types.ts +64 -1
- package/src/generated/vgi-client.ts +122 -28
- package/src/generated/vgi-protocol-schemas.ts +455 -58
- package/src/http/fetch.ts +129 -2
- package/src/index.core.ts +4 -2
- package/src/metadata/resolve.ts +43 -1
- package/src/metadata/serialize.ts +43 -5
- package/src/metadata/types.ts +17 -1
- package/src/protocol/dispatch.ts +1 -1
- package/src/protocol/handlers/aggregate.ts +23 -12
- package/src/protocol/handlers/catalog/admin.ts +47 -54
- package/src/protocol/handlers/catalog/index_methods.ts +6 -13
- package/src/protocol/handlers/catalog/macro.ts +12 -20
- package/src/protocol/handlers/catalog/shared.ts +12 -57
- package/src/protocol/handlers/catalog/table.ts +49 -112
- package/src/protocol/handlers/catalog/view.ts +18 -22
- package/src/protocol/handlers/function.ts +311 -18
- package/src/protocol/handlers/shared.ts +12 -3
- package/src/protocol/handlers/table-buffering.ts +11 -8
- package/src/protocol/serializers/bind.ts +69 -67
- package/src/protocol/serializers/cardinality.ts +7 -7
- package/src/protocol/serializers/init.ts +52 -10
- package/src/protocol/serializers/splits.ts +221 -0
- package/src/protocol/state-serializer.ts +55 -1
- package/src/protocol/types.ts +36 -3
- package/src/request-auth.ts +73 -0
- package/src/schema-path.ts +25 -0
- package/src/serve-entry.ts +69 -2
- package/src/split-token.ts +297 -0
- package/src/types.ts +9 -0
- package/src/worker-cf-entry.ts +2 -1
- package/src/worker.ts +79 -14
package/src/protocol/types.ts
CHANGED
|
@@ -91,11 +91,17 @@ export interface BindRequest {
|
|
|
91
91
|
/**
|
|
92
92
|
* Catalog schema that declares the function being bound. A worker may
|
|
93
93
|
* register the same name in more than one schema, so the bare name does not
|
|
94
|
-
* identify an implementation — resolution is by `(
|
|
94
|
+
* identify an implementation — resolution is by `(schema_path,
|
|
95
95
|
* function_name)`. Null for callers with no catalog context (COPY handler
|
|
96
|
-
* binds). Mirrors vgi-python's `BindRequest.
|
|
96
|
+
* binds). Mirrors vgi-python's `BindRequest.schema_path`.
|
|
97
97
|
*/
|
|
98
|
-
|
|
98
|
+
schema_path?: string[] | null;
|
|
99
|
+
/**
|
|
100
|
+
* Resolved name for every logical argument, in complete call order. Fixed
|
|
101
|
+
* parameters carry their declared names, unnamed varargs carry `null`, and
|
|
102
|
+
* named varargs retain the caller-provided name. `null` means unavailable.
|
|
103
|
+
*/
|
|
104
|
+
argument_names?: (string | null)[] | null;
|
|
99
105
|
}
|
|
100
106
|
|
|
101
107
|
export interface BindResponse {
|
|
@@ -125,6 +131,33 @@ export interface InitRequest {
|
|
|
125
131
|
* bytes the worker's combine() chose.
|
|
126
132
|
*/
|
|
127
133
|
finalize_state_id: Uint8Array | null;
|
|
134
|
+
/**
|
|
135
|
+
* The framework-stamped split envelopes this init redeems, or null when this
|
|
136
|
+
* is not a split init.
|
|
137
|
+
*
|
|
138
|
+
* A LIST because an engine whose partition count IS its concurrency bin-packs
|
|
139
|
+
* at planning time and reads a whole group per partition; DuckDB always sends
|
|
140
|
+
* exactly one. A worker handed several concatenates them in the order given.
|
|
141
|
+
*/
|
|
142
|
+
split_tokens: Uint8Array[] | null;
|
|
143
|
+
/**
|
|
144
|
+
* The VERIFIED payloads behind `split_tokens` — the worker's own bytes, with
|
|
145
|
+
* the envelope already opened and stripped.
|
|
146
|
+
*
|
|
147
|
+
* Populated by the protocol handler, which is what holds the signing key, so
|
|
148
|
+
* an unverified token never reaches a function. Null when this is not a split
|
|
149
|
+
* init.
|
|
150
|
+
*/
|
|
151
|
+
split_payloads?: Uint8Array[] | null;
|
|
152
|
+
/**
|
|
153
|
+
* A plain fetch limit, when the engine can supply one.
|
|
154
|
+
*
|
|
155
|
+
* Always null from DuckDB — `TableFunctionInitInput` carries no limit, and
|
|
156
|
+
* the Top-N `order_by_limit` means something different ("top K by that
|
|
157
|
+
* column"). DataFusion supplies a real value via `TableProvider::scan(limit)`.
|
|
158
|
+
* Over-producing is legal; the engine re-applies the limit above the scan.
|
|
159
|
+
*/
|
|
160
|
+
row_limit: number | null;
|
|
128
161
|
execution_id: Uint8Array | null;
|
|
129
162
|
init_opaque_data: Uint8Array | null;
|
|
130
163
|
/**
|
|
@@ -0,0 +1,73 @@
|
|
|
1
|
+
// Copyright 2025, 2026 Query Farm LLC - https://query.farm
|
|
2
|
+
// Request-scoped caller identity, for the code paths the RPC framework cannot
|
|
3
|
+
// hand a CallContext to.
|
|
4
|
+
//
|
|
5
|
+
// Every unary VGI method receives the caller's `AuthContext` directly — the
|
|
6
|
+
// dispatcher passes the OutputCollector, which *is* a CallContext. The stream
|
|
7
|
+
// `init` method does not: `@query-farm/vgi-rpc` types stream initialization as
|
|
8
|
+
// `(params) => state`, with no context parameter, on both the pipe and HTTP
|
|
9
|
+
// dispatchers. That is a framework signature this package cannot change.
|
|
10
|
+
//
|
|
11
|
+
// It matters because `init` opens two identity-bound AEAD envelopes:
|
|
12
|
+
// `attach_opaque_data` (sealed under `attachAad(auth)` by `catalog_attach`) and
|
|
13
|
+
// every split token (sealed under `splitTokenAad(body, auth)` by
|
|
14
|
+
// `table_function_plan`). Both of those minting sites ARE unary and therefore
|
|
15
|
+
// seal under the REAL principal. Opening them under a hard-coded `undefined`
|
|
16
|
+
// yields the anonymous identity tail, so the AAD only ever matches for an
|
|
17
|
+
// anonymous caller: an authenticated one gets `OpaqueDataRejectedError:
|
|
18
|
+
// attach_opaque_data not recognized` on the first scan. Anonymous workers never
|
|
19
|
+
// saw it, which is why it survived — the example HTTP workers authenticate
|
|
20
|
+
// nobody.
|
|
21
|
+
//
|
|
22
|
+
// The reference implementations solve this with an ambient, request-scoped
|
|
23
|
+
// identity rather than a parameter: vgi-python reads a `ContextVar` via
|
|
24
|
+
// `current_auth()`; vgi-go's RPC framework threads a `*CallContext` into
|
|
25
|
+
// `handleInit`. This is the TypeScript equivalent — an `AsyncLocalStorage`
|
|
26
|
+
// scope opened per HTTP request by `createVgiFetch` (see `http/fetch.ts`),
|
|
27
|
+
// read here.
|
|
28
|
+
//
|
|
29
|
+
// Deliberately a registry rather than a direct `node:async_hooks` import: this
|
|
30
|
+
// module is reachable from the protocol handlers, which also run under stdio,
|
|
31
|
+
// AF_UNIX and browser builds. Only the HTTP entry points install a scope, and
|
|
32
|
+
// they load `node:async_hooks` defensively. With no scope installed
|
|
33
|
+
// `currentRequestAuth()` returns `undefined`, which is exactly the previous
|
|
34
|
+
// behaviour — and correct, because a worker with no signing key seals nothing.
|
|
35
|
+
|
|
36
|
+
import type { AuthContext } from "@query-farm/vgi-rpc";
|
|
37
|
+
|
|
38
|
+
/** Mutable per-request cell. The scope is opened before the identity is known
|
|
39
|
+
* (authentication happens inside the handler), so the holder is published
|
|
40
|
+
* first and filled in when the framework reports the principal. */
|
|
41
|
+
export interface RequestAuthHolder {
|
|
42
|
+
auth?: AuthContext;
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
/** The slice of `AsyncLocalStorage` this module needs. Structural on purpose:
|
|
46
|
+
* it keeps `node:async_hooks` out of every non-HTTP build, and lets a test
|
|
47
|
+
* install a trivial stub. */
|
|
48
|
+
export interface RequestAuthScope {
|
|
49
|
+
getStore(): RequestAuthHolder | undefined;
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
let scope: RequestAuthScope | undefined;
|
|
53
|
+
|
|
54
|
+
/**
|
|
55
|
+
* Publish the scope `currentRequestAuth()` reads. Called once per process by
|
|
56
|
+
* the HTTP entry point; idempotent for the same scope.
|
|
57
|
+
*/
|
|
58
|
+
export function setRequestAuthScope(next: RequestAuthScope | undefined): void {
|
|
59
|
+
scope = next;
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
/**
|
|
63
|
+
* The authenticated principal of the request being served on this async
|
|
64
|
+
* context, or `undefined` when there is none — no scope installed (stdio /
|
|
65
|
+
* AF_UNIX / a host that has no `AsyncLocalStorage`), or an unauthenticated
|
|
66
|
+
* request.
|
|
67
|
+
*
|
|
68
|
+
* `undefined` and an anonymous `AuthContext` produce the same identity tail, so
|
|
69
|
+
* the two cases need not be distinguished by callers.
|
|
70
|
+
*/
|
|
71
|
+
export function currentRequestAuth(): AuthContext | undefined {
|
|
72
|
+
return scope?.getStore()?.auth;
|
|
73
|
+
}
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
// Copyright 2025, 2026 Query Farm LLC - https://query.farm
|
|
2
|
+
|
|
3
|
+
/** Raw identifier components of a catalog schema, from root to leaf. */
|
|
4
|
+
export type SchemaPath = string[];
|
|
5
|
+
|
|
6
|
+
/** Case-insensitive key used for catalog lookup and function dispatch. */
|
|
7
|
+
export function normalizeSchemaPath(path: Iterable<unknown> | string): string[] {
|
|
8
|
+
return typeof path === "string" ? [path] : Array.from(path, String);
|
|
9
|
+
}
|
|
10
|
+
|
|
11
|
+
export function schemaPathKey(path: Iterable<unknown> | string): string {
|
|
12
|
+
const normalized = normalizeSchemaPath(path);
|
|
13
|
+
if (normalized.length === 0 || normalized.some((part) => part.length === 0)) {
|
|
14
|
+
throw new Error("schema paths must contain at least one non-empty component");
|
|
15
|
+
}
|
|
16
|
+
return normalized.map((part) => part.toLowerCase()).join("\u0000");
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
export function schemaPathsEqual(left: Iterable<unknown> | string, right: Iterable<unknown> | string): boolean {
|
|
20
|
+
return schemaPathKey(left) === schemaPathKey(right);
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
export function schemaPathDisplay(path: Iterable<unknown> | string): string {
|
|
24
|
+
return normalizeSchemaPath(path).map((part) => JSON.stringify(part)).join(".");
|
|
25
|
+
}
|
package/src/serve-entry.ts
CHANGED
|
@@ -23,6 +23,14 @@
|
|
|
23
23
|
|
|
24
24
|
import type { CatalogInterface } from "./catalog/interface.js";
|
|
25
25
|
import type { FunctionRegistry } from "./functions/registry.js";
|
|
26
|
+
import {
|
|
27
|
+
irohForwardedHeaderIdentityProvider,
|
|
28
|
+
IROH_FORWARDED_ENDPOINT_HEADER,
|
|
29
|
+
observePeerIdentity,
|
|
30
|
+
peerIdentityPrimary,
|
|
31
|
+
type AuthenticateFn,
|
|
32
|
+
type PeerResolutionOptions,
|
|
33
|
+
} from "@query-farm/vgi-rpc";
|
|
26
34
|
import { createVgiFetch } from "./http/fetch.js";
|
|
27
35
|
|
|
28
36
|
/** Environment variables `serveVgiWorker` reads. Injectable for testing. */
|
|
@@ -38,6 +46,21 @@ export const DEFAULT_TOKEN_TTL = 3600;
|
|
|
38
46
|
/** Bytes of HMAC key `createHttpHandler` expects; 64 hex characters. */
|
|
39
47
|
export const SIGNING_KEY_BYTES = 32;
|
|
40
48
|
|
|
49
|
+
/** HTTP trust boundary for a colocated `vgi-iroh-bridge`. */
|
|
50
|
+
export interface IrohHttpBridgeOptions {
|
|
51
|
+
/** Worker-local namespace for EndpointId principals. */
|
|
52
|
+
issuer: string;
|
|
53
|
+
/** Exact bridge IP addresses. No CIDRs or hostnames. */
|
|
54
|
+
trustedProxyAddresses?: readonly string[];
|
|
55
|
+
/** Authenticate the EndpointId (default) or expose evidence only. */
|
|
56
|
+
authenticate?: boolean;
|
|
57
|
+
/**
|
|
58
|
+
* Host-specific physical peer and raw-header adapter. Required because a
|
|
59
|
+
* Fetch Request alone does not preserve duplicate header fields.
|
|
60
|
+
*/
|
|
61
|
+
peerResolutionContext?: (request: Request) => PeerResolutionOptions | Promise<PeerResolutionOptions>;
|
|
62
|
+
}
|
|
63
|
+
|
|
41
64
|
export interface ServeVgiWorkerOptions {
|
|
42
65
|
/** Short worker name shown on the landing page, e.g. "ishares". */
|
|
43
66
|
name: string;
|
|
@@ -54,6 +77,8 @@ export interface ServeVgiWorkerOptions {
|
|
|
54
77
|
repositoryUrl?: string;
|
|
55
78
|
/** Overrides `$PORT` (default 8787). Pass 0 to bind an ephemeral port. */
|
|
56
79
|
port?: number;
|
|
80
|
+
/** Bind address. Iroh bridge mode defaults to loopback; otherwise Bun's default. */
|
|
81
|
+
hostname?: string;
|
|
57
82
|
/** Overrides `$VGI_SIGNING_KEY`. 32 bytes. */
|
|
58
83
|
signingKey?: Uint8Array;
|
|
59
84
|
/** Overrides `$VGI_TOKEN_TTL` (default 3600 seconds). */
|
|
@@ -68,6 +93,11 @@ export interface ServeVgiWorkerOptions {
|
|
|
68
93
|
quiet?: boolean;
|
|
69
94
|
/** Environment source (default `process.env`). */
|
|
70
95
|
env?: ServeEnv;
|
|
96
|
+
/** Authenticates each request, returning the caller's `AuthContext`.
|
|
97
|
+
* Omit for an anonymous worker (the default). */
|
|
98
|
+
authenticate?: AuthenticateFn;
|
|
99
|
+
/** Preserve authenticated Iroh identity forwarded by an adjacent HTTP bridge. */
|
|
100
|
+
irohBridge?: IrohHttpBridgeOptions;
|
|
71
101
|
}
|
|
72
102
|
|
|
73
103
|
/** The slice of `Bun.serve`'s return value this module uses. */
|
|
@@ -75,11 +105,13 @@ export interface VgiHttpServer {
|
|
|
75
105
|
readonly port: number;
|
|
76
106
|
readonly url: URL;
|
|
77
107
|
stop(closeActiveConnections?: boolean): void;
|
|
108
|
+
requestIP?(request: Request): { address: string } | null;
|
|
78
109
|
}
|
|
79
110
|
|
|
80
111
|
interface BunLike {
|
|
81
112
|
serve(options: {
|
|
82
113
|
port: number;
|
|
114
|
+
hostname?: string;
|
|
83
115
|
fetch: (req: Request) => Promise<Response>;
|
|
84
116
|
}): VgiHttpServer;
|
|
85
117
|
}
|
|
@@ -184,6 +216,13 @@ export function createVgiWorkerFetch(
|
|
|
184
216
|
// mean "off" would silently re-enable it. `undefined` falls through to the env
|
|
185
217
|
// var and then to that same default.
|
|
186
218
|
const corsOrigins = opts.corsOrigins === null ? null : (opts.corsOrigins ?? env.CORS_ORIGINS);
|
|
219
|
+
const bridge = opts.irohBridge;
|
|
220
|
+
if (bridge && !bridge.peerResolutionContext) {
|
|
221
|
+
throw new Error(
|
|
222
|
+
"irohBridge requires a peerResolutionContext when createVgiWorkerFetch is used without serveVgiWorker",
|
|
223
|
+
);
|
|
224
|
+
}
|
|
225
|
+
const trustedProxyAddresses = bridge?.trustedProxyAddresses ?? ["127.0.0.1"];
|
|
187
226
|
|
|
188
227
|
return createVgiFetch({
|
|
189
228
|
protocol: { registry: opts.registry, catalogInterface: opts.catalogInterface },
|
|
@@ -194,6 +233,14 @@ export function createVgiWorkerFetch(
|
|
|
194
233
|
corsOrigins,
|
|
195
234
|
repositoryUrl: opts.repositoryUrl,
|
|
196
235
|
landingInfo: { name: opts.name, doc: opts.doc, version: opts.version },
|
|
236
|
+
authenticate: opts.authenticate,
|
|
237
|
+
peerIdentityProviders: bridge
|
|
238
|
+
? [irohForwardedHeaderIdentityProvider({ issuer: bridge.issuer, trustedProxyAddresses })]
|
|
239
|
+
: undefined,
|
|
240
|
+
peerAuthenticationPolicy: bridge
|
|
241
|
+
? (bridge.authenticate === false ? observePeerIdentity : peerIdentityPrimary("iroh"))
|
|
242
|
+
: undefined,
|
|
243
|
+
peerResolutionContext: bridge?.peerResolutionContext,
|
|
197
244
|
});
|
|
198
245
|
}
|
|
199
246
|
|
|
@@ -215,8 +262,28 @@ export function serveVgiWorker(opts: ServeVgiWorkerOptions): VgiHttpServer {
|
|
|
215
262
|
|
|
216
263
|
const env = opts.env ?? (process.env as ServeEnv);
|
|
217
264
|
const port = resolvePort(opts.port, env);
|
|
218
|
-
const
|
|
219
|
-
|
|
265
|
+
const hostname = opts.hostname ?? (opts.irohBridge ? "127.0.0.1" : undefined);
|
|
266
|
+
if (opts.irohBridge && !["127.0.0.1", "::1", "localhost"].includes(hostname ?? "")) {
|
|
267
|
+
throw new Error("an Iroh HTTP bridge upstream must bind loopback");
|
|
268
|
+
}
|
|
269
|
+
let server: VgiHttpServer;
|
|
270
|
+
const bridge = opts.irohBridge && !opts.irohBridge.peerResolutionContext
|
|
271
|
+
? {
|
|
272
|
+
...opts.irohBridge,
|
|
273
|
+
peerResolutionContext(request: Request): PeerResolutionOptions {
|
|
274
|
+
const immediatePeer = server.requestIP?.(request)?.address;
|
|
275
|
+
const forwarded = request.headers.get(IROH_FORWARDED_ENDPOINT_HEADER);
|
|
276
|
+
return {
|
|
277
|
+
immediatePeer,
|
|
278
|
+
headers: forwarded === null
|
|
279
|
+
? {}
|
|
280
|
+
: { [IROH_FORWARDED_ENDPOINT_HEADER]: [forwarded] },
|
|
281
|
+
};
|
|
282
|
+
},
|
|
283
|
+
}
|
|
284
|
+
: opts.irohBridge;
|
|
285
|
+
const fetch = createVgiWorkerFetch({ ...opts, irohBridge: bridge });
|
|
286
|
+
server = runtime.serve({ port, hostname, fetch });
|
|
220
287
|
|
|
221
288
|
if (!opts.quiet) {
|
|
222
289
|
const base = `http://localhost:${server.port}`;
|
|
@@ -0,0 +1,297 @@
|
|
|
1
|
+
// Copyright 2025, 2026 Query Farm LLC - https://query.farm
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* Split-token envelope: the framework's wrapper around a worker's split payload.
|
|
5
|
+
*
|
|
6
|
+
* A split token *names* a unit of scan work so a distributed engine can
|
|
7
|
+
* re-request exactly the work it was handed. The worker supplies only the
|
|
8
|
+
* payload; everything around it is stamped here, so an author cannot forget the
|
|
9
|
+
* consistency anchor or mis-bind the fingerprint, and never writes crypto.
|
|
10
|
+
*
|
|
11
|
+
* Layout (little-endian, fixed prefix) — byte-identical across every SDK:
|
|
12
|
+
*
|
|
13
|
+
* ```
|
|
14
|
+
* offset size field
|
|
15
|
+
* 0 1 format_version currently 1
|
|
16
|
+
* 1 1 flags bit0 = payload_sealed; bits 1-7 reserved, MUST be 0
|
|
17
|
+
* 2 2 anchor_len u16 LE
|
|
18
|
+
* 4 16 bind_fingerprint truncated SHA-256 of the bind identity
|
|
19
|
+
* 20 var consistency_anchor anchor_len bytes
|
|
20
|
+
* 20+n var payload the worker's own bytes
|
|
21
|
+
* ```
|
|
22
|
+
*
|
|
23
|
+
* **The header is plaintext on every transport; only the payload is sealed.**
|
|
24
|
+
* That is not a preference: a worker has no signing key on subprocess and unix,
|
|
25
|
+
* which is DuckDB's primary path, so a header readable only through AEAD would be
|
|
26
|
+
* unreadable exactly where DuckDB runs. It also matters for streaming — a
|
|
27
|
+
* checkpointed position must survive key rotation.
|
|
28
|
+
*
|
|
29
|
+
* @module
|
|
30
|
+
*/
|
|
31
|
+
|
|
32
|
+
import type { AuthContext } from "@query-farm/vgi-rpc";
|
|
33
|
+
|
|
34
|
+
import { identityTail, openBytes, sealBytes } from "./crypto.js";
|
|
35
|
+
|
|
36
|
+
/** Envelope format version. Checked unconditionally, before anything else. */
|
|
37
|
+
export const SPLIT_TOKEN_FORMAT_VERSION = 1;
|
|
38
|
+
|
|
39
|
+
/** bit0 of `flags`: the payload is AEAD-sealed rather than plaintext. */
|
|
40
|
+
const FLAG_PAYLOAD_SEALED = 0x01;
|
|
41
|
+
|
|
42
|
+
/** bits 1-7 are reserved and MUST be zero; a set bit is a forward-compat violation. */
|
|
43
|
+
const RESERVED_FLAGS_MASK = 0xfe;
|
|
44
|
+
|
|
45
|
+
const FINGERPRINT_LEN = 16;
|
|
46
|
+
const HEADER_LEN = 4 + FINGERPRINT_LEN;
|
|
47
|
+
|
|
48
|
+
/**
|
|
49
|
+
* Matches the reference default (`crypto.seal_bytes` version=1), so a token
|
|
50
|
+
* sealed by one SDK opens in another.
|
|
51
|
+
*/
|
|
52
|
+
const SEAL_VERSION = 1;
|
|
53
|
+
|
|
54
|
+
const AAD_PREFIX = new TextEncoder().encode("vgi.split_token.v1\u0000");
|
|
55
|
+
|
|
56
|
+
/** Stable error-kind strings, identical across SDKs. */
|
|
57
|
+
export const SPLIT_TOKEN_INVALID = "SPLIT_TOKEN_INVALID";
|
|
58
|
+
export const SPLIT_SNAPSHOT_EXPIRED = "SPLIT_SNAPSHOT_EXPIRED";
|
|
59
|
+
export const SPLIT_TRANSACTION_ENDED = "SPLIT_TRANSACTION_ENDED";
|
|
60
|
+
|
|
61
|
+
/**
|
|
62
|
+
* Why a split token was refused.
|
|
63
|
+
*
|
|
64
|
+
* The kind matters to a connector: only `SPLIT_SNAPSHOT_EXPIRED` means "re-run
|
|
65
|
+
* the query", and neither kind is retriable in place. Keeping the anchor in the
|
|
66
|
+
* PLAINTEXT header rather than in the AAD is what makes that distinction
|
|
67
|
+
* expressible — inside the AAD both collapse into one tag-check failure.
|
|
68
|
+
*/
|
|
69
|
+
export class SplitTokenError extends Error {
|
|
70
|
+
readonly kind: string;
|
|
71
|
+
|
|
72
|
+
constructor(kind: string, message: string) {
|
|
73
|
+
// The message carries the stable KIND, because the kind is the part a caller
|
|
74
|
+
// acts on: only SPLIT_SNAPSHOT_EXPIRED means "re-run the query", and a
|
|
75
|
+
// connector several layers up sees the message string rather than this
|
|
76
|
+
// class. Without it the three failures are indistinguishable to everyone
|
|
77
|
+
// downstream.
|
|
78
|
+
super(`[${kind}] ${message}`);
|
|
79
|
+
this.name = "SplitTokenError";
|
|
80
|
+
this.kind = kind;
|
|
81
|
+
}
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
function concat(...parts: Uint8Array[]): Uint8Array {
|
|
85
|
+
const total = parts.reduce((n, p) => n + p.length, 0);
|
|
86
|
+
const out = new Uint8Array(total);
|
|
87
|
+
let at = 0;
|
|
88
|
+
for (const p of parts) {
|
|
89
|
+
out.set(p, at);
|
|
90
|
+
at += p.length;
|
|
91
|
+
}
|
|
92
|
+
return out;
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
function equalBytes(a: Uint8Array, b: Uint8Array): boolean {
|
|
96
|
+
if (a.length !== b.length) return false;
|
|
97
|
+
let diff = 0;
|
|
98
|
+
for (let i = 0; i < a.length; i++) diff |= a[i] ^ b[i];
|
|
99
|
+
return diff === 0;
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
/**
|
|
103
|
+
* Derive the 16-byte binding check for a bind call.
|
|
104
|
+
*
|
|
105
|
+
* Minted **and** verified by the same worker, so it needs self-consistency only
|
|
106
|
+
* — it does not have to agree with any client, which is why the cross-SDK byte
|
|
107
|
+
* fixtures do not cover it. 16 bytes is a binding check, not a MAC: forgery
|
|
108
|
+
* resistance comes from the seal where a key exists, and from the uid trust
|
|
109
|
+
* boundary where one does not.
|
|
110
|
+
*
|
|
111
|
+
* Uses Web Crypto rather than a hash dependency so this works unchanged in the
|
|
112
|
+
* browser and Cloudflare Workers builds.
|
|
113
|
+
*/
|
|
114
|
+
export async function bindFingerprint(
|
|
115
|
+
schemaPath: string[],
|
|
116
|
+
functionName: string,
|
|
117
|
+
args: Uint8Array,
|
|
118
|
+
settings: Uint8Array,
|
|
119
|
+
projection: Uint8Array,
|
|
120
|
+
): Promise<Uint8Array> {
|
|
121
|
+
const enc = new TextEncoder();
|
|
122
|
+
const zero = Uint8Array.of(0);
|
|
123
|
+
const field = (label: string, value: Uint8Array) => concat(enc.encode(label), zero, value, zero);
|
|
124
|
+
const message = concat(
|
|
125
|
+
AAD_PREFIX,
|
|
126
|
+
field("schema_path", enc.encode(JSON.stringify(schemaPath))),
|
|
127
|
+
field("function_name", enc.encode(functionName)),
|
|
128
|
+
field("arguments", args),
|
|
129
|
+
field("settings", settings),
|
|
130
|
+
field("projection_ids", projection),
|
|
131
|
+
);
|
|
132
|
+
const digest = await crypto.subtle.digest("SHA-256", message as BufferSource);
|
|
133
|
+
return new Uint8Array(digest).slice(0, FINGERPRINT_LEN);
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
/**
|
|
137
|
+
* AAD for a sealed split payload: the plaintext header plus the caller identity.
|
|
138
|
+
*
|
|
139
|
+
* The identity half is load-bearing, not incidental — it stops a token minted for
|
|
140
|
+
* one principal being replayed by another, exactly as the attach envelope does. A
|
|
141
|
+
* split token names data (files, offsets, tenant partitions), so dropping it here
|
|
142
|
+
* while keeping it on attach would be a regression.
|
|
143
|
+
*/
|
|
144
|
+
function splitTokenAad(header: Uint8Array, auth: AuthContext | undefined): Uint8Array {
|
|
145
|
+
return concat(header, identityTail(auth));
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
/**
|
|
149
|
+
* Encode the consistency anchor.
|
|
150
|
+
*
|
|
151
|
+
* `catalog_version` is the counter that MOVES within an attach, so it is what a
|
|
152
|
+
* plan is pinned to; `resolved_data_version` is fixed at attach and would say
|
|
153
|
+
* nothing about staleness.
|
|
154
|
+
*/
|
|
155
|
+
export function splitAnchor(catalogVersion: bigint | number): Uint8Array {
|
|
156
|
+
const out = new Uint8Array(8);
|
|
157
|
+
new DataView(out.buffer).setBigInt64(0, BigInt(catalogVersion), true);
|
|
158
|
+
return out;
|
|
159
|
+
}
|
|
160
|
+
|
|
161
|
+
/** Stamp (and, when a key exists, seal) a worker payload into a split token. */
|
|
162
|
+
export async function buildSplitToken(opts: {
|
|
163
|
+
payload: Uint8Array;
|
|
164
|
+
fingerprint: Uint8Array;
|
|
165
|
+
anchor: Uint8Array;
|
|
166
|
+
signingKey?: Uint8Array;
|
|
167
|
+
auth?: AuthContext;
|
|
168
|
+
}): Promise<Uint8Array> {
|
|
169
|
+
const { payload, fingerprint, anchor, signingKey, auth } = opts;
|
|
170
|
+
if (fingerprint.length !== FINGERPRINT_LEN) {
|
|
171
|
+
throw new SplitTokenError(
|
|
172
|
+
SPLIT_TOKEN_INVALID,
|
|
173
|
+
`bind_fingerprint must be ${FINGERPRINT_LEN} bytes, got ${fingerprint.length}`,
|
|
174
|
+
);
|
|
175
|
+
}
|
|
176
|
+
if (anchor.length > 0xffff) {
|
|
177
|
+
throw new SplitTokenError(
|
|
178
|
+
SPLIT_TOKEN_INVALID,
|
|
179
|
+
`consistency_anchor too long: ${anchor.length} bytes exceeds u16`,
|
|
180
|
+
);
|
|
181
|
+
}
|
|
182
|
+
|
|
183
|
+
const head = new Uint8Array(4);
|
|
184
|
+
head[0] = SPLIT_TOKEN_FORMAT_VERSION;
|
|
185
|
+
head[1] = signingKey ? FLAG_PAYLOAD_SEALED : 0;
|
|
186
|
+
new DataView(head.buffer).setUint16(2, anchor.length, true);
|
|
187
|
+
const body = concat(head, fingerprint, anchor);
|
|
188
|
+
|
|
189
|
+
if (!signingKey) return concat(body, payload);
|
|
190
|
+
const sealed = await sealBytes(payload, signingKey, splitTokenAad(body, auth), SEAL_VERSION);
|
|
191
|
+
return concat(body, sealed);
|
|
192
|
+
}
|
|
193
|
+
|
|
194
|
+
/**
|
|
195
|
+
* Verify a split token and return the worker's payload.
|
|
196
|
+
*
|
|
197
|
+
* `expectedFingerprint` and `currentAnchor` are optional; omitting one skips that
|
|
198
|
+
* check.
|
|
199
|
+
*/
|
|
200
|
+
export async function openSplitToken(
|
|
201
|
+
token: Uint8Array,
|
|
202
|
+
opts: {
|
|
203
|
+
signingKey?: Uint8Array;
|
|
204
|
+
auth?: AuthContext;
|
|
205
|
+
expectedFingerprint?: Uint8Array;
|
|
206
|
+
currentAnchor?: Uint8Array;
|
|
207
|
+
} = {},
|
|
208
|
+
): Promise<Uint8Array> {
|
|
209
|
+
const { signingKey, auth, expectedFingerprint, currentAnchor } = opts;
|
|
210
|
+
|
|
211
|
+
if (token.length < HEADER_LEN) {
|
|
212
|
+
throw new SplitTokenError(
|
|
213
|
+
SPLIT_TOKEN_INVALID,
|
|
214
|
+
`split token too short: ${token.length} bytes, need at least ${HEADER_LEN}`,
|
|
215
|
+
);
|
|
216
|
+
}
|
|
217
|
+
const version = token[0];
|
|
218
|
+
const flags = token[1];
|
|
219
|
+
const view = new DataView(token.buffer, token.byteOffset, token.byteLength);
|
|
220
|
+
const anchorLen = view.getUint16(2, true);
|
|
221
|
+
|
|
222
|
+
if (version !== SPLIT_TOKEN_FORMAT_VERSION) {
|
|
223
|
+
throw new SplitTokenError(
|
|
224
|
+
SPLIT_TOKEN_INVALID,
|
|
225
|
+
`unsupported split-token format_version ${version}; this worker speaks ` +
|
|
226
|
+
`${SPLIT_TOKEN_FORMAT_VERSION}`,
|
|
227
|
+
);
|
|
228
|
+
}
|
|
229
|
+
if (flags & RESERVED_FLAGS_MASK) {
|
|
230
|
+
throw new SplitTokenError(
|
|
231
|
+
SPLIT_TOKEN_INVALID,
|
|
232
|
+
`split token sets reserved flag bits (flags=0x${flags.toString(16).padStart(2, "0")})`,
|
|
233
|
+
);
|
|
234
|
+
}
|
|
235
|
+
const sealed = (flags & FLAG_PAYLOAD_SEALED) !== 0;
|
|
236
|
+
|
|
237
|
+
// ---- The alg:none refusal. Load-bearing; do not relax. ----
|
|
238
|
+
// `flags` is attacker-controlled plaintext, so it may say "not sealed" on a
|
|
239
|
+
// token an attacker wrote by hand. A keyed worker that honoured that would
|
|
240
|
+
// redeem forged work without ever opening an envelope. The WORKER'S OWN KEY
|
|
241
|
+
// STATE decides, never the token.
|
|
242
|
+
if (signingKey && !sealed) {
|
|
243
|
+
throw new SplitTokenError(
|
|
244
|
+
SPLIT_TOKEN_INVALID,
|
|
245
|
+
"split token is unsealed but this worker holds a signing key; refusing. An unsealed " +
|
|
246
|
+
"token cannot be authenticated, so accepting one here would let any caller forge a " +
|
|
247
|
+
"split (alg:none).",
|
|
248
|
+
);
|
|
249
|
+
}
|
|
250
|
+
if (!signingKey && sealed) {
|
|
251
|
+
throw new SplitTokenError(
|
|
252
|
+
SPLIT_TOKEN_INVALID,
|
|
253
|
+
"split token is sealed but this worker holds no signing key; cannot open it",
|
|
254
|
+
);
|
|
255
|
+
}
|
|
256
|
+
|
|
257
|
+
const endOfAnchor = HEADER_LEN + anchorLen;
|
|
258
|
+
if (token.length < endOfAnchor) {
|
|
259
|
+
throw new SplitTokenError(
|
|
260
|
+
SPLIT_TOKEN_INVALID,
|
|
261
|
+
`split token truncated: anchor_len=${anchorLen} exceeds token length ${token.length}`,
|
|
262
|
+
);
|
|
263
|
+
}
|
|
264
|
+
|
|
265
|
+
const fingerprint = token.subarray(4, HEADER_LEN);
|
|
266
|
+
const anchor = token.subarray(HEADER_LEN, endOfAnchor);
|
|
267
|
+
const body = token.subarray(0, endOfAnchor);
|
|
268
|
+
const rest = token.subarray(endOfAnchor);
|
|
269
|
+
|
|
270
|
+
if (expectedFingerprint && !equalBytes(fingerprint, expectedFingerprint)) {
|
|
271
|
+
throw new SplitTokenError(
|
|
272
|
+
SPLIT_TOKEN_INVALID,
|
|
273
|
+
"split token was minted for a different bind (fingerprint mismatch)",
|
|
274
|
+
);
|
|
275
|
+
}
|
|
276
|
+
// Anchor check AFTER the bind check, and as its own kind: "read version N" is
|
|
277
|
+
// a different situation from "this token is not yours".
|
|
278
|
+
if (currentAnchor && !equalBytes(anchor, currentAnchor)) {
|
|
279
|
+
throw new SplitTokenError(SPLIT_SNAPSHOT_EXPIRED, "split snapshot expired; re-run the query");
|
|
280
|
+
}
|
|
281
|
+
|
|
282
|
+
if (!sealed) return new Uint8Array(rest);
|
|
283
|
+
try {
|
|
284
|
+
return await openBytes(
|
|
285
|
+
rest,
|
|
286
|
+
signingKey as Uint8Array,
|
|
287
|
+
splitTokenAad(body, auth),
|
|
288
|
+
SEAL_VERSION,
|
|
289
|
+
"split token",
|
|
290
|
+
);
|
|
291
|
+
} catch (err) {
|
|
292
|
+
throw new SplitTokenError(
|
|
293
|
+
SPLIT_TOKEN_INVALID,
|
|
294
|
+
`split token failed authentication: ${err instanceof Error ? err.message : String(err)}`,
|
|
295
|
+
);
|
|
296
|
+
}
|
|
297
|
+
}
|
package/src/types.ts
CHANGED
|
@@ -22,6 +22,15 @@ export enum NullHandling {
|
|
|
22
22
|
SPECIAL = "SPECIAL",
|
|
23
23
|
}
|
|
24
24
|
|
|
25
|
+
export enum ArgumentMonotonicity {
|
|
26
|
+
UNKNOWN = "UNKNOWN",
|
|
27
|
+
CONSTANT = "CONSTANT",
|
|
28
|
+
NON_DECREASING = "NON_DECREASING",
|
|
29
|
+
STRICTLY_INCREASING = "STRICTLY_INCREASING",
|
|
30
|
+
NON_INCREASING = "NON_INCREASING",
|
|
31
|
+
STRICTLY_DECREASING = "STRICTLY_DECREASING",
|
|
32
|
+
}
|
|
33
|
+
|
|
25
34
|
export enum OrderPreservation {
|
|
26
35
|
/** Output rows are in same order as input rows (DuckDB INSERTION_ORDER). */
|
|
27
36
|
PRESERVES_ORDER = "PRESERVES_ORDER",
|
package/src/worker-cf-entry.ts
CHANGED
|
@@ -17,7 +17,8 @@ export { createVgiFetch } from "./http/fetch.js";
|
|
|
17
17
|
export type { VgiFetchOptions } from "./http/fetch.js";
|
|
18
18
|
|
|
19
19
|
// Re-export the public types CF Worker authors need to wire a registry +
|
|
20
|
-
// catalog. They import from "vgi/worker-cf" rather than from
|
|
20
|
+
// catalog. They import from "@query-farm/vgi/worker-cf" rather than from
|
|
21
|
+
// "@query-farm/vgi" directly
|
|
21
22
|
// so the build-condition resolution picks the flechette Arrow backend.
|
|
22
23
|
export {
|
|
23
24
|
defineScalarFunction,
|