@query-farm/vgi 0.28.0 → 0.29.0

package/src/request-auth.ts

@@ -0,0 +1,73 @@
+// Copyright 2025, 2026 Query Farm LLC - https://query.farm
+// Request-scoped caller identity, for the code paths the RPC framework cannot
+// hand a CallContext to.
+//
+// Every unary VGI method receives the caller's `AuthContext` directly — the
+// dispatcher passes the OutputCollector, which *is* a CallContext. The stream
+// `init` method does not: `@query-farm/vgi-rpc` types stream initialization as
+// `(params) => state`, with no context parameter, on both the pipe and HTTP
+// dispatchers. That is a framework signature this package cannot change.
+//
+// It matters because `init` opens two identity-bound AEAD envelopes:
+// `attach_opaque_data` (sealed under `attachAad(auth)` by `catalog_attach`) and
+// every split token (sealed under `splitTokenAad(body, auth)` by
+// `table_function_plan`). Both of those minting sites ARE unary and therefore
+// seal under the REAL principal. Opening them under a hard-coded `undefined`
+// yields the anonymous identity tail, so the AAD only ever matches for an
+// anonymous caller: an authenticated one gets `OpaqueDataRejectedError:
+// attach_opaque_data not recognized` on the first scan. Anonymous workers never
+// saw it, which is why it survived — the example HTTP workers authenticate
+// nobody.
+//
+// The reference implementations solve this with an ambient, request-scoped
+// identity rather than a parameter: vgi-python reads a `ContextVar` via
+// `current_auth()`; vgi-go's RPC framework threads a `*CallContext` into
+// `handleInit`. This is the TypeScript equivalent — an `AsyncLocalStorage`
+// scope opened per HTTP request by `createVgiFetch` (see `http/fetch.ts`),
+// read here.
+//
+// Deliberately a registry rather than a direct `node:async_hooks` import: this
+// module is reachable from the protocol handlers, which also run under stdio,
+// AF_UNIX and browser builds. Only the HTTP entry points install a scope, and
+// they load `node:async_hooks` defensively. With no scope installed
+// `currentRequestAuth()` returns `undefined`, which is exactly the previous
+// behaviour — and correct, because a worker with no signing key seals nothing.
+
+import type { AuthContext } from "@query-farm/vgi-rpc";
+
+/** Mutable per-request cell. The scope is opened before the identity is known
+ *  (authentication happens inside the handler), so the holder is published
+ *  first and filled in when the framework reports the principal. */
+export interface RequestAuthHolder {
+  auth?: AuthContext;
+}
+
+/** The slice of `AsyncLocalStorage` this module needs. Structural on purpose:
+ *  it keeps `node:async_hooks` out of every non-HTTP build, and lets a test
+ *  install a trivial stub. */
+export interface RequestAuthScope {
+  getStore(): RequestAuthHolder | undefined;
+}
+
+let scope: RequestAuthScope | undefined;
+
+/**
+ * Publish the scope `currentRequestAuth()` reads. Called once per process by
+ * the HTTP entry point; idempotent for the same scope.
+ */
+export function setRequestAuthScope(next: RequestAuthScope | undefined): void {
+  scope = next;
+}
+
+/**
+ * The authenticated principal of the request being served on this async
+ * context, or `undefined` when there is none — no scope installed (stdio /
+ * AF_UNIX / a host that has no `AsyncLocalStorage`), or an unauthenticated
+ * request.
+ *
+ * `undefined` and an anonymous `AuthContext` produce the same identity tail, so
+ * the two cases need not be distinguished by callers.
+ */
+export function currentRequestAuth(): AuthContext | undefined {
+  return scope?.getStore()?.auth;
+}

package/src/serve-entry.ts

@@ -23,6 +23,7 @@
 
 import type { CatalogInterface } from "./catalog/interface.js";
 import type { FunctionRegistry } from "./functions/registry.js";
+import type { AuthenticateFn } from "@query-farm/vgi-rpc";
 import { createVgiFetch } from "./http/fetch.js";
 
 /** Environment variables `serveVgiWorker` reads. Injectable for testing. */
@@ -68,6 +69,9 @@ export interface ServeVgiWorkerOptions {
   quiet?: boolean;
   /** Environment source (default `process.env`). */
   env?: ServeEnv;
+  /** Authenticates each request, returning the caller's `AuthContext`.
+   *  Omit for an anonymous worker (the default). */
+  authenticate?: AuthenticateFn;
 }
 
 /** The slice of `Bun.serve`'s return value this module uses. */
@@ -194,6 +198,7 @@ export function createVgiWorkerFetch(
     corsOrigins,
     repositoryUrl: opts.repositoryUrl,
     landingInfo: { name: opts.name, doc: opts.doc, version: opts.version },
+    authenticate: opts.authenticate,
   });
 }
 

package/src/split-token.ts

@@ -0,0 +1,297 @@
+// Copyright 2025, 2026 Query Farm LLC - https://query.farm
+
+/**
+ * Split-token envelope: the framework's wrapper around a worker's split payload.
+ *
+ * A split token *names* a unit of scan work so a distributed engine can
+ * re-request exactly the work it was handed. The worker supplies only the
+ * payload; everything around it is stamped here, so an author cannot forget the
+ * consistency anchor or mis-bind the fingerprint, and never writes crypto.
+ *
+ * Layout (little-endian, fixed prefix) — byte-identical across every SDK:
+ *
+ * ```
+ * offset  size  field
+ * 0       1     format_version      currently 1
+ * 1       1     flags               bit0 = payload_sealed; bits 1-7 reserved, MUST be 0
+ * 2       2     anchor_len          u16 LE
+ * 4       16    bind_fingerprint    truncated SHA-256 of the bind identity
+ * 20      var   consistency_anchor  anchor_len bytes
+ * 20+n    var   payload             the worker's own bytes
+ * ```
+ *
+ * **The header is plaintext on every transport; only the payload is sealed.**
+ * That is not a preference: a worker has no signing key on subprocess and unix,
+ * which is DuckDB's primary path, so a header readable only through AEAD would be
+ * unreadable exactly where DuckDB runs. It also matters for streaming — a
+ * checkpointed position must survive key rotation.
+ *
+ * @module
+ */
+
+import type { AuthContext } from "@query-farm/vgi-rpc";
+
+import { identityTail, openBytes, sealBytes } from "./crypto.js";
+
+/** Envelope format version. Checked unconditionally, before anything else. */
+export const SPLIT_TOKEN_FORMAT_VERSION = 1;
+
+/** bit0 of `flags`: the payload is AEAD-sealed rather than plaintext. */
+const FLAG_PAYLOAD_SEALED = 0x01;
+
+/** bits 1-7 are reserved and MUST be zero; a set bit is a forward-compat violation. */
+const RESERVED_FLAGS_MASK = 0xfe;
+
+const FINGERPRINT_LEN = 16;
+const HEADER_LEN = 4 + FINGERPRINT_LEN;
+
+/**
+ * Matches the reference default (`crypto.seal_bytes` version=1), so a token
+ * sealed by one SDK opens in another.
+ */
+const SEAL_VERSION = 1;
+
+const AAD_PREFIX = new TextEncoder().encode("vgi.split_token.v1\u0000");
+
+/** Stable error-kind strings, identical across SDKs. */
+export const SPLIT_TOKEN_INVALID = "SPLIT_TOKEN_INVALID";
+export const SPLIT_SNAPSHOT_EXPIRED = "SPLIT_SNAPSHOT_EXPIRED";
+export const SPLIT_TRANSACTION_ENDED = "SPLIT_TRANSACTION_ENDED";
+
+/**
+ * Why a split token was refused.
+ *
+ * The kind matters to a connector: only `SPLIT_SNAPSHOT_EXPIRED` means "re-run
+ * the query", and neither kind is retriable in place. Keeping the anchor in the
+ * PLAINTEXT header rather than in the AAD is what makes that distinction
+ * expressible — inside the AAD both collapse into one tag-check failure.
+ */
+export class SplitTokenError extends Error {
+  readonly kind: string;
+
+  constructor(kind: string, message: string) {
+    // The message carries the stable KIND, because the kind is the part a caller
+    // acts on: only SPLIT_SNAPSHOT_EXPIRED means "re-run the query", and a
+    // connector several layers up sees the message string rather than this
+    // class. Without it the three failures are indistinguishable to everyone
+    // downstream.
+    super(`[${kind}] ${message}`);
+    this.name = "SplitTokenError";
+    this.kind = kind;
+  }
+}
+
+function concat(...parts: Uint8Array[]): Uint8Array {
+  const total = parts.reduce((n, p) => n + p.length, 0);
+  const out = new Uint8Array(total);
+  let at = 0;
+  for (const p of parts) {
+    out.set(p, at);
+    at += p.length;
+  }
+  return out;
+}
+
+function equalBytes(a: Uint8Array, b: Uint8Array): boolean {
+  if (a.length !== b.length) return false;
+  let diff = 0;
+  for (let i = 0; i < a.length; i++) diff |= a[i] ^ b[i];
+  return diff === 0;
+}
+
+/**
+ * Derive the 16-byte binding check for a bind call.
+ *
+ * Minted **and** verified by the same worker, so it needs self-consistency only
+ * — it does not have to agree with any client, which is why the cross-SDK byte
+ * fixtures do not cover it. 16 bytes is a binding check, not a MAC: forgery
+ * resistance comes from the seal where a key exists, and from the uid trust
+ * boundary where one does not.
+ *
+ * Uses Web Crypto rather than a hash dependency so this works unchanged in the
+ * browser and Cloudflare Workers builds.
+ */
+export async function bindFingerprint(
+  schemaName: string,
+  functionName: string,
+  args: Uint8Array,
+  settings: Uint8Array,
+  projection: Uint8Array,
+): Promise<Uint8Array> {
+  const enc = new TextEncoder();
+  const zero = Uint8Array.of(0);
+  const field = (label: string, value: Uint8Array) => concat(enc.encode(label), zero, value, zero);
+  const message = concat(
+    AAD_PREFIX,
+    field("schema_name", enc.encode(schemaName)),
+    field("function_name", enc.encode(functionName)),
+    field("arguments", args),
+    field("settings", settings),
+    field("projection_ids", projection),
+  );
+  const digest = await crypto.subtle.digest("SHA-256", message as BufferSource);
+  return new Uint8Array(digest).slice(0, FINGERPRINT_LEN);
+}
+
+/**
+ * AAD for a sealed split payload: the plaintext header plus the caller identity.
+ *
+ * The identity half is load-bearing, not incidental — it stops a token minted for
+ * one principal being replayed by another, exactly as the attach envelope does. A
+ * split token names data (files, offsets, tenant partitions), so dropping it here
+ * while keeping it on attach would be a regression.
+ */
+function splitTokenAad(header: Uint8Array, auth: AuthContext | undefined): Uint8Array {
+  return concat(header, identityTail(auth));
+}
+
+/**
+ * Encode the consistency anchor.
+ *
+ * `catalog_version` is the counter that MOVES within an attach, so it is what a
+ * plan is pinned to; `resolved_data_version` is fixed at attach and would say
+ * nothing about staleness.
+ */
+export function splitAnchor(catalogVersion: bigint | number): Uint8Array {
+  const out = new Uint8Array(8);
+  new DataView(out.buffer).setBigInt64(0, BigInt(catalogVersion), true);
+  return out;
+}
+
+/** Stamp (and, when a key exists, seal) a worker payload into a split token. */
+export async function buildSplitToken(opts: {
+  payload: Uint8Array;
+  fingerprint: Uint8Array;
+  anchor: Uint8Array;
+  signingKey?: Uint8Array;
+  auth?: AuthContext;
+}): Promise<Uint8Array> {
+  const { payload, fingerprint, anchor, signingKey, auth } = opts;
+  if (fingerprint.length !== FINGERPRINT_LEN) {
+    throw new SplitTokenError(
+      SPLIT_TOKEN_INVALID,
+      `bind_fingerprint must be ${FINGERPRINT_LEN} bytes, got ${fingerprint.length}`,
+    );
+  }
+  if (anchor.length > 0xffff) {
+    throw new SplitTokenError(
+      SPLIT_TOKEN_INVALID,
+      `consistency_anchor too long: ${anchor.length} bytes exceeds u16`,
+    );
+  }
+
+  const head = new Uint8Array(4);
+  head[0] = SPLIT_TOKEN_FORMAT_VERSION;
+  head[1] = signingKey ? FLAG_PAYLOAD_SEALED : 0;
+  new DataView(head.buffer).setUint16(2, anchor.length, true);
+  const body = concat(head, fingerprint, anchor);
+
+  if (!signingKey) return concat(body, payload);
+  const sealed = await sealBytes(payload, signingKey, splitTokenAad(body, auth), SEAL_VERSION);
+  return concat(body, sealed);
+}
+
+/**
+ * Verify a split token and return the worker's payload.
+ *
+ * `expectedFingerprint` and `currentAnchor` are optional; omitting one skips that
+ * check.
+ */
+export async function openSplitToken(
+  token: Uint8Array,
+  opts: {
+    signingKey?: Uint8Array;
+    auth?: AuthContext;
+    expectedFingerprint?: Uint8Array;
+    currentAnchor?: Uint8Array;
+  } = {},
+): Promise<Uint8Array> {
+  const { signingKey, auth, expectedFingerprint, currentAnchor } = opts;
+
+  if (token.length < HEADER_LEN) {
+    throw new SplitTokenError(
+      SPLIT_TOKEN_INVALID,
+      `split token too short: ${token.length} bytes, need at least ${HEADER_LEN}`,
+    );
+  }
+  const version = token[0];
+  const flags = token[1];
+  const view = new DataView(token.buffer, token.byteOffset, token.byteLength);
+  const anchorLen = view.getUint16(2, true);
+
+  if (version !== SPLIT_TOKEN_FORMAT_VERSION) {
+    throw new SplitTokenError(
+      SPLIT_TOKEN_INVALID,
+      `unsupported split-token format_version ${version}; this worker speaks ` +
+        `${SPLIT_TOKEN_FORMAT_VERSION}`,
+    );
+  }
+  if (flags & RESERVED_FLAGS_MASK) {
+    throw new SplitTokenError(
+      SPLIT_TOKEN_INVALID,
+      `split token sets reserved flag bits (flags=0x${flags.toString(16).padStart(2, "0")})`,
+    );
+  }
+  const sealed = (flags & FLAG_PAYLOAD_SEALED) !== 0;
+
+  // ---- The alg:none refusal. Load-bearing; do not relax. ----
+  // `flags` is attacker-controlled plaintext, so it may say "not sealed" on a
+  // token an attacker wrote by hand. A keyed worker that honoured that would
+  // redeem forged work without ever opening an envelope. The WORKER'S OWN KEY
+  // STATE decides, never the token.
+  if (signingKey && !sealed) {
+    throw new SplitTokenError(
+      SPLIT_TOKEN_INVALID,
+      "split token is unsealed but this worker holds a signing key; refusing. An unsealed " +
+        "token cannot be authenticated, so accepting one here would let any caller forge a " +
+        "split (alg:none).",
+    );
+  }
+  if (!signingKey && sealed) {
+    throw new SplitTokenError(
+      SPLIT_TOKEN_INVALID,
+      "split token is sealed but this worker holds no signing key; cannot open it",
+    );
+  }
+
+  const endOfAnchor = HEADER_LEN + anchorLen;
+  if (token.length < endOfAnchor) {
+    throw new SplitTokenError(
+      SPLIT_TOKEN_INVALID,
+      `split token truncated: anchor_len=${anchorLen} exceeds token length ${token.length}`,
+    );
+  }
+
+  const fingerprint = token.subarray(4, HEADER_LEN);
+  const anchor = token.subarray(HEADER_LEN, endOfAnchor);
+  const body = token.subarray(0, endOfAnchor);
+  const rest = token.subarray(endOfAnchor);
+
+  if (expectedFingerprint && !equalBytes(fingerprint, expectedFingerprint)) {
+    throw new SplitTokenError(
+      SPLIT_TOKEN_INVALID,
+      "split token was minted for a different bind (fingerprint mismatch)",
+    );
+  }
+  // Anchor check AFTER the bind check, and as its own kind: "read version N" is
+  // a different situation from "this token is not yours".
+  if (currentAnchor && !equalBytes(anchor, currentAnchor)) {
+    throw new SplitTokenError(SPLIT_SNAPSHOT_EXPIRED, "split snapshot expired; re-run the query");
+  }
+
+  if (!sealed) return new Uint8Array(rest);
+  try {
+    return await openBytes(
+      rest,
+      signingKey as Uint8Array,
+      splitTokenAad(body, auth),
+      SEAL_VERSION,
+      "split token",
+    );
+  } catch (err) {
+    throw new SplitTokenError(
+      SPLIT_TOKEN_INVALID,
+      `split token failed authentication: ${err instanceof Error ? err.message : String(err)}`,
+    );
+  }
+}

package/src/worker-cf-entry.ts

@@ -17,7 +17,8 @@ export { createVgiFetch } from "./http/fetch.js";
 export type { VgiFetchOptions } from "./http/fetch.js";
 
 // Re-export the public types CF Worker authors need to wire a registry +
-// catalog. They import from "vgi/worker-cf" rather than from "vgi" directly
+// catalog. They import from "@query-farm/vgi/worker-cf" rather than from
+// "@query-farm/vgi" directly
 // so the build-condition resolution picks the flechette Arrow backend.
 export {
   defineScalarFunction,

package/src/worker.ts

@@ -10,6 +10,26 @@ import type { CatalogDescriptor } from "./catalog/descriptors.js";
 import type { CatalogInterface } from "./catalog/interface.js";
 import { ReadOnlyCatalogInterface } from "./catalog/read-only.js";
 
+/**
+ * Startup/lifecycle tracing, off unless `VGI_DEBUG` is set to something other
+ * than "" or "0".
+ *
+ * These lines used to be unconditional, which put three of them in the log of
+ * every worker anyone ever deployed. They are genuinely useful when an ATTACH
+ * hangs and you need to know how far startup got, so they are kept rather than
+ * deleted — just behind a flag. Errors are NOT gated: `Worker error:` and
+ * `Worker init error:` always print.
+ *
+ * The message is a thunk so an unset flag costs no interpolation.
+ *
+ * stderr, never stdout: on the stdio transport stdout is the protocol.
+ */
+function debug(message: () => string): void {
+  const flag = process.env.VGI_DEBUG;
+  if (flag === undefined || flag === "" || flag === "0") return;
+  process.stderr.write(message() + "\n");
+}
+
 // argv parser for the launcher's `--unix PATH` / `--tcp [HOST:]PORT` /
 // `--idle-timeout SEC` contract. The C++ launcher (vgi extension) appends
 // these to the worker's argv whenever LOCATION uses the `launch:` scheme.
@@ -123,10 +143,10 @@ export class Worker {
   }
 
   run(argv: readonly string[] = process.argv.slice(2)): void {
-    process.stderr.write(`[worker] starting, pid=${process.pid}\n`);
+    debug(() => `[worker] starting, pid=${process.pid}`);
     try {
       const protocol = this.buildProtocol();
-      process.stderr.write(`[worker] protocol built\n`);
+      debug(() => `[worker] protocol built`);
 
       const launcher = parseLauncherArgs(argv);
       if (launcher.unixPath !== undefined) {
@@ -139,14 +159,14 @@ export class Worker {
         const idleTimeout = envOverride !== undefined && envOverride !== ""
           ? Number(envOverride)
           : launcher.idleTimeout;
-        process.stderr.write(
-          `[worker] AF_UNIX mode: ${launcher.unixPath} idle=${idleTimeout ?? 300}s${envOverride !== undefined && envOverride !== "" ? " (env override)" : ""}\n`,
+        debug(() =>
+          `[worker] AF_UNIX mode: ${launcher.unixPath} idle=${idleTimeout ?? 300}s${envOverride !== undefined && envOverride !== "" ? " (env override)" : ""}`,
         );
         serveUnix(protocol, {
           unixPath: launcher.unixPath,
           idleTimeout,
         }).then((handle) => handle.done).then(() => {
-          process.stderr.write(`[worker] serveUnix done (idle shutdown)\n`);
+          debug(() => `[worker] serveUnix done (idle shutdown)`);
         }).catch((err: any) => {
           process.stderr.write(`Worker error: ${err.message}\n${err.stack}\n`);
           process.exit(1);
@@ -164,15 +184,15 @@ export class Worker {
         const idleTimeout = envOverride !== undefined && envOverride !== ""
           ? Number(envOverride)
           : launcher.idleTimeout;
-        process.stderr.write(
-          `[worker] TCP mode: ${launcher.tcpHost}:${launcher.tcpPort} idle=${idleTimeout ?? 300}s${envOverride !== undefined && envOverride !== "" ? " (env override)" : ""}\n`,
+        debug(() =>
+          `[worker] TCP mode: ${launcher.tcpHost}:${launcher.tcpPort} idle=${idleTimeout ?? 300}s${envOverride !== undefined && envOverride !== "" ? " (env override)" : ""}`,
         );
         serveTcp(protocol, {
           host: launcher.tcpHost,
           port: launcher.tcpPort,
           idleTimeout,
         }).then((handle) => handle.done).then(() => {
-          process.stderr.write(`[worker] serveTcp done (idle shutdown)\n`);
+          debug(() => `[worker] serveTcp done (idle shutdown)`);
         }).catch((err: any) => {
           process.stderr.write(`Worker error: ${err.message}\n${err.stack}\n`);
           process.exit(1);
@@ -181,9 +201,9 @@ export class Worker {
       }
 
       const server = new VgiRpcServer(protocol);
-      process.stderr.write(`[worker] server created, calling run()\n`);
+      debug(() => `[worker] server created, calling run()`);
       server.run().then(() => {
-        process.stderr.write(`[worker] server.run() resolved cleanly\n`);
+        debug(() => `[worker] server.run() resolved cleanly`);
       }).catch((err) => {
         process.stderr.write(`Worker error: ${err.message}\n${err.stack}\n`);
         process.exit(1);