@query-farm/vgi 0.6.0

package/src/types.ts

@@ -0,0 +1,71 @@
+// Copyright 2025, 2026 Query Farm LLC - https://query.farm
+// VGI core types - enums and constants matching Python wire format exactly.
+
+export enum FunctionType {
+  SCALAR = "scalar",
+  TABLE = "table",
+  AGGREGATE = "aggregate",
+}
+
+export enum FunctionStability {
+  CONSISTENT = "CONSISTENT",
+  VOLATILE = "VOLATILE",
+  CONSISTENT_WITHIN_QUERY = "CONSISTENT_WITHIN_QUERY",
+}
+
+export enum NullHandling {
+  DEFAULT = "DEFAULT",
+  SPECIAL = "SPECIAL",
+}
+
+export enum OrderPreservation {
+  /** Output rows are in same order as input rows (DuckDB INSERTION_ORDER). */
+  PRESERVES_ORDER = "PRESERVES_ORDER",
+  /** Output order is undefined; may be reordered (DuckDB NO_ORDER). */
+  NO_ORDER_GUARANTEE = "NO_ORDER_GUARANTEE",
+  /** Output is in a fixed mandatory order; DuckDB serialises the pipeline
+   *  (single worker) to preserve it (DuckDB FIXED_ORDER). */
+  FIXED_ORDER = "FIXED_ORDER",
+}
+
+export enum OrderDependence {
+  ORDER_DEPENDENT = "ORDER_DEPENDENT",
+  NOT_ORDER_DEPENDENT = "NOT_ORDER_DEPENDENT",
+}
+
+export enum DistinctDependence {
+  DISTINCT_DEPENDENT = "DISTINCT_DEPENDENT",
+  NOT_DISTINCT_DEPENDENT = "NOT_DISTINCT_DEPENDENT",
+}
+
+export enum TableInOutPhase {
+  INPUT = "INPUT",
+  FINALIZE = "FINALIZE",
+  // Sink+Source (TableBufferingFunction) init phases. TABLE_BUFFERING is the
+  // sink-side init (persist init metadata so any pool worker can serve
+  // process/combine); TABLE_BUFFERING_FINALIZE is the per-finalize_state_id
+  // Source-stream init.
+  TABLE_BUFFERING = "TABLE_BUFFERING",
+  TABLE_BUFFERING_FINALIZE = "TABLE_BUFFERING_FINALIZE",
+}
+
+// Arrow schema metadata keys (must match Python exactly - bytes in Python, strings in TS)
+export const VGI_ARG_KEY = "vgi_arg";
+export const VGI_ARG_NAMED = "named";
+export const VGI_TYPE_KEY = "vgi_type";
+export const VGI_TYPE_TABLE = "table";
+export const VGI_TYPE_ANY = "any";
+export const VGI_VARARGS_KEY = "vgi_varargs";
+export const VGI_VARARGS_TRUE = "true";
+export const VGI_CONST_KEY = "vgi_const";
+export const VGI_CONST_TRUE = "true";
+// Per-argument description (UTF-8; presence-only). `vgi_doc_*` prefix reserved.
+export const VGI_DOC_KEY = "vgi_doc";
+
+export interface TableCardinality {
+  estimate: number | null;
+  max: number | null;
+}
+
+// Default max workers (matches Python DEFAULT_MAX_WORKERS = 99999)
+export const DEFAULT_MAX_WORKERS = 99999;

package/src/util/arrow/index.ts

@@ -0,0 +1,23 @@
+// Copyright 2025, 2026 Query Farm LLC - https://query.farm
+// Back-compat shim. Arrow utilities have moved to src/arrow/ (the backend
+// facade). This file is kept so existing imports of `../util/arrow/index.js`
+// continue to work during step 2 of the facade migration. Once all callers
+// have switched to `../arrow/index.js`, this file can be deleted.
+
+export {
+  emptyBatch,
+  batchFromRows,
+  batchFromColumns,
+  iterRows,
+  batchToScalarDict,
+  batchToSecretDict,
+  safeNumber,
+  decodeDictValue,
+  filterBatch,
+  projectSchema,
+  projectBatch,
+  serializeSchema,
+  deserializeSchema,
+  serializeBatch,
+  deserializeBatch,
+} from "../../arrow/index.js";

package/src/util/bytes.ts

@@ -0,0 +1,14 @@
+// Copyright 2025, 2026 Query Farm LLC - https://query.farm
+// Shared byte-conversion utility. Browser-safe (no bare Buffer.isBuffer).
+
+/** Convert any binary-ish value to a Uint8Array. Returns empty array for null/undefined. */
+export function toUint8Array(val: any): Uint8Array {
+  if (val instanceof Uint8Array) return val;
+  if (val instanceof ArrayBuffer) return new Uint8Array(val);
+  if (typeof Buffer !== "undefined" && Buffer.isBuffer(val))
+    return new Uint8Array(val);
+  if (val && val.buffer instanceof ArrayBuffer) {
+    return new Uint8Array(val.buffer, val.byteOffset, val.byteLength);
+  }
+  return new Uint8Array(0);
+}

package/src/util/statistics.ts

@@ -0,0 +1,40 @@
+// Copyright 2025, 2026 Query Farm LLC - https://query.farm
+// Column statistics: types and sparse-union wire serialization.
+// Mirrors vgi-python's serialize_column_statistics in catalog_interface.py.
+//
+// The wire format is a single RecordBatch with schema
+//   column_name: utf8
+//   min: sparse_union<T0, T1, ...>
+//   max: sparse_union<T0, T1, ...>
+//   has_null: bool
+//   has_not_null: bool
+//   distinct_count: int64
+//   contains_unicode: bool
+//   max_string_length: uint64
+// where Ti are the distinct Arrow types present among the stats' min/max values.
+// A row's type_id picks which union child holds its real min/max; the other
+// children carry null at that slot (sparse-union invariant).
+//
+// SparseUnion data construction differs significantly between arrow-js
+// (typeIds buffer + per-type children built via makeData) and flechette
+// (typeIdForValue closure on columnFromArray), so the actual batch
+// assembly is delegated to the active backend via the facade.
+
+import {
+  type ColumnStatistics,
+  buildStatisticsBatch,
+  serializeBatch,
+} from "../arrow/index.js";
+
+export type { ColumnStatistics };
+
+/**
+ * Serialize column statistics to IPC bytes per vgi-python's wire format.
+ * Returns an empty-stats batch when `stats` is empty (matching Python).
+ */
+export function serializeColumnStatistics(
+  stats: ColumnStatistics[],
+  _cacheMaxAgeSeconds?: number | null,
+): Uint8Array {
+  return serializeBatch(buildStatisticsBatch(stats));
+}

package/src/worker-cf-entry.ts

@@ -0,0 +1,172 @@
+// Copyright 2025, 2026 Query Farm LLC - https://query.farm
+// Cloudflare Workers entrypoint for the VGI worker.
+//
+// Imports the same building blocks as the Bun HTTP worker (Protocol, registry,
+// catalog interface, state serializer) but exposes the handler as `default.fetch`
+// — the shape `wrangler dev` and the Workers runtime expect.
+//
+// Build picks the flechette Arrow backend automatically because this entry is
+// referenced under the `workerd`/`worker`/`browser` conditional export, which
+// resolves `#arrow-impl` -> impl-flechette.
+//
+// Consumers wire their own catalog and function registry; the helper here is
+// `createVgiFetch` rather than a hard-coded singleton, so a CF Worker module
+// can compose multiple registries the same way the Bun worker does.
+
+import { createHttpHandler, unpackStateToken, type Protocol } from "@query-farm/vgi-rpc";
+import { arrowStateSerializer } from "./protocol/state-serializer.js";
+import { buildVgiProtocol, type ProtocolConfig } from "./protocol/dispatch.js";
+
+export interface VgiFetchOptions {
+  /** Wire-protocol config (registry + catalogInterface). */
+  protocol: Omit<ProtocolConfig, "recoverExchangeState">;
+  /** HMAC key for state-token signing. Pass a stable, secret 32-byte key
+   *  (e.g. derived from a Wrangler secret). Required because Workers don't
+   *  preserve in-memory state across requests/instances. */
+  signingKey: Uint8Array;
+  /** State-token TTL in seconds (default 3600). */
+  tokenTtl?: number;
+  /** URL path prefix for VGI requests (default "/vgi"). */
+  prefix?: string;
+  /** Server ID for state-token attribution (default "vgi-cf"). */
+  serverId?: string;
+}
+
+/**
+ * Build a fetch handler suitable for `export default { fetch }` in a CF
+ * Worker module. The returned handler is async-safe across Workers'
+ * isolate-per-request execution model — all state round-trips through the
+ * signed state token created here.
+ */
+export function createVgiFetch(opts: VgiFetchOptions): (req: Request) => Promise<Response> {
+  const tokenTtl = opts.tokenTtl ?? 3600;
+  const prefix = opts.prefix ?? "/vgi";
+  const serverId = opts.serverId ?? "vgi-cf";
+
+  const protocol: Protocol = buildVgiProtocol({
+    signingKey: opts.signingKey,
+    ...opts.protocol,
+    recoverExchangeState: async (opaqueData: Uint8Array) => {
+      const tokenString = new TextDecoder().decode(opaqueData);
+      // principal binding is enforced by the HTTP handler on the request that
+      // carried this token; the recovery path itself is not principal-scoped.
+      const unpacked = await unpackStateToken(tokenString, opts.signingKey, tokenTtl, undefined);
+      return arrowStateSerializer.deserialize(unpacked.stateBytes);
+    },
+  });
+
+  const handler = createHttpHandler(protocol, {
+    prefix,
+    serverId,
+    tokenKey: opts.signingKey,
+    tokenTtl,
+    stateSerializer: arrowStateSerializer,
+  });
+  return async (req: Request) => handler(req);
+}
+
+// Re-export the public types CF Worker authors need to wire a registry +
+// catalog. They import from "vgi/worker-cf" rather than from "vgi" directly
+// so the build-condition resolution picks the flechette Arrow backend.
+export {
+  defineScalarFunction,
+  defineTableFunction,
+  defineAggregate,
+  defineTableInOutFunction,
+  FunctionRegistry,
+  ReadOnlyCatalogInterface,
+  CompositeCatalogInterface,
+  CatalogInterface,
+  Arguments,
+  FunctionType,
+  serializeAttachOptionSpec,
+  serializeAttachOptionSpecs,
+  serializeColumnStatistics,
+  BindResultSchema,
+  ScanFunctionResultSchema,
+} from "./index.core.js";
+
+export type {
+  AttachOptionSpec,
+  ColumnStatistics,
+  CatalogAttachResult,
+  CatalogDescriptor,
+  CatalogInfo,
+  TableInfo,
+  AttachOpaqueData,
+  TransactionOpaqueData,
+  SchemaInfo,
+  VgiFunction,
+  TableProcessParams,
+} from "./index.core.js";
+
+export type { ProtocolConfig } from "./protocol/dispatch.js";
+
+// Backend-agnostic Arrow facade. Same source resolves to arrow-js on
+// Node/Bun and flechette under workerd/worker/browser via the package's
+// `#arrow-impl` conditional import. Worker authors should always go
+// through these factories instead of importing arrow-js / flechette
+// directly.
+export {
+  // Type guards
+  TypeId,
+  isNull,
+  isInt,
+  isFloat,
+  isBinary,
+  isUtf8,
+  isBool,
+  isDecimal,
+  isDate,
+  isTime,
+  isTimestamp,
+  isInterval,
+  isList,
+  isStruct,
+  isUnion,
+  isFixedSizeBinary,
+  isFixedSizeList,
+  isMap,
+  isDuration,
+  isDictionary,
+  isBatch,
+  // Active backend's surface
+  backend,
+  schema, field,
+  nullType, bool,
+  int, int8, int16, int32, int64,
+  uint8, uint16, uint32, uint64,
+  float16, float32, float64,
+  utf8, binary, fixedSizeBinary,
+  decimal, decimal128, decimal256,
+  date, dateDay, dateMillisecond,
+  time, timeSecond, timeMillisecond, timeMicrosecond, timeNanosecond,
+  timestamp, duration, interval,
+  list, fixedSizeList, struct, map,
+  dictionary,
+  union, sparseUnion, denseUnion,
+  TimeUnit, DateUnit, IntervalUnit, UnionMode,
+  emptyBatch, batchFromRows, batchFromColumns, columnFromArray,
+  iterRows, batchToScalarDict, batchToSecretDict, safeNumber, decodeDictValue,
+  filterBatch, projectSchema, projectBatch,
+  serializeSchema, deserializeSchema, serializeBatch, deserializeBatch,
+  buildStatisticsBatch,
+} from "./arrow/index.js";
+
+export type {
+  VgiTypeId,
+  VgiDataType,
+  VgiField,
+  VgiSchema,
+  VgiColumn,
+  VgiColumnData,
+  VgiBatch,
+  VgiBackendInfo,
+} from "./arrow/index.js";
+
+// State serializer (HMAC-signed token round-trip across stateless requests)
+export { arrowStateSerializer, EXCHANGE_STATE_SCHEMA } from "./protocol/state-serializer.js";
+
+// Re-export from vgi-rpc for convenience.
+export { OutputCollector, AuthContext } from "@query-farm/vgi-rpc";
+export type { LogMessage } from "@query-farm/vgi-rpc";

package/src/worker.ts

@@ -0,0 +1,212 @@
+// Copyright 2025, 2026 Query Farm LLC - https://query.farm
+// VGI Worker: main entry point for running a VGI function server.
+
+import { VgiRpcServer, serveUnix, serveTcp, serveStream as serveStreamRpc } from "@query-farm/vgi-rpc";
+import { FunctionRegistry } from "./functions/registry.js";
+import type { VgiFunction } from "./functions/types.js";
+import { buildVgiProtocol } from "./protocol/dispatch.js";
+import type { Protocol } from "@query-farm/vgi-rpc";
+import type { CatalogDescriptor } from "./catalog/descriptors.js";
+import type { CatalogInterface } from "./catalog/interface.js";
+import { ReadOnlyCatalogInterface } from "./catalog/read-only.js";
+
+// argv parser for the launcher's `--unix PATH` / `--tcp [HOST:]PORT` /
+// `--idle-timeout SEC` contract. The C++ launcher (vgi extension) appends
+// these to the worker's argv whenever LOCATION uses the `launch:` scheme.
+// Unknown args pass through.
+interface LauncherArgs {
+  unixPath?: string;
+  /** TCP bind host, defaulting to 127.0.0.1 when `--tcp` gives a bare port. */
+  tcpHost?: string;
+  /** TCP bind port; present iff `--tcp` was passed. */
+  tcpPort?: number;
+  idleTimeout?: number;
+}
+
+// Parse a `[HOST:]PORT` --tcp bind spec into a LauncherArgs. A bare PORT (no
+// colon) binds 127.0.0.1; an empty host (leading ":") also defaults to loopback.
+function parseTcpSpec(spec: string, out: LauncherArgs): void {
+  const idx = spec.lastIndexOf(":");
+  if (idx >= 0) {
+    const h = spec.slice(0, idx);
+    out.tcpHost = h === "" ? "127.0.0.1" : h;
+    out.tcpPort = Number(spec.slice(idx + 1));
+  } else {
+    out.tcpHost = "127.0.0.1";
+    out.tcpPort = Number(spec);
+  }
+}
+
+function parseLauncherArgs(argv: readonly string[]): LauncherArgs {
+  const out: LauncherArgs = {};
+  for (let i = 0; i < argv.length; i++) {
+    const a = argv[i];
+    if (a === "--unix" && i + 1 < argv.length) {
+      out.unixPath = argv[++i];
+    } else if (a.startsWith("--unix=")) {
+      out.unixPath = a.slice("--unix=".length);
+    } else if (a === "--tcp" && i + 1 < argv.length) {
+      parseTcpSpec(argv[++i], out);
+    } else if (a.startsWith("--tcp=")) {
+      parseTcpSpec(a.slice("--tcp=".length), out);
+    } else if (a === "--idle-timeout" && i + 1 < argv.length) {
+      out.idleTimeout = Number(argv[++i]);
+    } else if (a.startsWith("--idle-timeout=")) {
+      out.idleTimeout = Number(a.slice("--idle-timeout=".length));
+    }
+  }
+  return out;
+}
+
+export interface WorkerConfig {
+  functions?: VgiFunction[];
+  catalog?: CatalogDescriptor;
+  catalogInterface?: CatalogInterface;
+  /** Factory that receives the built ReadOnlyCatalogInterface and returns a custom catalog. */
+  catalogInterfaceFactory?: (base: ReadOnlyCatalogInterface) => CatalogInterface;
+  catalogName?: string;
+}
+
+export class Worker {
+  private _registry: FunctionRegistry;
+  private _catalogInterface?: CatalogInterface;
+  private _catalogName?: string;
+
+  constructor(config: WorkerConfig) {
+    this._registry = new FunctionRegistry();
+    this._catalogName = config.catalogName;
+
+    // Register explicit functions
+    if (config.functions) {
+      for (const func of config.functions) {
+        this._registry.register(func);
+      }
+    }
+
+    // Register functions from catalog descriptor
+    if (config.catalog) {
+      for (const schema of config.catalog.schemas) {
+        if (schema.functions) {
+          for (const func of schema.functions) {
+            this._registry.register(func);
+          }
+        }
+      }
+    }
+
+    // Set up catalog interface
+    if (config.catalogInterface) {
+      this._catalogInterface = config.catalogInterface;
+    } else if (config.catalog) {
+      const base = new ReadOnlyCatalogInterface(
+        config.catalog,
+        this._registry
+      );
+      this._catalogInterface = config.catalogInterfaceFactory
+        ? config.catalogInterfaceFactory(base)
+        : base;
+    }
+  }
+
+  run(argv: readonly string[] = process.argv.slice(2)): void {
+    process.stderr.write(`[worker] starting, pid=${process.pid}\n`);
+    try {
+      const protocol = this.buildProtocol();
+      process.stderr.write(`[worker] protocol built\n`);
+
+      const launcher = parseLauncherArgs(argv);
+      if (launcher.unixPath !== undefined) {
+        // VGI_WORKER_IDLE_TIMEOUT overrides whatever the C++ launcher
+        // appended via --idle-timeout. Useful in test setups where the
+        // launcher's default of 300 s leaves stale workers around long
+        // after the suite has finished — set 5 (or any small value) and
+        // the worker self-shuts-down promptly when the suite ends.
+        const envOverride = process.env.VGI_WORKER_IDLE_TIMEOUT;
+        const idleTimeout = envOverride !== undefined && envOverride !== ""
+          ? Number(envOverride)
+          : launcher.idleTimeout;
+        process.stderr.write(
+          `[worker] AF_UNIX mode: ${launcher.unixPath} idle=${idleTimeout ?? 300}s${envOverride !== undefined && envOverride !== "" ? " (env override)" : ""}\n`,
+        );
+        serveUnix(protocol, {
+          unixPath: launcher.unixPath,
+          idleTimeout,
+        }).then((handle) => handle.done).then(() => {
+          process.stderr.write(`[worker] serveUnix done (idle shutdown)\n`);
+        }).catch((err: any) => {
+          process.stderr.write(`Worker error: ${err.message}\n${err.stack}\n`);
+          process.exit(1);
+        });
+        return;
+      }
+
+      if (launcher.tcpPort !== undefined) {
+        // Raw Arrow-IPC framing over TCP — the network sibling of the
+        // AF_UNIX launcher transport. serveTcp prints the
+        // `TCP:<host>:<port>` discovery line itself (with the actual bound
+        // port when tcpPort is 0). NO auth/TLS — loopback/trusted networks
+        // only; use the HTTP transport for untrusted networks.
+        const envOverride = process.env.VGI_WORKER_IDLE_TIMEOUT;
+        const idleTimeout = envOverride !== undefined && envOverride !== ""
+          ? Number(envOverride)
+          : launcher.idleTimeout;
+        process.stderr.write(
+          `[worker] TCP mode: ${launcher.tcpHost}:${launcher.tcpPort} idle=${idleTimeout ?? 300}s${envOverride !== undefined && envOverride !== "" ? " (env override)" : ""}\n`,
+        );
+        serveTcp(protocol, {
+          host: launcher.tcpHost,
+          port: launcher.tcpPort,
+          idleTimeout,
+        }).then((handle) => handle.done).then(() => {
+          process.stderr.write(`[worker] serveTcp done (idle shutdown)\n`);
+        }).catch((err: any) => {
+          process.stderr.write(`Worker error: ${err.message}\n${err.stack}\n`);
+          process.exit(1);
+        });
+        return;
+      }
+
+      const server = new VgiRpcServer(protocol);
+      process.stderr.write(`[worker] server created, calling run()\n`);
+      server.run().then(() => {
+        process.stderr.write(`[worker] server.run() resolved cleanly\n`);
+      }).catch((err) => {
+        process.stderr.write(`Worker error: ${err.message}\n${err.stack}\n`);
+        process.exit(1);
+      });
+    } catch (err: any) {
+      process.stderr.write(`Worker init error: ${err.message}\n${err.stack}\n`);
+      process.exit(1);
+    }
+  }
+
+  /** Build the VGI protocol from this worker's registry + catalog interface. */
+  private buildProtocol(): Protocol {
+    return buildVgiProtocol({
+      registry: this._registry,
+      catalogInterface: this._catalogInterface,
+      catalogName: this._catalogName,
+    });
+  }
+
+  /**
+   * Serve this worker over a caller-provided byte-stream pair, instead of the
+   * argv-selected stdio/unix/tcp transports `run()` uses. Resolves when the
+   * `readable` ends.
+   *
+   * This is the seam for transports that aren't a process/socket — most notably
+   * a **Web Worker**: bridge the worker's `MessagePort` to a Node `Duplex`
+   * (readable side ← `port.on('message')`, writable side → `port.postMessage`)
+   * and pass it as both arguments. The same code runs in a browser Web Worker.
+   *
+   * @param readable incoming request bytes (web `ReadableStream` or Node `Readable`).
+   * @param writable outgoing response sink (a `net.Socket`/`Duplex`, or an fd
+   *   number). Omit for the stdout fd.
+   */
+  async serveStream(
+    readable: ReadableStream<Uint8Array> | NodeJS.ReadableStream,
+    writable?: number | import("node:net").Socket,
+  ): Promise<void> {
+    await serveStreamRpc(this.buildProtocol(), { readable, writable });
+  }
+}