@query-farm/vgi 0.6.0

package/src/crypto.ts

@@ -0,0 +1,143 @@
+// Copyright 2025, 2026 Query Farm LLC - https://query.farm
+// SPDX-License-Identifier: LicenseRef-QueryFarm-Source-Available-1.0
+
+// Catalog opaque-data AEAD envelopes.
+//
+// `attach_opaque_data` and `transaction_opaque_data` are implementation-chosen
+// byte strings the catalog returns and the client round-trips back. On HTTP
+// transport (where one worker authenticates many principals) the worker seals
+// each value in an authenticated-encrypted envelope whose AAD binds the
+// caller's (domain, principal); the transaction envelope additionally binds
+// its parent attach envelope. A value sealed for one principal — or one
+// attach — cannot be opened by another.
+//
+// Workers with no signing key (subprocess / unix transports) pass values
+// through unchanged, since OS process ownership already enforces identity.
+//
+// Wire format: version(1 byte) || nonce(24 bytes) || ciphertext+tag.
+// Matches vgi-python's vgi_rpc.crypto / vgi.worker and vgi-go's vgi/crypto.go.
+
+import { xchacha20poly1305 } from "@noble/ciphers/chacha.js";
+import type { AuthContext } from "@query-farm/vgi-rpc";
+
+const _UTF8 = new TextEncoder();
+
+// v2: the inner attach plaintext is uuid(16) || catalog_bytes — catalog_attach
+// prepends a framework-minted 16-byte UUID that storage shards on. Bumped from
+// 1 so a stale v1 token (no uuid prefix) is cleanly rejected at open.
+export const ATTACH_ENVELOPE_VERSION = 2;
+export const TRANSACTION_ENVELOPE_VERSION = 2;
+
+/** Width of the framework UUID prepended to every attach plaintext. */
+export const ATTACH_UUID_LEN = 16;
+
+const NONCE_LEN = 24; // XChaCha20-Poly1305 nonce
+const TAG_LEN = 16; // Poly1305 tag
+const MIN_ENVELOPE_LEN = 1 + NONCE_LEN + TAG_LEN;
+
+const ATTACH_AAD_PREFIX = _UTF8.encode("vgi.attach_opaque_data.v1\0");
+const TRANSACTION_AAD_PREFIX = _UTF8.encode("vgi.transaction_opaque_data.v1\0");
+
+/**
+ * Thrown when an envelope cannot be opened — wrong principal, wrong parent
+ * attach, tampered, malformed, or simply unknown. Every failure mode maps to
+ * this single error so a probing caller cannot distinguish them.
+ */
+export class OpaqueDataRejectedError extends Error {
+  constructor(field = "opaque data") {
+    super(`${field} not recognized`);
+    this.name = "OpaqueDataRejectedError";
+  }
+}
+
+function concat(...parts: Uint8Array[]): Uint8Array {
+  let total = 0;
+  for (const p of parts) total += p.length;
+  const out = new Uint8Array(total);
+  let off = 0;
+  for (const p of parts) {
+    out.set(p, off);
+    off += p.length;
+  }
+  return out;
+}
+
+/**
+ * Stretch/compress an operator-supplied key to the 32 bytes XChaCha20-Poly1305
+ * requires. Exactly-32-byte keys pass through unchanged; any other length is
+ * collapsed via SHA-256. Matches the vgi-python / vgi-go ports.
+ */
+async function normalizeKey(key: Uint8Array): Promise<Uint8Array> {
+  if (key.length === 32) return key;
+  return new Uint8Array(await crypto.subtle.digest("SHA-256", key as BufferSource));
+}
+
+/**
+ * Build the identity portion of an opaque-data AAD. Unauthenticated requests
+ * get a fixed anonymous tail, so an anonymous caller cannot open an envelope
+ * sealed for a real principal.
+ */
+export function identityTail(auth: AuthContext | undefined): Uint8Array {
+  if (!auth || !auth.authenticated) return _UTF8.encode("\0anonymous");
+  const domain = _UTF8.encode(auth.domain ?? "");
+  const principal = _UTF8.encode(auth.principal ?? "");
+  return concat(Uint8Array.of(0x01), domain, Uint8Array.of(0x00), principal);
+}
+
+/** AAD for an `attach_opaque_data` envelope: prefix + caller identity. */
+export function attachAad(auth: AuthContext | undefined): Uint8Array {
+  return concat(ATTACH_AAD_PREFIX, identityTail(auth));
+}
+
+/**
+ * AAD for a `transaction_opaque_data` envelope. Binds both the caller identity
+ * and the parent attach envelope, so a transaction value minted under one
+ * attach cannot be replayed against a different attach even by the same
+ * principal.
+ */
+export function transactionAad(
+  auth: AuthContext | undefined,
+  attachEnvelope: Uint8Array,
+): Uint8Array {
+  return concat(TRANSACTION_AAD_PREFIX, identityTail(auth), Uint8Array.of(0x00), attachEnvelope);
+}
+
+/** Seal `payload` into an AEAD envelope: version || nonce || ciphertext+tag. */
+export async function sealBytes(
+  payload: Uint8Array,
+  key: Uint8Array,
+  aad: Uint8Array,
+  version: number,
+): Promise<Uint8Array> {
+  const nonce = crypto.getRandomValues(new Uint8Array(NONCE_LEN));
+  const ciphertext = xchacha20poly1305(await normalizeKey(key), nonce, aad).encrypt(payload);
+  const out = new Uint8Array(1 + NONCE_LEN + ciphertext.length);
+  out[0] = version & 0xff;
+  out.set(nonce, 1);
+  out.set(ciphertext, 1 + NONCE_LEN);
+  return out;
+}
+
+/**
+ * Open and verify an envelope produced by {@link sealBytes}. Every failure
+ * mode — malformed, wrong version, tampered, wrong key, wrong AAD
+ * (cross-principal / cross-attach replay) — throws {@link OpaqueDataRejectedError}.
+ */
+export async function openBytes(
+  token: Uint8Array,
+  key: Uint8Array,
+  aad: Uint8Array,
+  version: number,
+  field = "opaque data",
+): Promise<Uint8Array> {
+  if (token.length < MIN_ENVELOPE_LEN || token[0] !== (version & 0xff)) {
+    throw new OpaqueDataRejectedError(field);
+  }
+  const nonce = token.subarray(1, 1 + NONCE_LEN);
+  const ciphertext = token.subarray(1 + NONCE_LEN);
+  try {
+    return xchacha20poly1305(await normalizeKey(key), nonce, aad).decrypt(ciphertext);
+  } catch {
+    throw new OpaqueDataRejectedError(field);
+  }
+}

package/src/errors.ts

@@ -0,0 +1,61 @@
+// Copyright 2025, 2026 Query Farm LLC - https://query.farm
+// VGI-specific error classes.
+
+export class VgiError extends Error {
+  constructor(message: string) {
+    super(message);
+    this.name = "VgiError";
+  }
+}
+
+export class RowCountMismatchError extends VgiError {
+  constructor(expected: number, actual: number) {
+    super(
+      `Scalar function output row count (${actual}) does not match ` +
+        `input row count (${expected})`
+    );
+    this.name = "RowCountMismatchError";
+  }
+}
+
+export class FunctionNotFoundError extends VgiError {
+  constructor(name: string, available?: string[]) {
+    super(`Unknown function '${name}'`);
+    this.name = "FunctionNotFoundError";
+  }
+}
+
+export class CatalogReadOnlyError extends VgiError {
+  constructor(operation: string) {
+    super(`catalog is read-only: ${operation} is not supported`);
+    this.name = "CatalogReadOnlyError";
+  }
+}
+
+export class CatalogNotFoundError extends VgiError {
+  constructor(entity: string, name: string) {
+    super(`${entity} '${name}' not found`);
+    this.name = "CatalogNotFoundError";
+  }
+}
+
+export class CatalogAlreadyExistsError extends VgiError {
+  constructor(entity: string, name: string) {
+    super(`${entity} '${name}' already exists`);
+    this.name = "CatalogAlreadyExistsError";
+  }
+}
+
+export class ArgumentValidationError extends VgiError {
+  constructor(message: string) {
+    super(message);
+    this.name = "ArgumentValidationError";
+  }
+}
+
+export class NoCatalogError extends VgiError {
+  constructor() {
+    super("No catalog is configured for this worker");
+    this.name = "NoCatalogError";
+  }
+}

package/src/filter-pushdown/collector.ts

@@ -0,0 +1,147 @@
+// Copyright 2025, 2026 Query Farm LLC - https://query.farm
+// FilteringOutputCollector — wraps an OutputCollector and applies pushdown
+// filters to every emitted batch — plus human-readable formatters used in
+// debug logs.
+
+import { type VgiBatch, type VgiSchema, isBatch } from "../arrow/index.js";
+import type { OutputCollector } from "@query-farm/vgi-rpc";
+import { batchFromColumns } from "../util/arrow/index.js";
+import { ComparisonOp, type ExprNode, type Filter } from "./types.js";
+import { PushdownFilters } from "./evaluate.js";
+
+/**
+ * Format pushdown filters as a human-readable SQL-like string.
+ * Returns "(none)" when no filters exist.
+ */
+export function formatPushedFilters(filters: PushdownFilters | undefined): string {
+  if (!filters || filters.filters.length === 0) return "(none)";
+  const sql = filters.toSql();
+  return sql || "(none)";
+}
+
+/**
+ * Format pushdown filters using Python-style `repr()` — e.g.
+ * `PushdownFilters([ConstantFilter(n < 4999)])`. Matches vgi-python's
+ * `_format_pushed_filters_safe` output so tests that pattern-match the
+ * repr (e.g. `dynamic_filter` checking for `ConstantFilter(n <`) work
+ * identically across workers.
+ */
+export function reprPushedFilters(filters: PushdownFilters | undefined): string {
+  if (!filters || filters.filters.length === 0) return "(none)";
+  const parts = filters.filters.map(reprFilter);
+  return `PushdownFilters([${parts.join(", ")}])`;
+}
+
+function opSymbol(op: ComparisonOp): string {
+  switch (op) {
+    case ComparisonOp.EQ: return "==";
+    case ComparisonOp.NE: return "!=";
+    case ComparisonOp.GT: return ">";
+    case ComparisonOp.GE: return ">=";
+    case ComparisonOp.LT: return "<";
+    case ComparisonOp.LE: return "<=";
+  }
+}
+
+function reprFilter(f: Filter): string {
+  switch (f.type) {
+    case "constant":
+      return `ConstantFilter(${f.columnName} ${opSymbol(f.op)} ${reprValue(f.value)})`;
+    case "is_null":
+      return `IsNullFilter(${f.columnName} IS NULL)`;
+    case "is_not_null":
+      return `IsNotNullFilter(${f.columnName} IS NOT NULL)`;
+    case "in": {
+      const values = [...f.values];
+      const preview = values.length > 5
+        ? `${JSON.stringify(values.slice(0, 3))}...(${values.length} total)`
+        : JSON.stringify(values);
+      return `InFilter(${f.columnName} IN ${preview})`;
+    }
+    case "and": {
+      const kids = f.children.map(reprFilter).join(" AND ");
+      return `AndFilter(${kids})`;
+    }
+    case "or": {
+      const kids = f.children.map(reprFilter).join(" OR ");
+      return `OrFilter(${kids})`;
+    }
+    case "struct":
+      return `StructFilter(${f.columnName}.${f.childName}: ${reprFilter(f.childFilter)})`;
+    case "expression":
+      return `ExpressionFilter(${f.columnName}: ${reprExpr(f.expr)})`;
+  }
+}
+
+function reprExpr(e: ExprNode): string {
+  switch (e.expr_type) {
+    case "column_ref": return `col#${e.index}`;
+    case "constant": return reprValue(e.value);
+    case "function": return `${e.function_name}(${e.children.map(reprExpr).join(", ")})`;
+    case "comparison": return `(${reprExpr(e.left)} ${opSymbol(e.op)} ${reprExpr(e.right)})`;
+    case "conjunction":
+      return `(${e.children.map(reprExpr).join(e.conjunction_type === "and" ? " AND " : " OR ")})`;
+  }
+}
+
+function reprValue(v: any): string {
+  if (typeof v === "string") return JSON.stringify(v);
+  if (typeof v === "bigint") return String(v);
+  if (v === null || v === undefined) return "null";
+  return String(v);
+}
+
+/**
+ * Wraps an OutputCollector to automatically apply pushdown filters
+ * to every emitted batch.
+ */
+export class FilteringOutputCollector {
+  constructor(
+    private inner: OutputCollector,
+    private filters: PushdownFilters,
+  ) {}
+
+  get outputSchema(): VgiSchema {
+    return this.inner.outputSchema;
+  }
+
+  get finished(): boolean {
+    return this.inner.finished;
+  }
+
+  emit(batch: VgiBatch, metadata?: Map<string, string>): void;
+  emit(columns: Record<string, any[]>): void;
+  emit(
+    batchOrColumns: VgiBatch | Record<string, any[]>,
+    metadata?: Map<string, string>,
+  ): void {
+    let batch: VgiBatch;
+    if (isBatch(batchOrColumns)) {
+      batch = batchOrColumns;
+    } else {
+      batch = batchFromColumns(batchOrColumns as Record<string, any[]>, this.inner.outputSchema as any);
+    }
+    const filtered = this.filters.apply(batch as any);
+    this.inner.emit(filtered as any, metadata);
+  }
+
+  emitRow(values: Record<string, any>): void {
+    const columns: Record<string, any[]> = {};
+    for (const [key, value] of Object.entries(values)) {
+      columns[key] = [value];
+    }
+    this.emit(columns);
+  }
+
+  finish(): void {
+    this.inner.finish();
+  }
+
+  clientLog(
+    level: string,
+    message: string,
+    extra?: Record<string, string>,
+  ): void {
+    this.inner.clientLog(level, message, extra);
+  }
+}

package/src/filter-pushdown/deserialize.ts

@@ -0,0 +1,253 @@
+// Copyright 2025, 2026 Query Farm LLC - https://query.farm
+// Deserialize pushdown filters from the wire format (Arrow RecordBatch with
+// JSON specs in column 0 and value-ref scalars in columns 1+).
+
+import { type VgiBatch, type VgiDataType, readCanonicalValue } from "../arrow/index.js";
+import { ComparisonOp, type ExprNode, type Filter } from "./types.js";
+import { PushdownFilters } from "./evaluate.js";
+
+function parseFilter(
+  spec: any,
+  getValue: (ref: number) => any,
+  getJoinKeysColumn?: (columnName: string) => any[] | null,
+): Filter | null {
+  const columnName: string = spec.column_name ?? "";
+  const columnIndex: number = spec.column_index ?? 0;
+
+  switch (spec.type) {
+    case "constant":
+      return {
+        type: "constant",
+        columnName,
+        columnIndex,
+        op: spec.op as ComparisonOp,
+        value: getValue(spec.value_ref),
+      };
+
+    case "is_null":
+      return { type: "is_null", columnName, columnIndex };
+
+    case "is_not_null":
+      return { type: "is_not_null", columnName, columnIndex };
+
+    case "in": {
+      const listScalar = getValue(spec.value_ref);
+      // listScalar is an Arrow list value — extract elements into a Set
+      const values = new Set<any>();
+      if (listScalar && typeof listScalar[Symbol.iterator] === "function") {
+        for (const v of listScalar) {
+          values.add(v);
+        }
+      } else if (Array.isArray(listScalar)) {
+        for (const v of listScalar) {
+          values.add(v);
+        }
+      }
+      return { type: "in", columnName, columnIndex, values };
+    }
+
+    case "and": {
+      // AND: drop unparseable children; the remaining ones still filter correctly
+      // (conjunction is weaker with fewer children, which is safe — extra rows
+      // pass but DuckDB re-filters client-side).
+      const andChildren = (spec.children ?? [])
+        .map((c: any) => parseFilter(c, getValue, getJoinKeysColumn))
+        .filter((f: Filter | null): f is Filter => f !== null);
+      if (andChildren.length === 0) return null;
+      return { type: "and", columnName, columnIndex, children: andChildren };
+    }
+
+    case "or": {
+      // OR: dropping a child would strengthen the filter (fewer rows pass),
+      // which is unsafe. If any child fails to parse, drop the whole OR.
+      const orChildren: Filter[] = [];
+      for (const c of spec.children ?? []) {
+        const parsed = parseFilter(c, getValue, getJoinKeysColumn);
+        if (parsed === null) return null;
+        orChildren.push(parsed);
+      }
+      return { type: "or", columnName, columnIndex, children: orChildren };
+    }
+
+    case "struct": {
+      const childFilter = parseFilter(spec.child_filter, getValue, getJoinKeysColumn);
+      if (childFilter === null) return null;
+      return {
+        type: "struct",
+        columnName,
+        columnIndex,
+        childIndex: spec.child_index ?? 0,
+        childName: spec.child_name ?? "",
+        childFilter,
+      };
+    }
+
+    case "join_keys": {
+      // vgi-python `FilterType.JOIN_KEYS`. DuckDB's dynamic_or_filter
+      // optimizer can promote IN / OR lists or join-predicate pushdowns into
+      // this shape. Look the keys_column up in the init request's join_keys
+      // batches and materialize as an InFilter. If no callback or no batch
+      // matches, fall through as null (graceful degradation).
+      const keysColumn: string = spec.keys_column ?? "";
+      if (!getJoinKeysColumn) return null;
+      const values = getJoinKeysColumn(keysColumn);
+      if (values === null) return null;
+      const set = new Set<any>(values);
+      return { type: "in", columnName, columnIndex, values: set };
+    }
+
+    case "expression": {
+      // Parse the serialized expression tree (DuckDB bound-expression → JSON
+      // emitted by FilterSerializer::SerializeExpression in C++). The tree
+      // may reference constants by `value_ref` — resolve those now so the
+      // evaluator sees plain JS values.
+      const expr = parseExprNode(spec.expr, getValue);
+      if (!expr) return null;
+      return { type: "expression", columnName, columnIndex, expr };
+    }
+
+    default:
+      throw new Error(`Unknown filter type: ${spec.type}`);
+  }
+}
+
+/**
+ * Build a getJoinKeysColumn lookup from an InitRequest's `joinKeys` batches.
+ * Each batch typically has a single column whose name is the join-keys column
+ * referenced by `spec.keys_column` on a join_keys filter.
+ */
+export function buildJoinKeysLookup(
+  joinKeyBatches: VgiBatch[],
+): (columnName: string) => any[] | null {
+  const index = new Map<string, any[]>();
+  for (const batch of joinKeyBatches) {
+    for (const field of batch.schema.fields) {
+      const col = batch.getChild(field.name);
+      if (!col) continue;
+      const type = field.type as unknown as VgiDataType;
+      const values: any[] = [];
+      for (let i = 0; i < batch.numRows; i++) {
+        // Canonical read so join keys match the column cells they filter
+        // against (built via readCanonicalValue in evaluate.ts).
+        values.push(readCanonicalValue(type, col, i));
+      }
+      index.set(field.name, values);
+    }
+  }
+  return (name: string) => index.get(name) ?? null;
+}
+
+// Parse a serialized expression subtree. Returns null if the shape doesn't
+// look like one we can evaluate (unknown expr_type); the caller skips the
+// whole filter in that case, which is equivalent to not pushing it down.
+function parseExprNode(
+  spec: any,
+  getValue: (ref: number) => any,
+): ExprNode | null {
+  if (!spec || typeof spec !== "object") return null;
+  switch (spec.expr_type) {
+    case "column_ref":
+      return { expr_type: "column_ref", index: Number(spec.index ?? 0) };
+    case "constant":
+      return { expr_type: "constant", value: getValue(Number(spec.value_ref)) };
+    case "function": {
+      const kids: ExprNode[] = [];
+      for (const c of spec.children ?? []) {
+        const k = parseExprNode(c, getValue);
+        if (!k) return null;
+        kids.push(k);
+      }
+      return { expr_type: "function", function_name: String(spec.function_name ?? ""), children: kids };
+    }
+    case "comparison": {
+      const left = parseExprNode(spec.left, getValue);
+      const right = parseExprNode(spec.right, getValue);
+      if (!left || !right) return null;
+      return {
+        expr_type: "comparison",
+        op: opFromString(spec.op),
+        left,
+        right,
+      };
+    }
+    case "conjunction": {
+      const kids: ExprNode[] = [];
+      for (const c of spec.children ?? []) {
+        const k = parseExprNode(c, getValue);
+        if (!k) return null;
+        kids.push(k);
+      }
+      return {
+        expr_type: "conjunction",
+        conjunction_type: spec.conjunction_type === "or" ? "or" : "and",
+        children: kids,
+      };
+    }
+    default:
+      return null;
+  }
+}
+
+function opFromString(s: string): ComparisonOp {
+  switch (s) {
+    case "eq": case "==": return ComparisonOp.EQ;
+    case "ne": case "!=": return ComparisonOp.NE;
+    case "gt": case ">": return ComparisonOp.GT;
+    case "ge": case ">=": return ComparisonOp.GE;
+    case "lt": case "<": return ComparisonOp.LT;
+    case "le": case "<=": return ComparisonOp.LE;
+    default: return ComparisonOp.EQ;
+  }
+}
+
+/**
+ * Deserialize pushdown filters from an Arrow RecordBatch.
+ *
+ * Wire format:
+ * - Column 0: UTF8 — JSON array of filter specs. Field metadata: vgi_filter_version: "1"
+ * - Columns 1+: Scalar values referenced by value_ref → column N+1
+ */
+export function deserializeFilters(
+  batch: VgiBatch,
+  getJoinKeysColumn?: (columnName: string) => any[] | null,
+): PushdownFilters {
+  // Validate version
+  const field0 = batch.schema.fields[0];
+  const version = field0.metadata?.get("vgi_filter_version");
+  if (!version) {
+    throw new Error(
+      "Filter batch missing vgi_filter_version metadata on field 0",
+    );
+  }
+  if (version !== "1") {
+    throw new Error(`Unsupported filter version: ${version}`);
+  }
+
+  // Parse JSON filter specs from column 0, row 0
+  const jsonCol = batch.getChildAt(0)!;
+  const jsonStr = jsonCol.get(0) as string;
+  let specs: any[];
+  try {
+    specs = JSON.parse(jsonStr);
+  } catch (e) {
+    throw new Error(`Failed to parse filter JSON: ${e}`);
+  }
+
+  // Value resolver: value_ref N → column N+1, row 0. Read through the canonical
+  // reader so filter literals land in the SAME representation as the column
+  // cells they're compared against (e.g. timestamp -> bigint on both backends),
+  // and so it works on flechette (no arrow-js `.get()`).
+  const getValue = (ref: number): any => {
+    const col = batch.getChildAt(ref + 1);
+    if (!col) {
+      throw new Error(`Filter value_ref ${ref} references non-existent column ${ref + 1}`);
+    }
+    const type = batch.schema.fields[ref + 1].type as unknown as VgiDataType;
+    return readCanonicalValue(type, col, 0);
+  };
+
+  const filters = specs
+    .map((spec) => parseFilter(spec, getValue, getJoinKeysColumn))
+    .filter((f): f is Filter => f !== null);
+  return new PushdownFilters(filters, version);
+}