@query-farm/vgi 0.6.0

package/src/arrow/impl-flechette/ipc.ts

@@ -0,0 +1,110 @@
+// Copyright 2025, 2026 Query Farm LLC - https://query.farm
+// Arrow IPC stream serialization/deserialization for the flechette backend.
+//
+// flechette's `tableFromIPC` accepts either Uint8Array or Uint8Array[] (the
+// latter for stream-format messages produced incrementally), and
+// `tableToIPC(table, { format: "stream" })` emits a self-contained stream.
+// The same Schema/RecordBatch shapes are produced and consumed here as in
+// impl-arrowjs (structurally compatible with VgiSchema / VgiBatch).
+
+import {
+  tableFromIPC,
+  tableToIPC,
+  columnFromArray,
+  field as f_field,
+  Table,
+} from "@query-farm/flechette";
+
+import type { VgiSchema, VgiBatch } from "../types.js";
+import { readFirstRecordBatchMeta } from "./message-meta.js";
+
+// flechette decoding options that match how vgi-typescript callers expect
+// values to come out: 64-bit ints stay as BigInt, decimals as scaled BigInts,
+// timestamps as BigInt nanoseconds, Maps as [k,v] arrays.
+const EXTRACT_OPTS = {
+  useBigInt: true,
+  useBigIntTimestamp: true,
+  useDecimalInt: true,
+  useMap: false,
+} as const;
+
+/**
+ * Serialize a Schema to Arrow IPC bytes.
+ *
+ * flechette has no schema-only serialization helper, so we encode an empty
+ * 0-row table that carries just the schema header. Decoders see the Schema
+ * + an immediate EOS marker — same wire shape arrow-js's
+ * `RecordBatchStreamWriter.close()` produces for an empty stream.
+ */
+export function serializeSchema(schema: VgiSchema): Uint8Array {
+  // Build the Table directly so we preserve per-field `nullable`/`metadata`.
+  // tableFromColumns rebuilds fields with nullable=true, which round-trips
+  // wrong on the C++ side (rejects schema-conforming payloads as schema
+  // mismatches). See build.ts for the same fix on batches.
+  const cols = schema.fields.map((f) => columnFromArray([], f.type as any));
+  const fields = schema.fields.map((f, i) =>
+    f_field(f.name, cols[i].type as any, (f as any).nullable ?? true, (f as any).metadata ?? null),
+  );
+  const flechSchema = {
+    version: 5,
+    endianness: 0,
+    fields,
+    metadata: (schema as any).metadata ?? null,
+  };
+  return tableToIPC(new Table(flechSchema as any, cols), { format: "stream" }) as Uint8Array;
+}
+
+/**
+ * Deserialize an Arrow Schema from IPC bytes. Returns the schema of the
+ * decoded table (which may be 0-row if the source was a schema-only stream).
+ */
+export function deserializeSchema(bytes: Uint8Array): VgiSchema {
+  const table = tableFromIPC(bytes, EXTRACT_OPTS);
+  // flechette's Table.schema is the structural shape we need.
+  return table.schema as unknown as VgiSchema;
+}
+
+/**
+ * Serialize a RecordBatch to Arrow IPC bytes (stream format).
+ *
+ * The batch passed in is structurally a flechette Table (1 record batch's
+ * worth of data). flechette's `tableToIPC` round-trips it as a complete
+ * stream — schema + one batch + EOS — which is what subprocess transport
+ * consumers expect per call.
+ */
+export function serializeBatch(batch: VgiBatch): Uint8Array {
+  // VgiBatch is structurally a flechette Table at this point.
+  return tableToIPC(batch as any, { format: "stream" }) as Uint8Array;
+}
+
+/**
+ * Deserialize a RecordBatch from Arrow IPC bytes.
+ *
+ * We treat the decoded Table as the "batch" — flechette Tables expose
+ * numRows / schema / getChild that satisfy VgiBatch's shape. For
+ * multi-batch streams this collapses to a single logical batch, matching
+ * the subprocess transport's one-batch-per-call convention.
+ */
+export function deserializeBatch(bytes: Uint8Array): VgiBatch {
+  const table: any = tableFromIPC(bytes, EXTRACT_OPTS);
+  // flechette's IPC parser ignores the Message-level `custom_metadata` field
+  // and reports row counts only via column children, so a "metadata-only"
+  // batch over a 0-field schema (legal in vgi-rpc — used to ferry state
+  // tokens / cancel signals) round-trips as numRows=0/metadata=undefined.
+  // Backfill both from the wire bytes so VgiBatch matches arrow-js's shape.
+  const meta = readFirstRecordBatchMeta(bytes);
+  if (meta === null) return table as VgiBatch;
+  const wantRows = table.numRows === 0 && meta.numRows > 0;
+  const wantMeta = !table.metadata && meta.metadata.size > 0;
+  if (!wantRows && !wantMeta) return table as VgiBatch;
+  // flechette Tables freeze their properties; wrap with a Proxy so reads of
+  // numRows / metadata see the backfilled values while everything else
+  // (schema, getChild, factory, …) flows through to the underlying Table.
+  return new Proxy(table, {
+    get(target, prop, receiver) {
+      if (wantRows && prop === "numRows") return meta.numRows;
+      if (wantMeta && prop === "metadata") return meta.metadata;
+      return Reflect.get(target, prop, receiver);
+    },
+  }) as unknown as VgiBatch;
+}

package/src/arrow/impl-flechette/iterate.ts

@@ -0,0 +1,121 @@
+// Copyright 2025, 2026 Query Farm LLC - https://query.farm
+// Read data out of flechette tables as plain JS values.
+
+import type { VgiBatch, VgiDataType } from "../types.js";
+import { codecFor } from "../codec/registry.js";
+import { readCanonicalValue } from "./canonical.js";
+
+/**
+ * Iterate rows of a batch (flechette Table) as plain objects, in the RICH
+ * representation (Date for date32/date64; canonical otherwise). Reads via the
+ * per-backend canonical reader then maps canonical -> rich through the codec —
+ * symmetric with the build path and identical to the arrow-js backend.
+ */
+export function* iterRows(
+  batch: VgiBatch,
+  repr: "rich" | "raw" = "rich",
+): Generator<Record<string, any>> {
+  const t = batch as any;
+  const fields = t.schema.fields;
+  const codecs = fields.map((f: any) => codecFor(f.type as VgiDataType));
+  for (let i = 0; i < t.numRows; i++) {
+    const row: Record<string, any> = {};
+    for (let fi = 0; fi < fields.length; fi++) {
+      const f = fields[fi];
+      const col = t.getChild(f.name);
+      if (!col) { row[f.name] = null; continue; }
+      const canonical = readCanonicalValue(f.type as VgiDataType, col, i);
+      const codec = codecs[fi];
+      row[f.name] = repr === "raw"
+        ? codec.canonicalToRaw(canonical)
+        : codec.canonicalToRich(canonical);
+    }
+    yield row;
+  }
+}
+
+/**
+ * Extract a single-row batch as a flat dict { col: value }, in the RICH
+ * representation. Routes through the canonical reader + codec (same as iterRows)
+ * so a temporal/decimal setting is represented identically to column data and
+ * identically to the arrow-js backend.
+ */
+export function batchToScalarDict(
+  batch: VgiBatch | null,
+): Record<string, any> {
+  if (!batch) return {};
+  const t = batch as any;
+  if (t.numRows === 0) return {};
+  const result: Record<string, any> = {};
+  for (const f of t.schema.fields) {
+    const col = t.getChild(f.name);
+    if (!col) continue;
+    const type = f.type as VgiDataType;
+    result[f.name] = codecFor(type).canonicalToRich(readCanonicalValue(type, col, 0));
+  }
+  return result;
+}
+
+/**
+ * No-op for flechette: dictionary columns auto-decode at extraction time.
+ * Provided for parity with impl-arrowjs (where the apache-arrow fork doesn't
+ * auto-decode in some paths).
+ */
+export function decodeDictValue(value: any, _index = 0): any {
+  return value;
+}
+
+/**
+ * Single-row batch → secret dict. Mirrors impl-arrowjs's same-named function.
+ */
+export function batchToSecretDict(
+  batch: VgiBatch | null,
+): Record<string, Record<string, any>> {
+  if (!batch) return {};
+  const t = batch as any;
+  if (t.numRows === 0) return {};
+  const result: Record<string, Record<string, any>> = {};
+  for (const f of t.schema.fields) {
+    const col = t.getChild(f.name);
+    if (!col) continue;
+    // Read in RICH form via the canonical reader + codec — a secret column is a
+    // struct scalar, surfacing as a plain { field: value } object (same path as
+    // iterRows), identical to the arrow-js backend.
+    const type = f.type as VgiDataType;
+    const val = codecFor(type).canonicalToRich(readCanonicalValue(type, col, 0));
+
+    let key = f.name;
+    let scope: string | undefined;
+    if (f.name.startsWith("secret_")) {
+      const secretType = f.metadata?.get?.("secret_type");
+      if (secretType) {
+        key = secretType;
+        scope = f.metadata?.get?.("scope") ?? undefined;
+      }
+    }
+
+    if (val && typeof val === "object" && !ArrayBuffer.isView(val)) {
+      const dict: Record<string, any> = { ...(val as Record<string, any>) };
+      if (key in result) {
+        throw new Error(
+          `batchToSecretDict: duplicate secret_type '${key}' (scope=${scope ?? "none"}).`,
+        );
+      }
+      result[key] = dict;
+      if (scope) result[`${key}:${scope}`] = dict;
+    } else if (val === null || val === undefined) {
+      // skip
+    } else {
+      result[key] = {};
+    }
+  }
+  return result;
+}
+
+/**
+ * BigInt → number coercion (lossy for values past 2^53).
+ */
+export function safeNumber(value: any): number {
+  if (typeof value === "bigint") return Number(value);
+  return value as number;
+}

package/src/arrow/impl-flechette/message-meta.ts

@@ -0,0 +1,175 @@
+// Copyright 2025, 2026 Query Farm LLC - https://query.farm
+// Minimal Arrow Message-envelope reader.
+//
+// flechette's IPC parser ignores the Message-level `custom_metadata` field
+// (vtable offset 12 in the Message FlatBuffer) and only surfaces row count
+// through column children — both of which are needed to round-trip vgi
+// "params" batches that carry zero-field schemas with auth/state tokens
+// in batch metadata. arrow-js exposes both via `RecordBatch.metadata` and
+// `RecordBatch.numRows`.
+//
+// This file is a self-contained, dependency-free reader that walks an IPC
+// stream and pulls out, for the first RecordBatch message it sees:
+//   * `numRows`   — from RecordBatch.length (Message header_value, RB vtable offset 4)
+//   * `metadata`  — Message.custom_metadata (Message vtable offset 12)
+//
+// Schema fields are 0/1/2/3/4; same for RecordBatch — see Arrow's
+// `format/Message.fbs` and `format/Schema.fbs`.
+
+const TYPE_SCHEMA = 1;
+const TYPE_RECORD_BATCH = 3;
+
+function readU32LE(b: Uint8Array, p: number): number {
+  return ((b[p] | (b[p + 1] << 8) | (b[p + 2] << 16) | (b[p + 3] << 24)) >>> 0);
+}
+function readI32LE(b: Uint8Array, p: number): number {
+  return readU32LE(b, p) | 0;
+}
+function readI16LE(b: Uint8Array, p: number): number {
+  const v = b[p] | (b[p + 1] << 8);
+  return v < 0x8000 ? v : v - 0x10000;
+}
+function readI64LE(b: Uint8Array, p: number): bigint {
+  const lo = BigInt(readU32LE(b, p));
+  const hi = BigInt(readU32LE(b, p + 4));
+  // Sign-extend if hi's top bit is set.
+  const top = hi & 0x80000000n;
+  const combined = (hi << 32n) | lo;
+  return top ? combined - (1n << 64n) : combined;
+}
+
+// FlatBuffer table accessor. Holds the absolute byte offset of the table
+// (root or sub-table) plus the parsed vtable field offsets.
+interface FbTable {
+  pos: number;
+  fields: Int16Array;
+}
+
+function readTable(buf: Uint8Array, tablePos: number): FbTable {
+  // First field at tablePos is the (signed) offset back to the vtable.
+  const vtableRel = readI32LE(buf, tablePos);
+  const vtablePos = tablePos - vtableRel;
+  const vtableSize = readI16LE(buf, vtablePos);
+  const fieldCount = (vtableSize - 4) >> 1;
+  const fields = new Int16Array(fieldCount);
+  for (let i = 0; i < fieldCount; i++) {
+    fields[i] = readI16LE(buf, vtablePos + 4 + i * 2);
+  }
+  return { pos: tablePos, fields };
+}
+
+/** Position of a field in the table buffer, or null if absent (offset 0). */
+function fieldAt(t: FbTable, fieldIndex: number): number | null {
+  if (fieldIndex >= t.fields.length) return null;
+  const off = t.fields[fieldIndex];
+  return off === 0 ? null : t.pos + off;
+}
+
+/** Follow a uoffset_t at `ptrPos` to the absolute target byte position. */
+function follow(buf: Uint8Array, ptrPos: number): number {
+  return ptrPos + readU32LE(buf, ptrPos);
+}
+
+function readString(buf: Uint8Array, strPos: number): string {
+  const len = readU32LE(buf, strPos);
+  return new TextDecoder().decode(buf.subarray(strPos + 4, strPos + 4 + len));
+}
+
+interface ParsedMessage {
+  /** Header type tag — 1 = Schema, 2 = DictionaryBatch, 3 = RecordBatch. */
+  headerType: number;
+  /** RecordBatch.length (rows). Zero if not a RecordBatch or absent. */
+  numRows: number;
+  /** Message-level body length (offset 10). Used to skip past the body. */
+  bodyLength: number;
+  /** Message.custom_metadata, parsed into a Map. Empty if absent. */
+  metadata: Map<string, string>;
+}
+
+function readMessageEnvelope(head: Uint8Array): ParsedMessage {
+  const rootOffset = readU32LE(head, 0);
+  const msg = readTable(head, rootOffset);
+
+  // Message field indices (Message.fbs):
+  //   0: version (int16)            vtable offset 4
+  //   1: headerType (uint8)         vtable offset 6
+  //   2: header   (table)           vtable offset 8
+  //   3: bodyLength (int64)         vtable offset 10
+  //   4: custom_metadata ([KeyVal]) vtable offset 12
+  const headerTypePos = fieldAt(msg, 1);
+  const headerType = headerTypePos === null ? 0 : head[headerTypePos];
+
+  const bodyLengthPos = fieldAt(msg, 3);
+  const bodyLengthBig = bodyLengthPos === null ? 0n : readI64LE(head, bodyLengthPos);
+  // Bodies are bounded by maxDecompressedRequestBytes upstream — safe to coerce.
+  const bodyLength = Number(bodyLengthBig);
+
+  let numRows = 0;
+  if (headerType === TYPE_RECORD_BATCH) {
+    const headerValuePos = fieldAt(msg, 2);
+    if (headerValuePos !== null) {
+      const rbPos = follow(head, headerValuePos);
+      const rb = readTable(head, rbPos);
+      // RecordBatch.length is field 0 (vtable offset 4), int64.
+      const lenPos = fieldAt(rb, 0);
+      numRows = lenPos === null ? 0 : Number(readI64LE(head, lenPos));
+    }
+  }
+
+  const metadata = new Map<string, string>();
+  const metaVecPtr = fieldAt(msg, 4);
+  if (metaVecPtr !== null) {
+    const vecPos = follow(head, metaVecPtr);
+    const vecLen = readU32LE(head, vecPos);
+    for (let i = 0; i < vecLen; i++) {
+      // Vector of tables: each entry is a uoffset_t (4 bytes) pointing to the table.
+      const itemPtr = vecPos + 4 + i * 4;
+      const kvPos = follow(head, itemPtr);
+      const kv = readTable(head, kvPos);
+      const keyPtr = fieldAt(kv, 0);
+      const valPtr = fieldAt(kv, 1);
+      const key = keyPtr === null ? "" : readString(head, follow(head, keyPtr));
+      const val = valPtr === null ? "" : readString(head, follow(head, valPtr));
+      metadata.set(key, val);
+    }
+  }
+
+  return { headerType, numRows, bodyLength, metadata };
+}
+
+/**
+ * Walk an Arrow IPC stream and return the first RecordBatch message's
+ * (numRows, metadata) — the bits flechette drops on the floor.
+ *
+ * Returns null if the stream contains no RecordBatch.
+ */
+export function readFirstRecordBatchMeta(
+  stream: Uint8Array,
+): { numRows: number; metadata: Map<string, string> } | null {
+  let pos = 0;
+  while (pos + 4 <= stream.length) {
+    let metaLen = readI32LE(stream, pos);
+    pos += 4;
+    // Continuation-marker form (post-Arrow 0.15): the real length follows.
+    if (metaLen === -1) {
+      if (pos + 4 > stream.length) return null;
+      metaLen = readI32LE(stream, pos);
+      pos += 4;
+    }
+    if (metaLen === 0) return null; // EOS
+    if (pos + metaLen > stream.length) return null;
+    const head = stream.subarray(pos, pos + metaLen);
+    pos += metaLen;
+    const parsed = readMessageEnvelope(head);
+    if (parsed.headerType === TYPE_RECORD_BATCH) {
+      return { numRows: parsed.numRows, metadata: parsed.metadata };
+    }
+    // Skip the body (Schema messages have bodyLength = 0).
+    pos += parsed.bodyLength;
+    if (parsed.headerType !== TYPE_SCHEMA && parsed.headerType !== 2 /* Dict */) {
+      // Unknown header — stop walking.
+      return null;
+    }
+  }
+  return null;
+}

package/src/arrow/impl-flechette/normalize-type.ts

@@ -0,0 +1,99 @@
+// Copyright 2025, 2026 Query Farm LLC - https://query.farm
+//
+// Normalize a (possibly foreign) Arrow DataType into a flechette-native type.
+//
+// App/worker code routinely builds schemas with concrete arrow-js DataType
+// instances (e.g. `new Int64()` from @query-farm/apache-arrow) rather than the
+// facade constructors. flechette's `columnFromArray` *tolerates* such foreign
+// type objects when reading values, but the resulting `Column.type` is the
+// foreign object — and flechette's IPC writer then serializes it incorrectly
+// (e.g. Utf8 round-trips to ["", "xy\0\0…"], Int64 throws BigInt→number on a
+// stale build). Reconstructing the type from its structural fields via
+// flechette's own constructors yields a writer-safe native type.
+//
+// Both arrow-js and flechette encode `typeId` with the same Arrow Type enum, so
+// we switch on it and read the shared structural props (with the one rename
+// flechette differs on: Int uses `signed`, arrow-js uses `isSigned`).
+
+import {
+  Type,
+  field as f_field,
+  nullType, bool, int, float, utf8, binary, fixedSizeBinary,
+  decimal, date, time, timestamp, duration, interval,
+  list, largeList, fixedSizeList, struct, map, dictionary, union,
+} from "@query-farm/flechette";
+
+function fField(f: any): any {
+  return f_field(f.name, toFlechetteType(f.type), f.nullable ?? true, f.metadata ?? null);
+}
+
+/**
+ * Rebuild `type` as a flechette-native DataType. Idempotent for types that are
+ * already native (reconstructed from the same structural props). Unknown
+ * typeIds pass through unchanged as a safety net.
+ */
+export function toFlechetteType(type: any): any {
+  if (type == null) return nullType();
+  switch (type.typeId) {
+    case Type.Null:
+      return nullType();
+    case Type.Bool:
+      return bool();
+    case Type.Int:
+      return int(type.bitWidth, type.isSigned ?? type.signed ?? true);
+    case Type.Float:
+      return float(type.precision);
+    case Type.Binary:
+    case Type.LargeBinary:
+      return binary();
+    case Type.Utf8:
+    case Type.LargeUtf8:
+      return utf8();
+    case Type.FixedSizeBinary:
+      return fixedSizeBinary(type.byteWidth ?? type.stride);
+    case Type.Decimal:
+      return decimal(type.precision, type.scale, type.bitWidth ?? 128);
+    case Type.Date:
+      return date(type.unit);
+    case Type.Time:
+      return time(type.unit);
+    case Type.Timestamp:
+      return timestamp(type.unit, type.timezone ?? null);
+    case Type.Duration:
+      return duration(type.unit);
+    case Type.Interval:
+      return interval(type.unit);
+    case Type.List:
+    case Type.LargeList:
+      return (type.typeId === Type.LargeList ? largeList : list)(fField(type.children[0]));
+    case Type.FixedSizeList:
+      return fixedSizeList(fField(type.children[0]), type.listSize);
+    case Type.Struct:
+      return struct(type.children.map(fField));
+    case Type.Map: {
+      // children[0] is the "entries" struct field holding [key, value].
+      const entries = type.children[0];
+      const [k, v] = entries.type.children;
+      return map(fField(k), fField(v), type.keysSorted ?? false);
+    }
+    case Type.Dictionary:
+      return dictionary(
+        toFlechetteType(type.dictionary),
+        toFlechetteType(type.indices),
+        type.ordered ?? type.isOrdered ?? false,
+        type.id,
+      );
+    case Type.Union:
+      return union(type.mode, type.children.map(fField), type.typeIds);
+    default:
+      return type;
+  }
+}
+
+/**
+ * Normalize every field's type in a schema-like object to flechette-native,
+ * preserving field name / nullable / metadata.
+ */
+export function normalizeSchemaFields(fields: readonly any[]): any[] {
+  return fields.map(fField);
+}

package/src/arrow/impl-flechette/project.ts

@@ -0,0 +1,64 @@
+// Copyright 2025, 2026 Query Farm LLC - https://query.farm
+// Schema/batch projection by column index for the flechette backend.
+
+import { tableFromColumns } from "@query-farm/flechette";
+import type { VgiSchema, VgiBatch } from "../types.js";
+
+function validateProjectionIds(
+  caller: string,
+  ids: number[],
+  fieldCount: number,
+): number[] {
+  for (const id of ids) {
+    if (id < -1) {
+      throw new Error(
+        `${caller}: unexpected negative projection ID ${id} (only -1 for row_id is allowed)`,
+      );
+    }
+    if (id >= fieldCount) {
+      console.warn(
+        `${caller}: projection ID ${id} is out of bounds (schema has ${fieldCount} fields), ignoring`,
+      );
+    }
+  }
+  return ids.filter((i) => i >= 0 && i < fieldCount);
+}
+
+export function projectSchema(
+  projectionIds: number[] | null,
+  schema: VgiSchema,
+): VgiSchema {
+  if (!projectionIds) return schema;
+  const valid = validateProjectionIds(
+    "projectSchema",
+    projectionIds,
+    schema.fields.length,
+  );
+  if (valid.length === 0) return schema;
+  return {
+    fields: valid.map((i) => schema.fields[i]),
+    metadata: schema.metadata,
+  } as VgiSchema;
+}
+
+export function projectBatch(
+  projectionIds: number[] | null,
+  batch: VgiBatch,
+): VgiBatch {
+  if (!projectionIds) return batch;
+  const t = batch as any;
+  const valid = validateProjectionIds(
+    "projectBatch",
+    projectionIds,
+    t.schema.fields.length,
+  );
+  if (valid.length === 0) return batch;
+
+  // Rebuild a Table with only the kept columns, in the requested order.
+  const cols: Record<string, any> = {};
+  for (const i of valid) {
+    const f = t.schema.fields[i];
+    cols[f.name] = t.getChild(f.name);
+  }
+  return tableFromColumns(cols) as unknown as VgiBatch;
+}