@query-farm/vgi 0.6.0

package/src/arrow/impl-arrowjs/ipc.ts

@@ -0,0 +1,67 @@
+// Copyright 2025, 2026 Query Farm LLC - https://query.farm
+// Arrow IPC stream serialization/deserialization for schemas and batches.
+
+import {
+  RecordBatch,
+  Schema,
+  RecordBatchStreamWriter,
+  RecordBatchReader,
+} from "@query-farm/apache-arrow";
+import type { VgiSchema, VgiBatch } from "../types.js";
+import { emptyBatch } from "./empty.js";
+
+/**
+ * Serialize a Schema to Arrow IPC bytes. Accepts arrow-js `Schema` or
+ * facade `VgiSchema` (the latter is satisfied structurally by arrow-js
+ * Schema instances at runtime).
+ */
+export function serializeSchema(schema: Schema | VgiSchema): Uint8Array {
+  const writer = new RecordBatchStreamWriter();
+  writer.reset(undefined, schema as Schema);
+  writer.close();
+  return writer.toUint8Array(true);
+}
+
+/**
+ * Deserialize an Arrow Schema from IPC bytes.
+ * Note: In Bun, reader.schema is always undefined, so we must read from the batch.
+ *
+ * Returns arrow-js's `Schema` (which structurally satisfies VgiSchema).
+ */
+export function deserializeSchema(bytes: Uint8Array): Schema {
+  const reader = RecordBatchReader.from(bytes);
+  const batches = [...reader];
+  if (batches.length > 0) {
+    return batches[0].schema;
+  }
+  if (reader.schema) return reader.schema;
+  throw new Error("Cannot deserialize schema from empty IPC stream");
+}
+
+/**
+ * Serialize a RecordBatch to Arrow IPC bytes. Accepts arrow-js `RecordBatch`
+ * or facade `VgiBatch`.
+ */
+export function serializeBatch(batch: RecordBatch | VgiBatch): Uint8Array {
+  const a = batch as RecordBatch;
+  const writer = new RecordBatchStreamWriter();
+  writer.reset(undefined, a.schema);
+  // Use _writeRecordBatch to bypass schema comparison bug
+  // (public write() silently drops batches on nullability mismatch)
+  (writer as any)._writeRecordBatch(a);
+  writer.close();
+  return writer.toUint8Array(true);
+}
+
+/**
+ * Deserialize a RecordBatch from Arrow IPC bytes.
+ */
+export function deserializeBatch(bytes: Uint8Array): RecordBatch {
+  const reader = RecordBatchReader.from(bytes);
+  const batches = [...reader];
+  if (batches.length === 0) {
+    const sch = reader.schema ?? new Schema([]);
+    return emptyBatch(sch);
+  }
+  return batches[0];
+}

package/src/arrow/impl-arrowjs/iterate.ts

@@ -0,0 +1,159 @@
+// Copyright 2025, 2026 Query Farm LLC - https://query.farm
+// Read data out of Arrow batches as plain JS values.
+
+import { RecordBatch } from "@query-farm/apache-arrow";
+import type { VgiBatch, VgiDataType } from "../types.js";
+import { codecFor } from "../codec/registry.js";
+import { readCanonicalValue } from "./canonical.js";
+
+/**
+ * Iterate rows of a RecordBatch as plain objects, in the RICH representation
+ * (Date for date32/date64; canonical otherwise). Reads via the per-backend
+ * canonical reader (lossless, backend-agnostic) then maps canonical -> rich
+ * through the codec — symmetric with the build path. Accepts arrow-js
+ * `RecordBatch` or facade `VgiBatch`.
+ */
+export function* iterRows(
+  batch: RecordBatch | VgiBatch,
+  repr: "rich" | "raw" = "rich",
+): Generator<Record<string, any>> {
+  const a = batch as RecordBatch;
+  const codecs = a.schema.fields.map((f) => codecFor(f.type as unknown as VgiDataType));
+  for (let i = 0; i < a.numRows; i++) {
+    const row: Record<string, any> = {};
+    for (let fi = 0; fi < a.schema.fields.length; fi++) {
+      const field = a.schema.fields[fi];
+      const col = a.getChild(field.name);
+      if (!col) { row[field.name] = null; continue; }
+      const canonical = readCanonicalValue(field.type as unknown as VgiDataType, col, i);
+      const codec = codecs[fi];
+      row[field.name] = repr === "raw"
+        ? codec.canonicalToRaw(canonical)
+        : codec.canonicalToRich(canonical);
+    }
+    yield row;
+  }
+}
+
+/**
+ * Extract single-row batch to a scalar dict, in the RICH representation. Routes
+ * through the canonical reader + codec (same as iterRows) so a temporal/decimal
+ * setting is represented identically to column data and across backends, and so
+ * Dictionary-encoded columns (DuckDB sends these for enum-shaped fields like
+ * SchemaObjectType) are decoded — readCanonicalValue handles dictionary decode.
+ */
+export function batchToScalarDict(
+  batch: RecordBatch | VgiBatch | null
+): Record<string, any> {
+  if (!batch) return {};
+  const a = batch as RecordBatch;
+  if (a.numRows === 0) return {};
+  const result: Record<string, any> = {};
+  for (const field of a.schema.fields) {
+    const col = a.getChild(field.name);
+    if (col) {
+      const type = field.type as unknown as VgiDataType;
+      const canonical = readCanonicalValue(type, col, 0);
+      result[field.name] = codecFor(type).canonicalToRich(canonical);
+    }
+  }
+  return result;
+}
+
+/**
+ * If `value` looks like a Dictionary-encoded Arrow scalar (Vector.get on a
+ * dict column on the apache-arrow fork returns the underlying Data, not the
+ * decoded string), pull out the decoded value at row `index`. Returns
+ * `value` unchanged when it isn't dict-shaped.
+ *
+ * Used at handler call sites where the incoming `params` came from the RPC
+ * layer's row extractor (which doesn't auto-decode dictionaries) and the
+ * handler needs the plain string.
+ */
+export function decodeDictValue(value: any, index = 0): any {
+  if (
+    value && typeof value === "object" &&
+    "dictionary" in value && "values" in value &&
+    typeof (value as any).dictionary?.get === "function"
+  ) {
+    // Respect the null bitmap — for nullable Dictionary columns DuckDB emits
+    // a "null" value as bitmap-bit=0 + index=0 (or whatever uninitialized
+    // memory holds). Without this check we'd return dictionary[0], turning
+    // "no ORDER BY pushdown" into a spurious "ASC NULLS_FIRST".
+    const data = value as any;
+    const nullBitmap = data.nullBitmap;
+    if (nullBitmap && nullBitmap.length > 0) {
+      const byte = nullBitmap[index >> 3];
+      if (((byte >> (index & 7)) & 1) === 0) return null;
+    }
+    const idx = Number(data.values[index]);
+    return (value as any).dictionary.get(idx);
+  }
+  return value;
+}
+
+/**
+ * Extract single-row batch to a secret dict (column per secret, each value is a struct).
+ * Handles both named secrets (column name = secret type) and scoped secrets
+ * (column name = "secret_N" with secret_type in field metadata).
+ */
+export function batchToSecretDict(
+  batch: RecordBatch | VgiBatch | null
+): Record<string, Record<string, any>> {
+  if (!batch) return {};
+  const a = batch as RecordBatch;
+  if (a.numRows === 0) return {};
+  const result: Record<string, Record<string, any>> = {};
+  for (const field of a.schema.fields) {
+    const col = a.getChild(field.name);
+    if (col) {
+      // Read in RICH form via the canonical reader + codec — a secret column is
+      // a struct scalar, which surfaces as a plain { field: value } object
+      // (same path as iterRows), so no Arrow-scalar toJSON() shimming is needed.
+      const type = field.type as unknown as VgiDataType;
+      const val = codecFor(type).canonicalToRich(readCanonicalValue(type, col, 0));
+
+      // Determine the key: for scoped secrets (secret_N), use secret_type from metadata
+      let key = field.name;
+      let scope: string | undefined;
+      if (field.name.startsWith("secret_")) {
+        const secretType = field.metadata?.get?.("secret_type");
+        if (secretType) {
+          key = secretType;
+          scope = field.metadata?.get?.("scope") ?? undefined;
+        }
+      }
+
+      if (val && typeof val === "object" && !ArrayBuffer.isView(val)) {
+        // Struct scalar -> already a plain object from the canonical reader.
+        const dict: Record<string, any> = { ...(val as Record<string, any>) };
+        if (key in result) {
+          throw new Error(
+            `batchToSecretDict: duplicate secret_type '${key}' (scope=${scope ?? "none"}). ` +
+            `Use scoped secrets with distinct scopes to avoid collisions.`
+          );
+        }
+        result[key] = dict;
+        // Store scope-qualified key for disambiguation
+        if (scope) {
+          result[`${key}:${scope}`] = dict;
+        }
+      } else if (val === null || val === undefined) {
+        // Skip null struct values (secret not found)
+      } else {
+        result[key] = {};
+      }
+    }
+  }
+  return result;
+}
+
+/**
+ * Safe number coercion for BigInt values.
+ */
+export function safeNumber(value: any): number {
+  if (typeof value === "bigint") {
+    return Number(value);
+  }
+  return value as number;
+}

package/src/arrow/impl-arrowjs/project.ts

@@ -0,0 +1,73 @@
+// Copyright 2025, 2026 Query Farm LLC - https://query.farm
+// Schema/batch projection by column index.
+
+import {
+  RecordBatch,
+  Schema,
+  Struct,
+  makeData,
+} from "@query-farm/apache-arrow";
+import type { VgiSchema, VgiBatch } from "../types.js";
+
+/**
+ * Validate and filter projection IDs. Returns only valid column indices.
+ * -1 is DuckDB's row_id sentinel (filtered out). IDs < -1 throw.
+ * IDs >= fieldCount are out-of-bounds and logged as warnings.
+ */
+function validateProjectionIds(
+  caller: string,
+  projectionIds: number[],
+  fieldCount: number,
+): number[] {
+  for (const id of projectionIds) {
+    if (id < -1) {
+      throw new Error(`${caller}: unexpected negative projection ID ${id} (only -1 for row_id is allowed)`);
+    }
+    if (id >= fieldCount) {
+      console.warn(`${caller}: projection ID ${id} is out of bounds (schema has ${fieldCount} fields), ignoring`);
+    }
+  }
+  return projectionIds.filter((i) => i >= 0 && i < fieldCount);
+}
+
+/**
+ * Project a schema by column indices, preserving only selected fields.
+ */
+export function projectSchema(
+  projectionIds: number[] | null,
+  schema: Schema | VgiSchema,
+): Schema {
+  const a = schema as Schema;
+  if (!projectionIds) return a;
+  const validIds = validateProjectionIds("projectSchema", projectionIds, a.fields.length);
+  if (validIds.length === 0) return a;
+  return new Schema(validIds.map((i) => a.fields[i]));
+}
+
+/**
+ * Project a RecordBatch by column indices.
+ */
+export function projectBatch(
+  projectionIds: number[] | null,
+  batch: RecordBatch | VgiBatch,
+): RecordBatch {
+  const a = batch as RecordBatch;
+  if (!projectionIds) return a;
+  const validIds = validateProjectionIds("projectBatch", projectionIds, a.schema.fields.length);
+  if (validIds.length === 0) return a;
+  const projectedSchema = projectSchema(projectionIds, a.schema);
+  const children = validIds.map((i) => {
+    const col = a.getChildAt(i);
+    return col!.data[0];
+  });
+
+  const structType = new Struct(projectedSchema.fields);
+  const data = makeData({
+    type: structType,
+    length: a.numRows,
+    children,
+    nullCount: 0,
+  });
+
+  return new RecordBatch(projectedSchema, data);
+}

package/src/arrow/impl-arrowjs/schema.ts

@@ -0,0 +1,256 @@
+// Copyright 2025, 2026 Query Farm LLC - https://query.farm
+// Factory-style constructors for arrow-js Schema/Field/DataType values.
+// Mirrors flechette's API so internal code can call the same factory
+// regardless of the active backend.
+
+import {
+  Schema as A_Schema,
+  Field as A_Field,
+  Null as A_Null,
+  Bool as A_Bool,
+  Int8 as A_Int8,
+  Int16 as A_Int16,
+  Int32 as A_Int32,
+  Int64 as A_Int64,
+  Uint8 as A_Uint8,
+  Uint16 as A_Uint16,
+  Uint32 as A_Uint32,
+  Uint64 as A_Uint64,
+  Float16 as A_Float16,
+  Float32 as A_Float32,
+  Float64 as A_Float64,
+  Utf8 as A_Utf8,
+  Binary as A_Binary,
+  Decimal as A_Decimal,
+  Date_ as A_Date,
+  Time as A_Time,
+  Timestamp as A_Timestamp,
+  Duration as A_Duration,
+  Interval as A_Interval,
+  List as A_List,
+  Struct as A_Struct,
+  Map_ as A_Map,
+  Dictionary as A_Dictionary,
+  SparseUnion as A_SparseUnion,
+  DenseUnion as A_DenseUnion,
+  FixedSizeBinary as A_FixedSizeBinary,
+  FixedSizeList as A_FixedSizeList,
+  DateUnit as A_DateUnit,
+  TimeUnit as A_TimeUnit,
+  IntervalUnit as A_IntervalUnit,
+  UnionMode as A_UnionMode,
+  type DataType,
+} from "@query-farm/apache-arrow";
+
+import type { VgiField, VgiSchema, VgiDataType } from "../types.js";
+
+// ----- TimeUnit / DateUnit / UnionMode constants ---------------------------
+// Numeric values match the Arrow IPC spec; both arrow-js and flechette
+// use the same integers, so internal code can freely cross backends.
+
+export const TimeUnit = {
+  SECOND: A_TimeUnit.SECOND,
+  MILLISECOND: A_TimeUnit.MILLISECOND,
+  MICROSECOND: A_TimeUnit.MICROSECOND,
+  NANOSECOND: A_TimeUnit.NANOSECOND,
+} as const;
+
+export const DateUnit = {
+  DAY: A_DateUnit.DAY,
+  MILLISECOND: A_DateUnit.MILLISECOND,
+} as const;
+
+export const IntervalUnit = {
+  YEAR_MONTH: A_IntervalUnit.YEAR_MONTH,
+  DAY_TIME: A_IntervalUnit.DAY_TIME,
+  MONTH_DAY_NANO: A_IntervalUnit.MONTH_DAY_NANO,
+} as const;
+
+export const UnionMode = {
+  Sparse: A_UnionMode.Sparse,
+  Dense: A_UnionMode.Dense,
+} as const;
+
+// ----- Schema / Field ------------------------------------------------------
+
+export function schema(
+  fields: readonly VgiField[],
+  metadata?: Map<string, string>,
+): VgiSchema {
+  return new A_Schema(
+    fields as A_Field[],
+    metadata ?? new Map(),
+  ) as unknown as VgiSchema;
+}
+
+export function field(
+  name: string,
+  type: VgiDataType,
+  nullable = true,
+  metadata?: Map<string, string>,
+): VgiField {
+  return new A_Field(
+    name,
+    type as DataType,
+    nullable,
+    metadata ?? new Map(),
+  ) as unknown as VgiField;
+}
+
+// ----- Primitive types -----------------------------------------------------
+
+export const nullType = (): VgiDataType => new A_Null() as unknown as VgiDataType;
+export const bool = (): VgiDataType => new A_Bool() as unknown as VgiDataType;
+
+export const int8 = (): VgiDataType => new A_Int8() as unknown as VgiDataType;
+export const int16 = (): VgiDataType => new A_Int16() as unknown as VgiDataType;
+export const int32 = (): VgiDataType => new A_Int32() as unknown as VgiDataType;
+export const int64 = (): VgiDataType => new A_Int64() as unknown as VgiDataType;
+export const uint8 = (): VgiDataType => new A_Uint8() as unknown as VgiDataType;
+export const uint16 = (): VgiDataType => new A_Uint16() as unknown as VgiDataType;
+export const uint32 = (): VgiDataType => new A_Uint32() as unknown as VgiDataType;
+export const uint64 = (): VgiDataType => new A_Uint64() as unknown as VgiDataType;
+export function int(bitWidth: 8 | 16 | 32 | 64 = 32, signed = true): VgiDataType {
+  if (signed) {
+    switch (bitWidth) {
+      case 8: return int8();
+      case 16: return int16();
+      case 32: return int32();
+      case 64: return int64();
+    }
+  } else {
+    switch (bitWidth) {
+      case 8: return uint8();
+      case 16: return uint16();
+      case 32: return uint32();
+      case 64: return uint64();
+    }
+  }
+  throw new Error(`int: unsupported bitWidth ${bitWidth}`);
+}
+
+export const float16 = (): VgiDataType => new A_Float16() as unknown as VgiDataType;
+export const float32 = (): VgiDataType => new A_Float32() as unknown as VgiDataType;
+export const float64 = (): VgiDataType => new A_Float64() as unknown as VgiDataType;
+
+export const utf8 = (): VgiDataType => new A_Utf8() as unknown as VgiDataType;
+export const binary = (): VgiDataType => new A_Binary() as unknown as VgiDataType;
+export const fixedSizeBinary = (byteWidth: number): VgiDataType =>
+  new A_FixedSizeBinary(byteWidth) as unknown as VgiDataType;
+
+// ----- Decimal -------------------------------------------------------------
+
+export function decimal(
+  precision: number,
+  scale: number,
+  bitWidth: 32 | 64 | 128 | 256 = 128,
+): VgiDataType {
+  // arrow-js's Decimal constructor signature is (scale, precision, bitWidth) —
+  // the OPPOSITE order of our facade (precision, scale). Passing them in facade
+  // order made arrow-js report `.precision`/`.scale` swapped versus flechette.
+  // Swap here so a facade-built decimal reports identical precision/scale on
+  // both backends.
+  return new A_Decimal(scale, precision, bitWidth) as unknown as VgiDataType;
+}
+export const decimal128 = (precision: number, scale: number): VgiDataType =>
+  decimal(precision, scale, 128);
+export const decimal256 = (precision: number, scale: number): VgiDataType =>
+  decimal(precision, scale, 256);
+
+// ----- Date / Time / Timestamp / Duration / Interval -----------------------
+
+export const date = (unit: number = A_DateUnit.MILLISECOND): VgiDataType =>
+  new A_Date(unit) as unknown as VgiDataType;
+export const dateDay = (): VgiDataType => date(A_DateUnit.DAY);
+export const dateMillisecond = (): VgiDataType => date(A_DateUnit.MILLISECOND);
+
+export function time(unit: number = A_TimeUnit.MILLISECOND, bitWidth?: 32 | 64): VgiDataType {
+  const bw = bitWidth ?? (unit <= A_TimeUnit.MILLISECOND ? 32 : 64);
+  return new A_Time(unit, bw) as unknown as VgiDataType;
+}
+export const timeSecond = (): VgiDataType => time(A_TimeUnit.SECOND, 32);
+export const timeMillisecond = (): VgiDataType => time(A_TimeUnit.MILLISECOND, 32);
+export const timeMicrosecond = (): VgiDataType => time(A_TimeUnit.MICROSECOND, 64);
+export const timeNanosecond = (): VgiDataType => time(A_TimeUnit.NANOSECOND, 64);
+
+export const timestamp = (
+  unit: number = A_TimeUnit.MILLISECOND,
+  timezone: string | null = null,
+): VgiDataType => new A_Timestamp(unit, timezone) as unknown as VgiDataType;
+
+export const duration = (unit: number = A_TimeUnit.MILLISECOND): VgiDataType =>
+  new A_Duration(unit) as unknown as VgiDataType;
+
+export const interval = (unit: number = A_IntervalUnit.MONTH_DAY_NANO): VgiDataType =>
+  new A_Interval(unit) as unknown as VgiDataType;
+
+// ----- Nested types --------------------------------------------------------
+
+export function list(child: VgiField | VgiDataType): VgiDataType {
+  const childField = isField(child) ? child : field("item", child as VgiDataType, true);
+  return new A_List(childField as A_Field) as unknown as VgiDataType;
+}
+
+export function fixedSizeList(child: VgiField | VgiDataType, listSize: number): VgiDataType {
+  const childField = isField(child) ? child : field("item", child as VgiDataType, true);
+  return new A_FixedSizeList(listSize, childField as A_Field) as unknown as VgiDataType;
+}
+
+export function struct(children: readonly VgiField[]): VgiDataType {
+  return new A_Struct(children as A_Field[]) as unknown as VgiDataType;
+}
+
+export function map(
+  keyField: VgiField | VgiDataType,
+  valueField: VgiField | VgiDataType,
+  keysSorted = false,
+): VgiDataType {
+  const k = isField(keyField) ? keyField : field("key", keyField as VgiDataType, false);
+  const v = isField(valueField) ? valueField : field("value", valueField as VgiDataType, true);
+  // arrow-js Map_ takes a Field whose type is Struct{key,value}
+  const entries = field("entries", struct([k, v]), false);
+  return new A_Map(entries as A_Field, keysSorted) as unknown as VgiDataType;
+}
+
+export function dictionary(
+  valueType: VgiDataType,
+  indexType: VgiDataType = int16(),
+  ordered = false,
+  id?: number,
+): VgiDataType {
+  return new A_Dictionary(
+    valueType as DataType,
+    indexType as any,
+    id,
+    ordered,
+  ) as unknown as VgiDataType;
+}
+
+export function union(
+  mode: number,
+  children: readonly (VgiField | VgiDataType)[],
+  typeIds?: number[],
+): VgiDataType {
+  const childFields = children.map((c, i) =>
+    isField(c) ? c : field(String(i), c as VgiDataType, true),
+  );
+  const ids = typeIds ?? childFields.map((_, i) => i);
+  const Ctor = mode === A_UnionMode.Dense ? A_DenseUnion : A_SparseUnion;
+  return new Ctor(ids, childFields as A_Field[]) as unknown as VgiDataType;
+}
+
+export const sparseUnion = (
+  children: readonly (VgiField | VgiDataType)[],
+  typeIds?: number[],
+): VgiDataType => union(A_UnionMode.Sparse, children, typeIds);
+
+export const denseUnion = (
+  children: readonly (VgiField | VgiDataType)[],
+  typeIds?: number[],
+): VgiDataType => union(A_UnionMode.Dense, children, typeIds);
+
+// Utility: distinguish Field from DataType. arrow-js Fields have `name` and
+// `type`; bare DataTypes only have `typeId` (and class-specific props).
+function isField(x: VgiField | VgiDataType): x is VgiField {
+  return typeof (x as VgiField).name === "string" && (x as VgiField).type != null;
+}