@query-farm/vgi 0.6.0

package/src/arrow/impl-flechette/schema.ts

@@ -0,0 +1,202 @@
+// Copyright 2025, 2026 Query Farm LLC - https://query.farm
+// Factory-style constructors for flechette Schema/Field/DataType values.
+// flechette already exposes these as factory functions, so most of this
+// file is direct re-exports. The few wrapper functions handle ergonomic
+// gaps (e.g. allowing a bare DataType for `list(child)` rather than
+// requiring a wrapped Field).
+
+import {
+  field as f_field,
+  nullType as f_nullType,
+  bool as f_bool,
+  int as f_int,
+  int8 as f_int8,
+  int16 as f_int16,
+  int32 as f_int32,
+  int64 as f_int64,
+  uint8 as f_uint8,
+  uint16 as f_uint16,
+  uint32 as f_uint32,
+  uint64 as f_uint64,
+  float as f_float,
+  float16 as f_float16,
+  float32 as f_float32,
+  float64 as f_float64,
+  utf8 as f_utf8,
+  binary as f_binary,
+  fixedSizeBinary as f_fixedSizeBinary,
+  decimal as f_decimal,
+  decimal128 as f_decimal128,
+  decimal256 as f_decimal256,
+  date as f_date,
+  dateDay as f_dateDay,
+  dateMillisecond as f_dateMillisecond,
+  time as f_time,
+  timeSecond as f_timeSecond,
+  timeMillisecond as f_timeMillisecond,
+  timeMicrosecond as f_timeMicrosecond,
+  timeNanosecond as f_timeNanosecond,
+  timestamp as f_timestamp,
+  duration as f_duration,
+  interval as f_interval,
+  list as f_list,
+  fixedSizeList as f_fixedSizeList,
+  struct as f_struct,
+  map as f_map,
+  dictionary as f_dictionary,
+  union as f_union,
+  TimeUnit as F_TimeUnit,
+  DateUnit as F_DateUnit,
+  IntervalUnit as F_IntervalUnit,
+  UnionMode as F_UnionMode,
+} from "@query-farm/flechette";
+
+import type { VgiField, VgiSchema, VgiDataType } from "../types.js";
+
+// ----- Unit / mode constants -----------------------------------------------
+
+export const TimeUnit = F_TimeUnit;
+export const DateUnit = F_DateUnit;
+export const IntervalUnit = F_IntervalUnit;
+export const UnionMode = F_UnionMode;
+
+// ----- Schema / Field ------------------------------------------------------
+
+export function schema(
+  fields: readonly VgiField[],
+  metadata?: Map<string, string>,
+): VgiSchema {
+  return {
+    fields: fields as readonly any[] as VgiField[],
+    metadata: metadata ?? new Map(),
+  } as VgiSchema;
+}
+
+export function field(
+  name: string,
+  type: VgiDataType,
+  nullable = true,
+  metadata?: Map<string, string>,
+): VgiField {
+  return f_field(
+    name,
+    type as any,
+    nullable,
+    metadata ?? new Map(),
+  ) as unknown as VgiField;
+}
+
+// ----- Primitive types -----------------------------------------------------
+
+export const nullType = (): VgiDataType => f_nullType() as unknown as VgiDataType;
+export const bool = (): VgiDataType => f_bool() as unknown as VgiDataType;
+
+export const int8 = (): VgiDataType => f_int8() as unknown as VgiDataType;
+export const int16 = (): VgiDataType => f_int16() as unknown as VgiDataType;
+export const int32 = (): VgiDataType => f_int32() as unknown as VgiDataType;
+export const int64 = (): VgiDataType => f_int64() as unknown as VgiDataType;
+export const uint8 = (): VgiDataType => f_uint8() as unknown as VgiDataType;
+export const uint16 = (): VgiDataType => f_uint16() as unknown as VgiDataType;
+export const uint32 = (): VgiDataType => f_uint32() as unknown as VgiDataType;
+export const uint64 = (): VgiDataType => f_uint64() as unknown as VgiDataType;
+export const int = (bitWidth: 8 | 16 | 32 | 64 = 32, signed = true): VgiDataType =>
+  f_int(bitWidth, signed) as unknown as VgiDataType;
+
+export const float16 = (): VgiDataType => f_float16() as unknown as VgiDataType;
+export const float32 = (): VgiDataType => f_float32() as unknown as VgiDataType;
+export const float64 = (): VgiDataType => f_float64() as unknown as VgiDataType;
+
+export const utf8 = (): VgiDataType => f_utf8() as unknown as VgiDataType;
+export const binary = (): VgiDataType => f_binary() as unknown as VgiDataType;
+export const fixedSizeBinary = (stride: number): VgiDataType =>
+  f_fixedSizeBinary(stride) as unknown as VgiDataType;
+
+// ----- Decimal -------------------------------------------------------------
+
+export function decimal(
+  precision: number,
+  scale: number,
+  bitWidth: 32 | 64 | 128 | 256 = 128,
+): VgiDataType {
+  return f_decimal(precision, scale, bitWidth) as unknown as VgiDataType;
+}
+export const decimal128 = (precision: number, scale: number): VgiDataType =>
+  f_decimal128(precision, scale) as unknown as VgiDataType;
+export const decimal256 = (precision: number, scale: number): VgiDataType =>
+  f_decimal256(precision, scale) as unknown as VgiDataType;
+
+// ----- Date / Time / Timestamp / Duration / Interval -----------------------
+
+export const date = (unit: number = F_DateUnit.MILLISECOND): VgiDataType =>
+  f_date(unit as any) as unknown as VgiDataType;
+export const dateDay = (): VgiDataType => f_dateDay() as unknown as VgiDataType;
+export const dateMillisecond = (): VgiDataType => f_dateMillisecond() as unknown as VgiDataType;
+
+export function time(unit: number = F_TimeUnit.MILLISECOND, _bitWidth?: 32 | 64): VgiDataType {
+  // bitWidth is implicit in flechette (derived from unit), so we ignore _bitWidth.
+  return f_time(unit as any) as unknown as VgiDataType;
+}
+export const timeSecond = (): VgiDataType => f_timeSecond() as unknown as VgiDataType;
+export const timeMillisecond = (): VgiDataType => f_timeMillisecond() as unknown as VgiDataType;
+export const timeMicrosecond = (): VgiDataType => f_timeMicrosecond() as unknown as VgiDataType;
+export const timeNanosecond = (): VgiDataType => f_timeNanosecond() as unknown as VgiDataType;
+
+export const timestamp = (
+  unit: number = F_TimeUnit.MILLISECOND,
+  timezone: string | null = null,
+): VgiDataType => f_timestamp(unit as any, timezone) as unknown as VgiDataType;
+
+export const duration = (unit: number = F_TimeUnit.MILLISECOND): VgiDataType =>
+  f_duration(unit as any) as unknown as VgiDataType;
+
+export const interval = (unit: number = F_IntervalUnit.MONTH_DAY_NANO): VgiDataType =>
+  f_interval(unit as any) as unknown as VgiDataType;
+
+// ----- Nested types --------------------------------------------------------
+
+export function list(child: VgiField | VgiDataType): VgiDataType {
+  return f_list(child as any) as unknown as VgiDataType;
+}
+
+export function fixedSizeList(child: VgiField | VgiDataType, listSize: number): VgiDataType {
+  return f_fixedSizeList(child as any, listSize) as unknown as VgiDataType;
+}
+
+export function struct(children: readonly VgiField[]): VgiDataType {
+  return f_struct(children as any) as unknown as VgiDataType;
+}
+
+export function map(
+  keyField: VgiField | VgiDataType,
+  valueField: VgiField | VgiDataType,
+  keysSorted = false,
+): VgiDataType {
+  return f_map(keyField as any, valueField as any, keysSorted) as unknown as VgiDataType;
+}
+
+export function dictionary(
+  valueType: VgiDataType,
+  indexType: VgiDataType = int16(),
+  ordered = false,
+  id?: number,
+): VgiDataType {
+  return f_dictionary(valueType as any, indexType as any, ordered, id) as unknown as VgiDataType;
+}
+
+export function union(
+  mode: number,
+  children: readonly (VgiField | VgiDataType)[],
+  typeIds?: number[],
+): VgiDataType {
+  return f_union(mode as any, children as any, typeIds) as unknown as VgiDataType;
+}
+
+export const sparseUnion = (
+  children: readonly (VgiField | VgiDataType)[],
+  typeIds?: number[],
+): VgiDataType => union(F_UnionMode.Sparse, children, typeIds);
+
+export const denseUnion = (
+  children: readonly (VgiField | VgiDataType)[],
+  typeIds?: number[],
+): VgiDataType => union(F_UnionMode.Dense, children, typeIds);

package/src/arrow/impl-flechette/statistics.ts

@@ -0,0 +1,198 @@
+// Copyright 2025, 2026 Query Farm LLC - https://query.farm
+// flechette impl of column-statistics serialization.
+//
+// The empty-stats batch (NDV-only path, no min/max) ships a one-child
+// sparse_union placeholder. Non-empty stats build a real sparse_union<T0,..>
+// min/max column whose per-row type code selects the child holding that row's
+// value — routed through the codec (rich -> canonical) and the canonical writer
+// so a date/timestamp/decimal min/max is represented identically to ordinary
+// column data and identically to the arrow-js backend.
+
+import {
+  columnFromArray as f_columnFromArray,
+  union as f_union,
+  field as f_field,
+  Table,
+  UnionMode,
+  type Column,
+} from "@query-farm/flechette";
+import type { VgiBatch, VgiDataType } from "../types.js";
+import { emptyBatch } from "./empty.js";
+import { codecFor } from "../codec/registry.js";
+import { prepareForFlechette } from "./canonical.js";
+import { toFlechetteType } from "./normalize-type.js";
+import {
+  bool,
+  field,
+  int64,
+  int8,
+  schema,
+  sparseUnion,
+  uint64,
+  utf8,
+} from "./schema.js";
+
+const COLUMN_OPTS = {
+  useBigInt: true,
+  useBigIntTimestamp: true,
+  useDecimalInt: true,
+} as const;
+
+export interface ColumnStatistics {
+  columnName: string;
+  arrowType: VgiDataType;
+  min: any;
+  max: any;
+  hasNull: boolean;
+  hasNotNull: boolean;
+  distinctCount: bigint | number | null;
+  containsUnicode: boolean | null;
+  maxStringLength: bigint | number | null;
+}
+
+// Stable, backend-agnostic type key. flechette types are plain objects whose
+// `.toString()` is "[object Object]" for every type (collapsing date32/date64/
+// timestamp/decimal into ONE union child), so dispatch on the structural fields
+// instead: typeId + unit + bitWidth + precision/scale + byteWidth.
+function typeKey(t: VgiDataType): string {
+  const a = t as any;
+  return [
+    t.typeId,
+    a.unit ?? "",
+    a.bitWidth ?? "",
+    a.precision ?? "",
+    a.scale ?? "",
+    a.byteWidth ?? "",
+  ].join(":");
+}
+
+// Canonical 8-column ColumnStatistics schema with a one-child sparse_union.
+// flechette can't build a 0-length sparse-union whose child is Null (its
+// columnBuilder rejects "Unsupported data type"); use Int8 as a structurally-
+// equivalent placeholder. A 0-row batch carries no values, so the child type
+// only matters for IPC schema serialization — the C++ extension treats any
+// 0-row stats blob as "stats are authoritative, just empty" regardless of
+// the union's child shape.
+function emptyStatsSchema() {
+  const placeholderChild = field("0", int8(), true);
+  const minMaxUnion = sparseUnion([placeholderChild], [0]);
+  return schema([
+    field("column_name", utf8(), false),
+    field("min", minMaxUnion, true),
+    field("max", minMaxUnion, true),
+    field("has_null", bool(), true),
+    field("has_not_null", bool(), true),
+    field("distinct_count", int64(), true),
+    field("contains_unicode", bool(), true),
+    field("max_string_length", uint64(), true),
+  ]);
+}
+
+/**
+ * Build a flechette Column for `type` from RICH values, routed through the codec
+ * (rich -> canonical) and the same per-type preparation the public canonical
+ * writer uses. Keeps stat values byte-identical to ordinary column data.
+ */
+function buildTypedColumn(values: any[], type: VgiDataType): Column<any> {
+  const codec = codecFor(type);
+  const canonical = values.map((v) => codec.richToCanonical(v));
+  const prepared = prepareForFlechette(type, canonical);
+  return f_columnFromArray(prepared, toFlechetteType(type) as any, COLUMN_OPTS);
+}
+
+/**
+ * Build the sparse-union min/max Column. `rowTypeCodes[i]` selects which child
+ * (one per distinct Arrow type) holds row i's value; the builder fills the
+ * other children with null at that slot (sparse-union invariant). Values are
+ * routed through codec rich -> canonical -> flechette prep so each child sees
+ * exactly what it would for ordinary column data.
+ */
+function buildUnionColumn(
+  values: any[],
+  rowTypeCodes: number[],
+  typeOrder: VgiDataType[],
+): Column<any> {
+  const typeIds = typeOrder.map((_t, i) => i);
+  const children = typeOrder.map((t, code) =>
+    f_field(String(code), toFlechetteType(t) as any, true),
+  );
+  // Per-row prepared value (only the selected child consumes it; non-selected
+  // children get null from the SparseUnionBuilder).
+  const prepared = values.map((v, i) => {
+    if (v == null) return null;
+    const type = typeOrder[rowTypeCodes[i]];
+    const codec = codecFor(type);
+    return prepareForFlechette(type, [codec.richToCanonical(v)])[0];
+  });
+  const unionType = f_union(
+    UnionMode.Sparse,
+    children as any,
+    typeIds,
+    (_value: any, index: number) => rowTypeCodes[index],
+  );
+  return f_columnFromArray(prepared, unionType as any, COLUMN_OPTS);
+}
+
+export function buildStatisticsBatch(stats: ColumnStatistics[]): VgiBatch {
+  if (stats.length === 0) {
+    return emptyBatch(emptyStatsSchema());
+  }
+
+  // Group distinct Arrow types in stable insertion order to assign type codes.
+  const typeOrder: VgiDataType[] = [];
+  const typeIdMap = new Map<string, number>();
+  const rowTypeCodes: number[] = [];
+  for (const s of stats) {
+    const key = typeKey(s.arrowType);
+    let code = typeIdMap.get(key);
+    if (code === undefined) {
+      code = typeOrder.length;
+      typeIdMap.set(key, code);
+      typeOrder.push(s.arrowType);
+    }
+    rowTypeCodes.push(code);
+  }
+
+  const minCol = buildUnionColumn(
+    stats.map((s) => s.min),
+    rowTypeCodes,
+    typeOrder,
+  );
+  const maxCol = buildUnionColumn(
+    stats.map((s) => s.max),
+    rowTypeCodes,
+    typeOrder,
+  );
+
+  const minMaxUnionType = minCol.type;
+
+  const childCols: Column<any>[] = [
+    buildTypedColumn(stats.map((s) => s.columnName), utf8()),
+    minCol,
+    maxCol,
+    buildTypedColumn(stats.map((s) => s.hasNull), bool()),
+    buildTypedColumn(stats.map((s) => s.hasNotNull), bool()),
+    buildTypedColumn(
+      stats.map((s) => (s.distinctCount == null ? null : BigInt(s.distinctCount))),
+      int64(),
+    ),
+    buildTypedColumn(stats.map((s) => s.containsUnicode), bool()),
+    buildTypedColumn(
+      stats.map((s) => (s.maxStringLength == null ? null : BigInt(s.maxStringLength))),
+      uint64(),
+    ),
+  ];
+
+  const fields = [
+    f_field("column_name", toFlechetteType(utf8()) as any, false),
+    f_field("min", minMaxUnionType as any, true),
+    f_field("max", minMaxUnionType as any, true),
+    f_field("has_null", toFlechetteType(bool()) as any, true),
+    f_field("has_not_null", toFlechetteType(bool()) as any, true),
+    f_field("distinct_count", toFlechetteType(int64()) as any, true),
+    f_field("contains_unicode", toFlechetteType(bool()) as any, true),
+    f_field("max_string_length", toFlechetteType(uint64()) as any, true),
+  ];
+  const flechSchema = { version: 5, endianness: 0, fields, metadata: null };
+  return new Table(flechSchema as any, childCols) as unknown as VgiBatch;
+}

package/src/arrow/index.ts

@@ -0,0 +1,129 @@
+// Copyright 2025, 2026 Query Farm LLC - https://query.farm
+// Public facade. Internal vgi-typescript code imports Arrow operations
+// from here; the active backend is selected at module-resolution time
+// via the `#arrow-impl` subpath in package.json's `imports` field.
+//
+// Selection conditions:
+//   workerd / worker / browser  -> impl-flechette (Cloudflare Workers, etc.)
+//   default                     -> impl-arrowjs   (Bun worker, Node)
+
+// Backend-agnostic types (defined in this package, satisfied structurally
+// by both arrow-js and flechette native values).
+export type {
+  VgiTypeId,
+  VgiDataType,
+  VgiField,
+  VgiSchema,
+  VgiColumn,
+  VgiColumnData,
+  VgiBatch,
+  VgiBackendInfo,
+  TaggedUnion,
+} from "./types.js";
+
+// Backend-agnostic predicates.
+export {
+  TypeId,
+  isNull,
+  isInt,
+  isFloat,
+  isBinary,
+  isUtf8,
+  isBool,
+  isDecimal,
+  isDate,
+  isTime,
+  isTimestamp,
+  isInterval,
+  isList,
+  isStruct,
+  isUnion,
+  isFixedSizeBinary,
+  isFixedSizeList,
+  isMap,
+  isDuration,
+  isDictionary,
+  isBatch,
+} from "./predicates.js";
+
+// Active backend's surface (IPC, build, iterate, utilities).
+export {
+  backend,
+  // batch utilities
+  emptyBatch, batchFromRows, batchFromColumns, columnFromArray,
+  iterRows, batchToScalarDict, batchToSecretDict, safeNumber, decodeDictValue,
+  filterBatch, projectSchema, projectBatch,
+  // canonical single-value read (backend-agnostic, lossless)
+  readCanonicalValue,
+  // IPC
+  serializeSchema, deserializeSchema, serializeBatch, deserializeBatch,
+  // Column statistics
+  buildStatisticsBatch,
+} from "#arrow-impl";
+
+export type { ColumnStatistics } from "#arrow-impl";
+
+// Type factories — exported from the TYPED facade layer (schema-types.ts) so
+// every factory carries a PRECISE nominal static descriptor (Date32Type,
+// TimestampType<'us'>, StructType<...>, …). Runtime values are identical to the
+// backend factories; the typed layer only refines the static return type, which
+// is what makes the rich/raw value mapping (RichValue/RawValue/ValueFor) and
+// the statically-typed define* author API work. Both backends report the same
+// static types from here.
+export {
+  schema, field,
+  nullType, bool,
+  int, int8, int16, int32, int64,
+  uint8, uint16, uint32, uint64,
+  float16, float32, float64,
+  utf8, binary, fixedSizeBinary,
+  decimal, decimal128, decimal256,
+  date, dateDay, dateMillisecond,
+  time, timeSecond, timeMillisecond, timeMicrosecond, timeNanosecond,
+  timestamp, duration, interval,
+  timestampSeconds, timestampMillis, timestampMicros, timestampNanos,
+  durationSeconds, durationMillis, durationMicros, durationNanos,
+  list, fixedSizeList, struct, map,
+  dictionary,
+  union, sparseUnion, denseUnion,
+  TimeUnit, DateUnit, IntervalUnit, UnionMode,
+} from "./schema-types.js";
+
+// Precise nominal type descriptors (the static face of the factory returns).
+export type {
+  NullDescriptor,
+  BoolType,
+  IntType, Int8Type, Int16Type, Int32Type, Int64Type,
+  Uint8Type, Uint16Type, Uint32Type, Uint64Type,
+  FloatType, Utf8Type, LargeUtf8Type, BinaryType, LargeBinaryType,
+  FixedSizeBinaryType, DecimalType,
+  Date32Type, Date64Type, Time32Type, Time64Type,
+  TimestampType, DurationType, IntervalType,
+  ListType, FixedSizeListType, StructType, MapType, DictionaryType, UnionType,
+  TypedField, TUnit,
+} from "./codec/type-descriptors.js";
+
+// Rich/raw value mapping.
+export type { RichValue, RawValue, ValueFor, Repr } from "./codec/repr.js";
+
+// Codec registry (single value rich<->canonical<->raw converters for manual use).
+export { codecFor } from "./codec/registry.js";
+export type { Codec } from "./codec/registry.js";
+
+// Branded raw scalar types + validating constructors / unwrappers.
+export type {
+  Branded,
+  Date32, Date64Ms,
+  Time32S, Time32Ms, Time64Us, Time64Ns,
+  TimestampSeconds, TimestampMillis, TimestampMicros, TimestampNanos,
+  DurationSeconds, DurationMillis, DurationMicros, DurationNanos,
+  UnscaledDecimal, Int64 as Int64Raw, Uint64 as Uint64Raw,
+} from "./codec/branded.js";
+export {
+  asDate32, asDate64Ms,
+  asTime32S, asTime32Ms, asTime64Us, asTime64Ns,
+  asTimestampSeconds, asTimestampMillis, asTimestampMicros, asTimestampNanos,
+  asDurationSeconds, asDurationMillis, asDurationMicros, asDurationNanos,
+  asUnscaledDecimal, asInt64, asUint64,
+  fromBranded, fromBrandedNumber, fromBrandedBigInt,
+} from "./codec/branded.js";

package/src/arrow/predicates.ts

@@ -0,0 +1,66 @@
+// Copyright 2025, 2026 Query Farm LLC - https://query.farm
+// Backend-agnostic Arrow type predicates. Both arrow-js and flechette
+// expose `typeId` on every DataType with identical numeric values, so a
+// single `t.typeId === N` check works regardless of the active backend.
+
+import type { VgiDataType } from "./types.js";
+
+export const TypeId = {
+  Null: 1,
+  Int: 2,
+  Float: 3,
+  Binary: 4,
+  Utf8: 5,
+  Bool: 6,
+  Decimal: 7,
+  Date: 8,
+  Time: 9,
+  Timestamp: 10,
+  Interval: 11,
+  List: 12,
+  Struct: 13,
+  Union: 14,
+  FixedSizeBinary: 15,
+  FixedSizeList: 16,
+  Map: 17,
+  Duration: 18,
+  LargeBinary: 19,
+  LargeUtf8: 20,
+  Dictionary: -1,
+} as const;
+
+export function isNull(t: VgiDataType): boolean { return t.typeId === TypeId.Null; }
+export function isInt(t: VgiDataType): boolean { return t.typeId === TypeId.Int; }
+export function isFloat(t: VgiDataType): boolean { return t.typeId === TypeId.Float; }
+export function isBinary(t: VgiDataType): boolean {
+  return t.typeId === TypeId.Binary || t.typeId === TypeId.LargeBinary;
+}
+export function isUtf8(t: VgiDataType): boolean {
+  return t.typeId === TypeId.Utf8 || t.typeId === TypeId.LargeUtf8;
+}
+export function isBool(t: VgiDataType): boolean { return t.typeId === TypeId.Bool; }
+export function isDecimal(t: VgiDataType): boolean { return t.typeId === TypeId.Decimal; }
+export function isDate(t: VgiDataType): boolean { return t.typeId === TypeId.Date; }
+export function isTime(t: VgiDataType): boolean { return t.typeId === TypeId.Time; }
+export function isTimestamp(t: VgiDataType): boolean { return t.typeId === TypeId.Timestamp; }
+export function isInterval(t: VgiDataType): boolean { return t.typeId === TypeId.Interval; }
+export function isList(t: VgiDataType): boolean { return t.typeId === TypeId.List; }
+export function isStruct(t: VgiDataType): boolean { return t.typeId === TypeId.Struct; }
+export function isUnion(t: VgiDataType): boolean { return t.typeId === TypeId.Union; }
+export function isFixedSizeBinary(t: VgiDataType): boolean { return t.typeId === TypeId.FixedSizeBinary; }
+export function isFixedSizeList(t: VgiDataType): boolean { return t.typeId === TypeId.FixedSizeList; }
+export function isMap(t: VgiDataType): boolean { return t.typeId === TypeId.Map; }
+export function isDuration(t: VgiDataType): boolean { return t.typeId === TypeId.Duration; }
+export function isDictionary(t: VgiDataType): boolean { return t.typeId === TypeId.Dictionary; }
+
+/** Runtime check: does `x` quack like a VgiBatch? Used at API boundaries
+ *  where callers may pass either a batch or a plain column dict. Duck-typed
+ *  because arrow-js and flechette use unrelated classes. */
+export function isBatch(x: unknown): x is import("./types.js").VgiBatch {
+  return (
+    x != null &&
+    typeof (x as any).numRows === "number" &&
+    (x as any).schema != null &&
+    Array.isArray((x as any).schema.fields)
+  );
+}