@query-farm/vgi 0.6.0

package/src/filter-pushdown/evaluate.ts

@@ -0,0 +1,591 @@
+// Copyright 2025, 2026 Query Farm LLC - https://query.farm
+// Filter pushdown evaluation: row-by-row predicate evaluation against
+// Arrow RecordBatches, plus the PushdownFilters class that owns a parsed
+// filter set and applies it to batches.
+
+import {
+  type VgiBatch,
+  type VgiSchema,
+  type VgiDataType,
+  schema,
+  readCanonicalValue,
+} from "../arrow/index.js";
+import { batchFromColumns, filterBatch } from "../util/arrow/index.js";
+import { codecFor } from "../arrow/codec/registry.js";
+import {
+  ComparisonOp,
+  type AndFilter,
+  type ConstantFilter,
+  type ExpressionFilter,
+  type ExprNode,
+  type Filter,
+  type InFilter,
+  type IsNotNullFilter,
+  type IsNullFilter,
+  type OrFilter,
+  type StructFilter,
+} from "./types.js";
+
+// ============================================================================
+// Comparison helpers
+// ============================================================================
+
+function compare(a: any, b: any, op: ComparisonOp): boolean {
+  switch (op) {
+    case ComparisonOp.EQ: return a === b;
+    case ComparisonOp.NE: return a !== b;
+    case ComparisonOp.GT: return a > b;
+    case ComparisonOp.GE: return a >= b;
+    case ComparisonOp.LT: return a < b;
+    case ComparisonOp.LE: return a <= b;
+  }
+}
+
+/**
+ * Read a column cell at `index` in CANONICAL form (the same representation the
+ * filter literals are deserialized into — see deserialize.ts `getValue`). Going
+ * through the per-backend canonical reader makes this work on BOTH Arrow
+ * backends (arrow-js's `col.get` and flechette's `col.at` are not interchange-
+ * able) and guarantees temporal/decimal cells and literals compare like-for-
+ * like (e.g. timestamp -> bigint on both sides). `colIndex` resolves the field
+ * type from the batch schema.
+ */
+function readCell(batch: VgiBatch, colIndex: number, index: number): any {
+  const col = batch.getChildAt(colIndex);
+  if (!col) return null;
+  const type = batch.schema.fields[colIndex].type as unknown as VgiDataType;
+  return readCanonicalValue(type, col, index);
+}
+
+// ============================================================================
+// Filter evaluation (row-by-row)
+// ============================================================================
+
+function evaluateFilter(
+  filter: Filter,
+  batch: VgiBatch,
+  mask: Uint8Array,
+): void {
+  switch (filter.type) {
+    case "constant":
+      evaluateConstant(filter, batch, mask);
+      break;
+    case "is_null":
+      evaluateIsNull(filter, batch, mask);
+      break;
+    case "is_not_null":
+      evaluateIsNotNull(filter, batch, mask);
+      break;
+    case "in":
+      evaluateIn(filter, batch, mask);
+      break;
+    case "and":
+      evaluateAnd(filter, batch, mask);
+      break;
+    case "or":
+      evaluateOr(filter, batch, mask);
+      break;
+    case "struct":
+      evaluateStruct(filter, batch, mask);
+      break;
+    case "expression":
+      evaluateExpression(filter, batch, mask);
+      break;
+  }
+}
+
+// Recursively evaluate an expression AST against a row of the batch, returning
+// the scalar value it produces. column_ref reads from the batch using the
+// caller-supplied remap (DuckDB rewrites column refs inside an ExpressionFilter
+// so index=0 means "this filter's column"); constant returns its pre-resolved
+// JS value; function dispatches on function_name; comparison/conjunction
+// combine children.
+function evaluateExpr(
+  node: ExprNode,
+  batch: VgiBatch,
+  rowIndex: number,
+  colRefToBatchIndex: (index: number) => number,
+): any {
+  switch (node.expr_type) {
+    case "column_ref":
+      return readCell(batch, colRefToBatchIndex(node.index), rowIndex);
+    case "constant":
+      return node.value;
+    case "function":
+      return evalFunction(
+        node.function_name,
+        node.children.map((c) => evaluateExpr(c, batch, rowIndex, colRefToBatchIndex)),
+      );
+    case "comparison": {
+      const left = evaluateExpr(node.left, batch, rowIndex, colRefToBatchIndex);
+      const right = evaluateExpr(node.right, batch, rowIndex, colRefToBatchIndex);
+      if (left == null || right == null) return null;
+      return compare(left, right, node.op);
+    }
+    case "conjunction": {
+      if (node.conjunction_type === "and") {
+        for (const c of node.children) {
+          const v = evaluateExpr(c, batch, rowIndex, colRefToBatchIndex);
+          if (v !== true) return v === false ? false : null;
+        }
+        return true;
+      }
+      let sawNull = false;
+      for (const c of node.children) {
+        const v = evaluateExpr(c, batch, rowIndex, colRefToBatchIndex);
+        if (v === true) return true;
+        if (v == null) sawNull = true;
+      }
+      return sawNull ? null : false;
+    }
+  }
+}
+
+// Dispatch DuckDB scalar-function names we support as pushdown filters.
+// Return null for any missing argument (SQL three-valued logic).
+function evalFunction(name: string, args: any[]): any {
+  if (args.some((a) => a == null)) return null;
+  switch (name) {
+    case "list_contains":
+    case "array_contains": {
+      // args[0] is an Arrow ListVector row (iterable), args[1] is the needle.
+      const [list, needle] = args;
+      if (!list || typeof list[Symbol.iterator] !== "function") return false;
+      for (const v of list) {
+        if (v === needle) return true;
+      }
+      return false;
+    }
+    case "starts_with":
+    case "prefix": {
+      const [hay, needle] = args;
+      return typeof hay === "string" && typeof needle === "string"
+        ? hay.startsWith(needle) : false;
+    }
+    case "contains": {
+      const [hay, needle] = args;
+      return typeof hay === "string" && typeof needle === "string"
+        ? hay.includes(needle) : false;
+    }
+    case "&&":
+    case "st_intersects_extent": {
+      // Bounding-box intersection between two WKB geometries. Parses each
+      // geometry's XY bbox and checks for overlap in both dimensions.
+      const [a, b] = args;
+      const ba = wkbBBox(a);
+      const bb = wkbBBox(b);
+      if (!ba || !bb) return false;
+      return !(ba.maxX < bb.minX || ba.minX > bb.maxX ||
+               ba.maxY < bb.minY || ba.minY > bb.maxY);
+    }
+    default:
+      // Unsupported functions shouldn't make it past the C++-side
+      // `ExpressionTreeIsSupported` gate, but be defensive: evaluate to null
+      // so the row is filtered out conservatively rather than silently
+      // producing wrong results.
+      return null;
+  }
+}
+
+// Parse the XY bounding box of a WKB geometry. Supports Point and Polygon for
+// ST_MakeEnvelope output; extend as new geometry kinds are needed.
+function wkbBBox(bytes: any): { minX: number; minY: number; maxX: number; maxY: number } | null {
+  if (!(bytes instanceof Uint8Array)) return null;
+  if (bytes.length < 5) return null;
+  const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
+  const le = view.getUint8(0) === 1;
+  const type = view.getUint32(1, le);
+  // Point (type=1): 21 bytes total (5 header + 8+8 coords)
+  if (type === 1 && bytes.length >= 21) {
+    const x = view.getFloat64(5, le);
+    const y = view.getFloat64(13, le);
+    return { minX: x, minY: y, maxX: x, maxY: y };
+  }
+  // Polygon (type=3): used by ST_MakeEnvelope. One ring, 4 points, close.
+  // Header 5 + num_rings 4 + num_points 4 + 4*16 coord bytes = 81 bytes.
+  if (type === 3 && bytes.length >= 81) {
+    const numRings = view.getUint32(5, le);
+    if (numRings < 1) return null;
+    const numPoints = view.getUint32(9, le);
+    let minX = Infinity, minY = Infinity, maxX = -Infinity, maxY = -Infinity;
+    let off = 13;
+    for (let i = 0; i < numPoints && off + 16 <= bytes.length; i++, off += 16) {
+      const x = view.getFloat64(off, le);
+      const y = view.getFloat64(off + 8, le);
+      if (x < minX) minX = x;
+      if (x > maxX) maxX = x;
+      if (y < minY) minY = y;
+      if (y > maxY) maxY = y;
+    }
+    if (minX === Infinity) return null;
+    return { minX, minY, maxX, maxY };
+  }
+  return null;
+}
+
+function evaluateExpression(
+  filter: ExpressionFilter,
+  batch: VgiBatch,
+  mask: Uint8Array,
+): void {
+  const n = batch.numRows;
+  // DuckDB's FilterSerializer rewrites bound references inside an
+  // ExpressionFilter so that column_ref index 0 refers to the filter's own
+  // column (see ReplaceWithBoundReference in DuckDB's optimizer). Higher
+  // indices on the same filter aren't produced — the filter is always
+  // scoped to a single column in the outer batch — so we map every
+  // column_ref back to `filter.columnIndex` by name, resolving through the
+  // current batch in case projection reordered columns.
+  const colName = filter.columnName;
+  let targetIdx = batch.schema.fields.findIndex((f) => f.name === colName);
+  if (targetIdx < 0) targetIdx = filter.columnIndex;
+  const colRef = (_nodeIndex: number) => targetIdx;
+  for (let i = 0; i < n; i++) {
+    if (!mask[i]) continue;
+    const v = evaluateExpr(filter.expr, batch, i, colRef);
+    mask[i] = v === true ? 1 : 0;
+  }
+}
+
+function evaluateConstant(
+  filter: ConstantFilter,
+  batch: VgiBatch,
+  mask: Uint8Array,
+): void {
+  const n = batch.numRows;
+  for (let i = 0; i < n; i++) {
+    if (!mask[i]) continue;
+    const cell = readCell(batch, filter.columnIndex, i);
+    if (cell === null) {
+      mask[i] = 0;
+    } else {
+      mask[i] = compare(cell, filter.value, filter.op) ? 1 : 0;
+    }
+  }
+}
+
+function evaluateIsNull(
+  filter: IsNullFilter,
+  batch: VgiBatch,
+  mask: Uint8Array,
+): void {
+  const n = batch.numRows;
+  for (let i = 0; i < n; i++) {
+    if (!mask[i]) continue;
+    mask[i] = (readCell(batch, filter.columnIndex, i) !== null) ? 0 : 1;
+  }
+}
+
+function evaluateIsNotNull(
+  filter: IsNotNullFilter,
+  batch: VgiBatch,
+  mask: Uint8Array,
+): void {
+  const n = batch.numRows;
+  for (let i = 0; i < n; i++) {
+    if (!mask[i]) continue;
+    mask[i] = (readCell(batch, filter.columnIndex, i) !== null) ? 1 : 0;
+  }
+}
+
+function evaluateIn(
+  filter: InFilter,
+  batch: VgiBatch,
+  mask: Uint8Array,
+): void {
+  const n = batch.numRows;
+  for (let i = 0; i < n; i++) {
+    if (!mask[i]) continue;
+    const cell = readCell(batch, filter.columnIndex, i);
+    if (cell === null) {
+      mask[i] = 0;
+    } else {
+      mask[i] = filter.values.has(cell) ? 1 : 0;
+    }
+  }
+}
+
+function evaluateAnd(
+  filter: AndFilter,
+  batch: VgiBatch,
+  mask: Uint8Array,
+): void {
+  // AND: start with all-pass, then narrow with each child
+  const n = batch.numRows;
+  const childMask = new Uint8Array(n);
+
+  for (const child of filter.children) {
+    childMask.fill(1);
+    evaluateFilter(child, batch, childMask);
+    for (let i = 0; i < n; i++) {
+      mask[i] &= childMask[i];
+    }
+  }
+}
+
+function evaluateOr(
+  filter: OrFilter,
+  batch: VgiBatch,
+  mask: Uint8Array,
+): void {
+  // OR: start with all-fail, then widen with each child
+  const n = batch.numRows;
+  const result = new Uint8Array(n); // all zeros
+  const childMask = new Uint8Array(n);
+
+  for (const child of filter.children) {
+    childMask.fill(1);
+    evaluateFilter(child, batch, childMask);
+    for (let i = 0; i < n; i++) {
+      result[i] |= childMask[i];
+    }
+  }
+
+  // Combine with incoming mask
+  for (let i = 0; i < n; i++) {
+    mask[i] &= result[i];
+  }
+}
+
+function evaluateStruct(
+  filter: StructFilter,
+  batch: VgiBatch,
+  mask: Uint8Array,
+): void {
+  const structType = batch.schema.fields[filter.columnIndex].type as unknown as VgiDataType;
+  const childField = (structType as any).children?.[filter.childIndex];
+  if (!childField) {
+    // Child field not found — fail all rows
+    mask.fill(0);
+    return;
+  }
+  const childType = childField.type as VgiDataType;
+
+  // Read the struct column in canonical form (backend-agnostic), pull out the
+  // target child's canonical value per row, and rebuild a single-column child
+  // batch via batchFromColumns (which re-runs canonical writes). Filter literals
+  // are compared in canonical form too (see deserialize.ts), so the child column
+  // cells and literals stay in matching representations across both backends.
+  const childCanonical = Array.from({ length: batch.numRows }, (_, i) => {
+    const structVal = readCell(batch, filter.columnIndex, i) as Record<string, unknown> | null;
+    return structVal == null ? null : (structVal[childField.name] ?? null);
+  });
+  // batchFromColumns expects RICH values; map canonical -> rich for the child.
+  const childCodec = codecFor(childType);
+  const childRich = childCanonical.map((v) => childCodec.canonicalToRich(v));
+
+  const childSchema = schema([
+    (structType as any).children[filter.childIndex],
+  ]);
+  const childBatch = batchFromColumns(
+    { [filter.childName]: childRich },
+    childSchema,
+  );
+
+  // Remap child filter's columnIndex to 0 for the wrapper batch
+  const remapped = { ...filter.childFilter, columnIndex: 0 };
+  evaluateFilter(remapped, childBatch, mask);
+}
+
+// ============================================================================
+// SQL formatting
+// ============================================================================
+
+const OP_SYMBOLS: Record<ComparisonOp, string> = {
+  [ComparisonOp.EQ]: "=",
+  [ComparisonOp.NE]: "!=",
+  [ComparisonOp.GT]: ">",
+  [ComparisonOp.GE]: ">=",
+  [ComparisonOp.LT]: "<",
+  [ComparisonOp.LE]: "<=",
+};
+
+function formatValue(v: any): string {
+  if (typeof v === "string") return `'${v}'`;
+  if (typeof v === "bigint") return String(v);
+  return String(v);
+}
+
+function filterToSql(f: Filter): string {
+  const col = f.columnName;
+
+  switch (f.type) {
+    case "constant":
+      return `${col} ${OP_SYMBOLS[f.op]} ${formatValue(f.value)}`;
+    case "is_null":
+      return `${col} IS NULL`;
+    case "is_not_null":
+      return `${col} IS NOT NULL`;
+    case "in": {
+      // Match Python's repr: summarize large lists as `N values`, otherwise
+      // render each element. Threshold matches vgi-python's InFilter.__repr__.
+      const size = f.values.size;
+      if (size > 20) {
+        return `${col} IN (${size} values)`;
+      }
+      const vals = [...f.values].map(formatValue).join(", ");
+      return `${col} IN (${vals})`;
+    }
+    case "and": {
+      const parts = f.children.map(filterToSql);
+      return `(${parts.join(" AND ")})`;
+    }
+    case "or": {
+      const parts = f.children.map(filterToSql);
+      return `(${parts.join(" OR ")})`;
+    }
+    case "struct": {
+      const nested = `${f.columnName}.${f.childName}`;
+      const remapped = { ...f.childFilter, columnName: nested };
+      return filterToSql(remapped);
+    }
+    case "expression":
+      // Expression filters don't have a natural SQL rendering here — the
+      // tree came from DuckDB's bound-expression serializer. Use a generic
+      // placeholder so debug output stays readable.
+      return `${f.columnName} MATCHES (expression)`;
+  }
+}
+
+// ============================================================================
+// PushdownFilters
+// ============================================================================
+
+export class PushdownFilters {
+  constructor(
+    readonly filters: Filter[],
+    readonly version: string,
+  ) {}
+
+  /** Convert filters to a human-readable SQL-like string. */
+  toSql(): string {
+    if (this.filters.length === 0) return "";
+    return this.filters.map(filterToSql).join(" AND ");
+  }
+
+  /** Evaluate filters against a batch, returning a Uint8Array mask (0=fail, 1=pass). */
+  evaluate(batch: VgiBatch): Uint8Array {
+    const n = batch.numRows;
+    const mask = new Uint8Array(n);
+    mask.fill(1);
+
+    for (const filter of this.filters) {
+      evaluateFilter(filter, batch, mask);
+    }
+
+    return mask;
+  }
+
+  /** Apply filters to a batch, returning only passing rows. */
+  apply(batch: VgiBatch): VgiBatch {
+    if (batch.numRows === 0 || this.filters.length === 0) return batch;
+
+    const mask = this.evaluate(batch);
+    // filterBatch rebuilds via the codec/canonical path (backend-agnostic,
+    // lossless), including its own all-pass / all-fail fast paths.
+    return filterBatch(batch, mask);
+  }
+
+  /**
+   * The set of column names referenced by the pushed-down filters, sorted
+   * ascending. Descends one level into AndFilter children (DuckDB pushes
+   * `col IN (...) AND col>=min` as a single AndFilter), consistent with
+   * `getColumnValues` / `hasFilterForColumn`. Mirrors the cross-language
+   * `filtered_columns` accessor (vgi-python / vgi-go / vgi-rust).
+   */
+  filteredColumns(): string[] {
+    const cols = new Set<string>();
+    for (const f of this.filters) {
+      if (f.type === "and") {
+        for (const c of f.children) cols.add(c.columnName);
+      } else {
+        cols.add(f.columnName);
+      }
+    }
+    return Array.from(cols).sort();
+  }
+
+  /**
+   * Whether any pushed-down filter references `columnName`. Mirrors the
+   * cross-language `has_filter_for_column` accessor.
+   */
+  hasFilterForColumn(columnName: string): boolean {
+    return this.collectColumnFilters(columnName).length > 0;
+  }
+
+  /**
+   * Discrete values a column could take, from equality (`=`) or `IN` filters.
+   * Useful for partition pruning (resolve the key set up front, fetch only
+   * those keys). Returns null when the predicate is not enumerable — no
+   * filter, a bare range, or an OR with a non-discrete branch.
+   *
+   * Mirrors vgi-python's `PushdownFilters.get_column_values`: descends one
+   * level into an AndFilter (DuckDB pushes `col IN (...) AND col>=min AND
+   * col<=max` as a single AndFilter from a semi-join), and unions OR branches
+   * only when every branch pins the column to discrete values.
+   */
+  getColumnValues(columnName: string): any[] | null {
+    for (const f of this.collectColumnFilters(columnName)) {
+      if (f.type === "constant" && f.op === ComparisonOp.EQ) {
+        return [f.value];
+      } else if (f.type === "in") {
+        return Array.from(f.values);
+      } else if (f.type === "or") {
+        const union = this.orDiscreteValues(f, columnName);
+        if (union !== null) return union;
+      }
+    }
+    return null;
+  }
+
+  /**
+   * Union of discrete values for `columnName` across all OR branches, iff every
+   * child constrains it to discrete values (`=` or `IN`); otherwise null (the
+   * column could take any value through a non-discrete branch). Returning one
+   * branch's values would be an unsafe subset.
+   */
+  private orDiscreteValues(orFilter: OrFilter, columnName: string): any[] | null {
+    const values: any[] = [];
+    for (const child of orFilter.children) {
+      if (child.columnName !== columnName) return null;
+      if (child.type === "constant" && child.op === ComparisonOp.EQ) {
+        values.push(child.value);
+      } else if (child.type === "in") {
+        for (const v of child.values) values.push(v);
+      } else {
+        return null;
+      }
+    }
+    if (values.length === 0) return null;
+    // Deduplicate, preserving first-seen order for stable output.
+    const seen = new Set<any>();
+    const deduped: any[] = [];
+    for (const v of values) {
+      if (!seen.has(v)) {
+        seen.add(v);
+        deduped.push(v);
+      }
+    }
+    return deduped;
+  }
+
+  /**
+   * Collect filters for a column from the top level and direct AND children
+   * (one level of descent, consistent with the Python accessor). Deeply nested
+   * AND/AND is not traversed.
+   */
+  private collectColumnFilters(columnName: string): Filter[] {
+    const result: Filter[] = [];
+    for (const f of this.filters) {
+      if (f.columnName !== columnName) continue;
+      if (f.type === "and") {
+        for (const c of f.children) {
+          if (c.columnName === columnName) result.push(c);
+        }
+      } else {
+        result.push(f);
+      }
+    }
+    return result;
+  }
+}

package/src/filter-pushdown/index.ts

@@ -0,0 +1,28 @@
+// Copyright 2025, 2026 Query Farm LLC - https://query.farm
+// Filter pushdown — types, deserialization from the wire format, evaluation
+// against Arrow batches, and an OutputCollector wrapper that applies filters
+// transparently. Ported from vgi-python/vgi/table_filter_pushdown.py.
+
+export {
+  ComparisonOp,
+  type AndFilter,
+  type ConstantFilter,
+  type ExpressionFilter,
+  type ExprNode,
+  type Filter,
+  type InFilter,
+  type IsNotNullFilter,
+  type IsNullFilter,
+  type OrFilter,
+  type StructFilter,
+} from "./types.js";
+
+export { PushdownFilters } from "./evaluate.js";
+
+export { buildJoinKeysLookup, deserializeFilters } from "./deserialize.js";
+
+export {
+  FilteringOutputCollector,
+  formatPushedFilters,
+  reprPushedFilters,
+} from "./collector.js";

package/src/filter-pushdown/types.ts

@@ -0,0 +1,91 @@
+// Copyright 2025, 2026 Query Farm LLC - https://query.farm
+// Filter pushdown type definitions.
+
+export enum ComparisonOp {
+  EQ = "eq",
+  NE = "ne",
+  GT = "gt",
+  GE = "ge",
+  LT = "lt",
+  LE = "le",
+}
+
+export interface ConstantFilter {
+  type: "constant";
+  columnName: string;
+  columnIndex: number;
+  op: ComparisonOp;
+  value: any;
+}
+
+export interface IsNullFilter {
+  type: "is_null";
+  columnName: string;
+  columnIndex: number;
+}
+
+export interface IsNotNullFilter {
+  type: "is_not_null";
+  columnName: string;
+  columnIndex: number;
+}
+
+export interface InFilter {
+  type: "in";
+  columnName: string;
+  columnIndex: number;
+  values: Set<any>;
+}
+
+export interface AndFilter {
+  type: "and";
+  columnName: string;
+  columnIndex: number;
+  children: Filter[];
+}
+
+export interface OrFilter {
+  type: "or";
+  columnName: string;
+  columnIndex: number;
+  children: Filter[];
+}
+
+export interface StructFilter {
+  type: "struct";
+  columnName: string;
+  columnIndex: number;
+  childIndex: number;
+  childName: string;
+  childFilter: Filter;
+}
+
+export type Filter =
+  | ConstantFilter
+  | IsNullFilter
+  | IsNotNullFilter
+  | InFilter
+  | AndFilter
+  | OrFilter
+  | StructFilter
+  | ExpressionFilter;
+
+/**
+ * Expression-tree filter. The expr is a parsed AST; evaluation matches
+ * DuckDB's semantics for the function names the table function advertises in
+ * `supportedExpressionFilters` (e.g. list_contains, starts_with, contains,
+ * &&/st_intersects_extent for spatial).
+ */
+export interface ExpressionFilter {
+  type: "expression";
+  columnName: string;
+  columnIndex: number;
+  expr: ExprNode;
+}
+
+export type ExprNode =
+  | { expr_type: "column_ref"; index: number }
+  | { expr_type: "constant"; value: any }
+  | { expr_type: "function"; function_name: string; children: ExprNode[] }
+  | { expr_type: "comparison"; op: ComparisonOp; left: ExprNode; right: ExprNode }
+  | { expr_type: "conjunction"; conjunction_type: "and" | "or"; children: ExprNode[] };