@query-farm/vgi 0.32.0 → 0.34.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@query-farm/vgi",
-  "version": "0.32.0",
+  "version": "0.34.0",
   "description": "TypeScript framework for extending DuckDB with user-defined functions via the Vector Gateway Interface (VGI) over Arrow IPC.",
   "license": "SEE LICENSE IN LICENSE",
   "author": "Query Farm LLC (https://query.farm)",
@@ -91,12 +91,12 @@
   },
   "peerDependencies": {
     "@query-farm/apache-arrow": "^21.1.1",
-    "@query-farm/vgi-rpc": ">=0.24.0 <0.25.0"
+    "@query-farm/vgi-rpc": ">=0.25.2 <0.26.0"
   },
   "devDependencies": {
     "@duckdb/node-api": "^1.5.5-r.4",
     "@query-farm/apache-arrow": "^21.1.1",
-    "@query-farm/vgi-rpc": "^0.24.0",
+    "@query-farm/vgi-rpc": "^0.25.2",
     "@types/bun": "latest",
     "typescript": "^7.0.2"
   },

package/src/arguments/arguments.ts

@@ -53,11 +53,20 @@ export class Arguments {
 
 /**
  * Unwrap an Arrow Scalar to a plain JS value.
+ *
+ * A `Date` is returned as-is. Arguments are decoded to their RICH values
+ * (deserializeArguments runs every value through the codec), and a JS `Date`
+ * is the rich value of date32/date64 — `valueOf()` would turn it into epoch
+ * MILLISECONDS, while a bare number in a date32 column means DAYS. So a DATE
+ * argument read through `get()` (every typed `params.args.x`, every scalar
+ * const) and written back out as a DATE came back 86,400,000x too far out:
+ * '2024-01-15' as 480510-12-09 on arrow-js, and an invalid Date on flechette.
  */
 function unwrapScalar(val: any): any {
   if (val === null || val === undefined) return null;
   // If it's already a primitive, return it
   if (typeof val !== "object") return val;
+  if (val instanceof Date) return val;
   // Arrow Scalar objects have a valueOf() method
   if (typeof val.valueOf === "function") {
     let v: any;

package/src/arrow/impl-arrowjs/canonical.ts

@@ -27,6 +27,7 @@
 import {
   Struct,
   List,
+  FixedSizeList,
   DataType,
   DateUnit,
   makeData,
@@ -54,6 +55,7 @@ export function writeCanonicalColumn(
 
 function buildColumnData(values: unknown[], type: DataType): any {
   if (DataType.isList(type)) return buildListData(values, type as List);
+  if (DataType.isFixedSizeList(type)) return buildFixedSizeListData(values, type as FixedSizeList);
   if (DataType.isMap(type)) return buildMapData(values, type as Map_);
   if (DataType.isStruct(type)) return buildStructData(values, type as Struct);
   if (DataType.isDecimal(type)) return buildDecimalData(values, type);
@@ -200,6 +202,48 @@ function buildListData(values: unknown[], listType: List): any {
   });
 }
 
+/**
+ * FixedSizeList (DuckDB's ARRAY) is built by hand, like List, so its items go
+ * through buildColumnData. Handing it to vectorFromArray fed arrow-js's own
+ * child builders CANONICAL values they read in other units: a date32
+ * day-number as epoch-ms (DATE[2], STRUCT(d DATE)[1], MAP(DATE, DATE)[1] and
+ * DATE[][1] all came back as 1970-01-01), and a bigint timestamp or unscaled
+ * decimal threw.
+ */
+function buildFixedSizeListData(values: unknown[], type: FixedSizeList): any {
+  const childField = type.children[0];
+  const listSize = type.listSize;
+  const numRows = values.length;
+  const allItems: unknown[] = [];
+  const nullBitmap = new Uint8Array(Math.ceil(numRows / 8));
+  let nullCount = 0;
+  for (let i = 0; i < numRows; i++) {
+    const row = values[i];
+    if (row == null) {
+      // A null row still occupies listSize child slots.
+      nullCount++;
+      for (let j = 0; j < listSize; j++) allItems.push(null);
+      continue;
+    }
+    nullBitmap[i >> 3] |= 1 << (i & 7);
+    const items = Array.isArray(row) ? row : Array.from(row as Iterable<unknown>);
+    if (items.length !== listSize) {
+      throw new TypeError(
+        `fixed_size_list[${listSize}]: row ${i} has ${items.length} items`,
+      );
+    }
+    for (const item of items) allItems.push(item);
+  }
+  const childData = allItems.length > 0
+    ? buildColumnData(allItems, childField.type)
+    : emptyDataForType(childField.type);
+  return makeData({
+    type, length: numRows, nullCount,
+    nullBitmap: nullCount > 0 ? nullBitmap : undefined,
+    child: childData,
+  });
+}
+
 function buildMapData(values: unknown[], mapType: Map_): any {
   const numRows = values.length;
   const entriesField = mapType.children[0];
@@ -285,6 +329,12 @@ function emptyDataForType(type: DataType): any {
       valueOffsets: new Int32Array([0]),
     });
   }
+  if (DataType.isFixedSizeList(type)) {
+    return makeData({
+      type, length: 0, nullCount: 0,
+      child: emptyDataForType((type as FixedSizeList).children[0].type),
+    });
+  }
   if (DataType.isStruct(type)) {
     const children = (type as any).children.map((cf: Field) => emptyDataForType(cf.type));
     return makeData({ type, length: 0, nullCount: 0, children });

package/src/arrow/impl-flechette/canonical.ts

@@ -118,12 +118,29 @@ export function prepareForFlechette(type: VgiDataType, values: unknown[]): unkno
     });
   }
   if (tid === 17) {
+    // The keys and values need the same preparation as a column of their
+    // type: a date32 key/value is a canonical day-number, which flechette's
+    // builder reads as epoch-ms, so MAP {DATE: DATE} came back as
+    // {1970-01-01=1970-01-01} unless it is turned into a JS Date first.
+    const entries = (type as any).children?.[0]?.type?.children;
+    const keyType = entries?.[0]?.type as VgiDataType | undefined;
+    const valueType = entries?.[1]?.type as VgiDataType | undefined;
+    const prepKey = keyType !== undefined && needsPrepare(keyType);
+    const prepValue = valueType !== undefined && needsPrepare(valueType);
     return values.map((v) => {
       if (v == null) return null;
-      if (v instanceof Map) return v;
-      if (Array.isArray(v)) return new Map(v as Array<[unknown, unknown]>);
-      if (typeof v === "object") return new Map(Object.entries(v as any));
-      return v;
+      let pairs: Array<[unknown, unknown]>;
+      if (v instanceof Map) pairs = Array.from(v.entries());
+      else if (Array.isArray(v)) pairs = v as Array<[unknown, unknown]>;
+      else if (typeof v === "object") pairs = Object.entries(v as any);
+      else return v;
+      if (prepKey || prepValue) {
+        pairs = pairs.map(([k, x]) => [
+          prepKey ? prepareForFlechette(keyType!, [k])[0] : k,
+          prepValue ? prepareForFlechette(valueType!, [x])[0] : x,
+        ]);
+      }
+      return new Map(pairs);
     });
   }
   if (tid === 13) {

package/src/arrow/impl-flechette/metadata.ts

@@ -16,7 +16,18 @@ import type { VgiBatch } from "../types.js";
  */
 export function withBatchMetadata(batch: VgiBatch, metadata: Map<string, string>): VgiBatch {
   const t = batch as any;
-  const clone = Object.assign(Object.create(Object.getPrototypeOf(t)), t);
+  // Copy the own properties as descriptors, not by assignment. vgi-rpc's reader
+  // pins a zero-column batch's row count as an own `numRows` over
+  // `Table.prototype.numRows`, a getter-only accessor, and `Object.assign`
+  // *assigns* each own property -- which throws in strict mode ("Attempted to
+  // assign to readonly property") over an inherited getter-only accessor.
+  const clone = Object.create(Object.getPrototypeOf(t), Object.getOwnPropertyDescriptors(t));
+  // This backend's own reader supplies that row count through a Proxy instead
+  // (see `deserializeBatch`), which no property copy carries: without this the
+  // clone would derive 0 rows from its absent columns.
+  if (clone.numRows !== t.numRows) {
+    Object.defineProperty(clone, "numRows", { value: t.numRows, configurable: true, enumerable: true });
+  }
   if (metadata && metadata.size > 0) {
     clone._vgiRecordMetadata = metadata;
     clone.metadata = metadata;

package/src/arrow/impl-flechette/normalize-type.ts

@@ -18,7 +18,7 @@
 import {
   Type,
   field as f_field,
-  nullType, bool, int, float, utf8, binary, fixedSizeBinary,
+  nullType, bool, int, float, utf8, largeUtf8, binary, largeBinary, fixedSizeBinary,
   decimal, date, time, timestamp, duration, interval,
   list, largeList, fixedSizeList, struct, map, dictionary, union,
 } from "@query-farm/flechette";
@@ -47,12 +47,19 @@ export function toFlechetteType(type: any): any {
       return aliasIntSigned(int(type.bitWidth, type.isSigned ?? type.signed ?? true));
     case Type.Float:
       return float(type.precision);
+    // Large variants stay large: flechette builds and encodes them, and
+    // folding them into Binary/Utf8 put a different type on the wire than the
+    // one declared (arrow-js writes LargeBinary/LargeUtf8). InitRequest
+    // declares `pushdown_filters` large_binary, so a flechette worker or
+    // client wrote that field as binary.
     case Type.Binary:
-    case Type.LargeBinary:
       return binary();
+    case Type.LargeBinary:
+      return largeBinary();
     case Type.Utf8:
-    case Type.LargeUtf8:
       return utf8();
+    case Type.LargeUtf8:
+      return largeUtf8();
     case Type.FixedSizeBinary:
       return fixedSizeBinary(type.byteWidth ?? type.stride);
     case Type.Decimal: