@query-farm/vgi 0.33.0 → 0.35.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@query-farm/vgi",
-  "version": "0.33.0",
+  "version": "0.35.0",
   "description": "TypeScript framework for extending DuckDB with user-defined functions via the Vector Gateway Interface (VGI) over Arrow IPC.",
   "license": "SEE LICENSE IN LICENSE",
   "author": "Query Farm LLC (https://query.farm)",
@@ -91,12 +91,12 @@
   },
   "peerDependencies": {
     "@query-farm/apache-arrow": "^21.1.1",
-    "@query-farm/vgi-rpc": ">=0.25.2 <0.26.0"
+    "@query-farm/vgi-rpc": ">=0.25.4 <0.26.0"
   },
   "devDependencies": {
     "@duckdb/node-api": "^1.5.5-r.4",
     "@query-farm/apache-arrow": "^21.1.1",
-    "@query-farm/vgi-rpc": "^0.25.2",
+    "@query-farm/vgi-rpc": "^0.25.4",
     "@types/bun": "latest",
     "typescript": "^7.0.2"
   },

package/src/arguments/arguments.ts

@@ -53,11 +53,20 @@ export class Arguments {
 
 /**
  * Unwrap an Arrow Scalar to a plain JS value.
+ *
+ * A `Date` is returned as-is. Arguments are decoded to their RICH values
+ * (deserializeArguments runs every value through the codec), and a JS `Date`
+ * is the rich value of date32/date64 — `valueOf()` would turn it into epoch
+ * MILLISECONDS, while a bare number in a date32 column means DAYS. So a DATE
+ * argument read through `get()` (every typed `params.args.x`, every scalar
+ * const) and written back out as a DATE came back 86,400,000x too far out:
+ * '2024-01-15' as 480510-12-09 on arrow-js, and an invalid Date on flechette.
  */
 function unwrapScalar(val: any): any {
   if (val === null || val === undefined) return null;
   // If it's already a primitive, return it
   if (typeof val !== "object") return val;
+  if (val instanceof Date) return val;
   // Arrow Scalar objects have a valueOf() method
   if (typeof val.valueOf === "function") {
     let v: any;

package/src/arrow/impl-arrowjs/canonical.ts

@@ -27,6 +27,7 @@
 import {
   Struct,
   List,
+  FixedSizeList,
   DataType,
   DateUnit,
   makeData,
@@ -54,6 +55,7 @@ export function writeCanonicalColumn(
 
 function buildColumnData(values: unknown[], type: DataType): any {
   if (DataType.isList(type)) return buildListData(values, type as List);
+  if (DataType.isFixedSizeList(type)) return buildFixedSizeListData(values, type as FixedSizeList);
   if (DataType.isMap(type)) return buildMapData(values, type as Map_);
   if (DataType.isStruct(type)) return buildStructData(values, type as Struct);
   if (DataType.isDecimal(type)) return buildDecimalData(values, type);
@@ -200,6 +202,48 @@ function buildListData(values: unknown[], listType: List): any {
   });
 }
 
+/**
+ * FixedSizeList (DuckDB's ARRAY) is built by hand, like List, so its items go
+ * through buildColumnData. Handing it to vectorFromArray fed arrow-js's own
+ * child builders CANONICAL values they read in other units: a date32
+ * day-number as epoch-ms (DATE[2], STRUCT(d DATE)[1], MAP(DATE, DATE)[1] and
+ * DATE[][1] all came back as 1970-01-01), and a bigint timestamp or unscaled
+ * decimal threw.
+ */
+function buildFixedSizeListData(values: unknown[], type: FixedSizeList): any {
+  const childField = type.children[0];
+  const listSize = type.listSize;
+  const numRows = values.length;
+  const allItems: unknown[] = [];
+  const nullBitmap = new Uint8Array(Math.ceil(numRows / 8));
+  let nullCount = 0;
+  for (let i = 0; i < numRows; i++) {
+    const row = values[i];
+    if (row == null) {
+      // A null row still occupies listSize child slots.
+      nullCount++;
+      for (let j = 0; j < listSize; j++) allItems.push(null);
+      continue;
+    }
+    nullBitmap[i >> 3] |= 1 << (i & 7);
+    const items = Array.isArray(row) ? row : Array.from(row as Iterable<unknown>);
+    if (items.length !== listSize) {
+      throw new TypeError(
+        `fixed_size_list[${listSize}]: row ${i} has ${items.length} items`,
+      );
+    }
+    for (const item of items) allItems.push(item);
+  }
+  const childData = allItems.length > 0
+    ? buildColumnData(allItems, childField.type)
+    : emptyDataForType(childField.type);
+  return makeData({
+    type, length: numRows, nullCount,
+    nullBitmap: nullCount > 0 ? nullBitmap : undefined,
+    child: childData,
+  });
+}
+
 function buildMapData(values: unknown[], mapType: Map_): any {
   const numRows = values.length;
   const entriesField = mapType.children[0];
@@ -285,6 +329,12 @@ function emptyDataForType(type: DataType): any {
       valueOffsets: new Int32Array([0]),
     });
   }
+  if (DataType.isFixedSizeList(type)) {
+    return makeData({
+      type, length: 0, nullCount: 0,
+      child: emptyDataForType((type as FixedSizeList).children[0].type),
+    });
+  }
   if (DataType.isStruct(type)) {
     const children = (type as any).children.map((cf: Field) => emptyDataForType(cf.type));
     return makeData({ type, length: 0, nullCount: 0, children });

package/src/arrow/impl-flechette/canonical.ts

@@ -118,12 +118,29 @@ export function prepareForFlechette(type: VgiDataType, values: unknown[]): unkno
     });
   }
   if (tid === 17) {
+    // The keys and values need the same preparation as a column of their
+    // type: a date32 key/value is a canonical day-number, which flechette's
+    // builder reads as epoch-ms, so MAP {DATE: DATE} came back as
+    // {1970-01-01=1970-01-01} unless it is turned into a JS Date first.
+    const entries = (type as any).children?.[0]?.type?.children;
+    const keyType = entries?.[0]?.type as VgiDataType | undefined;
+    const valueType = entries?.[1]?.type as VgiDataType | undefined;
+    const prepKey = keyType !== undefined && needsPrepare(keyType);
+    const prepValue = valueType !== undefined && needsPrepare(valueType);
     return values.map((v) => {
       if (v == null) return null;
-      if (v instanceof Map) return v;
-      if (Array.isArray(v)) return new Map(v as Array<[unknown, unknown]>);
-      if (typeof v === "object") return new Map(Object.entries(v as any));
-      return v;
+      let pairs: Array<[unknown, unknown]>;
+      if (v instanceof Map) pairs = Array.from(v.entries());
+      else if (Array.isArray(v)) pairs = v as Array<[unknown, unknown]>;
+      else if (typeof v === "object") pairs = Object.entries(v as any);
+      else return v;
+      if (prepKey || prepValue) {
+        pairs = pairs.map(([k, x]) => [
+          prepKey ? prepareForFlechette(keyType!, [k])[0] : k,
+          prepValue ? prepareForFlechette(valueType!, [x])[0] : x,
+        ]);
+      }
+      return new Map(pairs);
     });
   }
   if (tid === 13) {

package/src/client/client.ts

@@ -3,7 +3,11 @@
 // Works with any RpcClient (subprocess or HTTP transport).
 
 import { type VgiSchema, schema as schema_, type VgiField, field, type VgiBatch, type VgiDataType, utf8, binary, list } from "../arrow/index.js";
-import { type RpcClient, type StreamSession } from "@query-farm/vgi-rpc";
+// `import type`, not `import { type … }`: the latter keeps the statement and
+// emits `import {} from "@query-farm/vgi-rpc"`, a side-effect import of the
+// package root — which re-exports the whole framework, server included, into
+// every browser bundle. See the note in ./errors.
+import type { RpcClient, StreamSession } from "@query-farm/vgi-rpc";
 import {
   serializeBindRequest,
   deserializeBindResponse,

package/src/client/errors.ts

@@ -2,7 +2,16 @@
 // VgiClient error class + RpcClient wrapper that translates RpcError into
 // VgiClientError, preserving the worker's remote traceback.
 
-import { RpcError, type RpcClient } from "@query-farm/vgi-rpc";
+// RpcError comes from the `/connect` subpath, not the package root, and the
+// distinction is worth ~160 kB in every browser bundle. The root re-exports the
+// whole framework — protocol, dispatch, access log, the server — and a bundler
+// cannot drop it, so importing one error class from there shipped `RpcServer`
+// to browsers to satisfy an `instanceof`. `RpcClient` is a pure `import type`
+// for the same reason: `import { type X }` leaves the statement behind, and a
+// bare `import {} from "@query-farm/vgi-rpc"` is a side-effect import of that
+// same graph.
+import { RpcError } from "@query-farm/vgi-rpc/connect";
+import type { RpcClient } from "@query-farm/vgi-rpc";
 
 /** Error thrown by VgiClient when an RPC call fails or returns unexpected data. */
 export class VgiClientError extends Error {