@query-farm/vgi-rpc 0.7.2 → 0.7.4

package/dist/util/conform.d.ts

@@ -0,0 +1,18 @@
+import type { VgiBatch, VgiSchema } from "../arrow/index.js";
+/**
+ * Rebuild a batch's data to match the given schema's field types.
+ *
+ * Batches deserialized from IPC streams (e.g., from PyArrow) may use generic
+ * types (Float) instead of specific ones (Float64).  Arrow-JS's
+ * RecordBatchStreamWriter silently drops batches whose child Data types don't
+ * match the writer's schema.  Cloning each child Data with the schema's field
+ * type fixes the type metadata while preserving the underlying buffers.
+ *
+ * This is also used to cast compatible input types (e.g., int32→float64,
+ * float32→float64) when the input batch schema doesn't exactly match the
+ * method's declared input schema.  When the underlying buffer layout differs
+ * (e.g., 4-byte int32 vs 8-byte float64), we read the values and build a
+ * new vector with the target type.
+ */
+export declare function conformBatchToSchema(batch: VgiBatch, schema: VgiSchema): VgiBatch;
+//# sourceMappingURL=conform.d.ts.map

package/dist/util/conform.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"conform.d.ts","sourceRoot":"","sources":["../../src/util/conform.ts"],"names":[],"mappings":"AAYA,OAAO,KAAK,EAAE,QAAQ,EAAE,SAAS,EAAE,MAAM,mBAAmB,CAAC;AAkB7D;;;;;;;;;;;;;;GAcG;AACH,wBAAgB,oBAAoB,CAAC,KAAK,EAAE,QAAQ,EAAE,MAAM,EAAE,SAAS,GAAG,QAAQ,CAuDjF"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@query-farm/vgi-rpc",
-  "version": "0.7.2",
+  "version": "0.7.4",
   "license": "Apache-2.0",
   "homepage": "https://vgi-rpc-typescript.query.farm",
   "repository": {
@@ -50,7 +50,7 @@
   "dependencies": {
     "@noble/ciphers": "^2.2.0",
     "@query-farm/apache-arrow": "^21.1.1",
-    "@uwdata/flechette": "github:Query-farm/flechette#fix/timestamp-bigint-encode",
+    "@query-farm/flechette": "^2.4.0",
     "fzstd": "^0.1.1",
     "oauth4webapi": "^3.8.6"
   },
@@ -90,7 +90,7 @@
     "build": "bun run build:types && bun run build:js",
     "build:types": "bunx tsc -p tsconfig.build.json",
     "build:js": "bun build ./src/index.ts --outdir dist --target node --format esm --sourcemap=external --external @query-farm/apache-arrow",
-    "postinstall": "cd node_modules/@query-farm/apache-arrow && node -e \"const fs=require('fs');const p=JSON.parse(fs.readFileSync('package.json','utf8'));p.main='src/Arrow.node.ts';fs.writeFileSync('package.json',JSON.stringify(p,null,2)+'\\n')\" && cd ../../.. && bun scripts/patch-flechette.mjs",
+    "postinstall": "cd node_modules/@query-farm/apache-arrow && node -e \"const fs=require('fs');const p=JSON.parse(fs.readFileSync('package.json','utf8'));p.main='src/Arrow.node.ts';fs.writeFileSync('package.json',JSON.stringify(p,null,2)+'\\n')\"",
     "test": "bun test",
     "lint": "biome check .",
     "lint:fix": "biome check --write .",

package/src/arrow/impl-flechette/index.ts

@@ -36,8 +36,7 @@ import {
   tableFromIPC,
   tablesToIPC,
   tableToIPC,
-} from "@uwdata/flechette";
-
+} from "@query-farm/flechette";
 import type {
   IncrementalEncoder,
   VgiBackendInfo,
@@ -48,6 +47,7 @@ import type {
   VgiSchema,
 } from "../types.js";
 import { readFirstRecordBatchMeta } from "./message-meta.js";
+import { toFlechetteType } from "./normalize-type.js";
 
 export const backend: VgiBackendInfo = { name: "flechette", opaquePassthrough: false };
 
@@ -129,7 +129,7 @@ export function serializeSchema(s: VgiSchema): Uint8Array {
   // empty schemas, which then leaked into state-token-embedded schema
   // bytes and made `exchange_zero_columns` etc. see a 1-column schema
   // on the next round-trip.
-  const cols = s.fields.map((f) => f_columnFromArray([], f.type as any));
+  const cols = s.fields.map((f) => f_columnFromArray([], toFlechetteType(f.type) as any));
   const table = buildTablePreservingNullable(s, cols) as any;
   return tableToIPC(table, { format: "stream" }) as Uint8Array;
 }
@@ -169,7 +169,7 @@ export function deserializeBatch(bytes: Uint8Array): VgiBatch {
 // ----- Construction --------------------------------------------------------
 
 export function columnFromArray(values: any[], type: VgiDataType): VgiColumnData {
-  return f_columnFromArray(values, type as any, EXTRACT_OPTS) as VgiColumnData;
+  return f_columnFromArray(values, toFlechetteType(type) as any, EXTRACT_OPTS) as VgiColumnData;
 }
 
 // flechette's `tableFromColumns` discards per-field `nullable`/`metadata` —
@@ -213,7 +213,7 @@ export function singleRowBatch(s: VgiSchema, values: Record<string, any>): VgiBa
     if (f.type.typeId === 2 /* Int */ && (f.type as any).bitWidth === 64 && typeof val === "number") {
       val = BigInt(val);
     }
-    cols.push(f_columnFromArray(coerceValuesForType([val], f.type), f.type as any, EXTRACT_OPTS));
+    cols.push(f_columnFromArray(coerceValuesForType([val], f.type), toFlechetteType(f.type) as any, EXTRACT_OPTS));
   }
   return buildTablePreservingNullable(s, cols);
 }
@@ -223,7 +223,7 @@ export function batchFromColumns(s: VgiSchema, columns: Record<string, any[]>):
   const cols: Column<any>[] = [];
   for (const f of s.fields) {
     const vals = columns[f.name] ?? new Array(numRows).fill(null);
-    cols.push(f_columnFromArray(coerceValuesForType(vals, f.type), f.type as any, EXTRACT_OPTS));
+    cols.push(f_columnFromArray(coerceValuesForType(vals, f.type), toFlechetteType(f.type) as any, EXTRACT_OPTS));
   }
   return buildTablePreservingNullable(s, cols);
 }
@@ -240,7 +240,7 @@ export function batchFromColumnData(
 }
 
 export function emptyColumnData(type: VgiDataType): VgiColumnData {
-  return f_columnFromArray([], type as any, EXTRACT_OPTS) as VgiColumnData;
+  return f_columnFromArray([], toFlechetteType(type) as any, EXTRACT_OPTS) as VgiColumnData;
 }
 
 /**
@@ -255,7 +255,7 @@ export function emptyColumnData(type: VgiDataType): VgiColumnData {
 export function emptyBatchWithMetadata(s: VgiSchema, metadata?: Map<string, string>): VgiBatch {
   const cols: Record<string, Column<any>> = {};
   for (const f of s.fields) {
-    cols[f.name] = f_columnFromArray([], f.type as any, EXTRACT_OPTS);
+    cols[f.name] = f_columnFromArray([], toFlechetteType(f.type) as any, EXTRACT_OPTS);
   }
   const t = tableFromColumns(cols) as any;
   attachBatchMetadata(t, metadata);
@@ -274,7 +274,7 @@ export function singleRowBatchWithMetadata(
     if (f.type.typeId === 2 /* Int */ && (f.type as any).bitWidth === 64 && typeof val === "number") {
       val = BigInt(val);
     }
-    cols[f.name] = f_columnFromArray([val], f.type as any, EXTRACT_OPTS);
+    cols[f.name] = f_columnFromArray([val], toFlechetteType(f.type) as any, EXTRACT_OPTS);
   }
   const t = tableFromColumns(cols) as any;
   attachBatchMetadata(t, metadata);
@@ -386,7 +386,7 @@ export function conformBatchToSchema(batch: VgiBatch, schema: VgiSchema): VgiBat
     if (srcType.typeId === dstType.typeId) return srcCol;
     if (isNumericTypeId(srcType.typeId) && isNumericTypeId(dstType.typeId)) {
       mutated = true;
-      return f_columnFromArray(castNumericValues(srcCol, dstType), dstType as any, EXTRACT_OPTS);
+      return f_columnFromArray(castNumericValues(srcCol, dstType), toFlechetteType(dstType) as any, EXTRACT_OPTS);
     }
     return srcCol;
   });

package/src/arrow/impl-flechette/normalize-type.ts

@@ -0,0 +1,116 @@
+// Copyright 2025, 2026 Query Farm LLC - https://query.farm
+//
+// Normalize a (possibly foreign) Arrow DataType into a flechette-native type.
+//
+// App/worker code routinely builds schemas with concrete arrow-js DataType
+// instances (e.g. `new Int64()` from @query-farm/apache-arrow) rather than the
+// facade constructors. flechette's `columnFromArray` *tolerates* such foreign
+// type objects when reading values, but the resulting `Column.type` is the
+// foreign object — and flechette's IPC writer then serializes it incorrectly
+// (e.g. Utf8 round-trips to ["", "xy\0\0…"], Int64 throws BigInt→number on a
+// stale build). Reconstructing the type from its structural fields via
+// flechette's own constructors yields a writer-safe native type.
+//
+// Both arrow-js and flechette encode `typeId` with the same Arrow Type enum, so
+// we switch on it and read the shared structural props (with the one rename
+// flechette differs on: Int uses `signed`, arrow-js uses `isSigned`).
+
+import {
+  binary,
+  bool,
+  date,
+  decimal,
+  dictionary,
+  duration,
+  field as f_field,
+  fixedSizeBinary,
+  fixedSizeList,
+  float,
+  int,
+  interval,
+  largeList,
+  list,
+  map,
+  nullType,
+  struct,
+  Type,
+  time,
+  timestamp,
+  union,
+  utf8,
+} from "@query-farm/flechette";
+
+function fField(f: any): any {
+  return f_field(f.name, toFlechetteType(f.type), f.nullable ?? true, f.metadata ?? null);
+}
+
+/**
+ * Rebuild `type` as a flechette-native DataType. Idempotent for types that are
+ * already native (reconstructed from the same structural props). Unknown
+ * typeIds pass through unchanged as a safety net.
+ */
+export function toFlechetteType(type: any): any {
+  if (type == null) return nullType();
+  switch (type.typeId) {
+    case Type.Null:
+      return nullType();
+    case Type.Bool:
+      return bool();
+    case Type.Int:
+      return int(type.bitWidth, type.isSigned ?? type.signed ?? true);
+    case Type.Float:
+      return float(type.precision);
+    case Type.Binary:
+    case Type.LargeBinary:
+      return binary();
+    case Type.Utf8:
+    case Type.LargeUtf8:
+      return utf8();
+    case Type.FixedSizeBinary:
+      return fixedSizeBinary(type.byteWidth ?? type.stride);
+    case Type.Decimal:
+      return decimal(type.precision, type.scale, type.bitWidth ?? 128);
+    case Type.Date:
+      return date(type.unit);
+    case Type.Time:
+      return time(type.unit);
+    case Type.Timestamp:
+      return timestamp(type.unit, type.timezone ?? null);
+    case Type.Duration:
+      return duration(type.unit);
+    case Type.Interval:
+      return interval(type.unit);
+    case Type.List:
+    case Type.LargeList:
+      return (type.typeId === Type.LargeList ? largeList : list)(fField(type.children[0]));
+    case Type.FixedSizeList:
+      return fixedSizeList(fField(type.children[0]), type.listSize);
+    case Type.Struct:
+      return struct(type.children.map(fField));
+    case Type.Map: {
+      // children[0] is the "entries" struct field holding [key, value].
+      const entries = type.children[0];
+      const [k, v] = entries.type.children;
+      return map(fField(k), fField(v), type.keysSorted ?? false);
+    }
+    case Type.Dictionary:
+      return dictionary(
+        toFlechetteType(type.dictionary),
+        toFlechetteType(type.indices),
+        type.ordered ?? type.isOrdered ?? false,
+        type.id,
+      );
+    case Type.Union:
+      return union(type.mode, type.children.map(fField), type.typeIds);
+    default:
+      return type;
+  }
+}
+
+/**
+ * Normalize every field's type in a schema-like object to flechette-native,
+ * preserving field name / nullable / metadata.
+ */
+export function normalizeSchemaFields(fields: readonly any[]): any[] {
+  return fields.map(fField);
+}

package/src/dispatch/describe.ts

@@ -156,10 +156,12 @@ export async function buildDescribeBatch(
     const headerIpc = method.headerSchema ? serializeSchema(method.headerSchema) : null;
     headerSchemas.push(headerIpc);
 
-    let isExchange: boolean | null;
-    if (method.exchangeFn) isExchange = true;
-    else if (method.producerFn) isExchange = false;
-    else isExchange = null;
+    // v4 slim schema: `is_exchange` is vestigial and always null — producer
+    // vs exchange is already derivable from `method_type` + the presence of an
+    // input/params schema. The cross-language describe spec asserts null for
+    // every stream method (and the Python reference emits null for all), so we
+    // never populate it. See vgi-rpc conformance describe_stream_properties.
+    const isExchange: boolean | null = null;
     isExchanges.push(isExchange);
 
     hashRows.push({