@query-farm/vgi 0.17.0 → 0.19.0

package/dist/worker.d.ts

@@ -32,5 +32,5 @@ export declare class Worker {
      * @param writable outgoing response sink (a `net.Socket`/`Duplex`, or an fd
      *   number). Omit for the stdout fd.
      */
-    serveStream(readable: ReadableStream<Uint8Array> | NodeJS.ReadableStream, writable?: number | import("node:net").Socket): Promise<void>;
+    serveStream(readable: ReadableStream<Uint8Array> | NodeJS.ReadableStream, writable?: number | import("node:net").Socket | import("@query-farm/vgi-rpc").ByteSink): Promise<void>;
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@query-farm/vgi",
-  "version": "0.17.0",
+  "version": "0.19.0",
   "description": "TypeScript framework for extending DuckDB with user-defined functions via the Vector Gateway Interface (VGI) over Arrow IPC.",
   "license": "SEE LICENSE IN LICENSE",
   "author": "Query Farm LLC (https://query.farm)",
@@ -86,12 +86,12 @@
   },
   "peerDependencies": {
     "@query-farm/apache-arrow": "^21.1.1",
-    "@query-farm/vgi-rpc": "^0.13.0"
+    "@query-farm/vgi-rpc": "^0.14.0"
   },
   "devDependencies": {
     "@duckdb/node-api": "^1.5.2-r.1",
     "@query-farm/apache-arrow": "^21.1.1",
-    "@query-farm/vgi-rpc": "^0.13.0",
+    "@query-farm/vgi-rpc": "^0.14.0",
     "@types/bun": "latest",
     "typescript": "^6.0.3"
   },

package/src/arrow/impl-arrowjs/index.ts

@@ -41,6 +41,7 @@ export {
 export { filterBatch } from "./filter.js";
 export { projectSchema, projectBatch } from "./project.js";
 export { readCanonicalValue } from "./canonical.js";
+export { withBatchMetadata } from "./metadata.js";
 
 // IPC
 export {

package/src/arrow/impl-arrowjs/metadata.ts

@@ -0,0 +1,20 @@
+// Copyright 2025, 2026 Query Farm LLC - https://query.farm
+// Per-record-batch metadata attachment (arrow-js backend).
+
+import { RecordBatch } from "@query-farm/apache-arrow";
+import type { VgiBatch } from "../types.js";
+
+/**
+ * Re-emit a batch with a different per-record-batch metadata map (same schema
+ * + data, no copy). The @query-farm/apache-arrow fork's RecordBatch carries
+ * the map through IPC as the Message's `custom_metadata` field.
+ *
+ * Mirrors vgi-rpc's arrow facade helper of the same name — needed here so a
+ * worker can bake metadata into the batch object itself (vgi-rpc's HTTP
+ * exchange dispatch drops the separate emit-metadata argument for 0-row
+ * batches; a batch-carried map survives).
+ */
+export function withBatchMetadata(batch: VgiBatch, metadata: Map<string, string>): VgiBatch {
+  const a = batch as unknown as RecordBatch;
+  return new RecordBatch(a.schema, a.data, metadata) as unknown as VgiBatch;
+}

package/src/arrow/impl-flechette/index.ts

@@ -41,6 +41,7 @@ export {
 export { filterBatch } from "./filter.js";
 export { projectSchema, projectBatch } from "./project.js";
 export { readCanonicalValue } from "./canonical.js";
+export { withBatchMetadata } from "./metadata.js";
 
 // IPC
 export {

package/src/arrow/impl-flechette/metadata.ts

@@ -0,0 +1,25 @@
+// Copyright 2025, 2026 Query Farm LLC - https://query.farm
+// Per-record-batch metadata attachment (flechette backend).
+
+import type { VgiBatch } from "../types.js";
+
+/**
+ * Re-emit a batch with a different per-record-batch metadata map (same schema
+ * + data). Shallow-clones the flechette Table so the caller's reference is
+ * not mutated, then pins the map so that:
+ *  - `batch.metadata` surfaces it to consumer code (matching arrow-js's
+ *    RecordBatch.metadata getter behavior), and
+ *  - the patched flechette encoder picks it up via `_vgiRecordMetadata` and
+ *    emits it as the IPC Message's `custom_metadata` field.
+ *
+ * Mirrors vgi-rpc's arrow facade helper of the same name.
+ */
+export function withBatchMetadata(batch: VgiBatch, metadata: Map<string, string>): VgiBatch {
+  const t = batch as any;
+  const clone = Object.assign(Object.create(Object.getPrototypeOf(t)), t);
+  if (metadata && metadata.size > 0) {
+    clone._vgiRecordMetadata = metadata;
+    clone.metadata = metadata;
+  }
+  return clone as unknown as VgiBatch;
+}

package/src/arrow/index.ts

@@ -53,6 +53,8 @@ export {
   emptyBatch, batchFromRows, batchFromColumns, columnFromArray,
   iterRows, batchToScalarDict, batchToSecretDict, safeNumber, decodeDictValue,
   filterBatch, projectSchema, projectBatch,
+  // per-record-batch metadata attachment
+  withBatchMetadata,
   // canonical single-value read (backend-agnostic, lossless)
   readCanonicalValue,
   // IPC

package/src/catalog/descriptors.ts

@@ -60,18 +60,24 @@ export interface TableDescriptor {
    */
   statisticsCacheMaxAgeSeconds?: number | null;
   /**
-   * Dotted-path column references that MUST appear in a WHERE expression for
-   * any scan of this table. Top-level names (`"country"`) or struct subfields
-   * (`"bbox.xmin"`, `"nested.outer.inner"`). Empty/undefined (default) means
-   * no enforcement — the zero-cost fast path for every existing table.
+   * Required WHERE-filter groups in conjunctive normal form — an AND (outer
+   * list) of OR-groups (inner lists) of dotted-path column references that MUST
+   * appear in a WHERE expression for any scan of this table. A group is
+   * satisfied when any one of its paths has a filter; every group must be
+   * satisfied. So `[["accession_number"], ["ticker", "cik"]]` means
+   * "accession_number AND one of (ticker, cik)"; a single-path group
+   * `[["country"]]` is a plain mandatory filter. Paths are top-level names
+   * (`"country"`) or struct subfields (`"bbox.xmin"`, `"nested.outer.inner"`).
+   * Empty/undefined (default) means no enforcement — the zero-cost fast path
+   * for every existing table.
    *
    * Satisfaction is prefix-based: a present filter on a shorter path satisfies
    * any required path it is a prefix of. So a whole-struct filter on `bbox`
    * satisfies all of `bbox.xmin` / `.xmax` / `.ymin` / `.ymax`. The VGI DuckDB
-   * extension's optimizer pass consults this list at bind time and throws a
-   * `BinderException` listing any unsatisfied paths.
+   * extension's optimizer pass consults this at bind time and throws a
+   * `BinderException` listing any unsatisfied groups.
    */
-  requiredFieldFilterPaths?: string[];
+  requiredFilters?: string[][];
   supportsTimeTravel?: boolean;
   /**
    * Inline cardinality estimate/max surfaced through TableInfo. When set,

package/src/catalog/read-only.ts

@@ -235,26 +235,34 @@ export class ReadOnlyCatalogInterface extends CatalogInterface {
       const pkIndices = resolveIndexGroups(colSchema, t.primaryKey);
       const fkBytes = serializeForeignKeys(t.foreignKey, name);
 
-      // Validate required_field_filter_paths: the leading dotted segment of
-      // each path must be a real column on this table. Struct subfield
+      // Validate required_filters: an AND (outer list) of OR-groups (inner
+      // lists). Each OR-group must be non-empty, and the leading dotted segment
+      // of each path must be a real column on this table. Struct subfield
       // validity is not checked here — DuckDB's binder catches typos at scan
       // time, and the descriptor doesn't unpack STRUCT subfields. (Mirrors
       // vgi-python's Table descriptor validation.)
-      const requiredFieldFilterPaths = t.requiredFieldFilterPaths ?? [];
-      if (requiredFieldFilterPaths.length > 0) {
+      const requiredFilters = t.requiredFilters ?? [];
+      if (requiredFilters.length > 0) {
         const columnNames = new Set(colSchema.fields.map((f) => f.name));
-        for (const path of requiredFieldFilterPaths) {
-          if (!path) {
+        for (const group of requiredFilters) {
+          if (!group || group.length === 0) {
             throw new Error(
-              `Table '${t.name}': requiredFieldFilterPaths must not contain empty strings`
+              `Table '${t.name}': requiredFilters must not contain empty groups`
             );
           }
-          const head = path.split(".", 1)[0];
-          if (!columnNames.has(head)) {
-            throw new Error(
-              `Table '${t.name}': requiredFieldFilterPaths path '${path}' references ` +
-                `unknown column '${head}'. Available columns: ${[...columnNames].sort().join(", ")}`
-            );
+          for (const path of group) {
+            if (!path) {
+              throw new Error(
+                `Table '${t.name}': requiredFilters must not contain empty strings`
+              );
+            }
+            const head = path.split(".", 1)[0];
+            if (!columnNames.has(head)) {
+              throw new Error(
+                `Table '${t.name}': requiredFilters path '${path}' references ` +
+                  `unknown column '${head}'. Available columns: ${[...columnNames].sort().join(", ")}`
+              );
+            }
           }
         }
       }
@@ -292,7 +300,7 @@ export class ReadOnlyCatalogInterface extends CatalogInterface {
         cardinality_max: t.inlinedCardinality
           ? Number(t.inlinedCardinality.max)
           : null,
-        required_field_filter_paths: requiredFieldFilterPaths,
+        required_filters: requiredFilters,
       };
     }));
   }
@@ -437,6 +445,15 @@ export class ReadOnlyCatalogInterface extends CatalogInterface {
             (f.kind as string) === "table_buffering" && f.meta.sinkOrderDependent === true,
           requires_input_batch_index:
             (f.kind as string) === "table_buffering" && f.meta.requiresInputBatchIndex === true,
+          // Blended ("UNNEST-style") table-in-out: positional args ARE the
+          // per-row input columns (real typed args, no TABLE placeholder), so
+          // one registration serves literal / column / LATERAL call shapes.
+          // Set by defineRowTransformFunction; the C++ extension reads it to
+          // enter the in-out registration branch with real-typed args and
+          // drive the literal single-row scan-mode. False for scalars —
+          // mirrors Python's `input_from_args=False if is_scalar else ...`.
+          input_from_args:
+            f.kind === "table_in_out" && f.meta.inputFromArgs === true,
           required_settings: meta.requiredSettings,
           required_secrets: requiredSecrets.map((s) => ({
             secret_type: s.secret_type,

package/src/client/client.ts

@@ -170,6 +170,7 @@ export class VgiClient {
       tablesample_seed: ts?.seed == null ? null : BigInt(ts.seed),
       execution_id: opts?.executionId ?? null,
       init_opaque_data: null,
+      substream_id: null,
     };
 
     const initBatch = serializeInitRequest(initRequest);

package/src/functions/registry.ts

@@ -123,16 +123,40 @@ function filterByArgumentTypes(
         matched = r2.matched;
       }
     } else {
-      // Table functions: compare positional arg types
+      // Table functions: compare declared positional specs against the
+      // argument types.
       const posSpecs = specs
         .filter(s => typeof s.position === "number" && !s.isTableInput)
         .sort((a, b) => (a.position as number) - (b.position as number));
 
       const posTypes: (VgiDataType | null)[] = [];
-      const argsSchema = args.argumentsSchema;
-      for (let i = 0; i < args.positional.length; i++) {
-        const field = argsSchema?.fields.find(f => f.name === `positional_${i}`);
-        posTypes.push(field ? field.type : null);
+      if (func.meta.inputFromArgs) {
+        // Blended ("UNNEST-style"): the positional args ARE the per-row input
+        // columns, so they are NOT on the wire (args.positional is empty).
+        // Score the declared positional specs against the INPUT_SCHEMA column
+        // types instead (coercibly, via scoreTypes), so same-arity overloads
+        // disambiguate by type — matching what DuckDB's binder resolved.
+        // Expand a trailing varargs spec across the remaining input columns
+        // (mirrors the scalar column path).
+        if (inputSchema) {
+          let varArgsSpec: ArgumentSpec | null = null;
+          for (const spec of posSpecs) {
+            if (spec.isVarargs) varArgsSpec = spec;
+            const pos = spec.position as number;
+            posTypes.push(pos < inputSchema.fields.length ? inputSchema.fields[pos].type : null);
+          }
+          if (varArgsSpec) {
+            for (let i = (varArgsSpec.position as number) + 1; i < inputSchema.fields.length; i++) {
+              posTypes.push(inputSchema.fields[i].type);
+            }
+          }
+        }
+      } else {
+        const argsSchema = args.argumentsSchema;
+        for (let i = 0; i < args.positional.length; i++) {
+          const field = argsSchema?.fields.find(f => f.name === `positional_${i}`);
+          posTypes.push(field ? field.type : null);
+        }
       }
       const r = scoreTypes(posSpecs, posTypes);
       score += r.score;
@@ -207,6 +231,30 @@ export class FunctionRegistry {
         );
         const hasVarargs = posSpecs.some(s => s.isVarargs);
         const nonVarargs = posSpecs.filter(s => !s.isVarargs).length;
+        if (func.meta.inputFromArgs) {
+          // Blended ("UNNEST-style") overload: the positional params ARE the
+          // per-row input columns, so they are NOT on the wire
+          // (args.positional is empty). Resolve by INPUT-COLUMN count (arity)
+          // against the declared positional params instead — e.g.
+          // geo_encode(52,13) -> 2 input cols -> the 2-positional overload.
+          // Named (str-position) args still come from the wire arguments.
+          const numInputCols = context.inputSchema?.fields.length ?? 0;
+          if (hasVarargs) {
+            if (numInputCols < nonVarargs) return false;
+          } else if (numInputCols !== posSpecs.length) {
+            return false;
+          }
+          // Unknown named argument disqualifies the overload.
+          const validNamed = new Set(
+            func.argumentSpecs
+              .filter(s => typeof s.position === "string")
+              .map(s => s.position as string),
+          );
+          for (const key of args.named.keys()) {
+            if (!validNamed.has(key)) return false;
+          }
+          return true;
+        }
         if (hasVarargs) {
           return numArgs >= nonVarargs;
         }

package/src/functions/scalar.ts

@@ -44,6 +44,7 @@ import {
   type ArgumentConstraints,
 } from "../arguments/argument-spec.js";
 import { batchToScalarDict, batchToSecretDict, batchFromColumns, projectBatch, safeNumber } from "../util/arrow/index.js";
+import { cacheControlMetadata, type CacheControl } from "../cache-control.js";
 
 // ============================================================================
 // Scalar Bind Parameters
@@ -181,6 +182,15 @@ export interface ScalarFunctionConfig<
   maxWorkers?: number;
   requiredSettings?: string[];
   requiredSecrets?: string[];
+  /**
+   * Result-cache opt-in: when set, this CacheControl's `vgi.cache.*` metadata
+   * rides every output batch's custom metadata (per-batch — NOT the schema,
+   * which the IPC stream fixes at open), so the extension can memoize the
+   * scalar's output per distinct input value. A pure, deterministic scalar
+   * only — advertising this on a non-pure scalar serves stale rows. Mirrors
+   * vgi-python's `ScalarFunction.CACHE_CONTROL`.
+   */
+  cacheControl?: CacheControl;
 }
 
 export function defineScalarFunction<
@@ -394,7 +404,13 @@ export function defineScalarFunction<
             );
           }
 
-          out.emit(outputBatch);
+          // Result-cache opt-in: a scalar declaring cacheControl rides its
+          // vgi.cache.* keys on the emit path's per-batch custom metadata so
+          // the extension can memoize the output per distinct input value.
+          out.emit(
+            outputBatch,
+            config.cacheControl ? cacheControlMetadata(config.cacheControl) : undefined,
+          );
         },
       };
     },