@query-farm/vgi 0.27.0 → 0.29.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@query-farm/vgi",
-  "version": "0.27.0",
+  "version": "0.29.0",
   "description": "TypeScript framework for extending DuckDB with user-defined functions via the Vector Gateway Interface (VGI) over Arrow IPC.",
   "license": "SEE LICENSE IN LICENSE",
   "author": "Query Farm LLC (https://query.farm)",
@@ -90,12 +90,12 @@
   },
   "peerDependencies": {
     "@query-farm/apache-arrow": "^21.1.1",
-    "@query-farm/vgi-rpc": ">=0.20.0 <0.21.0"
+    "@query-farm/vgi-rpc": ">=0.22.0 <0.23.0"
   },
   "devDependencies": {
     "@duckdb/node-api": "^1.5.5-r.4",
     "@query-farm/apache-arrow": "^21.1.1",
-    "@query-farm/vgi-rpc": "^0.20.0",
+    "@query-farm/vgi-rpc": "^0.22.0",
     "@types/bun": "latest",
     "typescript": "^7.0.2"
   },

package/src/arguments/argument-spec.ts

@@ -167,6 +167,87 @@ function choiceEquals(choice: unknown, value: unknown): boolean {
   return choice === value;
 }
 
+/**
+ * Narrow a bind-time argument value from Arrow's bigint to a plain number.
+ *
+ * DuckDB types an integer literal as BIGINT, so `series(3)` arrives as `3n`
+ * whether or not anyone wanted 64 bits. Nobody writes `BigInt(1000)` for a page
+ * size, so the arg paths narrow — and they now all narrow the same way, which
+ * they did not before (two sites used safeNumber, two a bare Number).
+ *
+ * This is lossy above 2^53, and that is a deliberate call rather than an
+ * oversight. Every int64 *argument* across the fixtures and every shipped
+ * worker is a small control value — count, page_size, batch_size, increment,
+ * lag_minutes, restatement_lookback_days. The large int64s in the test suite
+ * are all somewhere else entirely:
+ *
+ *   - column data (copy_to/types.test, copy_to/parallel.test) — never touches
+ *     this path;
+ *   - filter-pushdown constants (filter_pushdown/integers.test pins 2^53+1 on
+ *     purpose) — deserialize.ts keeps those bigint, with its own regression
+ *     test;
+ *   - varargs values (constant_columns) — read raw off bindCall.arguments,
+ *     because extractArgs' copy of them is not what the function reads;
+ *   - time-travel versions — carried as strings (`at_value`).
+ *
+ * Preserving the bigint on overflow was tried and reverted: it buys nothing for
+ * any argument that exists, and costs a JS type that depends on magnitude, so a
+ * worker doing `args.n + 1` would work in every test and fail on one caller's
+ * input. If an argument ever genuinely needs the full range — a snowflake id, a
+ * nanosecond epoch — the fix is to make int64 arguments bigint by declared
+ * type, not by magnitude.
+ */
+export function narrowArgValue(value: unknown): unknown {
+  return typeof value === "bigint" ? Number(value) : value;
+}
+
+/**
+ * Reject a type *factory* where an Arrow *type* was expected.
+ *
+ * `params: { n: int64 }` (no parentheses) is accepted by the type system —
+ * `VgiDataType` is loose enough — and then silently produces an ArgumentSpec
+ * whose `arrowType` is a Function. Nothing complains until the schema is
+ * serialized or the bind arrives, and the error at that point names neither the
+ * argument nor the cause.
+ *
+ * The mistake is easy to make on purpose, too: the same identifier is a
+ * ready-made type on one entry point and a factory on another.
+ *
+ *   @query-farm/vgi            int, int32, float32, bool  →  type instances
+ *   @query-farm/vgi/worker-cf  int, int32, float32, bool  →  factories
+ *
+ * So moving a worker to Cloudflare can turn a correct declaration into a
+ * silently wrong one with no other edit. Failing here, at definition time,
+ * makes that a startup error naming the field.
+ */
+export function assertArrowType(value: unknown, where: string): void {
+  if (typeof value === "function") {
+    const name = (value as { name?: string }).name;
+    const call = name ? `${name}()` : "the factory";
+    throw new ArgumentValidationError(
+      `${where} is a type factory, not an Arrow type — call it: ${call}. ` +
+        `(Note that int, int32, float32 and bool are ready-made types on ` +
+        `@query-farm/vgi but factories on @query-farm/vgi/worker-cf, so the ` +
+        `same name may need parentheses on one entry point and not the other. ` +
+        `int64(), float64(), utf8() and the rest are factories everywhere.)`,
+    );
+  }
+  if (value === null || value === undefined) {
+    throw new ArgumentValidationError(`${where} is ${String(value)}, not an Arrow type.`);
+  }
+}
+
+/** {@link assertArrowType} over a record of declared types, e.g. `params`. */
+export function assertArrowTypes(
+  types: Record<string, unknown> | undefined,
+  where: string,
+): void {
+  if (!types) return;
+  for (const [name, type] of Object.entries(types)) {
+    assertArrowType(type, `${where}.${name}`);
+  }
+}
+
 /**
  * Enforce a bind-time const value against its declared {@link ArgumentConstraints},
  * throwing {@link ArgumentValidationError} on violation. Mirrors the Python SDK's

package/src/arguments/arguments.ts

@@ -3,7 +3,6 @@
 // Holds positional and named scalar values from DuckDB.
 
 import type { VgiSchema } from "../arrow/index.js";
-import { safeNumber } from "../util/arrow/index.js";
 
 export class Arguments {
   readonly positional: (any | null)[];

package/src/arrow/impl-arrowjs/index.ts

@@ -17,7 +17,7 @@ export {
   int8, int16, int32, int64,
   uint8, uint16, uint32, uint64,
   float16, float32, float64,
-  utf8, binary, fixedSizeBinary,
+  utf8, largeUtf8, binary, largeBinary, fixedSizeBinary,
   decimal, decimal128, decimal256,
   date, dateDay, dateMillisecond,
   time, timeSecond, timeMillisecond, timeMicrosecond, timeNanosecond,

package/src/arrow/impl-arrowjs/iterate.ts

@@ -151,8 +151,29 @@ export function batchToSecretDict(
 /**
  * Safe number coercion for BigInt values.
  */
+const MAX_SAFE = BigInt(Number.MAX_SAFE_INTEGER);
+const MIN_SAFE = -MAX_SAFE;
+
+/**
+ * Narrow a bigint to a number, refusing to do it lossily.
+ *
+ * Arrow int64/uint64 values arrive as bigint. Several bind-time argument paths
+ * want a plain number, and used to get one from a bare `Number(value)` — which
+ * silently rounds above 2^53, so an id or a nanosecond timestamp passed as a
+ * function argument came back subtly wrong with nothing logged anywhere.
+ *
+ * Throwing is the lesser evil: a caller that genuinely wants a lossy narrowing
+ * can still write `Number(v)` themselves, but nobody has to discover the loss
+ * from mismatched output weeks later.
+ */
 export function safeNumber(value: any): number {
   if (typeof value === "bigint") {
+    if (value > MAX_SAFE || value < MIN_SAFE) {
+      throw new RangeError(
+        `${value} cannot be represented exactly as a JS number ` +
+          `(|value| > Number.MAX_SAFE_INTEGER). Keep it as a bigint.`,
+      );
+    }
     return Number(value);
   }
   return value as number;

package/src/arrow/impl-arrowjs/schema.ts

@@ -20,7 +20,9 @@ import {
   Float32 as A_Float32,
   Float64 as A_Float64,
   Utf8 as A_Utf8,
+  LargeUtf8 as A_LargeUtf8,
   Binary as A_Binary,
+  LargeBinary as A_LargeBinary,
   Decimal as A_Decimal,
   Date_ as A_Date,
   Time as A_Time,
@@ -134,7 +136,9 @@ export const float32 = (): VgiDataType => new A_Float32() as unknown as VgiDataT
 export const float64 = (): VgiDataType => new A_Float64() as unknown as VgiDataType;
 
 export const utf8 = (): VgiDataType => new A_Utf8() as unknown as VgiDataType;
+export const largeUtf8 = (): VgiDataType => new A_LargeUtf8() as unknown as VgiDataType;
 export const binary = (): VgiDataType => new A_Binary() as unknown as VgiDataType;
+export const largeBinary = (): VgiDataType => new A_LargeBinary() as unknown as VgiDataType;
 export const fixedSizeBinary = (byteWidth: number): VgiDataType =>
   new A_FixedSizeBinary(byteWidth) as unknown as VgiDataType;
 

package/src/arrow/impl-flechette/index.ts

@@ -22,7 +22,7 @@ export {
   int8, int16, int32, int64,
   uint8, uint16, uint32, uint64,
   float16, float32, float64,
-  utf8, binary, fixedSizeBinary,
+  utf8, largeUtf8, binary, largeBinary, fixedSizeBinary,
   decimal, decimal128, decimal256,
   date, dateDay, dateMillisecond,
   time, timeSecond, timeMillisecond, timeMicrosecond, timeNanosecond,

package/src/arrow/impl-flechette/iterate.ts

@@ -115,7 +115,30 @@ export function batchToSecretDict(
 /**
  * BigInt → number coercion (lossy for values past 2^53).
  */
+const MAX_SAFE = BigInt(Number.MAX_SAFE_INTEGER);
+const MIN_SAFE = -MAX_SAFE;
+
+/**
+ * Narrow a bigint to a number, refusing to do it lossily.
+ *
+ * Arrow int64/uint64 values arrive as bigint. Several bind-time argument paths
+ * want a plain number, and used to get one from a bare `Number(value)` — which
+ * silently rounds above 2^53, so an id or a nanosecond timestamp passed as a
+ * function argument came back subtly wrong with nothing logged anywhere.
+ *
+ * Throwing is the lesser evil: a caller that genuinely wants a lossy narrowing
+ * can still write `Number(v)` themselves, but nobody has to discover the loss
+ * from mismatched output weeks later.
+ */
 export function safeNumber(value: any): number {
-  if (typeof value === "bigint") return Number(value);
+  if (typeof value === "bigint") {
+    if (value > MAX_SAFE || value < MIN_SAFE) {
+      throw new RangeError(
+        `${value} cannot be represented exactly as a JS number ` +
+          `(|value| > Number.MAX_SAFE_INTEGER). Keep it as a bigint.`,
+      );
+    }
+    return Number(value);
+  }
   return value as number;
 }

package/src/arrow/impl-flechette/schema.ts

@@ -23,7 +23,9 @@ import {
   float32 as f_float32,
   float64 as f_float64,
   utf8 as f_utf8,
+  largeUtf8 as f_largeUtf8,
   binary as f_binary,
+  largeBinary as f_largeBinary,
   fixedSizeBinary as f_fixedSizeBinary,
   decimal as f_decimal,
   decimal128 as f_decimal128,
@@ -111,7 +113,9 @@ export const float32 = (): VgiDataType => f_float32() as unknown as VgiDataType;
 export const float64 = (): VgiDataType => f_float64() as unknown as VgiDataType;
 
 export const utf8 = (): VgiDataType => f_utf8() as unknown as VgiDataType;
+export const largeUtf8 = (): VgiDataType => f_largeUtf8() as unknown as VgiDataType;
 export const binary = (): VgiDataType => f_binary() as unknown as VgiDataType;
+export const largeBinary = (): VgiDataType => f_largeBinary() as unknown as VgiDataType;
 export const fixedSizeBinary = (stride: number): VgiDataType =>
   f_fixedSizeBinary(stride) as unknown as VgiDataType;
 

package/src/arrow/index.ts

@@ -81,7 +81,7 @@ export {
   int, int8, int16, int32, int64,
   uint8, uint16, uint32, uint64,
   float16, float32, float64,
-  utf8, binary, fixedSizeBinary,
+  utf8, largeUtf8, binary, largeBinary, fixedSizeBinary,
   decimal, decimal128, decimal256,
   date, dateDay, dateMillisecond,
   time, timeSecond, timeMillisecond, timeMicrosecond, timeNanosecond,

package/src/arrow/schema-types.ts

@@ -25,7 +25,9 @@ import type {
   IntType,
   FloatType,
   Utf8Type,
+  LargeUtf8Type,
   BinaryType,
+  LargeBinaryType,
   FixedSizeBinaryType,
   DecimalType,
   Date32Type, Date64Type,
@@ -83,7 +85,9 @@ export const float32 = (): FloatType => impl.float32() as unknown as FloatType;
 export const float64 = (): FloatType => impl.float64() as unknown as FloatType;
 
 export const utf8 = (): Utf8Type => impl.utf8() as unknown as Utf8Type;
+export const largeUtf8 = (): LargeUtf8Type => impl.largeUtf8() as unknown as LargeUtf8Type;
 export const binary = (): BinaryType => impl.binary() as unknown as BinaryType;
+export const largeBinary = (): LargeBinaryType => impl.largeBinary() as unknown as LargeBinaryType;
 export const fixedSizeBinary = (byteWidth: number): FixedSizeBinaryType =>
   impl.fixedSizeBinary(byteWidth) as unknown as FixedSizeBinaryType;
 

package/src/arrow/types.ts

@@ -33,9 +33,43 @@ export interface VgiSchema {
   readonly metadata: Map<string, string>;
 }
 
-/** Column view. Generic dropped: arrow-js parameterizes on DataType, flechette
- *  on value type, so the facade exposes an erased shape. Callers cast at the
- *  use site (the column's static schema makes the value type known there). */
+/**
+ * Column view over one column of a batch.
+ *
+ * The value type is erased by default: arrow-js parameterizes on DataType and
+ * flechette on value type, so the facade cannot name one without picking a
+ * backend. Supply `T` at the use site — the column's declared type makes it
+ * known there — and the cast disappears:
+ *
+ * ```ts
+ * const ns = batch.getChildAt(0)! as Iterable<bigint | null>;
+ * for (const v of ns) { ... }        // v: bigint | null
+ * ```
+ *
+ * A type parameter here would read better, but it cannot be had cheaply: both
+ * backends' native batches are assigned to VgiBatch structurally, and a
+ * concrete `get(): unknown` does not satisfy a generic `get(): T`. Adding one
+ * would mean casting at ~25 internal sites to remove one cast in user code.
+ *
+ * **These are the backend's own values, not codec output.** `get()` and
+ * iteration return whatever the Arrow implementation stores, which for some
+ * types is not the value the SDK documents:
+ *
+ * | Arrow type       | here                          | {@link iterRows}      |
+ * | ---------------- | ----------------------------- | --------------------- |
+ * | `int64`          | `bigint`                      | `bigint`   (same)     |
+ * | `utf8`, `float64`| `string`, `number`            | same                  |
+ * | `decimal128`     | backend limbs (`DecimalBigNum`)| `bigint`, unscaled    |
+ * | `timestamp[us]`  | millisecond `number`          | microsecond `bigint`  |
+ * | `date32`         | millisecond `number`          | `Date`                |
+ *
+ * `repr: "raw"` on a function selects the *codec's* representation and has no
+ * effect on this path.
+ *
+ * So: reach for `getChildAt` on integer, float, boolean, string and binary
+ * columns, where it is the cheapest correct thing. For temporal, decimal and
+ * nested types go through {@link iterRows}, which runs the codec.
+ */
 export interface VgiColumn {
   readonly type: VgiDataType;
   readonly length: number;
@@ -46,7 +80,9 @@ export interface VgiColumn {
 export interface VgiBatch {
   readonly schema: VgiSchema;
   readonly numRows: number;
+  /** See {@link VgiColumn} — this does NOT run the codec. */
   getChild(name: string): VgiColumn | null;
+  /** See {@link VgiColumn} — this does NOT run the codec. */
   getChildAt(index: number): VgiColumn | null;
 }
 

package/src/catalog/interface.ts

@@ -164,7 +164,7 @@ export const encodeAttachCatalogInfo = (v: AttachCatalogInfo): Uint8Array =>
 import { deserializeBatch, serializeBatch, batchFromColumns } from "../util/arrow/index.js";
 import { schema as schema_, field as field_, type VgiDataType, type VgiField } from "../arrow/index.js";
 import { toUint8Array } from "../util/bytes.js";
-import { ScanBranchSchema } from "../generated/vgi-protocol-schemas.js";
+import { ScanBranchSchema, ScanFunctionResultSchema } from "../generated/vgi-protocol-schemas.js";
 
 /**
  * Result from `tableScanFunctionGet` — tells the VGI DuckDB extension which
@@ -181,6 +181,37 @@ export interface ScanFunctionResult {
   requiredExtensions: string[];
 }
 
+/**
+ * Encode a ScanFunctionResult wire record — the single-row batch shape
+ * `{ function_name, arguments, required_extensions }` that
+ * `catalog_table_scan_function_get` returns and that a function-backed table
+ * inlines into `TableInfo.scan_function`.
+ *
+ * It exists so there is exactly one place that names those columns. The
+ * inline-scan path used to restate the schema locally even though codegen
+ * emits it; a hand-written wire schema is how four SDKs ended up disagreeing
+ * about which columns were binary and which were large_binary.
+ *
+ * `argumentsBytes` is the already-serialized inner arguments batch (one column
+ * per argument: `arg_<index>` positional, bare name for named).
+ */
+export function encodeScanFunctionResult(
+  functionName: string,
+  argumentsBytes: Uint8Array,
+  requiredExtensions: string[] = [],
+): Uint8Array {
+  return serializeBatch(
+    batchFromColumns(
+      {
+        function_name: [functionName],
+        arguments: [argumentsBytes],
+        required_extensions: [requiredExtensions],
+      },
+      ScanFunctionResultSchema as any,
+    ),
+  );
+}
+
 /**
  * Decode a ScanFunctionResult from a wire-shape inner result dict.
  *
@@ -237,24 +268,59 @@ export function decodeScanFunctionResult(inner: Record<string, unknown>): ScanFu
 export function singleBranchResult(
   legacy: { function_name: string; arguments: unknown; required_extensions?: string[] },
 ): { branches: Uint8Array[]; required_extensions: string[] } {
-  const branchBatch = batchFromColumns(
-    {
-      function_name: [legacy.function_name],
-      arguments: [toUint8Array(legacy.arguments)],
-      branch_filter: [null],
-      writable: [false],
-      source_catalog: [null],
-      source_schema: [null],
-      source_table: [null],
-    },
-    ScanBranchSchema as any,
-  );
   return {
-    branches: [serializeBatch(branchBatch)],
+    branches: [
+      encodeScanBranch({ function_name: legacy.function_name, arguments: toUint8Array(legacy.arguments) }),
+    ],
     required_extensions: legacy.required_extensions ?? [],
   };
 }
 
+/**
+ * The ONE place `ScanBranch`'s columns are named.
+ *
+ * Both branch builders used to carry their own copy of this key list, and both
+ * copies were missing `format_name` / `format_locations` after those fields
+ * were added to the schema. `format_locations` is a LIST, and Arrow's assembler
+ * dereferences a list's `children[0]` while writing, so the omission did not
+ * degrade anything — it killed every scan through those builders with
+ * "Cannot read properties of undefined (reading 'slice')", including tables
+ * that predated format branches entirely. One key list is the fix; the
+ * completeness test in src/__tests__ pins it against ScanBranchSchema.
+ *
+ * Every column the schema declares must appear here, null included.
+ */
+function encodeScanBranch(cols: {
+  function_name: string;
+  arguments: Uint8Array;
+  branch_filter?: string | null;
+  writable?: boolean;
+  source_catalog?: string | null;
+  source_schema?: string | null;
+  source_table?: string | null;
+  format_name?: string | null;
+  format_locations?: string[] | null;
+  format_options?: Uint8Array | null;
+}): Uint8Array {
+  return serializeBatch(
+    batchFromColumns(
+      {
+        function_name: [cols.function_name],
+        arguments: [cols.arguments],
+        branch_filter: [cols.branch_filter ?? null],
+        writable: [cols.writable ?? false],
+        source_catalog: [cols.source_catalog ?? null],
+        source_schema: [cols.source_schema ?? null],
+        source_table: [cols.source_table ?? null],
+        format_name: [cols.format_name ?? null],
+        format_locations: [cols.format_locations ?? null],
+        format_options: [cols.format_options ?? null],
+      },
+      ScanBranchSchema as any,
+    ),
+  );
+}
+
 /**
  * One physical source backing a multi-branch scan, in the shape the
  * `buildScanBranchesResult` helper consumes. Mirrors vgi-python's
@@ -275,6 +341,16 @@ export interface ScanBranchInput {
   branchFilter?: string | null;
   /** Declares this branch as the INSERT target (at most one per table). */
   writable?: boolean;
+  /** Custom file FORMAT this branch reads, for a format-backed branch. */
+  formatName?: string | null;
+  /** Locations the format branch reads. */
+  formatLocations?: string[] | null;
+  /** Reader options for a format branch, as a 1-row IPC batch whose COLUMN
+   *  NAMES are the option names — the same shape `arguments` uses, because an
+   *  option value may be any Arrow type. These BECOME the reader's named
+   *  arguments, so without them a format branch cannot pass its reader a
+   *  delimiter, a header flag, or anything else. */
+  formatOptions?: Uint8Array | null;
   /**
    * Catalog-table branch (lakehouse federation): leave `functionName` empty and
    * set these to scan the base table
@@ -292,6 +368,26 @@ export interface ScanBranchInput {
  * positional args, the bare name for named args. Mirrors Python's
  * `ScanBranch.to_row_dict` argument handling.
  */
+/**
+ * Encode a format branch's reader options.
+ *
+ * Same shape as a branch's `arguments` blob — a 1-row batch whose COLUMN NAMES
+ * are the option names — because an option value may be any Arrow type, so no
+ * static schema could describe the map. Exported because a worker building a
+ * format branch needs it and has no other way to produce those bytes.
+ */
+export function encodeFormatOptions(
+  options: Record<string, { value: unknown; type: VgiDataType }>,
+): Uint8Array {
+  const fields: VgiField[] = [];
+  const values: Record<string, unknown[]> = {};
+  for (const [name, arg] of Object.entries(options)) {
+    fields.push(field_(name, arg.type, true));
+    values[name] = [arg.value];
+  }
+  return serializeBatch(batchFromColumns(values, schema_(fields)));
+}
+
 function serializeBranchArguments(branch: ScanBranchInput): Uint8Array {
   const fields: VgiField[] = [];
   const values: Record<string, unknown[]> = {};
@@ -328,21 +424,20 @@ export function buildScanBranchesResult(
   branches: ScanBranchInput[],
   requiredExtensions: string[] = [],
 ): { branches: Uint8Array[]; required_extensions: string[] } {
-  const serializedBranches = branches.map((branch) => {
-    const branchBatch = batchFromColumns(
-      {
-        function_name: [branch.functionName],
-        arguments: [serializeBranchArguments(branch)],
-        branch_filter: [branch.branchFilter ?? null],
-        writable: [branch.writable ?? false],
-        source_catalog: [branch.sourceCatalog ?? null],
-        source_schema: [branch.sourceSchema ?? null],
-        source_table: [branch.sourceTable ?? null],
-      },
-      ScanBranchSchema as any,
-    );
-    return serializeBatch(branchBatch);
-  });
+  const serializedBranches = branches.map((branch) =>
+    encodeScanBranch({
+      function_name: branch.functionName,
+      arguments: serializeBranchArguments(branch),
+      branch_filter: branch.branchFilter,
+      writable: branch.writable,
+      source_catalog: branch.sourceCatalog,
+      source_schema: branch.sourceSchema,
+      source_table: branch.sourceTable,
+      format_name: branch.formatName,
+      format_locations: branch.formatLocations,
+      format_options: branch.formatOptions,
+    }),
+  );
   return {
     branches: serializedBranches,
     required_extensions: requiredExtensions,

package/src/catalog/read-only.ts

@@ -17,6 +17,7 @@ import {
   type IndexConstraintType,
   type CopyFromFormatInfo,
   encodeAttachCatalogInfo,
+  encodeScanFunctionResult,
   encodeFunctionInfo,
 } from "./interface.js";
 import type { CatalogDescriptor, SchemaDescriptor, TableDescriptor, ViewDescriptor, MacroDescriptor, SettingDescriptor, SecretTypeDescriptor, ForeignKeyDef, DefaultValue } from "./descriptors.js";
@@ -473,6 +474,22 @@ export class ReadOnlyCatalogInterface extends CatalogInterface {
           supports_batch_index:
             ((f.kind as string) === "table_buffering" && f.meta.requiresInputBatchIndex === true) ||
             ((f.kind as string) === "table" && f.meta.supportsBatchIndex === true),
+          // Split-based scan planning (protocol 1.4.0). A function opts in by
+          // declaring `supportsSplits` AND implementing plan()/onSplit(): the
+          // declaration is what a distributed engine reads to decide it can
+          // retry a task, because a split NAMES its work and re-running one
+          // reads exactly the same rows.
+          supports_splits: (f.kind as string) === "table" && f.meta.supportsSplits === true,
+          // The worker applies pushed filters EXACTLY, so the engine may drop
+          // its own copy. Wrong answers if declared falsely.
+          filters_exactly_applied:
+            (f.kind as string) === "table" && f.meta.filtersExactlyApplied === true,
+          // Addressable positions in the data, for incremental/streaming reads.
+          supports_positions: (f.kind as string) === "table" && f.meta.supportsPositions === true,
+          // null means UNBOUNDED, not "expires immediately" — a client must not
+          // assume a TTL exists, or long-running streams are foreclosed.
+          split_token_ttl_seconds:
+            (f.kind as string) === "table" ? (f.meta.splitTokenTtlSeconds ?? null) : null,
           partition_kind:
             (f.kind as string) === "table" && f.meta.partitionKind
               ? f.meta.partitionKind
@@ -993,22 +1010,5 @@ function inlineScanFunction(
 ): Uint8Array {
   const argSchema = argumentSpecsToSchema(func.argumentSpecs);
   const argBytes = serializeArgsBatch(args ?? new Arguments(), argSchema);
-  const scanSchema = schema_([
-    field("function_name", utf8(), false),
-    field("arguments", binary(), false),
-    field(
-      "required_extensions",
-      list(field("item", utf8(), true)),
-      false,
-    ),
-  ]);
-  const batch = batchFromColumns(
-    {
-      function_name: [func.meta.name],
-      arguments: [argBytes],
-      required_extensions: [[] as string[]],
-    },
-    scanSchema,
-  );
-  return serializeBatch(batch);
+  return encodeScanFunctionResult(func.meta.name, argBytes);
 }

package/src/client/client.ts

@@ -70,7 +70,7 @@ import type {
  * Works with any RpcClient transport (subprocess or HTTP):
  * ```ts
  * import { subprocessConnect } from "@query-farm/vgi-rpc";
- * import { VgiClient, Arguments } from "vgi";
+ * import { VgiClient, Arguments } from "@query-farm/vgi";
  *
  * const rpc = subprocessConnect(["./my-worker"]);
  * const client = new VgiClient(rpc);
@@ -157,6 +157,10 @@ export class VgiClient {
       bind_call: bindRequest,
       output_schema: bindResponse.output_schema,
       bind_opaque_data: bindResponse.opaque_data,
+      // This client does not plan scans, so it never redeems a split, and it
+      // pushes no limit down.
+      split_tokens: null,
+      row_limit: null,
       projection_ids: opts?.projectionIds ?? null,
       pushdown_filters: opts?.pushdownFilters ?? null,
       join_keys: opts?.joinKeys ?? [],
@@ -1302,7 +1306,7 @@ export class VgiClient {
     transactionOpaqueData?: TransactionOpaqueData,
   ): Promise<void> {
     const schema = schema_([
-      field("attach_opaque_data", binary(), true),
+      field("attach_opaque_data", binary(), false),
       field("schema_name", utf8(), false),
       field("name", utf8(), false),
       field("macro_type", utf8(), false),

package/src/functions/aggregate.ts

@@ -20,6 +20,7 @@ import {
   type ArgumentSpec,
   type ArgumentConstraints,
 } from "../arguments/argument-spec.js";
+import { assertArrowType, assertArrowTypes } from "../arguments/argument-spec.js";
 import type {
   FunctionMeta,
   VgiFunction,
@@ -333,6 +334,8 @@ export function deleteExecutionState(executionId: Uint8Array): void {
 export function defineAggregate<TArgs = Record<string, any>, TState = any>(
   config: AggregateFunctionConfig<TArgs, TState>,
 ): VgiFunction & { aggregateConfig: AggregateFunctionConfig<TArgs, TState> } {
+  assertArrowTypes((config as any).args, `defineAggregate("${config.name}"): args`);
+
   const specs: ArgumentSpec[] = [];
   let posIdx = 0;
   const varargsSet = new Set(config.varargs ?? []);

package/src/functions/copy-from.ts

@@ -31,6 +31,7 @@ import type {
 import type { VgiFunction, FunctionMeta, StreamHandlers, FunctionExample } from "./types.js";
 import type { CopySecretLookup } from "./copy-to.js";
 import type { ArgumentSpec } from "../arguments/argument-spec.js";
+import { narrowArgValue } from "../arguments/argument-spec.js";
 import { batchToScalarDict, batchToSecretDict, safeNumber } from "../util/arrow/index.js";
 import { BoundStorage, storage as globalStorage } from "./storage.js";
 import type { TableProcessParams } from "./table.js";
@@ -165,7 +166,7 @@ export function defineCopyFromFunction<TArgs = Record<string, unknown>>(
         }
         val = opt.default;
       }
-      if (typeof val === "bigint") val = safeNumber(val);
+      val = narrowArgValue(val);
       if (opt.choices && !opt.choices.includes(val)) {
         throw new Error(
           `COPY (FORMAT ${config.format}): option '${name}' value ${JSON.stringify(val)} ` +

package/src/functions/copy-to.ts

@@ -38,6 +38,7 @@
 import { type VgiSchema, schema, type VgiBatch, type VgiDataType } from "../arrow/index.js";
 import type { BindRequest } from "../protocol/types.js";
 import type { FunctionMeta, FunctionExample } from "./types.js";
+import { narrowArgValue } from "../arguments/argument-spec.js";
 import { safeNumber } from "../util/arrow/index.js";
 import {
   defineTableBufferingFunction,
@@ -200,7 +201,7 @@ export function defineCopyToFunction<TArgs = Record<string, unknown>>(
         }
         val = opt.default;
       }
-      if (typeof val === "bigint") val = safeNumber(val);
+      val = narrowArgValue(val);
       if (opt.choices && !opt.choices.includes(val)) {
         throw new Error(
           `COPY (FORMAT ${config.format}): option '${name}' value ${JSON.stringify(val)} ` +