@query-farm/vgi 0.27.0 → 0.29.0

package/src/protocol/serializers/bind.ts

@@ -3,7 +3,7 @@
 // ArrowSerializableDataclass field declaration order so the format is
 // positional-compatible with the Python reader.
 
-import { type VgiSchema, schema, type VgiField, field, type VgiBatch, type VgiDataType, utf8, binary, bool, list, struct } from "../../arrow/index.js";
+import { type VgiSchema, schema, field, type VgiBatch, type VgiDataType, utf8, binary, bool, struct, dictionary, int16 } from "../../arrow/index.js";
 import { Arguments } from "../../arguments/arguments.js";
 import { FunctionType } from "../../types.js";
 import type { BindRequest, BindResponse, CopyFromContext, CopyToContext } from "../types.js";
@@ -16,59 +16,58 @@ import {
 import { toUint8Array, buildSingleRowBatch } from "./shared.js";
 import { serializeArguments, deserializeArguments } from "./arguments.js";
 
+// The BindRequest wire record. This must equal `BindRequestSchema` in
+// src/generated/vgi-protocol-schemas.ts EXACTLY — same columns, same order,
+// same types, same nullability — and src/__tests__/wire-schema-completeness.ts
+// asserts that it does.
+//
+// Exactly, not approximately. It is tempting to reason that every peer reads
+// these columns by name, so a nullable column with nothing to say can be left
+// out and a bind that opens no COPY scan can keep a shorter "legacy" shape.
+// That is what this file used to do, and it is wrong: vgi-rpc-go validates a
+// request against its declared parameter contract with `arrow.Schema.Equal`,
+// which is order-, name-, type- AND nullability-sensitive. A record short one
+// nullable column fails EVERY call of that shape, with a message that prints
+// two schemas in full and leaves the reader to diff them by eye.
+//
+// So `copy_from` and `copy_to` are always present and null on a non-COPY bind,
+// `schema_name` is LAST (not before them), and `function_type` is a dictionary
+// rather than plain utf8.
+const COPY_FROM_STRUCT_TYPE = struct([
+  field("format", utf8(), false),
+  field("file_path", utf8(), false),
+  field("expected_schema", binary(), false),
+]);
+
+const COPY_TO_STRUCT_TYPE = struct([
+  field("format", utf8(), false),
+  field("file_path", utf8(), false),
+]);
+
 const BIND_REQUEST_SCHEMA = schema([
   field("function_name", utf8(), false),
   field("arguments", binary(), false),
-  field("function_type", utf8(), false),
+  field("function_type", dictionary(utf8(), int16()), false),
   field("input_schema", binary(), true),
   field("settings", binary(), true),
   field("secrets", binary(), true),
   field("attach_opaque_data", binary(), true),
   field("transaction_opaque_data", binary(), true),
   field("resolved_secrets_provided", bool(), false),
-  // Time travel AT clause (additive, nullable, name-keyed -> wire-compatible).
-  // Both null when the scan has no AT clause. See BindRequest.at_unit.
+  // Time travel AT clause. Both null when the scan has no AT clause.
   field("at_unit", utf8(), true),
   field("at_value", utf8(), true),
+  // Null unless this bind opens a COPY scan / sink.
+  field("copy_from", COPY_FROM_STRUCT_TYPE, true),
+  field("copy_to", COPY_TO_STRUCT_TYPE, true),
   // Catalog schema owning the function. A worker may register one name in
   // several schemas, so the bare name is not a unique key — resolution is by
   // (schema_name, function_name). Null for callers with no catalog context.
   field("schema_name", utf8(), true),
 ]);
 
-// COPY ... FROM context — a nullable nested struct<format, file_path,
-// expected_schema>. Byte-for-byte the shape the C++ extension builds in
-// vgi_rpc_types.cpp (BuildBindRequest's copy_from branch) and the Python
-// BindRequest.copy_from field. Only appended to the serialized batch when the
-// bind actually opens a COPY scan, so ordinary scans keep the legacy wire shape.
-const COPY_FROM_STRUCT_TYPE = struct([
-  field("format", utf8(), false),
-  field("file_path", utf8(), false),
-  field("expected_schema", binary(), false),
-]);
-const COPY_FROM_FIELD = field("copy_from", COPY_FROM_STRUCT_TYPE, true);
-
-// COPY ... TO context — a nullable nested struct<format, file_path>.
-// Byte-for-byte the shape the C++ extension builds in vgi_rpc_types.cpp
-// (BuildBindRequest's copy_to branch) and the Python BindRequest.copy_to field.
-// Only appended to the serialized batch when the bind actually opens a COPY TO
-// sink, so ordinary scans keep the legacy wire shape.
-const COPY_TO_STRUCT_TYPE = struct([
-  field("format", utf8(), false),
-  field("file_path", utf8(), false),
-]);
-const COPY_TO_FIELD = field("copy_to", COPY_TO_STRUCT_TYPE, true);
-
-const BIND_RESPONSE_SCHEMA = schema([
-  field("output_schema", binary(), false),
-  field("opaque_data", binary(), true),
-  field("lookup_secret_types", list(field("item", utf8(), true)), false),
-  field("lookup_scopes", list(field("item", utf8(), true)), false),
-  field("lookup_names", list(field("item", utf8(), true)), false),
-]);
-
 export function serializeBindRequest(req: BindRequest): VgiBatch {
-  const row: Record<string, any> = {
+  return buildSingleRowBatch(BIND_REQUEST_SCHEMA, {
     function_name: req.function_name,
     arguments: serializeArguments(req.arguments),
     function_type: req.function_type,
@@ -80,33 +79,18 @@ export function serializeBindRequest(req: BindRequest): VgiBatch {
     resolved_secrets_provided: req.resolved_secrets_provided ?? false,
     at_unit: req.at_unit ?? null,
     at_value: req.at_value ?? null,
+    copy_from: req.copy_from
+      ? {
+          format: req.copy_from.format,
+          file_path: req.copy_from.file_path,
+          expected_schema: serializeSchema(req.copy_from.expected_schema),
+        }
+      : null,
+    copy_to: req.copy_to
+      ? { format: req.copy_to.format, file_path: req.copy_to.file_path }
+      : null,
     schema_name: req.schema_name ?? null,
-  };
-  // Append the copy_from / copy_to struct columns only for COPY scans/sinks, so
-  // non-COPY binds serialize to the exact legacy shape (fields matched by name
-  // on the reader). At most one is present in practice, but both are handled
-  // generically so the wire stays additive either way.
-  const extraFields: VgiField[] = [];
-  if (req.copy_from) {
-    extraFields.push(COPY_FROM_FIELD);
-    row.copy_from = {
-      format: req.copy_from.format,
-      file_path: req.copy_from.file_path,
-      expected_schema: serializeSchema(req.copy_from.expected_schema),
-    };
-  }
-  if (req.copy_to) {
-    extraFields.push(COPY_TO_FIELD);
-    row.copy_to = {
-      format: req.copy_to.format,
-      file_path: req.copy_to.file_path,
-    };
-  }
-  if (extraFields.length > 0) {
-    const schemaWithCopy = schema([...BIND_REQUEST_SCHEMA.fields, ...extraFields]);
-    return buildSingleRowBatch(schemaWithCopy, row);
-  }
-  return buildSingleRowBatch(BIND_REQUEST_SCHEMA, row);
+  });
 }
 
 function parseCopyFromContext(raw: any): CopyFromContext | null {
@@ -218,4 +202,13 @@ export function deserializeBindResponse(
   };
 }
 
-export { BIND_REQUEST_SCHEMA, BIND_RESPONSE_SCHEMA };
+// BIND_REQUEST_SCHEMA has no codegen counterpart — the generated
+// BindParamsSchema is only the `{request: binary}` envelope, so the inner
+// record is spelled out above. The BindResponse copy that used to sit beside
+// it did have one, and the two had already drifted: this file called
+// `opaque_data` nullable where the generated BindResultSchema calls it
+// non-null. Nobody read the copy (handlers/function.ts wraps with the
+// generated schema), which is exactly how a duplicate goes stale unnoticed —
+// and a stale copy is how a *correct* client ends up rejected at its first
+// call. Use BindResultSchema from src/generated/vgi-protocol-schemas.ts.
+export { BIND_REQUEST_SCHEMA };

package/src/protocol/serializers/cardinality.ts

@@ -1,17 +1,12 @@
 // Copyright 2025, 2026 Query Farm LLC - https://query.farm
 // TableFunctionCardinalityRequest / TableCardinality wire serialization.
 
-import { type VgiSchema, schema, type VgiField, field, type VgiDataType, binary, int64 } from "../../arrow/index.js";
+import { type VgiSchema, schema, type VgiField, field, type VgiDataType, binary } from "../../arrow/index.js";
 import type { TableFunctionCardinalityRequest, TableCardinality } from "../types.js";
 import { deserializeBatch, batchToScalarDict } from "../../util/arrow/index.js";
 import { toUint8Array } from "./shared.js";
 import { deserializeBindRequest } from "./bind.js";
 
-const TABLE_CARDINALITY_SCHEMA = schema([
-  field("estimate", int64(), true),
-  field("max", int64(), true),
-]);
-
 const TABLE_FUNCTION_CARDINALITY_REQUEST_SCHEMA = schema([
   field("bind_call", binary(), false),
   field("bind_opaque_data", binary(), true),
@@ -42,4 +37,9 @@ export function serializeTableCardinality(
   };
 }
 
-export { TABLE_CARDINALITY_SCHEMA, TABLE_FUNCTION_CARDINALITY_REQUEST_SCHEMA };
+// The TableCardinality result shape used to be restated here as well; it is
+// generated as TableFunctionCardinalityResultSchema, which is what
+// handlers/function.ts already wraps with. Only the request shape stays hand-
+// written, because codegen emits no schema for the inner request record (the
+// generated Params schema is just the `{request: binary}` envelope).
+export { TABLE_FUNCTION_CARDINALITY_REQUEST_SCHEMA };

package/src/protocol/serializers/init.ts

@@ -6,7 +6,7 @@
 //   bind_opaque_data: binary (nullable)
 //   projection_ids: list<int64> (nullable)
 //   pushdown_filters: large_binary (nullable) — nested IPC of filter batch
-//   join_keys: list<large_binary> (non-null list of IPC-serialized batches)
+//   join_keys: list<large_binary> (nullable list of IPC-serialized batches)
 //   phase: utf8 (nullable) — dictionary(int16,string) in Python
 //   execution_id: binary (nullable)
 //   init_opaque_data: binary (nullable)
@@ -21,6 +21,9 @@ import {
   int64,
   float64,
   list,
+  largeBinary,
+  dictionary,
+  int16,
 } from "../../arrow/index.js";
 // RecordBatch type alias kept for the joinKeys closure below — it's a runtime
 // value that lands in BindRequest, not a wire shape.
@@ -43,14 +46,31 @@ import { toUint8Array, buildSingleRowBatch } from "./shared.js";
 import { serializeBindRequest, deserializeBindRequest } from "./bind.js";
 import { decodeDictValue } from "../../util/arrow/index.js";
 
+// The InitRequest wire record. This must equal `InitRequestSchema` in
+// src/generated/vgi-protocol-schemas.ts EXACTLY — see the note on
+// BIND_REQUEST_SCHEMA in bind.ts for why "reads by name, so close enough" is
+// not good enough (vgi-rpc-go compares with `arrow.Schema.Equal`).
+//
+// This declaration had drifted in six ways at once, none of which any test
+// could see: it was missing `split_tokens` and `row_limit` entirely, typed
+// `pushdown_filters` and the `join_keys` items as 32-bit binary where the
+// protocol says large, typed `phase` and the two order-by enums as plain utf8
+// where the protocol says dictionary, and put `finalize_state_id` last instead
+// of ninth.
 const INIT_REQUEST_SCHEMA = makeSchema([
   field("bind_call", binary(), false),
   field("output_schema", binary(), false),
   field("bind_opaque_data", binary(), true),
-  field("projection_ids", list(field("item", int64(), false)), true),
-  field("pushdown_filters", binary(), true),
-  field("join_keys", list(field("item", binary(), true)), false),
-  field("phase", utf8(), true),
+  field("projection_ids", list(field("item", int64(), true)), true),
+  // Large: a serialized filter tree, a set of join keys, or a batch of split
+  // tokens can exceed the 2 GiB that 32-bit offsets address.
+  field("pushdown_filters", largeBinary(), true),
+  field("join_keys", list(field("item", largeBinary(), true)), true),
+  field("split_tokens", list(field("item", largeBinary(), true)), true),
+  field("row_limit", int64(), true),
+  field("phase", dictionary(utf8(), int16()), true),
+  // Buffered-table finalize stream identifier.
+  field("finalize_state_id", binary(), true),
   field("execution_id", binary(), true),
   field("init_opaque_data", binary(), true),
   // Per-substream identity for parallel streaming table-in-out (client-minted,
@@ -58,16 +78,21 @@ const INIT_REQUEST_SCHEMA = makeSchema([
   field("substream_id", binary(), true),
   // Order pushdown hints from DuckDB's RowGroupPruner (all null when no hint).
   field("order_by_column_name", utf8(), true),
-  field("order_by_direction", utf8(), true),
-  field("order_by_null_order", utf8(), true),
+  field("order_by_direction", dictionary(utf8(), int16()), true),
+  field("order_by_null_order", dictionary(utf8(), int16()), true),
   field("order_by_limit", int64(), true),
   // TABLESAMPLE pushdown hints from DuckDB's SamplingPushdown optimizer.
   field("tablesample_percentage", float64(), true),
   field("tablesample_seed", int64(), true),
-  // Buffered-table finalize stream identifier — C++ appends this last.
-  field("finalize_state_id", binary(), true),
 ]);
 
+// The init stream's header. Codegen emits nothing for it (the generated
+// InitParamsSchema covers only the request envelope), so it is hand-written —
+// which is exactly why it must be hand-written ONCE. It used to be restated at
+// the `protocol.exchange("init", ...)` registration in handlers/function.ts,
+// and two copies of a wire schema is how a *correct* client ends up rejected:
+// the copies disagree on some field's nullability and neither is wrong on its
+// own terms.
 const GLOBAL_INIT_RESPONSE_SCHEMA = makeSchema([
   field("execution_id", binary(), false),
   field("opaque_data", binary(), true),
@@ -87,7 +112,10 @@ export function serializeInitRequest(req: InitRequest): VgiBatch {
       ? serializeBatch(req.pushdown_filters)
       : null,
     join_keys: (req.join_keys ?? []).map((b) => serializeBatch(b)),
+    split_tokens: req.split_tokens ?? null,
+    row_limit: req.row_limit ?? null,
     phase: req.phase ?? null,
+    finalize_state_id: req.finalize_state_id ?? null,
     execution_id: req.execution_id ?? null,
     init_opaque_data: req.init_opaque_data ?? null,
     substream_id: req.substream_id ?? null,
@@ -97,7 +125,6 @@ export function serializeInitRequest(req: InitRequest): VgiBatch {
     order_by_limit: req.order_by_limit ?? null,
     tablesample_percentage: req.tablesample_percentage ?? null,
     tablesample_seed: req.tablesample_seed ?? null,
-    finalize_state_id: req.finalize_state_id ?? null,
   };
   return buildSingleRowBatch(INIT_REQUEST_SCHEMA, row);
 }
@@ -167,6 +194,7 @@ export function deserializeInitRequest(
       ? toUint8Array(params.bind_opaque_data)
       : null,
     projection_ids: projectionIds,
+    row_limit: params.row_limit == null ? null : Number(params.row_limit),
     pushdown_filters: params.pushdown_filters
       ? deserializeBatch(toUint8Array(params.pushdown_filters))
       : null,
@@ -175,6 +203,20 @@ export function deserializeInitRequest(
     finalize_state_id: params.finalize_state_id
       ? toUint8Array(params.finalize_state_id)
       : null,
+    // The envelopes this init redeems. Read defensively: the column is absent
+    // on a pre-1.4.0 client and null on every non-split scan, and neither is an
+    // error — only a present, non-empty list means "this is a split init".
+    split_tokens: (() => {
+      const raw = params.split_tokens;
+      if (raw == null) return null;
+      const iter: Iterable<unknown> = Array.isArray(raw)
+        ? raw
+        : typeof (raw as { [Symbol.iterator]?: unknown })[Symbol.iterator] === "function"
+          ? (raw as Iterable<unknown>)
+          : [];
+      const tokens = [...iter].filter((e) => e != null).map((e) => toUint8Array(e));
+      return tokens.length > 0 ? tokens : null;
+    })(),
     execution_id: params.execution_id
       ? toUint8Array(params.execution_id)
       : null,

package/src/protocol/serializers/splits.ts

@@ -0,0 +1,219 @@
+// Copyright 2025, 2026 Query Farm LLC - https://query.farm
+// TableFunctionPlanRequest / PlanResponse / ScanSplit wire serialization.
+//
+// Field ORDER and nullability are wire-significant: they mirror vgi/protocol.py
+// exactly, and the extension rejects a response whose schema disagrees.
+
+import { type VgiSchema, schema, field, binary, bool, int64, list } from "../../arrow/index.js";
+import {
+  ScanSplitSchema,
+  TableFunctionPlanResultSchema,
+} from "../../generated/vgi-protocol-schemas.js";
+import type { BindRequest } from "../types.js";
+import { deserializeBatch, batchToScalarDict } from "../../util/arrow/index.js";
+import { toUint8Array } from "./shared.js";
+import { deserializeBindRequest } from "./bind.js";
+
+/**
+ * One named, independently redeemable unit of scan work.
+ *
+ * A worker sets `payload` and nothing else; the framework stamps `token` from
+ * it, so an author cannot forget the consistency anchor or mis-bind the
+ * fingerprint. The client sends the TOKEN back, never the raw payload.
+ */
+export interface ScanSplit {
+  /** The worker's own bytes NAMING this unit of work. */
+  payload: Uint8Array;
+  /** Framework-stamped envelope. Set by the framework, never by a worker. */
+  token?: Uint8Array;
+  estimatedRows?: number | bigint | null;
+  /** True if `estimatedRows` is exact — unlocks COUNT(*) from statistics. */
+  rowsExact?: boolean;
+  /**
+   * Load-bearing for engines that bin-pack (DataFusion weight, Trino
+   * SplitWeight). Omitting it degrades them to round-robin by count; a greedily
+   * claiming client needs no cost model at all.
+   */
+  estimatedBytes?: number | bigint | null;
+  /** 2-row (min, max) batch in the existing `vgi_partition_values` encoding. */
+  partitionBounds?: Uint8Array | null;
+  columnStatistics?: Uint8Array | null;
+  locationIds?: number[] | null;
+  startPosition?: Uint8Array | null;
+  /** Absent means UNBOUNDED — a shard read forever. */
+  endPosition?: Uint8Array | null;
+}
+
+/** What a table function's `plan` hook returns. */
+export interface PlanResult {
+  /**
+   * One entry per unit of work. EMPTY is legal and means "no work": a
+   * fully-pruned scan reaches it, and the client must produce an empty result
+   * rather than an error.
+   */
+  splits: ScanSplit[];
+  /**
+   * Continued enumeration. More than one MUST partition the remaining
+   * enumeration disjointly and exhaustively. No client checks this: a dedup was
+   * tried and removed (it needed a set holding a copy of every token, it compared
+   * token bytes so it could never work on a keyed worker where each mint uses a
+   * fresh nonce, and the most a client can do with a duplicate is refuse anyway).
+   * Violating it returns DUPLICATE ROWS, silently.
+   */
+  nextCursors?: Uint8Array[];
+  /** NORMATIVE cap on redemption concurrency, not advisory. */
+  maxWorkers?: number | null;
+  estimatedTotalSplits?: number | bigint | null;
+  estimatedTotalRows?: number | bigint | null;
+  estimatedTotalBytes?: number | bigint | null;
+  /**
+   * The counter a stale token is detected against, and the anchor the framework
+   * stamps into every token in this plan.
+   */
+  catalogVersion?: number | bigint | null;
+  /**
+   * `catalog` (the default) or `transaction`. A transaction-scoped plan is not
+   * cacheable and is not redeemable after commit or rollback.
+   */
+  scope?: "catalog" | "transaction";
+  cacheMaxAgeSeconds?: number | null;
+}
+
+/** The request a client sends to divide a scan. */
+export interface TableFunctionPlanRequest {
+  bind_call: BindRequest;
+  bind_opaque_data: Uint8Array | null;
+  projection_ids: number[] | null;
+  pushdown_filters: Uint8Array | null;
+  /** The primary sizing lever: every engine is byte-driven. */
+  target_split_bytes: bigint | null;
+  /**
+   * The parallelism FLOOR — a small but expensive table still needs one reader
+   * per thread, which a byte target alone would not give it.
+   */
+  min_splits: bigint | null;
+  /**
+   * A place in the ENUMERATION of splits — NOT a place in the data. A cursor
+   * lives for one plan call; a position is checkpointed and must survive
+   * restarts, upgrades and key rotation.
+   */
+  cursor: Uint8Array | null;
+}
+
+/**
+ * The ScanSplit shape, from codegen rather than hand-written.
+ *
+ * It used to be spelled out here, and hand-written is how four SDKs ended up
+ * disagreeing about which of these columns were binary and which were
+ * large_binary — a disagreement the client surfaced as "the worker bypassed the
+ * framework", on every scan.
+ */
+export const SCAN_SPLIT_SCHEMA: VgiSchema = ScanSplitSchema as VgiSchema;
+
+/**
+ * The PlanResponse shape, taken from codegen rather than hand-written: it is
+ * regenerated from the canonical protocol, so it cannot drift.
+ */
+export const PLAN_RESPONSE_SCHEMA: VgiSchema = TableFunctionPlanResultSchema as VgiSchema;
+
+/**
+ * Decode a plan request.
+ *
+ * A client sends only the sizing inputs it actually has — the DuckDB extension
+ * sends seven columns, because it has no basis to invent a byte target it did
+ * not set — so every field but `bind_call` is read defensively.
+ */
+export function deserializePlanRequest(params: Record<string, any>): TableFunctionPlanRequest {
+  const bindCallBytes = toUint8Array(params.bind_call);
+  const bindCall = deserializeBindRequest(batchToScalarDict(deserializeBatch(bindCallBytes)));
+
+  let projectionIds: number[] | null = null;
+  if (params.projection_ids != null) {
+    const raw = params.projection_ids;
+    if (Array.isArray(raw)) projectionIds = raw.map(Number);
+    else if (raw && typeof raw[Symbol.iterator] === "function") projectionIds = [...raw].map(Number);
+  }
+
+  const asBigInt = (v: unknown): bigint | null =>
+    v == null ? null : typeof v === "bigint" ? v : BigInt(Number(v));
+
+  return {
+    bind_call: bindCall,
+    bind_opaque_data: params.bind_opaque_data ? toUint8Array(params.bind_opaque_data) : null,
+    projection_ids: projectionIds,
+    pushdown_filters: params.pushdown_filters ? toUint8Array(params.pushdown_filters) : null,
+    target_split_bytes: asBigInt(params.target_split_bytes),
+    min_splits: asBigInt(params.min_splits),
+    cursor: params.cursor ? toUint8Array(params.cursor) : null,
+  };
+}
+
+/**
+ * The raw bind fields the split-token fingerprint is derived from.
+ *
+ * Read from the UNDECODED bind_call dict rather than the parsed BindRequest,
+ * because the fingerprint must be reproducible byte-for-byte between the plan
+ * that minted a token and the init that redeems it. A decoded-then-re-encoded
+ * argument list is not guaranteed to round-trip to identical bytes; the wire
+ * bytes are.
+ */
+export function fingerprintInputs(bindCallBytes: Uint8Array): {
+  schemaName: string;
+  functionName: string;
+  args: Uint8Array;
+  settings: Uint8Array;
+} {
+  const dict = batchToScalarDict(deserializeBatch(bindCallBytes));
+  return {
+    schemaName: dict.schema_name == null ? "" : String(dict.schema_name),
+    functionName: String(dict.function_name ?? ""),
+    args: dict.arguments ? toUint8Array(dict.arguments) : new Uint8Array(0),
+    settings: dict.settings ? toUint8Array(dict.settings) : new Uint8Array(0),
+  };
+}
+
+/** Build the row dict for one `ScanSplit`. */
+export function scanSplitRow(split: ScanSplit, token: Uint8Array): Record<string, any> {
+  const asBigInt = (v: number | bigint | null | undefined): bigint | null =>
+    v == null ? null : typeof v === "bigint" ? v : BigInt(v);
+  return {
+    // Cleared, not forwarded. The payload is sealed INTO the token, and shipping
+    // the plaintext in the field beside the ciphertext made the seal decorative.
+    // No client reads it — the C++ side pulls `token` alone — and redemption
+    // recovers the payload from inside the envelope.
+    payload: new Uint8Array(0),
+    token,
+    estimated_rows: asBigInt(split.estimatedRows),
+    rows_exact: split.rowsExact === true,
+    estimated_bytes: asBigInt(split.estimatedBytes),
+    partition_bounds: split.partitionBounds ?? null,
+    column_statistics: split.columnStatistics ?? null,
+    location_ids: split.locationIds ?? null,
+    start_position: split.startPosition ?? null,
+    end_position: split.endPosition ?? null,
+  };
+}
+
+/** Build the row dict for a `PlanResponse`, given already-serialized splits. */
+export function planResponseRow(result: PlanResult, splitBlobs: Uint8Array[]): Record<string, any> {
+  const asBigInt = (v: number | bigint | null | undefined): bigint | null =>
+    v == null ? null : typeof v === "bigint" ? v : BigInt(v);
+  return {
+    splits: splitBlobs,
+    next_cursors: result.nextCursors ?? [],
+    execution_id: null,
+    init_opaque_data: new Uint8Array(0),
+    max_workers: asBigInt(result.maxWorkers),
+    estimated_total_splits: asBigInt(result.estimatedTotalSplits),
+    estimated_total_rows: asBigInt(result.estimatedTotalRows),
+    estimated_total_bytes: asBigInt(result.estimatedTotalBytes),
+    catalog_version: asBigInt(result.catalogVersion),
+    scope: result.scope ?? "catalog",
+    locations: null,
+    partitioning: [],
+    sort_order: [],
+    cache_max_age_seconds: asBigInt(result.cacheMaxAgeSeconds),
+    start_position: new Uint8Array(0),
+    end_position: new Uint8Array(0),
+  };
+}

package/src/protocol/state-serializer.ts

@@ -14,7 +14,9 @@ import {
   bool,
   float64,
   utf8,
+  list as makeList,
   nullType,
+  typeSignature,
   isStruct,
   isBinary,
   serializeBatch,
@@ -40,6 +42,58 @@ export const EXCHANGE_STATE_SCHEMA = makeSchema([
 const TEXT_ENCODER = new TextEncoder();
 const TEXT_DECODER = new TextDecoder();
 
+/**
+ * Child field name for an inferred list. Arrow matches list children by
+ * position, not name, so this is cosmetic — but keeping it at Arrow's
+ * conventional `item` makes a dumped schema read the way people expect.
+ */
+const LIST_CHILD_NAME = "item";
+
+/**
+ * Element type for an array with nothing to infer from — `[]`, or an array
+ * whose every element is null.
+ *
+ * Deliberately NOT `nullType()`, which is the semantically honest choice:
+ * flechette's column builder rejects a Null-typed list child ("Unsupported
+ * data type: Null"), so a `[]` in userState would round-trip on arrow-js and
+ * throw on flechette. Utf8 builds on both backends and, with no non-null
+ * element to carry, is indistinguishable on the way back out: `[]` returns
+ * `[]` and `[null, null]` returns `[null, null]`.
+ */
+const emptyListElementType = (): VgiDataType => utf8();
+
+/**
+ * Infer the single Arrow element type an array's contents share.
+ *
+ * Arrow lists are homogeneous, so a genuinely mixed array (`[1, "a"]`,
+ * or structs with differing field sets) has no faithful representation and is
+ * rejected rather than silently coerced. Nulls are skipped — they are
+ * representable at any element type — and an array with no non-null element
+ * falls back to {@link emptyListElementType}.
+ */
+function inferElementType(values: any[]): VgiDataType {
+  let elementType: VgiDataType | null = null;
+  let elementSig = "";
+  for (const v of values) {
+    if (v === null || v === undefined) continue;
+    const t = inferFieldType(v);
+    const sig = typeSignature(t);
+    if (elementType === null) {
+      elementType = t;
+      elementSig = sig;
+      continue;
+    }
+    if (sig !== elementSig) {
+      throw new Error(
+        `inferFieldType: arrays in userState must be homogeneous; ` +
+          `found both '${elementSig}' and '${sig}' in the same array ` +
+          `(split the values into separate fields, or normalize them to one type)`,
+      );
+    }
+  }
+  return elementType ?? emptyListElementType();
+}
+
 /** Infer an Arrow DataType from a JS value for user state serialization. */
 export function inferFieldType(value: any): VgiDataType {
   if (value === null || value === undefined) return nullType();
@@ -52,7 +106,7 @@ export function inferFieldType(value: any): VgiDataType {
       if (value instanceof Uint8Array || value instanceof ArrayBuffer) return binary();
       if (ArrayBuffer.isView(value)) return binary();
       if (Array.isArray(value)) {
-        throw new Error(`inferFieldType: arrays are not supported in userState (convert to a serializable form first)`);
+        return makeList(field(LIST_CHILD_NAME, inferElementType(value), true));
       }
       if (value instanceof Map) {
         throw new Error(`inferFieldType: Map is not supported in userState (use a plain object instead)`);

package/src/protocol/types.ts

@@ -125,6 +125,33 @@ export interface InitRequest {
    * bytes the worker's combine() chose.
    */
   finalize_state_id: Uint8Array | null;
+  /**
+   * The framework-stamped split envelopes this init redeems, or null when this
+   * is not a split init.
+   *
+   * A LIST because an engine whose partition count IS its concurrency bin-packs
+   * at planning time and reads a whole group per partition; DuckDB always sends
+   * exactly one. A worker handed several concatenates them in the order given.
+   */
+  split_tokens: Uint8Array[] | null;
+  /**
+   * The VERIFIED payloads behind `split_tokens` — the worker's own bytes, with
+   * the envelope already opened and stripped.
+   *
+   * Populated by the protocol handler, which is what holds the signing key, so
+   * an unverified token never reaches a function. Null when this is not a split
+   * init.
+   */
+  split_payloads?: Uint8Array[] | null;
+  /**
+   * A plain fetch limit, when the engine can supply one.
+   *
+   * Always null from DuckDB — `TableFunctionInitInput` carries no limit, and
+   * the Top-N `order_by_limit` means something different ("top K by that
+   * column"). DataFusion supplies a real value via `TableProvider::scan(limit)`.
+   * Over-producing is legal; the engine re-applies the limit above the scan.
+   */
+  row_limit: number | null;
   execution_id: Uint8Array | null;
   init_opaque_data: Uint8Array | null;
   /**