@query-farm/vgi 0.18.0 → 0.19.0

package/src/functions/table-in-out.ts

@@ -3,7 +3,7 @@
 // Two-phase: INPUT phase receives and transforms batches,
 // FINALIZE phase emits final results.
 
-import { type VgiSchema, schema, type VgiField, type VgiDataType, type VgiBatch, nullType } from "../arrow/index.js";
+import { type VgiSchema, schema, type VgiField, type VgiDataType, type VgiBatch, nullType, withBatchMetadata } from "../arrow/index.js";
 import type { OutputCollector } from "@query-farm/vgi-rpc";
 import { DEFAULT_MAX_WORKERS, TableInOutPhase } from "../types.js";
 import type {
@@ -20,6 +20,7 @@ import type {
 } from "./types.js";
 import type { ArgumentSpec } from "../arguments/argument-spec.js";
 import { batchToScalarDict, batchToSecretDict, projectSchema, projectBatch, emptyBatch } from "../util/arrow/index.js";
+import { CACHE_IF_MODIFIED_SINCE_KEY, CACHE_IF_NONE_MATCH_KEY } from "../cache-control.js";
 import {
   buildJoinKeysLookup,
   deserializeFilters,
@@ -51,6 +52,28 @@ export interface TableInOutProcessParams<TArgs = Record<string, any>> {
   pushdownFilters?: PushdownFilters;
   /** Shared storage for cross-phase and cross-worker data (SQLite-backed). */
   storage: BoundStorage;
+  /**
+   * Stable client-minted id for this streaming table-in-out substream.
+   * Present (identical across init / every process() / finalize) when the
+   * client fanned this function out across per-substream workers; use it to
+   * key per-substream accumulated state in shared storage so a finalize()
+   * that lands on a different HTTP backend than the process() calls still
+   * finds it. `null`/`undefined` for the serial path or an old client.
+   * Mirrors vgi-python's `ProcessParams.substream_id`.
+   */
+  substreamId?: Uint8Array | null;
+  /**
+   * Conditional-revalidation validator (exchange-mode result cache): the
+   * client holds a stale cached result for THIS input unit and asks the
+   * worker to confirm freshness cheaply. When set, process() may answer with
+   * a 0-row `cacheControlMetadata({ notModified: true, ... })` batch instead
+   * of recomputing. Rides the input batch's custom metadata (attached by the
+   * C++ WriteInputBatch). Undefined on a normal call.
+   */
+  ifNoneMatch?: string;
+  /** RFC 3339 Last-Modified validator for conditional revalidation.
+   *  Companion to {@link ifNoneMatch}. Undefined on a normal call. */
+  ifModifiedSince?: string;
 }
 
 // ============================================================================
@@ -273,6 +296,7 @@ export function defineTableInOutFunction<
         secrets,
         pushdownFilters,
         storage: boundStorage,
+        substreamId: request.substream_id ?? null,
       };
 
       const phase = request.phase;
@@ -354,9 +378,21 @@ export function defineTableInOutFunction<
           input: VgiBatch,
           out: OutputCollector
         ) => {
-          const wrappedOut = (config.autoApplyFilters && pushdownFilters)
-            ? new FilteringOutputCollector(out, pushdownFilters) as unknown as OutputCollector
-            : out;
+          // Surface conditional-revalidation validators off the input batch's
+          // custom metadata (attached by the C++ WriteInputBatch), so process()
+          // can answer a 0-row `notModified` batch instead of recomputing.
+          applyRevalidationValidators(eState.processParams, input);
+          // Bake emit metadata onto the batch (0-row HTTP survival), then
+          // reconcile emitted batches to the (possibly projected) output schema
+          // by name — a process() may emit its full declared schema and let the
+          // framework project (mirrors Python's OutputCollector.emit reconcile).
+          let wrappedOut: OutputCollector = makeMetadataBakingCollector(out);
+          if (projIds) {
+            wrappedOut = makeSchemaReconcilingCollector(wrappedOut, outputSchema);
+          }
+          if (config.autoApplyFilters && pushdownFilters) {
+            wrappedOut = new FilteringOutputCollector(wrappedOut, pushdownFilters) as unknown as OutputCollector;
+          }
           await processFn(eState.processParams, eState.state, input, wrappedOut);
 
           // Auto-persist accumulated state for FINALIZE recovery (matches Python/Go pattern)
@@ -371,3 +407,443 @@ export function defineTableInOutFunction<
     },
   };
 }
+
+// ============================================================================
+// Shared INPUT-phase helpers
+// ============================================================================
+
+/**
+ * Surface conditional-revalidation validators (exchange-mode result cache)
+ * from an input batch's custom metadata onto the process params. The C++
+ * WriteInputBatch attaches `vgi.cache.if_none_match` / `if_modified_since`
+ * when it holds a stale-but-revalidatable cached result for this input unit.
+ * Cleared (undefined) when absent so a later batch on the same stream doesn't
+ * see a stale validator.
+ */
+function applyRevalidationValidators(
+  params: { ifNoneMatch?: string; ifModifiedSince?: string },
+  input: VgiBatch,
+): void {
+  const md: Map<string, string> | undefined = (input as any)?.metadata;
+  params.ifNoneMatch = md?.get(CACHE_IF_NONE_MATCH_KEY);
+  params.ifModifiedSince = md?.get(CACHE_IF_MODIFIED_SINCE_KEY);
+}
+
+/**
+ * Wrap an OutputCollector so a pre-built batch emitted WITH metadata carries
+ * that metadata on the batch object itself (merged over any metadata the
+ * batch already had), in addition to the separate emit argument.
+ *
+ * Load-bearing for the HTTP transport: vgi-rpc's HTTP exchange dispatch
+ * merges the separate emit-metadata argument into the response batch only
+ * when `batch.numRows > 0` — a 0-row emit (e.g. a conditional-revalidation
+ * `notModified` reply) would silently lose its `vgi.cache.*` keys. A
+ * batch-carried map survives that path on both transports.
+ */
+function makeMetadataBakingCollector(inner: OutputCollector): OutputCollector {
+  return new Proxy(inner, {
+    get(target, prop, receiver) {
+      if (prop === "emit") {
+        return function (
+          batchOrColumns: VgiBatch | Record<string, any[]>,
+          metadata?: Map<string, string>,
+        ) {
+          if (
+            batchOrColumns &&
+            typeof (batchOrColumns as any).getChild === "function" &&
+            metadata &&
+            metadata.size > 0
+          ) {
+            const batch = batchOrColumns as VgiBatch;
+            const merged = new Map<string, string>((batch as any).metadata ?? []);
+            for (const [k, v] of metadata) merged.set(k, v);
+            return (target as any).emit(withBatchMetadata(batch, merged), merged);
+          }
+          return (target as any).emit(batchOrColumns, metadata);
+        };
+      }
+      const v = Reflect.get(target, prop, receiver);
+      return typeof v === "function" ? v.bind(target) : v;
+    },
+  });
+}
+
+/**
+ * Wrap an OutputCollector so each emitted pre-built batch is reconciled to the
+ * (possibly projected) target output schema by NAME before forwarding — a
+ * process() may emit its full declared schema and let the framework project.
+ * Mirrors vgi-rpc-python's `OutputCollector.emit` select step. A target column
+ * missing from the emitted batch throws a clear error.
+ */
+function makeSchemaReconcilingCollector(
+  inner: OutputCollector,
+  targetSchema: VgiSchema,
+): OutputCollector {
+  function reconcile(batch: VgiBatch): VgiBatch {
+    const batchFields = batch.schema.fields;
+    if (
+      batchFields.length === targetSchema.fields.length &&
+      targetSchema.fields.every((f, i) => batchFields[i].name === f.name)
+    ) {
+      return batch;
+    }
+    const ids: number[] = [];
+    for (const f of targetSchema.fields) {
+      const idx = batchFields.findIndex((bf) => bf.name === f.name);
+      if (idx < 0) {
+        throw new Error(
+          `emitted batch is missing projected output column '${f.name}' ` +
+            `(emitted: [${batchFields.map((bf) => bf.name).join(", ")}])`,
+        );
+      }
+      ids.push(idx);
+    }
+    return projectBatch(ids, batch) as unknown as VgiBatch;
+  }
+  return new Proxy(inner, {
+    get(target, prop, receiver) {
+      if (prop === "emit") {
+        return function (
+          batchOrColumns: VgiBatch | Record<string, any[]>,
+          metadata?: Map<string, string>,
+        ) {
+          if (batchOrColumns && typeof (batchOrColumns as any).getChild === "function") {
+            return (target as any).emit(reconcile(batchOrColumns as VgiBatch), metadata);
+          }
+          return (target as any).emit(batchOrColumns, metadata);
+        };
+      }
+      const v = Reflect.get(target, prop, receiver);
+      return typeof v === "function" ? v.bind(target) : v;
+    },
+  });
+}
+
+// ============================================================================
+// Per-output-row provenance (batched correlated LATERAL)
+// ============================================================================
+
+/** Metadata key carrying per-output-row provenance: base64 of a raw
+ *  little-endian int32[] mapping each output row to the input row that
+ *  produced it. Shared by string with the C++ extension and vgi-python. */
+export const PARENT_ROW_METADATA_KEY = "vgi_rpc.parent_row#b64";
+
+/**
+ * Fold per-output-row provenance into an emit metadata map.
+ *
+ * Used by the **batched correlated LATERAL** operator (blended
+ * RowTransformFunction under `FROM t, f(t.x)` / `LATERAL`): the C++ extension
+ * ships a whole input chunk to the worker in ONE exchange and reads ONE
+ * output batch, then maps each output row back to the input row that produced
+ * it via this array — so a 1->N fan-out or 1->0 filter can be batched instead
+ * of driven row-by-row.
+ *
+ * `parentRows[i]` is the 0-based index (into the input batch) of the row that
+ * produced output row `i`. Encoded as a raw little-endian `int32` array (NOT
+ * Arrow IPC), base64-encoded, under `vgi_rpc.parent_row#b64`. Absent metadata
+ * means an identity 1->1 map (the common case: the extension assumes it, and
+ * requires output rows == input rows).
+ *
+ * Contract: `parentRows.length` MUST equal the emitted batch's row count (a
+ * mismatch is a worker bug that would corrupt the stamping). Values are
+ * range-checked against the input width on the C++ side. Mirrors vgi-python's
+ * `_merge_parent_rows` / `out.emit(..., parent_rows=[...])`.
+ */
+export function parentRowsMetadata(
+  parentRows: number[],
+  outputRows: number,
+  extra?: Map<string, string>,
+): Map<string, string> {
+  if (parentRows.length !== outputRows) {
+    throw new Error(
+      `parentRowsMetadata: length ${parentRows.length} != output rows ${outputRows}; ` +
+        `parentRows must carry exactly one input-row index per emitted output row`,
+    );
+  }
+  const md = new Map<string, string>(extra ?? []);
+  if (outputRows === 0) {
+    // Nothing to map; skip the base64+pack for an empty emit.
+    return md;
+  }
+  const raw = new Uint8Array(parentRows.length * 4);
+  const dv = new DataView(raw.buffer);
+  for (let i = 0; i < parentRows.length; i++) {
+    dv.setInt32(i * 4, parentRows[i], true);
+  }
+  md.set(PARENT_ROW_METADATA_KEY, base64Encode(raw));
+  return md;
+}
+
+function base64Encode(bytes: Uint8Array): string {
+  if (typeof (globalThis as any).Buffer !== "undefined") {
+    return (globalThis as any).Buffer.from(bytes).toString("base64");
+  }
+  let bin = "";
+  for (let i = 0; i < bytes.length; i++) bin += String.fromCharCode(bytes[i]);
+  return (globalThis as any).btoa(bin);
+}
+
+// ============================================================================
+// Blended ("UNNEST-style") row-transform functions
+// ============================================================================
+
+/**
+ * Process params for a blended row-transform function. `args` carries only the
+ * NAMED (bind-time scalar) options — the positional args are the per-row input
+ * columns, read from `batch` in process() (by declared name for fixed args,
+ * positionally for varargs).
+ */
+export type RowTransformProcessParams<TArgs = Record<string, any>> =
+  TableInOutProcessParams<TArgs>;
+
+export interface RowTransformConfig<TArgs = Record<string, any>> {
+  name: string;
+  description?: string;
+  /**
+   * Positional args = the per-row INPUT COLUMNS (real typed args on the wire,
+   * no synthetic TABLE placeholder). Read from `batch` by declared name in
+   * process(); NOT surfaced on `params.args`.
+   */
+  args?: Record<string, VgiDataType>;
+  /**
+   * Trailing VARARGS input columns: the per-row input is N columns of the
+   * declared type. A varargs blended function has no per-column declared
+   * names (the C++ bind names them col0..colN-1), so process() reads the
+   * columns POSITIONALLY off `batch`.
+   */
+  varargs?: { name: string; type: VgiDataType; doc?: string };
+  /** Named (string-position) args stay bind-time scalars on `params.args`. */
+  namedArgs?: Record<string, VgiDataType>;
+  argDefaults?: Record<string, any>;
+  /** Per-argument descriptions keyed by arg name (surfaced as `vgi_doc`). */
+  argDocs?: Record<string, string>;
+  /** Bind: return the output schema. The input schema (the declared per-row
+   *  columns, typed by the C++ bind) is on `params.bindCall.input_schema`. */
+  onBind: (params: TableInOutBindParams<TArgs>) =>
+    | { outputSchema: VgiSchema; opaqueData?: Uint8Array }
+    | Promise<{ outputSchema: VgiSchema; opaqueData?: Uint8Array }>;
+  /**
+   * Per-row map: transform one input batch, emit exactly one output batch via
+   * `out`. 1->1, 1->N (with {@link parentRowsMetadata} provenance), and 1->0
+   * (a 0-row emit) all work. There is NO finalize — a blended function is a
+   * per-row map (DuckDB forbids FinalExecute under correlated LATERAL, one of
+   * the call shapes blended must serve). Accumulating functions use a classic
+   * TableInput table-in-out or a TableBufferingFunction.
+   */
+  process: (
+    params: RowTransformProcessParams<TArgs>,
+    batch: VgiBatch,
+    out: OutputCollector,
+  ) => void | Promise<void>;
+  // Metadata
+  projectionPushdown?: boolean;
+  filterPushdown?: boolean;
+  autoApplyFilters?: boolean;
+  stability?: FunctionStability;
+  examples?: FunctionExample[];
+  categories?: string[];
+  tags?: Record<string, string>;
+  maxWorkers?: number;
+  requiredSettings?: string[];
+  requiredSecrets?: string[];
+}
+
+/**
+ * Define a blended ("UNNEST-style") table-in-out function: its positional args
+ * ARE its per-row input columns, so ONE registration serves every call shape —
+ * `f(52, 13)` (literal -> one input row), `FROM t, f(t.x, t.y)` (columns ->
+ * streaming), and `LATERAL f(t.x, t.y)`. Mirrors vgi-python's
+ * `RowTransformFunction` (Phase B).
+ *
+ * Registers as a TABLE function with `FunctionInfo.input_from_args = true`;
+ * the worker's overload resolution matches blended overloads by INPUT-COLUMN
+ * count (the positional args are not on the wire). Map-shaped, no finalize.
+ */
+export function defineRowTransformFunction<
+  TArgs = Record<string, any>,
+>(config: RowTransformConfig<TArgs>): VgiFunction {
+  // Foot-gun guard (mirrors Python's resolve_metadata check): a blended
+  // function needs at least one positional Arg — its per-row input column.
+  const numPositional = Object.keys(config.args ?? {}).length + (config.varargs ? 1 : 0);
+  if (numPositional === 0) {
+    throw new Error(
+      `${config.name}: a blended row-transform function needs at least one ` +
+        `positional arg (its per-row input column); found none.`,
+    );
+  }
+
+  // Positional args are the per-row input columns — REAL typed args on the
+  // wire (no TableInput placeholder). Named args follow.
+  const specs: ArgumentSpec[] = [];
+  let posIdx = 0;
+  if (config.args) {
+    for (const [name, type] of Object.entries(config.args)) {
+      specs.push({
+        name,
+        position: posIdx++,
+        arrowType: type,
+        doc: config.argDocs?.[name],
+      });
+    }
+  }
+  if (config.varargs) {
+    specs.push({
+      name: config.varargs.name,
+      position: posIdx++,
+      arrowType: config.varargs.type,
+      isVarargs: true,
+      doc: config.varargs.doc ?? config.argDocs?.[config.varargs.name],
+    });
+  }
+  if (config.namedArgs) {
+    for (const [name, type] of Object.entries(config.namedArgs)) {
+      specs.push({
+        name,
+        position: name,
+        arrowType: type,
+        doc: config.argDocs?.[name],
+      });
+    }
+  }
+
+  const meta: FunctionMeta = {
+    name: config.name,
+    description: config.description,
+    stability: config.stability,
+    projectionPushdown: config.projectionPushdown,
+    filterPushdown: config.filterPushdown,
+    autoApplyFilters: config.autoApplyFilters,
+    examples: config.examples,
+    categories: config.categories,
+    tags: config.tags,
+    maxWorkers: config.maxWorkers,
+    requiredSettings: config.requiredSettings,
+    requiredSecrets: config.requiredSecrets,
+    // Map-shaped, per-row: a blended function never has a finalize.
+    hasFinalize: false,
+    inputFromArgs: true,
+  };
+
+  // Only NAMED args are on the wire — positional args are the input columns,
+  // absent from the wire arguments in every call shape.
+  function extractArgs(request: BindRequest): TArgs {
+    const args: Record<string, any> = {};
+    for (const spec of specs) {
+      if (typeof spec.position !== "string") continue;
+      const defaultVal =
+        config.argDefaults?.[spec.name] !== undefined
+          ? config.argDefaults[spec.name]
+          : undefined;
+      let val = request.arguments.get(spec.position, defaultVal ?? null);
+      if (typeof val === "bigint") val = Number(val);
+      args[spec.name] = val;
+    }
+    return args as TArgs;
+  }
+
+  return {
+    kind: "table_in_out",
+    meta,
+    argumentSpecs: specs,
+
+    async bind(request: BindRequest): Promise<BindResponse> {
+      const args = extractArgs(request);
+      const settings = batchToScalarDict(request.settings);
+      const secrets = batchToSecretDict(request.secrets);
+      const result = await config.onBind({
+        args,
+        bindCall: request,
+        settings,
+        secrets,
+      });
+      return {
+        output_schema: result.outputSchema,
+        opaque_data: result.opaqueData ?? null,
+      };
+    },
+
+    async globalInit(request: InitRequest): Promise<GlobalInitResponse> {
+      if (request.execution_id) {
+        return {
+          max_workers: DEFAULT_MAX_WORKERS,
+          execution_id: request.execution_id,
+          opaque_data: null,
+        };
+      }
+      const executionId = new Uint8Array(16);
+      crypto.getRandomValues(executionId);
+      return {
+        max_workers: config.maxWorkers ?? 1,
+        execution_id: executionId,
+        opaque_data: null,
+      };
+    },
+
+    createStreamHandlers(
+      request: InitRequest,
+      response: GlobalInitResponse,
+      _accumulatedState?: any,
+    ): StreamHandlers {
+      const args = extractArgs(request.bind_call);
+      const settings = batchToScalarDict(request.bind_call.settings);
+      const secrets = batchToSecretDict(request.bind_call.secrets);
+
+      const projIds = request.projection_ids && meta.projectionPushdown
+        ? request.projection_ids
+        : null;
+      const outputSchema = projIds
+        ? projectSchema(projIds, request.output_schema)
+        : request.output_schema;
+
+      const joinKeysLookup = buildJoinKeysLookup(request.join_keys);
+      const pushdownFilters = request.pushdown_filters
+        ? deserializeFilters(request.pushdown_filters, joinKeysLookup)
+        : undefined;
+
+      const boundStorage = new BoundStorage(
+        defaultStorage,
+        response.execution_id ?? new Uint8Array(16),
+      );
+
+      const processParams: RowTransformProcessParams<TArgs> = {
+        args,
+        initCall: request,
+        initResponse: response,
+        outputSchema,
+        settings,
+        secrets,
+        pushdownFilters,
+        storage: boundStorage,
+        substreamId: request.substream_id ?? null,
+      };
+
+      if (request.phase === TableInOutPhase.FINALIZE) {
+        throw new Error(
+          `${config.name}: a blended row-transform function has no FINALIZE ` +
+            `phase (it is a per-row map; has_finalize is advertised false).`,
+        );
+      }
+
+      return {
+        outputSchema,
+        inputSchema: request.bind_call.input_schema ?? undefined,
+        exchangeInit: () => ({ state: null, processParams }),
+        exchangeFn: async (
+          eState: { state: null; processParams: RowTransformProcessParams<TArgs> },
+          input: VgiBatch,
+          out: OutputCollector,
+        ) => {
+          applyRevalidationValidators(eState.processParams, input);
+          let wrappedOut: OutputCollector = makeMetadataBakingCollector(out);
+          if (projIds) {
+            wrappedOut = makeSchemaReconcilingCollector(wrappedOut, outputSchema);
+          }
+          if (config.autoApplyFilters && pushdownFilters) {
+            wrappedOut = new FilteringOutputCollector(wrappedOut, pushdownFilters) as unknown as OutputCollector;
+          }
+          await config.process(eState.processParams, input, wrappedOut);
+        },
+      };
+    },
+  };
+}

package/src/functions/types.ts

@@ -68,6 +68,18 @@ export interface FunctionMeta {
    * otherwise calling FinalExecute is unsupported and crashes the C++ side.
    */
   hasFinalize?: boolean;
+  /**
+   * Blended ("UNNEST-style") table-in-out: the function's positional args ARE
+   * its per-row input columns (real typed args, no synthetic TABLE
+   * placeholder), so ONE registration serves f(52,13) (literal -> 1 input
+   * row), FROM t, f(t.x, t.y) (columns -> streaming), and LATERAL f(t.x,t.y).
+   * Set by {@link defineRowTransformFunction}; surfaces as
+   * FunctionInfo.input_from_args. The C++ extension reads it to enter the
+   * in-out registration branch with real-typed args and drive the literal
+   * single-row scan-mode. Mirrors vgi-python's RowTransformFunction /
+   * ResolvedMetadata.input_from_args.
+   */
+  inputFromArgs?: boolean;
   /**
    * table_buffering: force ParallelSink=false in the C++ operator
    * (single-thread, source-ordered ingest).

package/src/generated/vgi-client.ts

@@ -2,7 +2,7 @@
 // GENERATED by vgi.codegen.ts_client. DO NOT EDIT BY HAND.
 //
 // Generator: vgi-gen-ts-client v1
-// Content hash: f3ac87321a0c
+// Content hash: 226553bd666c
 //
 // To regenerate:
 //   uv run --project ~/Development/vgi-python vgi-gen-ts-client \
@@ -138,6 +138,7 @@ export interface FunctionInfo {
   source_order_dependent: boolean;
   sink_order_dependent: boolean;
   requires_input_batch_index: boolean;
+  input_from_args: boolean;
   required_settings: string[];
   required_secrets: SecretLookupEntry[];
 }
@@ -471,6 +472,7 @@ export interface InitRequest {
   finalize_state_id?: Uint8Array | null;
   execution_id?: Uint8Array | null;
   init_opaque_data?: Uint8Array | null;
+  substream_id?: Uint8Array | null;
   order_by_column_name?: string | null;
   order_by_direction?: OrderByDirection | null;
   order_by_null_order?: OrderByNullOrder | null;

package/src/generated/vgi-protocol-schemas.ts

@@ -2,7 +2,7 @@
 // GENERATED by vgi.codegen.ts_schemas. DO NOT EDIT BY HAND.
 //
 // Generator: vgi-gen-ts-schemas v2
-// Content hash: b3c99cc5fac8
+// Content hash: 214d149e8f9c
 //
 // To regenerate:
 //   uv run --project ~/Development/vgi-python vgi-gen-ts-schemas \
@@ -114,6 +114,7 @@ export const FunctionInfoSchema = schema([
   field("source_order_dependent", bool(), false),
   field("sink_order_dependent", bool(), false),
   field("requires_input_batch_index", bool(), false),
+  field("input_from_args", bool(), false),
   field("required_settings", list(field("item", utf8(), true)), false),
   field("required_secrets", list(field("item", struct([field("secret_type", utf8(), false), field("scope", utf8(), true), field("secret_name", utf8(), true)]), true)), false),
 ]);

package/src/index.core.ts

@@ -39,7 +39,17 @@ export {
   type AggregateUpdateParams,
   type AggregateFinalizeParams,
 } from "./functions/aggregate.js";
-export { defineTableInOutFunction, type TableInOutConfig, type TableInOutBindParams, type TableInOutProcessParams } from "./functions/table-in-out.js";
+export {
+  defineTableInOutFunction,
+  defineRowTransformFunction,
+  parentRowsMetadata,
+  PARENT_ROW_METADATA_KEY,
+  type TableInOutConfig,
+  type TableInOutBindParams,
+  type TableInOutProcessParams,
+  type RowTransformConfig,
+  type RowTransformProcessParams,
+} from "./functions/table-in-out.js";
 export {
   defineTableBufferingFunction,
   type TableBufferingConfig,

package/src/protocol/serializers/init.ts

@@ -53,6 +53,9 @@ const INIT_REQUEST_SCHEMA = makeSchema([
   field("phase", utf8(), true),
   field("execution_id", binary(), true),
   field("init_opaque_data", binary(), true),
+  // Per-substream identity for parallel streaming table-in-out (client-minted,
+  // stable across this substream's init/process/finalize; null when absent).
+  field("substream_id", binary(), true),
   // Order pushdown hints from DuckDB's RowGroupPruner (all null when no hint).
   field("order_by_column_name", utf8(), true),
   field("order_by_direction", utf8(), true),
@@ -87,6 +90,7 @@ export function serializeInitRequest(req: InitRequest): VgiBatch {
     phase: req.phase ?? null,
     execution_id: req.execution_id ?? null,
     init_opaque_data: req.init_opaque_data ?? null,
+    substream_id: req.substream_id ?? null,
     order_by_column_name: req.order_by_column_name ?? null,
     order_by_direction: req.order_by_direction ?? null,
     order_by_null_order: req.order_by_null_order ?? null,
@@ -177,6 +181,9 @@ export function deserializeInitRequest(
     init_opaque_data: params.init_opaque_data
       ? toUint8Array(params.init_opaque_data)
       : null,
+    substream_id: params.substream_id
+      ? toUint8Array(params.substream_id)
+      : null,
     order_by_column_name: parseEnum(decodeDictValue(params.order_by_column_name)) ?? null,
     order_by_direction: parseDirection(decodeDictValue(params.order_by_direction)),
     order_by_null_order: parseNullOrder(decodeDictValue(params.order_by_null_order)),

package/src/protocol/types.ts

@@ -119,6 +119,18 @@ export interface InitRequest {
   finalize_state_id: Uint8Array | null;
   execution_id: Uint8Array | null;
   init_opaque_data: Uint8Array | null;
+  /**
+   * Per-substream identity for parallel streaming table-in-out functions.
+   * A stable, CLIENT-minted id for one substream (one DuckDB PipelineExecutor),
+   * identical across this substream's init / every process tick / finalize.
+   * Unlike a worker-minted `execution_id`, it survives an HTTP load balancer
+   * dispatching each request to an arbitrary backend: a finalize that lands on
+   * a different backend than the process() calls can still key the substream's
+   * accumulated state (in shared storage) by `substream_id`. `null` when the
+   * client did not supply one (serial path, non-table-in-out functions, old
+   * clients). Mirrors vgi-python's `InitRequest.substream_id`.
+   */
+  substream_id: Uint8Array | null;
   // Order pushdown hints from DuckDB's RowGroupPruner (null when no hint).
   order_by_column_name: string | null;
   order_by_direction: OrderByDirection | null;

package/src/worker.ts

@@ -205,7 +205,7 @@ export class Worker {
    */
   async serveStream(
     readable: ReadableStream<Uint8Array> | NodeJS.ReadableStream,
-    writable?: number | import("node:net").Socket,
+    writable?: number | import("node:net").Socket | import("@query-farm/vgi-rpc").ByteSink,
   ): Promise<void> {
     await serveStreamRpc(this.buildProtocol(), { readable, writable });
   }