@query-farm/vgi 0.18.0 → 0.19.0

package/dist/functions/scalar.d.ts

@@ -3,6 +3,7 @@ import type { AuthContext } from "@query-farm/vgi-rpc";
 import { FunctionStability, NullHandling } from "../types.js";
 import type { BindRequest } from "../protocol/types.js";
 import type { VgiFunction, FunctionExample } from "./types.js";
+import { type CacheControl } from "../cache-control.js";
 export interface ScalarBindParameters {
     /** Constant argument values (resolved at bind time) */
     constArgs: Record<string, any>;
@@ -108,5 +109,14 @@ export interface ScalarFunctionConfig<P extends Record<string, VgiDataType> = Re
     maxWorkers?: number;
     requiredSettings?: string[];
     requiredSecrets?: string[];
+    /**
+     * Result-cache opt-in: when set, this CacheControl's `vgi.cache.*` metadata
+     * rides every output batch's custom metadata (per-batch — NOT the schema,
+     * which the IPC stream fixes at open), so the extension can memoize the
+     * scalar's output per distinct input value. A pure, deterministic scalar
+     * only — advertising this on a non-pure scalar serves stale rows. Mirrors
+     * vgi-python's `ScalarFunction.CACHE_CONTROL`.
+     */
+    cacheControl?: CacheControl;
 }
 export declare function defineScalarFunction<P extends Record<string, VgiDataType> = Record<string, VgiDataType>, R extends VgiDataType = VgiDataType, M extends Repr = "rich">(config: ScalarFunctionConfig<P, R, M>): VgiFunction;

package/dist/functions/table-in-out.d.ts

@@ -21,6 +21,28 @@ export interface TableInOutProcessParams<TArgs = Record<string, any>> {
     pushdownFilters?: PushdownFilters;
     /** Shared storage for cross-phase and cross-worker data (SQLite-backed). */
     storage: BoundStorage;
+    /**
+     * Stable client-minted id for this streaming table-in-out substream.
+     * Present (identical across init / every process() / finalize) when the
+     * client fanned this function out across per-substream workers; use it to
+     * key per-substream accumulated state in shared storage so a finalize()
+     * that lands on a different HTTP backend than the process() calls still
+     * finds it. `null`/`undefined` for the serial path or an old client.
+     * Mirrors vgi-python's `ProcessParams.substream_id`.
+     */
+    substreamId?: Uint8Array | null;
+    /**
+     * Conditional-revalidation validator (exchange-mode result cache): the
+     * client holds a stale cached result for THIS input unit and asks the
+     * worker to confirm freshness cheaply. When set, process() may answer with
+     * a 0-row `cacheControlMetadata({ notModified: true, ... })` batch instead
+     * of recomputing. Rides the input batch's custom metadata (attached by the
+     * C++ WriteInputBatch). Undefined on a normal call.
+     */
+    ifNoneMatch?: string;
+    /** RFC 3339 Last-Modified validator for conditional revalidation.
+     *  Companion to {@link ifNoneMatch}. Undefined on a normal call. */
+    ifModifiedSince?: string;
 }
 export interface TableInOutConfig<TArgs = Record<string, any>, TState = null> {
     name: string;
@@ -62,3 +84,102 @@ export interface TableInOutConfig<TArgs = Record<string, any>, TState = null> {
     requiredSecrets?: string[];
 }
 export declare function defineTableInOutFunction<TArgs = Record<string, any>, TState = null>(config: TableInOutConfig<TArgs, TState>): VgiFunction;
+/** Metadata key carrying per-output-row provenance: base64 of a raw
+ *  little-endian int32[] mapping each output row to the input row that
+ *  produced it. Shared by string with the C++ extension and vgi-python. */
+export declare const PARENT_ROW_METADATA_KEY = "vgi_rpc.parent_row#b64";
+/**
+ * Fold per-output-row provenance into an emit metadata map.
+ *
+ * Used by the **batched correlated LATERAL** operator (blended
+ * RowTransformFunction under `FROM t, f(t.x)` / `LATERAL`): the C++ extension
+ * ships a whole input chunk to the worker in ONE exchange and reads ONE
+ * output batch, then maps each output row back to the input row that produced
+ * it via this array — so a 1->N fan-out or 1->0 filter can be batched instead
+ * of driven row-by-row.
+ *
+ * `parentRows[i]` is the 0-based index (into the input batch) of the row that
+ * produced output row `i`. Encoded as a raw little-endian `int32` array (NOT
+ * Arrow IPC), base64-encoded, under `vgi_rpc.parent_row#b64`. Absent metadata
+ * means an identity 1->1 map (the common case: the extension assumes it, and
+ * requires output rows == input rows).
+ *
+ * Contract: `parentRows.length` MUST equal the emitted batch's row count (a
+ * mismatch is a worker bug that would corrupt the stamping). Values are
+ * range-checked against the input width on the C++ side. Mirrors vgi-python's
+ * `_merge_parent_rows` / `out.emit(..., parent_rows=[...])`.
+ */
+export declare function parentRowsMetadata(parentRows: number[], outputRows: number, extra?: Map<string, string>): Map<string, string>;
+/**
+ * Process params for a blended row-transform function. `args` carries only the
+ * NAMED (bind-time scalar) options — the positional args are the per-row input
+ * columns, read from `batch` in process() (by declared name for fixed args,
+ * positionally for varargs).
+ */
+export type RowTransformProcessParams<TArgs = Record<string, any>> = TableInOutProcessParams<TArgs>;
+export interface RowTransformConfig<TArgs = Record<string, any>> {
+    name: string;
+    description?: string;
+    /**
+     * Positional args = the per-row INPUT COLUMNS (real typed args on the wire,
+     * no synthetic TABLE placeholder). Read from `batch` by declared name in
+     * process(); NOT surfaced on `params.args`.
+     */
+    args?: Record<string, VgiDataType>;
+    /**
+     * Trailing VARARGS input columns: the per-row input is N columns of the
+     * declared type. A varargs blended function has no per-column declared
+     * names (the C++ bind names them col0..colN-1), so process() reads the
+     * columns POSITIONALLY off `batch`.
+     */
+    varargs?: {
+        name: string;
+        type: VgiDataType;
+        doc?: string;
+    };
+    /** Named (string-position) args stay bind-time scalars on `params.args`. */
+    namedArgs?: Record<string, VgiDataType>;
+    argDefaults?: Record<string, any>;
+    /** Per-argument descriptions keyed by arg name (surfaced as `vgi_doc`). */
+    argDocs?: Record<string, string>;
+    /** Bind: return the output schema. The input schema (the declared per-row
+     *  columns, typed by the C++ bind) is on `params.bindCall.input_schema`. */
+    onBind: (params: TableInOutBindParams<TArgs>) => {
+        outputSchema: VgiSchema;
+        opaqueData?: Uint8Array;
+    } | Promise<{
+        outputSchema: VgiSchema;
+        opaqueData?: Uint8Array;
+    }>;
+    /**
+     * Per-row map: transform one input batch, emit exactly one output batch via
+     * `out`. 1->1, 1->N (with {@link parentRowsMetadata} provenance), and 1->0
+     * (a 0-row emit) all work. There is NO finalize — a blended function is a
+     * per-row map (DuckDB forbids FinalExecute under correlated LATERAL, one of
+     * the call shapes blended must serve). Accumulating functions use a classic
+     * TableInput table-in-out or a TableBufferingFunction.
+     */
+    process: (params: RowTransformProcessParams<TArgs>, batch: VgiBatch, out: OutputCollector) => void | Promise<void>;
+    projectionPushdown?: boolean;
+    filterPushdown?: boolean;
+    autoApplyFilters?: boolean;
+    stability?: FunctionStability;
+    examples?: FunctionExample[];
+    categories?: string[];
+    tags?: Record<string, string>;
+    maxWorkers?: number;
+    requiredSettings?: string[];
+    requiredSecrets?: string[];
+}
+/**
+ * Define a blended ("UNNEST-style") table-in-out function: its positional args
+ * ARE its per-row input columns, so ONE registration serves every call shape —
+ * `f(52, 13)` (literal -> one input row), `FROM t, f(t.x, t.y)` (columns ->
+ * streaming), and `LATERAL f(t.x, t.y)`. Mirrors vgi-python's
+ * `RowTransformFunction` (Phase B).
+ *
+ * Registers as a TABLE function with `FunctionInfo.input_from_args = true`;
+ * the worker's overload resolution matches blended overloads by INPUT-COLUMN
+ * count (the positional args are not on the wire). Map-shaped, no finalize.
+ */
+export declare function defineRowTransformFunction<TArgs = Record<string, any>>(config: RowTransformConfig<TArgs>): VgiFunction;

package/dist/functions/types.d.ts

@@ -48,6 +48,18 @@ export interface FunctionMeta {
      * otherwise calling FinalExecute is unsupported and crashes the C++ side.
      */
     hasFinalize?: boolean;
+    /**
+     * Blended ("UNNEST-style") table-in-out: the function's positional args ARE
+     * its per-row input columns (real typed args, no synthetic TABLE
+     * placeholder), so ONE registration serves f(52,13) (literal -> 1 input
+     * row), FROM t, f(t.x, t.y) (columns -> streaming), and LATERAL f(t.x,t.y).
+     * Set by {@link defineRowTransformFunction}; surfaces as
+     * FunctionInfo.input_from_args. The C++ extension reads it to enter the
+     * in-out registration branch with real-typed args and drive the literal
+     * single-row scan-mode. Mirrors vgi-python's RowTransformFunction /
+     * ResolvedMetadata.input_from_args.
+     */
+    inputFromArgs?: boolean;
     /**
      * table_buffering: force ParallelSink=false in the C++ operator
      * (single-thread, source-ordered ingest).

package/dist/generated/vgi-client.d.ts

@@ -108,6 +108,7 @@ export interface FunctionInfo {
     source_order_dependent: boolean;
     sink_order_dependent: boolean;
     requires_input_batch_index: boolean;
+    input_from_args: boolean;
     required_settings: string[];
     required_secrets: SecretLookupEntry[];
 }
@@ -394,6 +395,7 @@ export interface InitRequest {
     finalize_state_id?: Uint8Array | null;
     execution_id?: Uint8Array | null;
     init_opaque_data?: Uint8Array | null;
+    substream_id?: Uint8Array | null;
     order_by_column_name?: string | null;
     order_by_direction?: OrderByDirection | null;
     order_by_null_order?: OrderByNullOrder | null;

package/dist/index.core.d.ts

@@ -2,7 +2,7 @@ export { CACHE_ETAG_KEY, CACHE_EXPIRES_KEY, CACHE_IF_MODIFIED_SINCE_KEY, CACHE_I
 export { defineScalarFunction, type ScalarFunctionConfig, type ScalarBindParameters, type ScalarParameterDef } from "./functions/scalar.js";
 export { defineTableFunction, type TableFunctionConfig, type TableBindParams, type TableProcessParams } from "./functions/table.js";
 export { defineAggregate, GROUP_COLUMN_NAME, type AggregateFunctionConfig, type AggregateBindParams, type AggregateUpdateParams, type AggregateFinalizeParams, } from "./functions/aggregate.js";
-export { defineTableInOutFunction, type TableInOutConfig, type TableInOutBindParams, type TableInOutProcessParams } from "./functions/table-in-out.js";
+export { defineTableInOutFunction, defineRowTransformFunction, parentRowsMetadata, PARENT_ROW_METADATA_KEY, type TableInOutConfig, type TableInOutBindParams, type TableInOutProcessParams, type RowTransformConfig, type RowTransformProcessParams, } from "./functions/table-in-out.js";
 export { defineTableBufferingFunction, type TableBufferingConfig, type TableBufferingBindParams, type TableBufferingParams, type TableBufferingVgiFunction, } from "./functions/table-buffering.js";
 export { defineCopyFromFunction, type CopyFromFunctionConfig, type CopyFromOption, type CopyFromReadParams, } from "./functions/copy-from.js";
 export { defineCopyToFunction, type CopyToFunctionConfig, type CopyToOption, type CopyToWriteParams, type CopyToCloseParams, } from "./functions/copy-to.js";

package/dist/index.js

@@ -1586,6 +1586,14 @@ function projectBatch(projectionIds, batch) {
 }
 var init_project = () => {};
 
+// src/arrow/impl-arrowjs/metadata.ts
+import { RecordBatch as RecordBatch4 } from "@query-farm/apache-arrow";
+function withBatchMetadata(batch, metadata) {
+  const a = batch;
+  return new RecordBatch4(a.schema, a.data, metadata);
+}
+var init_metadata = () => {};
+
 // src/arrow/impl-arrowjs/ipc.ts
 import {
   Schema as Schema4,
@@ -1634,7 +1642,7 @@ import {
   Field as Field3,
   Null,
   makeData as makeData5,
-  RecordBatch as RecordBatch5,
+  RecordBatch as RecordBatch6,
   Struct as Struct5,
   SparseUnion,
   Schema as Schema5,
@@ -1746,7 +1754,7 @@ function assemble(columnNames, minData, maxData, hasNull, hasNotNull, distinctCo
   ];
   const structType = new Struct5(schema2.fields);
   const data = makeData5({ type: structType, length: n, children, nullCount: 0 });
-  return new RecordBatch5(schema2, data);
+  return new RecordBatch6(schema2, data);
 }
 var init_statistics = __esm(() => {
   init_registry();
@@ -1756,6 +1764,7 @@ var init_statistics = __esm(() => {
 // src/arrow/impl-arrowjs/index.ts
 var exports_impl_arrowjs = {};
 __export(exports_impl_arrowjs, {
+  withBatchMetadata: () => withBatchMetadata,
   utf8: () => utf8,
   union: () => union,
   uint8: () => uint8,
@@ -1830,6 +1839,7 @@ var init_impl_arrowjs = __esm(() => {
   init_filter();
   init_project();
   init_canonical();
+  init_metadata();
   init_ipc();
   init_statistics();
   backend = { name: "arrow-js" };
@@ -1928,6 +1938,7 @@ var init_branded = __esm(() => {
 // src/arrow/index.ts
 var exports_arrow = {};
 __export(exports_arrow, {
+  withBatchMetadata: () => withBatchMetadata,
   utf8: () => utf82,
   union: () => union2,
   uint8: () => uint82,
@@ -3085,7 +3096,7 @@ function defineScalarFunction(config) {
           if (outputBatch.numRows !== input.numRows) {
             throw new RowCountMismatchError(input.numRows, outputBatch.numRows);
           }
-          out.emit(outputBatch);
+          out.emit(outputBatch, config.cacheControl ? cacheControlMetadata(config.cacheControl) : undefined);
         }
       };
     }
@@ -4544,7 +4555,8 @@ function defineTableInOutFunction(config) {
         settings,
         secrets,
         pushdownFilters,
-        storage: boundStorage
+        storage: boundStorage,
+        substreamId: request.substream_id ?? null
       };
       const phase = request.phase;
       if (phase === "FINALIZE" /* FINALIZE */) {
@@ -4587,7 +4599,14 @@ function defineTableInOutFunction(config) {
         inputSchema: request.bind_call.input_schema ?? undefined,
         exchangeInit: () => ({ state, processParams }),
         exchangeFn: async (eState, input, out) => {
-          const wrappedOut = config.autoApplyFilters && pushdownFilters ? new FilteringOutputCollector(out, pushdownFilters) : out;
+          applyRevalidationValidators(eState.processParams, input);
+          let wrappedOut = makeMetadataBakingCollector(out);
+          if (projIds) {
+            wrappedOut = makeSchemaReconcilingCollector(wrappedOut, outputSchema);
+          }
+          if (config.autoApplyFilters && pushdownFilters) {
+            wrappedOut = new FilteringOutputCollector(wrappedOut, pushdownFilters);
+          }
           await processFn(eState.processParams, eState.state, input, wrappedOut);
           if (eState.state != null) {
             const serialized = serializeUserState(eState.state);
@@ -4600,6 +4619,230 @@ function defineTableInOutFunction(config) {
     }
   };
 }
+function applyRevalidationValidators(params, input) {
+  const md = input?.metadata;
+  params.ifNoneMatch = md?.get(CACHE_IF_NONE_MATCH_KEY);
+  params.ifModifiedSince = md?.get(CACHE_IF_MODIFIED_SINCE_KEY);
+}
+function makeMetadataBakingCollector(inner) {
+  return new Proxy(inner, {
+    get(target, prop, receiver) {
+      if (prop === "emit") {
+        return function(batchOrColumns, metadata) {
+          if (batchOrColumns && typeof batchOrColumns.getChild === "function" && metadata && metadata.size > 0) {
+            const batch = batchOrColumns;
+            const merged = new Map(batch.metadata ?? []);
+            for (const [k, v2] of metadata)
+              merged.set(k, v2);
+            return target.emit(withBatchMetadata(batch, merged), merged);
+          }
+          return target.emit(batchOrColumns, metadata);
+        };
+      }
+      const v = Reflect.get(target, prop, receiver);
+      return typeof v === "function" ? v.bind(target) : v;
+    }
+  });
+}
+function makeSchemaReconcilingCollector(inner, targetSchema) {
+  function reconcile(batch) {
+    const batchFields = batch.schema.fields;
+    if (batchFields.length === targetSchema.fields.length && targetSchema.fields.every((f, i) => batchFields[i].name === f.name)) {
+      return batch;
+    }
+    const ids = [];
+    for (const f of targetSchema.fields) {
+      const idx = batchFields.findIndex((bf) => bf.name === f.name);
+      if (idx < 0) {
+        throw new Error(`emitted batch is missing projected output column '${f.name}' ` + `(emitted: [${batchFields.map((bf) => bf.name).join(", ")}])`);
+      }
+      ids.push(idx);
+    }
+    return projectBatch(ids, batch);
+  }
+  return new Proxy(inner, {
+    get(target, prop, receiver) {
+      if (prop === "emit") {
+        return function(batchOrColumns, metadata) {
+          if (batchOrColumns && typeof batchOrColumns.getChild === "function") {
+            return target.emit(reconcile(batchOrColumns), metadata);
+          }
+          return target.emit(batchOrColumns, metadata);
+        };
+      }
+      const v = Reflect.get(target, prop, receiver);
+      return typeof v === "function" ? v.bind(target) : v;
+    }
+  });
+}
+var PARENT_ROW_METADATA_KEY = "vgi_rpc.parent_row#b64";
+function parentRowsMetadata(parentRows, outputRows, extra) {
+  if (parentRows.length !== outputRows) {
+    throw new Error(`parentRowsMetadata: length ${parentRows.length} != output rows ${outputRows}; ` + `parentRows must carry exactly one input-row index per emitted output row`);
+  }
+  const md = new Map(extra ?? []);
+  if (outputRows === 0) {
+    return md;
+  }
+  const raw = new Uint8Array(parentRows.length * 4);
+  const dv = new DataView(raw.buffer);
+  for (let i = 0;i < parentRows.length; i++) {
+    dv.setInt32(i * 4, parentRows[i], true);
+  }
+  md.set(PARENT_ROW_METADATA_KEY, base64Encode(raw));
+  return md;
+}
+function base64Encode(bytes) {
+  if (typeof globalThis.Buffer !== "undefined") {
+    return globalThis.Buffer.from(bytes).toString("base64");
+  }
+  let bin = "";
+  for (let i = 0;i < bytes.length; i++)
+    bin += String.fromCharCode(bytes[i]);
+  return globalThis.btoa(bin);
+}
+function defineRowTransformFunction(config) {
+  const numPositional = Object.keys(config.args ?? {}).length + (config.varargs ? 1 : 0);
+  if (numPositional === 0) {
+    throw new Error(`${config.name}: a blended row-transform function needs at least one ` + `positional arg (its per-row input column); found none.`);
+  }
+  const specs = [];
+  let posIdx = 0;
+  if (config.args) {
+    for (const [name, type] of Object.entries(config.args)) {
+      specs.push({
+        name,
+        position: posIdx++,
+        arrowType: type,
+        doc: config.argDocs?.[name]
+      });
+    }
+  }
+  if (config.varargs) {
+    specs.push({
+      name: config.varargs.name,
+      position: posIdx++,
+      arrowType: config.varargs.type,
+      isVarargs: true,
+      doc: config.varargs.doc ?? config.argDocs?.[config.varargs.name]
+    });
+  }
+  if (config.namedArgs) {
+    for (const [name, type] of Object.entries(config.namedArgs)) {
+      specs.push({
+        name,
+        position: name,
+        arrowType: type,
+        doc: config.argDocs?.[name]
+      });
+    }
+  }
+  const meta = {
+    name: config.name,
+    description: config.description,
+    stability: config.stability,
+    projectionPushdown: config.projectionPushdown,
+    filterPushdown: config.filterPushdown,
+    autoApplyFilters: config.autoApplyFilters,
+    examples: config.examples,
+    categories: config.categories,
+    tags: config.tags,
+    maxWorkers: config.maxWorkers,
+    requiredSettings: config.requiredSettings,
+    requiredSecrets: config.requiredSecrets,
+    hasFinalize: false,
+    inputFromArgs: true
+  };
+  function extractArgs(request) {
+    const args = {};
+    for (const spec of specs) {
+      if (typeof spec.position !== "string")
+        continue;
+      const defaultVal = config.argDefaults?.[spec.name] !== undefined ? config.argDefaults[spec.name] : undefined;
+      let val = request.arguments.get(spec.position, defaultVal ?? null);
+      if (typeof val === "bigint")
+        val = Number(val);
+      args[spec.name] = val;
+    }
+    return args;
+  }
+  return {
+    kind: "table_in_out",
+    meta,
+    argumentSpecs: specs,
+    async bind(request) {
+      const args = extractArgs(request);
+      const settings = batchToScalarDict(request.settings);
+      const secrets = batchToSecretDict(request.secrets);
+      const result = await config.onBind({
+        args,
+        bindCall: request,
+        settings,
+        secrets
+      });
+      return {
+        output_schema: result.outputSchema,
+        opaque_data: result.opaqueData ?? null
+      };
+    },
+    async globalInit(request) {
+      if (request.execution_id) {
+        return {
+          max_workers: DEFAULT_MAX_WORKERS,
+          execution_id: request.execution_id,
+          opaque_data: null
+        };
+      }
+      const executionId = new Uint8Array(16);
+      crypto.getRandomValues(executionId);
+      return {
+        max_workers: config.maxWorkers ?? 1,
+        execution_id: executionId,
+        opaque_data: null
+      };
+    },
+    createStreamHandlers(request, response, _accumulatedState) {
+      const args = extractArgs(request.bind_call);
+      const settings = batchToScalarDict(request.bind_call.settings);
+      const secrets = batchToSecretDict(request.bind_call.secrets);
+      const projIds = request.projection_ids && meta.projectionPushdown ? request.projection_ids : null;
+      const outputSchema = projIds ? projectSchema(projIds, request.output_schema) : request.output_schema;
+      const joinKeysLookup = buildJoinKeysLookup(request.join_keys);
+      const pushdownFilters = request.pushdown_filters ? deserializeFilters(request.pushdown_filters, joinKeysLookup) : undefined;
+      const boundStorage = new BoundStorage(storage, response.execution_id ?? new Uint8Array(16));
+      const processParams = {
+        args,
+        initCall: request,
+        initResponse: response,
+        outputSchema,
+        settings,
+        secrets,
+        pushdownFilters,
+        storage: boundStorage,
+        substreamId: request.substream_id ?? null
+      };
+      if (request.phase === "FINALIZE" /* FINALIZE */) {
+        throw new Error(`${config.name}: a blended row-transform function has no FINALIZE ` + `phase (it is a per-row map; has_finalize is advertised false).`);
+      }
+      return {
+        outputSchema,
+        inputSchema: request.bind_call.input_schema ?? undefined,
+        exchangeInit: () => ({ state: null, processParams }),
+        exchangeFn: async (eState, input, out) => {
+          applyRevalidationValidators(eState.processParams, input);
+          let wrappedOut = makeMetadataBakingCollector(out);
+          if (projIds) {
+            wrappedOut = makeSchemaReconcilingCollector(wrappedOut, outputSchema);
+          }
+          if (config.autoApplyFilters && pushdownFilters) {
+            wrappedOut = new FilteringOutputCollector(wrappedOut, pushdownFilters);
+          }
+          await config.process(eState.processParams, input, wrappedOut);
+        }
+      };
+    }
+  };
+}
 // src/functions/table-buffering.ts
 init_arrow();
 init_arrow();
@@ -4961,6 +5204,7 @@ var INIT_REQUEST_SCHEMA = schema2([
   field2("phase", utf82(), true),
   field2("execution_id", binary2(), true),
   field2("init_opaque_data", binary2(), true),
+  field2("substream_id", binary2(), true),
   field2("order_by_column_name", utf82(), true),
   field2("order_by_direction", utf82(), true),
   field2("order_by_null_order", utf82(), true),
@@ -4987,6 +5231,7 @@ function serializeInitRequest(req) {
     phase: req.phase ?? null,
     execution_id: req.execution_id ?? null,
     init_opaque_data: req.init_opaque_data ?? null,
+    substream_id: req.substream_id ?? null,
     order_by_column_name: req.order_by_column_name ?? null,
     order_by_direction: req.order_by_direction ?? null,
     order_by_null_order: req.order_by_null_order ?? null,
@@ -5048,6 +5293,7 @@ function deserializeInitRequest(params) {
     finalize_state_id: params.finalize_state_id ? toUint8Array2(params.finalize_state_id) : null,
     execution_id: params.execution_id ? toUint8Array2(params.execution_id) : null,
     init_opaque_data: params.init_opaque_data ? toUint8Array2(params.init_opaque_data) : null,
+    substream_id: params.substream_id ? toUint8Array2(params.substream_id) : null,
     order_by_column_name: parseEnum(decodeDictValue(params.order_by_column_name)) ?? null,
     order_by_direction: parseDirection(decodeDictValue(params.order_by_direction)),
     order_by_null_order: parseNullOrder(decodeDictValue(params.order_by_null_order)),
@@ -5836,6 +6082,7 @@ var FunctionInfoSchema = schema2([
   field2("source_order_dependent", bool2(), false),
   field2("sink_order_dependent", bool2(), false),
   field2("requires_input_batch_index", bool2(), false),
+  field2("input_from_args", bool2(), false),
   field2("required_settings", list2(field2("item", utf82(), true)), false),
   field2("required_secrets", list2(field2("item", struct2([field2("secret_type", utf82(), false), field2("scope", utf82(), true), field2("secret_name", utf82(), true)]), true)), false)
 ]);
@@ -6995,6 +7242,7 @@ class ReadOnlyCatalogInterface extends CatalogInterface {
         source_order_dependent: f.kind === "table_buffering" && f.meta.sourceOrderDependent === true,
         sink_order_dependent: f.kind === "table_buffering" && f.meta.sinkOrderDependent === true,
         requires_input_batch_index: f.kind === "table_buffering" && f.meta.requiresInputBatchIndex === true,
+        input_from_args: f.kind === "table_in_out" && f.meta.inputFromArgs === true,
         required_settings: meta.requiredSettings,
         required_secrets: requiredSecrets.map((s) => ({
           secret_type: s.secret_type,
@@ -7539,10 +7787,27 @@ function filterByArgumentTypes(candidates, args, inputSchema, isScalar) {
     } else {
       const posSpecs = specs.filter((s) => typeof s.position === "number" && !s.isTableInput).sort((a, b) => a.position - b.position);
       const posTypes = [];
-      const argsSchema = args.argumentsSchema;
-      for (let i = 0;i < args.positional.length; i++) {
-        const field3 = argsSchema?.fields.find((f) => f.name === `positional_${i}`);
-        posTypes.push(field3 ? field3.type : null);
+      if (func.meta.inputFromArgs) {
+        if (inputSchema) {
+          let varArgsSpec = null;
+          for (const spec of posSpecs) {
+            if (spec.isVarargs)
+              varArgsSpec = spec;
+            const pos = spec.position;
+            posTypes.push(pos < inputSchema.fields.length ? inputSchema.fields[pos].type : null);
+          }
+          if (varArgsSpec) {
+            for (let i = varArgsSpec.position + 1;i < inputSchema.fields.length; i++) {
+              posTypes.push(inputSchema.fields[i].type);
+            }
+          }
+        }
+      } else {
+        const argsSchema = args.argumentsSchema;
+        for (let i = 0;i < args.positional.length; i++) {
+          const field3 = argsSchema?.fields.find((f) => f.name === `positional_${i}`);
+          posTypes.push(field3 ? field3.type : null);
+        }
       }
       const r = scoreTypes(posSpecs, posTypes);
       score += r.score;
@@ -7601,6 +7866,21 @@ class FunctionRegistry {
         const posSpecs = func.argumentSpecs.filter((s) => typeof s.position === "number" && !s.isTableInput);
         const hasVarargs = posSpecs.some((s) => s.isVarargs);
         const nonVarargs = posSpecs.filter((s) => !s.isVarargs).length;
+        if (func.meta.inputFromArgs) {
+          const numInputCols = context.inputSchema?.fields.length ?? 0;
+          if (hasVarargs) {
+            if (numInputCols < nonVarargs)
+              return false;
+          } else if (numInputCols !== posSpecs.length) {
+            return false;
+          }
+          const validNamed = new Set(func.argumentSpecs.filter((s) => typeof s.position === "string").map((s) => s.position));
+          for (const key of args.named.keys()) {
+            if (!validNamed.has(key))
+              return false;
+          }
+          return true;
+        }
         if (hasVarargs) {
           return numArgs >= nonVarargs;
         }
@@ -10662,7 +10942,8 @@ class VgiClient {
       tablesample_percentage: ts?.percentage ?? null,
       tablesample_seed: ts?.seed == null ? null : BigInt(ts.seed),
       execution_id: opts?.executionId ?? null,
-      init_opaque_data: null
+      init_opaque_data: null,
+      substream_id: null
     };
     const initBatch = serializeInitRequest(initRequest);
     const session = await this.rpc.stream("init", wrapRequest(initBatch));
@@ -11510,6 +11791,7 @@ export {
   readCanonicalValue,
   projectSchema,
   projectBatch,
+  parentRowsMetadata,
   nullType2 as nullType,
   metadatasToArrow,
   map2 as map,
@@ -11581,6 +11863,7 @@ export {
   defineTableFunction,
   defineTableBufferingFunction,
   defineScalarFunction,
+  defineRowTransformFunction,
   defineCopyToFunction,
   defineCopyFromFunction,
   defineAggregate,
@@ -11648,6 +11931,7 @@ export {
   RowCountMismatchError,
   ReadOnlyCatalogInterface,
   PushdownFilters,
+  PARENT_ROW_METADATA_KEY,
   OutputCollector,
   OrderPreservation,
   OrderDependence,
@@ -11696,4 +11980,4 @@ export {
   ArgumentValidationError
 };
 
-//# debugId=B98CA527867E993364756E2164756E21
+//# debugId=8787F3CC30C0C60C64756E2164756E21