@query-farm/vgi 0.17.0 → 0.19.0

package/dist/functions/scalar.d.ts

@@ -3,6 +3,7 @@ import type { AuthContext } from "@query-farm/vgi-rpc";
 import { FunctionStability, NullHandling } from "../types.js";
 import type { BindRequest } from "../protocol/types.js";
 import type { VgiFunction, FunctionExample } from "./types.js";
+import { type CacheControl } from "../cache-control.js";
 export interface ScalarBindParameters {
     /** Constant argument values (resolved at bind time) */
     constArgs: Record<string, any>;
@@ -108,5 +109,14 @@ export interface ScalarFunctionConfig<P extends Record<string, VgiDataType> = Re
     maxWorkers?: number;
     requiredSettings?: string[];
     requiredSecrets?: string[];
+    /**
+     * Result-cache opt-in: when set, this CacheControl's `vgi.cache.*` metadata
+     * rides every output batch's custom metadata (per-batch — NOT the schema,
+     * which the IPC stream fixes at open), so the extension can memoize the
+     * scalar's output per distinct input value. A pure, deterministic scalar
+     * only — advertising this on a non-pure scalar serves stale rows. Mirrors
+     * vgi-python's `ScalarFunction.CACHE_CONTROL`.
+     */
+    cacheControl?: CacheControl;
 }
 export declare function defineScalarFunction<P extends Record<string, VgiDataType> = Record<string, VgiDataType>, R extends VgiDataType = VgiDataType, M extends Repr = "rich">(config: ScalarFunctionConfig<P, R, M>): VgiFunction;

package/dist/functions/table-in-out.d.ts

@@ -21,6 +21,28 @@ export interface TableInOutProcessParams<TArgs = Record<string, any>> {
     pushdownFilters?: PushdownFilters;
     /** Shared storage for cross-phase and cross-worker data (SQLite-backed). */
     storage: BoundStorage;
+    /**
+     * Stable client-minted id for this streaming table-in-out substream.
+     * Present (identical across init / every process() / finalize) when the
+     * client fanned this function out across per-substream workers; use it to
+     * key per-substream accumulated state in shared storage so a finalize()
+     * that lands on a different HTTP backend than the process() calls still
+     * finds it. `null`/`undefined` for the serial path or an old client.
+     * Mirrors vgi-python's `ProcessParams.substream_id`.
+     */
+    substreamId?: Uint8Array | null;
+    /**
+     * Conditional-revalidation validator (exchange-mode result cache): the
+     * client holds a stale cached result for THIS input unit and asks the
+     * worker to confirm freshness cheaply. When set, process() may answer with
+     * a 0-row `cacheControlMetadata({ notModified: true, ... })` batch instead
+     * of recomputing. Rides the input batch's custom metadata (attached by the
+     * C++ WriteInputBatch). Undefined on a normal call.
+     */
+    ifNoneMatch?: string;
+    /** RFC 3339 Last-Modified validator for conditional revalidation.
+     *  Companion to {@link ifNoneMatch}. Undefined on a normal call. */
+    ifModifiedSince?: string;
 }
 export interface TableInOutConfig<TArgs = Record<string, any>, TState = null> {
     name: string;
@@ -62,3 +84,102 @@ export interface TableInOutConfig<TArgs = Record<string, any>, TState = null> {
     requiredSecrets?: string[];
 }
 export declare function defineTableInOutFunction<TArgs = Record<string, any>, TState = null>(config: TableInOutConfig<TArgs, TState>): VgiFunction;
+/** Metadata key carrying per-output-row provenance: base64 of a raw
+ *  little-endian int32[] mapping each output row to the input row that
+ *  produced it. Shared by string with the C++ extension and vgi-python. */
+export declare const PARENT_ROW_METADATA_KEY = "vgi_rpc.parent_row#b64";
+/**
+ * Fold per-output-row provenance into an emit metadata map.
+ *
+ * Used by the **batched correlated LATERAL** operator (blended
+ * RowTransformFunction under `FROM t, f(t.x)` / `LATERAL`): the C++ extension
+ * ships a whole input chunk to the worker in ONE exchange and reads ONE
+ * output batch, then maps each output row back to the input row that produced
+ * it via this array — so a 1->N fan-out or 1->0 filter can be batched instead
+ * of driven row-by-row.
+ *
+ * `parentRows[i]` is the 0-based index (into the input batch) of the row that
+ * produced output row `i`. Encoded as a raw little-endian `int32` array (NOT
+ * Arrow IPC), base64-encoded, under `vgi_rpc.parent_row#b64`. Absent metadata
+ * means an identity 1->1 map (the common case: the extension assumes it, and
+ * requires output rows == input rows).
+ *
+ * Contract: `parentRows.length` MUST equal the emitted batch's row count (a
+ * mismatch is a worker bug that would corrupt the stamping). Values are
+ * range-checked against the input width on the C++ side. Mirrors vgi-python's
+ * `_merge_parent_rows` / `out.emit(..., parent_rows=[...])`.
+ */
+export declare function parentRowsMetadata(parentRows: number[], outputRows: number, extra?: Map<string, string>): Map<string, string>;
+/**
+ * Process params for a blended row-transform function. `args` carries only the
+ * NAMED (bind-time scalar) options — the positional args are the per-row input
+ * columns, read from `batch` in process() (by declared name for fixed args,
+ * positionally for varargs).
+ */
+export type RowTransformProcessParams<TArgs = Record<string, any>> = TableInOutProcessParams<TArgs>;
+export interface RowTransformConfig<TArgs = Record<string, any>> {
+    name: string;
+    description?: string;
+    /**
+     * Positional args = the per-row INPUT COLUMNS (real typed args on the wire,
+     * no synthetic TABLE placeholder). Read from `batch` by declared name in
+     * process(); NOT surfaced on `params.args`.
+     */
+    args?: Record<string, VgiDataType>;
+    /**
+     * Trailing VARARGS input columns: the per-row input is N columns of the
+     * declared type. A varargs blended function has no per-column declared
+     * names (the C++ bind names them col0..colN-1), so process() reads the
+     * columns POSITIONALLY off `batch`.
+     */
+    varargs?: {
+        name: string;
+        type: VgiDataType;
+        doc?: string;
+    };
+    /** Named (string-position) args stay bind-time scalars on `params.args`. */
+    namedArgs?: Record<string, VgiDataType>;
+    argDefaults?: Record<string, any>;
+    /** Per-argument descriptions keyed by arg name (surfaced as `vgi_doc`). */
+    argDocs?: Record<string, string>;
+    /** Bind: return the output schema. The input schema (the declared per-row
+     *  columns, typed by the C++ bind) is on `params.bindCall.input_schema`. */
+    onBind: (params: TableInOutBindParams<TArgs>) => {
+        outputSchema: VgiSchema;
+        opaqueData?: Uint8Array;
+    } | Promise<{
+        outputSchema: VgiSchema;
+        opaqueData?: Uint8Array;
+    }>;
+    /**
+     * Per-row map: transform one input batch, emit exactly one output batch via
+     * `out`. 1->1, 1->N (with {@link parentRowsMetadata} provenance), and 1->0
+     * (a 0-row emit) all work. There is NO finalize — a blended function is a
+     * per-row map (DuckDB forbids FinalExecute under correlated LATERAL, one of
+     * the call shapes blended must serve). Accumulating functions use a classic
+     * TableInput table-in-out or a TableBufferingFunction.
+     */
+    process: (params: RowTransformProcessParams<TArgs>, batch: VgiBatch, out: OutputCollector) => void | Promise<void>;
+    projectionPushdown?: boolean;
+    filterPushdown?: boolean;
+    autoApplyFilters?: boolean;
+    stability?: FunctionStability;
+    examples?: FunctionExample[];
+    categories?: string[];
+    tags?: Record<string, string>;
+    maxWorkers?: number;
+    requiredSettings?: string[];
+    requiredSecrets?: string[];
+}
+/**
+ * Define a blended ("UNNEST-style") table-in-out function: its positional args
+ * ARE its per-row input columns, so ONE registration serves every call shape —
+ * `f(52, 13)` (literal -> one input row), `FROM t, f(t.x, t.y)` (columns ->
+ * streaming), and `LATERAL f(t.x, t.y)`. Mirrors vgi-python's
+ * `RowTransformFunction` (Phase B).
+ *
+ * Registers as a TABLE function with `FunctionInfo.input_from_args = true`;
+ * the worker's overload resolution matches blended overloads by INPUT-COLUMN
+ * count (the positional args are not on the wire). Map-shaped, no finalize.
+ */
+export declare function defineRowTransformFunction<TArgs = Record<string, any>>(config: RowTransformConfig<TArgs>): VgiFunction;

package/dist/functions/types.d.ts

@@ -48,6 +48,18 @@ export interface FunctionMeta {
      * otherwise calling FinalExecute is unsupported and crashes the C++ side.
      */
     hasFinalize?: boolean;
+    /**
+     * Blended ("UNNEST-style") table-in-out: the function's positional args ARE
+     * its per-row input columns (real typed args, no synthetic TABLE
+     * placeholder), so ONE registration serves f(52,13) (literal -> 1 input
+     * row), FROM t, f(t.x, t.y) (columns -> streaming), and LATERAL f(t.x,t.y).
+     * Set by {@link defineRowTransformFunction}; surfaces as
+     * FunctionInfo.input_from_args. The C++ extension reads it to enter the
+     * in-out registration branch with real-typed args and drive the literal
+     * single-row scan-mode. Mirrors vgi-python's RowTransformFunction /
+     * ResolvedMetadata.input_from_args.
+     */
+    inputFromArgs?: boolean;
     /**
      * table_buffering: force ParallelSink=false in the C++ operator
      * (single-thread, source-ordered ingest).

package/dist/generated/vgi-client.d.ts

@@ -58,7 +58,7 @@ export interface TableInfo {
     cardinality_max?: number | null;
     column_statistics?: Uint8Array | null;
     bind_result?: Uint8Array | null;
-    required_field_filter_paths: string[];
+    required_filters: string[][];
 }
 export interface ViewInfo {
     comment?: string | null;
@@ -108,6 +108,7 @@ export interface FunctionInfo {
     source_order_dependent: boolean;
     sink_order_dependent: boolean;
     requires_input_batch_index: boolean;
+    input_from_args: boolean;
     required_settings: string[];
     required_secrets: SecretLookupEntry[];
 }
@@ -394,6 +395,7 @@ export interface InitRequest {
     finalize_state_id?: Uint8Array | null;
     execution_id?: Uint8Array | null;
     init_opaque_data?: Uint8Array | null;
+    substream_id?: Uint8Array | null;
     order_by_column_name?: string | null;
     order_by_direction?: OrderByDirection | null;
     order_by_null_order?: OrderByNullOrder | null;

package/dist/index.core.d.ts

@@ -2,7 +2,7 @@ export { CACHE_ETAG_KEY, CACHE_EXPIRES_KEY, CACHE_IF_MODIFIED_SINCE_KEY, CACHE_I
 export { defineScalarFunction, type ScalarFunctionConfig, type ScalarBindParameters, type ScalarParameterDef } from "./functions/scalar.js";
 export { defineTableFunction, type TableFunctionConfig, type TableBindParams, type TableProcessParams } from "./functions/table.js";
 export { defineAggregate, GROUP_COLUMN_NAME, type AggregateFunctionConfig, type AggregateBindParams, type AggregateUpdateParams, type AggregateFinalizeParams, } from "./functions/aggregate.js";
-export { defineTableInOutFunction, type TableInOutConfig, type TableInOutBindParams, type TableInOutProcessParams } from "./functions/table-in-out.js";
+export { defineTableInOutFunction, defineRowTransformFunction, parentRowsMetadata, PARENT_ROW_METADATA_KEY, type TableInOutConfig, type TableInOutBindParams, type TableInOutProcessParams, type RowTransformConfig, type RowTransformProcessParams, } from "./functions/table-in-out.js";
 export { defineTableBufferingFunction, type TableBufferingConfig, type TableBufferingBindParams, type TableBufferingParams, type TableBufferingVgiFunction, } from "./functions/table-buffering.js";
 export { defineCopyFromFunction, type CopyFromFunctionConfig, type CopyFromOption, type CopyFromReadParams, } from "./functions/copy-from.js";
 export { defineCopyToFunction, type CopyToFunctionConfig, type CopyToOption, type CopyToWriteParams, type CopyToCloseParams, } from "./functions/copy-to.js";