@query-farm/vgi 0.6.0

package/dist/functions/copy-to.d.ts

@@ -0,0 +1,121 @@
+import { type VgiBatch, type VgiDataType } from "../arrow/index.js";
+import type { BindRequest } from "../protocol/types.js";
+import type { FunctionExample } from "./types.js";
+import { type TableBufferingParams, type TableBufferingVgiFunction } from "./table-buffering.js";
+/**
+ * A single secret to resolve via the two-phase secret bind, returned from a COPY
+ * format's `onSecrets` hook. Mirrors a vgi-python `SecretLookupEntry`.
+ */
+export interface CopySecretLookup {
+    /** DuckDB secret type to resolve (e.g. `"s3"`, `"vgi_example"`). */
+    secretType: string;
+    /** Optional scope for longest-prefix matching — typically the COPY path. */
+    scope?: string;
+    /** Optional secret name for name-based lookup. */
+    name?: string;
+}
+/** Declaration of a single COPY option (a named function argument). */
+export interface CopyToOption {
+    /** Arrow type of the option value (e.g. `utf8()`, `int64()`). */
+    type: VgiDataType;
+    /** Per-option description, surfaced by `vgi_copy_formats()`. */
+    doc?: string;
+    /**
+     * Default value. When omitted the option is REQUIRED — the worker throws a
+     * clear error at COPY bind if the user does not supply it. Mirrors
+     * vgi-python's `Arg(..., default=...)` (no default => required).
+     */
+    default?: unknown;
+    /**
+     * Optional allowed value set, validated worker-side at bind. Mirrors
+     * vgi-python's `Arg(..., choices=[...])`.
+     */
+    choices?: unknown[];
+    /**
+     * Optional inclusive lower bound for numeric options, validated worker-side at
+     * bind. Mirrors vgi-python's `Arg(..., ge=...)`.
+     */
+    ge?: number;
+    /**
+     * Optional inclusive upper bound for numeric options, validated worker-side at
+     * bind. Mirrors vgi-python's `Arg(..., le=...)`.
+     */
+    le?: number;
+}
+/** Parameters for the per-batch `write` hook. */
+export interface CopyToWriteParams<TArgs = Record<string, unknown>> {
+    /** One input batch from the COPY source. */
+    batch: VgiBatch;
+    /** Parsed COPY options (defaults applied). */
+    options: TArgs;
+    /** Destination path from the `COPY ... TO 'path'` statement. */
+    filePath: string;
+    /** Full buffering parameters (settings, secrets, storage, executionId). */
+    params: TableBufferingParams<TArgs>;
+}
+/** Parameters for the terminal `close` hook. */
+export interface CopyToCloseParams<TArgs = Record<string, unknown>> {
+    /** Parsed COPY options (defaults applied). */
+    options: TArgs;
+    /** Destination path from the `COPY ... TO 'path'` statement. */
+    filePath: string;
+    /** Full buffering parameters (settings, secrets, storage, executionId). */
+    params: TableBufferingParams<TArgs>;
+}
+export interface CopyToFunctionConfig<TArgs = Record<string, unknown>> {
+    /** Handler name (the function's registered name; `Meta.name`). */
+    name: string;
+    /** SQL `FORMAT` identifier users type, e.g. `COPY t TO 'x' (FORMAT myfmt)`. */
+    format: string;
+    /** Function description (intrinsic documentation; `Meta.description`). */
+    description?: string;
+    /** Optional free-text comment surfaced by `vgi_copy_formats()`. */
+    comment?: string | null;
+    /** COPY direction; only `"to"` is supported here. */
+    direction?: string;
+    /**
+     * When true, the writer requires rows in source order — discovery advertises
+     * `ordered=true` and the extension uses a single-threaded sink
+     * (`REGULAR_COPY_TO_FILE`) so one worker receives every batch in source
+     * order. Mirrors vgi-python's `Meta.sink_order_dependent`.
+     */
+    ordered?: boolean;
+    /** COPY options, keyed by option name. The `file_path` is NOT an option. */
+    options?: Record<string, CopyToOption>;
+    categories?: string[];
+    tags?: Record<string, string>;
+    examples?: FunctionExample[];
+    requiredSettings?: string[];
+    requiredSecrets?: string[];
+    /**
+     * Optional secret-bind hook: forward CREATE SECRET credentials for
+     * secret-backed cloud writes (S3/GCS/HTTP/…). Called during bind (only on the
+     * first pass); return the secrets to resolve — typically scoped by the
+     * destination `filePath`. The framework's two-phase secret bind resolves each
+     * lookup from the caller's SecretManager and surfaces the resolved values on
+     * `params.secrets` at `write`/`close` time. Mirrors vgi-python's
+     * `CopyToFunction.on_secrets`.
+     */
+    onSecrets?: (params: {
+        options: TArgs;
+        filePath: string;
+        bindCall: BindRequest;
+    }) => CopySecretLookup[] | void;
+    /** Persist one input `batch` to a shard (called per sink batch). */
+    write: (params: CopyToWriteParams<TArgs>) => void | Promise<void>;
+    /**
+     * Write the destination and close it, once (called on the coordinator). Read
+     * the shards persisted by `write` and perform the terminal write. Called even
+     * for empty input. Return the row count (informational).
+     */
+    close: (params: CopyToCloseParams<TArgs>) => number | void | Promise<number | void>;
+}
+/**
+ * Define a custom `COPY ... TO` format writer, returned as a
+ * `TableBufferingVgiFunction` (kind `"table_buffering"`) carrying the
+ * `copyToFormat` metadata marker. Register it in the catalog's function list
+ * like any function; the catalog's `copyFromFormats()` introspection picks it
+ * up and advertises the format (direction `"to"`). The worker's
+ * `table_buffering_process` / `table_buffering_combine` RPCs drive write/close.
+ */
+export declare function defineCopyToFunction<TArgs = Record<string, unknown>>(config: CopyToFunctionConfig<TArgs>): TableBufferingVgiFunction;

package/dist/functions/registry.d.ts

@@ -0,0 +1,16 @@
+import type { VgiSchema } from "../arrow/index.js";
+import type { VgiFunction } from "./types.js";
+import type { Arguments } from "../arguments/arguments.js";
+export interface OverloadContext {
+    arguments?: Arguments;
+    inputSchema?: VgiSchema | null;
+    isScalar?: boolean;
+}
+export declare class FunctionRegistry {
+    private _functions;
+    register(func: VgiFunction): void;
+    get(name: string, context?: OverloadContext): VgiFunction;
+    has(name: string): boolean;
+    names(): string[];
+    all(): VgiFunction[];
+}

package/dist/functions/scalar.d.ts

@@ -0,0 +1,98 @@
+import { type VgiSchema, type VgiBatch, type VgiDataType, type ValueFor, type Repr } from "../arrow/index.js";
+import type { AuthContext } from "@query-farm/vgi-rpc";
+import { FunctionStability, NullHandling } from "../types.js";
+import type { BindRequest } from "../protocol/types.js";
+import type { VgiFunction, FunctionExample } from "./types.js";
+export interface ScalarBindParameters {
+    /** Constant argument values (resolved at bind time) */
+    constArgs: Record<string, any>;
+    /** Schema of all arguments including columnar params */
+    argumentsSchema: VgiSchema;
+    /** Settings from DuckDB */
+    settings: Record<string, any>;
+    /** Secrets from DuckDB */
+    secrets: Record<string, Record<string, any>>;
+    /** Original bind request — exposes attach_opaque_data, transaction_opaque_data, function_type, etc. */
+    bindCall: BindRequest;
+}
+/** Ordered parameter definition for scalar functions. */
+export interface ScalarParameterDef {
+    name: string;
+    type: VgiDataType;
+    /** If true, this is a constant parameter (scalar value resolved at bind time). */
+    const?: boolean;
+    /** If true, this parameter accepts variable number of arguments. */
+    varargs?: boolean;
+    /** Human-readable per-argument description (surfaced as `vgi_doc`). */
+    doc?: string;
+}
+/**
+ * The typed compute row for a scalar function: one property per declared
+ * columnar param, valued in the representation (`rich` / `raw`) selected by the
+ * function's `repr`. (Provided as a convenience type for authors who build rows
+ * out of the input batch.)
+ */
+export type ScalarComputeRow<P extends Record<string, VgiDataType>, M extends Repr> = {
+    [K in keyof P]: ValueFor<P[K], M> | null;
+};
+/** A single output value for a scalar function under representation `M`. */
+export type ScalarOutputValue<R extends VgiDataType, M extends Repr> = ValueFor<R, M> | null;
+/**
+ * Accepted compute return shapes: an array (or iterable) of output values in
+ * the declared representation, or a pre-built VgiBatch. The representation is
+ * enforced statically — returning a `Date` under `repr: 'raw'` (which expects a
+ * branded `Date32`/`Date64Ms`), or a branded value under `repr: 'rich'` (which
+ * expects a `Date`), is a COMPILE error.
+ */
+export type ScalarComputeResult<R extends VgiDataType, M extends Repr> = Array<ScalarOutputValue<R, M>> | Iterable<ScalarOutputValue<R, M>> | VgiBatch;
+export interface ScalarFunctionConfig<P extends Record<string, VgiDataType> = Record<string, VgiDataType>, R extends VgiDataType = VgiDataType, M extends Repr = "rich"> {
+    name: string;
+    description?: string;
+    /** Columnar params (receive Arrow arrays at process time) */
+    params?: P;
+    /** Constant params (receive scalar values resolved at bind time) */
+    constParams?: Record<string, VgiDataType>;
+    /**
+     * Per-argument descriptions keyed by param name, surfaced as `vgi_doc` field
+     * metadata (and via the extension's `vgi_function_arguments()`). Applies to
+     * `params` and `constParams`; for the ordered `parameters` list use each
+     * entry's `doc` field instead.
+     */
+    argDocs?: Record<string, string>;
+    /**
+     * Ordered parameter list. When provided, overrides `params` and `constParams`.
+     * Use this when parameters need non-default ordering (e.g. const, param, const).
+     */
+    parameters?: ScalarParameterDef[];
+    /** Output type (static) */
+    returns?: R;
+    /** Dynamic output type at bind time */
+    outputType?: (params: ScalarBindParameters) => VgiDataType | Promise<VgiDataType>;
+    /**
+     * Value representation for compute I/O. `'rich'` (default) uses JS `Date` for
+     * date32/date64 and plain number/bigint elsewhere. `'raw'` uses the branded
+     * unit-carrying aliases (Date32, TimestampMicros, UnscaledDecimal, …). The
+     * choice flows into compute()'s statically-checked return type and selects
+     * the runtime converter used to build the output column.
+     */
+    repr?: M;
+    /**
+     * Process: receives the columnar input batch + const values, returns the
+     * output column as an array/iterable of values (statically typed from
+     * `returns` and `repr`) or a pre-built VgiBatch.
+     */
+    compute: (batch: VgiBatch, consts: Record<string, any>, info: {
+        settings: Record<string, any>;
+        secrets: Record<string, Record<string, any>>;
+        auth: AuthContext;
+    }) => ScalarComputeResult<NoInfer<R>, NoInfer<M>>;
+    stability?: FunctionStability;
+    nullHandling?: NullHandling;
+    examples?: FunctionExample[];
+    categories?: string[];
+    tags?: Record<string, string>;
+    maxWorkers?: number;
+    requiredSettings?: string[];
+    requiredSecrets?: string[];
+}
+export declare function defineScalarFunction<P extends Record<string, VgiDataType> = Record<string, VgiDataType>, R extends VgiDataType = VgiDataType, M extends Repr = "rich">(config: ScalarFunctionConfig<P, R, M>): VgiFunction;

package/dist/functions/storage-cf-do.d.ts

@@ -0,0 +1,47 @@
+import type { FunctionStorage } from "./storage.js";
+/** Minimal fetch signature satisfied by both the global `fetch` and a
+ *  Cloudflare service-binding Fetcher (`env.<BINDING>`). */
+export type FetchLike = (input: string, init?: {
+    method?: string;
+    headers?: Record<string, string>;
+    body?: string;
+}) => Promise<Response>;
+export declare class FunctionStorageCfDo implements FunctionStorage {
+    private readonly _baseUrl;
+    private readonly _token;
+    /** Per-attach routing key (att-<hex uuid>); "" until pinned via forShard. */
+    private readonly _shardKey;
+    /** Fetch implementation. Defaults to the global `fetch`; pass a Cloudflare
+     *  service-binding Fetcher (env.<BINDING>) when the storage worker lives on
+     *  the same account/zone — a same-zone public-URL fetch is rejected (CF
+     *  error 1042), but a service binding routes worker-to-worker directly. */
+    private readonly _fetch;
+    constructor(opts: {
+        url: string;
+        token?: string | null;
+        shardKey?: string;
+        fetch?: FetchLike;
+    });
+    /** Build an instance from environment variables. */
+    static fromEnv(): FunctionStorageCfDo;
+    /**
+     * Return a view of this backend pinned to one shard key, so callers can route
+     * per logical ATTACH. Shares the URL/token; only the shard key differs.
+     */
+    forShard(shardKey: string): FunctionStorageCfDo;
+    workerPut(executionId: Uint8Array, workerId: number, state: Uint8Array): Promise<void>;
+    workerCollect(executionId: Uint8Array): Promise<Uint8Array[]>;
+    workerScan(executionId: Uint8Array): Promise<Array<[number, Uint8Array]>>;
+    queuePush(executionId: Uint8Array, items: Uint8Array[]): Promise<number>;
+    queuePop(executionId: Uint8Array): Promise<Uint8Array | null>;
+    queueClear(executionId: Uint8Array): Promise<number>;
+    stateGet(scopeId: Uint8Array, ns: Uint8Array, key: Uint8Array): Promise<Uint8Array | null>;
+    statePut(scopeId: Uint8Array, ns: Uint8Array, key: Uint8Array, value: Uint8Array): Promise<void>;
+    stateAppend(scopeId: Uint8Array, ns: Uint8Array, key: Uint8Array, item: Uint8Array): Promise<number>;
+    stateLogScan(scopeId: Uint8Array, ns: Uint8Array, key: Uint8Array, afterId?: number, limit?: number | null): Promise<Array<[number, Uint8Array]>>;
+    executionClear(scopeId: Uint8Array): Promise<number>;
+    private _statePutMany;
+    /** Drive state_scan / state_drain across pages. attemptId is null for scan. */
+    private _statePaged;
+    private _post;
+}

package/dist/functions/storage.d.ts

@@ -0,0 +1,86 @@
+export declare class UnknownInvocationError extends Error {
+    constructor(message?: string | Uint8Array);
+}
+export interface FunctionStorage {
+    workerPut(executionId: Uint8Array, workerId: number, state: Uint8Array): Promise<void>;
+    workerCollect(executionId: Uint8Array): Promise<Uint8Array[]>;
+    workerScan(executionId: Uint8Array): Promise<Array<[number, Uint8Array]>>;
+    queuePush(executionId: Uint8Array, items: Uint8Array[]): Promise<number>;
+    queuePop(executionId: Uint8Array): Promise<Uint8Array | null>;
+    queueClear(executionId: Uint8Array): Promise<number>;
+    stateGet(scopeId: Uint8Array, ns: Uint8Array, key: Uint8Array): Promise<Uint8Array | null>;
+    statePut(scopeId: Uint8Array, ns: Uint8Array, key: Uint8Array, value: Uint8Array): Promise<void>;
+    stateAppend(scopeId: Uint8Array, ns: Uint8Array, key: Uint8Array, item: Uint8Array): Promise<number>;
+    stateLogScan(scopeId: Uint8Array, ns: Uint8Array, key: Uint8Array, afterId?: number, limit?: number | null): Promise<Array<[number, Uint8Array]>>;
+    /** Wipe all state + log rows for a scope across every namespace. */
+    executionClear(scopeId: Uint8Array): Promise<number>;
+    /** Optional cleanup hook. Implementations holding a connection should release it here. */
+    close?(): Promise<void> | void;
+}
+export declare const FrameworkNS: {
+    readonly BUFFERING_INIT: Uint8Array<ArrayBufferLike>;
+    readonly STREAMING_FINALIZE: Uint8Array<ArrayBufferLike>;
+};
+/** Encode an int as 8-byte little-endian, matching Python's pack_int_key. */
+export declare function packIntKey(i: number | bigint): Uint8Array;
+export declare class FunctionStorageSqlite implements FunctionStorage {
+    readonly dbPath: string;
+    private _db;
+    constructor(dbPath?: string);
+    close(): void;
+    private _ensureTables;
+    stateGet(scopeId: Uint8Array, ns: Uint8Array, key: Uint8Array): Promise<Uint8Array | null>;
+    statePut(scopeId: Uint8Array, ns: Uint8Array, key: Uint8Array, value: Uint8Array): Promise<void>;
+    stateAppend(scopeId: Uint8Array, ns: Uint8Array, key: Uint8Array, item: Uint8Array): Promise<number>;
+    stateLogScan(scopeId: Uint8Array, ns: Uint8Array, key: Uint8Array, afterId?: number, limit?: number | null): Promise<Array<[number, Uint8Array]>>;
+    executionClear(scopeId: Uint8Array): Promise<number>;
+    workerPut(executionId: Uint8Array, workerId: number, state: Uint8Array): Promise<void>;
+    workerCollect(executionId: Uint8Array): Promise<Uint8Array[]>;
+    workerScan(executionId: Uint8Array): Promise<Array<[number, Uint8Array]>>;
+    queuePush(executionId: Uint8Array, items: Uint8Array[]): Promise<number>;
+    queuePop(executionId: Uint8Array): Promise<Uint8Array | null>;
+    queueClear(executionId: Uint8Array): Promise<number>;
+}
+export declare class BoundStorage {
+    private _base;
+    private _executionId;
+    constructor(storage: FunctionStorage, executionId: Uint8Array);
+    put(state: Uint8Array): Promise<void>;
+    collect(): Promise<Uint8Array[]>;
+    workerScan(): Promise<Array<[number, Uint8Array]>>;
+    queuePush(items: Uint8Array[]): Promise<number>;
+    queuePop(): Promise<Uint8Array | null>;
+    queueClear(): Promise<number>;
+    stateGet(ns: Uint8Array, key: Uint8Array): Promise<Uint8Array | null>;
+    statePut(ns: Uint8Array, key: Uint8Array, value: Uint8Array): Promise<void>;
+    stateAppend(ns: Uint8Array, key: Uint8Array, item: Uint8Array): Promise<number>;
+    stateLogScan(ns: Uint8Array, key: Uint8Array, afterId?: number, limit?: number | null): Promise<Array<[number, Uint8Array]>>;
+    executionClear(): Promise<number>;
+    static packIntKey(i: number | bigint): Uint8Array;
+}
+/**
+ * Resolve the FunctionStorage backend from environment variables.
+ *
+ * Mirrors vgi-python's `_resolve_storage` in `vgi/function.py`.
+ *
+ * `VGI_WORKER_SHARED_STORAGE` selects the backend (default: `sqlite`):
+ *   - `memory`        — `FunctionStorageSqlite` at `:memory:`. Process-local
+ *                       with no cross-process coordination — single-process
+ *                       deployments only. Ignores `VGI_WORKER_SQLITE_PATH`.
+ *   - `sqlite`        — `FunctionStorageSqlite`. Honors `VGI_WORKER_SQLITE_PATH`
+ *                       (including `:memory:`).
+ *   - `cloudflare-do` — `FunctionStorageCfDo`. Requires `VGI_CF_DO_URL`;
+ *                       optional `VGI_CF_DO_TOKEN` for bearer auth.
+ */
+export declare function resolveStorageFromEnv(): FunctionStorage;
+/**
+ * Override the default backend used by the `storage` singleton. Call once at
+ * worker startup to inject a pre-built backend that env-driven selection can't
+ * construct — notably a `FunctionStorageCfDo` wired to a Cloudflare
+ * service-binding Fetcher (the binding lives on `env`, not `process.env`, and a
+ * same-zone public-URL fetch is rejected with CF error 1042). Takes precedence
+ * over `VGI_WORKER_SHARED_STORAGE`.
+ */
+export declare function setStorage(impl: FunctionStorage): void;
+/** Default FunctionStorage instance. Backend is selected via env on first access. */
+export declare const storage: FunctionStorage;

package/dist/functions/table-buffering.d.ts

@@ -0,0 +1,99 @@
+import { type VgiSchema, type VgiDataType, type VgiBatch } from "../arrow/index.js";
+import type { OutputCollector } from "@query-farm/vgi-rpc";
+import type { BindRequest, InitRequest, GlobalInitResponse } from "../protocol/types.js";
+import type { TableCardinality } from "../types.js";
+import type { VgiFunction, FunctionExample } from "./types.js";
+import { FunctionStability } from "../types.js";
+import { BoundStorage } from "./storage.js";
+export interface TableBufferingBindParams<TArgs = Record<string, any>> {
+    args: TArgs;
+    bindCall: BindRequest;
+    settings: Record<string, any>;
+    secrets: Record<string, Record<string, any>>;
+    /** True on the second bind pass, after the connector resolved the secrets
+     *  requested via the onBind `lookupSecret*` return fields on the first pass. */
+    resolvedSecretsProvided: boolean;
+}
+export interface TableBufferingParams<TArgs = Record<string, any>> {
+    args: TArgs;
+    initCall: InitRequest;
+    outputSchema: VgiSchema;
+    settings: Record<string, any>;
+    secrets: Record<string, Record<string, any>>;
+    /** Shared cross-process storage scoped by execution_id. */
+    storage: BoundStorage;
+    /** Stable across coordinator + secondary workers for one DuckDB execution. */
+    executionId: Uint8Array;
+    /** Catalog attach identity (plaintext bytes). */
+    attachId: Uint8Array;
+    /** Hex/raw VGI transaction id, or null. */
+    transactionId: Uint8Array | null;
+    function_name: string;
+    /** Per-chunk batch_index when requiresInputBatchIndex=true; else null. */
+    batchIndex: number | null;
+    /** In-band log sink for the unary process()/combine() RPCs. */
+    clientLog: (level: string, message: string) => void;
+}
+export interface TableBufferingConfig<TArgs = Record<string, any>, TState = any> {
+    name: string;
+    description?: string;
+    args?: Record<string, VgiDataType>;
+    namedArgs?: Record<string, VgiDataType>;
+    argDefaults?: Record<string, any>;
+    /** Bind: default passes through input schema. May be async. Return non-empty
+     *  `lookupSecret*` lists on the first pass to request DuckDB secrets; the
+     *  connector then re-binds with `resolvedSecretsProvided` and the resolved
+     *  values are available in the process/finalize params' `secrets`. */
+    onBind?: (params: TableBufferingBindParams<TArgs>) => {
+        outputSchema: VgiSchema;
+        opaqueData?: Uint8Array;
+        lookupSecretTypes?: string[];
+        lookupScopes?: string[];
+        lookupNames?: string[];
+    } | Promise<{
+        outputSchema: VgiSchema;
+        opaqueData?: Uint8Array;
+        lookupSecretTypes?: string[];
+        lookupScopes?: string[];
+        lookupNames?: string[];
+    }>;
+    /** Sink: ingest one batch, return an opaque state_id. */
+    process: (batch: VgiBatch, params: TableBufferingParams<TArgs>) => Uint8Array | Promise<Uint8Array>;
+    /** Combine: group/merge state_ids, return finalize_state_ids. */
+    combine: (stateIds: Uint8Array[], params: TableBufferingParams<TArgs>) => Uint8Array[] | Promise<Uint8Array[]>;
+    /** Build the initial per-tick finalize state for a finalize_state_id. */
+    initialFinalizeState?: (finalizeStateId: Uint8Array, params: TableBufferingParams<TArgs>) => TState | Promise<TState>;
+    /** Source tick: emit one batch via out.emit / signal EOS via out.finish. */
+    finalize: (params: TableBufferingParams<TArgs>, finalizeStateId: Uint8Array, state: TState, out: OutputCollector) => void | Promise<void>;
+    cardinality?: (params: TableBufferingBindParams<TArgs>) => TableCardinality | Promise<TableCardinality>;
+    projectionPushdown?: boolean;
+    filterPushdown?: boolean;
+    autoApplyFilters?: boolean;
+    /** Force ParallelSink=false in the C++ operator (single-thread ingest). */
+    sinkOrderDependent?: boolean;
+    /** Force serial Source drain in finalize_queue order. */
+    sourceOrderDependent?: boolean;
+    /** Thread DuckDB's per-chunk batch_index into every process() call. */
+    requiresInputBatchIndex?: boolean;
+    stability?: FunctionStability;
+    examples?: FunctionExample[];
+    categories?: string[];
+    tags?: Record<string, string>;
+    maxWorkers?: number;
+    requiredSettings?: string[];
+    requiredSecrets?: string[];
+}
+/**
+ * A table_buffering VgiFunction also carries its callback config so the
+ * protocol's unary handlers (process/combine/destructor) can reach it.
+ */
+export interface TableBufferingVgiFunction extends VgiFunction {
+    bufferingConfig: TableBufferingConfig<any, any>;
+    bufferingExtractArgs: (request: BindRequest) => any;
+}
+export declare function defineTableBufferingFunction<TArgs = Record<string, any>, TState = any>(config: TableBufferingConfig<TArgs, TState>): TableBufferingVgiFunction;
+export declare function encodeBufferingInit(request: InitRequest, response: GlobalInitResponse): Uint8Array;
+export declare function decodeBufferingInit(payload: Uint8Array): {
+    request: InitRequest;
+    response: GlobalInitResponse;
+};

package/dist/functions/table-in-out.d.ts

@@ -0,0 +1,64 @@
+import { type VgiSchema, type VgiDataType, type VgiBatch } from "../arrow/index.js";
+import type { OutputCollector } from "@query-farm/vgi-rpc";
+import type { BindRequest, InitRequest, GlobalInitResponse } from "../protocol/types.js";
+import type { VgiFunction, FunctionExample } from "./types.js";
+import { type PushdownFilters } from "../filter-pushdown/index.js";
+import { FunctionStability } from "../types.js";
+import { BoundStorage } from "./storage.js";
+export interface TableInOutBindParams<TArgs = Record<string, any>> {
+    args: TArgs;
+    bindCall: BindRequest;
+    settings: Record<string, any>;
+    secrets: Record<string, Record<string, any>>;
+}
+export interface TableInOutProcessParams<TArgs = Record<string, any>> {
+    args: TArgs;
+    initCall: InitRequest;
+    initResponse: GlobalInitResponse;
+    outputSchema: VgiSchema;
+    settings: Record<string, any>;
+    secrets: Record<string, Record<string, any>>;
+    pushdownFilters?: PushdownFilters;
+    /** Shared storage for cross-phase and cross-worker data (SQLite-backed). */
+    storage: BoundStorage;
+}
+export interface TableInOutConfig<TArgs = Record<string, any>, TState = null> {
+    name: string;
+    description?: string;
+    args?: Record<string, VgiDataType>;
+    /** Named arguments (optional, DuckDB passes by name) */
+    namedArgs?: Record<string, VgiDataType>;
+    /** Argument defaults */
+    argDefaults?: Record<string, any>;
+    /** Bind: default passes through input schema. May be async. */
+    onBind?: (params: TableInOutBindParams<TArgs>) => {
+        outputSchema: VgiSchema;
+        opaqueData?: Uint8Array;
+    } | Promise<{
+        outputSchema: VgiSchema;
+        opaqueData?: Uint8Array;
+    }>;
+    onInit?: (params: {
+        args: TArgs;
+        initCall: InitRequest;
+        outputSchema: VgiSchema;
+        executionId: Uint8Array;
+    }) => GlobalInitResponse | Promise<GlobalInitResponse>;
+    initialState?: (params: TableInOutProcessParams<TArgs>) => TState;
+    /** Process: transform input batch, emit output via out */
+    process?: (params: TableInOutProcessParams<TArgs>, state: TState, batch: VgiBatch, out: OutputCollector) => void | Promise<void>;
+    /** Finalize: emit final batches after all input processed.
+     *  Receives all worker states collected from storage (matches Python's finish(params, states)). */
+    finalize?: (params: TableInOutProcessParams<TArgs>, states: TState[]) => VgiBatch[] | Promise<VgiBatch[]>;
+    projectionPushdown?: boolean;
+    filterPushdown?: boolean;
+    autoApplyFilters?: boolean;
+    stability?: FunctionStability;
+    examples?: FunctionExample[];
+    categories?: string[];
+    tags?: Record<string, string>;
+    maxWorkers?: number;
+    requiredSettings?: string[];
+    requiredSecrets?: string[];
+}
+export declare function defineTableInOutFunction<TArgs = Record<string, any>, TState = null>(config: TableInOutConfig<TArgs, TState>): VgiFunction;

package/dist/functions/table.d.ts

@@ -0,0 +1,118 @@
+import { type VgiSchema, type VgiDataType } from "../arrow/index.js";
+import type { OutputCollector } from "@query-farm/vgi-rpc";
+import type { BindRequest, InitRequest, GlobalInitResponse, TableCardinality } from "../protocol/types.js";
+import type { VgiFunction, FunctionExample } from "./types.js";
+import { type PushdownFilters } from "../filter-pushdown/index.js";
+import type { ColumnStatistics } from "../util/statistics.js";
+import { FunctionStability, type NullHandling, type OrderPreservation, type OrderDependence, type DistinctDependence } from "../types.js";
+import { BoundStorage } from "./storage.js";
+export interface TableBindParams<TArgs = Record<string, any>> {
+    args: TArgs;
+    bindCall: BindRequest;
+    settings: Record<string, any>;
+    secrets: Record<string, Record<string, any>>;
+    resolvedSecretsProvided: boolean;
+}
+export interface TableProcessParams<TArgs = Record<string, any>> {
+    args: TArgs;
+    initCall: InitRequest;
+    initResponse: GlobalInitResponse;
+    outputSchema: VgiSchema;
+    settings: Record<string, any>;
+    secrets: Record<string, Record<string, any>>;
+    pushdownFilters?: PushdownFilters;
+    storage?: BoundStorage;
+    /**
+     * AT (TIMESTAMP|VERSION) clause for this scan, or `undefined` when the scan
+     * has no AT clause. Carried on the per-scan bind embedded in the init request
+     * (`initCall.bind_call.at_unit` / `.at_value`), so function-backed tables can
+     * read time travel at init alongside their pushdown filters. Mirrors
+     * vgi-python's `ProcessParams.at_unit` / `.at_value`. See `BindRequest.at_unit`.
+     */
+    atUnit?: string;
+    atValue?: string;
+}
+export interface TableFunctionConfig<TArgs = Record<string, any>, TState = null> {
+    name: string;
+    description?: string;
+    /** Argument schema (positional args) */
+    args?: Record<string, VgiDataType>;
+    /** Argument docs */
+    argDocs?: Record<string, string>;
+    /** Argument defaults */
+    argDefaults?: Record<string, any>;
+    /** Names of args that accept variable number of arguments */
+    varargs?: string[];
+    /** Bind: return output schema. May be async — handlers `await` the result. */
+    onBind: (params: TableBindParams<TArgs>) => {
+        outputSchema: VgiSchema;
+        opaqueData?: Uint8Array;
+        lookupSecretTypes?: string[];
+        lookupScopes?: string[];
+        lookupNames?: string[];
+    } | Promise<{
+        outputSchema: VgiSchema;
+        opaqueData?: Uint8Array;
+        lookupSecretTypes?: string[];
+        lookupScopes?: string[];
+        lookupNames?: string[];
+    }>;
+    /** Init (optional). May be async — common when storage is HTTP-backed. */
+    onInit?: (params: {
+        args: TArgs;
+        initCall: InitRequest;
+        outputSchema: VgiSchema;
+        executionId: Uint8Array;
+        storage: BoundStorage;
+    }) => GlobalInitResponse | Promise<GlobalInitResponse>;
+    /** State factory */
+    initialState?: (params: TableProcessParams<TArgs>) => TState;
+    /** Process: emit batches via out, call out.finish() when done */
+    process: (params: TableProcessParams<TArgs>, state: TState, out: OutputCollector) => void | Promise<void>;
+    /** Cardinality hints */
+    cardinality?: (params: TableBindParams<TArgs>) => TableCardinality | Promise<TableCardinality>;
+    /**
+     * Per-column statistics for the function's output. Returned to DuckDB via
+     * the `table_function_statistics` RPC; the optimizer uses min/max to
+     * eliminate impossible filters at plan time (folding scans to
+     * EMPTY_RESULT). Return `null` or an empty array when bounds are unknown.
+     */
+    statistics?: (params: TableBindParams<TArgs>) => ColumnStatistics[] | null;
+    /**
+     * Per-execution diagnostics surfaced under EXPLAIN ANALYZE. DuckDB calls
+     * this once per parallel scan thread at pipeline FinishSource via the
+     * `table_function_dynamic_to_string` RPC. Return ordered key→value
+     * strings; the C++ extension merges these with the intrinsic keys
+     * (Function, Rows Read, Threads). The framework provides a BoundStorage
+     * keyed by the global execution_id so process() can persist counters
+     * that this callback then reads back — see profiling_demo for the
+     * canonical pattern.
+     */
+    dynamicToString?: (params: TableBindParams<TArgs>, executionId: Uint8Array, storage: BoundStorage) => Record<string, string> | Promise<Record<string, string>>;
+    projectionPushdown?: boolean;
+    filterPushdown?: boolean;
+    samplingPushdown?: boolean;
+    /** Opt in to DuckDB's late-materialization SEMI-join rewrite; requires a
+     *  UNIQUE, snapshot-stable rowid column. FunctionInfo.late_materialization. */
+    lateMaterialization?: boolean;
+    supportedExpressionFilters?: string[];
+    autoApplyFilters?: boolean;
+    stability?: FunctionStability;
+    examples?: FunctionExample[];
+    categories?: string[];
+    tags?: Record<string, string>;
+    maxWorkers?: number;
+    requiredSettings?: string[];
+    requiredSecrets?: string[];
+    /** Row-order preservation behavior; flows to DuckDB's
+     *  TableFunction::order_preservation_type. */
+    preservesOrder?: OrderPreservation;
+    nullHandling?: NullHandling;
+    orderDependent?: OrderDependence;
+    distinctDependent?: DistinctDependence;
+    /** Emit per-batch `vgi_batch_index`; FunctionInfo.supports_batch_index. */
+    supportsBatchIndex?: boolean;
+    /** Hive-style partition-columns mode; FunctionInfo.partition_kind. */
+    partitionKind?: "NOT_PARTITIONED" | "SINGLE_VALUE_PARTITIONS" | "OVERLAPPING_PARTITIONS" | "DISJOINT_PARTITIONS";
+}
+export declare function defineTableFunction<TArgs = Record<string, any>, TState = null>(config: TableFunctionConfig<TArgs, TState>): VgiFunction;