@query-farm/vgi 0.31.0 → 0.32.0

package/src/functions/storage.ts

@@ -84,9 +84,22 @@ function hexEncode(buf: Uint8Array): string {
 // FunctionStorage interface
 // ============================================================================
 
+/**
+ * Which slot of an execution's worker state a row occupies: a table-in-out
+ * substream's client-minted `substream_id` (raw bytes), or -- only for a client
+ * that sent none -- a numeric worker id, the process id (an int64 key). See
+ * {@link BoundStorage.put}.
+ */
+export type WorkerStateKey = number | Uint8Array;
+
 export interface FunctionStorage {
-  workerPut(executionId: Uint8Array, workerId: number, state: Uint8Array): Promise<void>;
+  /** Upsert `state` into the execution's worker-state slot `worker`. */
+  workerPut(executionId: Uint8Array, worker: WorkerStateKey, state: Uint8Array): Promise<void>;
+  /** Drain every worker-state slot of the execution, whatever its key. */
   workerCollect(executionId: Uint8Array): Promise<Uint8Array[]>;
+  /** Read every worker-state slot without draining. A slot keyed by a
+   *  numeric worker id reports that id; one keyed by bytes (a substream id)
+   *  has none and reports 0. */
   workerScan(executionId: Uint8Array): Promise<Array<[number, Uint8Array]>>;
   queuePush(executionId: Uint8Array, items: Uint8Array[]): Promise<number>;
   queuePop(executionId: Uint8Array): Promise<Uint8Array | null>;
@@ -148,8 +161,9 @@ function getDefaultDbPath(): string {
 }
 
 // Worker state rides the unified function_state table under a reserved
-// namespace, keyed by the worker/process id (8-byte big-endian) — exactly the
-// mapping the Cloudflare DO client uses, so both backends share one schema.
+// namespace, keyed by the substream id (its raw bytes) or, for a client that
+// sent none, the worker/process id (8-byte big-endian) — exactly the mapping
+// the Cloudflare DO client uses, so both backends share one schema.
 const NS_WORKER = new TextEncoder().encode("worker");
 
 /** Encode an int64 worker id as an 8-byte big-endian state key. */
@@ -159,6 +173,12 @@ function int64Key(v: number): Uint8Array {
   return b;
 }
 
+/** The state key for a worker-state slot: a substream id as-is, a numeric
+ *  worker id as an int64. */
+export function workerStateKey(worker: WorkerStateKey): Uint8Array {
+  return typeof worker === "number" ? int64Key(worker) : worker;
+}
+
 function int64FromKey(b: Uint8Array): number {
   if (b.length !== 8) return 0;
   return Number(new DataView(b.buffer, b.byteOffset, 8).getBigInt64(0, false));
@@ -300,18 +320,19 @@ export class FunctionStorageSqlite implements FunctionStorage {
     return total;
   }
 
-  // Worker state rides function_state under ns=worker, keyed by worker id.
+  // Worker state rides function_state under ns=worker, keyed by substream id
+  // or worker id.
 
   async workerPut(
     executionId: Uint8Array,
-    workerId: number,
+    worker: WorkerStateKey,
     state: Uint8Array
   ): Promise<void> {
     this._db
       .prepare(
         "INSERT OR REPLACE INTO function_state (scope_id, ns, key, value) VALUES (?, ?, ?, ?)"
       )
-      .run(bufferArg(executionId), bufferArg(NS_WORKER), bufferArg(int64Key(workerId)), bufferArg(state));
+      .run(bufferArg(executionId), bufferArg(NS_WORKER), bufferArg(workerStateKey(worker)), bufferArg(state));
   }
 
   async workerCollect(executionId: Uint8Array): Promise<Uint8Array[]> {
@@ -381,14 +402,36 @@ export class FunctionStorageSqlite implements FunctionStorage {
 export class BoundStorage {
   private _base: FunctionStorage;
   private _executionId: Uint8Array;
-
-  constructor(storage: FunctionStorage, executionId: Uint8Array) {
+  private _substreamId: Uint8Array | null;
+
+  /**
+   * @param substreamId The table-in-out substream this storage serves -- the
+   *   client-minted `InitRequest.substream_id`, identical across the
+   *   substream's init, every tick and its finalize -- or `null` when the
+   *   client sent none. Keys {@link put}.
+   */
+  constructor(storage: FunctionStorage, executionId: Uint8Array, substreamId: Uint8Array | null = null) {
     this._base = storage;
     this._executionId = executionId;
-  }
-
+    this._substreamId = substreamId;
+  }
+
+  /**
+   * Upsert this substream's state into the execution's worker state, which
+   * {@link collect} drains at finalize.
+   *
+   * One row per **substream**, keyed by its `substream_id`. Storage is
+   * already scoped to the execution, and every connection of a fanned-out
+   * scan shares that execution -- so the key has to tell those connections
+   * apart. The process id only does when each connection is its own process;
+   * under the launcher, TCP or HTTP one process serves many of them, and a
+   * per-process key let them overwrite each other's state so finish()
+   * undercounted. The pid remains only for a client that sends no
+   * `substream_id`, which it keys as before. Mirrors vgi-python's
+   * `TableInOutFunction.process` (072e543).
+   */
   put(state: Uint8Array): Promise<void> {
-    return this._base.workerPut(this._executionId, process.pid, state);
+    return this._base.workerPut(this._executionId, this._substreamId ?? process.pid, state);
   }
 
   collect(): Promise<Uint8Array[]> {

package/src/functions/table-buffering.ts

@@ -347,7 +347,6 @@ export function defineTableBufferingFunction<
     createStreamHandlers(
       request: InitRequest,
       response: GlobalInitResponse,
-      _accumulatedState?: any,
     ): StreamHandlers {
       const args = extractArgs(request.bind_call);
       const settings = batchToScalarDict(request.bind_call.settings);

package/src/functions/table-in-out.ts

@@ -3,7 +3,7 @@
 // Two-phase: INPUT phase receives and transforms batches,
 // FINALIZE phase emits final results.
 
-import { type VgiSchema, schema, type VgiField, type VgiDataType, type VgiBatch, nullType, withBatchMetadata, serializeBatch, deserializeBatch } from "../arrow/index.js";
+import { type VgiSchema, schema, type VgiField, type VgiDataType, type VgiBatch, nullType, isNull, withBatchMetadata, serializeBatch, deserializeBatch } from "../arrow/index.js";
 import type { OutputCollector } from "@query-farm/vgi-rpc";
 import { DEFAULT_MAX_WORKERS, TableInOutPhase } from "../types.js";
 import type {
@@ -33,6 +33,13 @@ import {
   FilteringOutputCollector,
   type PushdownFilters,
 } from "../filter-pushdown/index.js";
+import {
+  decodeDynamicFilterMetadata,
+  EMPTY_FILTER_HISTORY,
+  type FilterDeltaHistory,
+  recordFilterDelta,
+  replayFilterHistory,
+} from "../filter-pushdown/history.js";
 import { FunctionStability } from "../types.js";
 import { BoundStorage, storage as defaultStorage, FrameworkNS } from "./storage.js";
 import { serializeUserState, deserializeUserState } from "../protocol/state-serializer.js";
@@ -70,12 +77,14 @@ export interface TableInOutProcessParams<TArgs = Record<string, any>> {
   storage: BoundStorage;
   /**
    * Stable client-minted id for this streaming table-in-out substream.
-   * Present (identical across init / every process() / finalize) when the
-   * client fanned this function out across per-substream workers; use it to
-   * key per-substream accumulated state in shared storage so a finalize()
-   * that lands on a different HTTP backend than the process() calls still
-   * finds it. `null`/`undefined` for the serial path or an old client.
-   * Mirrors vgi-python's `ProcessParams.substream_id`.
+   * Present (identical across init / every process() / finalize) whenever the
+   * client sent one -- the DuckDB extension does for every substream. The
+   * framework keys the accumulated state it auto-persists by it
+   * ({@link BoundStorage.put}), so substreams of one execution that one
+   * process serves keep separate states and finalize() receives each; use it
+   * likewise for any per-substream state you keep in shared storage yourself.
+   * `null`/`undefined` for a client that sent none. Mirrors vgi-python's
+   * `ProcessParams.substream_id`.
    */
   substreamId?: Uint8Array | null;
   /**
@@ -298,7 +307,7 @@ export function defineTableInOutFunction<
     createStreamHandlers(
       request: InitRequest,
       response: GlobalInitResponse,
-      accumulatedState?: any,
+      filterHistory?: FilterDeltaHistory | null,
     ): StreamHandlers {
       const args = extractArgs(request.bind_call);
       const settings = batchToScalarDict(request.bind_call.settings);
@@ -315,20 +324,31 @@ export function defineTableInOutFunction<
       // Deserialize pushdown filters. Pass a join-keys column lookup so that
       // filters DuckDB promoted to join_keys (IN/OR lists, etc.) are
       // materialized as InFilters rather than silently dropped.
-      const pushdownFilters = request.pushdown_filters
-        ? deserializeFilters(request.pushdown_filters, {
-          outputSchema: request.output_schema,
-          joinKeyBatches: request.join_keys,
-          extensionFunctions: meta.additionalFilterFunctions,
-          runtimeAlgorithms: meta.runtimeFilterAlgorithms,
-          evaluationContexts: meta.filterEvaluationContexts,
-        })
-        : undefined;
-
-      // Create BoundStorage for cross-phase/cross-worker data sharing
+      // An HTTP continuation replays the dynamic-filter deltas its cursor
+      // carries, so the filters are the ones the previous exchange left.
+      const carriedFilterHistory = filterHistory ?? EMPTY_FILTER_HISTORY;
+      const pushdownFilters = replayFilterHistory(
+        request.pushdown_filters
+          ? deserializeFilters(request.pushdown_filters, {
+            outputSchema: request.output_schema,
+            joinKeyBatches: request.join_keys,
+            extensionFunctions: meta.additionalFilterFunctions,
+            runtimeAlgorithms: meta.runtimeFilterAlgorithms,
+            evaluationContexts: meta.filterEvaluationContexts,
+          })
+          : undefined,
+        carriedFilterHistory,
+      );
+
+      // Create BoundStorage for cross-phase/cross-worker data sharing. Keyed
+      // per substream, not per process: every connection of a fanned-out
+      // scan shares this execution's storage, and one process may serve many
+      // of them (see BoundStorage.put). Rebuilt from the sealed init request
+      // on every HTTP continuation, so the key is the same on every tick.
       const boundStorage = new BoundStorage(
         defaultStorage,
         response.execution_id ?? new Uint8Array(16),
+        request.substream_id ?? null,
       );
 
       const processParams: TableInOutProcessParams<TArgs> = {
@@ -382,12 +402,10 @@ export function defineTableInOutFunction<
           ) => {
             if (!pState.state.materialized) {
               const finalizeStates: TState[] = [];
-              if (accumulatedState != null) {
-                finalizeStates.push(accumulatedState as TState);
-              }
-              // Collect any states persisted by INPUT exchanges. Subprocess
-              // path reads from SQLite; HTTP path reads from CF DO / wherever
-              // the configured storage backend lives.
+              // Collect the states INPUT exchanges persisted -- the only
+              // source, on every transport: each INPUT turn upserts its
+              // substream's state here (SQLite, a CF DO, wherever the
+              // configured backend lives).
               const stored = await boundStorage.collect();
               for (const bytes of stored) {
                 finalizeStates.push(deserializeUserState(bytes) as TState);
@@ -467,11 +485,12 @@ export function defineTableInOutFunction<
       return {
         outputSchema,
         inputSchema: request.bind_call.input_schema,
-        exchangeInit: () => ({ state, processParams }),
+        exchangeInit: () => ({ state, processParams, filterHistory: carriedFilterHistory }),
         exchangeFn: async (
           eState: {
             state: TState;
             processParams: TableInOutProcessParams<TArgs>;
+            filterHistory: FilterDeltaHistory;
           },
           input: VgiBatch,
           out: OutputCollector
@@ -480,7 +499,7 @@ export function defineTableInOutFunction<
           // custom metadata (attached by the C++ WriteInputBatch), so process()
           // can answer a 0-row `notModified` batch instead of recomputing.
           applyRevalidationValidators(eState.processParams, input);
-          applyDynamicFilterUpdate(eState.processParams, input);
+          applyDynamicFilterUpdate(eState, input);
           // Bake emit metadata onto the batch (0-row HTTP survival), then
           // reconcile emitted batches to the (possibly projected) output schema
           // by name — a process() may emit its full declared schema and let the
@@ -529,27 +548,23 @@ function applyRevalidationValidators(
   params.ifModifiedSince = md?.get(CACHE_IF_MODIFIED_SINCE_KEY);
 }
 
+/**
+ * Apply the dynamic-filter delta an input batch carries (if any) and record it
+ * in the compacted history an HTTP cursor carries to the next exchange, so a
+ * tightened filter survives the turn boundary (the extension sends each delta
+ * once per stream). See `filter-pushdown/history.ts`.
+ */
 function applyDynamicFilterUpdate(
-  params: { pushdownFilters?: PushdownFilters },
+  eState: { processParams: { pushdownFilters?: PushdownFilters }; filterHistory: FilterDeltaHistory },
   input: VgiBatch,
 ): void {
   const encoded = ((input as any)?.metadata as Map<string, string> | undefined)?.get("vgi_pushdown_filters");
   if (!encoded) return;
-  if (!params.pushdownFilters) throw new Error("dynamic filter delta received without an initial snapshot");
-  if (new TextEncoder().encode(encoded).byteLength > (24 << 20)) {
-    throw new Error("base64 dynamic filter metadata exceeds the encoded-size limit");
-  }
-  if (encoded.length % 4 !== 0 ||
-      !/^(?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?$/.test(encoded)) {
-    throw new Error("dynamic filter metadata is not valid base64");
-  }
-  const binary = (globalThis as any).atob
-    ? (globalThis as any).atob(encoded)
-    : Buffer.from(encoded, "base64").toString("binary");
-  const bytes = Uint8Array.from(binary, (character: string) => character.charCodeAt(0));
-  if (bytes.byteLength > (17 << 20)) throw new Error("dynamic filter IPC payload exceeds the encoded-size limit");
-  const current = params.pushdownFilters;
-  params.pushdownFilters = current.applyDelta(deserializeBatch(bytes));
+  const current = eState.processParams.pushdownFilters;
+  if (!current) throw new Error("dynamic filter delta received without an initial snapshot");
+  const recorded = recordFilterDelta(current, eState.filterHistory, decodeDynamicFilterMetadata(encoded));
+  eState.processParams.pushdownFilters = recorded.filters;
+  eState.filterHistory = recorded.history;
 }
 
 /**
@@ -725,14 +740,17 @@ export interface RowTransformConfig<TArgs = Record<string, any>> {
   /**
    * Positional args = the per-row INPUT COLUMNS (real typed args on the wire,
    * no synthetic TABLE placeholder). Read from `batch` by declared name in
-   * process(); NOT surfaced on `params.args`.
+   * process(); NOT surfaced on `params.args`. A `Null` type declares the
+   * column ANY: the input column then arrives with whatever type DuckDB
+   * resolved for the call (see `params.bindCall.input_schema` in onBind).
    */
   args?: Record<string, VgiDataType>;
   /**
    * Trailing VARARGS input columns: the per-row input is N columns of the
    * declared type. A varargs blended function has no per-column declared
    * names (the C++ bind names them col0..colN-1), so process() reads the
-   * columns POSITIONALLY off `batch`.
+   * columns POSITIONALLY off `batch`. A `Null` element type declares ANY
+   * varargs: each runtime column resolves its own type.
    */
   varargs?: { name: string; type: VgiDataType; doc?: string };
   /** Named (string-position) args stay bind-time scalars on `params.args`. */
@@ -805,20 +823,30 @@ export function defineRowTransformFunction<
   let posIdx = 0;
   if (config.args) {
     for (const [name, type] of Object.entries(config.args)) {
+      // A Null arrow type declares the input column ANY (same convention as
+      // defineTableFunction / defineScalarFunction): advertised on the wire as
+      // vgi_type=any, and the client builds the input schema from the type
+      // DuckDB resolved for the call. Read the concrete type off
+      // bindCall.input_schema in onBind.
+      const isAny = isNull(type);
       specs.push({
         name,
         position: posIdx++,
-        arrowType: type,
+        arrowType: isAny ? nullType() : type,
+        isAnyType: isAny,
         doc: config.argDocs?.[name],
         ...constraintSpecFields(config.argConstraints?.[name]),
       });
     }
   }
   if (config.varargs) {
+    // ANY varargs: every runtime column resolves its own type.
+    const isAny = isNull(config.varargs.type);
     specs.push({
       name: config.varargs.name,
       position: posIdx++,
-      arrowType: config.varargs.type,
+      arrowType: isAny ? nullType() : config.varargs.type,
+      isAnyType: isAny,
       isVarargs: true,
       doc: config.varargs.doc ?? config.argDocs?.[config.varargs.name],
       ...constraintSpecFields(config.argConstraints?.[config.varargs.name]),
@@ -919,7 +947,7 @@ export function defineRowTransformFunction<
     createStreamHandlers(
       request: InitRequest,
       response: GlobalInitResponse,
-      _accumulatedState?: any,
+      filterHistory?: FilterDeltaHistory | null,
     ): StreamHandlers {
       const args = extractArgs(request.bind_call);
       const settings = batchToScalarDict(request.bind_call.settings);
@@ -932,19 +960,26 @@ export function defineRowTransformFunction<
         ? projectSchema(projIds, request.output_schema)
         : request.output_schema;
 
-      const pushdownFilters = request.pushdown_filters
-        ? deserializeFilters(request.pushdown_filters, {
-          outputSchema: request.output_schema,
-          joinKeyBatches: request.join_keys,
-          extensionFunctions: meta.additionalFilterFunctions,
-          runtimeAlgorithms: meta.runtimeFilterAlgorithms,
-          evaluationContexts: meta.filterEvaluationContexts,
-        })
-        : undefined;
+      // An HTTP continuation replays the dynamic-filter deltas its cursor
+      // carries, so the filters are the ones the previous exchange left.
+      const carriedFilterHistory = filterHistory ?? EMPTY_FILTER_HISTORY;
+      const pushdownFilters = replayFilterHistory(
+        request.pushdown_filters
+          ? deserializeFilters(request.pushdown_filters, {
+            outputSchema: request.output_schema,
+            joinKeyBatches: request.join_keys,
+            extensionFunctions: meta.additionalFilterFunctions,
+            runtimeAlgorithms: meta.runtimeFilterAlgorithms,
+            evaluationContexts: meta.filterEvaluationContexts,
+          })
+          : undefined,
+        carriedFilterHistory,
+      );
 
       const boundStorage = new BoundStorage(
         defaultStorage,
         response.execution_id ?? new Uint8Array(16),
+        request.substream_id ?? null,
       );
 
       const processParams: RowTransformProcessParams<TArgs> = {
@@ -986,14 +1021,18 @@ export function defineRowTransformFunction<
       return {
         outputSchema,
         inputSchema: request.bind_call.input_schema,
-        exchangeInit: () => ({ state: null, processParams }),
+        exchangeInit: () => ({ state: null, processParams, filterHistory: carriedFilterHistory }),
         exchangeFn: async (
-          eState: { state: null; processParams: RowTransformProcessParams<TArgs> },
+          eState: {
+            state: null;
+            processParams: RowTransformProcessParams<TArgs>;
+            filterHistory: FilterDeltaHistory;
+          },
           input: VgiBatch,
           out: OutputCollector,
         ) => {
           applyRevalidationValidators(eState.processParams, input);
-          applyDynamicFilterUpdate(eState.processParams, input);
+          applyDynamicFilterUpdate(eState, input);
           let wrappedOut: OutputCollector = makeMetadataBakingCollector(out);
           if (projIds) {
             wrappedOut = makeSchemaReconcilingCollector(wrappedOut, outputSchema);

package/src/functions/table.ts

@@ -9,7 +9,6 @@ import {
   isBatch,
   isNull,
   nullType,
-  deserializeBatch,
   readCanonicalValue,
 } from "../arrow/index.js";
 import { codecFor } from "../arrow/codec/registry.js";
@@ -47,6 +46,13 @@ import {
   FilteringOutputCollector,
   type PushdownFilters,
 } from "../filter-pushdown/index.js";
+import {
+  decodeDynamicFilterMetadata,
+  EMPTY_FILTER_HISTORY,
+  type FilterDeltaHistory,
+  recordFilterDelta,
+  replayFilterHistory,
+} from "../filter-pushdown/history.js";
 import type { ColumnStatistics } from "../util/statistics.js";
 import {
   FunctionStability,
@@ -58,30 +64,6 @@ import {
 import { BoundStorage, storage as globalStorage } from "./storage.js";
 import { CACHE_IF_MODIFIED_SINCE_KEY, CACHE_IF_NONE_MATCH_KEY } from "../cache-control.js";
 
-// Base64-decode a string into raw bytes. Used to unpack the dynamic filter
-// update DuckDB attaches to each tick batch's custom metadata.
-function base64Decode(s: string): Uint8Array {
-  if (new TextEncoder().encode(s).byteLength > (24 << 20)) {
-    throw new Error("base64 dynamic filter metadata exceeds the encoded-size limit");
-  }
-  if (s.length % 4 !== 0 || !/^(?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?$/.test(s)) {
-    throw new Error("dynamic filter metadata is not valid base64");
-  }
-  const bin = (globalThis as any).atob ? (globalThis as any).atob(s) : Buffer.from(s, "base64").toString("binary");
-  const out = new Uint8Array(bin.length);
-  for (let i = 0; i < bin.length; i++) out[i] = bin.charCodeAt(i);
-  if (out.byteLength > (17 << 20)) throw new Error("dynamic filter IPC payload exceeds the encoded-size limit");
-  return out;
-}
-
-// Read the first RecordBatch from an Arrow IPC stream buffer. Returns null if
-// the stream has no batches.
-function deserializeFilterBatch(bytes: Uint8Array): VgiBatch | null {
-  if (!bytes || bytes.length === 0) return null;
-  const batch = deserializeBatch(bytes);
-  return batch.numRows === 0 && batch.schema.fields.length === 0 ? null : batch;
-}
-
 // Wrap an OutputCollector so each emitted RecordBatch is projected-by-name
 // to the bound outputSchema before forwarding. Lenient: workers can emit
 // over-wide batches (full declared schema) and the framework drops the
@@ -191,6 +173,17 @@ export interface TableProcessParams<TArgs = Record<string, any>> {
   ifModifiedSince?: string;
 }
 
+/**
+ * A table stream's live handler state: the user's state, the params its
+ * process() sees, and the compacted dynamic-filter history an HTTP cursor
+ * carries to the next turn (see `filter-pushdown/history.ts`).
+ */
+interface TableHandlerState<TArgs, TState> {
+  state: TState;
+  processParams: TableProcessParams<TArgs>;
+  filterHistory: FilterDeltaHistory;
+}
+
 // ============================================================================
 // Functional API
 // ============================================================================
@@ -529,7 +522,8 @@ export function defineTableFunction<
 
     createStreamHandlers(
       request: InitRequest,
-      response: GlobalInitResponse
+      response: GlobalInitResponse,
+      filterHistory?: FilterDeltaHistory | null,
     ): StreamHandlers {
       const args = extractArgs(request.bind_call);
       const settings = batchToScalarDict(request.bind_call.settings);
@@ -553,9 +547,14 @@ export function defineTableFunction<
         runtimeAlgorithms: meta.runtimeFilterAlgorithms,
         evaluationContexts: meta.filterEvaluationContexts,
       };
-      const pushdownFilters = request.pushdown_filters
-        ? deserializeFilters(request.pushdown_filters, filterOptions)
-        : undefined;
+      // An HTTP continuation replays the dynamic-filter deltas its cursor
+      // carries, so process() sees the filters the previous turn left, not the
+      // init snapshot (the extension sends each delta once per stream).
+      const carriedFilterHistory = filterHistory ?? EMPTY_FILTER_HISTORY;
+      const pushdownFilters = replayFilterHistory(
+        request.pushdown_filters ? deserializeFilters(request.pushdown_filters, filterOptions) : undefined,
+        carriedFilterHistory,
+      );
 
       const boundStorage = new BoundStorage(globalStorage, response.execution_id);
 
@@ -593,9 +592,13 @@ export function defineTableFunction<
 
       return {
         outputSchema,
-        producerInit: () => ({ state, processParams }),
+        producerInit: (): TableHandlerState<TArgs, TState> => ({
+          state,
+          processParams,
+          filterHistory: carriedFilterHistory,
+        }),
         onTick: (
-          pState: { state: TState; processParams: TableProcessParams<TArgs> },
+          pState: TableHandlerState<TArgs, TState>,
           tickMetadata: Map<string, string> | undefined,
         ) => {
           if (!tickMetadata) return;
@@ -613,21 +616,21 @@ export function defineTableFunction<
           // Dynamic filter pushdown: DuckDB's Top-N optimizer tightens filters
           // between ticks and serializes an atomic v2 delta into the tick
           // batch's custom metadata under `vgi_pushdown_filters` (base64 of a
-          // filter IPC stream). Apply it to the prior snapshot so process()
-          // sees the revised advisory predicate state.
+          // filter IPC stream). Apply it to the prior state so process() sees
+          // the revised advisory predicate state, and record it in the
+          // compacted history an HTTP cursor carries to the next turn.
           const encoded = tickMetadata.get("vgi_pushdown_filters");
           if (!encoded) return;
           const current = pState.processParams.pushdownFilters;
           if (!current) throw new Error("dynamic filter delta received without an initial snapshot");
-          const bytes = base64Decode(encoded);
-          const filterBatch = deserializeFilterBatch(bytes);
-          if (!filterBatch) throw new Error("dynamic filter metadata must contain one RecordBatch");
           // Assignment happens only after complete validation, so a rejected
           // delta leaves the immutable prior state intact.
-          pState.processParams.pushdownFilters = current.applyDelta(filterBatch);
+          const recorded = recordFilterDelta(current, pState.filterHistory, decodeDynamicFilterMetadata(encoded));
+          pState.processParams.pushdownFilters = recorded.filters;
+          pState.filterHistory = recorded.history;
         },
         producerFn: async (
-          pState: { state: TState; processParams: TableProcessParams<TArgs> },
+          pState: TableHandlerState<TArgs, TState>,
           out: OutputCollector
         ) => {
           const current = pState.processParams.pushdownFilters;

package/src/functions/types.ts

@@ -26,6 +26,12 @@ import type { ArgumentSpec } from "../arguments/argument-spec.js";
 export interface HandlerState<T = any> {
   /** Serializable user state for HTTP exchange round-trips. */
   state: T;
+  /**
+   * The compacted dynamic-filter history, for a stream that applies
+   * `vgi_pushdown_filters` deltas. Carried in the HTTP cursor so the next turn
+   * rebuilds the filters this one left.
+   */
+  filterHistory?: import("../filter-pushdown/history.js").FilterDeltaHistory;
   [key: string]: any;
 }
 
@@ -229,10 +235,17 @@ export interface VgiFunction {
   defaultOutputSchema?: VgiSchema;
   bind(request: BindRequest): BindResponse | Promise<BindResponse>;
   globalInit(request: InitRequest): GlobalInitResponse | Promise<GlobalInitResponse>;
+  /**
+   * Build the stream's handlers. On an HTTP continuation `filterHistory` is
+   * the compacted dynamic-filter history the cursor carried: a function that
+   * applies `vgi_pushdown_filters` deltas replays it onto the init snapshot and
+   * keeps the (updated) history on its handler state as `filterHistory`, which
+   * the dispatcher writes back into the next cursor.
+   */
   createStreamHandlers(
     request: InitRequest,
     response: GlobalInitResponse,
-    accumulatedState?: any,
+    filterHistory?: import("../filter-pushdown/history.js").FilterDeltaHistory | null,
   ): StreamHandlers;
   cardinality?(request: TableFunctionCardinalityRequest): TableCardinality | Promise<TableCardinality>;
   /**

package/src/http/fetch.ts

@@ -13,7 +13,6 @@
 import {
   AuthContext,
   createHttpHandler,
-  unpackStateToken,
   type AuthenticateFn,
   type DispatchHook,
   type DispatchInfo,
@@ -24,13 +23,13 @@ import {
   type Protocol,
 } from "@query-farm/vgi-rpc";
 import { arrowStateSerializer } from "../protocol/state-serializer.js";
-import { buildVgiProtocol, VGI_PROTOCOL_NAME, type ProtocolConfig } from "../protocol/dispatch.js";
+import { buildVgiProtocol, type ProtocolConfig } from "../protocol/dispatch.js";
 import { createLandingRoutes, type LandingInfo } from "./landing.js";
 import { setRequestAuthScope, type RequestAuthHolder } from "../request-auth.js";
 
 export interface VgiFetchOptions {
   /** Wire-protocol config (registry + catalogInterface). */
-  protocol: Omit<ProtocolConfig, "recoverExchangeState">;
+  protocol: ProtocolConfig;
   /** HMAC key for state-token signing. Pass a stable, secret 32-byte key
    *  (e.g. derived from a Wrangler secret). Required because Workers don't
    *  preserve in-memory state across requests/instances. */
@@ -196,20 +195,6 @@ export function createVgiFetch(opts: VgiFetchOptions): (req: Request) => Promise
   const protocol: Protocol = buildVgiProtocol({
     signingKey: opts.signingKey,
     ...opts.protocol,
-    recoverExchangeState: async (opaqueData: Uint8Array) => {
-      const tokenString = new TextDecoder().decode(opaqueData);
-      // principal binding is enforced by the HTTP handler on the request that
-      // carried this token; the recovery path itself is not principal-scoped,
-      // so the scope names the protocol and leaves the identity anonymous.
-      // `protocol` is required since vgi-rpc 0.24.0: it is bound into the
-      // token's AEAD associated data, so a cursor minted under one hosted
-      // protocol cannot be opened under another.
-      const unpacked = await unpackStateToken(tokenString, opts.signingKey, tokenTtl, {
-        protocol: VGI_PROTOCOL_NAME,
-        principal: undefined,
-      });
-      return arrowStateSerializer.deserialize(unpacked.stateBytes);
-    },
   });
 
   const handler = createHttpHandler(protocol, {

package/src/index.core.ts

@@ -282,6 +282,7 @@ export type { ScalarComputeRow, ScalarOutputValue, ScalarComputeResult } from ".
 // Storage
 export {
   type FunctionStorage,
+  type WorkerStateKey,
   FunctionStorageSqlite,
   BoundStorage,
   UnknownInvocationError,