@query-farm/vgi 0.30.0 → 0.32.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@query-farm/vgi",
-  "version": "0.30.0",
+  "version": "0.32.0",
   "description": "TypeScript framework for extending DuckDB with user-defined functions via the Vector Gateway Interface (VGI) over Arrow IPC.",
   "license": "SEE LICENSE IN LICENSE",
   "author": "Query Farm LLC (https://query.farm)",
@@ -34,7 +34,8 @@
     "./src/index.ts",
     "./src/client-entry.ts",
     "./src/worker-cf-entry.ts",
-    "./src/serve-entry.ts"
+    "./src/serve-entry.ts",
+    "./src/arrow/impl-flechette/compat.ts"
   ],
   "main": "dist/index.js",
   "types": "dist/index.d.ts",
@@ -90,12 +91,12 @@
   },
   "peerDependencies": {
     "@query-farm/apache-arrow": "^21.1.1",
-    "@query-farm/vgi-rpc": ">=0.23.0 <0.24.0"
+    "@query-farm/vgi-rpc": ">=0.24.0 <0.25.0"
   },
   "devDependencies": {
     "@duckdb/node-api": "^1.5.5-r.4",
     "@query-farm/apache-arrow": "^21.1.1",
-    "@query-farm/vgi-rpc": "^0.23.0",
+    "@query-farm/vgi-rpc": "^0.24.0",
     "@types/bun": "latest",
     "typescript": "^7.0.2"
   },

package/src/catalog/item-encoding.ts

@@ -0,0 +1,59 @@
+// Copyright 2025, 2026 Query Farm LLC - https://query.farm
+// Catalog items that are built once and encoded once.
+//
+// A function listing from a descriptor-driven catalog depends only on static
+// metadata, yet every catalog_schema_contents_functions request used to rebuild
+// each FunctionInfo (argument and output schemas serialized to IPC, metadata
+// resolved) and then Arrow-encode it again: ~1.5 ms per function, ~236 ms for
+// the example worker's main schema, paid on every function-set load DuckDB
+// makes. vgi-python fixed the same cost by caching the listing and storing each
+// item's encoding on its (frozen dataclass) instance (6cc522a).
+//
+// JS objects are not frozen by default, and an encoding memo keyed by object
+// identity is only sound for an object that cannot change after it was
+// encoded. So the memo applies to FROZEN items only: a catalog opts in by
+// returning items it froze with `freezeCatalogItem` (ReadOnlyCatalogInterface
+// does), and every other item -- a catalog that builds fresh objects per call,
+// or keeps and mutates its own -- is encoded on every call exactly as before.
+
+import { encodeFunctionInfo, type FunctionInfo } from "../generated/vgi-client.js";
+
+/** Encoded bytes per frozen item instance. Weak, so it never pins an item. */
+const ENCODED = new WeakMap<object, Uint8Array>();
+
+/**
+ * Deep-freeze a catalog item (plain objects and arrays, recursively) so it can
+ * be shared between requests and its encoding memoized.
+ *
+ * Binary fields (`Uint8Array` and other ArrayBuffer views) are left as they are
+ * -- a typed array with elements cannot be frozen -- and must be treated as
+ * read-only by convention, like the rest of a shared item.
+ */
+export function freezeCatalogItem<T extends object>(item: T): Readonly<T> {
+  deepFreeze(item);
+  return item;
+}
+
+function deepFreeze(value: unknown): void {
+  if (value === null || typeof value !== "object" || Object.isFrozen(value) || ArrayBuffer.isView(value)) return;
+  const proto = Object.getPrototypeOf(value);
+  if (proto !== Object.prototype && proto !== Array.prototype && proto !== null) return;
+  for (const child of Object.values(value)) deepFreeze(child);
+  Object.freeze(value);
+}
+
+/**
+ * Arrow-encode a FunctionInfo, once per frozen instance.
+ *
+ * The bytes returned for a frozen item are shared between calls; callers copy
+ * them into a response and never write to them.
+ */
+export function encodeFunctionInfoOnce(info: FunctionInfo): Uint8Array {
+  if (!Object.isFrozen(info)) return encodeFunctionInfo(info);
+  let encoded = ENCODED.get(info);
+  if (encoded === undefined) {
+    encoded = encodeFunctionInfo(info);
+    ENCODED.set(info, encoded);
+  }
+  return encoded;
+}

package/src/catalog/read-only.ts

@@ -18,7 +18,6 @@ import {
   type CopyFromFormatInfo,
   encodeAttachCatalogInfo,
   encodeScanFunctionResult,
-  encodeFunctionInfo,
 } from "./interface.js";
 import { schemaDescriptorPath, type CatalogDescriptor, type SchemaDescriptor, type TableDescriptor, type ViewDescriptor, type MacroDescriptor, type SettingDescriptor, type SecretTypeDescriptor, type ForeignKeyDef, type DefaultValue } from "./descriptors.js";
 import { serializeColumnStatistics } from "../util/statistics.js";
@@ -32,12 +31,24 @@ import { FunctionStability, NullHandling, OrderPreservation, DEFAULT_MAX_WORKERS
 import { Arguments } from "../arguments/arguments.js";
 import { normalizeSchemaPath, schemaPathDisplay, schemaPathsEqual } from "../schema-path.js";
 import { ForeignKeyInfoSchema } from "../generated/vgi-protocol-schemas.js";
+import { encodeFunctionInfoOnce, freezeCatalogItem } from "./item-encoding.js";
 
 export class ReadOnlyCatalogInterface extends CatalogInterface {
   private _descriptor: CatalogDescriptor;
   private _registry: FunctionRegistry;
   private _attachments = new Map<string, AttachOpaqueData>();
   private _version = 1;
+  /**
+   * Function listings by schema and listing type, built on first request. A
+   * listing depends only on the registered functions' static metadata, so it
+   * is built once and every request returns the same frozen FunctionInfo
+   * instances -- which the listing handler then encodes once each (see
+   * item-encoding.ts). The descriptor is treated as immutable after the first
+   * listing, as the rest of this class already does.
+   */
+  private _functionListings = new Map<SchemaDescriptor, Map<FunctionListingKind, readonly FunctionInfo[]>>();
+  /** One frozen FunctionInfo per (declaring schema, function), shared by listings and `global_functions`. */
+  private _functionInfos = new Map<SchemaDescriptor, Map<VgiFunction, FunctionInfo>>();
 
   constructor(descriptor: CatalogDescriptor, registry: FunctionRegistry) {
     super();
@@ -169,7 +180,7 @@ export class ReadOnlyCatalogInterface extends CatalogInterface {
       // Functions the client should ALSO publish into its global (system.main)
       // namespace, under `global_function_prefix` + "_" + name. Empty unless
       // the descriptor opts in via `globalFunctions`.
-      global_functions: this._globalFunctionInfos().map(encodeFunctionInfo),
+      global_functions: this._globalFunctionInfos().map(encodeFunctionInfoOnce),
       global_function_prefix: this._descriptor.globalFunctionPrefix ?? "",
       resolved_data_version: null,
       resolved_implementation_version: null,
@@ -412,18 +423,53 @@ export class ReadOnlyCatalogInterface extends CatalogInterface {
     const schema = this._descriptor.schemas.find((s) => schemaPathsEqual(schemaDescriptorPath(s), path));
     if (!schema || !schema.functions) return [];
 
-    return schema.functions
-      .filter((f) => {
-        // Filter by SchemaObjectType (lowercase on the wire: scalar_function,
-        // table_function, aggregate_function). Unknown filter values pass
-        // everything through. Defensive String() coerces null/undefined.
-        const t = String(type ?? "").toUpperCase();
-        if (t === "SCALAR_FUNCTION") return f.kind === "scalar";
-        if (t === "TABLE_FUNCTION") return f.kind === "table" || f.kind === "table_in_out" || (f.kind as string) === "table_buffering";
-        if (t === "AGGREGATE_FUNCTION") return (f.kind as string) === "aggregate";
-        return true;
-      })
-      .map((f) => this._functionToInfo(f, path));
+    // Filter by SchemaObjectType (lowercase on the wire: scalar_function,
+    // table_function, aggregate_function). Unknown filter values pass
+    // everything through. Defensive String() coerces null/undefined.
+    const kind = functionListingKind(type);
+    const canonicalPath = schemaDescriptorPath(schema);
+    if (!sameSpelling(path, canonicalPath)) {
+      // Schema names match case-insensitively, and an item's schema_path keeps
+      // the caller's spelling, so a differently-spelled request is built fresh,
+      // exactly as before, rather than growing the cache per spelling.
+      return schema.functions.filter((f) => inListing(f, kind)).map((f) => this._functionToInfo(f, path));
+    }
+    let listings = this._functionListings.get(schema);
+    if (!listings) {
+      listings = new Map();
+      this._functionListings.set(schema, listings);
+    }
+    let listing = listings.get(kind);
+    if (!listing) {
+      listing = Object.freeze(
+        schema.functions.filter((f) => inListing(f, kind)).map((f) => this._sharedFunctionInfo(schema, f)),
+      );
+      listings.set(kind, listing);
+    }
+    // A fresh array of the shared (frozen) items: a caller may reorder or
+    // extend the array it gets without touching the cache.
+    return listing.slice();
+  }
+
+  /**
+   * The frozen FunctionInfo for `f` as declared in `schema`, built on first use.
+   * Shared between requests, so it must not (and, frozen, cannot) be mutated.
+   */
+  private _sharedFunctionInfo(schema: SchemaDescriptor, f: VgiFunction): FunctionInfo {
+    let infos = this._functionInfos.get(schema);
+    if (!infos) {
+      infos = new Map();
+      this._functionInfos.set(schema, infos);
+    }
+    let info = infos.get(f);
+    if (!info) {
+      // Cloned before freezing: the built item shares arrays with the function's
+      // own metadata (categories, monotonicity, ...), which must stay the
+      // author's to do with as they like.
+      info = freezeCatalogItem(structuredClone(this._functionToInfo(f, schemaDescriptorPath(schema))));
+      infos.set(f, info);
+    }
+    return info;
   }
 
   /**
@@ -602,22 +648,22 @@ export class ReadOnlyCatalogInterface extends CatalogInterface {
     const globals = this._descriptor.globalFunctions ?? [];
     if (globals.length === 0) return [];
 
-    const schemaOf = new Map<VgiFunction, string[]>();
+    const schemaOf = new Map<VgiFunction, SchemaDescriptor>();
     for (const sch of this._descriptor.schemas) {
       for (const fn of sch.functions ?? []) {
-        if (!schemaOf.has(fn)) schemaOf.set(fn, schemaDescriptorPath(sch));
+        if (!schemaOf.has(fn)) schemaOf.set(fn, sch);
       }
     }
 
     return globals.map((fn) => {
-      const schemaPath = schemaOf.get(fn);
-      if (schemaPath === undefined) {
+      const schema = schemaOf.get(fn);
+      if (schema === undefined) {
         throw new Error(
           `Catalog '${this._descriptor.name}': '${fn.meta.name}' is listed in ` +
             `globalFunctions but is not declared in any schema.`,
         );
       }
-      return this._functionToInfo(fn, schemaPath);
+      return this._sharedFunctionInfo(schema, fn);
     });
   }
 
@@ -1066,3 +1112,25 @@ function inlineScanFunction(
   const argBytes = serializeArgsBatch(args ?? new Arguments(), argSchema);
   return encodeScanFunctionResult(func.meta.name, argBytes, [], schemaPath);
 }
+
+/** Which functions a catalog_schema_contents_functions listing keeps. */
+type FunctionListingKind = "SCALAR_FUNCTION" | "TABLE_FUNCTION" | "AGGREGATE_FUNCTION" | "ALL";
+
+function functionListingKind(type: unknown): FunctionListingKind {
+  const t = String(type ?? "").toUpperCase();
+  return t === "SCALAR_FUNCTION" || t === "TABLE_FUNCTION" || t === "AGGREGATE_FUNCTION" ? t : "ALL";
+}
+
+function inListing(f: VgiFunction, kind: FunctionListingKind): boolean {
+  if (kind === "SCALAR_FUNCTION") return f.kind === "scalar";
+  if (kind === "TABLE_FUNCTION") return f.kind === "table" || f.kind === "table_in_out" || (f.kind as string) === "table_buffering";
+  if (kind === "AGGREGATE_FUNCTION") return (f.kind as string) === "aggregate";
+  return true;
+}
+
+/** Whether two schema paths are spelled identically (not just equal case-insensitively). */
+function sameSpelling(left: Iterable<unknown> | string, right: Iterable<unknown> | string): boolean {
+  const a = normalizeSchemaPath(left);
+  const b = normalizeSchemaPath(right);
+  return a.length === b.length && a.every((part, i) => part === b[i]);
+}

package/src/client/errors.ts

@@ -63,6 +63,25 @@ export function wrapRpcWithErrorEnrichment(rpc: RpcClient): RpcClient {
         throw e;
       }
     },
+    // The batch-level twins (vgi-rpc 0.24.0) get the same treatment: a caller
+    // holding encoded Arrow should still see the worker's traceback rather
+    // than a bare RpcError.
+    async callRaw(method, input) {
+      try {
+        return await rpc.callRaw(method, input);
+      } catch (e) {
+        if (e instanceof RpcError) throw VgiClientError.fromRpcError(e);
+        throw e;
+      }
+    },
+    async streamRaw(method, input, options) {
+      try {
+        return await rpc.streamRaw(method, input, options);
+      } catch (e) {
+        if (e instanceof RpcError) throw VgiClientError.fromRpcError(e);
+        throw e;
+      }
+    },
     describe: () => rpc.describe(),
     close: () => rpc.close(),
   };

package/src/filter-pushdown/evaluate.ts

@@ -22,9 +22,33 @@ import {
 
 type SqlBoolean = boolean | null;
 
-function readCell(batch: VgiBatch, index: number, row: number): unknown {
-  const column = batch.getChildAt(index);
-  return column ? readCanonicalValue(batch.schema.fields[index].type, column, row) : null;
+/**
+ * A batch's columns by name, resolved once per batch.
+ *
+ * Filters are evaluated row by row, and resolving a `column_ref` per row --
+ * a field search plus `getChildAt`, which builds a fresh Vector on arrow-js --
+ * cost more than the comparison itself: ~0.3 ms per 1000-row batch per
+ * referenced column. A batch is immutable, so what it resolves to never changes.
+ */
+const RESOLVED_COLUMNS = new WeakMap<object, Map<string, { type: VgiDataType; column: unknown } | null>>();
+
+function resolveColumn(batch: VgiBatch, columnName: string): { type: VgiDataType; column: unknown } | null {
+  let byName = RESOLVED_COLUMNS.get(batch);
+  if (!byName) {
+    byName = new Map();
+    RESOLVED_COLUMNS.set(batch, byName);
+  }
+  let resolved = byName.get(columnName);
+  if (resolved === undefined) {
+    const index = batch.schema.fields.findIndex((field) => field.name === columnName);
+    if (index < 0) {
+      throw new FilterV2Error(`filter column ${columnName} is unavailable in emitted batch`);
+    }
+    const column = batch.getChildAt(index);
+    resolved = column ? { type: batch.schema.fields[index].type, column } : null;
+    byName.set(columnName, resolved);
+  }
+  return resolved;
 }
 
 function deepEqual(left: unknown, right: unknown): boolean {
@@ -307,11 +331,8 @@ function evaluateCall(expression: Extract<FilterExpression, { node: "call" }>, a
 function evaluateExpression(expression: FilterExpression, batch: VgiBatch, row: number, context: EvaluationContext): unknown {
   switch (expression.node) {
     case "column_ref": {
-      const projectedIndex = batch.schema.fields.findIndex((field) => field.name === expression.columnName);
-      if (projectedIndex < 0) {
-        throw new FilterV2Error(`filter column ${expression.columnName} is unavailable in emitted batch`);
-      }
-      return readCell(batch, projectedIndex, row);
+      const resolved = resolveColumn(batch, expression.columnName);
+      return resolved ? readCanonicalValue(resolved.type, resolved.column as any, row) : null;
     }
     case "field_ref": {
       const parent = evaluateExpression(expression.expression, batch, row, context);
@@ -411,6 +432,32 @@ export class PushdownFilters {
     return applyFilterDelta(this, batch);
   }
 
+  /**
+   * These filters with their predicates arranged in `order`.
+   *
+   * Only the order changes, and `order` must name exactly the live predicate
+   * ids. Restores the order a stream had before its delta history was
+   * compacted: an id removed and later re-added moves to the end, which a
+   * shorter replay does not reproduce by itself.
+   */
+  withPredicateOrder(order: readonly string[]): PushdownFilters {
+    const predicates = this.predicates;
+    if (predicates.length === order.length && predicates.every((predicate, i) => predicate.id === order[i])) {
+      return this;
+    }
+    const byId = new Map(predicates.map((predicate) => [predicate.id, predicate]));
+    if (order.length !== byId.size || new Set(order).size !== order.length || !order.every((id) => byId.has(id))) {
+      throw new FilterV2Error("recorded predicate order does not match the replayed filter state");
+    }
+    return new PushdownFilters(
+      order.map((id) => byId.get(id)!),
+      this.evaluationContext,
+      this.options,
+      new Map(this.revisions),
+      new Set(this.requiredIds),
+    );
+  }
+
   evaluate(batch: VgiBatch): Uint8Array {
     const mask = new Uint8Array(batch.numRows);
     mask.fill(1);

package/src/filter-pushdown/history.ts

@@ -0,0 +1,239 @@
+// Copyright 2025, 2026 Query Farm LLC - https://query.farm
+// Dynamic-filter state across HTTP turns: a compacted delta history.
+//
+// A table stream's filters can change mid-scan. DuckDB's Top-N operator (and
+// friends) tightens a bound as rows arrive and ships the change as a v2 *delta*
+// on the next tick's `vgi_pushdown_filters` metadata -- and only then: the
+// extension sends each delta ONCE per stream, when a filter's value changes, not
+// on every tick. A byte-stream transport keeps the parsed filters in memory
+// between ticks. An HTTP stream cannot: every turn rebuilds its handlers from the
+// state token, so whatever a delta changed has to ride that token or it is lost
+// on the next turn, and the worker quietly falls back to the init snapshot.
+//
+// Carrying every delta ever received would be the opposite bug: the token grows
+// by a delta per tick and each turn replays all of them, quadratic in the tick
+// count (vgi-python 0283898). So the token carries a COMPACTED history -- for
+// each (id, revision) of the live state, tombstones included, the first delta
+// that carried it. Replaying just those reproduces the same predicates, values
+// and revisions: an earlier update to an id is overwritten by its current
+// revision, and a later one was stale when it arrived and is stale again on
+// replay. The history is therefore bounded by the number of predicate ids, not
+// the number of ticks. Replay cannot always reproduce predicate ORDER (an id
+// removed and later re-added moves to the end), so the live order is recorded
+// alongside and restored after replay.
+
+import { type VgiBatch, deserializeBatch, readCanonicalValue } from "../arrow/index.js";
+import { FilterV2Error } from "./deserialize.js";
+import type { PushdownFilters } from "./evaluate.js";
+
+/** Encoded-size limit on the base64 `vgi_pushdown_filters` value. */
+const MAX_ENCODED_DELTA_BYTES = 24 << 20;
+/** Decoded-size limit on one delta's Arrow IPC payload. */
+const MAX_DELTA_IPC_BYTES = 17 << 20;
+const BASE64 = /^(?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?$/;
+
+/**
+ * What an HTTP stream carries between turns to rebuild its dynamic filters.
+ *
+ * Both halves are plain data so the state serializer can put them in the
+ * cursor, and both are bounded by the number of predicate ids the scan has
+ * seen (see {@link recordFilterDelta}).
+ */
+export interface FilterDeltaHistory {
+  /** Arrow IPC bytes of each delta still needed, in arrival order. */
+  readonly deltas: readonly Uint8Array[];
+  /** The live predicate ids, in the order the stream had them. */
+  readonly order: readonly string[];
+}
+
+/** A stream that has applied no delta. */
+export const EMPTY_FILTER_HISTORY: FilterDeltaHistory = Object.freeze({
+  deltas: Object.freeze([]) as readonly Uint8Array[],
+  order: Object.freeze([]) as readonly string[],
+});
+
+/**
+ * Decode the base64 `vgi_pushdown_filters` metadata value into delta IPC bytes.
+ *
+ * Validates the encoding strictly and enforces the encoded and decoded size
+ * limits before anything is parsed.
+ */
+export function decodeDynamicFilterMetadata(encoded: string): Uint8Array {
+  if (new TextEncoder().encode(encoded).byteLength > MAX_ENCODED_DELTA_BYTES) {
+    throw new Error("base64 dynamic filter metadata exceeds the encoded-size limit");
+  }
+  if (encoded.length % 4 !== 0 || !BASE64.test(encoded)) {
+    throw new Error("dynamic filter metadata is not valid base64");
+  }
+  const binary = (globalThis as any).atob
+    ? (globalThis as any).atob(encoded)
+    : Buffer.from(encoded, "base64").toString("binary");
+  const bytes = new Uint8Array(binary.length);
+  for (let i = 0; i < binary.length; i++) bytes[i] = binary.charCodeAt(i);
+  if (bytes.byteLength > MAX_DELTA_IPC_BYTES) {
+    throw new Error("dynamic filter IPC payload exceeds the encoded-size limit");
+  }
+  return bytes;
+}
+
+/** Read one delta's IPC bytes into its single document batch. */
+export function decodeFilterDelta(bytes: Uint8Array): VgiBatch {
+  if (bytes.byteLength > MAX_DELTA_IPC_BYTES) {
+    throw new Error("dynamic filter IPC payload exceeds the encoded-size limit");
+  }
+  const batch = bytes.byteLength > 0 ? deserializeBatch(bytes) : null;
+  if (!batch || (batch.numRows === 0 && batch.schema.fields.length === 0)) {
+    throw new Error("dynamic filter metadata must contain one RecordBatch");
+  }
+  return batch as VgiBatch;
+}
+
+/**
+ * The `(id, revision)` of every update a delta document carries, as
+ * `id\0revision` keys.
+ *
+ * A structural read only, for bookkeeping over deltas that have already been
+ * applied -- and therefore validated -- by {@link PushdownFilters.applyDelta}.
+ * It neither applies nor validates anything.
+ */
+function deltaRevisionKeys(batch: VgiBatch): string[] {
+  const column = batch.getChildAt(0);
+  if (!column) throw new FilterV2Error("delta document has no filter_spec column");
+  const raw = readCanonicalValue(batch.schema.fields[0].type, column, 0);
+  const document = JSON.parse(String(raw)) as { updates?: Array<{ id: string; revision: number }> };
+  return (document.updates ?? []).map((update) => revisionKey(update.id, update.revision));
+}
+
+function revisionKey(id: string, revision: number): string {
+  return `${id}\u0000${revision}`;
+}
+
+/**
+ * Apply one tick's delta and return the new filters plus the history a cursor
+ * must carry to rebuild them.
+ *
+ * The history is compacted to the deltas that installed some predicate's
+ * CURRENT revision, tombstones included: for each `(id, revision)` of the live
+ * state, the first delta (in arrival order) carrying it. That delta is the one
+ * that was applied -- any later delta carrying the same revision was stale on
+ * arrival -- and a delta none of whose updates is current can never matter
+ * again, because revisions only move forward.
+ *
+ * @param current - The stream's filters before this delta.
+ * @param history - The compacted history those filters were rebuilt from.
+ * @param deltaBytes - The IPC bytes of the delta this tick carried.
+ */
+export function recordFilterDelta(
+  current: PushdownFilters,
+  history: FilterDeltaHistory,
+  deltaBytes: Uint8Array,
+): { filters: PushdownFilters; history: FilterDeltaHistory } {
+  const incoming = decodeFilterDelta(deltaBytes);
+  // Assignment happens only after complete validation, so a rejected delta
+  // leaves the caller's prior state (and history) intact.
+  const filters = current.applyDelta(incoming);
+  const wanted = new Set<string>();
+  for (const [id, revision] of filters.revisions) wanted.add(revisionKey(id, revision));
+  const deltas: Uint8Array[] = [];
+  const consider = (bytes: Uint8Array, batch: VgiBatch): void => {
+    let carries = false;
+    for (const key of deltaRevisionKeys(batch)) {
+      if (wanted.delete(key)) carries = true;
+    }
+    if (carries) deltas.push(bytes);
+  };
+  for (const bytes of history.deltas) consider(bytes, decodeFilterDelta(bytes));
+  consider(deltaBytes, incoming);
+  return { filters, history: { deltas, order: filters.predicates.map((predicate) => predicate.id) } };
+}
+
+const HISTORY_FORMAT_VERSION = 1;
+const UTF8_ENCODER = new TextEncoder();
+const UTF8_DECODER = new TextDecoder("utf-8", { fatal: true });
+
+/**
+ * Encode a history for the state token, or `null` when there is nothing to
+ * carry (no delta applied, which is every stream without dynamic filters).
+ *
+ * Layout (little-endian): `u8 version | u32 n | n x (u32 len, delta IPC) |
+ * u32 m | m x (u32 len, utf8 id)`.
+ */
+export function encodeFilterHistory(history: FilterDeltaHistory | null | undefined): Uint8Array | null {
+  if (!history || history.deltas.length === 0) return null;
+  const ids = history.order.map((id) => UTF8_ENCODER.encode(id));
+  let size = 1 + 4 + 4;
+  for (const delta of history.deltas) size += 4 + delta.byteLength;
+  for (const id of ids) size += 4 + id.byteLength;
+  const out = new Uint8Array(size);
+  const view = new DataView(out.buffer);
+  let offset = 0;
+  out[offset++] = HISTORY_FORMAT_VERSION;
+  const put = (chunks: readonly Uint8Array[]): void => {
+    view.setUint32(offset, chunks.length, true);
+    offset += 4;
+    for (const chunk of chunks) {
+      view.setUint32(offset, chunk.byteLength, true);
+      offset += 4;
+      out.set(chunk, offset);
+      offset += chunk.byteLength;
+    }
+  };
+  put(history.deltas);
+  put(ids);
+  return out;
+}
+
+/** Decode {@link encodeFilterHistory}'s bytes; `null`/empty is the empty history. */
+export function decodeFilterHistory(bytes: Uint8Array | null | undefined): FilterDeltaHistory {
+  if (!bytes || bytes.byteLength === 0) return EMPTY_FILTER_HISTORY;
+  const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
+  let offset = 0;
+  const need = (n: number): void => {
+    if (offset + n > bytes.byteLength) throw new Error("dynamic filter history in the state token is truncated");
+  };
+  need(1);
+  if (bytes[offset++] !== HISTORY_FORMAT_VERSION) {
+    throw new Error("dynamic filter history in the state token has an unknown format");
+  }
+  const take = (): Uint8Array[] => {
+    need(4);
+    const count = view.getUint32(offset, true);
+    offset += 4;
+    const chunks: Uint8Array[] = [];
+    for (let i = 0; i < count; i++) {
+      need(4);
+      const length = view.getUint32(offset, true);
+      offset += 4;
+      need(length);
+      // A copy, not a view: Arrow readers want their own aligned buffer.
+      chunks.push(bytes.slice(offset, offset + length));
+      offset += length;
+    }
+    return chunks;
+  };
+  const deltas = take();
+  const order = take().map((id) => UTF8_DECODER.decode(id));
+  if (offset !== bytes.byteLength) throw new Error("dynamic filter history in the state token has trailing bytes");
+  return { deltas, order };
+}
+
+/**
+ * Rebuild a turn's filters from the init snapshot plus a carried history.
+ *
+ * Every delta here was validated when it first arrived and has ridden a
+ * worker-sealed token since, but it is re-applied through the same strict
+ * parser -- a TS delta parse is structural, with no bind to skip.
+ *
+ * @param filters - The filters parsed from the init request's snapshot.
+ * @param history - The history {@link recordFilterDelta} produced.
+ */
+export function replayFilterHistory(
+  filters: PushdownFilters | undefined,
+  history: FilterDeltaHistory | null | undefined,
+): PushdownFilters | undefined {
+  if (!history || history.deltas.length === 0) return filters;
+  if (!filters) throw new Error("dynamic filter history carried without an initial snapshot");
+  let replayed = filters;
+  for (const bytes of history.deltas) replayed = replayed.applyDelta(decodeFilterDelta(bytes));
+  return replayed.withPredicateOrder(history.order);
+}

package/src/filter-pushdown/index.ts

@@ -35,6 +35,14 @@ export { PushdownFilters } from "./evaluate.js";
 
 export { FilterV2Error, applyFilterDelta, buildJoinKeysLookup, deserializeFilters } from "./deserialize.js";
 
+export {
+  EMPTY_FILTER_HISTORY,
+  type FilterDeltaHistory,
+  decodeDynamicFilterMetadata,
+  recordFilterDelta,
+  replayFilterHistory,
+} from "./history.js";
+
 export {
   FilteringOutputCollector,
   formatPushedFilters,

package/src/functions/storage-cf-do.ts

@@ -11,8 +11,8 @@
 //   Set `VGI_WORKER_SHARED_STORAGE=cloudflare-do` plus `VGI_CF_DO_URL`.
 //   Optionally set `VGI_CF_DO_TOKEN` for bearer auth.
 
-import type { FunctionStorage } from "./storage.js";
-import { UnknownInvocationError } from "./storage.js";
+import type { FunctionStorage, WorkerStateKey } from "./storage.js";
+import { UnknownInvocationError, workerStateKey } from "./storage.js";
 
 /** Minimal fetch signature satisfied by both the global `fetch` and a
  *  Cloudflare service-binding Fetcher (`env.<BINDING>`). */
@@ -77,10 +77,10 @@ export class FunctionStorageCfDo implements FunctionStorage {
     });
   }
 
-  // --- Worker state → ns=worker, key = int64(worker_id) ---
+  // --- Worker state → ns=worker, key = substream_id bytes, or int64(worker_id) ---
 
-  async workerPut(executionId: Uint8Array, workerId: number, state: Uint8Array): Promise<void> {
-    await this._statePutMany(executionId, NS_WORKER, [[int64Key(workerId), state]]);
+  async workerPut(executionId: Uint8Array, worker: WorkerStateKey, state: Uint8Array): Promise<void> {
+    await this._statePutMany(executionId, NS_WORKER, [[workerStateKey(worker), state]]);
   }
 
   async workerCollect(executionId: Uint8Array): Promise<Uint8Array[]> {
@@ -277,13 +277,8 @@ function newAttemptId(): string {
   return s;
 }
 
-/** Encode an int64 worker/group id as an 8-byte big-endian state key. */
-function int64Key(v: number): Uint8Array {
-  const b = new Uint8Array(8);
-  new DataView(b.buffer).setBigInt64(0, BigInt(v), false);
-  return b;
-}
-
+/** Decode an 8-byte big-endian state key to its int64 worker id; any other
+ *  key (a substream id) has none, and decodes to 0. */
 function int64FromKey(b: Uint8Array): number {
   if (b.length !== 8) return 0;
   return Number(new DataView(b.buffer, b.byteOffset, 8).getBigInt64(0, false));