@query-farm/vgi 0.31.0 → 0.33.0

package/README.md

@@ -41,7 +41,7 @@ npm install @query-farm/vgi
 Requires **Node.js ≥ 22.15** or **Bun**.
 
 > **Peer dependencies (since 0.3.0).** `@query-farm/apache-arrow` (`^21.1.1`) and
-> `@query-farm/vgi-rpc` (`^0.24.0`) are **peerDependencies** — install them directly
+> `@query-farm/vgi-rpc` (`^0.25.2`) are **peerDependencies** — install them directly
 > alongside `@query-farm/vgi`:
 >
 > ```bash

package/dist/catalog/item-encoding.d.ts

@@ -0,0 +1,17 @@
+import { type FunctionInfo } from "../generated/vgi-client.js";
+/**
+ * Deep-freeze a catalog item (plain objects and arrays, recursively) so it can
+ * be shared between requests and its encoding memoized.
+ *
+ * Binary fields (`Uint8Array` and other ArrayBuffer views) are left as they are
+ * -- a typed array with elements cannot be frozen -- and must be treated as
+ * read-only by convention, like the rest of a shared item.
+ */
+export declare function freezeCatalogItem<T extends object>(item: T): Readonly<T>;
+/**
+ * Arrow-encode a FunctionInfo, once per frozen instance.
+ *
+ * The bytes returned for a frozen item are shared between calls; callers copy
+ * them into a response and never write to them.
+ */
+export declare function encodeFunctionInfoOnce(info: FunctionInfo): Uint8Array;

package/dist/catalog/read-only.d.ts

@@ -7,6 +7,17 @@ export declare class ReadOnlyCatalogInterface extends CatalogInterface {
     private _registry;
     private _attachments;
     private _version;
+    /**
+     * Function listings by schema and listing type, built on first request. A
+     * listing depends only on the registered functions' static metadata, so it
+     * is built once and every request returns the same frozen FunctionInfo
+     * instances -- which the listing handler then encodes once each (see
+     * item-encoding.ts). The descriptor is treated as immutable after the first
+     * listing, as the rest of this class already does.
+     */
+    private _functionListings;
+    /** One frozen FunctionInfo per (declaring schema, function), shared by listings and `global_functions`. */
+    private _functionInfos;
     constructor(descriptor: CatalogDescriptor, registry: FunctionRegistry);
     /**
      * Which schema declares `fn`, by object identity, or null if no schema does.
@@ -50,6 +61,11 @@ export declare class ReadOnlyCatalogInterface extends CatalogInterface {
     } | null;
     schemaContentsViews(attachOpaqueData: AttachOpaqueData, path: string[], transactionOpaqueData?: TransactionOpaqueData): ViewInfo[];
     schemaContentsFunctions(attachOpaqueData: AttachOpaqueData, path: string[], type: string, transactionOpaqueData?: TransactionOpaqueData): FunctionInfo[];
+    /**
+     * The frozen FunctionInfo for `f` as declared in `schema`, built on first use.
+     * Shared between requests, so it must not (and, frozen, cannot) be mutated.
+     */
+    private _sharedFunctionInfo;
     /**
      * Build the wire `FunctionInfo` for one declared function.
      *

package/dist/filter-pushdown/evaluate.d.ts

@@ -11,6 +11,15 @@ export declare class PushdownFilters {
     constructor(predicates: FilterPredicate[], evaluationContext: EvaluationContext, options: DeserializeFilterOptions, revisions?: Map<string, number>, requiredIds?: Set<string>);
     get filters(): FilterExpression[];
     applyDelta(batch: VgiBatch): PushdownFilters;
+    /**
+     * These filters with their predicates arranged in `order`.
+     *
+     * Only the order changes, and `order` must name exactly the live predicate
+     * ids. Restores the order a stream had before its delta history was
+     * compacted: an id removed and later re-added moves to the end, which a
+     * shorter replay does not reproduce by itself.
+     */
+    withPredicateOrder(order: readonly string[]): PushdownFilters;
     evaluate(batch: VgiBatch): Uint8Array;
     apply(batch: VgiBatch): VgiBatch;
     filteredColumns(): string[];

package/dist/filter-pushdown/history.d.ts

@@ -0,0 +1,66 @@
+import { type VgiBatch } from "../arrow/index.js";
+import type { PushdownFilters } from "./evaluate.js";
+/**
+ * What an HTTP stream carries between turns to rebuild its dynamic filters.
+ *
+ * Both halves are plain data so the state serializer can put them in the
+ * cursor, and both are bounded by the number of predicate ids the scan has
+ * seen (see {@link recordFilterDelta}).
+ */
+export interface FilterDeltaHistory {
+    /** Arrow IPC bytes of each delta still needed, in arrival order. */
+    readonly deltas: readonly Uint8Array[];
+    /** The live predicate ids, in the order the stream had them. */
+    readonly order: readonly string[];
+}
+/** A stream that has applied no delta. */
+export declare const EMPTY_FILTER_HISTORY: FilterDeltaHistory;
+/**
+ * Decode the base64 `vgi_pushdown_filters` metadata value into delta IPC bytes.
+ *
+ * Validates the encoding strictly and enforces the encoded and decoded size
+ * limits before anything is parsed.
+ */
+export declare function decodeDynamicFilterMetadata(encoded: string): Uint8Array;
+/** Read one delta's IPC bytes into its single document batch. */
+export declare function decodeFilterDelta(bytes: Uint8Array): VgiBatch;
+/**
+ * Apply one tick's delta and return the new filters plus the history a cursor
+ * must carry to rebuild them.
+ *
+ * The history is compacted to the deltas that installed some predicate's
+ * CURRENT revision, tombstones included: for each `(id, revision)` of the live
+ * state, the first delta (in arrival order) carrying it. That delta is the one
+ * that was applied -- any later delta carrying the same revision was stale on
+ * arrival -- and a delta none of whose updates is current can never matter
+ * again, because revisions only move forward.
+ *
+ * @param current - The stream's filters before this delta.
+ * @param history - The compacted history those filters were rebuilt from.
+ * @param deltaBytes - The IPC bytes of the delta this tick carried.
+ */
+export declare function recordFilterDelta(current: PushdownFilters, history: FilterDeltaHistory, deltaBytes: Uint8Array): {
+    filters: PushdownFilters;
+    history: FilterDeltaHistory;
+};
+/**
+ * Encode a history for the state token, or `null` when there is nothing to
+ * carry (no delta applied, which is every stream without dynamic filters).
+ *
+ * Layout (little-endian): `u8 version | u32 n | n x (u32 len, delta IPC) |
+ * u32 m | m x (u32 len, utf8 id)`.
+ */
+export declare function encodeFilterHistory(history: FilterDeltaHistory | null | undefined): Uint8Array | null;
+/** Decode {@link encodeFilterHistory}'s bytes; `null`/empty is the empty history. */
+export declare function decodeFilterHistory(bytes: Uint8Array | null | undefined): FilterDeltaHistory;
+/**
+ * Rebuild a turn's filters from the init snapshot plus a carried history.
+ *
+ * Every delta here was validated when it first arrived and has ridden a
+ * worker-sealed token since, but it is re-applied through the same strict
+ * parser -- a TS delta parse is structural, with no bind to skip.
+ *
+ * @param filters - The filters parsed from the init request's snapshot.
+ * @param history - The history {@link recordFilterDelta} produced.
+ */
+export declare function replayFilterHistory(filters: PushdownFilters | undefined, history: FilterDeltaHistory | null | undefined): PushdownFilters | undefined;

package/dist/filter-pushdown/index.d.ts

@@ -1,4 +1,5 @@
 export { FILTER_ENCODING, FILTER_VERSION, DUCKDB_STANDARD_V1, NO_EVALUATION_CONTEXT, DUCKDB_SESSION_CONTEXT, ComparisonOp, type ArithmeticOp, type EvaluationContext, type Filter, type AndFilter, type ConstantFilter, type ExpressionFilter, type ExprNode, type InFilter, type IsNotNullFilter, type IsNullFilter, type OrFilter, type StructFilter, type FilterExpression, type FilterPredicate, type FilterSet, type FunctionIdentity, type PredicateMode, type PredicateSource, type StandardFilterFunction, } from "./types.js";
 export { PushdownFilters } from "./evaluate.js";
 export { FilterV2Error, applyFilterDelta, buildJoinKeysLookup, deserializeFilters } from "./deserialize.js";
+export { EMPTY_FILTER_HISTORY, type FilterDeltaHistory, decodeDynamicFilterMetadata, recordFilterDelta, replayFilterHistory, } from "./history.js";
 export { FilteringOutputCollector, formatPushedFilters, reprPushedFilters, } from "./collector.js";

package/dist/functions/storage-cf-do.d.ts

@@ -1,4 +1,4 @@
-import type { FunctionStorage } from "./storage.js";
+import type { FunctionStorage, WorkerStateKey } from "./storage.js";
 /** Minimal fetch signature satisfied by both the global `fetch` and a
  *  Cloudflare service-binding Fetcher (`env.<BINDING>`). */
 export type FetchLike = (input: string, init?: {
@@ -29,7 +29,7 @@ export declare class FunctionStorageCfDo implements FunctionStorage {
      * per logical ATTACH. Shares the URL/token; only the shard key differs.
      */
     forShard(shardKey: string): FunctionStorageCfDo;
-    workerPut(executionId: Uint8Array, workerId: number, state: Uint8Array): Promise<void>;
+    workerPut(executionId: Uint8Array, worker: WorkerStateKey, state: Uint8Array): Promise<void>;
     workerCollect(executionId: Uint8Array): Promise<Uint8Array[]>;
     workerScan(executionId: Uint8Array): Promise<Array<[number, Uint8Array]>>;
     queuePush(executionId: Uint8Array, items: Uint8Array[]): Promise<number>;

package/dist/functions/storage.d.ts

@@ -1,9 +1,21 @@
 export declare class UnknownInvocationError extends Error {
     constructor(message?: string | Uint8Array);
 }
+/**
+ * Which slot of an execution's worker state a row occupies: a table-in-out
+ * substream's client-minted `substream_id` (raw bytes), or -- only for a client
+ * that sent none -- a numeric worker id, the process id (an int64 key). See
+ * {@link BoundStorage.put}.
+ */
+export type WorkerStateKey = number | Uint8Array;
 export interface FunctionStorage {
-    workerPut(executionId: Uint8Array, workerId: number, state: Uint8Array): Promise<void>;
+    /** Upsert `state` into the execution's worker-state slot `worker`. */
+    workerPut(executionId: Uint8Array, worker: WorkerStateKey, state: Uint8Array): Promise<void>;
+    /** Drain every worker-state slot of the execution, whatever its key. */
     workerCollect(executionId: Uint8Array): Promise<Uint8Array[]>;
+    /** Read every worker-state slot without draining. A slot keyed by a
+     *  numeric worker id reports that id; one keyed by bytes (a substream id)
+     *  has none and reports 0. */
     workerScan(executionId: Uint8Array): Promise<Array<[number, Uint8Array]>>;
     queuePush(executionId: Uint8Array, items: Uint8Array[]): Promise<number>;
     queuePop(executionId: Uint8Array): Promise<Uint8Array | null>;
@@ -23,6 +35,9 @@ export declare const FrameworkNS: {
 };
 /** Encode an int as 8-byte little-endian, matching Python's pack_int_key. */
 export declare function packIntKey(i: number | bigint): Uint8Array;
+/** The state key for a worker-state slot: a substream id as-is, a numeric
+ *  worker id as an int64. */
+export declare function workerStateKey(worker: WorkerStateKey): Uint8Array;
 export declare class FunctionStorageSqlite implements FunctionStorage {
     readonly dbPath: string;
     private _db;
@@ -34,7 +49,7 @@ export declare class FunctionStorageSqlite implements FunctionStorage {
     stateAppend(scopeId: Uint8Array, ns: Uint8Array, key: Uint8Array, item: Uint8Array): Promise<number>;
     stateLogScan(scopeId: Uint8Array, ns: Uint8Array, key: Uint8Array, afterId?: number, limit?: number | null): Promise<Array<[number, Uint8Array]>>;
     executionClear(scopeId: Uint8Array): Promise<number>;
-    workerPut(executionId: Uint8Array, workerId: number, state: Uint8Array): Promise<void>;
+    workerPut(executionId: Uint8Array, worker: WorkerStateKey, state: Uint8Array): Promise<void>;
     workerCollect(executionId: Uint8Array): Promise<Uint8Array[]>;
     workerScan(executionId: Uint8Array): Promise<Array<[number, Uint8Array]>>;
     queuePush(executionId: Uint8Array, items: Uint8Array[]): Promise<number>;
@@ -44,7 +59,28 @@ export declare class FunctionStorageSqlite implements FunctionStorage {
 export declare class BoundStorage {
     private _base;
     private _executionId;
-    constructor(storage: FunctionStorage, executionId: Uint8Array);
+    private _substreamId;
+    /**
+     * @param substreamId The table-in-out substream this storage serves -- the
+     *   client-minted `InitRequest.substream_id`, identical across the
+     *   substream's init, every tick and its finalize -- or `null` when the
+     *   client sent none. Keys {@link put}.
+     */
+    constructor(storage: FunctionStorage, executionId: Uint8Array, substreamId?: Uint8Array | null);
+    /**
+     * Upsert this substream's state into the execution's worker state, which
+     * {@link collect} drains at finalize.
+     *
+     * One row per **substream**, keyed by its `substream_id`. Storage is
+     * already scoped to the execution, and every connection of a fanned-out
+     * scan shares that execution -- so the key has to tell those connections
+     * apart. The process id only does when each connection is its own process;
+     * under the launcher, TCP or HTTP one process serves many of them, and a
+     * per-process key let them overwrite each other's state so finish()
+     * undercounted. The pid remains only for a client that sends no
+     * `substream_id`, which it keys as before. Mirrors vgi-python's
+     * `TableInOutFunction.process` (072e543).
+     */
     put(state: Uint8Array): Promise<void>;
     collect(): Promise<Uint8Array[]>;
     workerScan(): Promise<Array<[number, Uint8Array]>>;

package/dist/functions/table-in-out.d.ts

@@ -26,12 +26,14 @@ export interface TableInOutProcessParams<TArgs = Record<string, any>> {
     storage: BoundStorage;
     /**
      * Stable client-minted id for this streaming table-in-out substream.
-     * Present (identical across init / every process() / finalize) when the
-     * client fanned this function out across per-substream workers; use it to
-     * key per-substream accumulated state in shared storage so a finalize()
-     * that lands on a different HTTP backend than the process() calls still
-     * finds it. `null`/`undefined` for the serial path or an old client.
-     * Mirrors vgi-python's `ProcessParams.substream_id`.
+     * Present (identical across init / every process() / finalize) whenever the
+     * client sent one -- the DuckDB extension does for every substream. The
+     * framework keys the accumulated state it auto-persists by it
+     * ({@link BoundStorage.put}), so substreams of one execution that one
+     * process serves keep separate states and finalize() receives each; use it
+     * likewise for any per-substream state you keep in shared storage yourself.
+     * `null`/`undefined` for a client that sent none. Mirrors vgi-python's
+     * `ProcessParams.substream_id`.
      */
     substreamId?: Uint8Array | null;
     /**
@@ -132,14 +134,17 @@ export interface RowTransformConfig<TArgs = Record<string, any>> {
     /**
      * Positional args = the per-row INPUT COLUMNS (real typed args on the wire,
      * no synthetic TABLE placeholder). Read from `batch` by declared name in
-     * process(); NOT surfaced on `params.args`.
+     * process(); NOT surfaced on `params.args`. A `Null` type declares the
+     * column ANY: the input column then arrives with whatever type DuckDB
+     * resolved for the call (see `params.bindCall.input_schema` in onBind).
      */
     args?: Record<string, VgiDataType>;
     /**
      * Trailing VARARGS input columns: the per-row input is N columns of the
      * declared type. A varargs blended function has no per-column declared
      * names (the C++ bind names them col0..colN-1), so process() reads the
-     * columns POSITIONALLY off `batch`.
+     * columns POSITIONALLY off `batch`. A `Null` element type declares ANY
+     * varargs: each runtime column resolves its own type.
      */
     varargs?: {
         name: string;

package/dist/functions/types.d.ts

@@ -7,6 +7,12 @@ import type { ArgumentSpec } from "../arguments/argument-spec.js";
 export interface HandlerState<T = any> {
     /** Serializable user state for HTTP exchange round-trips. */
     state: T;
+    /**
+     * The compacted dynamic-filter history, for a stream that applies
+     * `vgi_pushdown_filters` deltas. Carried in the HTTP cursor so the next turn
+     * rebuilds the filters this one left.
+     */
+    filterHistory?: import("../filter-pushdown/history.js").FilterDeltaHistory;
     [key: string]: any;
 }
 export interface FunctionExample {
@@ -196,7 +202,14 @@ export interface VgiFunction {
     defaultOutputSchema?: VgiSchema;
     bind(request: BindRequest): BindResponse | Promise<BindResponse>;
     globalInit(request: InitRequest): GlobalInitResponse | Promise<GlobalInitResponse>;
-    createStreamHandlers(request: InitRequest, response: GlobalInitResponse, accumulatedState?: any): StreamHandlers;
+    /**
+     * Build the stream's handlers. On an HTTP continuation `filterHistory` is
+     * the compacted dynamic-filter history the cursor carried: a function that
+     * applies `vgi_pushdown_filters` deltas replays it onto the init snapshot and
+     * keeps the (updated) history on its handler state as `filterHistory`, which
+     * the dispatcher writes back into the next cursor.
+     */
+    createStreamHandlers(request: InitRequest, response: GlobalInitResponse, filterHistory?: import("../filter-pushdown/history.js").FilterDeltaHistory | null): StreamHandlers;
     cardinality?(request: TableFunctionCardinalityRequest): TableCardinality | Promise<TableCardinality>;
     /**
      * Divide a scan into named, independently redeemable splits. Absent means the

package/dist/http/fetch.d.ts

@@ -3,7 +3,7 @@ import { type ProtocolConfig } from "../protocol/dispatch.js";
 import { type LandingInfo } from "./landing.js";
 export interface VgiFetchOptions {
     /** Wire-protocol config (registry + catalogInterface). */
-    protocol: Omit<ProtocolConfig, "recoverExchangeState">;
+    protocol: ProtocolConfig;
     /** HMAC key for state-token signing. Pass a stable, secret 32-byte key
      *  (e.g. derived from a Wrangler secret). Required because Workers don't
      *  preserve in-memory state across requests/instances. */

package/dist/index.core.d.ts

@@ -36,7 +36,7 @@ export { serializeSchema, deserializeSchema, serializeBatch, deserializeBatch, }
 export { schema, field, nullType, int8, int16, int64, uint8, uint16, uint32, uint64, float16, float64, utf8, largeUtf8, binary, largeBinary, fixedSizeBinary, decimal, decimal128, decimal256, date, dateDay, dateMillisecond, time, timeSecond, timeMillisecond, timeMicrosecond, timeNanosecond, timestamp, duration, interval, timestampSeconds, timestampMillis, timestampMicros, timestampNanos, durationSeconds, durationMillis, durationMicros, durationNanos, list, fixedSizeList, struct, map, dictionary, union, sparseUnion, denseUnion, TimeUnit, DateUnit, IntervalUnit, UnionMode, codecFor, asDate32, asDate64Ms, asTime32S, asTime32Ms, asTime64Us, asTime64Ns, asTimestampSeconds, asTimestampMillis, asTimestampMicros, asTimestampNanos, asDurationSeconds, asDurationMillis, asDurationMicros, asDurationNanos, asUnscaledDecimal, asInt64, asUint64, fromBranded, fromBrandedNumber, fromBrandedBigInt, TypeId, isNull, isInt, isFloat, isBinary, isUtf8, isBool, isDecimal, isDate, isTime, isTimestamp, isInterval, isList, isStruct, isUnion, isFixedSizeBinary, isFixedSizeList, isMap, isDuration, isDictionary, isBatch, typeSignature, readCanonicalValue, columnFromArray, decodeDictValue, backend, adoptArrowJsShape, } from "./arrow/index.js";
 export type { VgiTypeId, VgiDataType, VgiField, VgiSchema, VgiColumn, VgiColumnData, VgiBatch, VgiBackendInfo, TaggedUnion, NullDescriptor, BoolType, IntType, Int8Type, Int16Type, Int32Type, Int64Type, Uint8Type, Uint16Type, Uint32Type, Uint64Type, FloatType, Utf8Type, LargeUtf8Type, BinaryType, LargeBinaryType, FixedSizeBinaryType, DecimalType, Date32Type, Date64Type, Time32Type, Time64Type, TimestampType, DurationType, IntervalType, ListType, FixedSizeListType, StructType, MapType, DictionaryType, UnionType, TypedField, TUnit, RichValue, RawValue, ValueFor, Repr, Codec, Branded, Date32, Date64Ms, Time32S, Time32Ms, Time64Us, Time64Ns, TimestampSeconds, TimestampMillis, TimestampMicros, TimestampNanos, DurationSeconds, DurationMillis, DurationMicros, DurationNanos, UnscaledDecimal, Int64Raw, Uint64Raw, } from "./arrow/index.js";
 export type { ScalarComputeRow, ScalarOutputValue, ScalarComputeResult } from "./functions/scalar.js";
-export { type FunctionStorage, FunctionStorageSqlite, BoundStorage, UnknownInvocationError, resolveStorageFromEnv, setStorage, storage as functionStorage, } from "./functions/storage.js";
+export { type FunctionStorage, type WorkerStateKey, FunctionStorageSqlite, BoundStorage, UnknownInvocationError, resolveStorageFromEnv, setStorage, storage as functionStorage, } from "./functions/storage.js";
 export { FunctionStorageCfDo, type FetchLike } from "./functions/storage-cf-do.js";
 export { VgiClient, VgiClientError } from "./client/client.js";
 export type { VgiClientOptions, TableFunctionOptions, ScalarFunctionOptions, TableInOutFunctionOptions, OnCreateConflict, CatalogFunctionType as ClientCatalogFunctionType, CatalogAttachOptions, AttachOptionValue, } from "./client/types.js";