@query-farm/vgi 0.28.0 → 0.29.0

package/dist/protocol/handlers/catalog/shared.d.ts

@@ -60,11 +60,3 @@ export declare function makeGetCatalog(catalog: CatalogInterface | undefined): G
  */
 export declare function decodeOptionsBatch(bytes: any): Record<string, unknown>;
 export declare const emptyResultSchema: VgiSchema;
-export declare const attachOpaqueDataParam: VgiSchema;
-export declare const attachOpaqueDataTxnParams: VgiSchema;
-export declare const attachOpaqueDataNameTxnParams: VgiSchema;
-export declare const attachOpaqueDataSchemaNameTxnParams: VgiSchema;
-export declare const schemaNameIgnoreNotFoundTxnParams: VgiSchema;
-export declare const schemaNameCommentParams: VgiSchema;
-export declare const schemaNameRenameParams: VgiSchema;
-export declare const columnOpParams: VgiSchema;

package/dist/protocol/handlers/function.d.ts

@@ -11,6 +11,17 @@ export interface FunctionHandlerConfig {
      */
     catalogInterface?: {
         catalogNameForAttach(a: Uint8Array): string | null;
+        /**
+         * The catalog's current version, which is the consistency anchor every split
+         * token is stamped with AND checked against.
+         *
+         * Both sides read it from here on purpose. Minting from a different value
+         * than redemption compares against is not a subtle bug: it refuses every
+         * token, and the documented response to SPLIT_SNAPSHOT_EXPIRED is "re-run
+         * the query", which re-plans, mints the same mismatch and fails again — a
+         * livelock returning no rows, blaming the data for moving when it has not.
+         */
+        version?(attach: Uint8Array, txn?: Uint8Array): number | Promise<number>;
     };
 }
 export declare function registerFunctionMethods(protocol: Protocol, config: FunctionHandlerConfig): void;

package/dist/protocol/serializers/bind.d.ts

@@ -1,9 +1,8 @@
 import { type VgiSchema, type VgiBatch } from "../../arrow/index.js";
 import type { BindRequest, BindResponse } from "../types.js";
 declare const BIND_REQUEST_SCHEMA: VgiSchema;
-declare const BIND_RESPONSE_SCHEMA: VgiSchema;
 export declare function serializeBindRequest(req: BindRequest): VgiBatch;
 export declare function deserializeBindRequest(params: Record<string, any>): BindRequest;
 export declare function serializeBindResponse(resp: BindResponse): Record<string, any>;
 export declare function deserializeBindResponse(params: Record<string, any>): BindResponse;
-export { BIND_REQUEST_SCHEMA, BIND_RESPONSE_SCHEMA };
+export { BIND_REQUEST_SCHEMA };

package/dist/protocol/serializers/cardinality.d.ts

@@ -1,7 +1,6 @@
 import { type VgiSchema } from "../../arrow/index.js";
 import type { TableFunctionCardinalityRequest, TableCardinality } from "../types.js";
-declare const TABLE_CARDINALITY_SCHEMA: VgiSchema;
 declare const TABLE_FUNCTION_CARDINALITY_REQUEST_SCHEMA: VgiSchema;
 export declare function deserializeCardinalityRequest(params: Record<string, any>): TableFunctionCardinalityRequest;
 export declare function serializeTableCardinality(card: TableCardinality): Record<string, any>;
-export { TABLE_CARDINALITY_SCHEMA, TABLE_FUNCTION_CARDINALITY_REQUEST_SCHEMA };
+export { TABLE_FUNCTION_CARDINALITY_REQUEST_SCHEMA };

package/dist/protocol/serializers/splits.d.ts

@@ -0,0 +1,126 @@
+import { type VgiSchema } from "../../arrow/index.js";
+import type { BindRequest } from "../types.js";
+/**
+ * One named, independently redeemable unit of scan work.
+ *
+ * A worker sets `payload` and nothing else; the framework stamps `token` from
+ * it, so an author cannot forget the consistency anchor or mis-bind the
+ * fingerprint. The client sends the TOKEN back, never the raw payload.
+ */
+export interface ScanSplit {
+    /** The worker's own bytes NAMING this unit of work. */
+    payload: Uint8Array;
+    /** Framework-stamped envelope. Set by the framework, never by a worker. */
+    token?: Uint8Array;
+    estimatedRows?: number | bigint | null;
+    /** True if `estimatedRows` is exact — unlocks COUNT(*) from statistics. */
+    rowsExact?: boolean;
+    /**
+     * Load-bearing for engines that bin-pack (DataFusion weight, Trino
+     * SplitWeight). Omitting it degrades them to round-robin by count; a greedily
+     * claiming client needs no cost model at all.
+     */
+    estimatedBytes?: number | bigint | null;
+    /** 2-row (min, max) batch in the existing `vgi_partition_values` encoding. */
+    partitionBounds?: Uint8Array | null;
+    columnStatistics?: Uint8Array | null;
+    locationIds?: number[] | null;
+    startPosition?: Uint8Array | null;
+    /** Absent means UNBOUNDED — a shard read forever. */
+    endPosition?: Uint8Array | null;
+}
+/** What a table function's `plan` hook returns. */
+export interface PlanResult {
+    /**
+     * One entry per unit of work. EMPTY is legal and means "no work": a
+     * fully-pruned scan reaches it, and the client must produce an empty result
+     * rather than an error.
+     */
+    splits: ScanSplit[];
+    /**
+     * Continued enumeration. More than one MUST partition the remaining
+     * enumeration disjointly and exhaustively. No client checks this: a dedup was
+     * tried and removed (it needed a set holding a copy of every token, it compared
+     * token bytes so it could never work on a keyed worker where each mint uses a
+     * fresh nonce, and the most a client can do with a duplicate is refuse anyway).
+     * Violating it returns DUPLICATE ROWS, silently.
+     */
+    nextCursors?: Uint8Array[];
+    /** NORMATIVE cap on redemption concurrency, not advisory. */
+    maxWorkers?: number | null;
+    estimatedTotalSplits?: number | bigint | null;
+    estimatedTotalRows?: number | bigint | null;
+    estimatedTotalBytes?: number | bigint | null;
+    /**
+     * The counter a stale token is detected against, and the anchor the framework
+     * stamps into every token in this plan.
+     */
+    catalogVersion?: number | bigint | null;
+    /**
+     * `catalog` (the default) or `transaction`. A transaction-scoped plan is not
+     * cacheable and is not redeemable after commit or rollback.
+     */
+    scope?: "catalog" | "transaction";
+    cacheMaxAgeSeconds?: number | null;
+}
+/** The request a client sends to divide a scan. */
+export interface TableFunctionPlanRequest {
+    bind_call: BindRequest;
+    bind_opaque_data: Uint8Array | null;
+    projection_ids: number[] | null;
+    pushdown_filters: Uint8Array | null;
+    /** The primary sizing lever: every engine is byte-driven. */
+    target_split_bytes: bigint | null;
+    /**
+     * The parallelism FLOOR — a small but expensive table still needs one reader
+     * per thread, which a byte target alone would not give it.
+     */
+    min_splits: bigint | null;
+    /**
+     * A place in the ENUMERATION of splits — NOT a place in the data. A cursor
+     * lives for one plan call; a position is checkpointed and must survive
+     * restarts, upgrades and key rotation.
+     */
+    cursor: Uint8Array | null;
+}
+/**
+ * The ScanSplit shape, from codegen rather than hand-written.
+ *
+ * It used to be spelled out here, and hand-written is how four SDKs ended up
+ * disagreeing about which of these columns were binary and which were
+ * large_binary — a disagreement the client surfaced as "the worker bypassed the
+ * framework", on every scan.
+ */
+export declare const SCAN_SPLIT_SCHEMA: VgiSchema;
+/**
+ * The PlanResponse shape, taken from codegen rather than hand-written: it is
+ * regenerated from the canonical protocol, so it cannot drift.
+ */
+export declare const PLAN_RESPONSE_SCHEMA: VgiSchema;
+/**
+ * Decode a plan request.
+ *
+ * A client sends only the sizing inputs it actually has — the DuckDB extension
+ * sends seven columns, because it has no basis to invent a byte target it did
+ * not set — so every field but `bind_call` is read defensively.
+ */
+export declare function deserializePlanRequest(params: Record<string, any>): TableFunctionPlanRequest;
+/**
+ * The raw bind fields the split-token fingerprint is derived from.
+ *
+ * Read from the UNDECODED bind_call dict rather than the parsed BindRequest,
+ * because the fingerprint must be reproducible byte-for-byte between the plan
+ * that minted a token and the init that redeems it. A decoded-then-re-encoded
+ * argument list is not guaranteed to round-trip to identical bytes; the wire
+ * bytes are.
+ */
+export declare function fingerprintInputs(bindCallBytes: Uint8Array): {
+    schemaName: string;
+    functionName: string;
+    args: Uint8Array;
+    settings: Uint8Array;
+};
+/** Build the row dict for one `ScanSplit`. */
+export declare function scanSplitRow(split: ScanSplit, token: Uint8Array): Record<string, any>;
+/** Build the row dict for a `PlanResponse`, given already-serialized splits. */
+export declare function planResponseRow(result: PlanResult, splitBlobs: Uint8Array[]): Record<string, any>;

package/dist/protocol/types.d.ts

@@ -108,6 +108,33 @@ export interface InitRequest {
      * bytes the worker's combine() chose.
      */
     finalize_state_id: Uint8Array | null;
+    /**
+     * The framework-stamped split envelopes this init redeems, or null when this
+     * is not a split init.
+     *
+     * A LIST because an engine whose partition count IS its concurrency bin-packs
+     * at planning time and reads a whole group per partition; DuckDB always sends
+     * exactly one. A worker handed several concatenates them in the order given.
+     */
+    split_tokens: Uint8Array[] | null;
+    /**
+     * The VERIFIED payloads behind `split_tokens` — the worker's own bytes, with
+     * the envelope already opened and stripped.
+     *
+     * Populated by the protocol handler, which is what holds the signing key, so
+     * an unverified token never reaches a function. Null when this is not a split
+     * init.
+     */
+    split_payloads?: Uint8Array[] | null;
+    /**
+     * A plain fetch limit, when the engine can supply one.
+     *
+     * Always null from DuckDB — `TableFunctionInitInput` carries no limit, and
+     * the Top-N `order_by_limit` means something different ("top K by that
+     * column"). DataFusion supplies a real value via `TableProvider::scan(limit)`.
+     * Over-producing is legal; the engine re-applies the limit above the scan.
+     */
+    row_limit: number | null;
     execution_id: Uint8Array | null;
     init_opaque_data: Uint8Array | null;
     /**

package/dist/request-auth.d.ts

@@ -0,0 +1,28 @@
+import type { AuthContext } from "@query-farm/vgi-rpc";
+/** Mutable per-request cell. The scope is opened before the identity is known
+ *  (authentication happens inside the handler), so the holder is published
+ *  first and filled in when the framework reports the principal. */
+export interface RequestAuthHolder {
+    auth?: AuthContext;
+}
+/** The slice of `AsyncLocalStorage` this module needs. Structural on purpose:
+ *  it keeps `node:async_hooks` out of every non-HTTP build, and lets a test
+ *  install a trivial stub. */
+export interface RequestAuthScope {
+    getStore(): RequestAuthHolder | undefined;
+}
+/**
+ * Publish the scope `currentRequestAuth()` reads. Called once per process by
+ * the HTTP entry point; idempotent for the same scope.
+ */
+export declare function setRequestAuthScope(next: RequestAuthScope | undefined): void;
+/**
+ * The authenticated principal of the request being served on this async
+ * context, or `undefined` when there is none — no scope installed (stdio /
+ * AF_UNIX / a host that has no `AsyncLocalStorage`), or an unauthenticated
+ * request.
+ *
+ * `undefined` and an anonymous `AuthContext` produce the same identity tail, so
+ * the two cases need not be distinguished by callers.
+ */
+export declare function currentRequestAuth(): AuthContext | undefined;

package/dist/serve-entry.d.ts

@@ -1,5 +1,6 @@
 import type { CatalogInterface } from "./catalog/interface.js";
 import type { FunctionRegistry } from "./functions/registry.js";
+import type { AuthenticateFn } from "@query-farm/vgi-rpc";
 /** Environment variables `serveVgiWorker` reads. Injectable for testing. */
 export interface ServeEnv {
     PORT?: string;
@@ -40,6 +41,9 @@ export interface ServeVgiWorkerOptions {
     quiet?: boolean;
     /** Environment source (default `process.env`). */
     env?: ServeEnv;
+    /** Authenticates each request, returning the caller's `AuthContext`.
+     *  Omit for an anonymous worker (the default). */
+    authenticate?: AuthenticateFn;
 }
 /** The slice of `Bun.serve`'s return value this module uses. */
 export interface VgiHttpServer {