@query-farm/vgi 0.28.0 → 0.29.1

package/src/functions/table.ts

@@ -15,6 +15,10 @@ import {
 import { codecFor } from "../arrow/codec/registry.js";
 import type { OutputCollector } from "@query-farm/vgi-rpc";
 import { DEFAULT_MAX_WORKERS } from "../types.js";
+import type {
+  PlanResult,
+  TableFunctionPlanRequest,
+} from "../protocol/serializers/splits.js";
 import type {
   BindRequest,
   BindResponse,
@@ -34,10 +38,11 @@ import {
   validateConstConstraints,
   type ArgumentSpec,
   type ArgumentConstraints,
+  narrowArgValue,
 } from "../arguments/argument-spec.js";
+import { assertArrowType, assertArrowTypes } from "../arguments/argument-spec.js";
 import { batchToScalarDict, batchToSecretDict, projectSchema, safeNumber } from "../util/arrow/index.js";
 import {
-  buildJoinKeysLookup,
   deserializeFilters,
   FilteringOutputCollector,
   type PushdownFilters,
@@ -56,9 +61,16 @@ import { CACHE_IF_MODIFIED_SINCE_KEY, CACHE_IF_NONE_MATCH_KEY } from "../cache-c
 // Base64-decode a string into raw bytes. Used to unpack the dynamic filter
 // update DuckDB attaches to each tick batch's custom metadata.
 function base64Decode(s: string): Uint8Array {
+  if (new TextEncoder().encode(s).byteLength > (24 << 20)) {
+    throw new Error("base64 dynamic filter metadata exceeds the encoded-size limit");
+  }
+  if (s.length % 4 !== 0 || !/^(?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?$/.test(s)) {
+    throw new Error("dynamic filter metadata is not valid base64");
+  }
   const bin = (globalThis as any).atob ? (globalThis as any).atob(s) : Buffer.from(s, "base64").toString("binary");
   const out = new Uint8Array(bin.length);
   for (let i = 0; i < bin.length; i++) out[i] = bin.charCodeAt(i);
+  if (out.byteLength > (17 << 20)) throw new Error("dynamic filter IPC payload exceeds the encoded-size limit");
   return out;
 }
 
@@ -134,6 +146,8 @@ export interface TableBindParams<TArgs = Record<string, any>> {
   settings: Record<string, any>;
   secrets: Record<string, Record<string, any>>;
   resolvedSecretsProvided: boolean;
+  /** Resolved names aligned with the complete logical argument order. */
+  argumentNames?: (string | null)[] | null;
 }
 
 export interface TableProcessParams<TArgs = Record<string, any>> {
@@ -144,6 +158,15 @@ export interface TableProcessParams<TArgs = Record<string, any>> {
   settings: Record<string, any>;
   secrets: Record<string, Record<string, any>>;
   pushdownFilters?: PushdownFilters;
+  /**
+   * The verified payloads of the splits this init claimed, in the order given,
+   * or undefined when this is not a split scan.
+   *
+   * Undefined is meaningfully different from an empty array: undefined means
+   * the client did not plan (a split-only function should fail loudly rather
+   * than answer the same query differently), empty means a claim of no work.
+   */
+  splitPayloads?: Uint8Array[];
   storage?: BoundStorage;
   /**
    * AT (TIMESTAMP|VERSION) clause for this scan, or `undefined` when the scan
@@ -184,6 +207,8 @@ export interface TableFunctionConfig<
   argDocs?: Record<string, string>;
   /** Argument defaults */
   argDefaults?: Record<string, any>;
+  /** Authoritative typed defaults: one row, defaulted parameters only. */
+  parameterDefaultValues?: VgiBatch | null;
   /**
    * Per-argument discovery constraints (choices / ge / le / gt / lt / pattern),
    * keyed by argument name. Surfaced via `vgi_function_arguments()` for agent
@@ -227,6 +252,45 @@ export interface TableFunctionConfig<
   ) => void | Promise<void>;
   /** Cardinality hints */
   cardinality?: (params: TableBindParams<TArgs>) => TableCardinality | Promise<TableCardinality>;
+
+  /**
+   * Divide this scan into named, independently redeemable splits.
+   *
+   * Declaring it (together with `meta.supportsSplits`) is what opts a function
+   * into the split path; omitting it means the whole scan is one unit of work.
+   *
+   * A split *names* work rather than describing it. "These three files at
+   * version 47" survives a retry; "rows 0-999 of whatever this returns now"
+   * does not — and a distributed engine WILL retry, so the difference is
+   * correctness, not tidiness. The same split may also be redeemed more than
+   * once (recursive CTEs, re-collected DataFrames, task retry) and may be
+   * abandoned mid-stream (LIMIT, TopK, an empty join build side); neither is an
+   * error.
+   *
+   * Set only `payload` on each split. The framework stamps the consistency
+   * anchor, the bind fingerprint and (where a key exists) the seal.
+   *
+   * Size splits into comparable units of work and honour `targetSplitBytes`: a
+   * claiming client treats them as interchangeable because it cannot see
+   * per-split cost, so wildly uneven splits leave its makespan bounded by the
+   * largest one.
+   */
+  plan?: (
+    params: TableBindParams<TArgs>,
+    request: TableFunctionPlanRequest,
+  ) => PlanResult | Promise<PlanResult>;
+
+  /**
+   * Called on a split init with the VERIFIED payloads for the splits this
+   * connection claimed — the envelope is already opened and stripped, so
+   * unverified bytes never reach here.
+   *
+   * Any state carried from planning to reading must live in cross-process
+   * storage keyed by `execution_id`: the process that plans is, in the general
+   * case, not the process that reads — and under a distributed engine it is not
+   * even the same host.
+   */
+  onSplit?: (payloads: Uint8Array[], params: TableBindParams<TArgs>) => void | Promise<void>;
   /**
    * Per-column statistics for the function's output. Returned to DuckDB via
    * the `table_function_statistics` RPC; the optimizer uses min/max to
@@ -256,7 +320,10 @@ export interface TableFunctionConfig<
   /** Opt in to DuckDB's late-materialization SEMI-join rewrite; requires a
    *  UNIQUE, snapshot-stable rowid column. FunctionInfo.late_materialization. */
   lateMaterialization?: boolean;
-  supportedExpressionFilters?: string[];
+  filterSemanticProfiles?: string[];
+  additionalFilterFunctions?: import("./types.js").FilterFunctionCapability[];
+  runtimeFilterAlgorithms?: import("./types.js").FilterFunctionCapability[];
+  filterEvaluationContexts?: import("./types.js").EvaluationContextCapability[];
   autoApplyFilters?: boolean;
   stability?: FunctionStability;
   examples?: FunctionExample[];
@@ -275,12 +342,33 @@ export interface TableFunctionConfig<
   supportsBatchIndex?: boolean;
   /** Hive-style partition-columns mode; FunctionInfo.partition_kind. */
   partitionKind?: "NOT_PARTITIONED" | "SINGLE_VALUE_PARTITIONS" | "OVERLAPPING_PARTITIONS" | "DISJOINT_PARTITIONS";
+  /**
+   * This function divides its scan into named, independently redeemable splits;
+   * FunctionInfo.supports_splits. Declare it together with `plan` and `onSplit`.
+   *
+   * It is what a distributed engine reads to decide it can retry a task against
+   * this function, and what makes the client call `plan()` at all — without it a
+   * split-capable function is scanned as one undivided unit, silently.
+   */
+  supportsSplits?: boolean;
+  /**
+   * How long a minted split token stays redeemable, or undefined for UNBOUNDED
+   * (NOT "expires immediately"); FunctionInfo.split_token_ttl_seconds.
+   *
+   * A client refuses a plan whose TTL is below its own scheduling horizon,
+   * because an expired token is a failed query rather than a degradation:
+   * nothing re-plans when one expires, since a distributed engine retries the
+   * serialized task it was handed and has no path back to the planner.
+   */
+  splitTokenTtlSeconds?: number;
 }
 
 export function defineTableFunction<
   TArgs = Record<string, any>,
   TState = null,
 >(config: TableFunctionConfig<TArgs, TState>): VgiFunction {
+  assertArrowTypes((config as any).args, `defineTableFunction("${config.name}"): args`);
+
   // Build argument specs
   const specs: ArgumentSpec[] = [];
   let posIdx = 0;
@@ -298,7 +386,10 @@ export function defineTableFunction<
         isAnyType: isAny,
         isVarargs: varargsSet.has(name),
         doc: config.argDocs?.[name],
-        ...constraintSpecFields(config.argConstraints?.[name]),
+        ...constraintSpecFields(
+          config.argConstraints?.[name],
+          config.argDefaults?.[name],
+        ),
       });
     }
   }
@@ -311,11 +402,15 @@ export function defineTableFunction<
     filterPushdown: config.filterPushdown,
     samplingPushdown: config.samplingPushdown,
     lateMaterialization: config.lateMaterialization,
-    supportedExpressionFilters: config.supportedExpressionFilters,
+    filterSemanticProfiles: config.filterSemanticProfiles,
+    additionalFilterFunctions: config.additionalFilterFunctions,
+    runtimeFilterAlgorithms: config.runtimeFilterAlgorithms,
+    filterEvaluationContexts: config.filterEvaluationContexts,
     autoApplyFilters: config.autoApplyFilters,
     examples: config.examples,
     categories: config.categories,
     tags: config.tags,
+    parameterDefaultValues: config.parameterDefaultValues,
     maxWorkers: config.maxWorkers,
     requiredSettings: config.requiredSettings,
     requiredSecrets: config.requiredSecrets,
@@ -325,6 +420,8 @@ export function defineTableFunction<
     distinctDependent: config.distinctDependent,
     supportsBatchIndex: config.supportsBatchIndex,
     partitionKind: config.partitionKind,
+    supportsSplits: config.supportsSplits,
+    splitTokenTtlSeconds: config.splitTokenTtlSeconds,
   };
 
   function extractArgs(request: BindRequest): TArgs {
@@ -342,8 +439,9 @@ export function defineTableFunction<
         // converts positional args to named args)
         val = request.arguments.get(spec.name, defaultVal);
       }
-      // Arrow Int64 values come through as BigInt — coerce to number
-      if (typeof val === "bigint") val = safeNumber(val);
+      // Arrow Int64 arrives as bigint. Narrow when lossless, keep the
+      // bigint when not — see narrowArgValue.
+      val = narrowArgValue(val);
       // Enforce declared constraints at bind (table args are all bind-time).
       const constraints = config.argConstraints?.[spec.name];
       if (constraints) validateConstConstraints(spec.name, constraints, val);
@@ -364,6 +462,7 @@ export function defineTableFunction<
       const result = await config.onBind({
         args, bindCall: request, settings, secrets,
         resolvedSecretsProvided: request.resolved_secrets_provided ?? false,
+        argumentNames: request.argument_names ?? null,
       });
       return {
         output_schema: result.outputSchema,
@@ -378,6 +477,28 @@ export function defineTableFunction<
       const executionId = new Uint8Array(16);
       crypto.getRandomValues(executionId);
 
+      // A SPLIT init is neither primary nor secondary. It carries an
+      // execution_id like a secondary, so it must not re-run global init — but
+      // unlike a secondary it MUST run user code, because the payload is what
+      // names the work. The secondary branch below runs none, which is why this
+      // is a third branch rather than a flag on that one.
+      if (request.split_payloads) {
+        if (config.onSplit) {
+          await config.onSplit(request.split_payloads, {
+            args: extractArgs(request.bind_call),
+            bindCall: request.bind_call,
+            settings: batchToScalarDict(request.bind_call.settings),
+            secrets: batchToSecretDict(request.bind_call.secrets),
+            resolvedSecretsProvided: request.bind_call.resolved_secrets_provided ?? false,
+          });
+        }
+        return {
+          max_workers: config.maxWorkers ?? 1,
+          execution_id: request.execution_id ?? executionId,
+          opaque_data: null,
+        };
+      }
+
       if (request.execution_id) {
         // Secondary init - reuse execution ID
         return {
@@ -425,9 +546,15 @@ export function defineTableFunction<
       // Deserialize pushdown filters. Pass a join-keys column lookup so that
       // filters DuckDB promoted to join_keys (IN/OR lists, etc.) are
       // materialized as InFilters rather than silently dropped.
-      const joinKeysLookup = buildJoinKeysLookup(request.join_keys);
+      const filterOptions = {
+        outputSchema: request.output_schema,
+        joinKeyBatches: request.join_keys,
+        extensionFunctions: meta.additionalFilterFunctions,
+        runtimeAlgorithms: meta.runtimeFilterAlgorithms,
+        evaluationContexts: meta.filterEvaluationContexts,
+      };
       const pushdownFilters = request.pushdown_filters
-        ? deserializeFilters(request.pushdown_filters, joinKeysLookup)
+        ? deserializeFilters(request.pushdown_filters, filterOptions)
         : undefined;
 
       const boundStorage = new BoundStorage(globalStorage, response.execution_id);
@@ -440,11 +567,26 @@ export function defineTableFunction<
         settings,
         secrets,
         pushdownFilters,
+        splitPayloads: request.split_payloads ?? undefined,
         storage: boundStorage,
         atUnit: request.bind_call.at_unit ?? undefined,
         atValue: request.bind_call.at_value ?? undefined,
       };
 
+      if (request.phase != null) {
+        // Mirror image of the table-in-out guard: this is a plain producer
+        // (no input stream), but the caller sent a table-in-out init phase --
+        // i.e. drove it via table_in_out_function() instead of
+        // table_function(). Reject immediately rather than silently ignoring
+        // the phase/input stream and running as an ordinary producer while
+        // the caller is left feeding batches nobody reads.
+        throw new Error(
+          `${config.name}: is a plain table function (it takes no input row ` +
+            `stream) but was called with an init phase set -- call it via ` +
+            `table_function(), not table_in_out_function().`,
+        );
+      }
+
       const state = config.initialState
         ? config.initialState(processParams)
         : (null as TState);
@@ -469,23 +611,20 @@ export function defineTableFunction<
           if (ifModifiedSince !== undefined) pState.processParams.ifModifiedSince = ifModifiedSince;
 
           // Dynamic filter pushdown: DuckDB's Top-N optimizer tightens filters
-          // between ticks and serializes the current filter into the tick
+          // between ticks and serializes an atomic v2 delta into the tick
           // batch's custom metadata under `vgi_pushdown_filters` (base64 of a
-          // filter IPC stream). Decode and overwrite the current pushdown
-          // filters so process() sees the updated value.
+          // filter IPC stream). Apply it to the prior snapshot so process()
+          // sees the revised advisory predicate state.
           const encoded = tickMetadata.get("vgi_pushdown_filters");
           if (!encoded) return;
-          try {
-            const bytes = base64Decode(encoded);
-            const filterBatch = deserializeFilterBatch(bytes);
-            if (filterBatch) {
-              const updated = deserializeFilters(filterBatch, joinKeysLookup);
-              pState.processParams.pushdownFilters = updated;
-            }
-          } catch {
-            // Malformed dynamic-filter update: keep the previous filter. Not
-            // fatal — this is a best-effort optimization hint from DuckDB.
-          }
+          const current = pState.processParams.pushdownFilters;
+          if (!current) throw new Error("dynamic filter delta received without an initial snapshot");
+          const bytes = base64Decode(encoded);
+          const filterBatch = deserializeFilterBatch(bytes);
+          if (!filterBatch) throw new Error("dynamic filter metadata must contain one RecordBatch");
+          // Assignment happens only after complete validation, so a rejected
+          // delta leaves the immutable prior state intact.
+          pState.processParams.pushdownFilters = current.applyDelta(filterBatch);
         },
         producerFn: async (
           pState: { state: TState; processParams: TableProcessParams<TArgs> },
@@ -523,6 +662,32 @@ export function defineTableFunction<
         }
       : undefined,
 
+    // Wired through with decoded arguments, exactly as cardinality and
+    // statistics are.
+    //
+    // It was previously declared on the config type and never passed to the
+    // returned function, so the hook was DEAD: a worker could declare
+    // `supportsSplits` and a `plan`, advertise split capability on the wire, and
+    // silently serve the framework default of one empty-payload split. Nothing
+    // failed — the scan just quietly stopped being divided.
+    plan: config.plan
+      ? (request: TableFunctionPlanRequest) => {
+          const args = extractArgs(request.bind_call);
+          const settings = batchToScalarDict(request.bind_call.settings);
+          const secrets = batchToSecretDict(request.bind_call.secrets);
+          return config.plan!(
+            {
+              args,
+              bindCall: request.bind_call,
+              settings,
+              secrets,
+              resolvedSecretsProvided: request.bind_call.resolved_secrets_provided ?? false,
+            },
+            request,
+          );
+        }
+      : undefined,
+
     statistics: config.statistics
       ? (request: TableFunctionCardinalityRequest) => {
           const args = extractArgs(request.bind_call);

package/src/functions/types.ts

@@ -6,6 +6,7 @@ import type { OutputCollector } from "@query-farm/vgi-rpc";
 import type {
   FunctionStability,
   NullHandling,
+  ArgumentMonotonicity,
   OrderPreservation,
   OrderDependence,
   DistinctDependence,
@@ -34,14 +35,33 @@ export interface FunctionExample {
   expectedOutput?: string;
 }
 
+export interface FilterFunctionCapability {
+  namespace: string;
+  name: string;
+  version: number;
+}
+
+export interface EvaluationContextCapability {
+  profile: string;
+  providerFingerprint?: string | null;
+}
+
 export interface FunctionMeta {
   name: string;
   description?: string;
   stability?: FunctionStability;
   nullHandling?: NullHandling;
+  /** Scalar-only claims in argument declaration order. A vararg is one slot. */
+  argumentMonotonicity?: ArgumentMonotonicity[] | null;
   examples?: FunctionExample[];
   categories?: string[];
   tags?: Record<string, string>;
+  /**
+   * Authoritative typed parameter defaults. The batch must contain one row
+   * and only defaulted parameters, in signature order; a present null is an
+   * explicit null default. `vgi_default` field metadata remains discovery-only.
+   */
+  parameterDefaultValues?: VgiBatch | null;
   projectionPushdown?: boolean;
   filterPushdown?: boolean;
   samplingPushdown?: boolean;
@@ -54,7 +74,10 @@ export interface FunctionMeta {
    * for tables whose worker also guarantees a UNIQUE, snapshot-stable rowid.
    */
   lateMaterialization?: boolean;
-  supportedExpressionFilters?: string[];
+  filterSemanticProfiles?: string[];
+  additionalFilterFunctions?: FilterFunctionCapability[];
+  runtimeFilterAlgorithms?: FilterFunctionCapability[];
+  filterEvaluationContexts?: EvaluationContextCapability[];
   autoApplyFilters?: boolean;
   preservesOrder?: OrderPreservation;
   maxWorkers?: number;
@@ -102,6 +125,38 @@ export interface FunctionMeta {
    * `supports_batch_index`.
    */
   supportsBatchIndex?: boolean;
+  /**
+   * table (generator): the scan divides into named, independently redeemable
+   * splits (see the `plan` / `onSplit` hooks). Surfaces as FunctionInfo
+   * `supports_splits`.
+   *
+   * The declaration is what a distributed engine reads to decide whether it can
+   * retry a task: a split NAMES its work, so re-running one reads exactly the
+   * same rows. Without it, the division lives behind an opaque worker-side queue
+   * and a retry reads neither what it read before nor what it missed.
+   */
+  supportsSplits?: boolean;
+  /**
+   * table (generator): the worker applies every pushed filter EXACTLY, so the
+   * engine may drop its own copy rather than re-filtering. Wrong answers if
+   * declared falsely — leave unset unless certain.
+   */
+  filtersExactlyApplied?: boolean;
+  /**
+   * table (generator): addressable positions in the data, for incremental and
+   * streaming reads. The engine owns the checkpoint; this declares only that
+   * positions are meaningful to externalize.
+   */
+  supportsPositions?: boolean;
+  /**
+   * table (generator): how long a split token stays redeemable.
+   *
+   * UNSET MEANS UNBOUNDED, not "expires immediately" — a client must not assume
+   * a TTL exists, or long-running streams are foreclosed. A client rejects a
+   * plan whose TTL is below its own planning-to-task-start horizon, which on a
+   * busy cluster is hours rather than seconds.
+   */
+  splitTokenTtlSeconds?: number;
   /**
    * table (generator): Hive-style partition-columns mode. Functions declare
    * a PartitionKind and annotate bind-schema fields; emitted batches carry
@@ -180,6 +235,14 @@ export interface VgiFunction {
     accumulatedState?: any,
   ): StreamHandlers;
   cardinality?(request: TableFunctionCardinalityRequest): TableCardinality | Promise<TableCardinality>;
+  /**
+   * Divide a scan into named, independently redeemable splits. Absent means the
+   * function is not split-capable: the whole scan is one unit of work, and the
+   * framework answers with a single empty-payload split.
+   */
+  plan?(request: import("../protocol/serializers/splits.js").TableFunctionPlanRequest):
+    | import("../protocol/serializers/splits.js").PlanResult
+    | Promise<import("../protocol/serializers/splits.js").PlanResult>;
   /**
    * Per-column statistics for this table function's output given the user's
    * bind-time arguments. Returns null/[] when stats are unknown. Wired to the