@query-farm/vgi 0.28.0 → 0.29.0

package/src/functions/scalar.ts

@@ -41,6 +41,8 @@ import {
   argumentSpecsToSchema,
   constraintSpecFields,
   validateConstConstraints,
+  assertArrowType,
+  assertArrowTypes,
   type ArgumentConstraints,
 } from "../arguments/argument-spec.js";
 import { batchToScalarDict, batchToSecretDict, batchFromColumns, projectBatch, safeNumber } from "../util/arrow/index.js";
@@ -202,6 +204,15 @@ export function defineScalarFunction<
   R extends VgiDataType = VgiDataType,
   M extends Repr = "rich",
 >(config: ScalarFunctionConfig<P, R, M>): VgiFunction {
+  assertArrowTypes(config.params, `defineScalarFunction("${config.name}"): params`);
+  assertArrowTypes(config.constParams, `defineScalarFunction("${config.name}"): constParams`);
+  if (config.returns !== undefined) {
+    assertArrowType(config.returns, `defineScalarFunction("${config.name}"): returns`);
+  }
+  for (const p of config.parameters ?? []) {
+    assertArrowType(p.type, `defineScalarFunction("${config.name}"): parameters["${p.name}"].type`);
+  }
+
   const repr: Repr = config.repr ?? "rich";
   // Build argument specs
   const specs: ArgumentSpec[] = [];

package/src/functions/table-buffering.ts

@@ -40,6 +40,7 @@ import type {
   FunctionExample,
 } from "./types.js";
 import type { ArgumentSpec } from "../arguments/argument-spec.js";
+import { assertArrowType, assertArrowTypes } from "../arguments/argument-spec.js";
 import { batchToScalarDict, batchToSecretDict, projectSchema } from "../util/arrow/index.js";
 import { batchFromColumns, readCanonicalValue } from "../arrow/index.js";
 import { codecFor } from "../arrow/codec/registry.js";
@@ -182,6 +183,8 @@ export function defineTableBufferingFunction<
   TArgs = Record<string, any>,
   TState = any,
 >(config: TableBufferingConfig<TArgs, TState>): TableBufferingVgiFunction {
+  assertArrowTypes((config as any).args, `defineTableBufferingFunction("${config.name}"): args`);
+
   if (config.sinkOrderDependent && config.requiresInputBatchIndex) {
     throw new Error(
       `${config.name}: sinkOrderDependent and requiresInputBatchIndex are ` +

package/src/functions/table-in-out.ts

@@ -19,7 +19,8 @@ import type {
   FunctionExample,
 } from "./types.js";
 import type { ArgumentSpec } from "../arguments/argument-spec.js";
-import { batchToScalarDict, batchToSecretDict, projectSchema, projectBatch, emptyBatch } from "../util/arrow/index.js";
+import { assertArrowType, assertArrowTypes, narrowArgValue } from "../arguments/argument-spec.js";
+import { batchToScalarDict, batchToSecretDict, projectSchema, projectBatch, emptyBatch, safeNumber } from "../util/arrow/index.js";
 import { CACHE_IF_MODIFIED_SINCE_KEY, CACHE_IF_NONE_MATCH_KEY } from "../cache-control.js";
 import {
   buildJoinKeysLookup,
@@ -132,6 +133,8 @@ export function defineTableInOutFunction<
   TArgs = Record<string, any>,
   TState = null,
 >(config: TableInOutConfig<TArgs, TState>): VgiFunction {
+  assertArrowTypes((config as any).args, `defineTableInOutFunction("${config.name}"): args`);
+
 
   // Build argument specs
   const specs: ArgumentSpec[] = [];
@@ -192,8 +195,9 @@ export function defineTableInOutFunction<
           : undefined;
       // For named args, position is a string (the arg name)
       let val = request.arguments.get(spec.position, defaultVal);
-      // Arrow Int64 values come through as BigInt — coerce to number
-      if (typeof val === "bigint") val = Number(val);
+      // Arrow Int64 arrives as bigint. Narrow when lossless, keep the
+      // bigint when not — see narrowArgValue.
+      val = narrowArgValue(val);
       args[spec.name] = val;
     }
     return args as TArgs;
@@ -735,7 +739,7 @@ export function defineRowTransformFunction<
           ? config.argDefaults[spec.name]
           : undefined;
       let val = request.arguments.get(spec.position, defaultVal ?? null);
-      if (typeof val === "bigint") val = Number(val);
+      val = narrowArgValue(val);
       args[spec.name] = val;
     }
     return args as TArgs;

package/src/functions/table.ts

@@ -15,6 +15,10 @@ import {
 import { codecFor } from "../arrow/codec/registry.js";
 import type { OutputCollector } from "@query-farm/vgi-rpc";
 import { DEFAULT_MAX_WORKERS } from "../types.js";
+import type {
+  PlanResult,
+  TableFunctionPlanRequest,
+} from "../protocol/serializers/splits.js";
 import type {
   BindRequest,
   BindResponse,
@@ -34,7 +38,9 @@ import {
   validateConstConstraints,
   type ArgumentSpec,
   type ArgumentConstraints,
+  narrowArgValue,
 } from "../arguments/argument-spec.js";
+import { assertArrowType, assertArrowTypes } from "../arguments/argument-spec.js";
 import { batchToScalarDict, batchToSecretDict, projectSchema, safeNumber } from "../util/arrow/index.js";
 import {
   buildJoinKeysLookup,
@@ -144,6 +150,15 @@ export interface TableProcessParams<TArgs = Record<string, any>> {
   settings: Record<string, any>;
   secrets: Record<string, Record<string, any>>;
   pushdownFilters?: PushdownFilters;
+  /**
+   * The verified payloads of the splits this init claimed, in the order given,
+   * or undefined when this is not a split scan.
+   *
+   * Undefined is meaningfully different from an empty array: undefined means
+   * the client did not plan (a split-only function should fail loudly rather
+   * than answer the same query differently), empty means a claim of no work.
+   */
+  splitPayloads?: Uint8Array[];
   storage?: BoundStorage;
   /**
    * AT (TIMESTAMP|VERSION) clause for this scan, or `undefined` when the scan
@@ -227,6 +242,45 @@ export interface TableFunctionConfig<
   ) => void | Promise<void>;
   /** Cardinality hints */
   cardinality?: (params: TableBindParams<TArgs>) => TableCardinality | Promise<TableCardinality>;
+
+  /**
+   * Divide this scan into named, independently redeemable splits.
+   *
+   * Declaring it (together with `meta.supportsSplits`) is what opts a function
+   * into the split path; omitting it means the whole scan is one unit of work.
+   *
+   * A split *names* work rather than describing it. "These three files at
+   * version 47" survives a retry; "rows 0-999 of whatever this returns now"
+   * does not — and a distributed engine WILL retry, so the difference is
+   * correctness, not tidiness. The same split may also be redeemed more than
+   * once (recursive CTEs, re-collected DataFrames, task retry) and may be
+   * abandoned mid-stream (LIMIT, TopK, an empty join build side); neither is an
+   * error.
+   *
+   * Set only `payload` on each split. The framework stamps the consistency
+   * anchor, the bind fingerprint and (where a key exists) the seal.
+   *
+   * Size splits into comparable units of work and honour `targetSplitBytes`: a
+   * claiming client treats them as interchangeable because it cannot see
+   * per-split cost, so wildly uneven splits leave its makespan bounded by the
+   * largest one.
+   */
+  plan?: (
+    params: TableBindParams<TArgs>,
+    request: TableFunctionPlanRequest,
+  ) => PlanResult | Promise<PlanResult>;
+
+  /**
+   * Called on a split init with the VERIFIED payloads for the splits this
+   * connection claimed — the envelope is already opened and stripped, so
+   * unverified bytes never reach here.
+   *
+   * Any state carried from planning to reading must live in cross-process
+   * storage keyed by `execution_id`: the process that plans is, in the general
+   * case, not the process that reads — and under a distributed engine it is not
+   * even the same host.
+   */
+  onSplit?: (payloads: Uint8Array[], params: TableBindParams<TArgs>) => void | Promise<void>;
   /**
    * Per-column statistics for the function's output. Returned to DuckDB via
    * the `table_function_statistics` RPC; the optimizer uses min/max to
@@ -275,12 +329,33 @@ export interface TableFunctionConfig<
   supportsBatchIndex?: boolean;
   /** Hive-style partition-columns mode; FunctionInfo.partition_kind. */
   partitionKind?: "NOT_PARTITIONED" | "SINGLE_VALUE_PARTITIONS" | "OVERLAPPING_PARTITIONS" | "DISJOINT_PARTITIONS";
+  /**
+   * This function divides its scan into named, independently redeemable splits;
+   * FunctionInfo.supports_splits. Declare it together with `plan` and `onSplit`.
+   *
+   * It is what a distributed engine reads to decide it can retry a task against
+   * this function, and what makes the client call `plan()` at all — without it a
+   * split-capable function is scanned as one undivided unit, silently.
+   */
+  supportsSplits?: boolean;
+  /**
+   * How long a minted split token stays redeemable, or undefined for UNBOUNDED
+   * (NOT "expires immediately"); FunctionInfo.split_token_ttl_seconds.
+   *
+   * A client refuses a plan whose TTL is below its own scheduling horizon,
+   * because an expired token is a failed query rather than a degradation:
+   * nothing re-plans when one expires, since a distributed engine retries the
+   * serialized task it was handed and has no path back to the planner.
+   */
+  splitTokenTtlSeconds?: number;
 }
 
 export function defineTableFunction<
   TArgs = Record<string, any>,
   TState = null,
 >(config: TableFunctionConfig<TArgs, TState>): VgiFunction {
+  assertArrowTypes((config as any).args, `defineTableFunction("${config.name}"): args`);
+
   // Build argument specs
   const specs: ArgumentSpec[] = [];
   let posIdx = 0;
@@ -325,6 +400,8 @@ export function defineTableFunction<
     distinctDependent: config.distinctDependent,
     supportsBatchIndex: config.supportsBatchIndex,
     partitionKind: config.partitionKind,
+    supportsSplits: config.supportsSplits,
+    splitTokenTtlSeconds: config.splitTokenTtlSeconds,
   };
 
   function extractArgs(request: BindRequest): TArgs {
@@ -342,8 +419,9 @@ export function defineTableFunction<
         // converts positional args to named args)
         val = request.arguments.get(spec.name, defaultVal);
       }
-      // Arrow Int64 values come through as BigInt — coerce to number
-      if (typeof val === "bigint") val = safeNumber(val);
+      // Arrow Int64 arrives as bigint. Narrow when lossless, keep the
+      // bigint when not — see narrowArgValue.
+      val = narrowArgValue(val);
       // Enforce declared constraints at bind (table args are all bind-time).
       const constraints = config.argConstraints?.[spec.name];
       if (constraints) validateConstConstraints(spec.name, constraints, val);
@@ -378,6 +456,28 @@ export function defineTableFunction<
       const executionId = new Uint8Array(16);
       crypto.getRandomValues(executionId);
 
+      // A SPLIT init is neither primary nor secondary. It carries an
+      // execution_id like a secondary, so it must not re-run global init — but
+      // unlike a secondary it MUST run user code, because the payload is what
+      // names the work. The secondary branch below runs none, which is why this
+      // is a third branch rather than a flag on that one.
+      if (request.split_payloads) {
+        if (config.onSplit) {
+          await config.onSplit(request.split_payloads, {
+            args: extractArgs(request.bind_call),
+            bindCall: request.bind_call,
+            settings: batchToScalarDict(request.bind_call.settings),
+            secrets: batchToSecretDict(request.bind_call.secrets),
+            resolvedSecretsProvided: request.bind_call.resolved_secrets_provided ?? false,
+          });
+        }
+        return {
+          max_workers: config.maxWorkers ?? 1,
+          execution_id: request.execution_id ?? executionId,
+          opaque_data: null,
+        };
+      }
+
       if (request.execution_id) {
         // Secondary init - reuse execution ID
         return {
@@ -440,6 +540,7 @@ export function defineTableFunction<
         settings,
         secrets,
         pushdownFilters,
+        splitPayloads: request.split_payloads ?? undefined,
         storage: boundStorage,
         atUnit: request.bind_call.at_unit ?? undefined,
         atValue: request.bind_call.at_value ?? undefined,
@@ -523,6 +624,32 @@ export function defineTableFunction<
         }
       : undefined,
 
+    // Wired through with decoded arguments, exactly as cardinality and
+    // statistics are.
+    //
+    // It was previously declared on the config type and never passed to the
+    // returned function, so the hook was DEAD: a worker could declare
+    // `supportsSplits` and a `plan`, advertise split capability on the wire, and
+    // silently serve the framework default of one empty-payload split. Nothing
+    // failed — the scan just quietly stopped being divided.
+    plan: config.plan
+      ? (request: TableFunctionPlanRequest) => {
+          const args = extractArgs(request.bind_call);
+          const settings = batchToScalarDict(request.bind_call.settings);
+          const secrets = batchToSecretDict(request.bind_call.secrets);
+          return config.plan!(
+            {
+              args,
+              bindCall: request.bind_call,
+              settings,
+              secrets,
+              resolvedSecretsProvided: request.bind_call.resolved_secrets_provided ?? false,
+            },
+            request,
+          );
+        }
+      : undefined,
+
     statistics: config.statistics
       ? (request: TableFunctionCardinalityRequest) => {
           const args = extractArgs(request.bind_call);

package/src/functions/types.ts

@@ -102,6 +102,38 @@ export interface FunctionMeta {
    * `supports_batch_index`.
    */
   supportsBatchIndex?: boolean;
+  /**
+   * table (generator): the scan divides into named, independently redeemable
+   * splits (see the `plan` / `onSplit` hooks). Surfaces as FunctionInfo
+   * `supports_splits`.
+   *
+   * The declaration is what a distributed engine reads to decide whether it can
+   * retry a task: a split NAMES its work, so re-running one reads exactly the
+   * same rows. Without it, the division lives behind an opaque worker-side queue
+   * and a retry reads neither what it read before nor what it missed.
+   */
+  supportsSplits?: boolean;
+  /**
+   * table (generator): the worker applies every pushed filter EXACTLY, so the
+   * engine may drop its own copy rather than re-filtering. Wrong answers if
+   * declared falsely — leave unset unless certain.
+   */
+  filtersExactlyApplied?: boolean;
+  /**
+   * table (generator): addressable positions in the data, for incremental and
+   * streaming reads. The engine owns the checkpoint; this declares only that
+   * positions are meaningful to externalize.
+   */
+  supportsPositions?: boolean;
+  /**
+   * table (generator): how long a split token stays redeemable.
+   *
+   * UNSET MEANS UNBOUNDED, not "expires immediately" — a client must not assume
+   * a TTL exists, or long-running streams are foreclosed. A client rejects a
+   * plan whose TTL is below its own planning-to-task-start horizon, which on a
+   * busy cluster is hours rather than seconds.
+   */
+  splitTokenTtlSeconds?: number;
   /**
    * table (generator): Hive-style partition-columns mode. Functions declare
    * a PartitionKind and annotate bind-schema fields; emitted batches carry
@@ -180,6 +212,14 @@ export interface VgiFunction {
     accumulatedState?: any,
   ): StreamHandlers;
   cardinality?(request: TableFunctionCardinalityRequest): TableCardinality | Promise<TableCardinality>;
+  /**
+   * Divide a scan into named, independently redeemable splits. Absent means the
+   * function is not split-capable: the whole scan is one unit of work, and the
+   * framework answers with a single empty-payload split.
+   */
+  plan?(request: import("../protocol/serializers/splits.js").TableFunctionPlanRequest):
+    | import("../protocol/serializers/splits.js").PlanResult
+    | Promise<import("../protocol/serializers/splits.js").PlanResult>;
   /**
    * Per-column statistics for this table function's output given the user's
    * bind-time arguments. Returns null/[] when stats are unknown. Wired to the

package/src/generated/vgi-client.ts

@@ -2,7 +2,7 @@
 // GENERATED by vgi.codegen.ts_client. DO NOT EDIT BY HAND.
 //
 // Generator: vgi-gen-ts-client v1
-// Content hash: e0862da99312
+// Content hash: bda03237975e
 //
 // To regenerate:
 //   uv run --project ~/Development/vgi-python vgi-gen-ts-client \
@@ -129,6 +129,10 @@ export interface FunctionInfo {
   order_preservation?: OrderPreservation | null;
   max_workers?: number | null;
   supports_batch_index: boolean;
+  supports_splits: boolean;
+  filters_exactly_applied: boolean;
+  supports_positions: boolean;
+  split_token_ttl_seconds?: number | null;
   partition_kind: PartitionKind;
   order_dependent: OrderDependence;
   distinct_dependent: DistinctDependence;
@@ -368,6 +372,7 @@ export interface CatalogAttachRequest {
   options?: /** @record-batch */ Uint8Array | null;
   data_version_spec?: string | null;
   implementation_version?: string | null;
+  client_capabilities?: Uint8Array | null;
 }
 
 export interface CatalogAttachResult {
@@ -483,6 +488,8 @@ export interface InitRequest {
   projection_ids?: number[] | null;
   pushdown_filters?: /** @record-batch */ Uint8Array | null;
   join_keys?: /** @record-batch */ Uint8Array[] | null;
+  split_tokens?: Uint8Array[] | null;
+  row_limit?: number | null;
   phase?: TableInOutFunctionInitPhase | null;
   finalize_state_id?: Uint8Array | null;
   execution_id?: Uint8Array | null;
@@ -555,6 +562,61 @@ export interface TableFunctionDynamicToStringResponse {
   values: string[];
 }
 
+export interface TableFunctionPlanRequest {
+  bind_call: BindRequest;
+  bind_opaque_data?: Uint8Array | null;
+  projection_ids?: number[] | null;
+  pushdown_filters?: /** @record-batch */ Uint8Array | null;
+  join_keys?: /** @record-batch */ Uint8Array[] | null;
+  row_limit?: number | null;
+  target_split_bytes?: number | null;
+  min_splits?: number | null;
+  max_splits_per_response?: number | null;
+  cursor?: Uint8Array | null;
+  refined_filters?: /** @record-batch */ Uint8Array | null;
+  filters_complete: boolean;
+  start_position?: Uint8Array | null;
+  end_position?: Uint8Array | null;
+  order_by_column_name?: string | null;
+  order_by_direction?: OrderByDirection | null;
+  order_by_null_order?: OrderByNullOrder | null;
+  order_by_limit?: number | null;
+  tablesample_percentage?: number | null;
+  tablesample_seed?: number | null;
+}
+
+export interface ScanSplit {
+  payload: Uint8Array;
+  token: Uint8Array;
+  estimated_rows?: number | null;
+  rows_exact: boolean;
+  estimated_bytes?: number | null;
+  partition_bounds?: /** @record-batch */ Uint8Array | null;
+  column_statistics?: Uint8Array | null;
+  location_ids?: number[] | null;
+  start_position?: Uint8Array | null;
+  end_position?: Uint8Array | null;
+}
+
+export interface PlanResponse {
+  splits: ScanSplit[] | Uint8Array[];
+  next_cursors?: Uint8Array[] | null;
+  execution_id?: Uint8Array | null;
+  init_opaque_data?: Uint8Array | null;
+  max_workers?: number | null;
+  estimated_total_splits?: number | null;
+  estimated_total_rows?: number | null;
+  estimated_total_bytes?: number | null;
+  catalog_version?: number | null;
+  scope: string;
+  locations?: string[] | null;
+  partitioning?: Uint8Array[] | null;
+  sort_order?: Uint8Array[] | null;
+  cache_max_age_seconds?: number | null;
+  start_position?: Uint8Array | null;
+  end_position?: Uint8Array | null;
+}
+
 export interface TableFunctionStatisticsRequest {
   bind_call: BindRequest;
   bind_opaque_data?: Uint8Array | null;