@query-farm/vgi 0.27.0 → 0.29.0

package/dist/functions/table.d.ts

@@ -1,5 +1,6 @@
 import { type VgiSchema, type VgiDataType } from "../arrow/index.js";
 import type { OutputCollector } from "@query-farm/vgi-rpc";
+import type { PlanResult, TableFunctionPlanRequest } from "../protocol/serializers/splits.js";
 import type { BindRequest, InitRequest, GlobalInitResponse, TableCardinality } from "../protocol/types.js";
 import type { VgiFunction, FunctionExample } from "./types.js";
 import { type ArgumentConstraints } from "../arguments/argument-spec.js";
@@ -22,6 +23,15 @@ export interface TableProcessParams<TArgs = Record<string, any>> {
     settings: Record<string, any>;
     secrets: Record<string, Record<string, any>>;
     pushdownFilters?: PushdownFilters;
+    /**
+     * The verified payloads of the splits this init claimed, in the order given,
+     * or undefined when this is not a split scan.
+     *
+     * Undefined is meaningfully different from an empty array: undefined means
+     * the client did not plan (a split-only function should fail loudly rather
+     * than answer the same query differently), empty means a claim of no work.
+     */
+    splitPayloads?: Uint8Array[];
     storage?: BoundStorage;
     /**
      * AT (TIMESTAMP|VERSION) clause for this scan, or `undefined` when the scan
@@ -91,6 +101,40 @@ export interface TableFunctionConfig<TArgs = Record<string, any>, TState = null>
     process: (params: TableProcessParams<TArgs>, state: TState, out: OutputCollector) => void | Promise<void>;
     /** Cardinality hints */
     cardinality?: (params: TableBindParams<TArgs>) => TableCardinality | Promise<TableCardinality>;
+    /**
+     * Divide this scan into named, independently redeemable splits.
+     *
+     * Declaring it (together with `meta.supportsSplits`) is what opts a function
+     * into the split path; omitting it means the whole scan is one unit of work.
+     *
+     * A split *names* work rather than describing it. "These three files at
+     * version 47" survives a retry; "rows 0-999 of whatever this returns now"
+     * does not — and a distributed engine WILL retry, so the difference is
+     * correctness, not tidiness. The same split may also be redeemed more than
+     * once (recursive CTEs, re-collected DataFrames, task retry) and may be
+     * abandoned mid-stream (LIMIT, TopK, an empty join build side); neither is an
+     * error.
+     *
+     * Set only `payload` on each split. The framework stamps the consistency
+     * anchor, the bind fingerprint and (where a key exists) the seal.
+     *
+     * Size splits into comparable units of work and honour `targetSplitBytes`: a
+     * claiming client treats them as interchangeable because it cannot see
+     * per-split cost, so wildly uneven splits leave its makespan bounded by the
+     * largest one.
+     */
+    plan?: (params: TableBindParams<TArgs>, request: TableFunctionPlanRequest) => PlanResult | Promise<PlanResult>;
+    /**
+     * Called on a split init with the VERIFIED payloads for the splits this
+     * connection claimed — the envelope is already opened and stripped, so
+     * unverified bytes never reach here.
+     *
+     * Any state carried from planning to reading must live in cross-process
+     * storage keyed by `execution_id`: the process that plans is, in the general
+     * case, not the process that reads — and under a distributed engine it is not
+     * even the same host.
+     */
+    onSplit?: (payloads: Uint8Array[], params: TableBindParams<TArgs>) => void | Promise<void>;
     /**
      * Per-column statistics for the function's output. Returned to DuckDB via
      * the `table_function_statistics` RPC; the optimizer uses min/max to
@@ -134,5 +178,24 @@ export interface TableFunctionConfig<TArgs = Record<string, any>, TState = null>
     supportsBatchIndex?: boolean;
     /** Hive-style partition-columns mode; FunctionInfo.partition_kind. */
     partitionKind?: "NOT_PARTITIONED" | "SINGLE_VALUE_PARTITIONS" | "OVERLAPPING_PARTITIONS" | "DISJOINT_PARTITIONS";
+    /**
+     * This function divides its scan into named, independently redeemable splits;
+     * FunctionInfo.supports_splits. Declare it together with `plan` and `onSplit`.
+     *
+     * It is what a distributed engine reads to decide it can retry a task against
+     * this function, and what makes the client call `plan()` at all — without it a
+     * split-capable function is scanned as one undivided unit, silently.
+     */
+    supportsSplits?: boolean;
+    /**
+     * How long a minted split token stays redeemable, or undefined for UNBOUNDED
+     * (NOT "expires immediately"); FunctionInfo.split_token_ttl_seconds.
+     *
+     * A client refuses a plan whose TTL is below its own scheduling horizon,
+     * because an expired token is a failed query rather than a degradation:
+     * nothing re-plans when one expires, since a distributed engine retries the
+     * serialized task it was handed and has no path back to the planner.
+     */
+    splitTokenTtlSeconds?: number;
 }
 export declare function defineTableFunction<TArgs = Record<string, any>, TState = null>(config: TableFunctionConfig<TArgs, TState>): VgiFunction;

package/dist/functions/types.d.ts

@@ -82,6 +82,38 @@ export interface FunctionMeta {
      * `supports_batch_index`.
      */
     supportsBatchIndex?: boolean;
+    /**
+     * table (generator): the scan divides into named, independently redeemable
+     * splits (see the `plan` / `onSplit` hooks). Surfaces as FunctionInfo
+     * `supports_splits`.
+     *
+     * The declaration is what a distributed engine reads to decide whether it can
+     * retry a task: a split NAMES its work, so re-running one reads exactly the
+     * same rows. Without it, the division lives behind an opaque worker-side queue
+     * and a retry reads neither what it read before nor what it missed.
+     */
+    supportsSplits?: boolean;
+    /**
+     * table (generator): the worker applies every pushed filter EXACTLY, so the
+     * engine may drop its own copy rather than re-filtering. Wrong answers if
+     * declared falsely — leave unset unless certain.
+     */
+    filtersExactlyApplied?: boolean;
+    /**
+     * table (generator): addressable positions in the data, for incremental and
+     * streaming reads. The engine owns the checkpoint; this declares only that
+     * positions are meaningful to externalize.
+     */
+    supportsPositions?: boolean;
+    /**
+     * table (generator): how long a split token stays redeemable.
+     *
+     * UNSET MEANS UNBOUNDED, not "expires immediately" — a client must not assume
+     * a TTL exists, or long-running streams are foreclosed. A client rejects a
+     * plan whose TTL is below its own planning-to-task-start horizon, which on a
+     * busy cluster is hours rather than seconds.
+     */
+    splitTokenTtlSeconds?: number;
     /**
      * table (generator): Hive-style partition-columns mode. Functions declare
      * a PartitionKind and annotate bind-schema fields; emitted batches carry
@@ -146,6 +178,12 @@ export interface VgiFunction {
     globalInit(request: InitRequest): GlobalInitResponse | Promise<GlobalInitResponse>;
     createStreamHandlers(request: InitRequest, response: GlobalInitResponse, accumulatedState?: any): StreamHandlers;
     cardinality?(request: TableFunctionCardinalityRequest): TableCardinality | Promise<TableCardinality>;
+    /**
+     * Divide a scan into named, independently redeemable splits. Absent means the
+     * function is not split-capable: the whole scan is one unit of work, and the
+     * framework answers with a single empty-payload split.
+     */
+    plan?(request: import("../protocol/serializers/splits.js").TableFunctionPlanRequest): import("../protocol/serializers/splits.js").PlanResult | Promise<import("../protocol/serializers/splits.js").PlanResult>;
     /**
      * Per-column statistics for this table function's output given the user's
      * bind-time arguments. Returns null/[] when stats are unknown. Wired to the

package/dist/generated/vgi-client.d.ts

@@ -99,6 +99,10 @@ export interface FunctionInfo {
     order_preservation?: OrderPreservation | null;
     max_workers?: number | null;
     supports_batch_index: boolean;
+    supports_splits: boolean;
+    filters_exactly_applied: boolean;
+    supports_positions: boolean;
+    split_token_ttl_seconds?: number | null;
     partition_kind: PartitionKind;
     order_dependent: OrderDependence;
     distinct_dependent: DistinctDependence;
@@ -307,6 +311,7 @@ export interface CatalogAttachRequest {
     options?: /** @record-batch */ Uint8Array | null;
     data_version_spec?: string | null;
     implementation_version?: string | null;
+    client_capabilities?: Uint8Array | null;
 }
 export interface CatalogAttachResult {
     attach_opaque_data: Uint8Array;
@@ -406,6 +411,8 @@ export interface InitRequest {
     projection_ids?: number[] | null;
     pushdown_filters?: /** @record-batch */ Uint8Array | null;
     join_keys?: /** @record-batch */ Uint8Array[] | null;
+    split_tokens?: Uint8Array[] | null;
+    row_limit?: number | null;
     phase?: TableInOutFunctionInitPhase | null;
     finalize_state_id?: Uint8Array | null;
     execution_id?: Uint8Array | null;
@@ -467,6 +474,58 @@ export interface TableFunctionDynamicToStringResponse {
     keys: string[];
     values: string[];
 }
+export interface TableFunctionPlanRequest {
+    bind_call: BindRequest;
+    bind_opaque_data?: Uint8Array | null;
+    projection_ids?: number[] | null;
+    pushdown_filters?: /** @record-batch */ Uint8Array | null;
+    join_keys?: /** @record-batch */ Uint8Array[] | null;
+    row_limit?: number | null;
+    target_split_bytes?: number | null;
+    min_splits?: number | null;
+    max_splits_per_response?: number | null;
+    cursor?: Uint8Array | null;
+    refined_filters?: /** @record-batch */ Uint8Array | null;
+    filters_complete: boolean;
+    start_position?: Uint8Array | null;
+    end_position?: Uint8Array | null;
+    order_by_column_name?: string | null;
+    order_by_direction?: OrderByDirection | null;
+    order_by_null_order?: OrderByNullOrder | null;
+    order_by_limit?: number | null;
+    tablesample_percentage?: number | null;
+    tablesample_seed?: number | null;
+}
+export interface ScanSplit {
+    payload: Uint8Array;
+    token: Uint8Array;
+    estimated_rows?: number | null;
+    rows_exact: boolean;
+    estimated_bytes?: number | null;
+    partition_bounds?: /** @record-batch */ Uint8Array | null;
+    column_statistics?: Uint8Array | null;
+    location_ids?: number[] | null;
+    start_position?: Uint8Array | null;
+    end_position?: Uint8Array | null;
+}
+export interface PlanResponse {
+    splits: ScanSplit[] | Uint8Array[];
+    next_cursors?: Uint8Array[] | null;
+    execution_id?: Uint8Array | null;
+    init_opaque_data?: Uint8Array | null;
+    max_workers?: number | null;
+    estimated_total_splits?: number | null;
+    estimated_total_rows?: number | null;
+    estimated_total_bytes?: number | null;
+    catalog_version?: number | null;
+    scope: string;
+    locations?: string[] | null;
+    partitioning?: Uint8Array[] | null;
+    sort_order?: Uint8Array[] | null;
+    cache_max_age_seconds?: number | null;
+    start_position?: Uint8Array | null;
+    end_position?: Uint8Array | null;
+}
 export interface TableFunctionStatisticsRequest {
     bind_call: BindRequest;
     bind_opaque_data?: Uint8Array | null;

package/dist/generated/vgi-protocol-schemas.d.ts

@@ -9,7 +9,24 @@ export declare const CopyFromFormatInfoSchema: import("../index.core.js").VgiSch
 export declare const ScanFunctionResultSchema: import("../index.core.js").VgiSchema;
 export declare const ScanBranchesResultSchema: import("../index.core.js").VgiSchema;
 export declare const ScanBranchSchema: import("../index.core.js").VgiSchema;
+export declare const ScanSplitSchema: import("../index.core.js").VgiSchema;
 export declare const AttachCatalogInfoSchema: import("../index.core.js").VgiSchema;
+export declare const AggregateBindRequestSchema: import("../index.core.js").VgiSchema;
+export declare const AggregateCombineRequestSchema: import("../index.core.js").VgiSchema;
+export declare const AggregateDestructorRequestSchema: import("../index.core.js").VgiSchema;
+export declare const AggregateFinalizeRequestSchema: import("../index.core.js").VgiSchema;
+export declare const AggregateUpdateRequestSchema: import("../index.core.js").VgiSchema;
+export declare const BindRequestSchema: import("../index.core.js").VgiSchema;
+export declare const CatalogAttachRequestSchema: import("../index.core.js").VgiSchema;
+export declare const CopyFromContextSchema: import("../index.core.js").VgiSchema;
+export declare const CopyToContextSchema: import("../index.core.js").VgiSchema;
+export declare const GlobalInitResponseSchema: import("../index.core.js").VgiSchema;
+export declare const InitRequestSchema: import("../index.core.js").VgiSchema;
+export declare const TableBufferingCombineRequestSchema: import("../index.core.js").VgiSchema;
+export declare const TableBufferingDestructorRequestSchema: import("../index.core.js").VgiSchema;
+export declare const TableBufferingProcessRequestSchema: import("../index.core.js").VgiSchema;
+export declare const TableFunctionCardinalityRequestSchema: import("../index.core.js").VgiSchema;
+export declare const TableFunctionPlanRequestSchema: import("../index.core.js").VgiSchema;
 export declare const AggregateBindResultSchema: import("../index.core.js").VgiSchema;
 export declare const AggregateCombineResultSchema: import("../index.core.js").VgiSchema;
 export declare const AggregateDestructorResultSchema: import("../index.core.js").VgiSchema;
@@ -44,6 +61,7 @@ export declare const TableBufferingDestructorResultSchema: import("../index.core
 export declare const TableBufferingProcessResultSchema: import("../index.core.js").VgiSchema;
 export declare const TableFunctionCardinalityResultSchema: import("../index.core.js").VgiSchema;
 export declare const TableFunctionDynamicToStringResultSchema: import("../index.core.js").VgiSchema;
+export declare const TableFunctionPlanResultSchema: import("../index.core.js").VgiSchema;
 export declare const AggregateBindParamsSchema: import("../index.core.js").VgiSchema;
 export declare const AggregateCombineParamsSchema: import("../index.core.js").VgiSchema;
 export declare const AggregateDestructorParamsSchema: import("../index.core.js").VgiSchema;
@@ -113,4 +131,5 @@ export declare const TableBufferingDestructorParamsSchema: import("../index.core
 export declare const TableBufferingProcessParamsSchema: import("../index.core.js").VgiSchema;
 export declare const TableFunctionCardinalityParamsSchema: import("../index.core.js").VgiSchema;
 export declare const TableFunctionDynamicToStringParamsSchema: import("../index.core.js").VgiSchema;
+export declare const TableFunctionPlanParamsSchema: import("../index.core.js").VgiSchema;
 export declare const TableFunctionStatisticsParamsSchema: import("../index.core.js").VgiSchema;

package/dist/http/fetch.d.ts

@@ -1,3 +1,4 @@
+import { type AuthenticateFn } from "@query-farm/vgi-rpc";
 import { type ProtocolConfig } from "../protocol/dispatch.js";
 import { type LandingInfo } from "./landing.js";
 export interface VgiFetchOptions {
@@ -42,6 +43,15 @@ export interface VgiFetchOptions {
      *  its required name/doc/version; the CF entry had to remember, and
      *  vgi-open-meteo shipped for weeks without it. */
     landingInfo: LandingInfo;
+    /** Authenticates each incoming request, returning the caller's
+     *  {@link AuthContext}. Forwarded verbatim to `createHttpHandler`; omit it
+     *  and every caller is anonymous, which is what this package did before the
+     *  option existed.
+     *
+     *  Supplying it is what makes a worker able to serve more than one principal,
+     *  and the identity it returns is the one the worker seals `attach_opaque_data`
+     *  and split tokens under. */
+    authenticate?: AuthenticateFn;
 }
 /**
  * Build a fetch handler suitable for `export default { fetch }` in a CF

package/dist/index.core.d.ts

@@ -7,7 +7,7 @@ export { defineTableBufferingFunction, type TableBufferingConfig, type TableBuff
 export { defineCopyFromFunction, type CopyFromFunctionConfig, type CopyFromOption, type CopyFromReadParams, } from "./functions/copy-from.js";
 export { defineCopyToFunction, type CopyToFunctionConfig, type CopyToOption, type CopyToWriteParams, type CopyToCloseParams, } from "./functions/copy-to.js";
 export type { VgiFunction, FunctionMeta, StreamHandlers, FunctionExample, HandlerState } from "./functions/types.js";
-export { CatalogInterface, type CatalogAttachResult, type SchemaInfo, encodeSchemaInfo, decodeSchemaInfo, type TableInfo, encodeTableInfo, decodeTableInfo, type ViewInfo, encodeViewInfo, decodeViewInfo, type CatalogInfo, encodeCatalogInfo, decodeCatalogInfo, type FunctionInfo, encodeFunctionInfo, decodeFunctionInfo, type FunctionInfoOptions, type MacroInfo, encodeMacroInfo, decodeMacroInfo, type MacroType, type CopyFromFormatInfo, encodeCopyFromFormatInfo, type AttachOpaqueData, type TransactionOpaqueData, buildScanBranchesResult, type ScanBranchInput } from "./catalog/interface.js";
+export { CatalogInterface, type CatalogAttachResult, type SchemaInfo, encodeSchemaInfo, decodeSchemaInfo, type TableInfo, encodeTableInfo, decodeTableInfo, type ViewInfo, encodeViewInfo, decodeViewInfo, type CatalogInfo, encodeCatalogInfo, decodeCatalogInfo, type FunctionInfo, encodeFunctionInfo, decodeFunctionInfo, type FunctionInfoOptions, type MacroInfo, encodeMacroInfo, decodeMacroInfo, type MacroType, type CopyFromFormatInfo, encodeCopyFromFormatInfo, type AttachOpaqueData, type TransactionOpaqueData, buildScanBranchesResult, encodeFormatOptions, type ScanBranchInput } from "./catalog/interface.js";
 export type { CatalogDescriptor, SchemaDescriptor, TableDescriptor, ViewDescriptor, MacroDescriptor, SettingDescriptor, SecretTypeDescriptor, ForeignKeyDef, DefaultValue } from "./catalog/descriptors.js";
 export { ReadOnlyCatalogInterface } from "./catalog/read-only.js";
 export { CompositeCatalogInterface } from "./catalog/composite.js";
@@ -33,7 +33,7 @@ export { toUint8Array } from "./util/bytes.js";
 export { iterRows, batchFromRows, batchFromColumns, emptyBatch, filterBatch, projectSchema, projectBatch, batchToScalarDict, batchToSecretDict, safeNumber, } from "./util/arrow/index.js";
 export { type SecretFields, type SecretsDict, secretType, secretsOfType, secretForScope, secretForScopeOfType, } from "./secrets/helpers.js";
 export { serializeSchema, deserializeSchema, serializeBatch, deserializeBatch, } from "./util/arrow/index.js";
-export { schema, field, nullType, int8, int16, int64, uint8, uint16, uint32, uint64, float16, float64, utf8, binary, fixedSizeBinary, decimal, decimal128, decimal256, date, dateDay, dateMillisecond, time, timeSecond, timeMillisecond, timeMicrosecond, timeNanosecond, timestamp, duration, interval, timestampSeconds, timestampMillis, timestampMicros, timestampNanos, durationSeconds, durationMillis, durationMicros, durationNanos, list, fixedSizeList, struct, map, dictionary, union, sparseUnion, denseUnion, TimeUnit, DateUnit, IntervalUnit, UnionMode, codecFor, asDate32, asDate64Ms, asTime32S, asTime32Ms, asTime64Us, asTime64Ns, asTimestampSeconds, asTimestampMillis, asTimestampMicros, asTimestampNanos, asDurationSeconds, asDurationMillis, asDurationMicros, asDurationNanos, asUnscaledDecimal, asInt64, asUint64, fromBranded, fromBrandedNumber, fromBrandedBigInt, TypeId, isNull, isInt, isFloat, isBinary, isUtf8, isBool, isDecimal, isDate, isTime, isTimestamp, isInterval, isList, isStruct, isUnion, isFixedSizeBinary, isFixedSizeList, isMap, isDuration, isDictionary, isBatch, typeSignature, readCanonicalValue, columnFromArray, decodeDictValue, backend, adoptArrowJsShape, } from "./arrow/index.js";
+export { schema, field, nullType, int8, int16, int64, uint8, uint16, uint32, uint64, float16, float64, utf8, largeUtf8, binary, largeBinary, fixedSizeBinary, decimal, decimal128, decimal256, date, dateDay, dateMillisecond, time, timeSecond, timeMillisecond, timeMicrosecond, timeNanosecond, timestamp, duration, interval, timestampSeconds, timestampMillis, timestampMicros, timestampNanos, durationSeconds, durationMillis, durationMicros, durationNanos, list, fixedSizeList, struct, map, dictionary, union, sparseUnion, denseUnion, TimeUnit, DateUnit, IntervalUnit, UnionMode, codecFor, asDate32, asDate64Ms, asTime32S, asTime32Ms, asTime64Us, asTime64Ns, asTimestampSeconds, asTimestampMillis, asTimestampMicros, asTimestampNanos, asDurationSeconds, asDurationMillis, asDurationMicros, asDurationNanos, asUnscaledDecimal, asInt64, asUint64, fromBranded, fromBrandedNumber, fromBrandedBigInt, TypeId, isNull, isInt, isFloat, isBinary, isUtf8, isBool, isDecimal, isDate, isTime, isTimestamp, isInterval, isList, isStruct, isUnion, isFixedSizeBinary, isFixedSizeList, isMap, isDuration, isDictionary, isBatch, typeSignature, readCanonicalValue, columnFromArray, decodeDictValue, backend, adoptArrowJsShape, } from "./arrow/index.js";
 export type { VgiTypeId, VgiDataType, VgiField, VgiSchema, VgiColumn, VgiColumnData, VgiBatch, VgiBackendInfo, TaggedUnion, NullDescriptor, BoolType, IntType, Int8Type, Int16Type, Int32Type, Int64Type, Uint8Type, Uint16Type, Uint32Type, Uint64Type, FloatType, Utf8Type, LargeUtf8Type, BinaryType, LargeBinaryType, FixedSizeBinaryType, DecimalType, Date32Type, Date64Type, Time32Type, Time64Type, TimestampType, DurationType, IntervalType, ListType, FixedSizeListType, StructType, MapType, DictionaryType, UnionType, TypedField, TUnit, RichValue, RawValue, ValueFor, Repr, Codec, Branded, Date32, Date64Ms, Time32S, Time32Ms, Time64Us, Time64Ns, TimestampSeconds, TimestampMillis, TimestampMicros, TimestampNanos, DurationSeconds, DurationMillis, DurationMicros, DurationNanos, UnscaledDecimal, Int64Raw, Uint64Raw, } from "./arrow/index.js";
 export type { ScalarComputeRow, ScalarOutputValue, ScalarComputeResult } from "./functions/scalar.js";
 export { type FunctionStorage, FunctionStorageSqlite, BoundStorage, UnknownInvocationError, resolveStorageFromEnv, setStorage, storage as functionStorage, } from "./functions/storage.js";