@query-farm/vgi 0.27.0 → 0.29.0

package/README.md

@@ -387,13 +387,32 @@ For manual conversions outside a function, `codecFor(type)` returns the codec wi
 
 ### Factory name note
 
-The typed Arrow type factories `int`, `int32`, `float32`, and `bool` are **not**
-re-exported from the package root, because `@query-farm/vgi` already re-exports
-vgi-rpc argument builders of the same names. Import those four from the arrow facade
-(they ship as the typed factories there); the rest of the typed factory set
-(`int8`/`int16`/`int64`, `uint*`, `float16`/`float64`, `decimal*`, `dateDay`,
-`timestampMicros`, `struct`, `list`, `map`, …) is exported from the package root as
-usual.
+Four names — `int`, `int32`, `float32`, `bool` — mean different things depending on
+which entry point you import from:
+
+| Import from | `int` is | Write |
+| --- | --- | --- |
+| `@query-farm/vgi` | a ready-made `Int64` **type**, re-exported from vgi-rpc | `params: { n: int }` |
+| `@query-farm/vgi/worker-cf` | the typed **factory** `int(bitWidth?, signed?)` | `params: { n: int() }` |
+
+The package root also re-exports `float`, `str` and `bytes` as ready-made types;
+the Cloudflare entry does not have them at all (use `float64()`, `utf8()`,
+`binary()`).
+
+Everything else in the typed factory set — `int8`/`int16`/`int64`, `uint*`,
+`float16`/`float64`, `utf8`, `decimal*`, `dateDay`, `timestampMicros`, `struct`,
+`list`, `map`, … — is a **factory on both entry points** and is exported from the
+package root as usual.
+
+**Prefer the explicit factories in portable code.** `int64()` means the same thing
+everywhere; `int` does not, so moving a worker to Cloudflare Workers can turn a
+correct declaration into a wrong one with no other edit. Since 0.29.0 that mistake
+is caught at definition time rather than silently registering a `Function` as an
+argument type:
+
+```
+defineScalarFunction("double"): params.n is a type factory, not an Arrow type — call it: int64().
+```
 
 ## Migration: the type-handling break
 

package/dist/arguments/argument-spec.d.ts

@@ -58,6 +58,59 @@ export declare function formatRange(ge?: number, le?: number, gt?: number, lt?:
  * no field). Mirrors Python's `_constraint_kwargs`.
  */
 export declare function constraintSpecFields(constraints: ArgumentConstraints | undefined): Pick<ArgumentSpec, "defaultJson" | "choicesJson" | "rangeNotation" | "pattern">;
+/**
+ * Narrow a bind-time argument value from Arrow's bigint to a plain number.
+ *
+ * DuckDB types an integer literal as BIGINT, so `series(3)` arrives as `3n`
+ * whether or not anyone wanted 64 bits. Nobody writes `BigInt(1000)` for a page
+ * size, so the arg paths narrow — and they now all narrow the same way, which
+ * they did not before (two sites used safeNumber, two a bare Number).
+ *
+ * This is lossy above 2^53, and that is a deliberate call rather than an
+ * oversight. Every int64 *argument* across the fixtures and every shipped
+ * worker is a small control value — count, page_size, batch_size, increment,
+ * lag_minutes, restatement_lookback_days. The large int64s in the test suite
+ * are all somewhere else entirely:
+ *
+ *   - column data (copy_to/types.test, copy_to/parallel.test) — never touches
+ *     this path;
+ *   - filter-pushdown constants (filter_pushdown/integers.test pins 2^53+1 on
+ *     purpose) — deserialize.ts keeps those bigint, with its own regression
+ *     test;
+ *   - varargs values (constant_columns) — read raw off bindCall.arguments,
+ *     because extractArgs' copy of them is not what the function reads;
+ *   - time-travel versions — carried as strings (`at_value`).
+ *
+ * Preserving the bigint on overflow was tried and reverted: it buys nothing for
+ * any argument that exists, and costs a JS type that depends on magnitude, so a
+ * worker doing `args.n + 1` would work in every test and fail on one caller's
+ * input. If an argument ever genuinely needs the full range — a snowflake id, a
+ * nanosecond epoch — the fix is to make int64 arguments bigint by declared
+ * type, not by magnitude.
+ */
+export declare function narrowArgValue(value: unknown): unknown;
+/**
+ * Reject a type *factory* where an Arrow *type* was expected.
+ *
+ * `params: { n: int64 }` (no parentheses) is accepted by the type system —
+ * `VgiDataType` is loose enough — and then silently produces an ArgumentSpec
+ * whose `arrowType` is a Function. Nothing complains until the schema is
+ * serialized or the bind arrives, and the error at that point names neither the
+ * argument nor the cause.
+ *
+ * The mistake is easy to make on purpose, too: the same identifier is a
+ * ready-made type on one entry point and a factory on another.
+ *
+ *   @query-farm/vgi            int, int32, float32, bool  →  type instances
+ *   @query-farm/vgi/worker-cf  int, int32, float32, bool  →  factories
+ *
+ * So moving a worker to Cloudflare can turn a correct declaration into a
+ * silently wrong one with no other edit. Failing here, at definition time,
+ * makes that a startup error naming the field.
+ */
+export declare function assertArrowType(value: unknown, where: string): void;
+/** {@link assertArrowType} over a record of declared types, e.g. `params`. */
+export declare function assertArrowTypes(types: Record<string, unknown> | undefined, where: string): void;
 /**
  * Enforce a bind-time const value against its declared {@link ArgumentConstraints},
  * throwing {@link ArgumentValidationError} on violation. Mirrors the Python SDK's

package/dist/arrow/impl-arrowjs/index.d.ts

@@ -1,6 +1,6 @@
 import type { VgiBackendInfo } from "../types.js";
 export declare const backend: VgiBackendInfo;
-export { schema, field, nullType, bool, int, int8, int16, int32, int64, uint8, uint16, uint32, uint64, float16, float32, float64, utf8, binary, fixedSizeBinary, decimal, decimal128, decimal256, date, dateDay, dateMillisecond, time, timeSecond, timeMillisecond, timeMicrosecond, timeNanosecond, timestamp, duration, interval, list, fixedSizeList, struct, map, dictionary, union, sparseUnion, denseUnion, TimeUnit, DateUnit, IntervalUnit, UnionMode, } from "./schema.js";
+export { schema, field, nullType, bool, int, int8, int16, int32, int64, uint8, uint16, uint32, uint64, float16, float32, float64, utf8, largeUtf8, binary, largeBinary, fixedSizeBinary, decimal, decimal128, decimal256, date, dateDay, dateMillisecond, time, timeSecond, timeMillisecond, timeMicrosecond, timeNanosecond, timestamp, duration, interval, list, fixedSizeList, struct, map, dictionary, union, sparseUnion, denseUnion, TimeUnit, DateUnit, IntervalUnit, UnionMode, } from "./schema.js";
 export { emptyBatch } from "./empty.js";
 export { batchFromRows, batchFromColumns, columnFromArray } from "./build.js";
 export { iterRows, batchToScalarDict, batchToSecretDict, safeNumber, decodeDictValue, } from "./iterate.js";

package/dist/arrow/impl-arrowjs/iterate.d.ts

@@ -34,6 +34,15 @@ export declare function decodeDictValue(value: any, index?: number): any;
  */
 export declare function batchToSecretDict(batch: RecordBatch | VgiBatch | null): Record<string, Record<string, any>>;
 /**
- * Safe number coercion for BigInt values.
+ * Narrow a bigint to a number, refusing to do it lossily.
+ *
+ * Arrow int64/uint64 values arrive as bigint. Several bind-time argument paths
+ * want a plain number, and used to get one from a bare `Number(value)` — which
+ * silently rounds above 2^53, so an id or a nanosecond timestamp passed as a
+ * function argument came back subtly wrong with nothing logged anywhere.
+ *
+ * Throwing is the lesser evil: a caller that genuinely wants a lossy narrowing
+ * can still write `Number(v)` themselves, but nobody has to discover the loss
+ * from mismatched output weeks later.
  */
 export declare function safeNumber(value: any): number;

package/dist/arrow/impl-arrowjs/schema.d.ts

@@ -36,7 +36,9 @@ export declare const float16: () => VgiDataType;
 export declare const float32: () => VgiDataType;
 export declare const float64: () => VgiDataType;
 export declare const utf8: () => VgiDataType;
+export declare const largeUtf8: () => VgiDataType;
 export declare const binary: () => VgiDataType;
+export declare const largeBinary: () => VgiDataType;
 export declare const fixedSizeBinary: (byteWidth: number) => VgiDataType;
 export declare function decimal(precision: number, scale: number, bitWidth?: 32 | 64 | 128 | 256): VgiDataType;
 export declare const decimal128: (precision: number, scale: number) => VgiDataType;

package/dist/arrow/impl-flechette/index.d.ts

@@ -1,7 +1,7 @@
 import type { VgiBackendInfo } from "../types.js";
 import "./compat.js";
 export declare const backend: VgiBackendInfo;
-export { schema, field, nullType, bool, int, int8, int16, int32, int64, uint8, uint16, uint32, uint64, float16, float32, float64, utf8, binary, fixedSizeBinary, decimal, decimal128, decimal256, date, dateDay, dateMillisecond, time, timeSecond, timeMillisecond, timeMicrosecond, timeNanosecond, timestamp, duration, interval, list, fixedSizeList, struct, map, dictionary, union, sparseUnion, denseUnion, TimeUnit, DateUnit, IntervalUnit, UnionMode, } from "./schema.js";
+export { schema, field, nullType, bool, int, int8, int16, int32, int64, uint8, uint16, uint32, uint64, float16, float32, float64, utf8, largeUtf8, binary, largeBinary, fixedSizeBinary, decimal, decimal128, decimal256, date, dateDay, dateMillisecond, time, timeSecond, timeMillisecond, timeMicrosecond, timeNanosecond, timestamp, duration, interval, list, fixedSizeList, struct, map, dictionary, union, sparseUnion, denseUnion, TimeUnit, DateUnit, IntervalUnit, UnionMode, } from "./schema.js";
 export { emptyBatch } from "./empty.js";
 export { batchFromRows, batchFromColumns, columnFromArray } from "./build.js";
 export { iterRows, batchToScalarDict, batchToSecretDict, safeNumber, decodeDictValue, } from "./iterate.js";

package/dist/arrow/impl-flechette/iterate.d.ts

@@ -24,6 +24,15 @@ export declare function decodeDictValue(value: any, _index?: number): any;
  */
 export declare function batchToSecretDict(batch: VgiBatch | null): Record<string, Record<string, any>>;
 /**
- * BigInt → number coercion (lossy for values past 2^53).
+ * Narrow a bigint to a number, refusing to do it lossily.
+ *
+ * Arrow int64/uint64 values arrive as bigint. Several bind-time argument paths
+ * want a plain number, and used to get one from a bare `Number(value)` — which
+ * silently rounds above 2^53, so an id or a nanosecond timestamp passed as a
+ * function argument came back subtly wrong with nothing logged anywhere.
+ *
+ * Throwing is the lesser evil: a caller that genuinely wants a lossy narrowing
+ * can still write `Number(v)` themselves, but nobody has to discover the loss
+ * from mismatched output weeks later.
  */
 export declare function safeNumber(value: any): number;

package/dist/arrow/impl-flechette/schema.d.ts

@@ -21,7 +21,9 @@ export declare const float16: () => VgiDataType;
 export declare const float32: () => VgiDataType;
 export declare const float64: () => VgiDataType;
 export declare const utf8: () => VgiDataType;
+export declare const largeUtf8: () => VgiDataType;
 export declare const binary: () => VgiDataType;
+export declare const largeBinary: () => VgiDataType;
 export declare const fixedSizeBinary: (stride: number) => VgiDataType;
 export declare function decimal(precision: number, scale: number, bitWidth?: 32 | 64 | 128 | 256): VgiDataType;
 export declare const decimal128: (precision: number, scale: number) => VgiDataType;

package/dist/arrow/index.d.ts

@@ -2,7 +2,7 @@ export type { VgiTypeId, VgiDataType, VgiField, VgiSchema, VgiColumn, VgiColumnD
 export { TypeId, isNull, isInt, isFloat, isBinary, isUtf8, isBool, isDecimal, isDate, isTime, isTimestamp, isInterval, isList, isStruct, isUnion, isFixedSizeBinary, isFixedSizeList, isMap, isDuration, isDictionary, isBatch, typeSignature, } from "./predicates.js";
 export { backend, emptyBatch, batchFromRows, batchFromColumns, columnFromArray, iterRows, batchToScalarDict, batchToSecretDict, safeNumber, decodeDictValue, filterBatch, projectSchema, projectBatch, withBatchMetadata, adoptArrowJsShape, readCanonicalValue, serializeSchema, deserializeSchema, serializeBatch, deserializeBatch, buildStatisticsBatch, } from "#arrow-impl";
 export type { ColumnStatistics } from "#arrow-impl";
-export { schema, field, nullType, bool, int, int8, int16, int32, int64, uint8, uint16, uint32, uint64, float16, float32, float64, utf8, binary, fixedSizeBinary, decimal, decimal128, decimal256, date, dateDay, dateMillisecond, time, timeSecond, timeMillisecond, timeMicrosecond, timeNanosecond, timestamp, duration, interval, timestampSeconds, timestampMillis, timestampMicros, timestampNanos, durationSeconds, durationMillis, durationMicros, durationNanos, list, fixedSizeList, struct, map, dictionary, union, sparseUnion, denseUnion, TimeUnit, DateUnit, IntervalUnit, UnionMode, } from "./schema-types.js";
+export { schema, field, nullType, bool, int, int8, int16, int32, int64, uint8, uint16, uint32, uint64, float16, float32, float64, utf8, largeUtf8, binary, largeBinary, fixedSizeBinary, decimal, decimal128, decimal256, date, dateDay, dateMillisecond, time, timeSecond, timeMillisecond, timeMicrosecond, timeNanosecond, timestamp, duration, interval, timestampSeconds, timestampMillis, timestampMicros, timestampNanos, durationSeconds, durationMillis, durationMicros, durationNanos, list, fixedSizeList, struct, map, dictionary, union, sparseUnion, denseUnion, TimeUnit, DateUnit, IntervalUnit, UnionMode, } from "./schema-types.js";
 export type { NullDescriptor, BoolType, IntType, Int8Type, Int16Type, Int32Type, Int64Type, Uint8Type, Uint16Type, Uint32Type, Uint64Type, FloatType, Utf8Type, LargeUtf8Type, BinaryType, LargeBinaryType, FixedSizeBinaryType, DecimalType, Date32Type, Date64Type, Time32Type, Time64Type, TimestampType, DurationType, IntervalType, ListType, FixedSizeListType, StructType, MapType, DictionaryType, UnionType, TypedField, TUnit, } from "./codec/type-descriptors.js";
 export type { RichValue, RawValue, ValueFor, Repr } from "./codec/repr.js";
 export { codecFor } from "./codec/registry.js";

package/dist/arrow/schema-types.d.ts

@@ -1,6 +1,6 @@
 import * as impl from "#arrow-impl";
 import type { VgiField, VgiDataType } from "./types.js";
-import type { NullDescriptor, BoolType, Int8Type, Int16Type, Int32Type, Int64Type, Uint8Type, Uint16Type, Uint32Type, Uint64Type, IntType, FloatType, Utf8Type, BinaryType, FixedSizeBinaryType, DecimalType, Date32Type, Date64Type, Time32Type, Time64Type, TimestampType, DurationType, IntervalType, ListType, FixedSizeListType, StructType, MapType, DictionaryType, UnionType, TypedField, TUnit } from "./codec/type-descriptors.js";
+import type { NullDescriptor, BoolType, Int8Type, Int16Type, Int32Type, Int64Type, Uint8Type, Uint16Type, Uint32Type, Uint64Type, IntType, FloatType, Utf8Type, LargeUtf8Type, BinaryType, LargeBinaryType, FixedSizeBinaryType, DecimalType, Date32Type, Date64Type, Time32Type, Time64Type, TimestampType, DurationType, IntervalType, ListType, FixedSizeListType, StructType, MapType, DictionaryType, UnionType, TypedField, TUnit } from "./codec/type-descriptors.js";
 export declare const TimeUnit: {
     readonly SECOND: import("@query-farm/apache-arrow").TimeUnit.SECOND;
     readonly MILLISECOND: import("@query-farm/apache-arrow").TimeUnit.MILLISECOND;
@@ -35,7 +35,9 @@ export declare const float16: () => FloatType;
 export declare const float32: () => FloatType;
 export declare const float64: () => FloatType;
 export declare const utf8: () => Utf8Type;
+export declare const largeUtf8: () => LargeUtf8Type;
 export declare const binary: () => BinaryType;
+export declare const largeBinary: () => LargeBinaryType;
 export declare const fixedSizeBinary: (byteWidth: number) => FixedSizeBinaryType;
 export declare const decimal: (precision: number, scale: number, bitWidth?: 32 | 64 | 128 | 256) => DecimalType;
 export declare const decimal128: (precision: number, scale: number) => DecimalType;

package/dist/arrow/types.d.ts

@@ -20,9 +20,43 @@ export interface VgiSchema {
     readonly fields: readonly VgiField[];
     readonly metadata: Map<string, string>;
 }
-/** Column view. Generic dropped: arrow-js parameterizes on DataType, flechette
- *  on value type, so the facade exposes an erased shape. Callers cast at the
- *  use site (the column's static schema makes the value type known there). */
+/**
+ * Column view over one column of a batch.
+ *
+ * The value type is erased by default: arrow-js parameterizes on DataType and
+ * flechette on value type, so the facade cannot name one without picking a
+ * backend. Supply `T` at the use site — the column's declared type makes it
+ * known there — and the cast disappears:
+ *
+ * ```ts
+ * const ns = batch.getChildAt(0)! as Iterable<bigint | null>;
+ * for (const v of ns) { ... }        // v: bigint | null
+ * ```
+ *
+ * A type parameter here would read better, but it cannot be had cheaply: both
+ * backends' native batches are assigned to VgiBatch structurally, and a
+ * concrete `get(): unknown` does not satisfy a generic `get(): T`. Adding one
+ * would mean casting at ~25 internal sites to remove one cast in user code.
+ *
+ * **These are the backend's own values, not codec output.** `get()` and
+ * iteration return whatever the Arrow implementation stores, which for some
+ * types is not the value the SDK documents:
+ *
+ * | Arrow type       | here                          | {@link iterRows}      |
+ * | ---------------- | ----------------------------- | --------------------- |
+ * | `int64`          | `bigint`                      | `bigint`   (same)     |
+ * | `utf8`, `float64`| `string`, `number`            | same                  |
+ * | `decimal128`     | backend limbs (`DecimalBigNum`)| `bigint`, unscaled    |
+ * | `timestamp[us]`  | millisecond `number`          | microsecond `bigint`  |
+ * | `date32`         | millisecond `number`          | `Date`                |
+ *
+ * `repr: "raw"` on a function selects the *codec's* representation and has no
+ * effect on this path.
+ *
+ * So: reach for `getChildAt` on integer, float, boolean, string and binary
+ * columns, where it is the cheapest correct thing. For temporal, decimal and
+ * nested types go through {@link iterRows}, which runs the codec.
+ */
 export interface VgiColumn {
     readonly type: VgiDataType;
     readonly length: number;
@@ -32,7 +66,9 @@ export interface VgiColumn {
 export interface VgiBatch {
     readonly schema: VgiSchema;
     readonly numRows: number;
+    /** See {@link VgiColumn} — this does NOT run the codec. */
     getChild(name: string): VgiColumn | null;
+    /** See {@link VgiColumn} — this does NOT run the codec. */
     getChildAt(index: number): VgiColumn | null;
 }
 export interface VgiBackendInfo {

package/dist/catalog/interface.d.ts

@@ -96,6 +96,21 @@ export interface ScanFunctionResult {
     /** DuckDB extensions to load before calling the function. */
     requiredExtensions: string[];
 }
+/**
+ * Encode a ScanFunctionResult wire record — the single-row batch shape
+ * `{ function_name, arguments, required_extensions }` that
+ * `catalog_table_scan_function_get` returns and that a function-backed table
+ * inlines into `TableInfo.scan_function`.
+ *
+ * It exists so there is exactly one place that names those columns. The
+ * inline-scan path used to restate the schema locally even though codegen
+ * emits it; a hand-written wire schema is how four SDKs ended up disagreeing
+ * about which columns were binary and which were large_binary.
+ *
+ * `argumentsBytes` is the already-serialized inner arguments batch (one column
+ * per argument: `arg_<index>` positional, bare name for named).
+ */
+export declare function encodeScanFunctionResult(functionName: string, argumentsBytes: Uint8Array, requiredExtensions?: string[]): Uint8Array;
 /**
  * Decode a ScanFunctionResult from a wire-shape inner result dict.
  *
@@ -149,6 +164,16 @@ export interface ScanBranchInput {
     branchFilter?: string | null;
     /** Declares this branch as the INSERT target (at most one per table). */
     writable?: boolean;
+    /** Custom file FORMAT this branch reads, for a format-backed branch. */
+    formatName?: string | null;
+    /** Locations the format branch reads. */
+    formatLocations?: string[] | null;
+    /** Reader options for a format branch, as a 1-row IPC batch whose COLUMN
+     *  NAMES are the option names — the same shape `arguments` uses, because an
+     *  option value may be any Arrow type. These BECOME the reader's named
+     *  arguments, so without them a format branch cannot pass its reader a
+     *  delimiter, a header flag, or anything else. */
+    formatOptions?: Uint8Array | null;
     /**
      * Catalog-table branch (lakehouse federation): leave `functionName` empty and
      * set these to scan the base table
@@ -159,6 +184,24 @@ export interface ScanBranchInput {
     sourceSchema?: string | null;
     sourceTable?: string | null;
 }
+/**
+ * Serialize a single branch's arguments to the nested-IPC form `ScanBranch`
+ * expects: a 1-row batch with one column per argument — `arg_<index>` for
+ * positional args, the bare name for named args. Mirrors Python's
+ * `ScanBranch.to_row_dict` argument handling.
+ */
+/**
+ * Encode a format branch's reader options.
+ *
+ * Same shape as a branch's `arguments` blob — a 1-row batch whose COLUMN NAMES
+ * are the option names — because an option value may be any Arrow type, so no
+ * static schema could describe the map. Exported because a worker building a
+ * format branch needs it and has no other way to produce those bytes.
+ */
+export declare function encodeFormatOptions(options: Record<string, {
+    value: unknown;
+    type: VgiDataType;
+}>): Uint8Array;
 /**
  * Build a multi-branch `ScanBranchesResult` wire dict from explicit branch
  * definitions. Each branch becomes its own 1-row `ScanBranchSchema` IPC

package/dist/client/client.d.ts

@@ -10,7 +10,7 @@ import type { VgiClientOptions, TableFunctionOptions, ScalarFunctionOptions, Tab
  * Works with any RpcClient transport (subprocess or HTTP):
  * ```ts
  * import { subprocessConnect } from "@query-farm/vgi-rpc";
- * import { VgiClient, Arguments } from "vgi";
+ * import { VgiClient, Arguments } from "@query-farm/vgi";
  *
  * const rpc = subprocessConnect(["./my-worker"]);
  * const client = new VgiClient(rpc);