@query-farm/vgi 0.6.0

package/src/protocol/serializers/init.ts

@@ -0,0 +1,247 @@
+// Copyright 2025, 2026 Query Farm LLC - https://query.farm
+// InitRequest / GlobalInitResponse wire serialization. Schema mirrors
+// vgi-python's InitRequest dataclass — outer wire shape:
+//   bind_call: binary (nested IPC of BindRequest)
+//   output_schema: binary (nullable serialized Schema)
+//   bind_opaque_data: binary (nullable)
+//   projection_ids: list<int64> (nullable)
+//   pushdown_filters: large_binary (nullable) — nested IPC of filter batch
+//   join_keys: list<large_binary> (non-null list of IPC-serialized batches)
+//   phase: utf8 (nullable) — dictionary(int16,string) in Python
+//   execution_id: binary (nullable)
+//   init_opaque_data: binary (nullable)
+// Order pushdown hints + TABLESAMPLE pushdown hints follow.
+
+import {
+  type VgiBatch,
+  schema as makeSchema,
+  field,
+  utf8,
+  binary,
+  int64,
+  float64,
+  list,
+} from "../../arrow/index.js";
+// RecordBatch type alias kept for the joinKeys closure below — it's a runtime
+// value that lands in BindRequest, not a wire shape.
+type RecordBatch = VgiBatch;
+import { TableInOutPhase } from "../../types.js";
+import {
+  OrderByDirection,
+  OrderByNullOrder,
+  type InitRequest,
+  type GlobalInitResponse,
+} from "../types.js";
+import {
+  serializeSchema,
+  deserializeSchema,
+  serializeBatch,
+  deserializeBatch,
+  batchToScalarDict,
+} from "../../util/arrow/index.js";
+import { toUint8Array, buildSingleRowBatch } from "./shared.js";
+import { serializeBindRequest, deserializeBindRequest } from "./bind.js";
+import { decodeDictValue } from "../../util/arrow/index.js";
+
+const INIT_REQUEST_SCHEMA = makeSchema([
+  field("bind_call", binary(), false),
+  field("output_schema", binary(), false),
+  field("bind_opaque_data", binary(), true),
+  field("projection_ids", list(field("item", int64(), false)), true),
+  field("pushdown_filters", binary(), true),
+  field("join_keys", list(field("item", binary(), true)), false),
+  field("phase", utf8(), true),
+  field("execution_id", binary(), true),
+  field("init_opaque_data", binary(), true),
+  // Order pushdown hints from DuckDB's RowGroupPruner (all null when no hint).
+  field("order_by_column_name", utf8(), true),
+  field("order_by_direction", utf8(), true),
+  field("order_by_null_order", utf8(), true),
+  field("order_by_limit", int64(), true),
+  // TABLESAMPLE pushdown hints from DuckDB's SamplingPushdown optimizer.
+  field("tablesample_percentage", float64(), true),
+  field("tablesample_seed", int64(), true),
+  // Buffered-table finalize stream identifier — C++ appends this last.
+  field("finalize_state_id", binary(), true),
+]);
+
+const GLOBAL_INIT_RESPONSE_SCHEMA = makeSchema([
+  field("execution_id", binary(), false),
+  field("opaque_data", binary(), true),
+  field("max_workers", int64(), false),
+]);
+
+export function serializeInitRequest(req: InitRequest): VgiBatch {
+  const bindCallBatch = serializeBindRequest(req.bind_call);
+  const bindCallBytes = serializeBatch(bindCallBatch);
+
+  const row: Record<string, any> = {
+    bind_call: bindCallBytes,
+    output_schema: serializeSchema(req.output_schema),
+    bind_opaque_data: req.bind_opaque_data ?? null,
+    projection_ids: req.projection_ids?.map((n) => BigInt(n)) ?? null,
+    pushdown_filters: req.pushdown_filters
+      ? serializeBatch(req.pushdown_filters)
+      : null,
+    join_keys: (req.join_keys ?? []).map((b) => serializeBatch(b)),
+    phase: req.phase ?? null,
+    execution_id: req.execution_id ?? null,
+    init_opaque_data: req.init_opaque_data ?? null,
+    order_by_column_name: req.order_by_column_name ?? null,
+    order_by_direction: req.order_by_direction ?? null,
+    order_by_null_order: req.order_by_null_order ?? null,
+    order_by_limit: req.order_by_limit ?? null,
+    tablesample_percentage: req.tablesample_percentage ?? null,
+    tablesample_seed: req.tablesample_seed ?? null,
+    finalize_state_id: req.finalize_state_id ?? null,
+  };
+  return buildSingleRowBatch(INIT_REQUEST_SCHEMA, row);
+}
+
+export function deserializeInitRequest(
+  params: Record<string, any>
+): InitRequest {
+  const bindCallBytes = toUint8Array(params.bind_call);
+  const bindCallBatch = deserializeBatch(bindCallBytes);
+  // Extract the single row as params via the codec/canonical path.
+  const bindParams = batchToScalarDict(bindCallBatch);
+
+  const bindCall = deserializeBindRequest(bindParams);
+
+  // Parse projection_ids - may be a list/array of Int32
+  let projectionIds: number[] | null = null;
+  if (params.projection_ids != null) {
+    const raw = params.projection_ids;
+    if (Array.isArray(raw)) {
+      projectionIds = raw.map(Number);
+    } else if (raw && typeof raw[Symbol.iterator] === "function") {
+      projectionIds = [...raw].map(Number);
+    }
+  }
+
+  // Parse phase. DuckDB sends this as a Dictionary on the wire; vgi-rpc's
+  // row extractor doesn't auto-decode dictionaries, so values come in as raw
+  // Arrow Data objects. decodeDictValue resolves them to the underlying string.
+  let phase: TableInOutPhase | null = null;
+  const phaseRaw = decodeDictValue(params.phase);
+  if (phaseRaw != null) {
+    const phaseStr = String(phaseRaw);
+    if (phaseStr === "INPUT") {
+      phase = TableInOutPhase.INPUT;
+    } else if (phaseStr === "FINALIZE") {
+      phase = TableInOutPhase.FINALIZE;
+    } else if (phaseStr === "TABLE_BUFFERING") {
+      phase = TableInOutPhase.TABLE_BUFFERING;
+    } else if (phaseStr === "TABLE_BUFFERING_FINALIZE") {
+      phase = TableInOutPhase.TABLE_BUFFERING_FINALIZE;
+    }
+  }
+
+  // Parse join_keys - list of IPC-serialized RecordBatches
+  const joinKeys: RecordBatch[] = [];
+  if (params.join_keys != null) {
+    const raw = params.join_keys;
+    const iter: Iterable<any> = Array.isArray(raw)
+      ? raw
+      : typeof raw[Symbol.iterator] === "function"
+      ? raw
+      : [];
+    for (const entry of iter) {
+      if (entry == null) continue;
+      try {
+        joinKeys.push(deserializeBatch(toUint8Array(entry)));
+      } catch {
+        // Skip malformed batches rather than failing init.
+      }
+    }
+  }
+
+  return {
+    bind_call: bindCall,
+    output_schema: deserializeSchema(toUint8Array(params.output_schema)),
+    bind_opaque_data: params.bind_opaque_data
+      ? toUint8Array(params.bind_opaque_data)
+      : null,
+    projection_ids: projectionIds,
+    pushdown_filters: params.pushdown_filters
+      ? deserializeBatch(toUint8Array(params.pushdown_filters))
+      : null,
+    join_keys: joinKeys,
+    phase,
+    finalize_state_id: params.finalize_state_id
+      ? toUint8Array(params.finalize_state_id)
+      : null,
+    execution_id: params.execution_id
+      ? toUint8Array(params.execution_id)
+      : null,
+    init_opaque_data: params.init_opaque_data
+      ? toUint8Array(params.init_opaque_data)
+      : null,
+    order_by_column_name: parseEnum(decodeDictValue(params.order_by_column_name)) ?? null,
+    order_by_direction: parseDirection(decodeDictValue(params.order_by_direction)),
+    order_by_null_order: parseNullOrder(decodeDictValue(params.order_by_null_order)),
+    order_by_limit: parseBigInt(params.order_by_limit),
+    tablesample_percentage: parseNumber(params.tablesample_percentage),
+    tablesample_seed: parseBigInt(params.tablesample_seed),
+  };
+}
+
+function parseEnum(v: any): string | null {
+  if (v === null || v === undefined) return null;
+  return String(v);
+}
+function parseDirection(v: any): OrderByDirection | null {
+  if (v === null || v === undefined) return null;
+  const s = String(v);
+  if (s === "ASC") return OrderByDirection.ASC;
+  if (s === "DESC") return OrderByDirection.DESC;
+  return null;
+}
+function parseNullOrder(v: any): OrderByNullOrder | null {
+  if (v === null || v === undefined) return null;
+  const s = String(v);
+  if (s === "NULLS_FIRST") return OrderByNullOrder.NULLS_FIRST;
+  if (s === "NULLS_LAST") return OrderByNullOrder.NULLS_LAST;
+  return null;
+}
+function parseBigInt(v: any): bigint | null {
+  if (v === null || v === undefined) return null;
+  if (typeof v === "bigint") return v;
+  if (typeof v === "number") return BigInt(v);
+  try {
+    return BigInt(String(v));
+  } catch {
+    return null;
+  }
+}
+function parseNumber(v: any): number | null {
+  if (v === null || v === undefined) return null;
+  if (typeof v === "number") return v;
+  if (typeof v === "bigint") return Number(v);
+  const n = Number(v);
+  return Number.isNaN(n) ? null : n;
+}
+
+export function serializeGlobalInitResponse(
+  resp: GlobalInitResponse
+): Record<string, any> {
+  return {
+    execution_id: resp.execution_id,
+    opaque_data: resp.opaque_data ?? null,
+    max_workers: resp.max_workers,
+  };
+}
+
+export function deserializeGlobalInitResponse(
+  params: Record<string, any>
+): GlobalInitResponse {
+  return {
+    execution_id: toUint8Array(params.execution_id),
+    opaque_data: params.opaque_data
+      ? toUint8Array(params.opaque_data)
+      : null,
+    max_workers: Number(params.max_workers),
+  };
+}
+
+export { INIT_REQUEST_SCHEMA, GLOBAL_INIT_RESPONSE_SCHEMA };

package/src/protocol/serializers/shared.ts

@@ -0,0 +1,46 @@
+// Copyright 2025, 2026 Query Farm LLC - https://query.farm
+// Shared helpers for protocol wire serialization.
+
+import {
+  type VgiSchema,
+  type VgiBatch,
+  isInt,
+  batchFromColumns,
+} from "../../arrow/index.js";
+import { toUint8Array as toUint8ArrayBase } from "../../util/bytes.js";
+
+/**
+ * Coerce a wire value (string / Uint8Array / Buffer / etc.) to a Uint8Array.
+ * Throws when the input is non-null but produces zero bytes — that signals a
+ * truly unsupported shape, distinct from a legitimate empty Uint8Array.
+ */
+export function toUint8Array(val: any): Uint8Array {
+  if (typeof val === "string") return new TextEncoder().encode(val);
+  const result = toUint8ArrayBase(val);
+  if (result.length === 0 && val != null) {
+    throw new Error(`Cannot convert ${typeof val} to Uint8Array`);
+  }
+  return result;
+}
+
+/**
+ * Build a single-row RecordBatch from a {fieldName: value} dict matching the
+ * given schema. Auto-coerces JS numbers to BigInt for Int64 fields.
+ */
+export function buildSingleRowBatch(
+  schema: VgiSchema,
+  values: Record<string, any>,
+): VgiBatch {
+  // Build column dict, coercing Int64 numbers to BigInt where the schema
+  // declares one — vectorFromArray/columnFromArray rejects plain Number for
+  // 64-bit ints under both backends.
+  const cols: Record<string, any[]> = {};
+  for (const f of schema.fields) {
+    let val = values[f.name];
+    if (isInt(f.type) && (f.type as any).bitWidth === 64 && typeof val === "number") {
+      val = BigInt(val);
+    }
+    cols[f.name] = [val];
+  }
+  return batchFromColumns(cols, schema);
+}

package/src/protocol/state-serializer.ts

@@ -0,0 +1,181 @@
+// Copyright 2025, 2026 Query Farm LLC - https://query.farm
+// Arrow IPC state serializer for HTTP transport.
+// Replaces JSON serialization with Arrow IPC for the exchange state token,
+// avoiding base64/hex overhead and staying consistent with the rest of the protocol.
+
+import {
+  type VgiDataType,
+  type VgiField,
+  schema as makeSchema,
+  field,
+  struct as makeStruct,
+  binary,
+  int64,
+  bool,
+  float64,
+  utf8,
+  nullType,
+  isStruct,
+  isBinary,
+  serializeBatch,
+  deserializeBatch,
+  batchFromColumns,
+  readCanonicalValue,
+} from "../arrow/index.js";
+import { codecFor } from "../arrow/codec/registry.js";
+import type { StateSerializer } from "@query-farm/vgi-rpc";
+import { toUint8Array } from "../util/bytes.js";
+
+/** Schema for the exchange state carried in HTTP state tokens. */
+export const EXCHANGE_STATE_SCHEMA = makeSchema([
+  field("function_name", binary(), false),
+  field("init_request", binary(), false),
+  field("execution_id", binary(), false),
+  field("max_workers", int64(), false),
+  field("opaque_data", binary(), true),
+  field("is_producer", bool(), false),
+  field("user_state", binary(), true),
+]);
+
+const TEXT_ENCODER = new TextEncoder();
+const TEXT_DECODER = new TextDecoder();
+
+/** Infer an Arrow DataType from a JS value for user state serialization. */
+export function inferFieldType(value: any): VgiDataType {
+  if (value === null || value === undefined) return nullType();
+  switch (typeof value) {
+    case "number": return float64();
+    case "bigint": return int64();
+    case "string": return utf8();
+    case "boolean": return bool();
+    case "object":
+      if (value instanceof Uint8Array || value instanceof ArrayBuffer) return binary();
+      if (ArrayBuffer.isView(value)) return binary();
+      if (Array.isArray(value)) {
+        throw new Error(`inferFieldType: arrays are not supported in userState (convert to a serializable form first)`);
+      }
+      if (value instanceof Map) {
+        throw new Error(`inferFieldType: Map is not supported in userState (use a plain object instead)`);
+      }
+      if (value instanceof Set) {
+        throw new Error(`inferFieldType: Set is not supported in userState (use an array or plain object instead)`);
+      }
+      if (value instanceof Date) {
+        throw new Error(`inferFieldType: Date is not supported in userState (use a number (epoch ms) or ISO string instead)`);
+      }
+      if (value instanceof RegExp) {
+        throw new Error(`inferFieldType: RegExp is not supported in userState (use a string pattern instead)`);
+      }
+      // Plain object → Struct
+      const fields = Object.entries(value).map(
+        ([k, v]) => field(k, inferFieldType(v), true),
+      );
+      return makeStruct(fields);
+    default:
+      throw new Error(`inferFieldType: unsupported type '${typeof value}'`);
+  }
+}
+
+/**
+ * Serialize userState to Arrow IPC bytes.
+ * Infers schema from the JS object at runtime. Arrow IPC is self-describing,
+ * so deserialization doesn't need the schema ahead of time.
+ *
+ * For an empty object `{}`, we emit a 0-row batch with empty schema —
+ * `deserializeUserState` recognizes that shape and returns `{}` rather than
+ * `null`. (`null` is reserved for "no userState declared at all".)
+ */
+export function serializeUserState(userState: any): Uint8Array | null {
+  if (userState == null) return null;
+  const entries = Object.entries(userState);
+  if (entries.length === 0) {
+    // 0-row, 0-field batch — round-trips back to {} via deserializeUserState.
+    return serializeBatch(batchFromColumns({}, makeSchema([])));
+  }
+  const fields = entries.map(
+    ([k, v]) => field(k, inferFieldType(v), true),
+  );
+  const sch = makeSchema(fields);
+  const columns: Record<string, any[]> = {};
+  for (const [key, val] of entries) {
+    columns[key] = [val];
+  }
+  return serializeBatch(batchFromColumns(columns, sch));
+}
+
+/**
+ * Extract a typed value from an Arrow column in RICH form via the codec /
+ * canonical path. Backend-agnostic and lossless (BigInt for Int64, Uint8Array
+ * for Binary, plain objects for Struct). userState forbids Date/temporal types
+ * (see {@link inferFieldType}), so rich == canonical for everything stored here.
+ */
+function extractTypedValue(col: any, index: number, type: VgiDataType): any {
+  return codecFor(type).canonicalToRich(readCanonicalValue(type, col, index));
+}
+
+/**
+ * Deserialize userState from Arrow IPC bytes.
+ * Reconstructs a plain JS object, preserving BigInt for Int64 columns.
+ */
+export function deserializeUserState(bytes: Uint8Array | null): any {
+  if (bytes == null) return null;
+  const batch = deserializeBatch(bytes);
+  if (batch.numRows === 0 && batch.schema.fields.length === 0) {
+    return {};
+  }
+  if (batch.numRows === 0) return null;
+  const result: Record<string, any> = {};
+  for (const f of batch.schema.fields) {
+    const col = batch.getChild(f.name);
+    if (!col) { result[f.name] = null; continue; }
+    result[f.name] = extractTypedValue(col, 0, f.type);
+  }
+  return result;
+}
+
+/** Arrow IPC state serializer — stores all data as native binary columns. */
+export const arrowStateSerializer: StateSerializer = {
+  serialize(state: any): Uint8Array {
+    const columns: Record<string, any[]> = {
+      function_name: [TEXT_ENCODER.encode(state.functionName)],
+      init_request: [state.initRequestIpc],
+      execution_id: [state.executionId],
+      max_workers: [state.maxWorkers],
+      opaque_data: [state.opaqueData ?? null],
+      is_producer: [state.isProducer],
+      user_state: [serializeUserState(state.userState)],
+    };
+    const batch = batchFromColumns(columns, EXCHANGE_STATE_SCHEMA);
+    return serializeBatch(batch);
+  },
+
+  deserialize(bytes: Uint8Array): any {
+    const batch = deserializeBatch(bytes);
+    const fieldByName = new Map(batch.schema.fields.map((f) => [f.name, f]));
+    const get = (name: string) => {
+      const col = batch.getChild(name);
+      const f = fieldByName.get(name);
+      if (!col || !f) return null;
+      const type = f.type as unknown as VgiDataType;
+      return codecFor(type).canonicalToRich(readCanonicalValue(type, col, 0));
+    };
+
+    const isProducer = get("is_producer");
+    const opaqueDataRaw = get("opaque_data");
+    const fnNameRaw = get("function_name");
+
+    return {
+      functionName: fnNameRaw != null ? TEXT_DECODER.decode(toUint8Array(fnNameRaw)) : "",
+      initRequestIpc: toUint8Array(get("init_request")),
+      executionId: toUint8Array(get("execution_id")),
+      maxWorkers: Number(get("max_workers")),
+      opaqueData: opaqueDataRaw != null ? toUint8Array(opaqueDataRaw) : null,
+      isProducer,
+      // vgi-rpc dispatch reads __isProducer to choose producer vs exchange mode
+      __isProducer: isProducer,
+      userState: deserializeUserState(
+        get("user_state") != null ? toUint8Array(get("user_state")) : null,
+      ),
+    };
+  },
+};

package/src/protocol/types.ts

@@ -0,0 +1,153 @@
+// Copyright 2025, 2026 Query Farm LLC - https://query.farm
+// Protocol-level types for the VGI worker. Snake_case to match the Python
+// side and the generated vgi-client.ts wire interfaces. These are the
+// *deserialized* shapes handlers see — fields that cross the wire as `binary`
+// here hold the parsed Arrow object (Schema / RecordBatch / Arguments) rather
+// than raw bytes, so the handler doesn't have to deserialize itself.
+//
+// Domain-layer types (FunctionMeta, TableFunctionConfig, TableProcessParams,
+// StreamHandlers, VgiClientOptions, etc.) remain camelCase as JS-idiomatic
+// public API surfaces. Conversions happen at the layer boundary in
+// src/protocol/serialize.ts and the function/catalog dispatch sites.
+
+import type { VgiSchema, VgiBatch } from "../arrow/index.js";
+import type { Arguments } from "../arguments/arguments.js";
+import { FunctionType, TableInOutPhase, type TableCardinality } from "../types.js";
+
+/**
+ * Context for a `COPY ... FROM` read, threaded onto {@link BindRequest}.
+ *
+ * Present only when the bind/init opens a COPY-FROM scan (`null`/absent
+ * otherwise — set by the VGI extension's `copy_from_bind`). `InitRequest`
+ * embeds the same `BindRequest` as `bind_call`, so process()/init also reach it
+ * via `initCall.bind_call.copy_from`. The handler's options arrive through the
+ * normal `BindRequest.arguments` (built from the COPY options), so they are not
+ * duplicated here. Mirrors vgi-python's `protocol.CopyFromContext`.
+ */
+export interface CopyFromContext {
+  /** The `FORMAT` name resolved at COPY bind time. */
+  format: string;
+  /** The source path from the `COPY ... FROM 'path'` statement. */
+  file_path: string;
+  /**
+   * The COPY target's column schema (name + type, in target order). The worker
+   * must bind its output to, and emit columns whose types match, this schema
+   * exactly — DuckDB inserts no cast between the scan and the INSERT. Holds the
+   * parsed Arrow Schema (the wire field is binary; deserialized here).
+   */
+  expected_schema: VgiSchema;
+}
+
+/**
+ * Context for a `COPY ... TO` write, threaded onto {@link BindRequest}.
+ *
+ * Present only when the bind/init opens a COPY-TO sink (`null`/absent
+ * otherwise — set by the VGI extension's `copy_to_bind`). `InitRequest`
+ * embeds the same `BindRequest` as `bind_call`, so process()/combine() also
+ * reach it via `initCall.bind_call.copy_to`. The handler's options arrive
+ * through the normal `BindRequest.arguments`; the **source** columns ride the
+ * existing `BindRequest.input_schema` (so they are not duplicated here).
+ * Mirrors vgi-python's `protocol.CopyToContext`.
+ */
+export interface CopyToContext {
+  /** The `FORMAT` name resolved at COPY bind time. */
+  format: string;
+  /** The destination path from the `COPY ... TO 'path'` statement. */
+  file_path: string;
+}
+
+export interface BindRequest {
+  function_name: string;
+  arguments: Arguments;
+  function_type: FunctionType;
+  input_schema: VgiSchema | null;
+  settings: VgiBatch | null;
+  secrets: VgiBatch | null;
+  attach_opaque_data: Uint8Array | null;
+  transaction_opaque_data: Uint8Array | null;
+  resolved_secrets_provided: boolean;
+  /**
+   * COPY ... FROM context — `null`/absent unless this bind/init opens a
+   * COPY-FROM scan. Additive + name-keyed, so ordinary scans (which omit it on
+   * the wire) deserialize to `null`. Mirrors vgi-python's `BindRequest.copy_from`.
+   */
+  copy_from?: CopyFromContext | null;
+  /**
+   * COPY ... TO context — `null`/absent unless this bind/init opens a COPY-TO
+   * sink. Additive + name-keyed, same wire-safe rationale as `copy_from`.
+   * Mirrors vgi-python's `BindRequest.copy_to`.
+   */
+  copy_to?: CopyToContext | null;
+  /**
+   * Time travel: the AT (TIMESTAMP|VERSION ...) clause for this scan, threaded
+   * from DuckDB's per-reference bind. Both `null` when the scan has no AT clause.
+   * For inline-bound (function-backed) tables the actual on_bind RPC runs once at
+   * attach with no AT, so the per-scan AT is carried on the bind request embedded
+   * in each InitRequest — read it at init via `init_call.bind_call.at_unit` (or
+   * `TableProcessParams.atUnit`). Mirrors vgi-python's `BindRequest.at_unit`.
+   */
+  at_unit?: string | null;
+  at_value?: string | null;
+}
+
+export interface BindResponse {
+  output_schema: VgiSchema;
+  opaque_data: Uint8Array | null;
+  lookup_secret_types?: string[];
+  lookup_scopes?: string[];
+  lookup_names?: string[];
+}
+
+export interface InitRequest {
+  bind_call: BindRequest;
+  output_schema: VgiSchema;
+  bind_opaque_data: Uint8Array | null;
+  projection_ids: number[] | null;
+  pushdown_filters: VgiBatch | null;
+  /**
+   * Join-key value batches, one per join-keys column. Keyed by the column
+   * name inside each batch's schema. Populated when DuckDB promotes
+   * IN/OR lists or join predicates to batched join-keys pushdowns.
+   */
+  join_keys: VgiBatch[];
+  phase: TableInOutPhase | null;
+  /**
+   * Buffered-table finalize stream: which finalize_state_id this stream
+   * serves. Set when phase=TABLE_BUFFERING_FINALIZE; null otherwise. Opaque
+   * bytes the worker's combine() chose.
+   */
+  finalize_state_id: Uint8Array | null;
+  execution_id: Uint8Array | null;
+  init_opaque_data: Uint8Array | null;
+  // Order pushdown hints from DuckDB's RowGroupPruner (null when no hint).
+  order_by_column_name: string | null;
+  order_by_direction: OrderByDirection | null;
+  order_by_null_order: OrderByNullOrder | null;
+  order_by_limit: bigint | null;
+  // TABLESAMPLE pushdown hints.
+  tablesample_percentage: number | null;
+  tablesample_seed: bigint | null;
+}
+
+export enum OrderByDirection {
+  ASC = "ASC",
+  DESC = "DESC",
+}
+
+export enum OrderByNullOrder {
+  NULLS_FIRST = "NULLS_FIRST",
+  NULLS_LAST = "NULLS_LAST",
+}
+
+export interface GlobalInitResponse {
+  max_workers: number;
+  execution_id: Uint8Array;
+  opaque_data: Uint8Array | null;
+}
+
+export interface TableFunctionCardinalityRequest {
+  bind_call: BindRequest;
+  bind_opaque_data: Uint8Array | null;
+}
+
+export { TableCardinality };

package/src/secrets/helpers.ts

@@ -0,0 +1,70 @@
+// Type- and scope-aware selection over resolved secrets.
+//
+// Resolved secrets reach a worker as a plain object keyed by each secret's
+// unique DuckDB secret name (so several secrets of the same type — e.g. one per
+// S3 bucket — coexist). Each secret carries its connector-serialized `type` (the
+// DuckDB secret type) and `scope` (newline-joined scope prefixes) fields, plus
+// type-specific fields. These helpers mirror `vgi::Secrets` in the Rust SDK.
+
+export type SecretFields = Record<string, any>;
+export type SecretsDict = Record<string, SecretFields>;
+
+function scalarStr(v: any): string {
+  if (v === null || v === undefined) return "";
+  return typeof v === "string" ? v : String(v);
+}
+
+/** The DuckDB secret type of the named secret (its serialized `type` field). */
+export function secretType(secrets: SecretsDict, name: string): string | undefined {
+  const f = secrets?.[name];
+  return f && "type" in f ? scalarStr(f.type) : undefined;
+}
+
+/** Every resolved secret whose serialized `type` field matches `type`. */
+export function secretsOfType(secrets: SecretsDict, type: string): SecretFields[] {
+  return Object.values(secrets ?? {}).filter((f) => scalarStr(f.type) === type);
+}
+
+/**
+ * The secret whose `scope` is the longest prefix of `path`. The connector
+ * serializes each secret's scope as a newline-joined list of prefixes; a secret
+ * with no (or empty) scope matches as a last-resort fallback. Returns undefined
+ * only when there are no candidate secrets.
+ */
+export function secretForScope(secrets: SecretsDict, path: string): SecretFields | undefined {
+  return selectForScope(secrets, path, undefined);
+}
+
+/** Like {@link secretForScope} but only over secrets of `type`. */
+export function secretForScopeOfType(
+  secrets: SecretsDict,
+  path: string,
+  type: string,
+): SecretFields | undefined {
+  return selectForScope(secrets, path, type);
+}
+
+function selectForScope(
+  secrets: SecretsDict,
+  path: string,
+  type: string | undefined,
+): SecretFields | undefined {
+  let best: SecretFields | undefined;
+  let bestLen = -1;
+  let fallback: SecretFields | undefined;
+  for (const fields of Object.values(secrets ?? {})) {
+    if (type !== undefined && scalarStr(fields.type) !== type) continue;
+    const scope = scalarStr(fields.scope);
+    if (!scope) {
+      if (!fallback) fallback = fields;
+      continue;
+    }
+    for (const prefix of scope.split("\n")) {
+      if (prefix && path.startsWith(prefix) && prefix.length > bestLen) {
+        bestLen = prefix.length;
+        best = fields;
+      }
+    }
+  }
+  return best ?? fallback;
+}