@query-farm/vgi 0.6.0

package/src/client/errors.ts

@@ -0,0 +1,69 @@
+// Copyright 2025, 2026 Query Farm LLC - https://query.farm
+// VgiClient error class + RpcClient wrapper that translates RpcError into
+// VgiClientError, preserving the worker's remote traceback.
+
+import { RpcError, type RpcClient } from "@query-farm/vgi-rpc";
+
+/** Error thrown by VgiClient when an RPC call fails or returns unexpected data. */
+export class VgiClientError extends Error {
+  /** Remote traceback from the worker, when an RpcError carried one. */
+  readonly remoteTraceback?: string;
+  /** Underlying error type from the worker (e.g. "ValueError"), when known. */
+  readonly errorType?: string;
+
+  constructor(
+    message: string,
+    options?: { remoteTraceback?: string; errorType?: string; cause?: unknown },
+  ) {
+    super(message, options?.cause != null ? { cause: options.cause } : undefined);
+    this.name = "VgiClientError";
+    if (options?.remoteTraceback) this.remoteTraceback = options.remoteTraceback;
+    if (options?.errorType) this.errorType = options.errorType;
+  }
+
+  /**
+   * Build a VgiClientError from an underlying RpcError, preserving the
+   * remote error type, message, and traceback so the worker-raised exception
+   * shows up at the top of the stack rather than buried under VGI framing.
+   * Mirrors Python's ClientError.from_rpc_error.
+   */
+  static fromRpcError(e: RpcError): VgiClientError {
+    const head = `${e.errorType}: ${e.errorMessage}`;
+    const message = e.remoteTraceback
+      ? `${head}\n\nRemote traceback:\n${e.remoteTraceback}`
+      : head;
+    return new VgiClientError(message, {
+      remoteTraceback: e.remoteTraceback || undefined,
+      errorType: e.errorType,
+      cause: e,
+    });
+  }
+}
+
+/**
+ * Wrap an RpcClient so any RpcError thrown by call/stream is rethrown as a
+ * VgiClientError carrying the worker's remote traceback. Other errors
+ * propagate unchanged.
+ */
+export function wrapRpcWithErrorEnrichment(rpc: RpcClient): RpcClient {
+  return {
+    async call(method, params) {
+      try {
+        return await rpc.call(method, params);
+      } catch (e) {
+        if (e instanceof RpcError) throw VgiClientError.fromRpcError(e);
+        throw e;
+      }
+    },
+    async stream(method, params) {
+      try {
+        return await rpc.stream(method, params);
+      } catch (e) {
+        if (e instanceof RpcError) throw VgiClientError.fromRpcError(e);
+        throw e;
+      }
+    },
+    describe: () => rpc.describe(),
+    close: () => rpc.close(),
+  };
+}

package/src/client/helpers.ts

@@ -0,0 +1,56 @@
+// Copyright 2025, 2026 Query Farm LLC - https://query.farm
+// Internal helpers shared by VgiClient methods.
+
+import { toUint8Array } from "../util/bytes.js";
+
+/**
+ * Deserialize a List<Binary> of serialized info batches into typed info objects.
+ */
+export function deserializeInfoList<T>(
+  items: unknown,
+  deserializeFn: (bytes: Uint8Array) => T,
+): T[] {
+  if (!items) return [];
+  const arr: unknown[] = Array.isArray(items)
+    ? items
+    : [...(items as Iterable<unknown>)];
+  return arr
+    .filter((b) => b != null)
+    .map((b) => deserializeFn(toUint8Array(b)));
+}
+
+/**
+ * Deserialize Arrow Map entries into a plain Record<string, string>.
+ */
+export function deserializeTags(mapVal: unknown): Record<string, string> {
+  const tags: Record<string, string> = {};
+  if (!mapVal) return tags;
+  const iterable = mapVal as { [Symbol.iterator]?: () => Iterator<unknown> };
+  if (typeof iterable[Symbol.iterator] !== "function") return tags;
+  for (const entry of iterable as Iterable<unknown>) {
+    if (Array.isArray(entry)) {
+      tags[String(entry[0])] = String(entry[1]);
+    } else if (entry && typeof entry === "object") {
+      const e = entry as Record<string, unknown> & { [0]?: unknown; [1]?: unknown };
+      tags[String(e.key ?? e[0] ?? "")] = String(e.value ?? e[1] ?? "");
+    }
+  }
+  return tags;
+}
+
+/**
+ * Convert sync/async iterable to async iterator.
+ */
+export function toAsyncIterator<T>(
+  input: Iterable<T> | AsyncIterable<T>,
+): AsyncIterator<T> {
+  if (Symbol.asyncIterator in (input as any)) {
+    return (input as AsyncIterable<T>)[Symbol.asyncIterator]();
+  }
+  const syncIter = (input as Iterable<T>)[Symbol.iterator]();
+  return {
+    async next() {
+      return syncIter.next();
+    },
+  };
+}

package/src/client/protocol.ts

@@ -0,0 +1,33 @@
+// Copyright 2025, 2026 Query Farm LLC - https://query.farm
+// Wrap/unwrap helpers for the VGI double-wrapping protocol.
+// Mirrors the server's dispatch.ts:64-84 (unwrapRequest/wrapResult).
+
+import {
+  serializeBatch,
+  deserializeBatch,
+  batchToScalarDict,
+} from "../util/arrow/index.js";
+import { toUint8Array } from "../util/bytes.js";
+import type { VgiBatch } from "../arrow/index.js";
+export { toUint8Array };
+
+/**
+ * Wrap a RecordBatch as a { request: Uint8Array } dict for RPC params.
+ * DuckDB wraps ArrowSerializableDataclass params in a single "request" Binary column.
+ */
+export function wrapRequest(batch: VgiBatch): { request: Uint8Array } {
+  return { request: serializeBatch(batch) };
+}
+
+/**
+ * Unwrap a { result: Uint8Array } response dict.
+ * The "result" Binary column contains a serialized inner batch;
+ * returns its row 0 as a flat dict.
+ */
+export function unwrapResult(
+  result: Record<string, any>,
+): Record<string, any> {
+  const resultBytes = toUint8Array(result.result);
+  const innerBatch = deserializeBatch(resultBytes);
+  return batchToScalarDict(innerBatch);
+}

package/src/client/types.ts

@@ -0,0 +1,140 @@
+// Copyright 2025, 2026 Query Farm LLC - https://query.farm
+// Client option types for VgiClient.
+
+import type { VgiBatch } from "../arrow/index.js";
+import type { Arguments } from "../arguments/arguments.js";
+import type { AttachOptionValue } from "../catalog/attach-options.js";
+import type { BindResponse } from "../protocol/types.js";
+import { OrderByDirection, OrderByNullOrder } from "../protocol/types.js";
+
+export type { AttachOptionValue };
+export { OrderByDirection, OrderByNullOrder };
+
+/** ORDER BY pushdown hint passed at init time. */
+export interface OrderByPushdown {
+  columnName: string;
+  direction: OrderByDirection;
+  nullOrder: OrderByNullOrder;
+  /** Optional LIMIT to combine with ORDER BY. */
+  limit?: bigint | number | null;
+}
+
+/** TABLESAMPLE pushdown hint passed at init time. */
+export interface TablesamplePushdown {
+  /** Sample percentage in [0, 100]. */
+  percentage: number;
+  /** Optional sampling seed. */
+  seed?: bigint | number | null;
+}
+
+/** Callback invoked after a successful bind, before init. */
+export type BindResultCallback = (response: BindResponse) => void;
+
+/**
+ * Options bag for VgiClient.catalogAttach.
+ *
+ * Pass `options` as a plain key→value map — the client serializes to an
+ * Arrow RecordBatch with per-value type inference (see AttachOptionValue).
+ * For Arrow types that inference can't express (Decimal, Timestamp, exact
+ * int width, nested struct), use `optionsBytes` to supply pre-serialized
+ * bytes. Providing both throws.
+ *
+ * `dataVersionSpec` / `implementationVersion` are sent to versioned
+ * catalogs for attach-time validation; workers that aren't versioned
+ * ignore them.
+ */
+export interface CatalogAttachOptions {
+  options?: Record<string, AttachOptionValue>;
+  optionsBytes?: Uint8Array;
+  dataVersionSpec?: string | null;
+  implementationVersion?: string | null;
+}
+
+/** Conflict resolution strategy for create operations. */
+export type OnCreateConflict = "error" | "ignore" | "replace";
+
+/** DuckDB catalog function type filter (sent as uppercase wire values). */
+export type CatalogFunctionType = "SCALAR_FUNCTION" | "TABLE_FUNCTION";
+
+/** Macro type filter for schema contents listing (uppercase wire values). */
+export type CatalogMacroType = "SCALAR_MACRO" | "TABLE_MACRO";
+
+/** Options for constructing a VgiClient. */
+export interface VgiClientOptions {
+  /** Pre-existing attach ID to bind this client to a specific catalog. */
+  attachOpaqueData?: Uint8Array;
+}
+
+/** Options for calling a table function. */
+export interface TableFunctionOptions {
+  /** Name of the function to call. */
+  functionName: string;
+  /** Positional and named arguments. */
+  arguments?: Arguments;
+  /** Column indices to project (filter pushdown). */
+  projectionIds?: number[];
+  /** Filter pushdown batch. */
+  pushdownFilters?: VgiBatch;
+  /** DuckDB settings to pass to the function. */
+  settings?: VgiBatch;
+  /** Transaction ID for transactional catalogs. */
+  transactionOpaqueData?: Uint8Array;
+  /** Attach ID to bind this call to a specific catalog attach. Overrides the client-level attachOpaqueData. */
+  attachOpaqueData?: Uint8Array;
+  /** ORDER BY pushdown hint from DuckDB's RowGroupPruner. */
+  orderBy?: OrderByPushdown;
+  /** TABLESAMPLE pushdown hint from DuckDB's SamplingPushdown optimizer. */
+  tablesample?: TablesamplePushdown;
+  /** Join-key value batches, one per join-keys column. */
+  joinKeys?: VgiBatch[];
+  /** Invoked after bind, before init. Receives the bind response. */
+  onBind?: BindResultCallback;
+}
+
+/** Options for calling a scalar function. */
+export interface ScalarFunctionOptions {
+  /** Name of the function to call. */
+  functionName: string;
+  /** Input batches to process. */
+  input: Iterable<VgiBatch> | AsyncIterable<VgiBatch>;
+  /** Positional and named arguments. */
+  arguments?: Arguments;
+  /** DuckDB settings to pass to the function. */
+  settings?: VgiBatch;
+  /** DuckDB secrets to pass to the function. */
+  secrets?: VgiBatch;
+  /** Transaction ID for transactional catalogs. */
+  transactionOpaqueData?: Uint8Array;
+  /** Attach ID to bind this call to a specific catalog attach. Overrides the client-level attachOpaqueData. */
+  attachOpaqueData?: Uint8Array;
+  /** Invoked after bind, before init. Receives the bind response. */
+  onBind?: BindResultCallback;
+}
+
+/** Options for calling a table-in-out function. */
+export interface TableInOutFunctionOptions {
+  /** Name of the function to call. */
+  functionName: string;
+  /** Input batches to process. */
+  input: Iterable<VgiBatch> | AsyncIterable<VgiBatch>;
+  /** Positional and named arguments. */
+  arguments?: Arguments;
+  /** Column indices to project (filter pushdown). */
+  projectionIds?: number[];
+  /** Filter pushdown batch. */
+  pushdownFilters?: VgiBatch;
+  /** DuckDB settings to pass to the function. */
+  settings?: VgiBatch;
+  /** Transaction ID for transactional catalogs. */
+  transactionOpaqueData?: Uint8Array;
+  /** Attach ID to bind this call to a specific catalog attach. Overrides the client-level attachOpaqueData. */
+  attachOpaqueData?: Uint8Array;
+  /** ORDER BY pushdown hint from DuckDB's RowGroupPruner. */
+  orderBy?: OrderByPushdown;
+  /** TABLESAMPLE pushdown hint from DuckDB's SamplingPushdown optimizer. */
+  tablesample?: TablesamplePushdown;
+  /** Join-key value batches, one per join-keys column. */
+  joinKeys?: VgiBatch[];
+  /** Invoked after bind, before init. Receives the bind response. */
+  onBind?: BindResultCallback;
+}

package/src/client-entry.ts

@@ -0,0 +1,112 @@
+// Copyright 2025, 2026 Query Farm LLC - https://query.farm
+// VGI Client — browser-safe entry point.
+//
+// Import from "vgi/client" instead of "vgi" when targeting browsers, Deno,
+// or any environment that cannot load Node.js built-ins. This entry point
+// excludes server-side code (Worker, Protocol, Storage) that depends on
+// node:fs, node:os, and process.
+//
+// Usage:
+//   import { VgiClient, Arguments } from "vgi/client";
+//   import { httpConnect } from "@query-farm/vgi-rpc";      // or your own RpcClient
+//
+//   const rpc = httpConnect("https://my-vgi-server/");
+//   const client = new VgiClient(rpc);
+
+// Client
+export { VgiClient, VgiClientError } from "./client/client.js";
+export type {
+  VgiClientOptions,
+  TableFunctionOptions,
+  ScalarFunctionOptions,
+  TableInOutFunctionOptions,
+  OnCreateConflict,
+  CatalogFunctionType,
+} from "./client/types.js";
+
+// Arguments
+export { Arguments } from "./arguments/arguments.js";
+export {
+  type ArgumentSpec,
+  argumentSpecsToSchema,
+  schemaToArgumentSpecs,
+} from "./arguments/argument-spec.js";
+
+// Core enums
+export {
+  FunctionType,
+  FunctionStability,
+  NullHandling,
+  OrderPreservation,
+  OrderDependence,
+  DistinctDependence,
+  TableInOutPhase,
+} from "./types.js";
+
+// Protocol types (needed for advanced client use)
+export type {
+  BindRequest,
+  BindResponse,
+  InitRequest,
+  GlobalInitResponse,
+  TableCardinality,
+} from "./protocol/types.js";
+
+// Catalog info types
+export {
+  type SchemaInfo,
+  encodeSchemaInfo,
+  decodeSchemaInfo,
+  type TableInfo,
+  encodeTableInfo,
+  decodeTableInfo,
+  type ViewInfo,
+  encodeViewInfo,
+  decodeViewInfo,
+  type FunctionInfo,
+  encodeFunctionInfo,
+  decodeFunctionInfo,
+  type MacroInfo,
+  encodeMacroInfo,
+  decodeMacroInfo,
+  type CatalogAttachResult,
+  type AttachOpaqueData,
+  type TransactionOpaqueData,
+} from "./catalog/interface.js";
+
+// Errors
+export {
+  VgiError,
+  RowCountMismatchError,
+  FunctionNotFoundError,
+  CatalogReadOnlyError,
+  CatalogNotFoundError,
+  CatalogAlreadyExistsError,
+  ArgumentValidationError,
+  NoCatalogError,
+} from "./errors.js";
+
+// Arrow utilities
+export {
+  iterRows,
+  batchFromRows,
+  batchFromColumns,
+  emptyBatch,
+  filterBatch,
+  projectSchema,
+  projectBatch,
+  batchToScalarDict,
+  safeNumber,
+  serializeSchema,
+  deserializeSchema,
+  serializeBatch,
+  deserializeBatch,
+} from "./util/arrow/index.js";
+
+// Byte utilities
+export { toUint8Array } from "./util/bytes.js";
+
+// Re-export transport types from vgi-rpc (type-only — zero runtime cost).
+// Users import the actual connect functions (httpConnect, pipeConnect) from
+// "@query-farm/vgi-rpc" directly so their bundler can tree-shake the server-side code.
+export type { RpcClient, StreamSession, LogMessage } from "@query-farm/vgi-rpc";

package/src/codec/asd.ts

@@ -0,0 +1,147 @@
+// Copyright 2025, 2026 Query Farm LLC - https://query.farm
+// SPDX-License-Identifier: LicenseRef-QueryFarm-Source-Available-1.0
+
+/**
+ * Generic ArrowSerializableDataclass (ASD) codec.
+ *
+ * Bridges between typed TS objects (snake_case, matching vgi-python dataclass
+ * field names) and Arrow IPC bytes (single-row batch with the dataclass'
+ * ARROW_SCHEMA). Used by the generated vgi-client.ts encode<X>/decode<X>
+ * wrappers to make the typed client interfaces honest at the wire boundary.
+ *
+ * Encoding delegates to the facade's `batchFromRows` (which handles the
+ * complex types — Decimal/BigInt/List/Map/Struct/Dictionary — uniformly
+ * across both arrow-js and flechette backends). Decoding pulls each field's
+ * value with `batchToScalarDict`, then runs `normalizeValue` to convert
+ * Arrow representations (MapRow, BigInt, etc.) back into plain JS shapes.
+ */
+
+import {
+  type VgiSchema,
+  type VgiField,
+  type VgiDataType,
+  isBinary,
+  isBool,
+  isDictionary,
+  isInt,
+  isList,
+  isMap,
+  isStruct,
+  batchFromRows,
+  serializeBatch,
+  deserializeBatch,
+} from "../arrow/index.js";
+
+/**
+ * Encode a typed object as a single-row Arrow IPC stream using `schema`.
+ * Matches Python's `ArrowSerializableDataclass.serialize_to_bytes()`.
+ */
+export function encodeASD(
+  schema: VgiSchema,
+  obj: Record<string, any>,
+): Uint8Array {
+  return serializeBatch(batchFromRows([obj], schema));
+}
+
+/**
+ * Decode a single-row Arrow IPC stream into a typed object using `schema`.
+ * Matches Python's `ArrowSerializableDataclass.deserialize_from_bytes()`.
+ */
+export function decodeASD<T>(
+  schema: VgiSchema,
+  bytes: Uint8Array,
+): T {
+  const batch = deserializeBatch(bytes);
+  if (batch.numRows === 0) {
+    const names = schema.fields.map((f) => f.name).join(",");
+    throw new Error(`decodeASD: empty batch (expected 1 row for ${names})`);
+  }
+  const out: Record<string, any> = {};
+  for (const field of schema.fields) {
+    const col = batch.getChild(field.name);
+    const raw = col ? col.get(0) : null;
+    out[field.name] = normalizeValue(raw, field.type);
+  }
+  return out as T;
+}
+
+// --------------------------------------------------------------------------- //
+// Decode side: Arrow value -> normalized JS value
+// --------------------------------------------------------------------------- //
+
+function normalizeValue(raw: any, type: VgiDataType): any {
+  if (raw == null) return null;
+
+  if (isList(type)) {
+    const childType = (type as any).children[0].type as VgiDataType;
+    const out: any[] = [];
+    for (const item of raw) out.push(normalizeValue(item, childType));
+    return out;
+  }
+
+  if (isMap(type)) {
+    const out: Record<string, string> = {};
+    // arrow-js MapRow is iterable of [k,v]; flechette returns [[k,v],...] arrays.
+    if (raw[Symbol.iterator]) {
+      for (const entry of raw) {
+        if (Array.isArray(entry)) {
+          out[String(entry[0])] = String(entry[1] ?? "");
+        } else if (entry && typeof entry === "object") {
+          const k = entry.key ?? entry[0];
+          const v = entry.value ?? entry[1];
+          out[String(k)] = v == null ? "" : String(v);
+        }
+      }
+    } else if (typeof raw === "object") {
+      // Plain-object map (flechette without useMap): Object.entries fallback
+      for (const [k, v] of Object.entries(raw)) {
+        out[String(k)] = v == null ? "" : String(v);
+      }
+    }
+    return out;
+  }
+
+  if (isStruct(type)) {
+    const childFields = (type as any).children as VgiField[];
+    const out: Record<string, any> = {};
+    for (const cf of childFields) {
+      out[cf.name] = normalizeValue(raw[cf.name], cf.type);
+    }
+    return out;
+  }
+
+  if (isDictionary(type)) {
+    return raw == null ? null : String(raw);
+  }
+
+  if (isBinary(type)) {
+    return toUint8Array(raw);
+  }
+
+  if (isInt(type) && (type as any).bitWidth === 64) {
+    // Prefer number when representable; callers that need BigInt can convert.
+    if (typeof raw === "bigint") {
+      if (raw >= BigInt(Number.MIN_SAFE_INTEGER) && raw <= BigInt(Number.MAX_SAFE_INTEGER)) {
+        return Number(raw);
+      }
+      return raw;
+    }
+    return raw;
+  }
+
+  if (isBool(type)) {
+    return Boolean(raw);
+  }
+
+  // Utf8, Int8/16/32, Float32/64 — pass through as-is.
+  return raw;
+}
+
+function toUint8Array(val: any): Uint8Array {
+  if (val instanceof Uint8Array) return val;
+  if (val instanceof ArrayBuffer) return new Uint8Array(val);
+  if (val && val.buffer instanceof ArrayBuffer) {
+    return new Uint8Array(val.buffer, val.byteOffset, val.byteLength);
+  }
+  return new Uint8Array(0);
+}