@query-farm/vgi 0.6.0

package/src/catalog/composite.ts

@@ -0,0 +1,180 @@
+// Copyright 2025, 2026 Query Farm LLC - https://query.farm
+// CompositeCatalogInterface — fan one worker out across multiple
+// ReadOnlyCatalogInterface instances, picking which one handles each call by
+// the `name` passed at attach time, then by the `attachOpaqueData` returned for all
+// subsequent calls.
+//
+// This mirrors vgi-python's MetaWorker pattern, where one worker process
+// can serve several distinct catalogs (`example`, `projection_repro`,
+// `schema_reconcile`, …) from a single LOCATION.
+
+import type {
+  AttachOpaqueData,
+  CatalogAttachResult,
+  CatalogInfo,
+  CopyFromFormatInfo,
+  FunctionInfo,
+  IndexInfo,
+  MacroInfo,
+  MacroType,
+  SchemaInfo,
+  TableInfo,
+  TransactionOpaqueData,
+  ViewInfo,
+} from "./interface.js";
+import { CatalogInterface } from "./interface.js";
+
+function bufferEquals(a: AttachOpaqueData, b: AttachOpaqueData): boolean {
+  if (a.length !== b.length) return false;
+  for (let i = 0; i < a.length; i++) if (a[i] !== b[i]) return false;
+  return true;
+}
+
+// Subprocess transport spawns a pool of worker processes; DuckDB routes calls
+// across them round-robin. Each worker has its own CompositeCatalog instance,
+// so we can't keep routing state in memory — the attach happens in worker A
+// but later calls land on worker B with an empty route table. To route across
+// workers without a shared store, we encode the backend index in the first
+// byte of attach_opaque_data. Every worker decodes the same byte and routes the same
+// way, so the route is implicit in the attach_opaque_data itself.
+const ROUTE_BYTE = 0;
+
+export class CompositeCatalogInterface extends CatalogInterface {
+  constructor(private readonly _backends: CatalogInterface[]) {
+    super();
+    if (_backends.length > 256) {
+      throw new Error("CompositeCatalog: at most 256 backends supported");
+    }
+  }
+
+  private _route(attachOpaqueData: AttachOpaqueData): CatalogInterface {
+    const idx = attachOpaqueData[ROUTE_BYTE];
+    const route = this._backends[idx];
+    if (!route) {
+      throw new Error(`CompositeCatalog: no backend at route-byte index ${idx} (have ${this._backends.length})`);
+    }
+    return route;
+  }
+
+  catalogs(): string[] {
+    const all: string[] = [];
+    for (const b of this._backends) all.push(...b.catalogs());
+    return all;
+  }
+
+  async catalogsInfo(): Promise<CatalogInfo[]> {
+    const all: CatalogInfo[] = [];
+    for (const b of this._backends) {
+      if (b.catalogsInfo) {
+        all.push(...(await b.catalogsInfo()));
+      } else {
+        for (const name of b.catalogs()) {
+          all.push({ name, implementation_version: null, data_version_spec: null, attach_option_specs: [], releases: [] });
+        }
+      }
+    }
+    return all;
+  }
+
+  async attach(
+    name: string,
+    options?: Record<string, unknown>,
+    dataVersionSpec?: string | null,
+    implementationVersion?: string | null,
+  ): Promise<CatalogAttachResult> {
+    for (let i = 0; i < this._backends.length; i++) {
+      const b = this._backends[i];
+      if (b.catalogs().includes(name)) {
+        const result = await b.attach(name, options, dataVersionSpec, implementationVersion);
+        // Stamp the route-byte so other workers in the pool can decode the
+        // backend without needing in-memory state. Mutate in place so the
+        // wire returns the rewritten id.
+        const stamped = new Uint8Array(result.attach_opaque_data);
+        stamped[ROUTE_BYTE] = i;
+        result.attach_opaque_data = stamped;
+        return result;
+      }
+    }
+    throw new Error(`No worker handles catalog '${name}'`);
+  }
+
+  async detach(attachOpaqueData: AttachOpaqueData): Promise<void> {
+    await this._route(attachOpaqueData).detach(attachOpaqueData);
+  }
+
+  async version(attachOpaqueData: AttachOpaqueData, transactionOpaqueData?: TransactionOpaqueData): Promise<number> {
+    return await this._route(attachOpaqueData).version(attachOpaqueData, transactionOpaqueData);
+  }
+
+  async schemas(attachOpaqueData: AttachOpaqueData, transactionOpaqueData?: TransactionOpaqueData): Promise<SchemaInfo[]> {
+    return await this._route(attachOpaqueData).schemas(attachOpaqueData, transactionOpaqueData);
+  }
+
+  override async schemaGet(attachOpaqueData: AttachOpaqueData, name: string, transactionOpaqueData?: TransactionOpaqueData): Promise<SchemaInfo | null> {
+    return await this._route(attachOpaqueData).schemaGet(attachOpaqueData, name, transactionOpaqueData);
+  }
+
+  override async schemaContentsTables(attachOpaqueData: AttachOpaqueData, name: string, transactionOpaqueData?: TransactionOpaqueData): Promise<TableInfo[]> {
+    return await this._route(attachOpaqueData).schemaContentsTables(attachOpaqueData, name, transactionOpaqueData);
+  }
+
+  override async schemaContentsViews(attachOpaqueData: AttachOpaqueData, name: string, transactionOpaqueData?: TransactionOpaqueData): Promise<ViewInfo[]> {
+    return await this._route(attachOpaqueData).schemaContentsViews(attachOpaqueData, name, transactionOpaqueData);
+  }
+
+  override async schemaContentsFunctions(attachOpaqueData: AttachOpaqueData, name: string, type: string, transactionOpaqueData?: TransactionOpaqueData): Promise<FunctionInfo[]> {
+    return await this._route(attachOpaqueData).schemaContentsFunctions(attachOpaqueData, name, type, transactionOpaqueData);
+  }
+
+  override async schemaContentsMacros(attachOpaqueData: AttachOpaqueData, name: string, type: string, transactionOpaqueData?: TransactionOpaqueData): Promise<MacroInfo[]> {
+    return await this._route(attachOpaqueData).schemaContentsMacros(attachOpaqueData, name, type, transactionOpaqueData);
+  }
+
+  override async schemaContentsIndexes(attachOpaqueData: AttachOpaqueData, name: string, transactionOpaqueData?: TransactionOpaqueData): Promise<IndexInfo[]> {
+    return await this._route(attachOpaqueData).schemaContentsIndexes(attachOpaqueData, name, transactionOpaqueData);
+  }
+
+  override async indexGet(attachOpaqueData: AttachOpaqueData, schemaName: string, name: string, transactionOpaqueData?: TransactionOpaqueData): Promise<IndexInfo | null> {
+    return await this._route(attachOpaqueData).indexGet(attachOpaqueData, schemaName, name, transactionOpaqueData);
+  }
+
+  override async tableGet(attachOpaqueData: AttachOpaqueData, schemaName: string, name: string, atUnit?: string, atValue?: string, transactionOpaqueData?: TransactionOpaqueData): Promise<TableInfo | null> {
+    return await this._route(attachOpaqueData).tableGet(attachOpaqueData, schemaName, name, atUnit, atValue, transactionOpaqueData);
+  }
+
+  override async tableScanFunctionGet(attachOpaqueData: AttachOpaqueData, schemaName: string, name: string, atUnit?: string, atValue?: string, transactionOpaqueData?: TransactionOpaqueData): Promise<any> {
+    return await this._route(attachOpaqueData).tableScanFunctionGet(attachOpaqueData, schemaName, name, atUnit, atValue, transactionOpaqueData);
+  }
+
+  override async tableScanBranchesGet(attachOpaqueData: AttachOpaqueData, schemaName: string, name: string, atUnit?: string, atValue?: string, transactionOpaqueData?: TransactionOpaqueData): Promise<any> {
+    return await this._route(attachOpaqueData).tableScanBranchesGet(attachOpaqueData, schemaName, name, atUnit, atValue, transactionOpaqueData);
+  }
+
+  override async tableColumnStatisticsGet(attachOpaqueData: AttachOpaqueData, schemaName: string, name: string, transactionOpaqueData?: TransactionOpaqueData): Promise<{ bytes: Uint8Array; cacheMaxAgeSeconds: number | null } | null> {
+    return await this._route(attachOpaqueData).tableColumnStatisticsGet(attachOpaqueData, schemaName, name, transactionOpaqueData);
+  }
+
+  override async viewGet(attachOpaqueData: AttachOpaqueData, schemaName: string, name: string, transactionOpaqueData?: TransactionOpaqueData): Promise<ViewInfo | null> {
+    return await this._route(attachOpaqueData).viewGet(attachOpaqueData, schemaName, name, transactionOpaqueData);
+  }
+
+  override async macroGet(attachOpaqueData: AttachOpaqueData, schemaName: string, name: string, transactionOpaqueData?: TransactionOpaqueData): Promise<MacroInfo | null> {
+    return await this._route(attachOpaqueData).macroGet(attachOpaqueData, schemaName, name, transactionOpaqueData);
+  }
+
+  override async copyFromFormats(attachOpaqueData: AttachOpaqueData, transactionOpaqueData?: TransactionOpaqueData): Promise<CopyFromFormatInfo[]> {
+    return await this._route(attachOpaqueData).copyFromFormats(attachOpaqueData, transactionOpaqueData);
+  }
+
+  override async transactionBegin(attachOpaqueData: AttachOpaqueData): Promise<Uint8Array | null> {
+    return await this._route(attachOpaqueData).transactionBegin(attachOpaqueData);
+  }
+
+  override async transactionCommit(attachOpaqueData: AttachOpaqueData, transactionOpaqueData: TransactionOpaqueData): Promise<void> {
+    await this._route(attachOpaqueData).transactionCommit(attachOpaqueData, transactionOpaqueData);
+  }
+
+  override async transactionRollback(attachOpaqueData: AttachOpaqueData, transactionOpaqueData: TransactionOpaqueData): Promise<void> {
+    await this._route(attachOpaqueData).transactionRollback(attachOpaqueData, transactionOpaqueData);
+  }
+}

package/src/catalog/descriptors.ts

@@ -0,0 +1,164 @@
+// Copyright 2025, 2026 Query Farm LLC - https://query.farm
+// Declarative catalog descriptor types.
+
+import type { VgiSchema, VgiDataType } from "../arrow/index.js";
+import type { VgiFunction } from "../functions/types.js";
+import type { Arguments } from "../arguments/arguments.js";
+import type { ColumnStatistics } from "../util/statistics.js";
+
+export interface SettingDescriptor {
+  name: string;
+  description: string;
+  type: VgiDataType;
+  /** Must be a JS primitive compatible with `type` (string, number, bigint, boolean). */
+  defaultValue?: string | number | bigint | boolean;
+}
+
+export interface ForeignKeyDef {
+  columns: string[];
+  referencedTable: string;
+  referencedColumns: string[];
+  referencedSchema?: string;
+}
+
+export type DefaultValue = string | number | boolean | null;
+
+export interface TableDescriptor {
+  name: string;
+  columns?: VgiSchema;
+  function?: VgiFunction;
+  arguments?: Arguments;
+  notNull?: string[];
+  unique?: string[][];
+  check?: string[];
+  primaryKey?: string[][];
+  foreignKey?: ForeignKeyDef[];
+  defaults?: Record<string, DefaultValue>;
+  /** Per-column comment strings, applied as field metadata `comment`. */
+  columnComments?: Record<string, string>;
+  /**
+   * Generated (virtual) columns: map of column name → SQL expression computed
+   * from other physical columns. Applied as Arrow field metadata
+   * `generated_expression`, which the DuckDB VGI extension reads at table
+   * registration. The backing scan function should not return these columns —
+   * DuckDB evaluates the expressions client-side.
+   */
+  generatedColumns?: Record<string, string>;
+  /**
+   * Per-column statistics for the optimizer. Keys are column names; values
+   * are ColumnStatistics records with typed min/max and Arrow type info.
+   * DuckDB uses these for plan-time filter elimination and join reordering
+   * via the `catalog_table_column_statistics_get` RPC.
+   */
+  statistics?: Record<string, ColumnStatistics>;
+  /**
+   * Cache TTL for this table's column statistics. DuckDB caches the result
+   * of `catalog_table_column_statistics_get` for up to this many seconds.
+   * `undefined`/null means cache indefinitely; `0` means never cache
+   * (always re-fetch).
+   */
+  statisticsCacheMaxAgeSeconds?: number | null;
+  /**
+   * Dotted-path column references that MUST appear in a WHERE expression for
+   * any scan of this table. Top-level names (`"country"`) or struct subfields
+   * (`"bbox.xmin"`, `"nested.outer.inner"`). Empty/undefined (default) means
+   * no enforcement — the zero-cost fast path for every existing table.
+   *
+   * Satisfaction is prefix-based: a present filter on a shorter path satisfies
+   * any required path it is a prefix of. So a whole-struct filter on `bbox`
+   * satisfies all of `bbox.xmin` / `.xmax` / `.ymin` / `.ymax`. The VGI DuckDB
+   * extension's optimizer pass consults this list at bind time and throws a
+   * `BinderException` listing any unsatisfied paths.
+   */
+  requiredFieldFilterPaths?: string[];
+  supportsTimeTravel?: boolean;
+  /**
+   * Inline cardinality estimate/max surfaced through TableInfo. When set,
+   * the C++ extension uses these directly and skips the per-bind
+   * table_function_cardinality RPC. Only meaningful for function-backed
+   * tables; static-column tables don't have a function to call.
+   */
+  inlinedCardinality?: { estimate: bigint; max: bigint };
+  comment?: string;
+  tags?: Record<string, string>;
+}
+
+export interface ViewDescriptor {
+  name: string;
+  definition: string;
+  comment?: string;
+  /**
+   * Per-column comments keyed by the view's output column name. Unlike tables
+   * (whose column comments ride along as Arrow field metadata), a view ships
+   * only its SQL `definition`, so column comments need their own channel. The
+   * C++ extension aligns these by name against the bound output columns and
+   * feeds them into `CreateViewInfo.column_comments_map`; names that don't
+   * match a bound column are ignored.
+   */
+  columnComments?: Record<string, string>;
+  tags?: Record<string, string>;
+}
+
+export interface MacroDescriptor {
+  name: string;
+  macroType: "scalar" | "table";
+  parameters: string[];
+  parameterDefaultValues?: Uint8Array | null;
+  /**
+   * Optional mapping of parameter name to a human/agent-facing description.
+   * Keys must appear in `parameters`. Descriptions flow over the wire via the
+   * macro `arguments_schema`'s `vgi_doc` field metadata (the same channel
+   * functions use for per-argument docs), so the DuckDB extension's
+   * `vgi_function_arguments()` can surface them. Empty/omitted = no docs.
+   */
+  parameterDocs?: Record<string, string>;
+  definition: string;
+  comment?: string;
+  tags?: Record<string, string>;
+}
+
+export interface IndexDescriptor {
+  name: string;
+  tableName: string;
+  expressions: string[];
+  /** UNIQUE / PRIMARY / NONE; defaults to NONE. */
+  constraintType?: "NONE" | "UNIQUE" | "PRIMARY";
+  indexType?: string;
+  options?: Record<string, string>;
+  comment?: string;
+  tags?: Record<string, string>;
+}
+
+export interface SchemaDescriptor {
+  name: string;
+  tables?: TableDescriptor[];
+  views?: ViewDescriptor[];
+  macros?: MacroDescriptor[];
+  functions?: VgiFunction[];
+  indexes?: IndexDescriptor[];
+  comment?: string;
+  tags?: Record<string, string>;
+}
+
+export interface SecretTypeDescriptor {
+  name: string;
+  description: string;
+  schema: VgiSchema; // Field metadata {redact: "true"} marks redacted fields
+}
+
+export interface CatalogDescriptor {
+  name: string;
+  defaultSchema?: string;
+  schemas: SchemaDescriptor[];
+  settings?: SettingDescriptor[];
+  secretTypes?: SecretTypeDescriptor[];
+  comment?: string;
+  tags?: Record<string, string>;
+  /**
+   * Homepage for this catalog — repo, docs, or dataset landing page. Surfaced
+   * through the `catalog_catalogs` discovery record as `CatalogInfo.source_url`,
+   * which DuckDB exposes via `duckdb_databases()`. Optional; omitted/`undefined`
+   * advertises a null `source_url`.
+   */
+  sourceUrl?: string;
+}