@query-farm/vgi 0.38.0 → 0.39.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@query-farm/vgi",
-  "version": "0.38.0",
+  "version": "0.39.0",
   "description": "TypeScript framework for extending DuckDB with user-defined functions via the Vector Gateway Interface (VGI) over Arrow IPC.",
   "license": "SEE LICENSE IN LICENSE",
   "author": "Query Farm LLC (https://query.farm)",

package/src/client/client.ts

@@ -42,6 +42,7 @@ import {
   type CatalogAttachResult,
   type SchemaContentsInfo,
   decodeIndexInfo,
+  type IndexInfo,
   type AttachOpaqueData,
   type TransactionOpaqueData,
   decodeCatalogInfo,
@@ -72,6 +73,8 @@ import type {
   OrderByPushdown,
   TablesamplePushdown,
   AttachOptionValue,
+  CatalogSnapshot,
+  LoadCatalogOptions,
 } from "./types.js";
 
 /**
@@ -696,6 +699,9 @@ export class VgiClient {
    * current the worker answers `not_modified` with no schemas. `etag` null
    * means the worker does not revalidate. Each schema's wire `path` is checked
    * against its decoded `SchemaInfo.path`.
+   *
+   * This is the raw RPC. To enumerate a catalog, use `loadCatalog`, which
+   * checks the capability, revalidates and falls back to the per-schema RPCs.
    */
   async catalogContents(
     attachOpaqueData: AttachOpaqueData,
@@ -734,6 +740,132 @@ export class VgiClient {
     };
   }
 
+  /**
+   * Enumerate the whole catalog: every schema with all of its tables, views,
+   * scalar/aggregate/table functions, scalar/table macros and indexes.
+   *
+   * Honors the attach result's `supports_catalog_contents` the way the DuckDB
+   * extension does:
+   *
+   * - advertised (and `useCatalogContents` not false, no transaction): one
+   *   `catalog_contents` call. With `previous.etag` it is conditional
+   *   (`if_none_match`); `not_modified` returns `previous`'s contents.
+   * - not advertised: `catalog_schemas` + per-schema
+   *   `catalog_schema_contents_*` calls -- `catalog_contents` is never sent.
+   *   Kinds whose `estimated_object_count` is exactly 0 are skipped.
+   * - advertised but the call fails (or answers malformed, e.g. a
+   *   `SchemaContents.path` that disagrees with its `SchemaInfo.path`): falls
+   *   back to the per-schema calls and reports the failure in
+   *   `fallback_error`.
+   *
+   * A full answer older than `previous` (a lagging replica) is retried once,
+   * then the per-schema path is used.
+   */
+  async loadCatalog(
+    attach: Pick<CatalogAttachResult, "attach_opaque_data" | "supports_catalog_contents">,
+    opts?: LoadCatalogOptions,
+  ): Promise<CatalogSnapshot> {
+    const attachOpaqueData = attach.attach_opaque_data;
+    const previous = opts?.previous ?? null;
+    const useContents =
+      attach.supports_catalog_contents === true &&
+      opts?.useCatalogContents !== false &&
+      opts?.transactionOpaqueData == null;
+    if (!useContents) {
+      return this._loadCatalogPerSchema(attachOpaqueData, opts?.transactionOpaqueData, null);
+    }
+
+    let fallbackError: string | null = null;
+    const ifNoneMatch = previous?.etag ?? null;
+    const knownVersion = previous?.catalog_version ?? 0;
+    for (let attempt = 0; attempt < 2; attempt++) {
+      let response: Awaited<ReturnType<VgiClient["catalogContents"]>>;
+      try {
+        response = await this.catalogContents(attachOpaqueData, ifNoneMatch);
+      } catch (err) {
+        fallbackError = err instanceof Error ? err.message : String(err);
+        break;
+      }
+      if (response.not_modified) {
+        if (previous == null || ifNoneMatch == null) {
+          fallbackError = "catalog_contents answered not_modified to an unconditional request";
+          break;
+        }
+        return {
+          ...previous,
+          // 0 = unknown: keep the version already known.
+          catalog_version: response.catalog_version !== 0 ? response.catalog_version : previous.catalog_version,
+          etag: response.etag ?? ifNoneMatch,
+          not_modified: true,
+          fallback_error: null,
+        };
+      }
+      // Version adoption: a snapshot at least as new as the known version is
+      // current; 0 means unknown and is accepted. An older one is retried once.
+      if (response.catalog_version !== 0 && knownVersion !== 0 && response.catalog_version < knownVersion) {
+        fallbackError = `catalog_contents snapshot version ${response.catalog_version} is older than the known version ${knownVersion}`;
+        continue;
+      }
+      return {
+        catalog_version: response.catalog_version,
+        etag: response.etag,
+        source: "catalog_contents",
+        not_modified: false,
+        fallback_error: null,
+        schemas: response.schemas,
+      };
+    }
+    return this._loadCatalogPerSchema(attachOpaqueData, undefined, fallbackError);
+  }
+
+  /** `loadCatalog` without `catalog_contents`: catalog_schemas + per-schema calls. */
+  private async _loadCatalogPerSchema(
+    attachOpaqueData: AttachOpaqueData,
+    transactionOpaqueData: TransactionOpaqueData | undefined,
+    fallbackError: string | null,
+  ): Promise<CatalogSnapshot> {
+    const schemas = await this.schemas(attachOpaqueData, transactionOpaqueData);
+    const out: SchemaContentsInfo[] = [];
+    // Sequential on purpose: a stdio RpcClient carries one call at a time.
+    for (const schema of schemas) {
+      const counts = schema.estimated_object_count ?? {};
+      const path = schema.path;
+      // Only an explicit 0 is a guarantee; a missing count is unknown. Decoded
+      // int64 counts are numbers, or bigints beyond the safe range, so compare
+      // numerically.
+      const has = (kind: string) => {
+        const n = (counts as Record<string, unknown>)[kind];
+        return n == null || Number(n) !== 0;
+      };
+      const t = transactionOpaqueData;
+      out.push({
+        schema,
+        tables: has("table") ? await this.schemaContentsTables(attachOpaqueData, path, t) : [],
+        views: has("view") ? await this.schemaContentsViews(attachOpaqueData, path, t) : [],
+        scalar_functions: has("scalar_function")
+          ? await this.schemaContentsFunctions(attachOpaqueData, path, "SCALAR_FUNCTION", t)
+          : [],
+        aggregate_functions: has("aggregate_function")
+          ? await this.schemaContentsFunctions(attachOpaqueData, path, "AGGREGATE_FUNCTION", t)
+          : [],
+        table_functions: has("table_function")
+          ? await this.schemaContentsFunctions(attachOpaqueData, path, "TABLE_FUNCTION", t)
+          : [],
+        scalar_macros: has("macro") ? await this.schemaContentsMacros(attachOpaqueData, path, "SCALAR_MACRO", t) : [],
+        table_macros: has("macro") ? await this.schemaContentsMacros(attachOpaqueData, path, "TABLE_MACRO", t) : [],
+        indexes: has("index") ? await this.schemaContentsIndexes(attachOpaqueData, path, t) : [],
+      });
+    }
+    return {
+      catalog_version: null,
+      etag: null,
+      source: "per_schema",
+      not_modified: false,
+      fallback_error: fallbackError,
+      schemas: out,
+    };
+  }
+
   /** Detach a previously-attached catalog. */
   async catalogDetach(attachOpaqueData: AttachOpaqueData): Promise<void> {
     await this.rpc.call("catalog_detach", { attach_opaque_data: attachOpaqueData });
@@ -955,6 +1087,22 @@ export class VgiClient {
     return deserializeInfoList(inner.items, decodeFunctionInfo);
   }
 
+  /** List indexes in a schema. */
+  async schemaContentsIndexes(
+    attachOpaqueData: AttachOpaqueData,
+    path: string[] | string,
+    transactionOpaqueData?: TransactionOpaqueData,
+  ): Promise<IndexInfo[]> {
+    const result = await this.rpc.call("catalog_schema_contents_indexes", {
+      attach_opaque_data: attachOpaqueData,
+      path: normalizeSchemaPath(path),
+      transaction_opaque_data: transactionOpaqueData ?? null,
+    });
+    if (!result) return [];
+    const inner = unwrapResult(result);
+    return deserializeInfoList(inner.items, decodeIndexInfo);
+  }
+
   /** Get a table by name, or null if not found. */
   async tableGet(
     attachOpaqueData: AttachOpaqueData,

package/src/client/types.ts

@@ -5,6 +5,7 @@ import type { VgiBatch } from "../arrow/index.js";
 import type { Arguments } from "../arguments/arguments.js";
 import type { AttachOptionValue } from "../catalog/attach-options.js";
 import type { BindResponse } from "../protocol/types.js";
+import type { SchemaContentsInfo, TransactionOpaqueData } from "../catalog/interface.js";
 import { OrderByDirection, OrderByNullOrder } from "../protocol/types.js";
 
 export type { AttachOptionValue };
@@ -56,7 +57,7 @@ export interface CatalogAttachOptions {
 export type OnCreateConflict = "error" | "ignore" | "replace";
 
 /** DuckDB catalog function type filter (sent as uppercase wire values). */
-export type CatalogFunctionType = "SCALAR_FUNCTION" | "TABLE_FUNCTION";
+export type CatalogFunctionType = "SCALAR_FUNCTION" | "AGGREGATE_FUNCTION" | "TABLE_FUNCTION";
 
 /** Macro type filter for schema contents listing (uppercase wire values). */
 export type CatalogMacroType = "SCALAR_MACRO" | "TABLE_MACRO";
@@ -151,3 +152,59 @@ export interface TableInOutFunctionOptions {
    */
   hasFinalize?: boolean;
 }
+
+/**
+ * A whole-catalog snapshot returned by `VgiClient.loadCatalog`: every schema
+ * with all of its tables, views, functions, macros and indexes.
+ *
+ * Hold on to it and pass it back as `LoadCatalogOptions.previous` to
+ * revalidate: when it carries an `etag` the client sends that as
+ * `if_none_match`, and a `not_modified` answer returns this snapshot's
+ * contents unchanged.
+ */
+export interface CatalogSnapshot {
+  /**
+   * Version the snapshot was taken at, as reported by `catalog_contents`.
+   * `null` when it was assembled from the per-schema RPCs (which carry no
+   * version); 0 means the worker does not track versions.
+   */
+  catalog_version: number | null;
+  /** Validator to revalidate with; null = the worker does not revalidate. */
+  etag: string | null;
+  /**
+   * How the contents were obtained: one `catalog_contents` call, or
+   * `catalog_schemas` + per-schema `catalog_schema_contents_*` calls.
+   */
+  source: "catalog_contents" | "per_schema";
+  /** True when this load was a `not_modified` revalidation of `previous`. */
+  not_modified: boolean;
+  /**
+   * Set when the catalog advertised `catalog_contents` but the call failed
+   * (or returned a malformed answer) and the client fell back to the
+   * per-schema RPCs: the failure's message.
+   */
+  fallback_error: string | null;
+  /** One entry per schema, parents before children. */
+  schemas: SchemaContentsInfo[];
+}
+
+/** Options for `VgiClient.loadCatalog`. */
+export interface LoadCatalogOptions {
+  /**
+   * A snapshot from an earlier load of the same attach. Its `etag` (if any)
+   * is sent as `if_none_match`; a `not_modified` answer keeps its contents.
+   */
+  previous?: CatalogSnapshot | null;
+  /**
+   * Set false to never call `catalog_contents` (the client-side equivalent of
+   * DuckDB's `SET vgi_catalog_contents = false`). Default true: it is used
+   * whenever the attach result advertised `supports_catalog_contents`.
+   */
+  useCatalogContents?: boolean;
+  /**
+   * Enumerate inside this transaction. `catalog_contents` returns only the
+   * committed catalog (it takes no transaction), so a transaction-scoped load
+   * always uses the per-schema RPCs.
+   */
+  transactionOpaqueData?: TransactionOpaqueData;
+}

package/src/codec/asd.ts

@@ -175,23 +175,27 @@ function normalizeValue(raw: any, type: VgiDataType): any {
   }
 
   if (isMap(type)) {
-    const out: Record<string, string> = {};
+    // Values are decoded with the map's declared value type, like list items
+    // and struct children: a map<utf8, int64> yields numbers (bigint beyond
+    // the safe range), not strings, and a null value stays null.
+    const entries = (type as any).children[0].type as VgiDataType;
+    const valueType = ((entries as any).children as VgiField[])[1].type;
+    const out: Record<string, any> = {};
+    const put = (k: unknown, v: unknown): void => {
+      out[String(k)] = normalizeValue(v, valueType);
+    };
     // arrow-js MapRow is iterable of [k,v]; flechette returns [[k,v],...] arrays.
     if (raw[Symbol.iterator]) {
       for (const entry of raw) {
         if (Array.isArray(entry)) {
-          out[String(entry[0])] = String(entry[1] ?? "");
+          put(entry[0], entry[1]);
         } else if (entry && typeof entry === "object") {
-          const k = entry.key ?? entry[0];
-          const v = entry.value ?? entry[1];
-          out[String(k)] = v == null ? "" : String(v);
+          put(entry.key ?? entry[0], entry.value ?? entry[1]);
         }
       }
     } else if (typeof raw === "object") {
       // Plain-object map (flechette without useMap): Object.entries fallback
-      for (const [k, v] of Object.entries(raw)) {
-        out[String(k)] = v == null ? "" : String(v);
-      }
+      for (const [k, v] of Object.entries(raw)) put(k, v);
     }
     return out;
   }

package/src/functions/copy-to.ts

@@ -75,7 +75,7 @@ export interface CopyToOption {
    * Optional allowed value set, validated worker-side at bind. Mirrors
    * vgi-python's `Arg(..., choices=[...])`.
    */
-  choices?: unknown[];
+  choices?: readonly unknown[];
   /**
    * Optional inclusive lower bound for numeric options, validated worker-side at
    * bind. Mirrors vgi-python's `Arg(..., ge=...)`.

package/src/index.core.ts

@@ -303,6 +303,8 @@ export type {
   CatalogFunctionType as ClientCatalogFunctionType,
   CatalogAttachOptions,
   AttachOptionValue,
+  CatalogSnapshot,
+  LoadCatalogOptions,
 } from "./client/types.js";
 
 // Re-export from vgi-rpc for convenience