@query-farm/vgi 0.38.0 → 0.40.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@query-farm/vgi",
-  "version": "0.38.0",
+  "version": "0.40.0",
   "description": "TypeScript framework for extending DuckDB with user-defined functions via the Vector Gateway Interface (VGI) over Arrow IPC.",
   "license": "SEE LICENSE IN LICENSE",
   "author": "Query Farm LLC (https://query.farm)",
@@ -91,12 +91,12 @@
   },
   "peerDependencies": {
     "@query-farm/apache-arrow": "^21.1.1",
-    "@query-farm/vgi-rpc": "^0.26.0"
+    "@query-farm/vgi-rpc": "^0.27.0"
   },
   "devDependencies": {
     "@duckdb/node-api": "^1.5.5-r.4",
     "@query-farm/apache-arrow": "^21.1.1",
-    "@query-farm/vgi-rpc": "^0.26.0",
+    "@query-farm/vgi-rpc": "^0.27.0",
     "@types/bun": "latest",
     "typescript": "^7.0.2"
   },

package/src/client/client.ts

@@ -42,6 +42,7 @@ import {
   type CatalogAttachResult,
   type SchemaContentsInfo,
   decodeIndexInfo,
+  type IndexInfo,
   type AttachOpaqueData,
   type TransactionOpaqueData,
   decodeCatalogInfo,
@@ -72,6 +73,8 @@ import type {
   OrderByPushdown,
   TablesamplePushdown,
   AttachOptionValue,
+  CatalogSnapshot,
+  LoadCatalogOptions,
 } from "./types.js";
 
 /**
@@ -696,6 +699,9 @@ export class VgiClient {
    * current the worker answers `not_modified` with no schemas. `etag` null
    * means the worker does not revalidate. Each schema's wire `path` is checked
    * against its decoded `SchemaInfo.path`.
+   *
+   * This is the raw RPC. To enumerate a catalog, use `loadCatalog`, which
+   * checks the capability, revalidates and falls back to the per-schema RPCs.
    */
   async catalogContents(
     attachOpaqueData: AttachOpaqueData,
@@ -734,6 +740,132 @@ export class VgiClient {
     };
   }
 
+  /**
+   * Enumerate the whole catalog: every schema with all of its tables, views,
+   * scalar/aggregate/table functions, scalar/table macros and indexes.
+   *
+   * Honors the attach result's `supports_catalog_contents` the way the DuckDB
+   * extension does:
+   *
+   * - advertised (and `useCatalogContents` not false, no transaction): one
+   *   `catalog_contents` call. With `previous.etag` it is conditional
+   *   (`if_none_match`); `not_modified` returns `previous`'s contents.
+   * - not advertised: `catalog_schemas` + per-schema
+   *   `catalog_schema_contents_*` calls -- `catalog_contents` is never sent.
+   *   Kinds whose `estimated_object_count` is exactly 0 are skipped.
+   * - advertised but the call fails (or answers malformed, e.g. a
+   *   `SchemaContents.path` that disagrees with its `SchemaInfo.path`): falls
+   *   back to the per-schema calls and reports the failure in
+   *   `fallback_error`.
+   *
+   * A full answer older than `previous` (a lagging replica) is retried once,
+   * then the per-schema path is used.
+   */
+  async loadCatalog(
+    attach: Pick<CatalogAttachResult, "attach_opaque_data" | "supports_catalog_contents">,
+    opts?: LoadCatalogOptions,
+  ): Promise<CatalogSnapshot> {
+    const attachOpaqueData = attach.attach_opaque_data;
+    const previous = opts?.previous ?? null;
+    const useContents =
+      attach.supports_catalog_contents === true &&
+      opts?.useCatalogContents !== false &&
+      opts?.transactionOpaqueData == null;
+    if (!useContents) {
+      return this._loadCatalogPerSchema(attachOpaqueData, opts?.transactionOpaqueData, null);
+    }
+
+    let fallbackError: string | null = null;
+    const ifNoneMatch = previous?.etag ?? null;
+    const knownVersion = previous?.catalog_version ?? 0;
+    for (let attempt = 0; attempt < 2; attempt++) {
+      let response: Awaited<ReturnType<VgiClient["catalogContents"]>>;
+      try {
+        response = await this.catalogContents(attachOpaqueData, ifNoneMatch);
+      } catch (err) {
+        fallbackError = err instanceof Error ? err.message : String(err);
+        break;
+      }
+      if (response.not_modified) {
+        if (previous == null || ifNoneMatch == null) {
+          fallbackError = "catalog_contents answered not_modified to an unconditional request";
+          break;
+        }
+        return {
+          ...previous,
+          // 0 = unknown: keep the version already known.
+          catalog_version: response.catalog_version !== 0 ? response.catalog_version : previous.catalog_version,
+          etag: response.etag ?? ifNoneMatch,
+          not_modified: true,
+          fallback_error: null,
+        };
+      }
+      // Version adoption: a snapshot at least as new as the known version is
+      // current; 0 means unknown and is accepted. An older one is retried once.
+      if (response.catalog_version !== 0 && knownVersion !== 0 && response.catalog_version < knownVersion) {
+        fallbackError = `catalog_contents snapshot version ${response.catalog_version} is older than the known version ${knownVersion}`;
+        continue;
+      }
+      return {
+        catalog_version: response.catalog_version,
+        etag: response.etag,
+        source: "catalog_contents",
+        not_modified: false,
+        fallback_error: null,
+        schemas: response.schemas,
+      };
+    }
+    return this._loadCatalogPerSchema(attachOpaqueData, undefined, fallbackError);
+  }
+
+  /** `loadCatalog` without `catalog_contents`: catalog_schemas + per-schema calls. */
+  private async _loadCatalogPerSchema(
+    attachOpaqueData: AttachOpaqueData,
+    transactionOpaqueData: TransactionOpaqueData | undefined,
+    fallbackError: string | null,
+  ): Promise<CatalogSnapshot> {
+    const schemas = await this.schemas(attachOpaqueData, transactionOpaqueData);
+    const out: SchemaContentsInfo[] = [];
+    // Sequential on purpose: a stdio RpcClient carries one call at a time.
+    for (const schema of schemas) {
+      const counts = schema.estimated_object_count ?? {};
+      const path = schema.path;
+      // Only an explicit 0 is a guarantee; a missing count is unknown. Decoded
+      // int64 counts are numbers, or bigints beyond the safe range, so compare
+      // numerically.
+      const has = (kind: string) => {
+        const n = (counts as Record<string, unknown>)[kind];
+        return n == null || Number(n) !== 0;
+      };
+      const t = transactionOpaqueData;
+      out.push({
+        schema,
+        tables: has("table") ? await this.schemaContentsTables(attachOpaqueData, path, t) : [],
+        views: has("view") ? await this.schemaContentsViews(attachOpaqueData, path, t) : [],
+        scalar_functions: has("scalar_function")
+          ? await this.schemaContentsFunctions(attachOpaqueData, path, "SCALAR_FUNCTION", t)
+          : [],
+        aggregate_functions: has("aggregate_function")
+          ? await this.schemaContentsFunctions(attachOpaqueData, path, "AGGREGATE_FUNCTION", t)
+          : [],
+        table_functions: has("table_function")
+          ? await this.schemaContentsFunctions(attachOpaqueData, path, "TABLE_FUNCTION", t)
+          : [],
+        scalar_macros: has("macro") ? await this.schemaContentsMacros(attachOpaqueData, path, "SCALAR_MACRO", t) : [],
+        table_macros: has("macro") ? await this.schemaContentsMacros(attachOpaqueData, path, "TABLE_MACRO", t) : [],
+        indexes: has("index") ? await this.schemaContentsIndexes(attachOpaqueData, path, t) : [],
+      });
+    }
+    return {
+      catalog_version: null,
+      etag: null,
+      source: "per_schema",
+      not_modified: false,
+      fallback_error: fallbackError,
+      schemas: out,
+    };
+  }
+
   /** Detach a previously-attached catalog. */
   async catalogDetach(attachOpaqueData: AttachOpaqueData): Promise<void> {
     await this.rpc.call("catalog_detach", { attach_opaque_data: attachOpaqueData });
@@ -955,6 +1087,22 @@ export class VgiClient {
     return deserializeInfoList(inner.items, decodeFunctionInfo);
   }
 
+  /** List indexes in a schema. */
+  async schemaContentsIndexes(
+    attachOpaqueData: AttachOpaqueData,
+    path: string[] | string,
+    transactionOpaqueData?: TransactionOpaqueData,
+  ): Promise<IndexInfo[]> {
+    const result = await this.rpc.call("catalog_schema_contents_indexes", {
+      attach_opaque_data: attachOpaqueData,
+      path: normalizeSchemaPath(path),
+      transaction_opaque_data: transactionOpaqueData ?? null,
+    });
+    if (!result) return [];
+    const inner = unwrapResult(result);
+    return deserializeInfoList(inner.items, decodeIndexInfo);
+  }
+
   /** Get a table by name, or null if not found. */
   async tableGet(
     attachOpaqueData: AttachOpaqueData,

package/src/client/types.ts

@@ -5,6 +5,7 @@ import type { VgiBatch } from "../arrow/index.js";
 import type { Arguments } from "../arguments/arguments.js";
 import type { AttachOptionValue } from "../catalog/attach-options.js";
 import type { BindResponse } from "../protocol/types.js";
+import type { SchemaContentsInfo, TransactionOpaqueData } from "../catalog/interface.js";
 import { OrderByDirection, OrderByNullOrder } from "../protocol/types.js";
 
 export type { AttachOptionValue };
@@ -56,7 +57,7 @@ export interface CatalogAttachOptions {
 export type OnCreateConflict = "error" | "ignore" | "replace";
 
 /** DuckDB catalog function type filter (sent as uppercase wire values). */
-export type CatalogFunctionType = "SCALAR_FUNCTION" | "TABLE_FUNCTION";
+export type CatalogFunctionType = "SCALAR_FUNCTION" | "AGGREGATE_FUNCTION" | "TABLE_FUNCTION";
 
 /** Macro type filter for schema contents listing (uppercase wire values). */
 export type CatalogMacroType = "SCALAR_MACRO" | "TABLE_MACRO";
@@ -151,3 +152,59 @@ export interface TableInOutFunctionOptions {
    */
   hasFinalize?: boolean;
 }
+
+/**
+ * A whole-catalog snapshot returned by `VgiClient.loadCatalog`: every schema
+ * with all of its tables, views, functions, macros and indexes.
+ *
+ * Hold on to it and pass it back as `LoadCatalogOptions.previous` to
+ * revalidate: when it carries an `etag` the client sends that as
+ * `if_none_match`, and a `not_modified` answer returns this snapshot's
+ * contents unchanged.
+ */
+export interface CatalogSnapshot {
+  /**
+   * Version the snapshot was taken at, as reported by `catalog_contents`.
+   * `null` when it was assembled from the per-schema RPCs (which carry no
+   * version); 0 means the worker does not track versions.
+   */
+  catalog_version: number | null;
+  /** Validator to revalidate with; null = the worker does not revalidate. */
+  etag: string | null;
+  /**
+   * How the contents were obtained: one `catalog_contents` call, or
+   * `catalog_schemas` + per-schema `catalog_schema_contents_*` calls.
+   */
+  source: "catalog_contents" | "per_schema";
+  /** True when this load was a `not_modified` revalidation of `previous`. */
+  not_modified: boolean;
+  /**
+   * Set when the catalog advertised `catalog_contents` but the call failed
+   * (or returned a malformed answer) and the client fell back to the
+   * per-schema RPCs: the failure's message.
+   */
+  fallback_error: string | null;
+  /** One entry per schema, parents before children. */
+  schemas: SchemaContentsInfo[];
+}
+
+/** Options for `VgiClient.loadCatalog`. */
+export interface LoadCatalogOptions {
+  /**
+   * A snapshot from an earlier load of the same attach. Its `etag` (if any)
+   * is sent as `if_none_match`; a `not_modified` answer keeps its contents.
+   */
+  previous?: CatalogSnapshot | null;
+  /**
+   * Set false to never call `catalog_contents` (the client-side equivalent of
+   * DuckDB's `SET vgi_catalog_contents = false`). Default true: it is used
+   * whenever the attach result advertised `supports_catalog_contents`.
+   */
+  useCatalogContents?: boolean;
+  /**
+   * Enumerate inside this transaction. `catalog_contents` returns only the
+   * committed catalog (it takes no transaction), so a transaction-scoped load
+   * always uses the per-schema RPCs.
+   */
+  transactionOpaqueData?: TransactionOpaqueData;
+}

package/src/codec/asd.ts

@@ -175,23 +175,27 @@ function normalizeValue(raw: any, type: VgiDataType): any {
   }
 
   if (isMap(type)) {
-    const out: Record<string, string> = {};
+    // Values are decoded with the map's declared value type, like list items
+    // and struct children: a map<utf8, int64> yields numbers (bigint beyond
+    // the safe range), not strings, and a null value stays null.
+    const entries = (type as any).children[0].type as VgiDataType;
+    const valueType = ((entries as any).children as VgiField[])[1].type;
+    const out: Record<string, any> = {};
+    const put = (k: unknown, v: unknown): void => {
+      out[String(k)] = normalizeValue(v, valueType);
+    };
     // arrow-js MapRow is iterable of [k,v]; flechette returns [[k,v],...] arrays.
     if (raw[Symbol.iterator]) {
       for (const entry of raw) {
         if (Array.isArray(entry)) {
-          out[String(entry[0])] = String(entry[1] ?? "");
+          put(entry[0], entry[1]);
         } else if (entry && typeof entry === "object") {
-          const k = entry.key ?? entry[0];
-          const v = entry.value ?? entry[1];
-          out[String(k)] = v == null ? "" : String(v);
+          put(entry.key ?? entry[0], entry.value ?? entry[1]);
         }
       }
     } else if (typeof raw === "object") {
       // Plain-object map (flechette without useMap): Object.entries fallback
-      for (const [k, v] of Object.entries(raw)) {
-        out[String(k)] = v == null ? "" : String(v);
-      }
+      for (const [k, v] of Object.entries(raw)) put(k, v);
     }
     return out;
   }

package/src/functions/copy-to.ts

@@ -75,7 +75,7 @@ export interface CopyToOption {
    * Optional allowed value set, validated worker-side at bind. Mirrors
    * vgi-python's `Arg(..., choices=[...])`.
    */
-  choices?: unknown[];
+  choices?: readonly unknown[];
   /**
    * Optional inclusive lower bound for numeric options, validated worker-side at
    * bind. Mirrors vgi-python's `Arg(..., ge=...)`.

package/src/index.core.ts

@@ -303,6 +303,8 @@ export type {
   CatalogFunctionType as ClientCatalogFunctionType,
   CatalogAttachOptions,
   AttachOptionValue,
+  CatalogSnapshot,
+  LoadCatalogOptions,
 } from "./client/types.js";
 
 // Re-export from vgi-rpc for convenience
@@ -316,6 +318,7 @@ export type { RpcClient, StreamSession, LogMessage } from "@query-farm/vgi-rpc";
 // GrantRefusedError to decline a grant.
 export {
   buildRpcServer,
+  grantKeysFromArgv,
   INTROSPECT_PRINCIPALS_ENV,
   type BuildRpcServerOptions,
   type HostingOptions,
@@ -323,6 +326,7 @@ export {
 } from "./rpc-server.js";
 export {
   AuthUnavailableError,
+  GrantKeys,
   GrantRefusedError,
   IdentityUnavailableError,
   StatusError,

package/src/rpc-server.ts

@@ -22,6 +22,7 @@
 // exactly as it would against a single-protocol server.
 
 import {
+  GrantKeys,
   type GrantMinter,
   IdentityImpl,
   type Protocol,
@@ -117,6 +118,20 @@ export interface HostingOptions {
   /** Maximum age of the caller's authentication for `issue_grant`, in
    *  seconds. vgi-rpc's default when omitted. */
   maxAuthAge?: number;
+
+  /**
+   * Sealed-grant keys (vgi-rpc `GrantKeys`), turning on the framework's own
+   * grants on HTTP: `issue_grant` mints `vgig1.` tokens (unless
+   * {@link mintGrant} is supplied) and the worker's HTTP authentication accepts
+   * them back as bearer credentials, as the grant's owner. When omitted,
+   * `VGI_RPC_GRANT_KEYS` (comma-separated base64, exactly 32 bytes each, first
+   * mints, all verify), `VGI_RPC_GRANT_AUDIENCE` and
+   * `VGI_RPC_GRANT_MAX_TTL_SECONDS` are read; `serveVgiWorker` also accepts
+   * `--grant-key KEY` (repeatable). Unset everywhere, nothing changes. A
+   * malformed key refuses to start. `null` turns grants off regardless of the
+   * environment.
+   */
+  grantKeys?: GrantKeys | null;
 }
 
 /** Server-level options passed through to `VgiRpcServer`. */
@@ -156,6 +171,9 @@ export function buildRpcServer(
     serverId: options.serverId,
     protocols,
     identity,
+    // Resolved here, per transport, rather than by vgi-rpc's own environment
+    // default: identity (and so grants) is hosted on HTTP only.
+    grantKeys: null,
   });
 }
 
@@ -224,10 +242,15 @@ function buildIdentity(
   env: Record<string, string | undefined> | undefined,
 ): IdentityImpl | undefined {
   const { resolveToken, mintGrant } = hosting;
-  if (!resolveToken && !mintGrant) return undefined;
+  // Explicit option, else the environment. Read at server build, so a
+  // malformed key stops the worker before it serves anything.
+  const grantKeys = hosting.grantKeys === undefined ? GrantKeys.fromEnv(env ?? processEnv()) : hosting.grantKeys;
+  if (!resolveToken && !mintGrant && !grantKeys) return undefined;
   return new IdentityImpl({
     resolveToken,
+    // Absent with grant keys configured, vgi-rpc mints sealed grants itself.
     mintGrant,
+    grantKeys: grantKeys ?? undefined,
     // Only meaningful for `introspect_token`; a worker that mints but resolves
     // nothing is not an oracle and needs no allowlist.
     introspectPrincipals: resolveToken ? resolveIntrospectPrincipals(hosting.introspectPrincipals, env) : undefined,
@@ -271,3 +294,34 @@ function resolveIntrospectPrincipals(
   }
   return principals;
 }
+
+function processEnv(): Record<string, string | undefined> | undefined {
+  return (globalThis as { process?: { env?: Record<string, string | undefined> } }).process?.env;
+}
+
+/**
+ * Collect `--grant-key KEY` / `--grant-key=KEY` (repeatable, first mints) from
+ * an argv, as a `GrantKeys` -- or `undefined` when none was passed, so the
+ * environment still applies. Audience and lifetime come from
+ * `VGI_RPC_GRANT_AUDIENCE` / `VGI_RPC_GRANT_MAX_TTL_SECONDS`.
+ *
+ * @throws Error A malformed key: a worker refuses to start.
+ */
+export function grantKeysFromArgv(
+  argv: readonly string[],
+  env: Record<string, string | undefined> | undefined = processEnv(),
+): GrantKeys | undefined {
+  const keys: string[] = [];
+  for (let i = 0; i < argv.length; i++) {
+    const a = argv[i];
+    if (a === "--grant-key") {
+      if (i + 1 >= argv.length) throw new Error("--grant-key requires a value");
+      keys.push(argv[++i]);
+    } else if (a.startsWith("--grant-key=")) {
+      keys.push(a.slice("--grant-key=".length));
+    }
+  }
+  if (keys.length === 0) return undefined;
+  const fromEnv = GrantKeys.fromEnv({ ...(env ?? {}), VGI_RPC_GRANT_KEYS: keys.join(",") });
+  return fromEnv ?? undefined;
+}

package/src/serve-entry.ts

@@ -29,10 +29,11 @@ import {
   observePeerIdentity,
   peerIdentityPrimary,
   type AuthenticateFn,
+  GrantKeys,
   type PeerResolutionOptions,
 } from "@query-farm/vgi-rpc";
 import { createVgiFetch } from "./http/fetch.js";
-import type { HostingOptions } from "./rpc-server.js";
+import { grantKeysFromArgv, type HostingOptions } from "./rpc-server.js";
 
 /** Environment variables `serveVgiWorker` reads. Injectable for testing. */
 export interface ServeEnv {
@@ -40,6 +41,14 @@ export interface ServeEnv {
   VGI_SIGNING_KEY?: string;
   VGI_TOKEN_TTL?: string;
   CORS_ORIGINS?: string;
+  /** Sealed-grant keys: comma-separated base64, exactly 32 bytes each, first mints. */
+  VGI_RPC_GRANT_KEYS?: string;
+  /** Audience bound into sealed grants. */
+  VGI_RPC_GRANT_AUDIENCE?: string;
+  /** Sealed-grant lifetime ceiling, seconds (default 7 days). */
+  VGI_RPC_GRANT_MAX_TTL_SECONDS?: string;
+  /** Principals allowed to call `introspect_token`, comma-separated. */
+  VGI_INTROSPECT_PRINCIPALS?: string;
 }
 
 export const DEFAULT_PORT = 8787;
@@ -94,6 +103,10 @@ export interface ServeVgiWorkerOptions extends HostingOptions {
   quiet?: boolean;
   /** Environment source (default `process.env`). */
   env?: ServeEnv;
+  /** Command-line arguments to read `--grant-key KEY` (repeatable, first mints)
+   *  from. Default `process.argv.slice(2)`. A key here takes precedence over
+   *  `VGI_RPC_GRANT_KEYS`; an explicit {@link grantKeys} over both. */
+  argv?: readonly string[];
   /** Authenticates each request, returning the caller's `AuthContext`.
    *  Omit for an anonymous worker (the default). */
   authenticate?: AuthenticateFn;
@@ -247,6 +260,11 @@ export function createVgiWorkerFetch(
     mintGrant: opts.mintGrant,
     introspectPrincipals: opts.introspectPrincipals,
     maxAuthAge: opts.maxAuthAge,
+    grantKeys:
+      opts.grantKeys !== undefined
+        ? opts.grantKeys
+        : (grantKeysFromArgv(opts.argv ?? process.argv.slice(2), env as Record<string, string | undefined>) ??
+          GrantKeys.fromEnv(env as Record<string, string | undefined>)),
   });
 }
 

package/src/worker-cf-entry.ts

@@ -18,6 +18,7 @@ export type { VgiFetchOptions } from "./http/fetch.js";
 export type { HostingOptions } from "./rpc-server.js";
 export {
   AuthUnavailableError,
+  GrantKeys,
   GrantRefusedError,
   IdentityUnavailableError,
   StatusError,