@query-farm/vgi 0.38.0 → 0.40.0

package/README.md

@@ -615,6 +615,37 @@ never return `null` for one: both read as a definitive answer. Return `null` onl
 when the store answered and the credential is unknown; throw `GrantRefusedError` to
 decline a grant.
 
+### Sealed grants: minted credentials that log their owner in
+
+`issue_grant` mints a credential meant for unattended automation to present later
+as an ordinary bearer. Configure **grant keys** and the worker both mints and
+accepts them, with no storage and no code:
+
+```console
+$ VGI_RPC_GRANT_KEYS="$(openssl rand -base64 32)" bun run scripts/serve.ts
+# or: bun run scripts/serve.ts --grant-key "$(openssl rand -base64 32)"
+```
+
+- `VGI_RPC_GRANT_KEYS`: comma-separated standard base64, exactly 32 bytes each. The
+  first mints; every key verifies. Rotate by adding the new key first and removing
+  the old one once its grants have expired. `--grant-key KEY` (repeatable) on
+  `serveVgiWorker`, or the `grantKeys` option (`GrantKeys.parse([...])`, required on
+  Cloudflare Workers, where there is no process environment) work too.
+- Optional `VGI_RPC_GRANT_AUDIENCE` (bind grants to this deployment) and
+  `VGI_RPC_GRANT_MAX_TTL_SECONDS` (default 7 days).
+- With keys set, HTTP hosts `issue_grant` (minted by the framework unless you supply
+  `mintGrant`) and the worker's HTTP authentication accepts `vgig1.` grants after
+  your own `authenticate`, as the grant's owner (domain `grant`). A caller
+  authenticated by a grant has no `auth_time`, so it cannot mint another grant.
+- A worker that supplies `resolveToken` also has bearers your `authenticate` did not
+  accept resolved through it (domain `token`); an `AuthUnavailableError` from the hook
+  is a 503 with `Retry-After`, never a 401.
+- Your `authenticate` must throw (not return anonymous) for a bearer it does not
+  recognise, or the chain ends before grants are tried.
+- No key set, nothing changes. A malformed key stops the worker at startup.
+- Sealed grants are not individually revocable: keep the lifetime short; removing a
+  key revokes every grant it minted.
+
 ## Runtimes & entry points
 
 The package ships a backend-agnostic Arrow facade and selects an implementation at

package/dist/client/client.d.ts

@@ -1,9 +1,9 @@
 import { type VgiBatch } from "../arrow/index.js";
 import type { RpcClient } from "@query-farm/vgi-rpc";
-import { type SchemaInfo, type TableInfo, type ViewInfo, type FunctionInfo, type MacroInfo, type MacroType, type CatalogAttachResult, type SchemaContentsInfo, type AttachOpaqueData, type TransactionOpaqueData, type CatalogInfo, type ScanFunctionResult } from "../catalog/interface.js";
+import { type SchemaInfo, type TableInfo, type ViewInfo, type FunctionInfo, type MacroInfo, type MacroType, type CatalogAttachResult, type SchemaContentsInfo, type IndexInfo, type AttachOpaqueData, type TransactionOpaqueData, type CatalogInfo, type ScanFunctionResult } from "../catalog/interface.js";
 import { VgiClientError } from "./errors.js";
 export { VgiClientError };
-import type { VgiClientOptions, TableFunctionOptions, ScalarFunctionOptions, TableInOutFunctionOptions, OnCreateConflict, CatalogFunctionType, CatalogMacroType, CatalogAttachOptions, AttachOptionValue } from "./types.js";
+import type { VgiClientOptions, TableFunctionOptions, ScalarFunctionOptions, TableInOutFunctionOptions, OnCreateConflict, CatalogFunctionType, CatalogMacroType, CatalogAttachOptions, AttachOptionValue, CatalogSnapshot, LoadCatalogOptions } from "./types.js";
 /**
  * High-level client for calling VGI worker functions and catalog API.
  *
@@ -103,6 +103,9 @@ export declare class VgiClient {
      * current the worker answers `not_modified` with no schemas. `etag` null
      * means the worker does not revalidate. Each schema's wire `path` is checked
      * against its decoded `SchemaInfo.path`.
+     *
+     * This is the raw RPC. To enumerate a catalog, use `loadCatalog`, which
+     * checks the capability, revalidates and falls back to the per-schema RPCs.
      */
     catalogContents(attachOpaqueData: AttachOpaqueData, ifNoneMatch?: string | null): Promise<{
         catalog_version: number;
@@ -110,6 +113,30 @@ export declare class VgiClient {
         not_modified: boolean;
         schemas: SchemaContentsInfo[];
     }>;
+    /**
+     * Enumerate the whole catalog: every schema with all of its tables, views,
+     * scalar/aggregate/table functions, scalar/table macros and indexes.
+     *
+     * Honors the attach result's `supports_catalog_contents` the way the DuckDB
+     * extension does:
+     *
+     * - advertised (and `useCatalogContents` not false, no transaction): one
+     *   `catalog_contents` call. With `previous.etag` it is conditional
+     *   (`if_none_match`); `not_modified` returns `previous`'s contents.
+     * - not advertised: `catalog_schemas` + per-schema
+     *   `catalog_schema_contents_*` calls -- `catalog_contents` is never sent.
+     *   Kinds whose `estimated_object_count` is exactly 0 are skipped.
+     * - advertised but the call fails (or answers malformed, e.g. a
+     *   `SchemaContents.path` that disagrees with its `SchemaInfo.path`): falls
+     *   back to the per-schema calls and reports the failure in
+     *   `fallback_error`.
+     *
+     * A full answer older than `previous` (a lagging replica) is retried once,
+     * then the per-schema path is used.
+     */
+    loadCatalog(attach: Pick<CatalogAttachResult, "attach_opaque_data" | "supports_catalog_contents">, opts?: LoadCatalogOptions): Promise<CatalogSnapshot>;
+    /** `loadCatalog` without `catalog_contents`: catalog_schemas + per-schema calls. */
+    private _loadCatalogPerSchema;
     /** Detach a previously-attached catalog. */
     catalogDetach(attachOpaqueData: AttachOpaqueData): Promise<void>;
     /**
@@ -153,6 +180,8 @@ export declare class VgiClient {
     schemaContentsViews(attachOpaqueData: AttachOpaqueData, path: string[] | string, transactionOpaqueData?: TransactionOpaqueData): Promise<ViewInfo[]>;
     /** List functions in a schema, filtered by type. */
     schemaContentsFunctions(attachOpaqueData: AttachOpaqueData, path: string[] | string, type: CatalogFunctionType, transactionOpaqueData?: TransactionOpaqueData): Promise<FunctionInfo[]>;
+    /** List indexes in a schema. */
+    schemaContentsIndexes(attachOpaqueData: AttachOpaqueData, path: string[] | string, transactionOpaqueData?: TransactionOpaqueData): Promise<IndexInfo[]>;
     /** Get a table by name, or null if not found. */
     tableGet(attachOpaqueData: AttachOpaqueData, schemaPath: string[] | string, name: string, transactionOpaqueData?: TransactionOpaqueData): Promise<TableInfo | null>;
     /** Create a new table. */

package/dist/client/types.d.ts

@@ -2,6 +2,7 @@ import type { VgiBatch } from "../arrow/index.js";
 import type { Arguments } from "../arguments/arguments.js";
 import type { AttachOptionValue } from "../catalog/attach-options.js";
 import type { BindResponse } from "../protocol/types.js";
+import type { SchemaContentsInfo, TransactionOpaqueData } from "../catalog/interface.js";
 import { OrderByDirection, OrderByNullOrder } from "../protocol/types.js";
 export type { AttachOptionValue };
 export { OrderByDirection, OrderByNullOrder };
@@ -46,7 +47,7 @@ export interface CatalogAttachOptions {
 /** Conflict resolution strategy for create operations. */
 export type OnCreateConflict = "error" | "ignore" | "replace";
 /** DuckDB catalog function type filter (sent as uppercase wire values). */
-export type CatalogFunctionType = "SCALAR_FUNCTION" | "TABLE_FUNCTION";
+export type CatalogFunctionType = "SCALAR_FUNCTION" | "AGGREGATE_FUNCTION" | "TABLE_FUNCTION";
 /** Macro type filter for schema contents listing (uppercase wire values). */
 export type CatalogMacroType = "SCALAR_MACRO" | "TABLE_MACRO";
 /** Options for constructing a VgiClient. */
@@ -136,3 +137,57 @@ export interface TableInOutFunctionOptions {
      */
     hasFinalize?: boolean;
 }
+/**
+ * A whole-catalog snapshot returned by `VgiClient.loadCatalog`: every schema
+ * with all of its tables, views, functions, macros and indexes.
+ *
+ * Hold on to it and pass it back as `LoadCatalogOptions.previous` to
+ * revalidate: when it carries an `etag` the client sends that as
+ * `if_none_match`, and a `not_modified` answer returns this snapshot's
+ * contents unchanged.
+ */
+export interface CatalogSnapshot {
+    /**
+     * Version the snapshot was taken at, as reported by `catalog_contents`.
+     * `null` when it was assembled from the per-schema RPCs (which carry no
+     * version); 0 means the worker does not track versions.
+     */
+    catalog_version: number | null;
+    /** Validator to revalidate with; null = the worker does not revalidate. */
+    etag: string | null;
+    /**
+     * How the contents were obtained: one `catalog_contents` call, or
+     * `catalog_schemas` + per-schema `catalog_schema_contents_*` calls.
+     */
+    source: "catalog_contents" | "per_schema";
+    /** True when this load was a `not_modified` revalidation of `previous`. */
+    not_modified: boolean;
+    /**
+     * Set when the catalog advertised `catalog_contents` but the call failed
+     * (or returned a malformed answer) and the client fell back to the
+     * per-schema RPCs: the failure's message.
+     */
+    fallback_error: string | null;
+    /** One entry per schema, parents before children. */
+    schemas: SchemaContentsInfo[];
+}
+/** Options for `VgiClient.loadCatalog`. */
+export interface LoadCatalogOptions {
+    /**
+     * A snapshot from an earlier load of the same attach. Its `etag` (if any)
+     * is sent as `if_none_match`; a `not_modified` answer keeps its contents.
+     */
+    previous?: CatalogSnapshot | null;
+    /**
+     * Set false to never call `catalog_contents` (the client-side equivalent of
+     * DuckDB's `SET vgi_catalog_contents = false`). Default true: it is used
+     * whenever the attach result advertised `supports_catalog_contents`.
+     */
+    useCatalogContents?: boolean;
+    /**
+     * Enumerate inside this transaction. `catalog_contents` returns only the
+     * committed catalog (it takes no transaction), so a transaction-scoped load
+     * always uses the per-schema RPCs.
+     */
+    transactionOpaqueData?: TransactionOpaqueData;
+}

package/dist/client-entry.js

@@ -2388,21 +2388,23 @@ function normalizeValue(raw, type) {
     return out;
   }
   if (isMap(type)) {
+    const entries = type.children[0].type;
+    const valueType = entries.children[1].type;
     const out = {};
+    const put = (k, v) => {
+      out[String(k)] = normalizeValue(v, valueType);
+    };
     if (raw[Symbol.iterator]) {
       for (const entry of raw) {
         if (Array.isArray(entry)) {
-          out[String(entry[0])] = String(entry[1] ?? "");
+          put(entry[0], entry[1]);
         } else if (entry && typeof entry === "object") {
-          const k = entry.key ?? entry[0];
-          const v = entry.value ?? entry[1];
-          out[String(k)] = v == null ? "" : String(v);
+          put(entry.key ?? entry[0], entry.value ?? entry[1]);
         }
       }
     } else if (typeof raw === "object") {
-      for (const [k, v] of Object.entries(raw)) {
-        out[String(k)] = v == null ? "" : String(v);
-      }
+      for (const [k, v] of Object.entries(raw))
+        put(k, v);
     }
     return out;
   }
@@ -3973,6 +3975,84 @@ class VgiClient {
       schemas
     };
   }
+  async loadCatalog(attach, opts) {
+    const attachOpaqueData = attach.attach_opaque_data;
+    const previous = opts?.previous ?? null;
+    const useContents = attach.supports_catalog_contents === true && opts?.useCatalogContents !== false && opts?.transactionOpaqueData == null;
+    if (!useContents) {
+      return this._loadCatalogPerSchema(attachOpaqueData, opts?.transactionOpaqueData, null);
+    }
+    let fallbackError = null;
+    const ifNoneMatch = previous?.etag ?? null;
+    const knownVersion = previous?.catalog_version ?? 0;
+    for (let attempt = 0;attempt < 2; attempt++) {
+      let response;
+      try {
+        response = await this.catalogContents(attachOpaqueData, ifNoneMatch);
+      } catch (err) {
+        fallbackError = err instanceof Error ? err.message : String(err);
+        break;
+      }
+      if (response.not_modified) {
+        if (previous == null || ifNoneMatch == null) {
+          fallbackError = "catalog_contents answered not_modified to an unconditional request";
+          break;
+        }
+        return {
+          ...previous,
+          catalog_version: response.catalog_version !== 0 ? response.catalog_version : previous.catalog_version,
+          etag: response.etag ?? ifNoneMatch,
+          not_modified: true,
+          fallback_error: null
+        };
+      }
+      if (response.catalog_version !== 0 && knownVersion !== 0 && response.catalog_version < knownVersion) {
+        fallbackError = `catalog_contents snapshot version ${response.catalog_version} is older than the known version ${knownVersion}`;
+        continue;
+      }
+      return {
+        catalog_version: response.catalog_version,
+        etag: response.etag,
+        source: "catalog_contents",
+        not_modified: false,
+        fallback_error: null,
+        schemas: response.schemas
+      };
+    }
+    return this._loadCatalogPerSchema(attachOpaqueData, undefined, fallbackError);
+  }
+  async _loadCatalogPerSchema(attachOpaqueData, transactionOpaqueData, fallbackError) {
+    const schemas = await this.schemas(attachOpaqueData, transactionOpaqueData);
+    const out = [];
+    for (const schema of schemas) {
+      const counts = schema.estimated_object_count ?? {};
+      const path = schema.path;
+      const has = (kind) => {
+        const n = counts[kind];
+        return n == null || Number(n) !== 0;
+      };
+      const t = transactionOpaqueData;
+      out.push({
+        schema,
+        tables: has("table") ? await this.schemaContentsTables(attachOpaqueData, path, t) : [],
+        views: has("view") ? await this.schemaContentsViews(attachOpaqueData, path, t) : [],
+        scalar_functions: has("scalar_function") ? await this.schemaContentsFunctions(attachOpaqueData, path, "SCALAR_FUNCTION", t) : [],
+        aggregate_functions: has("aggregate_function") ? await this.schemaContentsFunctions(attachOpaqueData, path, "AGGREGATE_FUNCTION", t) : [],
+        table_functions: has("table_function") ? await this.schemaContentsFunctions(attachOpaqueData, path, "TABLE_FUNCTION", t) : [],
+        scalar_macros: has("macro") ? await this.schemaContentsMacros(attachOpaqueData, path, "SCALAR_MACRO", t) : [],
+        table_macros: has("macro") ? await this.schemaContentsMacros(attachOpaqueData, path, "TABLE_MACRO", t) : [],
+        indexes: has("index") ? await this.schemaContentsIndexes(attachOpaqueData, path, t) : []
+      });
+    }
+    return {
+      catalog_version: null,
+      etag: null,
+      source: "per_schema",
+      not_modified: false,
+      fallback_error: fallbackError,
+      schemas: out
+    };
+  }
   async catalogDetach(attachOpaqueData) {
     await this.rpc.call("catalog_detach", { attach_opaque_data: attachOpaqueData });
   }
@@ -4111,6 +4191,17 @@ class VgiClient {
     const inner = unwrapResult(result);
     return deserializeInfoList(inner.items, decodeFunctionInfo);
   }
+  async schemaContentsIndexes(attachOpaqueData, path, transactionOpaqueData) {
+    const result = await this.rpc.call("catalog_schema_contents_indexes", {
+      attach_opaque_data: attachOpaqueData,
+      path: normalizeSchemaPath(path),
+      transaction_opaque_data: transactionOpaqueData ?? null
+    });
+    if (!result)
+      return [];
+    const inner = unwrapResult(result);
+    return deserializeInfoList(inner.items, decodeIndexInfo);
+  }
   async tableGet(attachOpaqueData, schemaPath, name, transactionOpaqueData) {
     const result = await this.rpc.call("catalog_table_get", {
       attach_opaque_data: attachOpaqueData,
@@ -4549,4 +4640,4 @@ export {
   typeSignature
 };
 
-//# debugId=AD4326E5920F408164756E2164756E21
+//# debugId=A700197CD6EACFA764756E2164756E21