@query-farm/vgi 0.24.1 → 0.25.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@query-farm/vgi",
-  "version": "0.24.1",
+  "version": "0.25.0",
   "description": "TypeScript framework for extending DuckDB with user-defined functions via the Vector Gateway Interface (VGI) over Arrow IPC.",
   "license": "SEE LICENSE IN LICENSE",
   "author": "Query Farm LLC (https://query.farm)",
@@ -90,12 +90,12 @@
   },
   "peerDependencies": {
     "@query-farm/apache-arrow": "^21.1.1",
-    "@query-farm/vgi-rpc": ">=0.18.1 <0.19.0"
+    "@query-farm/vgi-rpc": ">=0.19.0 <0.20.0"
   },
   "devDependencies": {
     "@duckdb/node-api": "^1.5.2-r.1",
     "@query-farm/apache-arrow": "^21.1.1",
-    "@query-farm/vgi-rpc": "^0.18.1",
+    "@query-farm/vgi-rpc": "^0.19.0",
     "@types/bun": "latest",
     "typescript": "^6.0.3"
   },

package/src/catalog/attach-option.ts

@@ -17,6 +17,11 @@
 //     default_value: Binary nullable -- serialized single-row RecordBatch with
 //                                       the default value under "value", or
 //                                       null when no default is declared
+//     required: Bool nullable -- the caller must supply this option at ATTACH
+//                                time. Nullable and appended LAST so a peer
+//                                that predates the column reads the batch by
+//                                name and simply doesn't see it; absent and
+//                                explicit-null both mean "not required".
 //
 // The extension reads the outer `attach_option_specs: list<binary>` column
 // from CatalogInfo and validates user-supplied ATTACH options against the
@@ -28,10 +33,15 @@ import {
   field,
   utf8,
   binary,
+  bool,
   batchFromColumns,
+  iterRows,
   serializeBatch,
+  deserializeBatch,
   serializeSchema,
+  deserializeSchema,
 } from "../arrow/index.js";
+import { toUint8Array } from "../util/bytes.js";
 
 /**
  * Declarative spec for a single attach-time option.
@@ -55,6 +65,16 @@ export interface AttachOptionSpec {
    * the options dict delivered to attach()).
    */
   default?: unknown;
+  /**
+   * The caller must supply this option at ATTACH time. A catalog that cannot
+   * be attached without it advertises that fact at discovery, so a client can
+   * say so before attempting the attach rather than surfacing a failure that
+   * reads like an empty catalog.
+   *
+   * Mutually exclusive with `default` — an option that falls back to a value
+   * is by definition satisfiable without the caller.
+   */
+  required?: boolean;
 }
 
 const SPEC_SCHEMA = makeSchema([
@@ -62,6 +82,7 @@ const SPEC_SCHEMA = makeSchema([
   field("description", utf8(), false),
   field("type", binary(), false),
   field("default_value", binary(), true),
+  field("required", bool(), true),
 ]);
 
 /**
@@ -69,6 +90,17 @@ const SPEC_SCHEMA = makeSchema([
  * (one IPC-serialized RecordBatch with a single row).
  */
 export function serializeAttachOptionSpec(spec: AttachOptionSpec): Uint8Array {
+  // Mirrors AttachOptionSpec.__post_init__ on the Python side: an option that
+  // falls back to a value is always satisfiable without the caller, so the
+  // combination is a declaration bug rather than a runtime condition.
+  if (spec.required && spec.default !== undefined && spec.default !== null) {
+    throw new Error(
+      `Attach option '${spec.name}' is required but also declares a default ` +
+        `(${JSON.stringify(spec.default)}); an option with a default is always ` +
+        "satisfiable without the caller. Drop one.",
+    );
+  }
+
   // `type` is encoded as a serialized Arrow Schema with one field named
   // "value" of the option's DataType. This lets the extension peek at the
   // logical type without a separate enum.
@@ -91,6 +123,9 @@ export function serializeAttachOptionSpec(spec: AttachOptionSpec): Uint8Array {
       description: [spec.description],
       type: [typeBytes],
       default_value: [defaultBytes],
+      // Written explicitly rather than left null so a reader sees `false`, not
+      // NULL, for an option that simply isn't required.
+      required: [spec.required ?? false],
     },
     SPEC_SCHEMA,
   );
@@ -105,3 +140,96 @@ export function serializeAttachOptionSpecs(
 ): Uint8Array[] {
   return Array.from(specs, serializeAttachOptionSpec);
 }
+
+/**
+ * Deserialize one AttachOptionSpec from the wire format above.
+ *
+ * Reads by column name, so a spec written by a peer that predates the
+ * `required` column deserializes with `required: false` rather than failing —
+ * the same tolerance the Python and C++ readers have.
+ */
+export function deserializeAttachOptionSpec(
+  bytes: Uint8Array,
+): AttachOptionSpec {
+  const row = [...iterRows(deserializeBatch(bytes))][0] as
+    | Record<string, unknown>
+    | undefined;
+  if (row === undefined) {
+    throw new Error("Cannot deserialize AttachOptionSpec from an empty batch");
+  }
+
+  // `type` is a serialized Arrow Schema whose single "value" field carries the
+  // option's DataType.
+  const typeSchema = deserializeSchema(toUint8Array(row.type as Uint8Array));
+  const type = typeSchema.fields[0]!.type;
+
+  let value: unknown;
+  const defaultBytes = row.default_value;
+  if (defaultBytes != null) {
+    const asBytes = toUint8Array(defaultBytes as Uint8Array);
+    if (asBytes.length > 0) {
+      const defaultRow = [...iterRows(deserializeBatch(asBytes))][0] as
+        | Record<string, unknown>
+        | undefined;
+      value = defaultRow?.value;
+    }
+  }
+
+  return {
+    name: String(row.name),
+    description: String(row.description ?? ""),
+    type,
+    ...(value === undefined || value === null ? {} : { default: value }),
+    required: row.required === true,
+  };
+}
+
+/** Deserialize many specs — the shape `CatalogInfo.attach_option_specs` holds. */
+export function deserializeAttachOptionSpecs(
+  specs: Iterable<Uint8Array>,
+): AttachOptionSpec[] {
+  return Array.from(specs, deserializeAttachOptionSpec);
+}
+
+/**
+ * Raised when `attach` omits options declared `required`.
+ *
+ * Carries the option names in `missing` so a caller can act on them without
+ * parsing the message. Mirrors `MissingAttachOptionsError` on the Python side,
+ * message included — the extension's integration suite matches on its text.
+ */
+export class MissingAttachOptionsError extends Error {
+  readonly missing: string[];
+
+  constructor(catalogName: string, missing: string[]) {
+    const joined = missing.map((name) => `'${name}'`).join(", ");
+    super(
+      `Catalog '${catalogName}' cannot be attached without the required ` +
+        `option${missing.length > 1 ? "s" : ""} ${joined}.`,
+    );
+    this.name = "MissingAttachOptionsError";
+    this.missing = missing;
+  }
+}
+
+/**
+ * Throw if any `required` spec has no corresponding entry in `options`.
+ *
+ * Option names are matched case-insensitively, mirroring DuckDB's handling of
+ * ATTACH option keys.
+ *
+ * @throws {MissingAttachOptionsError} If a required option was not supplied.
+ */
+export function validateRequiredAttachOptions(
+  catalogName: string,
+  specs: Iterable<AttachOptionSpec>,
+  options: Record<string, unknown>,
+): void {
+  const supplied = new Set(Object.keys(options).map((key) => key.toLowerCase()));
+  const missing = Array.from(specs)
+    .filter((spec) => spec.required && !supplied.has(spec.name.toLowerCase()))
+    .map((spec) => spec.name);
+  if (missing.length > 0) {
+    throw new MissingAttachOptionsError(catalogName, missing);
+  }
+}

package/src/catalog/read-only.ts

@@ -26,6 +26,7 @@ import type { FunctionRegistry } from "../functions/registry.js";
 import { argumentSpecsToSchema, macroArgumentsSchema } from "../arguments/argument-spec.js";
 import { serializeSchema, serializeBatch, emptyBatch, batchFromColumns } from "../util/arrow/index.js";
 import { resolveMetadata } from "../metadata/resolve.js";
+import { type AttachOptionSpec, validateRequiredAttachOptions } from "./attach-option.js";
 import { FunctionStability, NullHandling, OrderPreservation, DEFAULT_MAX_WORKERS } from "../types.js";
 import { Arguments } from "../arguments/arguments.js";
 
@@ -58,6 +59,19 @@ export class ReadOnlyCatalogInterface extends CatalogInterface {
     return [this._descriptor.name];
   }
 
+  /**
+   * Attach-time options this catalog declares, for the named catalog.
+   *
+   * Empty by default — the declarative descriptor path carries no attach
+   * options. A subclass that advertises specs in `catalogsInfo()` should
+   * return the same specs here so `attach()` enforces the `required` ones it
+   * advertised. Takes the catalog name because one interface may serve
+   * several catalogs with different option sets.
+   */
+  attachOptionSpecs(_name: string): AttachOptionSpec[] {
+    return [];
+  }
+
   // Advertise the catalog's discovery record. The declarative path carries no
   // version metadata or attach option specs, but it does surface the optional
   // `source_url` from the descriptor so it flows through `catalog_catalogs`
@@ -82,6 +96,12 @@ export class ReadOnlyCatalogInterface extends CatalogInterface {
     if (!this.catalogs().includes(name)) {
       throw new Error(`No worker handles catalog '${name}'`);
     }
+    // Refuse an attach that omits a declared-required option, so the caller
+    // gets the option name rather than a catalog that looks mysteriously
+    // empty. A subclass that overrides `attach` outright bypasses this and
+    // should call `validateRequiredAttachOptions` itself — mirrors
+    // CatalogInterface.catalog_attach on the Python side.
+    validateRequiredAttachOptions(name, this.attachOptionSpecs(name), options ?? {});
 
     const attachOpaqueData = new Uint8Array(16);
     crypto.getRandomValues(attachOpaqueData);

package/src/client-entry.ts

@@ -103,6 +103,17 @@ export {
   deserializeBatch,
 } from "./util/arrow/index.js";
 
+// Arrow type identification — lets a consumer render a DataType for display
+// without reaching into the backend implementation.
+export { TypeId, typeSignature } from "./arrow/predicates.js";
+
+// Attach options — the page-facing decoder for CatalogInfo.attach_option_specs.
+export {
+  type AttachOptionSpec,
+  deserializeAttachOptionSpec,
+  deserializeAttachOptionSpecs,
+} from "./catalog/attach-option.js";
+
 // Byte utilities
 export { toUint8Array } from "./util/bytes.js";
 

package/src/http/fetch.ts

@@ -13,7 +13,7 @@
 import { createHttpHandler, unpackStateToken, type Protocol } from "@query-farm/vgi-rpc";
 import { arrowStateSerializer } from "../protocol/state-serializer.js";
 import { buildVgiProtocol, type ProtocolConfig } from "../protocol/dispatch.js";
-import type { LandingDescribeProvider } from "./describe-json.js";
+import type { LandingInfo } from "@query-farm/vgi-rpc";
 
 export interface VgiFetchOptions {
   /** Wire-protocol config (registry + catalogInterface). */
@@ -36,10 +36,10 @@ export interface VgiFetchOptions {
   /** Public source-repository URL, surfaced on the landing page. */
   repositoryUrl?: string;
   /** Enables the standardized VGI landing surface: `GET /` serves the shared
-   *  landing.html, `GET /describe.json` the catalog contract, and
-   *  `GET /describe/{catalog}/{schema}/{table}.json` lazy per-object columns.
-   *  Build one with `createLandingDescribe(catalogInterface, workerInfo)`. */
-  landingDescribe?: LandingDescribeProvider;
+   *  landing.html plus a JSON status document carrying this identity, and
+   *  `GET /vgi-client.js` serves the browser client build the page reads the
+   *  catalog with. */
+  landingInfo?: LandingInfo;
 }
 
 /**
@@ -74,7 +74,7 @@ export function createVgiFetch(opts: VgiFetchOptions): (req: Request) => Promise
     stateSerializer: arrowStateSerializer,
     corsOrigins: opts.corsOrigins,
     repositoryUrl: opts.repositoryUrl,
-    landingDescribe: opts.landingDescribe,
+    landingInfo: opts.landingInfo,
   });
   return async (req: Request) => handler(req);
 }

package/src/index.core.ts

@@ -153,25 +153,18 @@ export { arrowStateSerializer, EXCHANGE_STATE_SCHEMA, serializeUserState, deseri
 // Protocol assembly (for embedding the worker dispatcher in a custom HTTP server).
 export { buildVgiProtocol, type ProtocolConfig } from "./protocol/dispatch.js";
 
-// VGI HTTP landing surface producer (describe.json contract). Pass
-// `createLandingDescribe(catalogInterface, workerInfo)` to `createHttpHandler`'s
-// `landingDescribe` option. See docs/http-landing-contract.md.
-export {
-  buildColumnsJson,
-  buildDescribeJson,
-  createLandingDescribe,
-  CUPOLA_BASE,
-  LANDING_SCHEMA_VERSION,
-  type LandingDescribeProvider,
-  type WorkerDescribeInfo,
-} from "./http/describe-json.js";
+// VGI HTTP landing surface. The shared landing.html reads catalog metadata by
+// speaking the protocol through the client bundle the worker serves beside it,
+// so a worker supplies only its own identity via `createHttpHandler`'s
+// `landingInfo` option.
+export type { LandingInfo } from "@query-farm/vgi-rpc";
 
 // Wire schemas — needed by workers that synthesize TableInfo.bind_result /
 // TableInfo.scan_function bytes to short-circuit per-scan RPCs.
 export { BindResultSchema, ScanFunctionResultSchema } from "./generated/vgi-protocol-schemas.js";
 
 // Attach-option specs (for catalogs that advertise typed ATTACH options).
-export { type AttachOptionSpec, serializeAttachOptionSpec, serializeAttachOptionSpecs } from "./catalog/attach-option.js";
+export { type AttachOptionSpec, serializeAttachOptionSpec, serializeAttachOptionSpecs, deserializeAttachOptionSpec, deserializeAttachOptionSpecs } from "./catalog/attach-option.js";
 
 // Column statistics (for inlining on TableInfo.column_statistics).
 export { type ColumnStatistics, serializeColumnStatistics } from "./util/statistics.js";

package/src/serve-entry.ts

@@ -23,7 +23,6 @@
 
 import type { CatalogInterface } from "./catalog/interface.js";
 import type { FunctionRegistry } from "./functions/registry.js";
-import { createLandingDescribe } from "./http/describe-json.js";
 import { createVgiFetch } from "./http/fetch.js";
 
 /** Environment variables `serveVgiWorker` reads. Injectable for testing. */
@@ -192,11 +191,7 @@ export function createVgiWorkerFetch(
     serverId: opts.serverId ?? opts.name,
     corsOrigins,
     repositoryUrl: opts.repositoryUrl,
-    landingDescribe: createLandingDescribe(opts.catalogInterface, {
-      name: opts.name,
-      doc: opts.doc,
-      version: opts.version,
-    }),
+    landingInfo: { name: opts.name, doc: opts.doc, version: opts.version },
   });
 }
 
@@ -225,7 +220,7 @@ export function serveVgiWorker(opts: ServeVgiWorkerOptions): VgiHttpServer {
     const base = `http://localhost:${server.port}`;
     console.log(`${opts.name} VGI HTTP worker listening on ${base}`);
     console.log(`  landing page   ${base}/`);
-    console.log(`  describe.json  ${base}/describe.json`);
+    console.log(`  client bundle  ${base}/vgi-client.js`);
     console.log(`  health         ${base}/health`);
     console.log(`  attach         ATTACH '${opts.name}' AS ${opts.name} (TYPE vgi, LOCATION '${base}');`);
   }

package/src/worker-cf-entry.ts

@@ -53,10 +53,10 @@ export type {
 
 export type { ProtocolConfig } from "./protocol/dispatch.js";
 
-// Landing surface. CF workers pass the provider as `landingDescribe` so
-// `GET /` serves the standardized VGI landing page for the worker's catalog.
-export { createLandingDescribe } from "./http/describe-json.js";
-export type { LandingDescribeProvider, WorkerDescribeInfo } from "./http/describe-json.js";
+// Landing surface. CF workers pass their identity as `landingInfo` so `GET /`
+// serves the standardized VGI landing page, and `GET /vgi-client.js` the
+// browser client build it reads the catalog with.
+export type { LandingInfo } from "@query-farm/vgi-rpc";
 
 // Backend-agnostic Arrow facade. Same source resolves to arrow-js on
 // Node/Bun and flechette under workerd/worker/browser via the package's

package/dist/http/describe-json.d.ts

@@ -1,62 +0,0 @@
-import type { CatalogInterface } from "../catalog/interface.js";
-export declare const LANDING_SCHEMA_VERSION = 1;
-export declare const CUPOLA_BASE = "https://cupola.query-farm.services";
-/**
- * Producer for the VGI landing surface, passed to `createHttpHandler`'s
- * `landingDescribe` option. Declared locally (structurally identical to
- * `@query-farm/vgi-rpc`'s `LandingDescribeProvider`) so this package does not
- * hard-couple to a specific vgi-rpc version.
- */
-export interface LandingDescribeProvider {
-    describe(ctx: {
-        serverId: string;
-        oauth: boolean;
-    }): unknown | Promise<unknown>;
-    columns(catalog: string, schema: string, table: string): unknown | null | Promise<unknown | null>;
-}
-/** Identity of the worker software surfaced in `describe.json` → `worker`. */
-export interface WorkerDescribeInfo {
-    /** Worker name (e.g. class or catalog name). */
-    name: string;
-    /** One-line worker description. */
-    doc?: string;
-    /** Worker software version. */
-    version?: string;
-}
-interface ColumnJson {
-    name: string;
-    type: string;
-    comment?: string;
-}
-/**
- * Build the full `describe.json` document for a worker's catalog interface.
- *
- * `oauth` and `serverId` are supplied by the HTTP server (they depend on runtime
- * config, not the catalog).
- */
-export declare function buildDescribeJson(iface: CatalogInterface, worker: WorkerDescribeInfo, opts?: {
-    oauth?: boolean;
-    serverId?: string;
-    cupolaBase?: string;
-}): Promise<Record<string, unknown>>;
-/**
- * Build the lazy per-object column payload for one table or view.
- *
- * Returns `{ columns: [{ name, type, comment? }] }`, or `null` when the object
- * can't be found. Tables deserialize their Arrow column schema; views expose
- * their declared column comments (types are only known after binding the SQL,
- * which the worker does not do here).
- */
-export declare function buildColumnsJson(iface: CatalogInterface, catalog: string, schema: string, table: string): Promise<{
-    columns: ColumnJson[];
-} | null>;
-/**
- * Build a {@link LandingDescribeProvider} to pass to `createHttpHandler` from
- * `@query-farm/vgi-rpc`. Wires the worker's catalog interface into the
- * standardized landing surface (`GET {prefix}/describe.json` and the lazy
- * per-object column endpoint).
- */
-export declare function createLandingDescribe(iface: CatalogInterface, worker: WorkerDescribeInfo, opts?: {
-    cupolaBase?: string;
-}): LandingDescribeProvider;
-export {};