@query-farm/vgi 0.24.0 → 0.24.2

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@query-farm/vgi",
-  "version": "0.24.0",
+  "version": "0.24.2",
   "description": "TypeScript framework for extending DuckDB with user-defined functions via the Vector Gateway Interface (VGI) over Arrow IPC.",
   "license": "SEE LICENSE IN LICENSE",
   "author": "Query Farm LLC (https://query.farm)",
@@ -69,15 +69,18 @@
   "files": [
     "dist",
     "src",
+    "scripts/postinstall.cjs",
     "!src/**/__tests__/**",
     "!src/**/*.test.ts",
     "!src/**/*.spec.ts"
   ],
+  "//imports": "`browser` maps to arrow-js DELIBERATELY, not flechette. Measured on the client surface (VgiClient + Arguments + batch helpers), a minified browser bundle is 84 KB gzip via arrow-js vs 116 KB via flechette — arrow-js tree-shakes to the subset the client touches, while flechette links as a unit. Flechette still wins for `worker-cf`, which exercises far more of the Arrow API and cannot take a peer dep. Keep `browser` AFTER `workerd`/`worker`: conditions match in key order, so a workerd build (which sets both) must hit flechette first.",
   "imports": {
     "#arrow-impl": {
       "flechette": "./src/arrow/impl-flechette/index.ts",
       "workerd": "./src/arrow/impl-flechette/index.ts",
       "worker": "./src/arrow/impl-flechette/index.ts",
+      "browser": "./src/arrow/impl-arrowjs/index.ts",
       "default": "./src/arrow/impl-arrowjs/index.ts"
     }
   },
@@ -87,12 +90,12 @@
   },
   "peerDependencies": {
     "@query-farm/apache-arrow": "^21.1.1",
-    "@query-farm/vgi-rpc": "^0.17.0"
+    "@query-farm/vgi-rpc": ">=0.18.1 <0.19.0"
   },
   "devDependencies": {
     "@duckdb/node-api": "^1.5.2-r.1",
     "@query-farm/apache-arrow": "^21.1.1",
-    "@query-farm/vgi-rpc": "^0.17.0",
+    "@query-farm/vgi-rpc": "^0.18.1",
     "@types/bun": "latest",
     "typescript": "^6.0.3"
   },

package/scripts/postinstall.cjs

@@ -0,0 +1,47 @@
+#!/usr/bin/env node
+// Copyright 2025, 2026 Query Farm LLC - https://query.farm
+// Postinstall: deduplicate @query-farm/apache-arrow across @query-farm/vgi-rpc
+// to avoid dual-package instanceof issues.
+//
+// This is a LOCAL DEVELOPMENT-only fixup for the vgi-typescript monorepo, where
+// @query-farm/vgi-rpc is a `file:` link with its own nested copy of apache-arrow.
+// It must NOT run when this package is installed as a dependency of someone
+// else's project — so it no-ops unless the sibling vgi-rpc-typescript checkout
+// is present next to this repo. (Published consumers get a single hoisted
+// apache-arrow from the registry, so no dedup is needed there.)
+const fs = require("fs");
+const path = require("path");
+
+const PKG = "@query-farm/apache-arrow";
+
+// Resolve everything relative to the repo root (this file lives in <repo>/scripts),
+// not the process cwd, so behavior doesn't depend on where the installer runs us.
+const repoRoot = path.resolve(__dirname, "..");
+
+// Dev-monorepo signal: the sibling vgi-rpc source checkout. When this package is
+// installed as a dependency, that path resolves inside the consumer's node_modules
+// and won't exist — so we bail and leave their tree untouched.
+const devSibling = path.resolve(repoRoot, "..", "vgi-rpc-typescript", "package.json");
+if (!fs.existsSync(devSibling)) {
+  process.exit(0);
+}
+
+// Deduplicate: replace vgi-rpc's nested copy with a symlink to ours.
+//   Without this, two separate DataType classes exist at runtime and
+//   instanceof checks fail across package boundaries.
+//   Bun uses per-file symlinks for file: deps, so module resolution follows
+//   the real path. We must target the REAL node_modules of vgi-rpc-typescript.
+const ourArrow = path.join(repoRoot, "node_modules", PKG);
+const rpcPkgLink = path.join(repoRoot, "node_modules", "@query-farm", "vgi-rpc", "package.json");
+try {
+  const realPkgPath = fs.realpathSync(rpcPkgLink);
+  const rpcRoot = path.dirname(realPkgPath);
+  const rpcArrow = path.join(rpcRoot, "node_modules", PKG);
+  const st = fs.lstatSync(rpcArrow);
+  if (!st.isSymbolicLink()) {
+    fs.rmSync(rpcArrow, { recursive: true });
+    fs.symlinkSync(ourArrow, rpcArrow);
+  }
+} catch (e) {
+  if (e.code !== "ENOENT") throw e;
+}

package/src/catalog/attach-option.ts

@@ -17,6 +17,11 @@
 //     default_value: Binary nullable -- serialized single-row RecordBatch with
 //                                       the default value under "value", or
 //                                       null when no default is declared
+//     required: Bool nullable -- the caller must supply this option at ATTACH
+//                                time. Nullable and appended LAST so a peer
+//                                that predates the column reads the batch by
+//                                name and simply doesn't see it; absent and
+//                                explicit-null both mean "not required".
 //
 // The extension reads the outer `attach_option_specs: list<binary>` column
 // from CatalogInfo and validates user-supplied ATTACH options against the
@@ -28,6 +33,7 @@ import {
   field,
   utf8,
   binary,
+  bool,
   batchFromColumns,
   serializeBatch,
   serializeSchema,
@@ -55,6 +61,16 @@ export interface AttachOptionSpec {
    * the options dict delivered to attach()).
    */
   default?: unknown;
+  /**
+   * The caller must supply this option at ATTACH time. A catalog that cannot
+   * be attached without it advertises that fact at discovery, so a client can
+   * say so before attempting the attach rather than surfacing a failure that
+   * reads like an empty catalog.
+   *
+   * Mutually exclusive with `default` — an option that falls back to a value
+   * is by definition satisfiable without the caller.
+   */
+  required?: boolean;
 }
 
 const SPEC_SCHEMA = makeSchema([
@@ -62,6 +78,7 @@ const SPEC_SCHEMA = makeSchema([
   field("description", utf8(), false),
   field("type", binary(), false),
   field("default_value", binary(), true),
+  field("required", bool(), true),
 ]);
 
 /**
@@ -69,6 +86,17 @@ const SPEC_SCHEMA = makeSchema([
  * (one IPC-serialized RecordBatch with a single row).
  */
 export function serializeAttachOptionSpec(spec: AttachOptionSpec): Uint8Array {
+  // Mirrors AttachOptionSpec.__post_init__ on the Python side: an option that
+  // falls back to a value is always satisfiable without the caller, so the
+  // combination is a declaration bug rather than a runtime condition.
+  if (spec.required && spec.default !== undefined && spec.default !== null) {
+    throw new Error(
+      `Attach option '${spec.name}' is required but also declares a default ` +
+        `(${JSON.stringify(spec.default)}); an option with a default is always ` +
+        "satisfiable without the caller. Drop one.",
+    );
+  }
+
   // `type` is encoded as a serialized Arrow Schema with one field named
   // "value" of the option's DataType. This lets the extension peek at the
   // logical type without a separate enum.
@@ -91,6 +119,9 @@ export function serializeAttachOptionSpec(spec: AttachOptionSpec): Uint8Array {
       description: [spec.description],
       type: [typeBytes],
       default_value: [defaultBytes],
+      // Written explicitly rather than left null so a reader sees `false`, not
+      // NULL, for an option that simply isn't required.
+      required: [spec.required ?? false],
     },
     SPEC_SCHEMA,
   );
@@ -105,3 +136,46 @@ export function serializeAttachOptionSpecs(
 ): Uint8Array[] {
   return Array.from(specs, serializeAttachOptionSpec);
 }
+
+/**
+ * Raised when `attach` omits options declared `required`.
+ *
+ * Carries the option names in `missing` so a caller can act on them without
+ * parsing the message. Mirrors `MissingAttachOptionsError` on the Python side,
+ * message included — the extension's integration suite matches on its text.
+ */
+export class MissingAttachOptionsError extends Error {
+  readonly missing: string[];
+
+  constructor(catalogName: string, missing: string[]) {
+    const joined = missing.map((name) => `'${name}'`).join(", ");
+    super(
+      `Catalog '${catalogName}' cannot be attached without the required ` +
+        `option${missing.length > 1 ? "s" : ""} ${joined}.`,
+    );
+    this.name = "MissingAttachOptionsError";
+    this.missing = missing;
+  }
+}
+
+/**
+ * Throw if any `required` spec has no corresponding entry in `options`.
+ *
+ * Option names are matched case-insensitively, mirroring DuckDB's handling of
+ * ATTACH option keys.
+ *
+ * @throws {MissingAttachOptionsError} If a required option was not supplied.
+ */
+export function validateRequiredAttachOptions(
+  catalogName: string,
+  specs: Iterable<AttachOptionSpec>,
+  options: Record<string, unknown>,
+): void {
+  const supplied = new Set(Object.keys(options).map((key) => key.toLowerCase()));
+  const missing = Array.from(specs)
+    .filter((spec) => spec.required && !supplied.has(spec.name.toLowerCase()))
+    .map((spec) => spec.name);
+  if (missing.length > 0) {
+    throw new MissingAttachOptionsError(catalogName, missing);
+  }
+}

package/src/catalog/read-only.ts

@@ -26,6 +26,7 @@ import type { FunctionRegistry } from "../functions/registry.js";
 import { argumentSpecsToSchema, macroArgumentsSchema } from "../arguments/argument-spec.js";
 import { serializeSchema, serializeBatch, emptyBatch, batchFromColumns } from "../util/arrow/index.js";
 import { resolveMetadata } from "../metadata/resolve.js";
+import { type AttachOptionSpec, validateRequiredAttachOptions } from "./attach-option.js";
 import { FunctionStability, NullHandling, OrderPreservation, DEFAULT_MAX_WORKERS } from "../types.js";
 import { Arguments } from "../arguments/arguments.js";
 
@@ -58,6 +59,19 @@ export class ReadOnlyCatalogInterface extends CatalogInterface {
     return [this._descriptor.name];
   }
 
+  /**
+   * Attach-time options this catalog declares, for the named catalog.
+   *
+   * Empty by default — the declarative descriptor path carries no attach
+   * options. A subclass that advertises specs in `catalogsInfo()` should
+   * return the same specs here so `attach()` enforces the `required` ones it
+   * advertised. Takes the catalog name because one interface may serve
+   * several catalogs with different option sets.
+   */
+  attachOptionSpecs(_name: string): AttachOptionSpec[] {
+    return [];
+  }
+
   // Advertise the catalog's discovery record. The declarative path carries no
   // version metadata or attach option specs, but it does surface the optional
   // `source_url` from the descriptor so it flows through `catalog_catalogs`
@@ -82,6 +96,12 @@ export class ReadOnlyCatalogInterface extends CatalogInterface {
     if (!this.catalogs().includes(name)) {
       throw new Error(`No worker handles catalog '${name}'`);
     }
+    // Refuse an attach that omits a declared-required option, so the caller
+    // gets the option name rather than a catalog that looks mysteriously
+    // empty. A subclass that overrides `attach` outright bypasses this and
+    // should call `validateRequiredAttachOptions` itself — mirrors
+    // CatalogInterface.catalog_attach on the Python side.
+    validateRequiredAttachOptions(name, this.attachOptionSpecs(name), options ?? {});
 
     const attachOpaqueData = new Uint8Array(16);
     crypto.getRandomValues(attachOpaqueData);