@query-farm/vgi 0.7.0 → 0.9.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@query-farm/vgi",
-  "version": "0.7.0",
+  "version": "0.9.0",
   "description": "TypeScript framework for extending DuckDB with user-defined functions via the Vector Gateway Interface (VGI) over Arrow IPC.",
   "license": "SEE LICENSE IN LICENSE",
   "author": "Query Farm LLC (https://query.farm)",

package/src/arguments/argument-spec.ts

@@ -14,6 +14,10 @@ import {
   VGI_CONST_KEY,
   VGI_CONST_TRUE,
   VGI_DOC_KEY,
+  VGI_DEFAULT_KEY,
+  VGI_CHOICES_KEY,
+  VGI_RANGE_KEY,
+  VGI_PATTERN_KEY,
 } from "../types.js";
 
 export interface ArgumentSpec {
@@ -25,6 +29,126 @@ export interface ArgumentSpec {
   isVarargs?: boolean;
   isConst?: boolean;
   doc?: string;
+  /**
+   * Discovery-facing constraint metadata, pre-encoded to their surfaced UTF-8
+   * forms (presence-only — `undefined` means the constraint is absent and the
+   * corresponding field-metadata key is omitted entirely). Mirrors Python's
+   * `ArgumentSpec.default_json` / `choices_json` / `range_notation` / `pattern`
+   * so `argumentSpecsToSchema`/`schemaToArgumentSpecs` round-trip symmetrically.
+   */
+  defaultJson?: string;
+  choicesJson?: string;
+  rangeNotation?: string;
+  pattern?: string;
+}
+
+/**
+ * Raw per-argument constraint declarations, as authored on a function's
+ * argument descriptor. Encoded into the pre-computed {@link ArgumentSpec}
+ * constraint fields via {@link constraintSpecFields}.
+ */
+export interface ArgumentConstraints {
+  /** Closed set of allowed values (JSON-encoded into `vgi_choices`). */
+  choices?: readonly unknown[];
+  /** Inclusive lower bound (`>=`). */
+  ge?: number;
+  /** Inclusive upper bound (`<=`). */
+  le?: number;
+  /** Exclusive lower bound (`>`). */
+  gt?: number;
+  /** Exclusive upper bound (`<`). */
+  lt?: number;
+  /** Regex the value must match (surfaced as `vgi_pattern`, as-is). */
+  pattern?: string;
+  /** Default value (JSON-encoded into `vgi_default`); optional args only. */
+  default?: unknown;
+}
+
+/** `JSON.stringify`, falling back to the value's string form when unserializable. */
+function safeJson(value: unknown): string {
+  try {
+    const encoded = JSON.stringify(value);
+    // JSON.stringify returns undefined for functions/symbols/undefined.
+    if (encoded === undefined) {
+      return JSON.stringify(String(value));
+    }
+    return encoded;
+  } catch {
+    return JSON.stringify(String(value));
+  }
+}
+
+/**
+ * Build interval notation from an argument's numeric bounds.
+ *
+ * Inclusive bounds (`ge`/`le`) render as square brackets, exclusive bounds
+ * (`gt`/`lt`) as parentheses, and an open side as `-inf`/`+inf`. Returns
+ * `undefined` when the argument has no numeric bound at all.
+ *
+ * Examples: `ge=0,le=10` -> `"[0, 10]"`; `gt=0` -> `"(0, +inf)"`;
+ * `ge=1,lt=10` -> `"[1, 10)"`.
+ */
+export function formatRange(
+  ge?: number,
+  le?: number,
+  gt?: number,
+  lt?: number,
+): string | undefined {
+  if (ge === undefined && le === undefined && gt === undefined && lt === undefined) {
+    return undefined;
+  }
+  let low: string;
+  if (gt !== undefined) {
+    low = `(${gt}`;
+  } else if (ge !== undefined) {
+    low = `[${ge}`;
+  } else {
+    low = "(-inf";
+  }
+  let high: string;
+  if (lt !== undefined) {
+    high = `${lt})`;
+  } else if (le !== undefined) {
+    high = `${le}]`;
+  } else {
+    high = "+inf)";
+  }
+  return `${low}, ${high}`;
+}
+
+/**
+ * Encode raw {@link ArgumentConstraints} into the pre-computed constraint
+ * fields of an {@link ArgumentSpec} (presence-only — absent constraints yield
+ * no field). Mirrors Python's `_constraint_kwargs`.
+ */
+export function constraintSpecFields(
+  constraints: ArgumentConstraints | undefined,
+): Pick<ArgumentSpec, "defaultJson" | "choicesJson" | "rangeNotation" | "pattern"> {
+  const fields: Pick<
+    ArgumentSpec,
+    "defaultJson" | "choicesJson" | "rangeNotation" | "pattern"
+  > = {};
+  if (constraints === undefined) return fields;
+
+  if ("default" in constraints && constraints.default !== undefined) {
+    fields.defaultJson = safeJson(constraints.default);
+  }
+  if (constraints.choices !== undefined) {
+    fields.choicesJson = safeJson([...constraints.choices]);
+  }
+  const range = formatRange(
+    constraints.ge,
+    constraints.le,
+    constraints.gt,
+    constraints.lt,
+  );
+  if (range !== undefined) {
+    fields.rangeNotation = range;
+  }
+  if (constraints.pattern !== undefined) {
+    fields.pattern = constraints.pattern;
+  }
+  return fields;
 }
 
 function argumentSpecSortKey(spec: ArgumentSpec): [number, number | string] {
@@ -76,6 +200,20 @@ export function argumentSpecsToSchema(specs: ArgumentSpec[]): VgiSchema {
       metadata.set(VGI_DOC_KEY, spec.doc);
     }
 
+    // Per-argument constraint metadata (presence-only; already value-encoded).
+    if (spec.defaultJson !== undefined) {
+      metadata.set(VGI_DEFAULT_KEY, spec.defaultJson);
+    }
+    if (spec.choicesJson !== undefined) {
+      metadata.set(VGI_CHOICES_KEY, spec.choicesJson);
+    }
+    if (spec.rangeNotation !== undefined) {
+      metadata.set(VGI_RANGE_KEY, spec.rangeNotation);
+    }
+    if (spec.pattern !== undefined) {
+      metadata.set(VGI_PATTERN_KEY, spec.pattern);
+    }
+
     const field = field_(
       spec.name,
       spec.arrowType,
@@ -184,6 +322,12 @@ export function schemaToArgumentSpecs(schema: VgiSchema): ArgumentSpec[] {
     const isConst = metadata.get(VGI_CONST_KEY) === VGI_CONST_TRUE;
     const doc = metadata.get(VGI_DOC_KEY) ?? "";
 
+    // Per-argument constraint metadata (absent key -> undefined).
+    const defaultJson = metadata.get(VGI_DEFAULT_KEY);
+    const choicesJson = metadata.get(VGI_CHOICES_KEY);
+    const rangeNotation = metadata.get(VGI_RANGE_KEY);
+    const pattern = metadata.get(VGI_PATTERN_KEY);
+
     specs.push({
       name: field.name,
       position,
@@ -193,6 +337,10 @@ export function schemaToArgumentSpecs(schema: VgiSchema): ArgumentSpec[] {
       isVarargs,
       isConst,
       doc,
+      defaultJson,
+      choicesJson,
+      rangeNotation,
+      pattern,
     });
   }
 

package/src/catalog/descriptors.ts

@@ -2,6 +2,7 @@
 // Declarative catalog descriptor types.
 
 import type { VgiSchema, VgiDataType } from "../arrow/index.js";
+import type { AttachCatalogInfo } from "./interface.js";
 import type { VgiFunction } from "../functions/types.js";
 import type { Arguments } from "../arguments/arguments.js";
 import type { ColumnStatistics } from "../util/statistics.js";
@@ -152,6 +153,11 @@ export interface CatalogDescriptor {
   schemas: SchemaDescriptor[];
   settings?: SettingDescriptor[];
   secretTypes?: SecretTypeDescriptor[];
+  /**
+   * Companion catalogs (lakehouse federation) the client should ATTACH when this
+   * VGI catalog attaches. Surfaced via `catalog_attach.attach_catalogs`.
+   */
+  attachCatalogs?: AttachCatalogInfo[];
   comment?: string;
   tags?: Record<string, string>;
   /**

package/src/catalog/interface.ts

@@ -74,7 +74,10 @@ export { encodeIndexInfo, decodeIndexInfo } from "../generated/vgi-client.js";
 // generic ASD codec with the generated schema. Mirrors vgi-python's
 // catalog_interface.CopyFromFormatInfo.
 import { encodeASD } from "../codec/asd.js";
-import { CopyFromFormatInfoSchema } from "../generated/vgi-protocol-schemas.js";
+import {
+  AttachCatalogInfoSchema,
+  CopyFromFormatInfoSchema,
+} from "../generated/vgi-protocol-schemas.js";
 
 export interface CopyFromFormatInfo {
   /** Free-text comment (CopyFromFunction's `copyFromComment`). */
@@ -117,6 +120,42 @@ export const encodeCopyFromFormatInfo = (v: CopyFromFormatInfo): Uint8Array =>
     ordered: v.ordered ?? false,
   });
 
+/**
+ * A companion catalog the client should ATTACH when this VGI catalog attaches
+ * (lakehouse federation). Serialized into `CatalogAttachResult.attach_catalogs`;
+ * the C++ VGI extension attaches each at VGI-attach time so multi-branch
+ * catalog-table branches can resolve tables in a companion DuckLake / Iceberg /
+ * Postgres / DuckDB.
+ */
+export interface AttachCatalogInfo {
+  /** ATTACH alias; also a catalog-table ScanBranch's `sourceCatalog`. */
+  alias: string;
+  /** ATTACH target — a path or DSN (e.g. `ducklake:sqlite:/data/meta.sqlite`). */
+  target: string;
+  /** DuckDB db type; empty => inferred from the target scheme prefix. */
+  db_type?: string;
+  /** Extra ATTACH options forwarded verbatim (e.g. DuckLake DATA_PATH). */
+  options?: Record<string, string>;
+  /** Attach hidden (excluded from duckdb_databases()); still resolvable. */
+  hidden?: boolean;
+  /** When true, attach failure fails the whole VGI ATTACH. */
+  required?: boolean;
+  /** Optional named secret to inject into the companion's ATTACH options. */
+  secret_ref?: string;
+}
+
+/** Encode an AttachCatalogInfo to Arrow IPC bytes (single-row batch). */
+export const encodeAttachCatalogInfo = (v: AttachCatalogInfo): Uint8Array =>
+  encodeASD(AttachCatalogInfoSchema, {
+    alias: v.alias,
+    target: v.target,
+    db_type: v.db_type ?? "",
+    options: v.options ?? {},
+    hidden: v.hidden ?? false,
+    required: v.required ?? false,
+    secret_ref: v.secret_ref ?? "",
+  });
+
 // ============================================================================
 // ScanFunctionResult — typed result for catalog_table_scan_function_get.
 // Mirrors vgi-python's ScanFunctionResult (catalog_interface.py:380).
@@ -204,6 +243,9 @@ export function singleBranchResult(
       arguments: [toUint8Array(legacy.arguments)],
       branch_filter: [null],
       writable: [false],
+      source_catalog: [null],
+      source_schema: [null],
+      source_table: [null],
     },
     ScanBranchSchema as any,
   );
@@ -233,6 +275,15 @@ export interface ScanBranchInput {
   branchFilter?: string | null;
   /** Declares this branch as the INSERT target (at most one per table). */
   writable?: boolean;
+  /**
+   * Catalog-table branch (lakehouse federation): leave `functionName` empty and
+   * set these to scan the base table
+   * `sourceCatalog.sourceSchema.sourceTable` in a companion catalog instead of
+   * calling a table function.
+   */
+  sourceCatalog?: string | null;
+  sourceSchema?: string | null;
+  sourceTable?: string | null;
 }
 
 /**
@@ -284,6 +335,9 @@ export function buildScanBranchesResult(
         arguments: [serializeBranchArguments(branch)],
         branch_filter: [branch.branchFilter ?? null],
         writable: [branch.writable ?? false],
+        source_catalog: [branch.sourceCatalog ?? null],
+        source_schema: [branch.sourceSchema ?? null],
+        source_table: [branch.sourceTable ?? null],
       },
       ScanBranchSchema as any,
     );

package/src/catalog/read-only.ts

@@ -16,6 +16,7 @@ import {
   type IndexInfo,
   type IndexConstraintType,
   type CopyFromFormatInfo,
+  encodeAttachCatalogInfo,
 } from "./interface.js";
 import type { CatalogDescriptor, SchemaDescriptor, TableDescriptor, ViewDescriptor, MacroDescriptor, SettingDescriptor, SecretTypeDescriptor, ForeignKeyDef, DefaultValue } from "./descriptors.js";
 import { serializeColumnStatistics } from "../util/statistics.js";
@@ -80,6 +81,10 @@ export class ReadOnlyCatalogInterface extends CatalogInterface {
       serializeSecretType(s)
     );
 
+    const attachCatalogs = (this._descriptor.attachCatalogs ?? []).map((c) =>
+      encodeAttachCatalogInfo(c)
+    );
+
     // Auto-derive supportsTimeTravel from table descriptors
     const hasTimeTravel = this._descriptor.schemas.some(
       (s) => s.tables?.some((t) => t.supportsTimeTravel) ?? false
@@ -95,6 +100,7 @@ export class ReadOnlyCatalogInterface extends CatalogInterface {
       default_schema: this._descriptor.defaultSchema ?? "main",
       settings,
       secret_types: secretTypes,
+      attach_catalogs: attachCatalogs,
       comment: this._descriptor.comment ?? null,
       tags: this._descriptor.tags ?? {},
       // Advertise true so DuckDB routes catalog_table_column_statistics_get

package/src/client/client.ts

@@ -621,6 +621,9 @@ export class VgiClient {
       secret_types: inner.secret_types
         ? (Array.isArray(inner.secret_types) ? inner.secret_types : [...inner.secret_types]).map(toUint8Array)
         : [],
+      attach_catalogs: inner.attach_catalogs
+        ? (Array.isArray(inner.attach_catalogs) ? inner.attach_catalogs : [...inner.attach_catalogs]).map(toUint8Array)
+        : [],
       comment: inner.comment ?? null,
       tags: deserializeTags(inner.tags),
       supports_column_statistics: inner.supports_column_statistics ?? false,

package/src/functions/scalar.ts

@@ -37,7 +37,7 @@ import type {
   FunctionExample,
 } from "./types.js";
 import type { ArgumentSpec } from "../arguments/argument-spec.js";
-import { argumentSpecsToSchema } from "../arguments/argument-spec.js";
+import { argumentSpecsToSchema, constraintSpecFields } from "../arguments/argument-spec.js";
 import { batchToScalarDict, batchToSecretDict, batchFromColumns, projectBatch, safeNumber } from "../util/arrow/index.js";
 
 // ============================================================================
@@ -71,6 +71,22 @@ export interface ScalarParameterDef {
   varargs?: boolean;
   /** Human-readable per-argument description (surfaced as `vgi_doc`). */
   doc?: string;
+  // Discovery-facing validation constraints (surfaced via
+  // `vgi_function_arguments()`). All optional; see `ArgumentConstraints`.
+  /** Closed set of allowed values (surfaced as `vgi_choices`). */
+  choices?: readonly unknown[];
+  /** Inclusive lower bound, `>=` (folded into `vgi_range`). */
+  ge?: number;
+  /** Inclusive upper bound, `<=` (folded into `vgi_range`). */
+  le?: number;
+  /** Exclusive lower bound, `>` (folded into `vgi_range`). */
+  gt?: number;
+  /** Exclusive upper bound, `<` (folded into `vgi_range`). */
+  lt?: number;
+  /** Regex the value must match (surfaced as `vgi_pattern`). */
+  pattern?: string;
+  /** Default value (surfaced as `vgi_default`, JSON-encoded). */
+  default?: unknown;
 }
 
 /**
@@ -183,6 +199,7 @@ export function defineScalarFunction<
         isConst: p.const,
         isVarargs: p.varargs,
         doc: p.doc,
+        ...constraintSpecFields(p),
       });
     }
   } else {

package/src/generated/vgi-client.ts

@@ -2,7 +2,7 @@
 // GENERATED by vgi.codegen.ts_client. DO NOT EDIT BY HAND.
 //
 // Generator: vgi-gen-ts-client v1
-// Content hash: 101207be9452
+// Content hash: 5ddab596625f
 //
 // To regenerate:
 //   uv run --project ~/Development/vgi-python vgi-gen-ts-client \
@@ -366,6 +366,7 @@ export interface CatalogAttachResult {
   default_schema: string;
   settings: Uint8Array[];
   secret_types: Uint8Array[];
+  attach_catalogs: Uint8Array[];
   comment?: string | null;
   tags: Record<string, string>;
   supports_column_statistics: boolean;

package/src/generated/vgi-protocol-schemas.ts

@@ -2,7 +2,7 @@
 // GENERATED by vgi.codegen.ts_schemas. DO NOT EDIT BY HAND.
 //
 // Generator: vgi-gen-ts-schemas v2
-// Content hash: d028e7cb81cd
+// Content hash: 673bfdc9573a
 //
 // To regenerate:
 //   uv run --project ~/Development/vgi-python vgi-gen-ts-schemas \
@@ -175,6 +175,20 @@ export const ScanBranchSchema = schema([
   field("arguments", binary(), false),
   field("branch_filter", utf8(), true),
   field("writable", bool(), false),
+  field("source_catalog", utf8(), true),
+  field("source_schema", utf8(), true),
+  field("source_table", utf8(), true),
+]);
+
+// Origin: AttachCatalogInfo
+export const AttachCatalogInfoSchema = schema([
+  field("alias", utf8(), false),
+  field("target", utf8(), false),
+  field("db_type", utf8(), false),
+  field("options", map(field("key", utf8(), false), field("value", utf8(), true), false), false),
+  field("hidden", bool(), false),
+  field("required", bool(), false),
+  field("secret_ref", utf8(), false),
 ]);
 
 // Origin: method 'aggregate_bind' result
@@ -246,6 +260,7 @@ export const CatalogAttachResultSchema = schema([
   field("default_schema", utf8(), false),
   field("settings", list(field("item", binary(), true)), false),
   field("secret_types", list(field("item", binary(), true)), false),
+  field("attach_catalogs", list(field("item", binary(), true)), false),
   field("comment", utf8(), true),
   field("tags", map(field("key", utf8(), false), field("value", utf8(), true), false), false),
   field("supports_column_statistics", bool(), false),

package/src/index.core.ts

@@ -75,7 +75,7 @@ export type {
 
 // Arguments
 export { Arguments } from "./arguments/arguments.js";
-export { type ArgumentSpec, argumentSpecsToSchema, schemaToArgumentSpecs, macroArgumentsSchema, macroParameterDocsFromSchema } from "./arguments/argument-spec.js";
+export { type ArgumentSpec, type ArgumentConstraints, argumentSpecsToSchema, schemaToArgumentSpecs, macroArgumentsSchema, macroParameterDocsFromSchema, formatRange, constraintSpecFields } from "./arguments/argument-spec.js";
 
 // Errors
 export {

package/src/protocol/handlers/catalog/admin.ts

@@ -100,6 +100,7 @@ export function registerCatalogAdminMethods(protocol: Protocol, getCatalog: GetC
         default_schema: result.default_schema ?? "main",
         settings: result.settings ?? [],
         secret_types: result.secret_types ?? [],
+        attach_catalogs: result.attach_catalogs ?? [],
         comment: result.comment ?? null,
         tags: result.tags ?? {},
         // True so DuckDB will route catalog_table_column_statistics_get RPCs

package/src/types.ts

@@ -62,6 +62,20 @@ export const VGI_CONST_TRUE = "true";
 // Per-argument description (UTF-8; presence-only). `vgi_doc_*` prefix reserved.
 export const VGI_DOC_KEY = "vgi_doc";
 
+// Per-argument constraint metadata for agent discovery. All presence-only
+// (omitted when absent) and value-encoded as UTF-8 so the surfaced column type
+// stays uniform regardless of the arg's value type:
+//   vgi_default — JSON scalar (the arg's default value; optional args only)
+//   vgi_choices — JSON array (the closed set of allowed values)
+//   vgi_range   — interval notation built from ge/le/gt/lt (e.g. "[0, 100]",
+//                 "(0, +inf)", "[1, 10)"); a discovery surface, not raw bounds
+//   vgi_pattern — raw regex the value must match (open set)
+// Kept byte-for-byte in sync with the C++ reader and the Python/other SDKs.
+export const VGI_DEFAULT_KEY = "vgi_default";
+export const VGI_CHOICES_KEY = "vgi_choices";
+export const VGI_RANGE_KEY = "vgi_range";
+export const VGI_PATTERN_KEY = "vgi_pattern";
+
 export interface TableCardinality {
   estimate: number | null;
   max: number | null;