@query-farm/vgi 0.22.0 → 0.24.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@query-farm/vgi",
-  "version": "0.22.0",
+  "version": "0.24.0",
   "description": "TypeScript framework for extending DuckDB with user-defined functions via the Vector Gateway Interface (VGI) over Arrow IPC.",
   "license": "SEE LICENSE IN LICENSE",
   "author": "Query Farm LLC (https://query.farm)",
@@ -87,12 +87,12 @@
   },
   "peerDependencies": {
     "@query-farm/apache-arrow": "^21.1.1",
-    "@query-farm/vgi-rpc": "^0.15.4"
+    "@query-farm/vgi-rpc": "^0.17.0"
   },
   "devDependencies": {
     "@duckdb/node-api": "^1.5.2-r.1",
     "@query-farm/apache-arrow": "^21.1.1",
-    "@query-farm/vgi-rpc": "^0.15.4",
+    "@query-farm/vgi-rpc": "^0.17.0",
     "@types/bun": "latest",
     "typescript": "^6.0.3"
   },

package/src/catalog/descriptors.ts

@@ -173,4 +173,22 @@ export interface CatalogDescriptor {
    * advertises a null `source_url`.
    */
   sourceUrl?: string;
+  /**
+   * Functions this catalog asks the client to *additionally* publish into its
+   * global (DuckDB `system.main`) function namespace, on top of the normal
+   * schema-qualified registration. Surfaced as IPC-serialized `FunctionInfo`
+   * records on `catalog_attach.global_functions`.
+   *
+   * Every entry must also be declared in exactly one of this catalog's
+   * schemas — the `FunctionInfo.schema_name` it carries is the bind-dispatch
+   * key, not a sentinel. Mirrors vgi-python's `Catalog.global_functions`.
+   */
+  globalFunctions?: VgiFunction[];
+  /**
+   * Prefix applied client-side to every `globalFunctions` entry's name when it
+   * is published globally (e.g. prefix `vgi_example` + `global_scalar` →
+   * `vgi_example_global_scalar`). The advertised `FunctionInfo.name` stays
+   * unprefixed. Surfaced as `catalog_attach.global_function_prefix`.
+   */
+  globalFunctionPrefix?: string;
 }

package/src/catalog/read-only.ts

@@ -17,6 +17,7 @@ import {
   type IndexConstraintType,
   type CopyFromFormatInfo,
   encodeAttachCatalogInfo,
+  encodeFunctionInfo,
 } from "./interface.js";
 import type { CatalogDescriptor, SchemaDescriptor, TableDescriptor, ViewDescriptor, MacroDescriptor, SettingDescriptor, SecretTypeDescriptor, ForeignKeyDef, DefaultValue } from "./descriptors.js";
 import { serializeColumnStatistics } from "../util/statistics.js";
@@ -119,6 +120,11 @@ export class ReadOnlyCatalogInterface extends CatalogInterface {
       // Advertise true so DuckDB routes catalog_table_column_statistics_get
       // for tables whose TableInfo.supports_column_statistics is also true.
       supports_column_statistics: true,
+      // Functions the client should ALSO publish into its global (system.main)
+      // namespace, under `global_function_prefix` + "_" + name. Empty unless
+      // the descriptor opts in via `globalFunctions`.
+      global_functions: this._globalFunctionInfos().map(encodeFunctionInfo),
+      global_function_prefix: this._descriptor.globalFunctionPrefix ?? "",
       resolved_data_version: null,
       resolved_implementation_version: null,
     };
@@ -374,7 +380,18 @@ export class ReadOnlyCatalogInterface extends CatalogInterface {
         if (t === "AGGREGATE_FUNCTION") return (f.kind as string) === "aggregate";
         return true;
       })
-      .map((f) => {
+      .map((f) => this._functionToInfo(f, name));
+  }
+
+  /**
+   * Build the wire `FunctionInfo` for one declared function.
+   *
+   * `schemaName` is the schema the function actually lives in — it is the
+   * bind-dispatch key, so a function advertised through more than one route
+   * (schema contents *and* `global_functions`) must carry the same value in
+   * both. Shared by `schemaContentsFunctions` and `_globalFunctionInfos`.
+   */
+  private _functionToInfo(f: VgiFunction, name: string): FunctionInfo {
         const meta = resolveMetadata(f);
         const argSchema = argumentSpecsToSchema(f.argumentSpecs);
         const argBytes = serializeSchema(argSchema);
@@ -474,7 +491,39 @@ export class ReadOnlyCatalogInterface extends CatalogInterface {
             secret_name: s.secret_name,
           })),
         };
-      });
+  }
+
+  /**
+   * Build `FunctionInfo` records for `CatalogDescriptor.globalFunctions`.
+   *
+   * Each entry's `schema_name` is the schema it actually lives in — the
+   * globally visible name is derived client-side from
+   * `global_function_prefix`, so `name` stays unprefixed here. An entry not
+   * declared in any schema is a descriptor bug, so it throws rather than
+   * advertising a function no bind can resolve. Mirrors vgi-python's
+   * `_global_function_infos`.
+   */
+  private _globalFunctionInfos(): FunctionInfo[] {
+    const globals = this._descriptor.globalFunctions ?? [];
+    if (globals.length === 0) return [];
+
+    const schemaOf = new Map<VgiFunction, string>();
+    for (const sch of this._descriptor.schemas) {
+      for (const fn of sch.functions ?? []) {
+        if (!schemaOf.has(fn)) schemaOf.set(fn, sch.name);
+      }
+    }
+
+    return globals.map((fn) => {
+      const schemaName = schemaOf.get(fn);
+      if (schemaName === undefined) {
+        throw new Error(
+          `Catalog '${this._descriptor.name}': '${fn.meta.name}' is listed in ` +
+            `globalFunctions but is not declared in any schema.`,
+        );
+      }
+      return this._functionToInfo(fn, schemaName);
+    });
   }
 
   override schemaContentsMacros(

package/src/client/client.ts

@@ -628,6 +628,10 @@ export class VgiClient {
       comment: inner.comment ?? null,
       tags: deserializeTags(inner.tags),
       supports_column_statistics: inner.supports_column_statistics ?? false,
+      global_functions: inner.global_functions
+        ? (Array.isArray(inner.global_functions) ? inner.global_functions : [...inner.global_functions]).map(toUint8Array)
+        : [],
+      global_function_prefix: inner.global_function_prefix ?? "",
       resolved_data_version: inner.resolved_data_version ?? null,
       resolved_implementation_version: inner.resolved_implementation_version ?? null,
     };

package/src/generated/vgi-client.ts

@@ -2,7 +2,7 @@
 // GENERATED by vgi.codegen.ts_client. DO NOT EDIT BY HAND.
 //
 // Generator: vgi-gen-ts-client v1
-// Content hash: 80ec56060e88
+// Content hash: e0862da99312
 //
 // To regenerate:
 //   uv run --project ~/Development/vgi-python vgi-gen-ts-client \
@@ -384,6 +384,8 @@ export interface CatalogAttachResult {
   comment?: string | null;
   tags: Record<string, string>;
   supports_column_statistics: boolean;
+  global_functions: Uint8Array[];
+  global_function_prefix: string;
   resolved_data_version?: string | null;
   resolved_implementation_version?: string | null;
 }

package/src/generated/vgi-protocol-schemas.ts

@@ -2,7 +2,7 @@
 // GENERATED by vgi.codegen.ts_schemas. DO NOT EDIT BY HAND.
 //
 // Generator: vgi-gen-ts-schemas v2
-// Content hash: 214d149e8f9c
+// Content hash: ae533ce2196c
 //
 // To regenerate:
 //   uv run --project ~/Development/vgi-python vgi-gen-ts-schemas \
@@ -265,6 +265,8 @@ export const CatalogAttachResultSchema = schema([
   field("comment", utf8(), true),
   field("tags", map(field("key", utf8(), false), field("value", utf8(), true), false), false),
   field("supports_column_statistics", bool(), false),
+  field("global_functions", list(field("item", binary(), true)), false),
+  field("global_function_prefix", utf8(), false),
   field("resolved_data_version", utf8(), true),
   field("resolved_implementation_version", utf8(), true),
 ]);

package/src/protocol/handlers/aggregate.ts

@@ -41,8 +41,20 @@ import {
 } from "./shared.js";
 
 export function registerAggregateMethods(protocol: Protocol, registry: FunctionRegistry): void {
-  function resolveAggregate(name: string): AggregateFunctionConfig<any, any> {
-    const func = registry.get(name, { arguments: new Arguments(), inputSchema: null, isScalar: false }) as any;
+  // `schemaName` scopes resolution to the schema the caller named. Every
+  // aggregate RPC carries it (protocol 1.2.0), so an aggregate name declared in
+  // two schemas dispatches to the one the request names rather than colliding
+  // on the flat by-name index. `null` (no schema) keeps the cross-schema lookup.
+  function resolveAggregate(
+    name: string,
+    schemaName?: string | null,
+  ): AggregateFunctionConfig<any, any> {
+    const func = registry.get(name, {
+      arguments: new Arguments(),
+      inputSchema: null,
+      isScalar: false,
+      schemaName: schemaName ?? null,
+    }) as any;
     if (!func || func.kind !== "aggregate") {
       throw new Error(`Function '${name}' is not an aggregate function`);
     }
@@ -58,7 +70,7 @@ export function registerAggregateMethods(protocol: Protocol, registry: FunctionR
     handler: async (params) => {
       const innerParams = unwrapRequest(params.request);
       const functionName: string = innerParams.function_name;
-      const cfg = resolveAggregate(functionName);
+      const cfg = resolveAggregate(functionName, innerParams.schema_name ?? null);
       const argBytes = toUint8Array(innerParams.arguments);
       const args = deserializeArguments(argBytes);
       const inputSchema = innerParams.input_schema
@@ -109,7 +121,7 @@ export function registerAggregateMethods(protocol: Protocol, registry: FunctionR
       const innerParams = unwrapRequest(params.request);
       const functionName: string = innerParams.function_name;
       const executionId = toUint8Array(innerParams.execution_id);
-      const cfg = resolveAggregate(functionName);
+      const cfg = resolveAggregate(functionName, innerParams.schema_name ?? null);
       const batch = readSingleBatch(toUint8Array(innerParams.input_batch));
       if (!batch || batch.numRows === 0) return wrapResult({}, makeSchema([]));
 
@@ -176,7 +188,7 @@ export function registerAggregateMethods(protocol: Protocol, registry: FunctionR
       const innerParams = unwrapRequest(params.request);
       const functionName: string = innerParams.function_name;
       const executionId = toUint8Array(innerParams.execution_id);
-      const cfg = resolveAggregate(functionName);
+      const cfg = resolveAggregate(functionName, innerParams.schema_name ?? null);
       const batch = readSingleBatch(toUint8Array(innerParams.merge_batch));
       if (!batch || batch.numRows === 0) return wrapResult({}, makeSchema([]));
 
@@ -225,7 +237,7 @@ export function registerAggregateMethods(protocol: Protocol, registry: FunctionR
       const innerParams = unwrapRequest(params.request);
       const functionName: string = innerParams.function_name;
       const executionId = toUint8Array(innerParams.execution_id);
-      const cfg = resolveAggregate(functionName);
+      const cfg = resolveAggregate(functionName, innerParams.schema_name ?? null);
       const outputSchema = deserializeSchema(toUint8Array(innerParams.output_schema));
       const gidBatch = readSingleBatch(toUint8Array(innerParams.group_ids_batch));
       const groupIds: bigint[] = [];

package/src/protocol/handlers/catalog/admin.ts

@@ -108,6 +108,12 @@ export function registerCatalogAdminMethods(protocol: Protocol, getCatalog: GetC
         // is also true. Catalogs that never serve column stats can override
         // this in attach() via CatalogAttachResult.supports_column_statistics.
         supports_column_statistics: result.supports_column_statistics ?? true,
+        // Protocol 1.3.0: functions the worker asks the client to publish into
+        // its global namespace. Empty unless the catalog advertises them —
+        // but the columns must still be present, since the extension pins the
+        // full CatalogAttachResult field list on the response schema.
+        global_functions: result.global_functions ?? [],
+        global_function_prefix: result.global_function_prefix ?? "",
         resolved_data_version: result.resolved_data_version ?? null,
         resolved_implementation_version: result.resolved_implementation_version ?? null,
       }, CatalogAttachResultSchema);

package/src/protocol/handlers/table-buffering.ts

@@ -43,11 +43,19 @@ export function registerTableBufferingMethods(
   registry: FunctionRegistry,
   signingKey?: Uint8Array,
 ): void {
-  function resolveBuffering(name: string): TableBufferingVgiFunction {
+  // `schemaName` scopes resolution to the schema the caller named. The
+  // buffering process/combine RPCs carry it (protocol 1.2.0), so a name
+  // declared in two schemas resolves to the one the request names rather than
+  // colliding on the flat by-name index. `null` keeps the cross-schema lookup.
+  function resolveBuffering(
+    name: string,
+    schemaName?: string | null,
+  ): TableBufferingVgiFunction {
     const func = registry.get(name, {
       arguments: new Arguments(),
       inputSchema: null,
       isScalar: false,
+      schemaName: schemaName ?? null,
     }) as any;
     if (!func || func.kind !== "table_buffering") {
       throw new Error(`Function '${name}' is not a table_buffering function`);
@@ -75,7 +83,7 @@ export function registerTableBufferingMethods(
         : rawAttach;
     const transactionId = inner.transaction_id ? toUint8Array(inner.transaction_id) : null;
 
-    const func = resolveBuffering(functionName);
+    const func = resolveBuffering(functionName, inner.schema_name ?? null);
     const bound = new BoundStorage(defaultStorage, executionId);
     const payload = await bound.stateGet(
       FrameworkNS.BUFFERING_INIT,