@query-farm/vgi 0.21.0 → 0.22.0

package/dist/functions/registry.d.ts

@@ -5,11 +5,42 @@ export interface OverloadContext {
     arguments?: Arguments;
     inputSchema?: VgiSchema | null;
     isScalar?: boolean;
+    /**
+     * Catalog schema that declares the function, from `BindRequest.schema_name`.
+     * A worker may register the same name in more than one schema, so the bare
+     * name is not a unique key. When set, resolution is scoped to that schema and
+     * never falls back to another one; when absent, every schema is searched.
+     */
+    schemaName?: string | null;
+    /**
+     * Catalog that owns the function, resolved from the bind's
+     * `attach_opaque_data`. Two catalogs served by one worker may each declare
+     * the same schema *and* function name, in which case only the catalog tells
+     * them apart. Absent for calls with no attachment.
+     */
+    catalogName?: string | null;
 }
 export declare class FunctionRegistry {
     private _functions;
+    private _bySchema;
+    private _byCatalog;
     register(func: VgiFunction): void;
+    /**
+     * Record that `func` is declared in `schemaName`, in addition to the flat
+     * by-name index. Idempotent, and independent of `register()` so a function
+     * reachable from several schemas (e.g. a scan function referenced by tables in
+     * more than one schema) resolves from any of them.
+     */
+    registerInSchema(func: VgiFunction, schemaName: string, catalogName?: string): void;
+    /** Schemas that declare `functionName`, sorted — for error messages. */
+    schemasFor(functionName: string): string[];
     get(name: string, context?: OverloadContext): VgiFunction;
+    /**
+     * Pick one function from an already-scoped candidate list, by argument shape
+     * and types. Scoping (catalog / schema) happens in `get()`; by the time this
+     * runs, every candidate is a legitimate overload of the same name.
+     */
+    private _disambiguate;
     has(name: string): boolean;
     names(): string[];
     all(): VgiFunction[];

package/dist/functions/scalar.d.ts

@@ -112,10 +112,14 @@ export interface ScalarFunctionConfig<P extends Record<string, VgiDataType> = Re
     /**
      * Result-cache opt-in: when set, this CacheControl's `vgi.cache.*` metadata
      * rides every output batch's custom metadata (per-batch — NOT the schema,
-     * which the IPC stream fixes at open), so the extension can memoize the
-     * scalar's output per distinct input value. A pure, deterministic scalar
-     * only — advertising this on a non-pure scalar serves stale rows. Mirrors
+     * which the IPC stream fixes at open). A pure, deterministic scalar only —
+     * advertising this on a non-pure scalar serves stale rows. Mirrors
      * vgi-python's `ScalarFunction.CACHE_CONTROL`.
+     *
+     * Per-VALUE memoization (the extension remembering one output per distinct
+     * input value) additionally requires `perValue: true`, which is off by
+     * default because a memo serve only beats a worker call when the call is
+     * expensive — see {@link CacheControl.perValue}.
      */
     cacheControl?: CacheControl;
 }

package/dist/generated/vgi-client.d.ts

@@ -141,6 +141,7 @@ export interface AggregateBindRequest {
     settings?: /** @record-batch */ Uint8Array | null;
     secrets?: /** @record-batch */ Uint8Array | null;
     attach_opaque_data?: Uint8Array | null;
+    schema_name?: string | null;
 }
 export interface AggregateBindResponse {
     output_schema: Uint8Array;
@@ -151,6 +152,7 @@ export interface AggregateCombineRequest {
     execution_id: Uint8Array;
     merge_batch: Uint8Array;
     attach_opaque_data?: Uint8Array | null;
+    schema_name?: string | null;
 }
 export interface AggregateCombineResponse {
 }
@@ -159,6 +161,7 @@ export interface AggregateDestructorRequest {
     execution_id: Uint8Array;
     group_ids_batch: Uint8Array;
     attach_opaque_data?: Uint8Array | null;
+    schema_name?: string | null;
 }
 export interface AggregateDestructorResponse {
 }
@@ -168,6 +171,7 @@ export interface AggregateFinalizeRequest {
     group_ids_batch: Uint8Array;
     output_schema: Uint8Array;
     attach_opaque_data?: Uint8Array | null;
+    schema_name?: string | null;
 }
 export interface AggregateFinalizeResponse {
     result_batch: Uint8Array;
@@ -177,6 +181,7 @@ export interface AggregateStreamingChunkRequest {
     execution_id: Uint8Array;
     input_batch: Uint8Array;
     attach_opaque_data?: Uint8Array | null;
+    schema_name?: string | null;
 }
 export interface AggregateStreamingChunkResponse {
     result_batch: Uint8Array;
@@ -185,6 +190,7 @@ export interface AggregateStreamingCloseRequest {
     function_name: string;
     execution_id: Uint8Array;
     attach_opaque_data?: Uint8Array | null;
+    schema_name?: string | null;
 }
 export interface AggregateStreamingCloseResponse {
 }
@@ -198,6 +204,7 @@ export interface AggregateStreamingOpenRequest {
     settings?: /** @record-batch */ Uint8Array | null;
     secrets?: /** @record-batch */ Uint8Array | null;
     attach_opaque_data?: Uint8Array | null;
+    schema_name?: string | null;
 }
 export interface AggregateStreamingOpenResponse {
     execution_id: Uint8Array;
@@ -207,6 +214,7 @@ export interface AggregateUpdateRequest {
     execution_id: Uint8Array;
     input_batch: Uint8Array;
     attach_opaque_data?: Uint8Array | null;
+    schema_name?: string | null;
 }
 export interface AggregateUpdateResponse {
 }
@@ -218,6 +226,7 @@ export interface AggregateWindowRequest {
     frame_starts: number[];
     frame_ends: number[];
     attach_opaque_data?: Uint8Array | null;
+    schema_name?: string | null;
 }
 export interface AggregateWindowResponse {
     result_batch: Uint8Array;
@@ -232,6 +241,7 @@ export interface AggregateWindowBatchRequest {
     frame_starts: number[];
     frame_ends: number[];
     attach_opaque_data?: Uint8Array | null;
+    schema_name?: string | null;
 }
 export interface AggregateWindowBatchResponse {
     result_batch: Uint8Array;
@@ -241,6 +251,7 @@ export interface AggregateWindowDestructorRequest {
     execution_id: Uint8Array;
     partition_id: number;
     attach_opaque_data?: Uint8Array | null;
+    schema_name?: string | null;
 }
 export interface AggregateWindowDestructorResponse {
 }
@@ -255,6 +266,7 @@ export interface AggregateWindowInitRequest {
     frame_stats: Uint8Array;
     all_valid: Uint8Array;
     attach_opaque_data?: Uint8Array | null;
+    schema_name?: string | null;
 }
 export interface AggregateWindowInitResponse {
 }
@@ -281,6 +293,7 @@ export interface BindRequest {
     at_value?: string | null;
     copy_from?: CopyFromContext | null;
     copy_to?: CopyToContext | null;
+    schema_name?: string | null;
 }
 export interface BindResponse {
     output_schema: Uint8Array;
@@ -409,6 +422,7 @@ export interface TableBufferingCombineRequest {
     state_ids: Uint8Array[];
     attach_opaque_data?: Uint8Array | null;
     transaction_id?: Uint8Array | null;
+    schema_name?: string | null;
 }
 export interface TableBufferingCombineResponse {
     finalize_state_ids: Uint8Array[];
@@ -418,6 +432,7 @@ export interface TableBufferingDestructorRequest {
     execution_id: Uint8Array;
     attach_opaque_data?: Uint8Array | null;
     transaction_id?: Uint8Array | null;
+    schema_name?: string | null;
 }
 export interface TableBufferingDestructorResponse {
 }
@@ -428,6 +443,7 @@ export interface TableBufferingProcessRequest {
     attach_opaque_data?: Uint8Array | null;
     transaction_id?: Uint8Array | null;
     batch_index?: number | null;
+    schema_name?: string | null;
 }
 export interface TableBufferingProcessResponse {
     state_id: Uint8Array;

package/dist/index.core.d.ts

@@ -1,4 +1,4 @@
-export { CACHE_ETAG_KEY, CACHE_EXPIRES_KEY, CACHE_IF_MODIFIED_SINCE_KEY, CACHE_IF_NONE_MATCH_KEY, CACHE_LAST_MODIFIED_KEY, CACHE_NO_STORE_KEY, CACHE_NOT_MODIFIED_KEY, CACHE_PARTITION_SCOPE_KEY, CACHE_REVALIDATABLE_KEY, CACHE_SCOPE_CATALOG, CACHE_SCOPE_KEY, CACHE_SCOPE_TRANSACTION, CACHE_STALE_IF_ERROR_KEY, CACHE_STALE_WHILE_REVALIDATE_KEY, CACHE_TTL_KEY, type CacheControl, cacheControlMetadata, type CacheScope, } from "./cache-control.js";
+export { CACHE_ETAG_KEY, CACHE_EXPIRES_KEY, CACHE_IF_MODIFIED_SINCE_KEY, CACHE_IF_NONE_MATCH_KEY, CACHE_LAST_MODIFIED_KEY, CACHE_NO_STORE_KEY, CACHE_NOT_MODIFIED_KEY, CACHE_PARTITION_SCOPE_KEY, CACHE_PER_VALUE_KEY, CACHE_REVALIDATABLE_KEY, CACHE_SCOPE_CATALOG, CACHE_SCOPE_KEY, CACHE_SCOPE_TRANSACTION, CACHE_STALE_IF_ERROR_KEY, CACHE_STALE_WHILE_REVALIDATE_KEY, CACHE_TTL_KEY, type CacheControl, cacheControlMetadata, type CacheScope, } from "./cache-control.js";
 export { defineScalarFunction, type ScalarFunctionConfig, type ScalarBindParameters, type ScalarParameterDef } from "./functions/scalar.js";
 export { defineTableFunction, type TableFunctionConfig, type TableBindParams, type TableProcessParams } from "./functions/table.js";
 export { defineAggregate, GROUP_COLUMN_NAME, type AggregateFunctionConfig, type AggregateBindParams, type AggregateUpdateParams, type AggregateFinalizeParams, } from "./functions/aggregate.js";

package/dist/index.js

@@ -2611,6 +2611,7 @@ var CACHE_STALE_WHILE_REVALIDATE_KEY = "vgi.cache.stale_while_revalidate";
 var CACHE_STALE_IF_ERROR_KEY = "vgi.cache.stale_if_error";
 var CACHE_NOT_MODIFIED_KEY = "vgi.cache.not_modified";
 var CACHE_PARTITION_SCOPE_KEY = "vgi.cache.partition_scope";
+var CACHE_PER_VALUE_KEY = "vgi.cache.per_value";
 var CACHE_IF_NONE_MATCH_KEY = "vgi.cache.if_none_match";
 var CACHE_IF_MODIFIED_SINCE_KEY = "vgi.cache.if_modified_since";
 var CACHE_SCOPE_CATALOG = "catalog";
@@ -2649,6 +2650,8 @@ function cacheControlMetadata(cc, extra) {
     md.set(CACHE_NOT_MODIFIED_KEY, "1");
   if (cc.partitionScope)
     md.set(CACHE_PARTITION_SCOPE_KEY, "1");
+  if (cc.perValue)
+    md.set(CACHE_PER_VALUE_KEY, "1");
   return md;
 }
 // src/functions/scalar.ts
@@ -5105,7 +5108,8 @@ var BIND_REQUEST_SCHEMA = schema2([
   field2("transaction_opaque_data", binary2(), true),
   field2("resolved_secrets_provided", bool2(), false),
   field2("at_unit", utf82(), true),
-  field2("at_value", utf82(), true)
+  field2("at_value", utf82(), true),
+  field2("schema_name", utf82(), true)
 ]);
 var COPY_FROM_STRUCT_TYPE = struct2([
   field2("format", utf82(), false),
@@ -5137,7 +5141,8 @@ function serializeBindRequest(req) {
     transaction_opaque_data: req.transaction_opaque_data ?? null,
     resolved_secrets_provided: req.resolved_secrets_provided ?? false,
     at_unit: req.at_unit ?? null,
-    at_value: req.at_value ?? null
+    at_value: req.at_value ?? null,
+    schema_name: req.schema_name ?? null
   };
   const extraFields = [];
   if (req.copy_from) {
@@ -5209,6 +5214,7 @@ function deserializeBindRequest(params) {
     resolved_secrets_provided: params.resolved_secrets_provided ?? false,
     at_unit: params.at_unit ? String(params.at_unit) : null,
     at_value: params.at_value ? String(params.at_value) : null,
+    schema_name: params.schema_name ? String(params.schema_name) : null,
     copy_from: parseCopyFromContext(params.copy_from),
     copy_to: parseCopyToContext(params.copy_to)
   };
@@ -6786,6 +6792,10 @@ function isPromise(v) {
 }
 
 class CatalogInterface {
+  catalogNameForAttach(_attachOpaqueData) {
+    const names = this.catalogs();
+    return names.length === 1 ? names[0] : null;
+  }
   create(name, onConflict, options) {
     throw new CatalogReadOnlyError("create");
   }
@@ -7020,6 +7030,17 @@ class ReadOnlyCatalogInterface extends CatalogInterface {
     super();
     this._descriptor = descriptor;
     this._registry = registry;
+    for (const sch of descriptor.schemas ?? []) {
+      for (const fn of sch.functions ?? [])
+        registry.registerInSchema(fn, sch.name, descriptor.name);
+      for (const tbl of sch.tables ?? []) {
+        for (const key of ["function", "insertFunction", "updateFunction", "deleteFunction"]) {
+          const fn = tbl[key];
+          if (fn)
+            registry.registerInSchema(fn, sch.name, descriptor.name);
+        }
+      }
+    }
   }
   catalogs() {
     return [this._descriptor.name];
@@ -7642,6 +7663,13 @@ class CompositeCatalogInterface extends CatalogInterface {
     }
     return route;
   }
+  catalogNameForAttach(attachOpaqueData) {
+    try {
+      return this._route(attachOpaqueData).catalogNameForAttach(attachOpaqueData);
+    } catch {
+      return null;
+    }
+  }
   catalogs() {
     const all = [];
     for (const b of this._backends)
@@ -7870,9 +7898,17 @@ function filterByArgumentTypes(candidates, args, inputSchema, isScalar) {
   const maxScore = Math.max(...scored.map((s) => s.score));
   return scored.filter((s) => s.score === maxScore).map((s) => s.func);
 }
+function schemaKey(schemaName, functionName) {
+  return `${schemaName.toLowerCase()}\x00${functionName}`;
+}
+function catalogKey(catalogName, schemaName, functionName) {
+  return `${catalogName.toLowerCase()}\x00${schemaKey(schemaName, functionName)}`;
+}
 
 class FunctionRegistry {
   _functions = new Map;
+  _bySchema = new Map;
+  _byCatalog = new Map;
   register(func) {
     const name = func.meta.name;
     if (!this._functions.has(name)) {
@@ -7880,11 +7916,57 @@ class FunctionRegistry {
     }
     this._functions.get(name).push(func);
   }
+  registerInSchema(func, schemaName, catalogName) {
+    const add = (map3, key) => {
+      const bucket = map3.get(key);
+      if (!bucket) {
+        map3.set(key, [func]);
+        return;
+      }
+      if (!bucket.includes(func))
+        bucket.push(func);
+    };
+    add(this._bySchema, schemaKey(schemaName, func.meta.name));
+    if (catalogName) {
+      add(this._byCatalog, catalogKey(catalogName, schemaName, func.meta.name));
+    }
+  }
+  schemasFor(functionName) {
+    const out = [];
+    for (const key of this._bySchema.keys()) {
+      const [schema3, name] = key.split("\x00");
+      if (name === functionName)
+        out.push(schema3);
+    }
+    return out.sort();
+  }
   get(name, context) {
-    const candidates = this._functions.get(name);
+    let candidates = this._functions.get(name);
+    const catalogName = context?.catalogName;
+    const schemaName = context?.schemaName;
+    if (catalogName && schemaName) {
+      const scoped = this._byCatalog.get(catalogKey(catalogName, schemaName, name));
+      if (scoped && scoped.length > 0) {
+        return this._disambiguate(name, scoped, context);
+      }
+    }
+    if (schemaName) {
+      const scoped = this._bySchema.get(schemaKey(schemaName, name));
+      if (scoped && scoped.length > 0) {
+        candidates = scoped;
+      } else if (candidates && candidates.length > 0) {
+        const schemas = this.schemasFor(name);
+        if (schemas.length > 0) {
+          throw new Error(`Function '${name}' is not registered in schema '${schemaName}'. ` + `It is available in: [${schemas.join(", ")}]`);
+        }
+      }
+    }
     if (!candidates || candidates.length === 0) {
       throw new FunctionNotFoundError(name, this.names());
     }
+    return this._disambiguate(name, candidates, context);
+  }
+  _disambiguate(name, candidates, context) {
     if (candidates.length === 1) {
       return candidates[0];
     }
@@ -8170,11 +8252,21 @@ function wrapResult(values, innerSchema) {
   const batch = batchFromColumns(Object.fromEntries(a.fields.map((f) => [f.name, [values[f.name] ?? null]])), a);
   return { result: serializeBatch(batch) };
 }
-function overloadContext(req) {
+function overloadContext(req, catalogInterface) {
+  let catalogName = null;
+  if (catalogInterface && req.attach_opaque_data) {
+    try {
+      catalogName = catalogInterface.catalogNameForAttach(req.attach_opaque_data);
+    } catch {
+      catalogName = null;
+    }
+  }
   return {
     arguments: req.arguments,
     inputSchema: req.input_schema,
-    isScalar: String(req.function_type).toLowerCase() === "scalar"
+    isScalar: String(req.function_type).toLowerCase() === "scalar",
+    schemaName: req.schema_name ?? null,
+    catalogName
   };
 }
 function recoverFinalizeState(request, recoverExchangeState) {
@@ -9147,7 +9239,7 @@ var columnOpParams = schema2([
 
 // src/protocol/handlers/function.ts
 function registerFunctionMethods(protocol, config) {
-  const { registry, signingKey } = config;
+  const { registry, signingKey, catalogInterface } = config;
   async function stripAttach(attach, ctx) {
     if (attach == null)
       return null;
@@ -9156,6 +9248,15 @@ function registerFunctionMethods(protocol, config) {
       return env;
     return openAttach(env, ctx?.auth, signingKey);
   }
+  async function strippedContext(bindCall, ctx) {
+    let attach = null;
+    try {
+      attach = await stripAttach(bindCall.attach_opaque_data, ctx);
+    } catch {
+      attach = null;
+    }
+    return overloadContext({ ...bindCall, attach_opaque_data: attach }, catalogInterface);
+  }
   const initHeaderSchema = schema2([
     field2("execution_id", binary2(), false),
     field2("opaque_data", binary2(), true),
@@ -9172,7 +9273,7 @@ function registerFunctionMethods(protocol, config) {
       const innerParams = unwrapRequest(params.request);
       const request = deserializeBindRequest(innerParams);
       request.attach_opaque_data = await stripAttach(request.attach_opaque_data, ctx);
-      const func = registry.get(request.function_name, overloadContext(request));
+      const func = registry.get(request.function_name, overloadContext(request, catalogInterface));
       const response = await func.bind(request);
       const serialized = serializeBindResponse(response);
       return wrapResult(serialized, BindResultSchema);
@@ -9187,7 +9288,7 @@ function registerFunctionMethods(protocol, config) {
       const innerParams = unwrapRequest(params.request);
       const request = deserializeInitRequest(innerParams);
       request.bind_call.attach_opaque_data = await stripAttach(request.bind_call.attach_opaque_data, undefined);
-      const func = registry.get(request.bind_call.function_name, overloadContext(request.bind_call));
+      const func = registry.get(request.bind_call.function_name, overloadContext(request.bind_call, catalogInterface));
       const initResponse = await func.globalInit(request);
       const accumulatedState = recoverFinalizeState(request, config.recoverExchangeState);
       const handlers = func.createStreamHandlers(request, initResponse, accumulatedState);
@@ -9227,7 +9328,7 @@ function registerFunctionMethods(protocol, config) {
         const initRequestBatch = deserializeBatch(state.initRequestIpc);
         const initRequestDict = batchToScalarDict(initRequestBatch);
         const request = deserializeInitRequest(initRequestDict);
-        const func = registry.get(state.functionName, overloadContext(request.bind_call));
+        const func = registry.get(state.functionName, await strippedContext(request.bind_call));
         const executionId = state.executionId;
         const opaqueData = state.opaqueData ?? null;
         const initResponse = {
@@ -9275,7 +9376,7 @@ function registerFunctionMethods(protocol, config) {
     handler: async (params) => {
       const innerParams = unwrapRequest(params.request);
       const request = deserializeCardinalityRequest(innerParams);
-      const func = registry.get(request.bind_call.function_name, overloadContext(request.bind_call));
+      const func = registry.get(request.bind_call.function_name, await strippedContext(request.bind_call));
       let cardResult;
       if (func.cardinality) {
         cardResult = serializeTableCardinality(await func.cardinality(request));
@@ -9288,10 +9389,10 @@ function registerFunctionMethods(protocol, config) {
   protocol.unary("table_function_statistics", {
     params: REQUEST_PARAMS_SCHEMA,
     result: RESULT_BINARY_NULLABLE_SCHEMA,
-    handler: (params) => {
+    handler: async (params) => {
       const innerParams = unwrapRequest(params.request);
       const request = deserializeCardinalityRequest(innerParams);
-      const func = registry.get(request.bind_call.function_name, overloadContext(request.bind_call));
+      const func = registry.get(request.bind_call.function_name, await strippedContext(request.bind_call));
       if (!func.statistics)
         return { result: null };
       const stats = func.statistics(request);
@@ -9311,7 +9412,7 @@ function registerFunctionMethods(protocol, config) {
       const bindCall = deserializeBindRequest(bindParams);
       const globalExecutionId = toUint8Array(innerParams.global_execution_id);
       const bindOpaqueData = innerParams.bind_opaque_data ? toUint8Array(innerParams.bind_opaque_data) : null;
-      const func = registry.get(bindCall.function_name, overloadContext(bindCall));
+      const func = registry.get(bindCall.function_name, await strippedContext(bindCall));
       const map3 = func.dynamicToString ? await func.dynamicToString({ bindCall, bindOpaqueData, globalExecutionId }) : {};
       const keys = Object.keys(map3);
       const values = keys.map((k) => map3[k] ?? "");
@@ -10311,7 +10412,8 @@ function buildVgiProtocol(config) {
   registerFunctionMethods(protocol, {
     registry: config.registry,
     recoverExchangeState: config.recoverExchangeState,
-    signingKey: config.signingKey
+    signingKey: config.signingKey,
+    catalogInterface: config.catalogInterface
   });
   registerAggregateMethods(protocol, config.registry);
   registerTableBufferingMethods(protocol, config.registry, config.signingKey);
@@ -11696,7 +11798,7 @@ class Worker {
   _catalogInterface;
   _catalogName;
   constructor(config) {
-    this._registry = new FunctionRegistry;
+    this._registry = config.registry ?? new FunctionRegistry;
     this._catalogName = config.catalogName;
     if (config.functions) {
       for (const func of config.functions) {
@@ -11708,6 +11810,7 @@ class Worker {
         if (schema3.functions) {
           for (const func of schema3.functions) {
             this._registry.register(func);
+            this._registry.registerInSchema(func, schema3.name, config.catalog.name);
           }
         }
       }
@@ -12017,6 +12120,7 @@ export {
   CACHE_SCOPE_KEY,
   CACHE_SCOPE_CATALOG,
   CACHE_REVALIDATABLE_KEY,
+  CACHE_PER_VALUE_KEY,
   CACHE_PARTITION_SCOPE_KEY,
   CACHE_NO_STORE_KEY,
   CACHE_NOT_MODIFIED_KEY,
@@ -12032,4 +12136,4 @@ export {
   ArgumentValidationError
 };
 
-//# debugId=16E563A7CD4C6BEC64756E2164756E21
+//# debugId=AC11F66207EB748964756E2164756E21