@query-farm/vgi 0.20.0 → 0.22.0

package/dist/util/arrow/index.d.ts

@@ -1 +1 @@
-export { emptyBatch, batchFromRows, batchFromColumns, iterRows, batchToScalarDict, batchToSecretDict, safeNumber, decodeDictValue, filterBatch, projectSchema, projectBatch, serializeSchema, deserializeSchema, serializeBatch, deserializeBatch, } from "../../arrow/index.js";
+export { emptyBatch, batchFromRows, batchFromColumns, iterRows, batchToScalarDict, batchToSecretDict, safeNumber, decodeDictValue, filterBatch, projectSchema, projectBatch, serializeSchema, deserializeSchema, serializeBatch, deserializeBatch, adoptArrowJsShape, } from "../../arrow/index.js";

package/dist/worker-cf-entry.js

@@ -95,6 +95,42 @@ function isDuration(t) {
 function isDictionary(t) {
   return t.typeId === TypeId.Dictionary;
 }
+function typeSignature(t) {
+  if (t == null)
+    return "null";
+  const a = t;
+  const kids = () => Array.isArray(a.children) ? a.children.map((c) => `${c?.name ?? ""}:${typeSignature(c?.type ?? c)}`).join(",") : "";
+  switch (t.typeId) {
+    case TypeId.Int:
+      return `int${a.bitWidth}${a.isSigned ?? a.signed ?? true ? "" : "u"}`;
+    case TypeId.Float:
+      return `float${a.precision}`;
+    case TypeId.Decimal:
+      return `decimal${a.bitWidth ?? 128}(${a.precision},${a.scale})`;
+    case TypeId.Date:
+    case TypeId.Time:
+    case TypeId.Duration:
+      return `${t.typeId}[${a.unit}]`;
+    case TypeId.Timestamp:
+      return `ts[${a.unit},${a.timezone ?? ""}]`;
+    case TypeId.Interval:
+      return `iv[${a.unit}]`;
+    case TypeId.FixedSizeBinary:
+      return `fsb(${a.byteWidth ?? a.stride})`;
+    case TypeId.FixedSizeList:
+      return `fsl(${a.listSize ?? a.stride},${kids()})`;
+    case TypeId.List:
+    case TypeId.Struct:
+    case TypeId.Map:
+      return `${t.typeId}<${kids()}>`;
+    case TypeId.Union:
+      return `union(${a.mode},${kids()})`;
+    case TypeId.Dictionary:
+      return `dict(${typeSignature(a.dictionary)},${typeSignature(a.indices)},${a.ordered ?? a.isOrdered ?? false})`;
+    default:
+      return Array.isArray(a.children) ? `${t.typeId}<${kids()}>` : String(t.typeId);
+  }
+}
 function isBatch(x) {
   return x != null && typeof x.numRows === "number" && x.schema != null && Array.isArray(x.schema.fields);
 }
@@ -1825,11 +1861,15 @@ __export(exports_impl_arrowjs, {
   batchFromRows: () => batchFromRows,
   batchFromColumns: () => batchFromColumns,
   backend: () => backend,
+  adoptArrowJsShape: () => adoptArrowJsShape,
   UnionMode: () => UnionMode,
   TimeUnit: () => TimeUnit,
   IntervalUnit: () => IntervalUnit,
   DateUnit: () => DateUnit
 });
+function adoptArrowJsShape(batch) {
+  return batch;
+}
 var backend;
 var init_impl_arrowjs = __esm(() => {
   init_schema();
@@ -1945,6 +1985,7 @@ __export(exports_arrow, {
   uint64: () => uint642,
   uint32: () => uint322,
   uint16: () => uint162,
+  typeSignature: () => typeSignature,
   timestampSeconds: () => timestampSeconds,
   timestampNanos: () => timestampNanos,
   timestampMillis: () => timestampMillis,
@@ -2048,6 +2089,7 @@ __export(exports_arrow, {
   asDurationMicros: () => asDurationMicros,
   asDate64Ms: () => asDate64Ms,
   asDate32: () => asDate32,
+  adoptArrowJsShape: () => adoptArrowJsShape,
   UnionMode: () => UnionMode2,
   TypeId: () => TypeId,
   TimeUnit: () => TimeUnit2,
@@ -2893,7 +2935,8 @@ var BIND_REQUEST_SCHEMA = schema2([
   field2("transaction_opaque_data", binary2(), true),
   field2("resolved_secrets_provided", bool2(), false),
   field2("at_unit", utf82(), true),
-  field2("at_value", utf82(), true)
+  field2("at_value", utf82(), true),
+  field2("schema_name", utf82(), true)
 ]);
 var COPY_FROM_STRUCT_TYPE = struct2([
   field2("format", utf82(), false),
@@ -2961,6 +3004,7 @@ function deserializeBindRequest(params) {
     resolved_secrets_provided: params.resolved_secrets_provided ?? false,
     at_unit: params.at_unit ? String(params.at_unit) : null,
     at_value: params.at_value ? String(params.at_value) : null,
+    schema_name: params.schema_name ? String(params.schema_name) : null,
     copy_from: parseCopyFromContext(params.copy_from),
     copy_to: parseCopyToContext(params.copy_to)
   };
@@ -3791,11 +3835,21 @@ function wrapResult(values, innerSchema) {
   const batch = batchFromColumns(Object.fromEntries(a.fields.map((f) => [f.name, [values[f.name] ?? null]])), a);
   return { result: serializeBatch(batch) };
 }
-function overloadContext(req) {
+function overloadContext(req, catalogInterface) {
+  let catalogName = null;
+  if (catalogInterface && req.attach_opaque_data) {
+    try {
+      catalogName = catalogInterface.catalogNameForAttach(req.attach_opaque_data);
+    } catch {
+      catalogName = null;
+    }
+  }
   return {
     arguments: req.arguments,
     inputSchema: req.input_schema,
-    isScalar: String(req.function_type).toLowerCase() === "scalar"
+    isScalar: String(req.function_type).toLowerCase() === "scalar",
+    schemaName: req.schema_name ?? null,
+    catalogName
   };
 }
 function recoverFinalizeState(request, recoverExchangeState) {
@@ -4810,7 +4864,7 @@ var columnOpParams = schema2([
 
 // src/protocol/handlers/function.ts
 function registerFunctionMethods(protocol, config) {
-  const { registry, signingKey } = config;
+  const { registry, signingKey, catalogInterface } = config;
   async function stripAttach(attach, ctx) {
     if (attach == null)
       return null;
@@ -4819,6 +4873,15 @@ function registerFunctionMethods(protocol, config) {
       return env;
     return openAttach(env, ctx?.auth, signingKey);
   }
+  async function strippedContext(bindCall, ctx) {
+    let attach = null;
+    try {
+      attach = await stripAttach(bindCall.attach_opaque_data, ctx);
+    } catch {
+      attach = null;
+    }
+    return overloadContext({ ...bindCall, attach_opaque_data: attach }, catalogInterface);
+  }
   const initHeaderSchema = schema2([
     field2("execution_id", binary2(), false),
     field2("opaque_data", binary2(), true),
@@ -4835,7 +4898,7 @@ function registerFunctionMethods(protocol, config) {
       const innerParams = unwrapRequest(params.request);
       const request = deserializeBindRequest(innerParams);
       request.attach_opaque_data = await stripAttach(request.attach_opaque_data, ctx);
-      const func = registry.get(request.function_name, overloadContext(request));
+      const func = registry.get(request.function_name, overloadContext(request, catalogInterface));
       const response = await func.bind(request);
       const serialized = serializeBindResponse(response);
       return wrapResult(serialized, BindResultSchema);
@@ -4850,7 +4913,7 @@ function registerFunctionMethods(protocol, config) {
       const innerParams = unwrapRequest(params.request);
       const request = deserializeInitRequest(innerParams);
       request.bind_call.attach_opaque_data = await stripAttach(request.bind_call.attach_opaque_data, undefined);
-      const func = registry.get(request.bind_call.function_name, overloadContext(request.bind_call));
+      const func = registry.get(request.bind_call.function_name, overloadContext(request.bind_call, catalogInterface));
       const initResponse = await func.globalInit(request);
       const accumulatedState = recoverFinalizeState(request, config.recoverExchangeState);
       const handlers = func.createStreamHandlers(request, initResponse, accumulatedState);
@@ -4880,6 +4943,7 @@ function registerFunctionMethods(protocol, config) {
       return state;
     },
     exchange: async (state, input, out) => {
+      input = adoptArrowJsShape(input);
       let handlers;
       let handlerState;
       if (state._handlers) {
@@ -4889,7 +4953,7 @@ function registerFunctionMethods(protocol, config) {
         const initRequestBatch = deserializeBatch(state.initRequestIpc);
         const initRequestDict = batchToScalarDict(initRequestBatch);
         const request = deserializeInitRequest(initRequestDict);
-        const func = registry.get(state.functionName, overloadContext(request.bind_call));
+        const func = registry.get(state.functionName, await strippedContext(request.bind_call));
         const executionId = state.executionId;
         const opaqueData = state.opaqueData ?? null;
         const initResponse = {
@@ -4937,7 +5001,7 @@ function registerFunctionMethods(protocol, config) {
     handler: async (params) => {
       const innerParams = unwrapRequest(params.request);
       const request = deserializeCardinalityRequest(innerParams);
-      const func = registry.get(request.bind_call.function_name, overloadContext(request.bind_call));
+      const func = registry.get(request.bind_call.function_name, await strippedContext(request.bind_call));
       let cardResult;
       if (func.cardinality) {
         cardResult = serializeTableCardinality(await func.cardinality(request));
@@ -4950,10 +5014,10 @@ function registerFunctionMethods(protocol, config) {
   protocol.unary("table_function_statistics", {
     params: REQUEST_PARAMS_SCHEMA,
     result: RESULT_BINARY_NULLABLE_SCHEMA,
-    handler: (params) => {
+    handler: async (params) => {
       const innerParams = unwrapRequest(params.request);
       const request = deserializeCardinalityRequest(innerParams);
-      const func = registry.get(request.bind_call.function_name, overloadContext(request.bind_call));
+      const func = registry.get(request.bind_call.function_name, await strippedContext(request.bind_call));
       if (!func.statistics)
         return { result: null };
       const stats = func.statistics(request);
@@ -4973,7 +5037,7 @@ function registerFunctionMethods(protocol, config) {
       const bindCall = deserializeBindRequest(bindParams);
       const globalExecutionId = toUint8Array(innerParams.global_execution_id);
       const bindOpaqueData = innerParams.bind_opaque_data ? toUint8Array(innerParams.bind_opaque_data) : null;
-      const func = registry.get(bindCall.function_name, overloadContext(bindCall));
+      const func = registry.get(bindCall.function_name, await strippedContext(bindCall));
       const map3 = func.dynamicToString ? await func.dynamicToString({ bindCall, bindOpaqueData, globalExecutionId }) : {};
       const keys = Object.keys(map3);
       const values = keys.map((k) => map3[k] ?? "");
@@ -6454,6 +6518,10 @@ function isPromise(v) {
 }
 
 class CatalogInterface {
+  catalogNameForAttach(_attachOpaqueData) {
+    const names = this.catalogs();
+    return names.length === 1 ? names[0] : null;
+  }
   create(name, onConflict, options) {
     throw new CatalogReadOnlyError("create");
   }
@@ -7188,7 +7256,8 @@ function buildVgiProtocol(config) {
   registerFunctionMethods(protocol, {
     registry: config.registry,
     recoverExchangeState: config.recoverExchangeState,
-    signingKey: config.signingKey
+    signingKey: config.signingKey,
+    catalogInterface: config.catalogInterface
   });
   registerAggregateMethods(protocol, config.registry);
   registerTableBufferingMethods(protocol, config.registry, config.signingKey);
@@ -7236,6 +7305,8 @@ var CACHE_REVALIDATABLE_KEY = "vgi.cache.revalidatable";
 var CACHE_STALE_WHILE_REVALIDATE_KEY = "vgi.cache.stale_while_revalidate";
 var CACHE_STALE_IF_ERROR_KEY = "vgi.cache.stale_if_error";
 var CACHE_NOT_MODIFIED_KEY = "vgi.cache.not_modified";
+var CACHE_PARTITION_SCOPE_KEY = "vgi.cache.partition_scope";
+var CACHE_PER_VALUE_KEY = "vgi.cache.per_value";
 var CACHE_IF_NONE_MATCH_KEY = "vgi.cache.if_none_match";
 var CACHE_IF_MODIFIED_SINCE_KEY = "vgi.cache.if_modified_since";
 var CACHE_SCOPE_CATALOG = "catalog";
@@ -7272,6 +7343,10 @@ function cacheControlMetadata(cc, extra) {
     md.set(CACHE_STALE_IF_ERROR_KEY, String(cc.staleIfError));
   if (cc.notModified)
     md.set(CACHE_NOT_MODIFIED_KEY, "1");
+  if (cc.partitionScope)
+    md.set(CACHE_PARTITION_SCOPE_KEY, "1");
+  if (cc.perValue)
+    md.set(CACHE_PER_VALUE_KEY, "1");
   return md;
 }
 
@@ -8042,6 +8117,17 @@ class ReadOnlyCatalogInterface extends CatalogInterface {
     super();
     this._descriptor = descriptor;
     this._registry = registry;
+    for (const sch of descriptor.schemas ?? []) {
+      for (const fn of sch.functions ?? [])
+        registry.registerInSchema(fn, sch.name, descriptor.name);
+      for (const tbl of sch.tables ?? []) {
+        for (const key of ["function", "insertFunction", "updateFunction", "deleteFunction"]) {
+          const fn = tbl[key];
+          if (fn)
+            registry.registerInSchema(fn, sch.name, descriptor.name);
+        }
+      }
+    }
   }
   catalogs() {
     return [this._descriptor.name];
@@ -8664,6 +8750,13 @@ class CompositeCatalogInterface extends CatalogInterface {
     }
     return route;
   }
+  catalogNameForAttach(attachOpaqueData) {
+    try {
+      return this._route(attachOpaqueData).catalogNameForAttach(attachOpaqueData);
+    } catch {
+      return null;
+    }
+  }
   catalogs() {
     const all = [];
     for (const b of this._backends)
@@ -8761,8 +8854,11 @@ class CompositeCatalogInterface extends CatalogInterface {
 init_arrow();
 var EXACT_MATCH_SCORE = 2;
 var FAMILY_MATCH_SCORE = 1;
+function typesIdentical(a, b) {
+  return typeSignature(a) === typeSignature(b);
+}
 function typesCompatible(actual, declared) {
-  if (actual.toString() === declared.toString())
+  if (typesIdentical(actual, declared))
     return true;
   if (isInt(actual) && isInt(declared))
     return true;
@@ -8790,7 +8886,7 @@ function scoreTypes(specs, actualTypes) {
     const actual = actualTypes[i];
     if (actual === null)
       continue;
-    if (actual.toString() === spec.arrowType.toString()) {
+    if (typesIdentical(actual, spec.arrowType)) {
       score += EXACT_MATCH_SCORE;
     } else if (typesCompatible(actual, spec.arrowType)) {
       score += FAMILY_MATCH_SCORE;
@@ -8803,7 +8899,7 @@ function scoreTypes(specs, actualTypes) {
       const actual = actualTypes[i];
       if (actual === null)
         continue;
-      if (actual.toString() === varArgsSpec.arrowType.toString()) {
+      if (typesIdentical(actual, varArgsSpec.arrowType)) {
         score += EXACT_MATCH_SCORE;
       } else if (typesCompatible(actual, varArgsSpec.arrowType)) {
         score += FAMILY_MATCH_SCORE;
@@ -8889,9 +8985,17 @@ function filterByArgumentTypes(candidates, args, inputSchema, isScalar) {
   const maxScore = Math.max(...scored.map((s) => s.score));
   return scored.filter((s) => s.score === maxScore).map((s) => s.func);
 }
+function schemaKey(schemaName, functionName) {
+  return `${schemaName.toLowerCase()}\x00${functionName}`;
+}
+function catalogKey(catalogName, schemaName, functionName) {
+  return `${catalogName.toLowerCase()}\x00${schemaKey(schemaName, functionName)}`;
+}
 
 class FunctionRegistry {
   _functions = new Map;
+  _bySchema = new Map;
+  _byCatalog = new Map;
   register(func) {
     const name = func.meta.name;
     if (!this._functions.has(name)) {
@@ -8899,11 +9003,57 @@ class FunctionRegistry {
     }
     this._functions.get(name).push(func);
   }
+  registerInSchema(func, schemaName, catalogName) {
+    const add = (map3, key) => {
+      const bucket = map3.get(key);
+      if (!bucket) {
+        map3.set(key, [func]);
+        return;
+      }
+      if (!bucket.includes(func))
+        bucket.push(func);
+    };
+    add(this._bySchema, schemaKey(schemaName, func.meta.name));
+    if (catalogName) {
+      add(this._byCatalog, catalogKey(catalogName, schemaName, func.meta.name));
+    }
+  }
+  schemasFor(functionName) {
+    const out = [];
+    for (const key of this._bySchema.keys()) {
+      const [schema3, name] = key.split("\x00");
+      if (name === functionName)
+        out.push(schema3);
+    }
+    return out.sort();
+  }
   get(name, context) {
-    const candidates = this._functions.get(name);
+    let candidates = this._functions.get(name);
+    const catalogName = context?.catalogName;
+    const schemaName = context?.schemaName;
+    if (catalogName && schemaName) {
+      const scoped = this._byCatalog.get(catalogKey(catalogName, schemaName, name));
+      if (scoped && scoped.length > 0) {
+        return this._disambiguate(name, scoped, context);
+      }
+    }
+    if (schemaName) {
+      const scoped = this._bySchema.get(schemaKey(schemaName, name));
+      if (scoped && scoped.length > 0) {
+        candidates = scoped;
+      } else if (candidates && candidates.length > 0) {
+        const schemas = this.schemasFor(name);
+        if (schemas.length > 0) {
+          throw new Error(`Function '${name}' is not registered in schema '${schemaName}'. ` + `It is available in: [${schemas.join(", ")}]`);
+        }
+      }
+    }
     if (!candidates || candidates.length === 0) {
       throw new FunctionNotFoundError(name, this.names());
     }
+    return this._disambiguate(name, candidates, context);
+  }
+  _disambiguate(name, candidates, context) {
     if (candidates.length === 1) {
       return candidates[0];
     }
@@ -9531,4 +9681,4 @@ export {
   Arguments
 };
 
-//# debugId=9CBEC1BA4EDD0C8364756E2164756E21
+//# debugId=0DAB6D0BDC711A3364756E2164756E21