@query-farm/vgi 0.18.0 → 0.19.1

package/dist/worker-cf-entry.js

@@ -1586,6 +1586,14 @@ function projectBatch(projectionIds, batch) {
 }
 var init_project = () => {};
 
+// src/arrow/impl-arrowjs/metadata.ts
+import { RecordBatch as RecordBatch4 } from "@query-farm/apache-arrow";
+function withBatchMetadata(batch, metadata) {
+  const a = batch;
+  return new RecordBatch4(a.schema, a.data, metadata);
+}
+var init_metadata = () => {};
+
 // src/arrow/impl-arrowjs/ipc.ts
 import {
   Schema as Schema4,
@@ -1634,7 +1642,7 @@ import {
   Field as Field3,
   Null,
   makeData as makeData5,
-  RecordBatch as RecordBatch5,
+  RecordBatch as RecordBatch6,
   Struct as Struct5,
   SparseUnion,
   Schema as Schema5,
@@ -1746,7 +1754,7 @@ function assemble(columnNames, minData, maxData, hasNull, hasNotNull, distinctCo
   ];
   const structType = new Struct5(schema2.fields);
   const data = makeData5({ type: structType, length: n, children, nullCount: 0 });
-  return new RecordBatch5(schema2, data);
+  return new RecordBatch6(schema2, data);
 }
 var init_statistics = __esm(() => {
   init_registry();
@@ -1756,6 +1764,7 @@ var init_statistics = __esm(() => {
 // src/arrow/impl-arrowjs/index.ts
 var exports_impl_arrowjs = {};
 __export(exports_impl_arrowjs, {
+  withBatchMetadata: () => withBatchMetadata,
   utf8: () => utf8,
   union: () => union,
   uint8: () => uint8,
@@ -1830,6 +1839,7 @@ var init_impl_arrowjs = __esm(() => {
   init_filter();
   init_project();
   init_canonical();
+  init_metadata();
   init_ipc();
   init_statistics();
   backend = { name: "arrow-js" };
@@ -1928,6 +1938,7 @@ var init_branded = __esm(() => {
 // src/arrow/index.ts
 var exports_arrow = {};
 __export(exports_arrow, {
+  withBatchMetadata: () => withBatchMetadata,
   utf8: () => utf82,
   union: () => union2,
   uint8: () => uint82,
@@ -2975,6 +2986,7 @@ var INIT_REQUEST_SCHEMA = schema2([
   field2("phase", utf82(), true),
   field2("execution_id", binary2(), true),
   field2("init_opaque_data", binary2(), true),
+  field2("substream_id", binary2(), true),
   field2("order_by_column_name", utf82(), true),
   field2("order_by_direction", utf82(), true),
   field2("order_by_null_order", utf82(), true),
@@ -3039,6 +3051,7 @@ function deserializeInitRequest(params) {
     finalize_state_id: params.finalize_state_id ? toUint8Array2(params.finalize_state_id) : null,
     execution_id: params.execution_id ? toUint8Array2(params.execution_id) : null,
     init_opaque_data: params.init_opaque_data ? toUint8Array2(params.init_opaque_data) : null,
+    substream_id: params.substream_id ? toUint8Array2(params.substream_id) : null,
     order_by_column_name: parseEnum(decodeDictValue(params.order_by_column_name)) ?? null,
     order_by_direction: parseDirection(decodeDictValue(params.order_by_direction)),
     order_by_null_order: parseNullOrder(decodeDictValue(params.order_by_null_order)),
@@ -3215,6 +3228,7 @@ var FunctionInfoSchema = schema2([
   field2("source_order_dependent", bool2(), false),
   field2("sink_order_dependent", bool2(), false),
   field2("requires_input_batch_index", bool2(), false),
+  field2("input_from_args", bool2(), false),
   field2("required_settings", list2(field2("item", utf82(), true)), false),
   field2("required_secrets", list2(field2("item", struct2([field2("secret_type", utf82(), false), field2("scope", utf82(), true), field2("secret_name", utf82(), true)]), true)), false)
 ]);
@@ -7210,6 +7224,58 @@ function createVgiFetch(opts) {
 }
 // src/functions/scalar.ts
 init_arrow();
+
+// src/cache-control.ts
+var CACHE_TTL_KEY = "vgi.cache.ttl";
+var CACHE_EXPIRES_KEY = "vgi.cache.expires";
+var CACHE_NO_STORE_KEY = "vgi.cache.no_store";
+var CACHE_SCOPE_KEY = "vgi.cache.scope";
+var CACHE_ETAG_KEY = "vgi.cache.etag";
+var CACHE_LAST_MODIFIED_KEY = "vgi.cache.last_modified";
+var CACHE_REVALIDATABLE_KEY = "vgi.cache.revalidatable";
+var CACHE_STALE_WHILE_REVALIDATE_KEY = "vgi.cache.stale_while_revalidate";
+var CACHE_STALE_IF_ERROR_KEY = "vgi.cache.stale_if_error";
+var CACHE_NOT_MODIFIED_KEY = "vgi.cache.not_modified";
+var CACHE_IF_NONE_MATCH_KEY = "vgi.cache.if_none_match";
+var CACHE_IF_MODIFIED_SINCE_KEY = "vgi.cache.if_modified_since";
+var CACHE_SCOPE_CATALOG = "catalog";
+var CACHE_SCOPE_TRANSACTION = "transaction";
+var VALID_SCOPES = [CACHE_SCOPE_CATALOG, CACHE_SCOPE_TRANSACTION];
+function cacheControlMetadata(cc, extra) {
+  const scope = cc.scope ?? CACHE_SCOPE_CATALOG;
+  if (!VALID_SCOPES.includes(scope)) {
+    throw new Error(`CacheControl.scope must be one of ${VALID_SCOPES.join(", ")}, got ${JSON.stringify(scope)}`);
+  }
+  for (const name of ["ttl", "staleWhileRevalidate", "staleIfError"]) {
+    const value = cc[name];
+    if (value !== undefined && value < 0) {
+      throw new Error(`CacheControl.${name} must be >= 0, got ${value}`);
+    }
+  }
+  const md = new Map(extra ?? []);
+  if (cc.ttl !== undefined)
+    md.set(CACHE_TTL_KEY, String(cc.ttl));
+  if (cc.expires !== undefined)
+    md.set(CACHE_EXPIRES_KEY, cc.expires);
+  if (cc.noStore)
+    md.set(CACHE_NO_STORE_KEY, "1");
+  md.set(CACHE_SCOPE_KEY, scope);
+  if (cc.etag !== undefined)
+    md.set(CACHE_ETAG_KEY, cc.etag);
+  if (cc.lastModified !== undefined)
+    md.set(CACHE_LAST_MODIFIED_KEY, cc.lastModified);
+  if (cc.revalidatable)
+    md.set(CACHE_REVALIDATABLE_KEY, "1");
+  if (cc.staleWhileRevalidate !== undefined)
+    md.set(CACHE_STALE_WHILE_REVALIDATE_KEY, String(cc.staleWhileRevalidate));
+  if (cc.staleIfError !== undefined)
+    md.set(CACHE_STALE_IF_ERROR_KEY, String(cc.staleIfError));
+  if (cc.notModified)
+    md.set(CACHE_NOT_MODIFIED_KEY, "1");
+  return md;
+}
+
+// src/functions/scalar.ts
 function defineScalarFunction(config) {
   const repr = config.repr ?? "rich";
   const specs = [];
@@ -7356,7 +7422,7 @@ function defineScalarFunction(config) {
           if (outputBatch.numRows !== input.numRows) {
             throw new RowCountMismatchError(input.numRows, outputBatch.numRows);
           }
-          out.emit(outputBatch);
+          out.emit(outputBatch, config.cacheControl ? cacheControlMetadata(config.cacheControl) : undefined);
         }
       };
     }
@@ -7366,12 +7432,6 @@ function defineScalarFunction(config) {
 init_arrow();
 init_registry();
 init_storage();
-
-// src/cache-control.ts
-var CACHE_IF_NONE_MATCH_KEY = "vgi.cache.if_none_match";
-var CACHE_IF_MODIFIED_SINCE_KEY = "vgi.cache.if_modified_since";
-
-// src/functions/table.ts
 function base64Decode(s) {
   const bin = globalThis.atob ? globalThis.atob(s) : Buffer.from(s, "base64").toString("binary");
   const out = new Uint8Array(bin.length);
@@ -7762,7 +7822,8 @@ function defineTableInOutFunction(config) {
         settings,
         secrets,
         pushdownFilters,
-        storage: boundStorage
+        storage: boundStorage,
+        substreamId: request.substream_id ?? null
       };
       const phase = request.phase;
       if (phase === "FINALIZE" /* FINALIZE */) {
@@ -7805,7 +7866,14 @@ function defineTableInOutFunction(config) {
         inputSchema: request.bind_call.input_schema ?? undefined,
         exchangeInit: () => ({ state, processParams }),
         exchangeFn: async (eState, input, out) => {
-          const wrappedOut = config.autoApplyFilters && pushdownFilters ? new FilteringOutputCollector(out, pushdownFilters) : out;
+          applyRevalidationValidators(eState.processParams, input);
+          let wrappedOut = makeMetadataBakingCollector(out);
+          if (projIds) {
+            wrappedOut = makeSchemaReconcilingCollector(wrappedOut, outputSchema);
+          }
+          if (config.autoApplyFilters && pushdownFilters) {
+            wrappedOut = new FilteringOutputCollector(wrappedOut, pushdownFilters);
+          }
           await processFn(eState.processParams, eState.state, input, wrappedOut);
           if (eState.state != null) {
             const serialized = serializeUserState(eState.state);
@@ -7818,6 +7886,62 @@ function defineTableInOutFunction(config) {
     }
   };
 }
+function applyRevalidationValidators(params, input) {
+  const md = input?.metadata;
+  params.ifNoneMatch = md?.get(CACHE_IF_NONE_MATCH_KEY);
+  params.ifModifiedSince = md?.get(CACHE_IF_MODIFIED_SINCE_KEY);
+}
+function makeMetadataBakingCollector(inner) {
+  return new Proxy(inner, {
+    get(target, prop, receiver) {
+      if (prop === "emit") {
+        return function(batchOrColumns, metadata) {
+          if (batchOrColumns && typeof batchOrColumns.getChild === "function" && metadata && metadata.size > 0) {
+            const batch = batchOrColumns;
+            const merged = new Map(batch.metadata ?? []);
+            for (const [k, v2] of metadata)
+              merged.set(k, v2);
+            return target.emit(withBatchMetadata(batch, merged), merged);
+          }
+          return target.emit(batchOrColumns, metadata);
+        };
+      }
+      const v = Reflect.get(target, prop, receiver);
+      return typeof v === "function" ? v.bind(target) : v;
+    }
+  });
+}
+function makeSchemaReconcilingCollector(inner, targetSchema) {
+  function reconcile(batch) {
+    const batchFields = batch.schema.fields;
+    if (batchFields.length === targetSchema.fields.length && targetSchema.fields.every((f, i) => batchFields[i].name === f.name)) {
+      return batch;
+    }
+    const ids = [];
+    for (const f of targetSchema.fields) {
+      const idx = batchFields.findIndex((bf) => bf.name === f.name);
+      if (idx < 0) {
+        throw new Error(`emitted batch is missing projected output column '${f.name}' ` + `(emitted: [${batchFields.map((bf) => bf.name).join(", ")}])`);
+      }
+      ids.push(idx);
+    }
+    return projectBatch(ids, batch);
+  }
+  return new Proxy(inner, {
+    get(target, prop, receiver) {
+      if (prop === "emit") {
+        return function(batchOrColumns, metadata) {
+          if (batchOrColumns && typeof batchOrColumns.getChild === "function") {
+            return target.emit(reconcile(batchOrColumns), metadata);
+          }
+          return target.emit(batchOrColumns, metadata);
+        };
+      }
+      const v = Reflect.get(target, prop, receiver);
+      return typeof v === "function" ? v.bind(target) : v;
+    }
+  });
+}
 // src/catalog/read-only.ts
 init_arrow();
 
@@ -8185,6 +8309,7 @@ class ReadOnlyCatalogInterface extends CatalogInterface {
         source_order_dependent: f.kind === "table_buffering" && f.meta.sourceOrderDependent === true,
         sink_order_dependent: f.kind === "table_buffering" && f.meta.sinkOrderDependent === true,
         requires_input_batch_index: f.kind === "table_buffering" && f.meta.requiresInputBatchIndex === true,
+        input_from_args: f.kind === "table_in_out" && f.meta.inputFromArgs === true,
         required_settings: meta.requiredSettings,
         required_secrets: requiredSecrets.map((s) => ({
           secret_type: s.secret_type,
@@ -8729,10 +8854,27 @@ function filterByArgumentTypes(candidates, args, inputSchema, isScalar) {
     } else {
       const posSpecs = specs.filter((s) => typeof s.position === "number" && !s.isTableInput).sort((a, b) => a.position - b.position);
       const posTypes = [];
-      const argsSchema = args.argumentsSchema;
-      for (let i = 0;i < args.positional.length; i++) {
-        const field3 = argsSchema?.fields.find((f) => f.name === `positional_${i}`);
-        posTypes.push(field3 ? field3.type : null);
+      if (func.meta.inputFromArgs) {
+        if (inputSchema) {
+          let varArgsSpec = null;
+          for (const spec of posSpecs) {
+            if (spec.isVarargs)
+              varArgsSpec = spec;
+            const pos = spec.position;
+            posTypes.push(pos < inputSchema.fields.length ? inputSchema.fields[pos].type : null);
+          }
+          if (varArgsSpec) {
+            for (let i = varArgsSpec.position + 1;i < inputSchema.fields.length; i++) {
+              posTypes.push(inputSchema.fields[i].type);
+            }
+          }
+        }
+      } else {
+        const argsSchema = args.argumentsSchema;
+        for (let i = 0;i < args.positional.length; i++) {
+          const field3 = argsSchema?.fields.find((f) => f.name === `positional_${i}`);
+          posTypes.push(field3 ? field3.type : null);
+        }
       }
       const r = scoreTypes(posSpecs, posTypes);
       score += r.score;
@@ -8791,6 +8933,21 @@ class FunctionRegistry {
         const posSpecs = func.argumentSpecs.filter((s) => typeof s.position === "number" && !s.isTableInput);
         const hasVarargs = posSpecs.some((s) => s.isVarargs);
         const nonVarargs = posSpecs.filter((s) => !s.isVarargs).length;
+        if (func.meta.inputFromArgs) {
+          const numInputCols = context.inputSchema?.fields.length ?? 0;
+          if (hasVarargs) {
+            if (numInputCols < nonVarargs)
+              return false;
+          } else if (numInputCols !== posSpecs.length) {
+            return false;
+          }
+          const validNamed = new Set(func.argumentSpecs.filter((s) => typeof s.position === "string").map((s) => s.position));
+          for (const key of args.named.keys()) {
+            if (!validNamed.has(key))
+              return false;
+          }
+          return true;
+        }
         if (hasVarargs) {
           return numArgs >= nonVarargs;
         }
@@ -9374,4 +9531,4 @@ export {
   Arguments
 };
 
-//# debugId=520242AE9AD7B67464756E2164756E21
+//# debugId=9CBEC1BA4EDD0C8364756E2164756E21