@query-farm/vgi 0.17.0 → 0.19.0

package/dist/index.js

@@ -1586,6 +1586,14 @@ function projectBatch(projectionIds, batch) {
 }
 var init_project = () => {};
 
+// src/arrow/impl-arrowjs/metadata.ts
+import { RecordBatch as RecordBatch4 } from "@query-farm/apache-arrow";
+function withBatchMetadata(batch, metadata) {
+  const a = batch;
+  return new RecordBatch4(a.schema, a.data, metadata);
+}
+var init_metadata = () => {};
+
 // src/arrow/impl-arrowjs/ipc.ts
 import {
   Schema as Schema4,
@@ -1634,7 +1642,7 @@ import {
   Field as Field3,
   Null,
   makeData as makeData5,
-  RecordBatch as RecordBatch5,
+  RecordBatch as RecordBatch6,
   Struct as Struct5,
   SparseUnion,
   Schema as Schema5,
@@ -1746,7 +1754,7 @@ function assemble(columnNames, minData, maxData, hasNull, hasNotNull, distinctCo
   ];
   const structType = new Struct5(schema2.fields);
   const data = makeData5({ type: structType, length: n, children, nullCount: 0 });
-  return new RecordBatch5(schema2, data);
+  return new RecordBatch6(schema2, data);
 }
 var init_statistics = __esm(() => {
   init_registry();
@@ -1756,6 +1764,7 @@ var init_statistics = __esm(() => {
 // src/arrow/impl-arrowjs/index.ts
 var exports_impl_arrowjs = {};
 __export(exports_impl_arrowjs, {
+  withBatchMetadata: () => withBatchMetadata,
   utf8: () => utf8,
   union: () => union,
   uint8: () => uint8,
@@ -1830,6 +1839,7 @@ var init_impl_arrowjs = __esm(() => {
   init_filter();
   init_project();
   init_canonical();
+  init_metadata();
   init_ipc();
   init_statistics();
   backend = { name: "arrow-js" };
@@ -1928,6 +1938,7 @@ var init_branded = __esm(() => {
 // src/arrow/index.ts
 var exports_arrow = {};
 __export(exports_arrow, {
+  withBatchMetadata: () => withBatchMetadata,
   utf8: () => utf82,
   union: () => union2,
   uint8: () => uint82,
@@ -3085,7 +3096,7 @@ function defineScalarFunction(config) {
           if (outputBatch.numRows !== input.numRows) {
             throw new RowCountMismatchError(input.numRows, outputBatch.numRows);
           }
-          out.emit(outputBatch);
+          out.emit(outputBatch, config.cacheControl ? cacheControlMetadata(config.cacheControl) : undefined);
         }
       };
     }
@@ -4544,7 +4555,8 @@ function defineTableInOutFunction(config) {
         settings,
         secrets,
         pushdownFilters,
-        storage: boundStorage
+        storage: boundStorage,
+        substreamId: request.substream_id ?? null
       };
       const phase = request.phase;
       if (phase === "FINALIZE" /* FINALIZE */) {
@@ -4587,7 +4599,14 @@ function defineTableInOutFunction(config) {
         inputSchema: request.bind_call.input_schema ?? undefined,
         exchangeInit: () => ({ state, processParams }),
         exchangeFn: async (eState, input, out) => {
-          const wrappedOut = config.autoApplyFilters && pushdownFilters ? new FilteringOutputCollector(out, pushdownFilters) : out;
+          applyRevalidationValidators(eState.processParams, input);
+          let wrappedOut = makeMetadataBakingCollector(out);
+          if (projIds) {
+            wrappedOut = makeSchemaReconcilingCollector(wrappedOut, outputSchema);
+          }
+          if (config.autoApplyFilters && pushdownFilters) {
+            wrappedOut = new FilteringOutputCollector(wrappedOut, pushdownFilters);
+          }
           await processFn(eState.processParams, eState.state, input, wrappedOut);
           if (eState.state != null) {
             const serialized = serializeUserState(eState.state);
@@ -4600,6 +4619,230 @@ function defineTableInOutFunction(config) {
     }
   };
 }
+function applyRevalidationValidators(params, input) {
+  const md = input?.metadata;
+  params.ifNoneMatch = md?.get(CACHE_IF_NONE_MATCH_KEY);
+  params.ifModifiedSince = md?.get(CACHE_IF_MODIFIED_SINCE_KEY);
+}
+function makeMetadataBakingCollector(inner) {
+  return new Proxy(inner, {
+    get(target, prop, receiver) {
+      if (prop === "emit") {
+        return function(batchOrColumns, metadata) {
+          if (batchOrColumns && typeof batchOrColumns.getChild === "function" && metadata && metadata.size > 0) {
+            const batch = batchOrColumns;
+            const merged = new Map(batch.metadata ?? []);
+            for (const [k, v2] of metadata)
+              merged.set(k, v2);
+            return target.emit(withBatchMetadata(batch, merged), merged);
+          }
+          return target.emit(batchOrColumns, metadata);
+        };
+      }
+      const v = Reflect.get(target, prop, receiver);
+      return typeof v === "function" ? v.bind(target) : v;
+    }
+  });
+}
+function makeSchemaReconcilingCollector(inner, targetSchema) {
+  function reconcile(batch) {
+    const batchFields = batch.schema.fields;
+    if (batchFields.length === targetSchema.fields.length && targetSchema.fields.every((f, i) => batchFields[i].name === f.name)) {
+      return batch;
+    }
+    const ids = [];
+    for (const f of targetSchema.fields) {
+      const idx = batchFields.findIndex((bf) => bf.name === f.name);
+      if (idx < 0) {
+        throw new Error(`emitted batch is missing projected output column '${f.name}' ` + `(emitted: [${batchFields.map((bf) => bf.name).join(", ")}])`);
+      }
+      ids.push(idx);
+    }
+    return projectBatch(ids, batch);
+  }
+  return new Proxy(inner, {
+    get(target, prop, receiver) {
+      if (prop === "emit") {
+        return function(batchOrColumns, metadata) {
+          if (batchOrColumns && typeof batchOrColumns.getChild === "function") {
+            return target.emit(reconcile(batchOrColumns), metadata);
+          }
+          return target.emit(batchOrColumns, metadata);
+        };
+      }
+      const v = Reflect.get(target, prop, receiver);
+      return typeof v === "function" ? v.bind(target) : v;
+    }
+  });
+}
+var PARENT_ROW_METADATA_KEY = "vgi_rpc.parent_row#b64";
+function parentRowsMetadata(parentRows, outputRows, extra) {
+  if (parentRows.length !== outputRows) {
+    throw new Error(`parentRowsMetadata: length ${parentRows.length} != output rows ${outputRows}; ` + `parentRows must carry exactly one input-row index per emitted output row`);
+  }
+  const md = new Map(extra ?? []);
+  if (outputRows === 0) {
+    return md;
+  }
+  const raw = new Uint8Array(parentRows.length * 4);
+  const dv = new DataView(raw.buffer);
+  for (let i = 0;i < parentRows.length; i++) {
+    dv.setInt32(i * 4, parentRows[i], true);
+  }
+  md.set(PARENT_ROW_METADATA_KEY, base64Encode(raw));
+  return md;
+}
+function base64Encode(bytes) {
+  if (typeof globalThis.Buffer !== "undefined") {
+    return globalThis.Buffer.from(bytes).toString("base64");
+  }
+  let bin = "";
+  for (let i = 0;i < bytes.length; i++)
+    bin += String.fromCharCode(bytes[i]);
+  return globalThis.btoa(bin);
+}
+function defineRowTransformFunction(config) {
+  const numPositional = Object.keys(config.args ?? {}).length + (config.varargs ? 1 : 0);
+  if (numPositional === 0) {
+    throw new Error(`${config.name}: a blended row-transform function needs at least one ` + `positional arg (its per-row input column); found none.`);
+  }
+  const specs = [];
+  let posIdx = 0;
+  if (config.args) {
+    for (const [name, type] of Object.entries(config.args)) {
+      specs.push({
+        name,
+        position: posIdx++,
+        arrowType: type,
+        doc: config.argDocs?.[name]
+      });
+    }
+  }
+  if (config.varargs) {
+    specs.push({
+      name: config.varargs.name,
+      position: posIdx++,
+      arrowType: config.varargs.type,
+      isVarargs: true,
+      doc: config.varargs.doc ?? config.argDocs?.[config.varargs.name]
+    });
+  }
+  if (config.namedArgs) {
+    for (const [name, type] of Object.entries(config.namedArgs)) {
+      specs.push({
+        name,
+        position: name,
+        arrowType: type,
+        doc: config.argDocs?.[name]
+      });
+    }
+  }
+  const meta = {
+    name: config.name,
+    description: config.description,
+    stability: config.stability,
+    projectionPushdown: config.projectionPushdown,
+    filterPushdown: config.filterPushdown,
+    autoApplyFilters: config.autoApplyFilters,
+    examples: config.examples,
+    categories: config.categories,
+    tags: config.tags,
+    maxWorkers: config.maxWorkers,
+    requiredSettings: config.requiredSettings,
+    requiredSecrets: config.requiredSecrets,
+    hasFinalize: false,
+    inputFromArgs: true
+  };
+  function extractArgs(request) {
+    const args = {};
+    for (const spec of specs) {
+      if (typeof spec.position !== "string")
+        continue;
+      const defaultVal = config.argDefaults?.[spec.name] !== undefined ? config.argDefaults[spec.name] : undefined;
+      let val = request.arguments.get(spec.position, defaultVal ?? null);
+      if (typeof val === "bigint")
+        val = Number(val);
+      args[spec.name] = val;
+    }
+    return args;
+  }
+  return {
+    kind: "table_in_out",
+    meta,
+    argumentSpecs: specs,
+    async bind(request) {
+      const args = extractArgs(request);
+      const settings = batchToScalarDict(request.settings);
+      const secrets = batchToSecretDict(request.secrets);
+      const result = await config.onBind({
+        args,
+        bindCall: request,
+        settings,
+        secrets
+      });
+      return {
+        output_schema: result.outputSchema,
+        opaque_data: result.opaqueData ?? null
+      };
+    },
+    async globalInit(request) {
+      if (request.execution_id) {
+        return {
+          max_workers: DEFAULT_MAX_WORKERS,
+          execution_id: request.execution_id,
+          opaque_data: null
+        };
+      }
+      const executionId = new Uint8Array(16);
+      crypto.getRandomValues(executionId);
+      return {
+        max_workers: config.maxWorkers ?? 1,
+        execution_id: executionId,
+        opaque_data: null
+      };
+    },
+    createStreamHandlers(request, response, _accumulatedState) {
+      const args = extractArgs(request.bind_call);
+      const settings = batchToScalarDict(request.bind_call.settings);
+      const secrets = batchToSecretDict(request.bind_call.secrets);
+      const projIds = request.projection_ids && meta.projectionPushdown ? request.projection_ids : null;
+      const outputSchema = projIds ? projectSchema(projIds, request.output_schema) : request.output_schema;
+      const joinKeysLookup = buildJoinKeysLookup(request.join_keys);
+      const pushdownFilters = request.pushdown_filters ? deserializeFilters(request.pushdown_filters, joinKeysLookup) : undefined;
+      const boundStorage = new BoundStorage(storage, response.execution_id ?? new Uint8Array(16));
+      const processParams = {
+        args,
+        initCall: request,
+        initResponse: response,
+        outputSchema,
+        settings,
+        secrets,
+        pushdownFilters,
+        storage: boundStorage,
+        substreamId: request.substream_id ?? null
+      };
+      if (request.phase === "FINALIZE" /* FINALIZE */) {
+        throw new Error(`${config.name}: a blended row-transform function has no FINALIZE ` + `phase (it is a per-row map; has_finalize is advertised false).`);
+      }
+      return {
+        outputSchema,
+        inputSchema: request.bind_call.input_schema ?? undefined,
+        exchangeInit: () => ({ state: null, processParams }),
+        exchangeFn: async (eState, input, out) => {
+          applyRevalidationValidators(eState.processParams, input);
+          let wrappedOut = makeMetadataBakingCollector(out);
+          if (projIds) {
+            wrappedOut = makeSchemaReconcilingCollector(wrappedOut, outputSchema);
+          }
+          if (config.autoApplyFilters && pushdownFilters) {
+            wrappedOut = new FilteringOutputCollector(wrappedOut, pushdownFilters);
+          }
+          await config.process(eState.processParams, input, wrappedOut);
+        }
+      };
+    }
+  };
+}
 // src/functions/table-buffering.ts
 init_arrow();
 init_arrow();
@@ -4961,6 +5204,7 @@ var INIT_REQUEST_SCHEMA = schema2([
   field2("phase", utf82(), true),
   field2("execution_id", binary2(), true),
   field2("init_opaque_data", binary2(), true),
+  field2("substream_id", binary2(), true),
   field2("order_by_column_name", utf82(), true),
   field2("order_by_direction", utf82(), true),
   field2("order_by_null_order", utf82(), true),
@@ -4987,6 +5231,7 @@ function serializeInitRequest(req) {
     phase: req.phase ?? null,
     execution_id: req.execution_id ?? null,
     init_opaque_data: req.init_opaque_data ?? null,
+    substream_id: req.substream_id ?? null,
     order_by_column_name: req.order_by_column_name ?? null,
     order_by_direction: req.order_by_direction ?? null,
     order_by_null_order: req.order_by_null_order ?? null,
@@ -5048,6 +5293,7 @@ function deserializeInitRequest(params) {
     finalize_state_id: params.finalize_state_id ? toUint8Array2(params.finalize_state_id) : null,
     execution_id: params.execution_id ? toUint8Array2(params.execution_id) : null,
     init_opaque_data: params.init_opaque_data ? toUint8Array2(params.init_opaque_data) : null,
+    substream_id: params.substream_id ? toUint8Array2(params.substream_id) : null,
     order_by_column_name: parseEnum(decodeDictValue(params.order_by_column_name)) ?? null,
     order_by_direction: parseDirection(decodeDictValue(params.order_by_direction)),
     order_by_null_order: parseNullOrder(decodeDictValue(params.order_by_null_order)),
@@ -5796,7 +6042,7 @@ var TableInfoSchema = schema2([
   field2("cardinality_max", int642(), false),
   field2("column_statistics", binary2(), false),
   field2("bind_result", binary2(), false),
-  field2("required_field_filter_paths", list2(field2("item", utf82(), true)), false)
+  field2("required_filters", list2(field2("item", list2(field2("item", utf82(), true)), true)), false)
 ]);
 var ViewInfoSchema = schema2([
   field2("comment", utf82(), true),
@@ -5836,6 +6082,7 @@ var FunctionInfoSchema = schema2([
   field2("source_order_dependent", bool2(), false),
   field2("sink_order_dependent", bool2(), false),
   field2("requires_input_batch_index", bool2(), false),
+  field2("input_from_args", bool2(), false),
   field2("required_settings", list2(field2("item", utf82(), true)), false),
   field2("required_secrets", list2(field2("item", struct2([field2("secret_type", utf82(), false), field2("scope", utf82(), true), field2("secret_name", utf82(), true)]), true)), false)
 ]);
@@ -6868,16 +7115,21 @@ class ReadOnlyCatalogInterface extends CatalogInterface {
       const uniqueIndices = resolveIndexGroups(colSchema, t.unique);
       const pkIndices = resolveIndexGroups(colSchema, t.primaryKey);
       const fkBytes = serializeForeignKeys(t.foreignKey, name);
-      const requiredFieldFilterPaths = t.requiredFieldFilterPaths ?? [];
-      if (requiredFieldFilterPaths.length > 0) {
+      const requiredFilters = t.requiredFilters ?? [];
+      if (requiredFilters.length > 0) {
         const columnNames = new Set(colSchema.fields.map((f) => f.name));
-        for (const path2 of requiredFieldFilterPaths) {
-          if (!path2) {
-            throw new Error(`Table '${t.name}': requiredFieldFilterPaths must not contain empty strings`);
+        for (const group of requiredFilters) {
+          if (!group || group.length === 0) {
+            throw new Error(`Table '${t.name}': requiredFilters must not contain empty groups`);
           }
-          const head = path2.split(".", 1)[0];
-          if (!columnNames.has(head)) {
-            throw new Error(`Table '${t.name}': requiredFieldFilterPaths path '${path2}' references ` + `unknown column '${head}'. Available columns: ${[...columnNames].sort().join(", ")}`);
+          for (const path2 of group) {
+            if (!path2) {
+              throw new Error(`Table '${t.name}': requiredFilters must not contain empty strings`);
+            }
+            const head = path2.split(".", 1)[0];
+            if (!columnNames.has(head)) {
+              throw new Error(`Table '${t.name}': requiredFilters path '${path2}' references ` + `unknown column '${head}'. Available columns: ${[...columnNames].sort().join(", ")}`);
+            }
           }
         }
       }
@@ -6903,7 +7155,7 @@ class ReadOnlyCatalogInterface extends CatalogInterface {
         delete_function: null,
         cardinality_estimate: t.inlinedCardinality ? Number(t.inlinedCardinality.estimate) : null,
         cardinality_max: t.inlinedCardinality ? Number(t.inlinedCardinality.max) : null,
-        required_field_filter_paths: requiredFieldFilterPaths
+        required_filters: requiredFilters
       };
     }));
   }
@@ -6990,6 +7242,7 @@ class ReadOnlyCatalogInterface extends CatalogInterface {
         source_order_dependent: f.kind === "table_buffering" && f.meta.sourceOrderDependent === true,
         sink_order_dependent: f.kind === "table_buffering" && f.meta.sinkOrderDependent === true,
         requires_input_batch_index: f.kind === "table_buffering" && f.meta.requiresInputBatchIndex === true,
+        input_from_args: f.kind === "table_in_out" && f.meta.inputFromArgs === true,
         required_settings: meta.requiredSettings,
         required_secrets: requiredSecrets.map((s) => ({
           secret_type: s.secret_type,
@@ -7534,10 +7787,27 @@ function filterByArgumentTypes(candidates, args, inputSchema, isScalar) {
     } else {
       const posSpecs = specs.filter((s) => typeof s.position === "number" && !s.isTableInput).sort((a, b) => a.position - b.position);
       const posTypes = [];
-      const argsSchema = args.argumentsSchema;
-      for (let i = 0;i < args.positional.length; i++) {
-        const field3 = argsSchema?.fields.find((f) => f.name === `positional_${i}`);
-        posTypes.push(field3 ? field3.type : null);
+      if (func.meta.inputFromArgs) {
+        if (inputSchema) {
+          let varArgsSpec = null;
+          for (const spec of posSpecs) {
+            if (spec.isVarargs)
+              varArgsSpec = spec;
+            const pos = spec.position;
+            posTypes.push(pos < inputSchema.fields.length ? inputSchema.fields[pos].type : null);
+          }
+          if (varArgsSpec) {
+            for (let i = varArgsSpec.position + 1;i < inputSchema.fields.length; i++) {
+              posTypes.push(inputSchema.fields[i].type);
+            }
+          }
+        }
+      } else {
+        const argsSchema = args.argumentsSchema;
+        for (let i = 0;i < args.positional.length; i++) {
+          const field3 = argsSchema?.fields.find((f) => f.name === `positional_${i}`);
+          posTypes.push(field3 ? field3.type : null);
+        }
       }
       const r = scoreTypes(posSpecs, posTypes);
       score += r.score;
@@ -7596,6 +7866,21 @@ class FunctionRegistry {
         const posSpecs = func.argumentSpecs.filter((s) => typeof s.position === "number" && !s.isTableInput);
         const hasVarargs = posSpecs.some((s) => s.isVarargs);
         const nonVarargs = posSpecs.filter((s) => !s.isVarargs).length;
+        if (func.meta.inputFromArgs) {
+          const numInputCols = context.inputSchema?.fields.length ?? 0;
+          if (hasVarargs) {
+            if (numInputCols < nonVarargs)
+              return false;
+          } else if (numInputCols !== posSpecs.length) {
+            return false;
+          }
+          const validNamed = new Set(func.argumentSpecs.filter((s) => typeof s.position === "string").map((s) => s.position));
+          for (const key of args.named.keys()) {
+            if (!validNamed.has(key))
+              return false;
+          }
+          return true;
+        }
         if (hasVarargs) {
           return numArgs >= nonVarargs;
         }
@@ -10657,7 +10942,8 @@ class VgiClient {
       tablesample_percentage: ts?.percentage ?? null,
       tablesample_seed: ts?.seed == null ? null : BigInt(ts.seed),
       execution_id: opts?.executionId ?? null,
-      init_opaque_data: null
+      init_opaque_data: null,
+      substream_id: null
     };
     const initBatch = serializeInitRequest(initRequest);
     const session = await this.rpc.stream("init", wrapRequest(initBatch));
@@ -11505,6 +11791,7 @@ export {
   readCanonicalValue,
   projectSchema,
   projectBatch,
+  parentRowsMetadata,
   nullType2 as nullType,
   metadatasToArrow,
   map2 as map,
@@ -11576,6 +11863,7 @@ export {
   defineTableFunction,
   defineTableBufferingFunction,
   defineScalarFunction,
+  defineRowTransformFunction,
   defineCopyToFunction,
   defineCopyFromFunction,
   defineAggregate,
@@ -11643,6 +11931,7 @@ export {
   RowCountMismatchError,
   ReadOnlyCatalogInterface,
   PushdownFilters,
+  PARENT_ROW_METADATA_KEY,
   OutputCollector,
   OrderPreservation,
   OrderDependence,
@@ -11691,4 +11980,4 @@ export {
   ArgumentValidationError
 };
 
-//# debugId=0510FF19793B06C764756E2164756E21
+//# debugId=8787F3CC30C0C60C64756E2164756E21