@query-farm/vgi 0.25.1 → 0.25.2

package/dist/worker-cf-entry.d.ts

@@ -1,7 +1,7 @@
 export { createVgiFetch } from "./http/fetch.js";
 export type { VgiFetchOptions } from "./http/fetch.js";
-export { defineScalarFunction, defineTableFunction, defineAggregate, defineTableInOutFunction, FunctionRegistry, ReadOnlyCatalogInterface, CompositeCatalogInterface, CatalogInterface, Arguments, FunctionType, serializeAttachOptionSpec, serializeAttachOptionSpecs, serializeColumnStatistics, BindResultSchema, ScanFunctionResultSchema, } from "./index.core.js";
-export type { AttachOptionSpec, ColumnStatistics, CatalogAttachResult, CatalogDescriptor, CatalogInfo, TableInfo, AttachOpaqueData, TransactionOpaqueData, SchemaInfo, VgiFunction, TableProcessParams, } from "./index.core.js";
+export { defineScalarFunction, defineTableFunction, defineAggregate, defineTableInOutFunction, defineRowTransformFunction, parentRowsMetadata, PARENT_ROW_METADATA_KEY, constraintSpecFields, FunctionRegistry, ReadOnlyCatalogInterface, CompositeCatalogInterface, CatalogInterface, Arguments, FunctionType, serializeAttachOptionSpec, serializeAttachOptionSpecs, serializeColumnStatistics, BindResultSchema, ScanFunctionResultSchema, } from "./index.core.js";
+export type { AttachOptionSpec, ColumnStatistics, CatalogAttachResult, CatalogDescriptor, CatalogInfo, TableInfo, AttachOpaqueData, TransactionOpaqueData, SchemaInfo, VgiFunction, TableProcessParams, ArgumentSpec, ArgumentConstraints, RowTransformConfig, RowTransformProcessParams, TableInOutBindParams, } from "./index.core.js";
 export type { ProtocolConfig } from "./protocol/dispatch.js";
 export type { LandingInfo } from "@query-farm/vgi-rpc";
 export { TypeId, isNull, isInt, isFloat, isBinary, isUtf8, isBool, isDecimal, isDate, isTime, isTimestamp, isInterval, isList, isStruct, isUnion, isFixedSizeBinary, isFixedSizeList, isMap, isDuration, isDictionary, isBatch, backend, schema, field, nullType, bool, int, int8, int16, int32, int64, uint8, uint16, uint32, uint64, float16, float32, float64, utf8, binary, fixedSizeBinary, decimal, decimal128, decimal256, date, dateDay, dateMillisecond, time, timeSecond, timeMillisecond, timeMicrosecond, timeNanosecond, timestamp, duration, interval, list, fixedSizeList, struct, map, dictionary, union, sparseUnion, denseUnion, TimeUnit, DateUnit, IntervalUnit, UnionMode, emptyBatch, batchFromRows, batchFromColumns, columnFromArray, iterRows, batchToScalarDict, batchToSecretDict, safeNumber, decodeDictValue, filterBatch, projectSchema, projectBatch, serializeSchema, deserializeSchema, serializeBatch, deserializeBatch, buildStatisticsBatch, } from "./arrow/index.js";

package/dist/worker-cf-entry.js

@@ -8026,6 +8026,174 @@ function makeSchemaReconcilingCollector(inner, targetSchema) {
     }
   });
 }
+var PARENT_ROW_METADATA_KEY = "vgi_rpc.parent_row#b64";
+function parentRowsMetadata(parentRows, outputRows, extra) {
+  if (parentRows.length !== outputRows) {
+    throw new Error(`parentRowsMetadata: length ${parentRows.length} != output rows ${outputRows}; ` + `parentRows must carry exactly one input-row index per emitted output row`);
+  }
+  const md = new Map(extra ?? []);
+  if (outputRows === 0) {
+    return md;
+  }
+  const raw = new Uint8Array(parentRows.length * 4);
+  const dv = new DataView(raw.buffer);
+  for (let i = 0;i < parentRows.length; i++) {
+    dv.setInt32(i * 4, parentRows[i], true);
+  }
+  md.set(PARENT_ROW_METADATA_KEY, base64Encode(raw));
+  return md;
+}
+function base64Encode(bytes) {
+  if (typeof globalThis.Buffer !== "undefined") {
+    return globalThis.Buffer.from(bytes).toString("base64");
+  }
+  let bin = "";
+  for (let i = 0;i < bytes.length; i++)
+    bin += String.fromCharCode(bytes[i]);
+  return globalThis.btoa(bin);
+}
+function defineRowTransformFunction(config) {
+  const numPositional = Object.keys(config.args ?? {}).length + (config.varargs ? 1 : 0);
+  if (numPositional === 0) {
+    throw new Error(`${config.name}: a blended row-transform function needs at least one ` + `positional arg (its per-row input column); found none.`);
+  }
+  const specs = [];
+  let posIdx = 0;
+  if (config.args) {
+    for (const [name, type] of Object.entries(config.args)) {
+      specs.push({
+        name,
+        position: posIdx++,
+        arrowType: type,
+        doc: config.argDocs?.[name]
+      });
+    }
+  }
+  if (config.varargs) {
+    specs.push({
+      name: config.varargs.name,
+      position: posIdx++,
+      arrowType: config.varargs.type,
+      isVarargs: true,
+      doc: config.varargs.doc ?? config.argDocs?.[config.varargs.name]
+    });
+  }
+  if (config.namedArgs) {
+    for (const [name, type] of Object.entries(config.namedArgs)) {
+      specs.push({
+        name,
+        position: name,
+        arrowType: type,
+        doc: config.argDocs?.[name]
+      });
+    }
+  }
+  const meta = {
+    name: config.name,
+    description: config.description,
+    stability: config.stability,
+    projectionPushdown: config.projectionPushdown,
+    filterPushdown: config.filterPushdown,
+    autoApplyFilters: config.autoApplyFilters,
+    examples: config.examples,
+    categories: config.categories,
+    tags: config.tags,
+    maxWorkers: config.maxWorkers,
+    requiredSettings: config.requiredSettings,
+    requiredSecrets: config.requiredSecrets,
+    hasFinalize: false,
+    inputFromArgs: true
+  };
+  function extractArgs(request) {
+    const args = {};
+    for (const spec of specs) {
+      if (typeof spec.position !== "string")
+        continue;
+      const defaultVal = config.argDefaults?.[spec.name] !== undefined ? config.argDefaults[spec.name] : undefined;
+      let val = request.arguments.get(spec.position, defaultVal ?? null);
+      if (typeof val === "bigint")
+        val = Number(val);
+      args[spec.name] = val;
+    }
+    return args;
+  }
+  return {
+    kind: "table_in_out",
+    meta,
+    argumentSpecs: specs,
+    async bind(request) {
+      const args = extractArgs(request);
+      const settings = batchToScalarDict(request.settings);
+      const secrets = batchToSecretDict(request.secrets);
+      const result = await config.onBind({
+        args,
+        bindCall: request,
+        settings,
+        secrets
+      });
+      return {
+        output_schema: result.outputSchema,
+        opaque_data: result.opaqueData ?? null
+      };
+    },
+    async globalInit(request) {
+      if (request.execution_id) {
+        return {
+          max_workers: DEFAULT_MAX_WORKERS,
+          execution_id: request.execution_id,
+          opaque_data: null
+        };
+      }
+      const executionId = new Uint8Array(16);
+      crypto.getRandomValues(executionId);
+      return {
+        max_workers: config.maxWorkers ?? 1,
+        execution_id: executionId,
+        opaque_data: null
+      };
+    },
+    createStreamHandlers(request, response, _accumulatedState) {
+      const args = extractArgs(request.bind_call);
+      const settings = batchToScalarDict(request.bind_call.settings);
+      const secrets = batchToSecretDict(request.bind_call.secrets);
+      const projIds = request.projection_ids && meta.projectionPushdown ? request.projection_ids : null;
+      const outputSchema = projIds ? projectSchema(projIds, request.output_schema) : request.output_schema;
+      const joinKeysLookup = buildJoinKeysLookup(request.join_keys);
+      const pushdownFilters = request.pushdown_filters ? deserializeFilters(request.pushdown_filters, joinKeysLookup) : undefined;
+      const boundStorage = new BoundStorage(storage, response.execution_id ?? new Uint8Array(16));
+      const processParams = {
+        args,
+        initCall: request,
+        initResponse: response,
+        outputSchema,
+        settings,
+        secrets,
+        pushdownFilters,
+        storage: boundStorage,
+        substreamId: request.substream_id ?? null
+      };
+      if (request.phase === "FINALIZE" /* FINALIZE */) {
+        throw new Error(`${config.name}: a blended row-transform function has no FINALIZE ` + `phase (it is a per-row map; has_finalize is advertised false).`);
+      }
+      return {
+        outputSchema,
+        inputSchema: request.bind_call.input_schema ?? undefined,
+        exchangeInit: () => ({ state: null, processParams }),
+        exchangeFn: async (eState, input, out) => {
+          applyRevalidationValidators(eState.processParams, input);
+          let wrappedOut = makeMetadataBakingCollector(out);
+          if (projIds) {
+            wrappedOut = makeSchemaReconcilingCollector(wrappedOut, outputSchema);
+          }
+          if (config.autoApplyFilters && pushdownFilters) {
+            wrappedOut = new FilteringOutputCollector(wrappedOut, pushdownFilters);
+          }
+          await config.process(eState.processParams, input, wrappedOut);
+        }
+      };
+    }
+  };
+}
 // src/catalog/read-only.ts
 init_arrow();
 
@@ -9244,6 +9412,7 @@ export {
   safeNumber,
   projectSchema,
   projectBatch,
+  parentRowsMetadata,
   nullType2 as nullType,
   map2 as map,
   list2 as list,
@@ -9290,6 +9459,7 @@ export {
   defineTableInOutFunction,
   defineTableFunction,
   defineScalarFunction,
+  defineRowTransformFunction,
   defineAggregate,
   decodeDictValue,
   decimal2562 as decimal256,
@@ -9299,6 +9469,7 @@ export {
   dateDay2 as dateDay,
   date2 as date,
   createVgiFetch,
+  constraintSpecFields,
   columnFromArray,
   buildStatisticsBatch,
   bool2 as bool,
@@ -9314,6 +9485,7 @@ export {
   TimeUnit2 as TimeUnit,
   ScanFunctionResultSchema,
   ReadOnlyCatalogInterface,
+  PARENT_ROW_METADATA_KEY,
   OutputCollector2 as OutputCollector,
   IntervalUnit2 as IntervalUnit,
   FunctionType,
@@ -9327,4 +9499,4 @@ export {
   Arguments
 };
 
-//# debugId=3225E125CCD893FA64756E2164756E21
+//# debugId=372B5FB94FE209FC64756E2164756E21