@query-farm/vgi 0.28.0 → 0.29.1

package/src/functions/table-in-out.ts

@@ -3,7 +3,7 @@
 // Two-phase: INPUT phase receives and transforms batches,
 // FINALIZE phase emits final results.
 
-import { type VgiSchema, schema, type VgiField, type VgiDataType, type VgiBatch, nullType, withBatchMetadata } from "../arrow/index.js";
+import { type VgiSchema, schema, type VgiField, type VgiDataType, type VgiBatch, nullType, withBatchMetadata, serializeBatch, deserializeBatch } from "../arrow/index.js";
 import type { OutputCollector } from "@query-farm/vgi-rpc";
 import { DEFAULT_MAX_WORKERS, TableInOutPhase } from "../types.js";
 import type {
@@ -18,19 +18,33 @@ import type {
   StreamHandlers,
   FunctionExample,
 } from "./types.js";
-import type { ArgumentSpec } from "../arguments/argument-spec.js";
-import { batchToScalarDict, batchToSecretDict, projectSchema, projectBatch, emptyBatch } from "../util/arrow/index.js";
+import type { ArgumentConstraints, ArgumentSpec } from "../arguments/argument-spec.js";
+import {
+  assertArrowType,
+  assertArrowTypes,
+  constraintSpecFields,
+  narrowArgValue,
+  validateConstConstraints,
+} from "../arguments/argument-spec.js";
+import { batchToScalarDict, batchToSecretDict, projectSchema, projectBatch, emptyBatch, safeNumber } from "../util/arrow/index.js";
 import { CACHE_IF_MODIFIED_SINCE_KEY, CACHE_IF_NONE_MATCH_KEY } from "../cache-control.js";
 import {
-  buildJoinKeysLookup,
   deserializeFilters,
   FilteringOutputCollector,
   type PushdownFilters,
 } from "../filter-pushdown/index.js";
 import { FunctionStability } from "../types.js";
-import { BoundStorage, storage as defaultStorage } from "./storage.js";
+import { BoundStorage, storage as defaultStorage, FrameworkNS } from "./storage.js";
 import { serializeUserState, deserializeUserState } from "../protocol/state-serializer.js";
 
+/**
+ * State-log key the streaming FINALIZE drain uses within
+ * {@link FrameworkNS.STREAMING_FINALIZE}. There is exactly one flush per
+ * execution, so a single empty key is enough — matches vgi-python's
+ * `_STREAMING_FINALIZE_KEY = b""`.
+ */
+const STREAMING_FINALIZE_KEY = new Uint8Array(0);
+
 // ============================================================================
 // Table In-Out parameter bundles (reuse table function's)
 // ============================================================================
@@ -40,6 +54,8 @@ export interface TableInOutBindParams<TArgs = Record<string, any>> {
   bindCall: BindRequest;
   settings: Record<string, any>;
   secrets: Record<string, Record<string, any>>;
+  /** Resolved names aligned with the complete logical argument order. */
+  argumentNames?: (string | null)[] | null;
 }
 
 export interface TableInOutProcessParams<TArgs = Record<string, any>> {
@@ -91,6 +107,12 @@ export interface TableInOutConfig<
   namedArgs?: Record<string, VgiDataType>;
   /** Argument defaults */
   argDefaults?: Record<string, any>;
+  /** Authoritative typed defaults: one row, defaulted parameters only. */
+  parameterDefaultValues?: VgiBatch | null;
+  /** Argument descriptions keyed by argument name. */
+  argDocs?: Record<string, string>;
+  /** Discovery constraints, also enforced for scalar arguments at bind. */
+  argConstraints?: Record<string, ArgumentConstraints>;
   /** Bind: default passes through input schema. May be async. */
   onBind?: (params: TableInOutBindParams<TArgs>) =>
     | { outputSchema: VgiSchema; opaqueData?: Uint8Array }
@@ -132,6 +154,8 @@ export function defineTableInOutFunction<
   TArgs = Record<string, any>,
   TState = null,
 >(config: TableInOutConfig<TArgs, TState>): VgiFunction {
+  assertArrowTypes((config as any).args, `defineTableInOutFunction("${config.name}"): args`);
+
 
   // Build argument specs
   const specs: ArgumentSpec[] = [];
@@ -144,6 +168,11 @@ export function defineTableInOutFunction<
         name,
         position: posIdx++,
         arrowType: type,
+        doc: config.argDocs?.[name],
+        ...constraintSpecFields(
+          config.argConstraints?.[name],
+          config.argDefaults?.[name],
+        ),
       });
     }
   }
@@ -162,6 +191,11 @@ export function defineTableInOutFunction<
         name,
         position: name, // string position = named arg
         arrowType: type,
+        doc: config.argDocs?.[name],
+        ...constraintSpecFields(
+          config.argConstraints?.[name],
+          config.argDefaults?.[name],
+        ),
       });
     }
   }
@@ -176,6 +210,7 @@ export function defineTableInOutFunction<
     examples: config.examples,
     categories: config.categories,
     tags: config.tags,
+    parameterDefaultValues: config.parameterDefaultValues,
     maxWorkers: config.maxWorkers,
     requiredSettings: config.requiredSettings,
     requiredSecrets: config.requiredSecrets,
@@ -192,8 +227,11 @@ export function defineTableInOutFunction<
           : undefined;
       // For named args, position is a string (the arg name)
       let val = request.arguments.get(spec.position, defaultVal);
-      // Arrow Int64 values come through as BigInt — coerce to number
-      if (typeof val === "bigint") val = Number(val);
+      // Arrow Int64 arrives as bigint. Narrow when lossless, keep the
+      // bigint when not — see narrowArgValue.
+      val = narrowArgValue(val);
+      const constraints = config.argConstraints?.[spec.name];
+      if (constraints) validateConstConstraints(spec.name, constraints, val);
       args[spec.name] = val;
     }
     return args as TArgs;
@@ -214,6 +252,7 @@ export function defineTableInOutFunction<
           bindCall: request,
           settings,
           secrets,
+          argumentNames: request.argument_names ?? null,
         });
         return {
           output_schema: result.outputSchema,
@@ -276,9 +315,14 @@ export function defineTableInOutFunction<
       // Deserialize pushdown filters. Pass a join-keys column lookup so that
       // filters DuckDB promoted to join_keys (IN/OR lists, etc.) are
       // materialized as InFilters rather than silently dropped.
-      const joinKeysLookup = buildJoinKeysLookup(request.join_keys);
       const pushdownFilters = request.pushdown_filters
-        ? deserializeFilters(request.pushdown_filters, joinKeysLookup)
+        ? deserializeFilters(request.pushdown_filters, {
+          outputSchema: request.output_schema,
+          joinKeyBatches: request.join_keys,
+          extensionFunctions: meta.additionalFilterFunctions,
+          runtimeAlgorithms: meta.runtimeFilterAlgorithms,
+          evaluationContexts: meta.filterEvaluationContexts,
+        })
         : undefined;
 
       // Create BoundStorage for cross-phase/cross-worker data sharing
@@ -302,18 +346,41 @@ export function defineTableInOutFunction<
       const phase = request.phase;
 
       if (phase === TableInOutPhase.FINALIZE) {
-        // FINALIZE phase: producer mode.
-        // The actual collect+finalize work is deferred into the first
-        // producerFn call so the (now-async) FunctionStorage can be awaited
-        // there. Storing the materialized batches on the per-call state.
+        // FINALIZE phase: producer mode, drained by cursor out of the
+        // execution-scoped state log.
+        //
+        // Over HTTP a producer stream is strictly LOCK-STEP: at most one data
+        // batch per response, after which the worker hands back a continuation
+        // token and the client comes back for the next batch. A finalize()
+        // returning N batches therefore has to survive N-1 continuations — and
+        // a continuation rebuilds these handlers from scratch, with nothing of
+        // the previous turn left in memory.
+        //
+        // The prior shape kept the materialized batches on the handler object
+        // (a `batches` field beside `state`), and only `state` rides the state
+        // token. So the continuation found `batches == null` and re-ran the
+        // user's finalize() — whose inputs `boundStorage.collect()` had already
+        // DESTRUCTIVELY drained (DELETE ... RETURNING). Second call: no states,
+        // zero batches, immediate finish. The stream truncated after its first
+        // batch, silently, and only over HTTP (a byte-stream transport runs
+        // every tick inside one process, so the field survived).
+        //
+        // Same fix as vgi-python's BufferedFinalizeState: call the user's
+        // finalize() exactly ONCE, append each returned batch's IPC bytes to
+        // the state log under FrameworkNS.STREAMING_FINALIZE, and have every
+        // tick drain exactly ONE row past a cursor. Only the POSITION rides the
+        // state token, never the rows. finalize() is never re-run: its contract
+        // is to DRAIN accumulated state, so a second call is not obliged to
+        // return the same rows.
+        const finalizeLogKey = STREAMING_FINALIZE_KEY;
         return {
           outputSchema,
-          producerInit: () => ({ state: { batchIdx: 0 }, batches: null as VgiBatch[] | null }),
+          producerInit: () => ({ state: { cursor: -1, materialized: false } }),
           producerFn: async (
-            pState: { state: { batchIdx: number }; batches: VgiBatch[] | null },
+            pState: { state: { cursor: number; materialized: boolean } },
             out: OutputCollector
           ) => {
-            if (pState.batches == null) {
+            if (!pState.state.materialized) {
               const finalizeStates: TState[] = [];
               if (accumulatedState != null) {
                 finalizeStates.push(accumulatedState as TState);
@@ -332,24 +399,55 @@ export function defineTableInOutFunction<
                     : (null as TState),
                 );
               }
-              pState.batches = config.finalize
+              const batches: VgiBatch[] = config.finalize
                 ? await config.finalize(processParams, finalizeStates)
                 : [];
+              for (const batch of batches) {
+                await boundStorage.stateAppend(
+                  FrameworkNS.STREAMING_FINALIZE,
+                  finalizeLogKey,
+                  serializeBatch(batch),
+                );
+              }
+              // Set before the first emit so the flag is part of the userState
+              // this turn serializes — a continuation must never re-run
+              // finalize(), even when the log came back empty.
+              pState.state.materialized = true;
             }
-            if (pState.state.batchIdx >= pState.batches.length) {
+            const rows = await boundStorage.stateLogScan(
+              FrameworkNS.STREAMING_FINALIZE,
+              finalizeLogKey,
+              pState.state.cursor,
+              1,
+            );
+            if (rows.length === 0) {
               out.finish();
               return;
             }
-            out.emit(pState.batches[pState.state.batchIdx]);
-            pState.state.batchIdx++;
-            if (pState.state.batchIdx >= pState.batches.length) {
-              out.finish();
-            }
+            const [logId, bytes] = rows[0];
+            out.emit(deserializeBatch(bytes) as unknown as VgiBatch);
+            pState.state.cursor = logId;
           },
         };
       }
 
-      // INPUT phase: exchange mode
+      // INPUT phase: exchange mode. Any phase other than FINALIZE falls
+      // through to here, including a missing/undefined phase -- i.e. a
+      // caller that drove this function via table_function() instead of
+      // table_in_out_function(input=...). Reject that immediately rather
+      // than silently degrading to an empty inputSchema below, which
+      // disables the transport layer's own schema-conformance check and
+      // leaves both sides waiting on each other forever (confirmed live: a
+      // non-terminating exchange/continuation loop against a real deployed
+      // worker, not an error).
+      if (request.bind_call.input_schema == null) {
+        throw new Error(
+          `${config.name}: is a table-in-out function (it requires an input row ` +
+            `stream) but no input schema was supplied -- call it via ` +
+            `table_in_out_function(input=...), not table_function().`,
+        );
+      }
+
       const state = config.initialState
         ? config.initialState(processParams)
         : (null as TState);
@@ -368,7 +466,7 @@ export function defineTableInOutFunction<
 
       return {
         outputSchema,
-        inputSchema: request.bind_call.input_schema ?? undefined,
+        inputSchema: request.bind_call.input_schema,
         exchangeInit: () => ({ state, processParams }),
         exchangeFn: async (
           eState: {
@@ -382,6 +480,7 @@ export function defineTableInOutFunction<
           // custom metadata (attached by the C++ WriteInputBatch), so process()
           // can answer a 0-row `notModified` batch instead of recomputing.
           applyRevalidationValidators(eState.processParams, input);
+          applyDynamicFilterUpdate(eState.processParams, input);
           // Bake emit metadata onto the batch (0-row HTTP survival), then
           // reconcile emitted batches to the (possibly projected) output schema
           // by name — a process() may emit its full declared schema and let the
@@ -390,8 +489,9 @@ export function defineTableInOutFunction<
           if (projIds) {
             wrappedOut = makeSchemaReconcilingCollector(wrappedOut, outputSchema);
           }
-          if (config.autoApplyFilters && pushdownFilters) {
-            wrappedOut = new FilteringOutputCollector(wrappedOut, pushdownFilters) as unknown as OutputCollector;
+          const currentFilters = eState.processParams.pushdownFilters;
+          if (config.autoApplyFilters && currentFilters) {
+            wrappedOut = new FilteringOutputCollector(wrappedOut, currentFilters) as unknown as OutputCollector;
           }
           await processFn(eState.processParams, eState.state, input, wrappedOut);
 
@@ -429,6 +529,29 @@ function applyRevalidationValidators(
   params.ifModifiedSince = md?.get(CACHE_IF_MODIFIED_SINCE_KEY);
 }
 
+function applyDynamicFilterUpdate(
+  params: { pushdownFilters?: PushdownFilters },
+  input: VgiBatch,
+): void {
+  const encoded = ((input as any)?.metadata as Map<string, string> | undefined)?.get("vgi_pushdown_filters");
+  if (!encoded) return;
+  if (!params.pushdownFilters) throw new Error("dynamic filter delta received without an initial snapshot");
+  if (new TextEncoder().encode(encoded).byteLength > (24 << 20)) {
+    throw new Error("base64 dynamic filter metadata exceeds the encoded-size limit");
+  }
+  if (encoded.length % 4 !== 0 ||
+      !/^(?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?$/.test(encoded)) {
+    throw new Error("dynamic filter metadata is not valid base64");
+  }
+  const binary = (globalThis as any).atob
+    ? (globalThis as any).atob(encoded)
+    : Buffer.from(encoded, "base64").toString("binary");
+  const bytes = Uint8Array.from(binary, (character: string) => character.charCodeAt(0));
+  if (bytes.byteLength > (17 << 20)) throw new Error("dynamic filter IPC payload exceeds the encoded-size limit");
+  const current = params.pushdownFilters;
+  params.pushdownFilters = current.applyDelta(deserializeBatch(bytes));
+}
+
 /**
  * Wrap an OutputCollector so a pre-built batch emitted WITH metadata carries
  * that metadata on the batch object itself (merged over any metadata the
@@ -615,8 +738,12 @@ export interface RowTransformConfig<TArgs = Record<string, any>> {
   /** Named (string-position) args stay bind-time scalars on `params.args`. */
   namedArgs?: Record<string, VgiDataType>;
   argDefaults?: Record<string, any>;
+  /** Authoritative typed defaults: one row, defaulted parameters only. */
+  parameterDefaultValues?: VgiBatch | null;
   /** Per-argument descriptions keyed by arg name (surfaced as `vgi_doc`). */
   argDocs?: Record<string, string>;
+  /** Discovery constraints, also enforced for named scalar arguments at bind. */
+  argConstraints?: Record<string, ArgumentConstraints>;
   /** Bind: return the output schema. The input schema (the declared per-row
    *  columns, typed by the C++ bind) is on `params.bindCall.input_schema`. */
   onBind: (params: TableInOutBindParams<TArgs>) =>
@@ -683,6 +810,7 @@ export function defineRowTransformFunction<
         position: posIdx++,
         arrowType: type,
         doc: config.argDocs?.[name],
+        ...constraintSpecFields(config.argConstraints?.[name]),
       });
     }
   }
@@ -693,6 +821,7 @@ export function defineRowTransformFunction<
       arrowType: config.varargs.type,
       isVarargs: true,
       doc: config.varargs.doc ?? config.argDocs?.[config.varargs.name],
+      ...constraintSpecFields(config.argConstraints?.[config.varargs.name]),
     });
   }
   if (config.namedArgs) {
@@ -702,6 +831,10 @@ export function defineRowTransformFunction<
         position: name,
         arrowType: type,
         doc: config.argDocs?.[name],
+        ...constraintSpecFields(
+          config.argConstraints?.[name],
+          config.argDefaults?.[name],
+        ),
       });
     }
   }
@@ -716,6 +849,7 @@ export function defineRowTransformFunction<
     examples: config.examples,
     categories: config.categories,
     tags: config.tags,
+    parameterDefaultValues: config.parameterDefaultValues,
     maxWorkers: config.maxWorkers,
     requiredSettings: config.requiredSettings,
     requiredSecrets: config.requiredSecrets,
@@ -735,7 +869,9 @@ export function defineRowTransformFunction<
           ? config.argDefaults[spec.name]
           : undefined;
       let val = request.arguments.get(spec.position, defaultVal ?? null);
-      if (typeof val === "bigint") val = Number(val);
+      val = narrowArgValue(val);
+      const constraints = config.argConstraints?.[spec.name];
+      if (constraints) validateConstConstraints(spec.name, constraints, val);
       args[spec.name] = val;
     }
     return args as TArgs;
@@ -755,6 +891,7 @@ export function defineRowTransformFunction<
         bindCall: request,
         settings,
         secrets,
+        argumentNames: request.argument_names ?? null,
       });
       return {
         output_schema: result.outputSchema,
@@ -795,9 +932,14 @@ export function defineRowTransformFunction<
         ? projectSchema(projIds, request.output_schema)
         : request.output_schema;
 
-      const joinKeysLookup = buildJoinKeysLookup(request.join_keys);
       const pushdownFilters = request.pushdown_filters
-        ? deserializeFilters(request.pushdown_filters, joinKeysLookup)
+        ? deserializeFilters(request.pushdown_filters, {
+          outputSchema: request.output_schema,
+          joinKeyBatches: request.join_keys,
+          extensionFunctions: meta.additionalFilterFunctions,
+          runtimeAlgorithms: meta.runtimeFilterAlgorithms,
+          evaluationContexts: meta.filterEvaluationContexts,
+        })
         : undefined;
 
       const boundStorage = new BoundStorage(
@@ -824,9 +966,26 @@ export function defineRowTransformFunction<
         );
       }
 
+      if (request.bind_call.input_schema == null) {
+        // A caller drove this function via table_function() (or the bare
+        // producer path) instead of table_in_out_function(input=...) -- a
+        // blended row-transform function's positional args ARE its per-row
+        // input columns, so it cannot run with no input stream at all. Left
+        // unguarded, this doesn't fail here -- it silently degrades to an
+        // empty inputSchema, which disables the transport layer's own
+        // schema-conformance check downstream and leaves both sides waiting
+        // on each other forever: confirmed live, a non-terminating exchange/
+        // continuation loop against a real deployed worker, not an error.
+        throw new Error(
+          `${config.name}: is a table-in-out function (its positional args are ` +
+            `per-row input columns) but no input schema was supplied -- call it ` +
+            `via table_in_out_function(input=...), not table_function().`,
+        );
+      }
+
       return {
         outputSchema,
-        inputSchema: request.bind_call.input_schema ?? undefined,
+        inputSchema: request.bind_call.input_schema,
         exchangeInit: () => ({ state: null, processParams }),
         exchangeFn: async (
           eState: { state: null; processParams: RowTransformProcessParams<TArgs> },
@@ -834,12 +993,14 @@ export function defineRowTransformFunction<
           out: OutputCollector,
         ) => {
           applyRevalidationValidators(eState.processParams, input);
+          applyDynamicFilterUpdate(eState.processParams, input);
           let wrappedOut: OutputCollector = makeMetadataBakingCollector(out);
           if (projIds) {
             wrappedOut = makeSchemaReconcilingCollector(wrappedOut, outputSchema);
           }
-          if (config.autoApplyFilters && pushdownFilters) {
-            wrappedOut = new FilteringOutputCollector(wrappedOut, pushdownFilters) as unknown as OutputCollector;
+          const currentFilters = eState.processParams.pushdownFilters;
+          if (config.autoApplyFilters && currentFilters) {
+            wrappedOut = new FilteringOutputCollector(wrappedOut, currentFilters) as unknown as OutputCollector;
           }
           await config.process(eState.processParams, input, wrappedOut);
         },