@query-farm/vgi 0.29.0 → 0.30.0

package/src/filter-pushdown/index.ts

@@ -4,22 +4,36 @@
 // transparently. Ported from vgi-python/vgi/table_filter_pushdown.py.
 
 export {
+  FILTER_ENCODING,
+  FILTER_VERSION,
+  DUCKDB_STANDARD_V1,
+  NO_EVALUATION_CONTEXT,
+  DUCKDB_SESSION_CONTEXT,
   ComparisonOp,
+  type ArithmeticOp,
+  type EvaluationContext,
+  type Filter,
   type AndFilter,
   type ConstantFilter,
   type ExpressionFilter,
   type ExprNode,
-  type Filter,
   type InFilter,
   type IsNotNullFilter,
   type IsNullFilter,
   type OrFilter,
   type StructFilter,
+  type FilterExpression,
+  type FilterPredicate,
+  type FilterSet,
+  type FunctionIdentity,
+  type PredicateMode,
+  type PredicateSource,
+  type StandardFilterFunction,
 } from "./types.js";
 
 export { PushdownFilters } from "./evaluate.js";
 
-export { buildJoinKeysLookup, deserializeFilters } from "./deserialize.js";
+export { FilterV2Error, applyFilterDelta, buildJoinKeysLookup, deserializeFilters } from "./deserialize.js";
 
 export {
   FilteringOutputCollector,

package/src/filter-pushdown/types.ts

@@ -1,91 +1,108 @@
 // Copyright 2025, 2026 Query Farm LLC - https://query.farm
-// Filter pushdown type definitions.
+
+import type { VgiBatch, VgiDataType, VgiField } from "../arrow/index.js";
+
+export const FILTER_ENCODING = "vgi.filters.v2";
+export const FILTER_VERSION = "2";
+export const DUCKDB_STANDARD_V1 = "vgi.duckdb.standard.v1";
+export const NO_EVALUATION_CONTEXT = "vgi.none.v1";
+export const DUCKDB_SESSION_CONTEXT = "vgi.duckdb.session.v1";
 
 export enum ComparisonOp {
   EQ = "eq",
   NE = "ne",
-  GT = "gt",
-  GE = "ge",
   LT = "lt",
   LE = "le",
+  GT = "gt",
+  GE = "ge",
+  DISTINCT_FROM = "distinct_from",
+  NOT_DISTINCT_FROM = "not_distinct_from",
 }
 
-export interface ConstantFilter {
-  type: "constant";
-  columnName: string;
-  columnIndex: number;
-  op: ComparisonOp;
-  value: any;
-}
-
-export interface IsNullFilter {
-  type: "is_null";
-  columnName: string;
-  columnIndex: number;
-}
+export type ArithmeticOp = "add" | "subtract" | "multiply" | "divide" | "modulo";
+export type PredicateMode = "required" | "advisory";
+export type PredicateSource = "query" | "join" | "top_n" | "split_refinement" | "other";
+export type StandardFilterFunction = "starts_with" | "ends_with" | "contains" | "list_contains";
 
-export interface IsNotNullFilter {
-  type: "is_not_null";
-  columnName: string;
-  columnIndex: number;
+export interface FunctionIdentity {
+  namespace: string;
+  name: string;
+  version: number;
 }
 
-export interface InFilter {
-  type: "in";
-  columnName: string;
-  columnIndex: number;
-  values: Set<any>;
+export interface EvaluationContext {
+  profile: string;
+  timeZone?: string;
+  calendar?: string;
+  defaultCollation?: string;
+  ieeeFloatingPointOps?: boolean;
+  integerDivision?: boolean;
+  providerFingerprint?: string;
 }
 
-export interface AndFilter {
-  type: "and";
-  columnName: string;
-  columnIndex: number;
-  children: Filter[];
-}
+export type FilterExpression =
+  | { node: "column_ref"; columnIndex: number; columnName: string; dataType?: VgiDataType }
+  | { node: "field_ref"; expression: FilterExpression; fieldIndex: number; fieldName: string; dataType: VgiDataType }
+  | { node: "literal"; valueRef: number; field: VgiField; value: unknown }
+  | { node: "comparison"; op: ComparisonOp; left: FilterExpression; right: FilterExpression }
+  | { node: "and" | "or"; children: FilterExpression[] }
+  | { node: "not"; expression: FilterExpression }
+  | { node: "is_null"; expression: FilterExpression; negated: boolean }
+  | { node: "in"; expression: FilterExpression; set: FilterSet; negated: boolean }
+  | { node: "cast"; expression: FilterExpression; typeRef: number; field: VgiField }
+  | { node: "arithmetic"; op: ArithmeticOp; left: FilterExpression; right: FilterExpression }
+  | { node: "negate"; expression: FilterExpression }
+  | {
+      node: "call";
+      function: StandardFilterFunction | FunctionIdentity;
+      arguments: FilterExpression[];
+      options?: Record<string, unknown>;
+    }
+  | {
+      node: "runtime_filter";
+      algorithm: FunctionIdentity;
+      input: FilterExpression;
+      artifactRef: number;
+      field: VgiField;
+      artifact: unknown;
+      nullHandling: "pass" | "reject";
+      supported: boolean;
+    };
 
-export interface OrFilter {
-  type: "or";
-  columnName: string;
-  columnIndex: number;
-  children: Filter[];
-}
+export type FilterSet =
+  | { kind: "literal"; valueRef: number; field: VgiField; values: unknown[] }
+  | {
+      kind: "external";
+      batchIndex: number;
+      columnIndex: number;
+      columnName: string;
+      values: unknown[];
+      batch: VgiBatch;
+    };
 
-export interface StructFilter {
-  type: "struct";
-  columnName: string;
-  columnIndex: number;
-  childIndex: number;
-  childName: string;
-  childFilter: Filter;
+export interface FilterPredicate {
+  id: string;
+  revision: number;
+  mode: PredicateMode;
+  source: PredicateSource;
+  expression: FilterExpression;
 }
 
-export type Filter =
-  | ConstantFilter
-  | IsNullFilter
-  | IsNotNullFilter
-  | InFilter
-  | AndFilter
-  | OrFilter
-  | StructFilter
-  | ExpressionFilter;
-
-/**
- * Expression-tree filter. The expr is a parsed AST; evaluation matches
- * DuckDB's semantics for the function names the table function advertises in
- * `supportedExpressionFilters` (e.g. list_contains, starts_with, contains,
- * &&/st_intersects_extent for spatial).
- */
-export interface ExpressionFilter {
-  type: "expression";
-  columnName: string;
-  columnIndex: number;
-  expr: ExprNode;
+export interface FilterCapabilities {
+  extensionFunctions?: ReadonlyArray<FunctionIdentity>;
+  runtimeAlgorithms?: ReadonlyArray<FunctionIdentity>;
+  evaluationContexts?: ReadonlyArray<{ profile: string; providerFingerprint?: string | null }>;
 }
 
-export type ExprNode =
-  | { expr_type: "column_ref"; index: number }
-  | { expr_type: "constant"; value: any }
-  | { expr_type: "function"; function_name: string; children: ExprNode[] }
-  | { expr_type: "comparison"; op: ComparisonOp; left: ExprNode; right: ExprNode }
-  | { expr_type: "conjunction"; conjunction_type: "and" | "or"; children: ExprNode[] };
+// Source aliases ease the protocol-2.0 transition for applications that imported
+// the old names. The wire model itself is v2-only.
+export type Filter = FilterExpression;
+export type ExprNode = FilterExpression;
+export type ConstantFilter = FilterExpression;
+export type IsNullFilter = FilterExpression;
+export type IsNotNullFilter = FilterExpression;
+export type InFilter = FilterExpression;
+export type AndFilter = FilterExpression;
+export type OrFilter = FilterExpression;
+export type StructFilter = FilterExpression;
+export type ExpressionFilter = FilterExpression;

package/src/functions/aggregate.ts

@@ -64,6 +64,8 @@ export interface AggregateBindParams<TArgs = Record<string, any>> {
   inputSchema: VgiSchema | null;
   settings: Record<string, any>;
   secrets: Record<string, Record<string, any>>;
+  /** Resolved names aligned with the complete logical argument order. */
+  argumentNames: (string | null)[] | null;
 }
 
 export interface AggregateUpdateParams<TArgs = Record<string, any>, TState = any> {
@@ -128,6 +130,8 @@ export interface AggregateFunctionConfig<TArgs = Record<string, any>, TState = a
   onBind?: (params: AggregateBindParams<TArgs>) => VgiDataType | Promise<VgiDataType>;
   /** Optional per-arg default values (positional only here). */
   argDefaults?: Record<string, any>;
+  /** Authoritative typed defaults: one row, defaulted parameters only. */
+  parameterDefaultValues?: VgiBatch | null;
   /**
    * Per-argument discovery constraints (choices / ge / le / gt / lt / pattern),
    * keyed by argument name. Surfaced via `vgi_function_arguments()` and enforced
@@ -356,7 +360,10 @@ export function defineAggregate<TArgs = Record<string, any>, TState = any>(
         isAnyType,
         isVarargs: varargsSet.has(name),
         isConst,
-        ...constraintSpecFields(config.argConstraints?.[name]),
+        ...constraintSpecFields(
+          config.argConstraints?.[name],
+          config.argDefaults?.[name],
+        ),
       });
     }
   }
@@ -367,6 +374,7 @@ export function defineAggregate<TArgs = Record<string, any>, TState = any>(
     examples: config.examples,
     categories: config.categories,
     tags: config.tags,
+    parameterDefaultValues: config.parameterDefaultValues,
     nullHandling: config.nullHandling as any,
     requiredSecrets: config.requiredSecrets,
   };

package/src/functions/registry.ts

@@ -11,18 +11,19 @@ import type { VgiFunction } from "./types.js";
 import type { Arguments } from "../arguments/arguments.js";
 import type { ArgumentSpec } from "../arguments/argument-spec.js";
 import { FunctionNotFoundError } from "../errors.js";
+import { schemaPathDisplay, schemaPathKey } from "../schema-path.js";
 
 export interface OverloadContext {
   arguments?: Arguments;
   inputSchema?: VgiSchema | null;
   isScalar?: boolean;
   /**
-   * Catalog schema that declares the function, from `BindRequest.schema_name`.
+   * Catalog schema that declares the function, from `BindRequest.schema_path`.
    * A worker may register the same name in more than one schema, so the bare
    * name is not a unique key. When set, resolution is scoped to that schema and
    * never falls back to another one; when absent, every schema is searched.
    */
-  schemaName?: string | null;
+  schemaPath?: string[] | null;
   /**
    * Catalog that owns the function, resolved from the bind's
    * `attach_opaque_data`. Two catalogs served by one worker may each declare
@@ -205,13 +206,13 @@ function filterByArgumentTypes(
 }
 
 /** Key for the schema-scoped index: lowercased schema, then function name. */
-function schemaKey(schemaName: string, functionName: string): string {
-  return `${schemaName.toLowerCase()}\u0000${functionName}`;
+function schemaKey(schemaPath: string[], functionName: string): string {
+  return `${schemaPathKey(schemaPath)}\u0001${functionName}`;
 }
 
 /** Key for the catalog-scoped index: lowercased catalog, then the schema key. */
-function catalogKey(catalogName: string, schemaName: string, functionName: string): string {
-  return `${catalogName.toLowerCase()}\u0000${schemaKey(schemaName, functionName)}`;
+function catalogKey(catalogName: string, schemaPath: string[], functionName: string): string {
+  return `${catalogName.toLowerCase()}\u0000${schemaKey(schemaPath, functionName)}`;
 }
 
 export class FunctionRegistry {
@@ -234,12 +235,12 @@ export class FunctionRegistry {
   }
 
   /**
-   * Record that `func` is declared in `schemaName`, in addition to the flat
+   * Record that `func` is declared in `schemaPath`, in addition to the flat
    * by-name index. Idempotent, and independent of `register()` so a function
    * reachable from several schemas (e.g. a scan function referenced by tables in
    * more than one schema) resolves from any of them.
    */
-  registerInSchema(func: VgiFunction, schemaName: string, catalogName?: string): void {
+  registerInSchema(func: VgiFunction, schemaPath: string[], catalogName?: string): void {
     const add = (map: Map<string, VgiFunction[]>, key: string) => {
       const bucket = map.get(key);
       if (!bucket) {
@@ -248,20 +249,21 @@ export class FunctionRegistry {
       }
       if (!bucket.includes(func)) bucket.push(func);
     };
-    add(this._bySchema, schemaKey(schemaName, func.meta.name));
+    add(this._bySchema, schemaKey(schemaPath, func.meta.name));
     if (catalogName) {
-      add(this._byCatalog, catalogKey(catalogName, schemaName, func.meta.name));
+      add(this._byCatalog, catalogKey(catalogName, schemaPath, func.meta.name));
     }
   }
 
   /** Schemas that declare `functionName`, sorted — for error messages. */
-  schemasFor(functionName: string): string[] {
-    const out: string[] = [];
+  schemasFor(functionName: string): string[][] {
+    const out: string[][] = [];
     for (const key of this._bySchema.keys()) {
-      const [schema, name] = key.split("\u0000");
-      if (name === functionName) out.push(schema);
+      const separator = key.lastIndexOf("\u0001");
+      const name = key.slice(separator + 1);
+      if (name === functionName) out.push(key.slice(0, separator).split("\u0000"));
     }
-    return out.sort();
+    return out.sort((left, right) => schemaPathKey(left).localeCompare(schemaPathKey(right)));
   }
 
   get(name: string, context?: OverloadContext): VgiFunction {
@@ -273,23 +275,23 @@ export class FunctionRegistry {
     // Catalog first: it is the only key that separates two catalogs declaring
     // the same schema and name.
     const catalogName = context?.catalogName;
-    const schemaName = context?.schemaName;
-    if (catalogName && schemaName) {
-      const scoped = this._byCatalog.get(catalogKey(catalogName, schemaName, name));
+    const schemaPath = context?.schemaPath;
+    if (catalogName && schemaPath) {
+      const scoped = this._byCatalog.get(catalogKey(catalogName, schemaPath, name));
       if (scoped && scoped.length > 0) {
         return this._disambiguate(name, scoped, context);
       }
     }
-    if (schemaName) {
-      const scoped = this._bySchema.get(schemaKey(schemaName, name));
+    if (schemaPath) {
+      const scoped = this._bySchema.get(schemaKey(schemaPath, name));
       if (scoped && scoped.length > 0) {
         candidates = scoped;
       } else if (candidates && candidates.length > 0) {
         const schemas = this.schemasFor(name);
         if (schemas.length > 0) {
           throw new Error(
-            `Function '${name}' is not registered in schema '${schemaName}'. ` +
-              `It is available in: [${schemas.join(", ")}]`,
+            `Function '${name}' is not registered in schema ${schemaPathDisplay(schemaPath)}. ` +
+              `It is available in: [${schemas.map(schemaPathDisplay).join(", ")}]`,
           );
         }
       }

package/src/functions/scalar.ts

@@ -21,6 +21,7 @@ import {
   FunctionType,
   FunctionStability,
   NullHandling,
+  ArgumentMonotonicity,
   DEFAULT_MAX_WORKERS,
 } from "../types.js";
 import { RowCountMismatchError } from "../errors.js";
@@ -63,6 +64,8 @@ export interface ScalarBindParameters {
   secrets: Record<string, Record<string, any>>;
   /** Original bind request — exposes attach_opaque_data, transaction_opaque_data, function_type, etc. */
   bindCall: BindRequest;
+  /** Resolved names aligned with the complete logical argument order. */
+  argumentNames: (string | null)[] | null;
 }
 
 // ============================================================================
@@ -178,9 +181,12 @@ export interface ScalarFunctionConfig<
   // Metadata
   stability?: FunctionStability;
   nullHandling?: NullHandling;
+  /** One scalar monotonicity claim per ordered parameter declaration. */
+  argumentMonotonicity?: ArgumentMonotonicity[] | null;
   examples?: FunctionExample[];
   categories?: string[];
   tags?: Record<string, string>;
+  parameterDefaultValues?: VgiBatch | null;
   maxWorkers?: number;
   requiredSettings?: string[];
   requiredSecrets?: string[];
@@ -262,6 +268,23 @@ export function defineScalarFunction<
     }
   }
 
+  if (config.argumentMonotonicity !== undefined && config.argumentMonotonicity !== null) {
+    if (config.argumentMonotonicity.length !== specs.length) {
+      throw new Error(
+        `defineScalarFunction("${config.name}"): argumentMonotonicity has ` +
+        `${config.argumentMonotonicity.length} entries, expected ${specs.length} declaration slots`,
+      );
+    }
+    const allowed = new Set<string>(Object.values(ArgumentMonotonicity));
+    for (const [index, value] of config.argumentMonotonicity.entries()) {
+      if (!allowed.has(value)) {
+        throw new Error(
+          `defineScalarFunction("${config.name}"): argumentMonotonicity[${index}] has unknown value ${String(value)}`,
+        );
+      }
+    }
+  }
+
   // Raw const-arg constraints for bind-time enforcement, keyed by arg name.
   // Only the ordered `parameters` API carries constraints (the legacy
   // params/constParams path does not), so only its const params appear here.
@@ -287,9 +310,11 @@ export function defineScalarFunction<
     description: config.description,
     stability: config.stability,
     nullHandling: config.nullHandling,
+    argumentMonotonicity: config.argumentMonotonicity,
     examples: config.examples,
     categories: config.categories,
     tags: config.tags,
+    parameterDefaultValues: config.parameterDefaultValues,
     maxWorkers: config.maxWorkers,
     requiredSettings: config.requiredSettings,
     requiredSecrets: config.requiredSecrets,
@@ -339,6 +364,7 @@ export function defineScalarFunction<
           settings,
           secrets,
           bindCall: request,
+          argumentNames: request.argument_names ?? null,
         });
       } else if (config.returns) {
         outputType = config.returns;

package/src/functions/table-buffering.ts

@@ -39,13 +39,17 @@ import type {
   StreamHandlers,
   FunctionExample,
 } from "./types.js";
-import type { ArgumentSpec } from "../arguments/argument-spec.js";
-import { assertArrowType, assertArrowTypes } from "../arguments/argument-spec.js";
+import type { ArgumentConstraints, ArgumentSpec } from "../arguments/argument-spec.js";
+import {
+  assertArrowType,
+  assertArrowTypes,
+  constraintSpecFields,
+  validateConstConstraints,
+} from "../arguments/argument-spec.js";
 import { batchToScalarDict, batchToSecretDict, projectSchema } from "../util/arrow/index.js";
 import { batchFromColumns, readCanonicalValue } from "../arrow/index.js";
 import { codecFor } from "../arrow/codec/registry.js";
 import {
-  buildJoinKeysLookup,
   deserializeFilters,
   FilteringOutputCollector,
   type PushdownFilters,
@@ -71,6 +75,8 @@ export interface TableBufferingBindParams<TArgs = Record<string, any>> {
   /** True on the second bind pass, after the connector resolved the secrets
    *  requested via the onBind `lookupSecret*` return fields on the first pass. */
   resolvedSecretsProvided: boolean;
+  /** Resolved names aligned with the complete logical argument order. */
+  argumentNames?: (string | null)[] | null;
 }
 
 export interface TableBufferingParams<TArgs = Record<string, any>> {
@@ -107,6 +113,12 @@ export interface TableBufferingConfig<
   args?: Record<string, VgiDataType>;
   namedArgs?: Record<string, VgiDataType>;
   argDefaults?: Record<string, any>;
+  /** Authoritative typed defaults: one row, defaulted parameters only. */
+  parameterDefaultValues?: VgiBatch | null;
+  /** Argument descriptions keyed by argument name. */
+  argDocs?: Record<string, string>;
+  /** Discovery constraints, also enforced for scalar arguments at bind. */
+  argConstraints?: Record<string, ArgumentConstraints>;
   /** Bind: default passes through input schema. May be async. Return non-empty
    *  `lookupSecret*` lists on the first pass to request DuckDB secrets; the
    *  connector then re-binds with `resolvedSecretsProvided` and the resolved
@@ -196,7 +208,16 @@ export function defineTableBufferingFunction<
   let posIdx = 0;
   if (config.args) {
     for (const [name, type] of Object.entries(config.args)) {
-      specs.push({ name, position: posIdx++, arrowType: type });
+      specs.push({
+        name,
+        position: posIdx++,
+        arrowType: type,
+        doc: config.argDocs?.[name],
+        ...constraintSpecFields(
+          config.argConstraints?.[name],
+          config.argDefaults?.[name],
+        ),
+      });
     }
   }
   specs.push({
@@ -207,7 +228,16 @@ export function defineTableBufferingFunction<
   });
   if (config.namedArgs) {
     for (const [name, type] of Object.entries(config.namedArgs)) {
-      specs.push({ name, position: name, arrowType: type });
+      specs.push({
+        name,
+        position: name,
+        arrowType: type,
+        doc: config.argDocs?.[name],
+        ...constraintSpecFields(
+          config.argConstraints?.[name],
+          config.argDefaults?.[name],
+        ),
+      });
     }
   }
 
@@ -221,6 +251,7 @@ export function defineTableBufferingFunction<
     examples: config.examples,
     categories: config.categories,
     tags: config.tags,
+    parameterDefaultValues: config.parameterDefaultValues,
     maxWorkers: config.maxWorkers,
     requiredSettings: config.requiredSettings,
     requiredSecrets: config.requiredSecrets,
@@ -242,6 +273,8 @@ export function defineTableBufferingFunction<
           : undefined;
       let val = request.arguments.get(spec.position, defaultVal);
       if (typeof val === "bigint") val = Number(val);
+      const constraints = config.argConstraints?.[spec.name];
+      if (constraints) validateConstConstraints(spec.name, constraints, val);
       args[spec.name] = val;
     }
     return args as TArgs;
@@ -265,6 +298,7 @@ export function defineTableBufferingFunction<
           settings,
           secrets,
           resolvedSecretsProvided: request.resolved_secrets_provided ?? false,
+          argumentNames: request.argument_names ?? null,
         });
         return {
           output_schema: result.outputSchema,
@@ -326,9 +360,14 @@ export function defineTableBufferingFunction<
         ? projectSchema(projIds, request.output_schema)
         : request.output_schema;
 
-      const joinKeysLookup = buildJoinKeysLookup(request.join_keys);
       const pushdownFilters = request.pushdown_filters
-        ? deserializeFilters(request.pushdown_filters, joinKeysLookup)
+        ? deserializeFilters(request.pushdown_filters, {
+          outputSchema: request.output_schema,
+          joinKeyBatches: request.join_keys,
+          extensionFunctions: meta.additionalFilterFunctions,
+          runtimeAlgorithms: meta.runtimeFilterAlgorithms,
+          evaluationContexts: meta.filterEvaluationContexts,
+        })
         : undefined;
 
       const bound = new BoundStorage(