@query-farm/vgi 0.28.0 → 0.29.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +52 -8
- package/dist/arguments/argument-spec.d.ts +57 -2
- package/dist/arrow/impl-arrowjs/index.d.ts +1 -1
- package/dist/arrow/impl-arrowjs/iterate.d.ts +10 -1
- package/dist/arrow/impl-arrowjs/schema.d.ts +2 -0
- package/dist/arrow/impl-flechette/index.d.ts +1 -1
- package/dist/arrow/impl-flechette/iterate.d.ts +10 -1
- package/dist/arrow/impl-flechette/schema.d.ts +2 -0
- package/dist/arrow/index.d.ts +1 -1
- package/dist/arrow/schema-types.d.ts +3 -1
- package/dist/arrow/types.d.ts +39 -3
- package/dist/catalog/composite.d.ts +13 -13
- package/dist/catalog/descriptors.d.ts +7 -3
- package/dist/catalog/interface.d.ts +108 -38
- package/dist/catalog/read-only.d.ts +32 -15
- package/dist/client/client.d.ts +37 -30
- package/dist/client/types.d.ts +2 -0
- package/dist/client-entry.d.ts +1 -1
- package/dist/client-entry.js +791 -447
- package/dist/client-entry.js.map +18 -17
- package/dist/filter-pushdown/capabilities.d.ts +3 -0
- package/dist/filter-pushdown/deserialize.d.ts +14 -15
- package/dist/filter-pushdown/evaluate.d.ts +14 -44
- package/dist/filter-pushdown/index.d.ts +2 -2
- package/dist/filter-pushdown/types.d.ts +120 -68
- package/dist/functions/aggregate.d.ts +4 -0
- package/dist/functions/registry.d.ts +5 -5
- package/dist/functions/scalar.d.ts +6 -1
- package/dist/functions/table-buffering.d.ts +9 -0
- package/dist/functions/table-in-out.d.ts +13 -0
- package/dist/functions/table.d.ts +72 -2
- package/dist/functions/types.d.ts +60 -2
- package/dist/generated/vgi-client.d.ts +114 -26
- package/dist/generated/vgi-protocol-schemas.d.ts +34 -0
- package/dist/http/fetch.d.ts +16 -0
- package/dist/index.core.d.ts +4 -3
- package/dist/index.js +3396 -2064
- package/dist/index.js.map +55 -49
- package/dist/metadata/types.d.ts +15 -2
- package/dist/protocol/handlers/catalog/shared.d.ts +0 -8
- package/dist/protocol/handlers/function.d.ts +11 -0
- package/dist/protocol/handlers/shared.d.ts +3 -1
- package/dist/protocol/serializers/bind.d.ts +1 -2
- package/dist/protocol/serializers/cardinality.d.ts +1 -2
- package/dist/protocol/serializers/splits.d.ts +126 -0
- package/dist/protocol/types.d.ts +36 -3
- package/dist/request-auth.d.ts +28 -0
- package/dist/schema-path.d.ts +7 -0
- package/dist/serve-entry.d.ts +25 -0
- package/dist/serve-entry.js +1129 -569
- package/dist/serve-entry.js.map +36 -32
- package/dist/split-token.d.ts +88 -0
- package/dist/types.d.ts +8 -0
- package/dist/worker-cf-entry.js +2928 -1689
- package/dist/worker-cf-entry.js.map +51 -44
- package/dist/worker.d.ts +1 -1
- package/package.json +3 -3
- package/src/arguments/argument-spec.ts +93 -4
- package/src/arguments/arguments.ts +0 -1
- package/src/arrow/impl-arrowjs/index.ts +1 -1
- package/src/arrow/impl-arrowjs/iterate.ts +21 -0
- package/src/arrow/impl-arrowjs/schema.ts +4 -0
- package/src/arrow/impl-flechette/index.ts +1 -1
- package/src/arrow/impl-flechette/iterate.ts +24 -1
- package/src/arrow/impl-flechette/schema.ts +4 -0
- package/src/arrow/index.ts +1 -1
- package/src/arrow/schema-types.ts +4 -0
- package/src/arrow/types.ts +39 -3
- package/src/catalog/composite.ts +26 -26
- package/src/catalog/descriptors.ts +12 -3
- package/src/catalog/interface.ts +217 -72
- package/src/catalog/read-only.ts +141 -84
- package/src/client/client.ts +97 -73
- package/src/client/types.ts +2 -0
- package/src/client-entry.ts +2 -1
- package/src/filter-pushdown/capabilities.ts +13 -0
- package/src/filter-pushdown/collector.ts +22 -35
- package/src/filter-pushdown/deserialize.ts +657 -221
- package/src/filter-pushdown/evaluate.ts +410 -499
- package/src/filter-pushdown/index.ts +16 -2
- package/src/filter-pushdown/types.ts +88 -71
- package/src/functions/aggregate.ts +12 -1
- package/src/functions/copy-from.ts +2 -1
- package/src/functions/copy-to.ts +2 -1
- package/src/functions/registry.ts +24 -22
- package/src/functions/scalar.ts +37 -0
- package/src/functions/table-buffering.ts +48 -6
- package/src/functions/table-in-out.ts +194 -33
- package/src/functions/table.ts +187 -22
- package/src/functions/types.ts +64 -1
- package/src/generated/vgi-client.ts +122 -28
- package/src/generated/vgi-protocol-schemas.ts +455 -58
- package/src/http/fetch.ts +129 -2
- package/src/index.core.ts +4 -2
- package/src/metadata/resolve.ts +43 -1
- package/src/metadata/serialize.ts +43 -5
- package/src/metadata/types.ts +17 -1
- package/src/protocol/dispatch.ts +1 -1
- package/src/protocol/handlers/aggregate.ts +23 -12
- package/src/protocol/handlers/catalog/admin.ts +47 -54
- package/src/protocol/handlers/catalog/index_methods.ts +6 -13
- package/src/protocol/handlers/catalog/macro.ts +12 -20
- package/src/protocol/handlers/catalog/shared.ts +12 -57
- package/src/protocol/handlers/catalog/table.ts +49 -112
- package/src/protocol/handlers/catalog/view.ts +18 -22
- package/src/protocol/handlers/function.ts +311 -18
- package/src/protocol/handlers/shared.ts +12 -3
- package/src/protocol/handlers/table-buffering.ts +11 -8
- package/src/protocol/serializers/bind.ts +69 -67
- package/src/protocol/serializers/cardinality.ts +7 -7
- package/src/protocol/serializers/init.ts +52 -10
- package/src/protocol/serializers/splits.ts +221 -0
- package/src/protocol/state-serializer.ts +55 -1
- package/src/protocol/types.ts +36 -3
- package/src/request-auth.ts +73 -0
- package/src/schema-path.ts +25 -0
- package/src/serve-entry.ts +69 -2
- package/src/split-token.ts +297 -0
- package/src/types.ts +9 -0
- package/src/worker-cf-entry.ts +2 -1
- package/src/worker.ts +79 -14
|
@@ -4,22 +4,36 @@
|
|
|
4
4
|
// transparently. Ported from vgi-python/vgi/table_filter_pushdown.py.
|
|
5
5
|
|
|
6
6
|
export {
|
|
7
|
+
FILTER_ENCODING,
|
|
8
|
+
FILTER_VERSION,
|
|
9
|
+
DUCKDB_STANDARD_V1,
|
|
10
|
+
NO_EVALUATION_CONTEXT,
|
|
11
|
+
DUCKDB_SESSION_CONTEXT,
|
|
7
12
|
ComparisonOp,
|
|
13
|
+
type ArithmeticOp,
|
|
14
|
+
type EvaluationContext,
|
|
15
|
+
type Filter,
|
|
8
16
|
type AndFilter,
|
|
9
17
|
type ConstantFilter,
|
|
10
18
|
type ExpressionFilter,
|
|
11
19
|
type ExprNode,
|
|
12
|
-
type Filter,
|
|
13
20
|
type InFilter,
|
|
14
21
|
type IsNotNullFilter,
|
|
15
22
|
type IsNullFilter,
|
|
16
23
|
type OrFilter,
|
|
17
24
|
type StructFilter,
|
|
25
|
+
type FilterExpression,
|
|
26
|
+
type FilterPredicate,
|
|
27
|
+
type FilterSet,
|
|
28
|
+
type FunctionIdentity,
|
|
29
|
+
type PredicateMode,
|
|
30
|
+
type PredicateSource,
|
|
31
|
+
type StandardFilterFunction,
|
|
18
32
|
} from "./types.js";
|
|
19
33
|
|
|
20
34
|
export { PushdownFilters } from "./evaluate.js";
|
|
21
35
|
|
|
22
|
-
export { buildJoinKeysLookup, deserializeFilters } from "./deserialize.js";
|
|
36
|
+
export { FilterV2Error, applyFilterDelta, buildJoinKeysLookup, deserializeFilters } from "./deserialize.js";
|
|
23
37
|
|
|
24
38
|
export {
|
|
25
39
|
FilteringOutputCollector,
|
|
@@ -1,91 +1,108 @@
|
|
|
1
1
|
// Copyright 2025, 2026 Query Farm LLC - https://query.farm
|
|
2
|
-
|
|
2
|
+
|
|
3
|
+
import type { VgiBatch, VgiDataType, VgiField } from "../arrow/index.js";
|
|
4
|
+
|
|
5
|
+
export const FILTER_ENCODING = "vgi.filters.v2";
|
|
6
|
+
export const FILTER_VERSION = "2";
|
|
7
|
+
export const DUCKDB_STANDARD_V1 = "vgi.duckdb.standard.v1";
|
|
8
|
+
export const NO_EVALUATION_CONTEXT = "vgi.none.v1";
|
|
9
|
+
export const DUCKDB_SESSION_CONTEXT = "vgi.duckdb.session.v1";
|
|
3
10
|
|
|
4
11
|
export enum ComparisonOp {
|
|
5
12
|
EQ = "eq",
|
|
6
13
|
NE = "ne",
|
|
7
|
-
GT = "gt",
|
|
8
|
-
GE = "ge",
|
|
9
14
|
LT = "lt",
|
|
10
15
|
LE = "le",
|
|
16
|
+
GT = "gt",
|
|
17
|
+
GE = "ge",
|
|
18
|
+
DISTINCT_FROM = "distinct_from",
|
|
19
|
+
NOT_DISTINCT_FROM = "not_distinct_from",
|
|
11
20
|
}
|
|
12
21
|
|
|
13
|
-
export
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
op: ComparisonOp;
|
|
18
|
-
value: any;
|
|
19
|
-
}
|
|
20
|
-
|
|
21
|
-
export interface IsNullFilter {
|
|
22
|
-
type: "is_null";
|
|
23
|
-
columnName: string;
|
|
24
|
-
columnIndex: number;
|
|
25
|
-
}
|
|
22
|
+
export type ArithmeticOp = "add" | "subtract" | "multiply" | "divide" | "modulo";
|
|
23
|
+
export type PredicateMode = "required" | "advisory";
|
|
24
|
+
export type PredicateSource = "query" | "join" | "top_n" | "split_refinement" | "other";
|
|
25
|
+
export type StandardFilterFunction = "starts_with" | "ends_with" | "contains" | "list_contains";
|
|
26
26
|
|
|
27
|
-
export interface
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
27
|
+
export interface FunctionIdentity {
|
|
28
|
+
namespace: string;
|
|
29
|
+
name: string;
|
|
30
|
+
version: number;
|
|
31
31
|
}
|
|
32
32
|
|
|
33
|
-
export interface
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
33
|
+
export interface EvaluationContext {
|
|
34
|
+
profile: string;
|
|
35
|
+
timeZone?: string;
|
|
36
|
+
calendar?: string;
|
|
37
|
+
defaultCollation?: string;
|
|
38
|
+
ieeeFloatingPointOps?: boolean;
|
|
39
|
+
integerDivision?: boolean;
|
|
40
|
+
providerFingerprint?: string;
|
|
38
41
|
}
|
|
39
42
|
|
|
40
|
-
export
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
}
|
|
43
|
+
export type FilterExpression =
|
|
44
|
+
| { node: "column_ref"; columnIndex: number; columnName: string; dataType?: VgiDataType }
|
|
45
|
+
| { node: "field_ref"; expression: FilterExpression; fieldIndex: number; fieldName: string; dataType: VgiDataType }
|
|
46
|
+
| { node: "literal"; valueRef: number; field: VgiField; value: unknown }
|
|
47
|
+
| { node: "comparison"; op: ComparisonOp; left: FilterExpression; right: FilterExpression }
|
|
48
|
+
| { node: "and" | "or"; children: FilterExpression[] }
|
|
49
|
+
| { node: "not"; expression: FilterExpression }
|
|
50
|
+
| { node: "is_null"; expression: FilterExpression; negated: boolean }
|
|
51
|
+
| { node: "in"; expression: FilterExpression; set: FilterSet; negated: boolean }
|
|
52
|
+
| { node: "cast"; expression: FilterExpression; typeRef: number; field: VgiField }
|
|
53
|
+
| { node: "arithmetic"; op: ArithmeticOp; left: FilterExpression; right: FilterExpression }
|
|
54
|
+
| { node: "negate"; expression: FilterExpression }
|
|
55
|
+
| {
|
|
56
|
+
node: "call";
|
|
57
|
+
function: StandardFilterFunction | FunctionIdentity;
|
|
58
|
+
arguments: FilterExpression[];
|
|
59
|
+
options?: Record<string, unknown>;
|
|
60
|
+
}
|
|
61
|
+
| {
|
|
62
|
+
node: "runtime_filter";
|
|
63
|
+
algorithm: FunctionIdentity;
|
|
64
|
+
input: FilterExpression;
|
|
65
|
+
artifactRef: number;
|
|
66
|
+
field: VgiField;
|
|
67
|
+
artifact: unknown;
|
|
68
|
+
nullHandling: "pass" | "reject";
|
|
69
|
+
supported: boolean;
|
|
70
|
+
};
|
|
46
71
|
|
|
47
|
-
export
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
72
|
+
export type FilterSet =
|
|
73
|
+
| { kind: "literal"; valueRef: number; field: VgiField; values: unknown[] }
|
|
74
|
+
| {
|
|
75
|
+
kind: "external";
|
|
76
|
+
batchIndex: number;
|
|
77
|
+
columnIndex: number;
|
|
78
|
+
columnName: string;
|
|
79
|
+
values: unknown[];
|
|
80
|
+
batch: VgiBatch;
|
|
81
|
+
};
|
|
53
82
|
|
|
54
|
-
export interface
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
childFilter: Filter;
|
|
83
|
+
export interface FilterPredicate {
|
|
84
|
+
id: string;
|
|
85
|
+
revision: number;
|
|
86
|
+
mode: PredicateMode;
|
|
87
|
+
source: PredicateSource;
|
|
88
|
+
expression: FilterExpression;
|
|
61
89
|
}
|
|
62
90
|
|
|
63
|
-
export
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
|
67
|
-
| InFilter
|
|
68
|
-
| AndFilter
|
|
69
|
-
| OrFilter
|
|
70
|
-
| StructFilter
|
|
71
|
-
| ExpressionFilter;
|
|
72
|
-
|
|
73
|
-
/**
|
|
74
|
-
* Expression-tree filter. The expr is a parsed AST; evaluation matches
|
|
75
|
-
* DuckDB's semantics for the function names the table function advertises in
|
|
76
|
-
* `supportedExpressionFilters` (e.g. list_contains, starts_with, contains,
|
|
77
|
-
* &&/st_intersects_extent for spatial).
|
|
78
|
-
*/
|
|
79
|
-
export interface ExpressionFilter {
|
|
80
|
-
type: "expression";
|
|
81
|
-
columnName: string;
|
|
82
|
-
columnIndex: number;
|
|
83
|
-
expr: ExprNode;
|
|
91
|
+
export interface FilterCapabilities {
|
|
92
|
+
extensionFunctions?: ReadonlyArray<FunctionIdentity>;
|
|
93
|
+
runtimeAlgorithms?: ReadonlyArray<FunctionIdentity>;
|
|
94
|
+
evaluationContexts?: ReadonlyArray<{ profile: string; providerFingerprint?: string | null }>;
|
|
84
95
|
}
|
|
85
96
|
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
97
|
+
// Source aliases ease the protocol-2.0 transition for applications that imported
|
|
98
|
+
// the old names. The wire model itself is v2-only.
|
|
99
|
+
export type Filter = FilterExpression;
|
|
100
|
+
export type ExprNode = FilterExpression;
|
|
101
|
+
export type ConstantFilter = FilterExpression;
|
|
102
|
+
export type IsNullFilter = FilterExpression;
|
|
103
|
+
export type IsNotNullFilter = FilterExpression;
|
|
104
|
+
export type InFilter = FilterExpression;
|
|
105
|
+
export type AndFilter = FilterExpression;
|
|
106
|
+
export type OrFilter = FilterExpression;
|
|
107
|
+
export type StructFilter = FilterExpression;
|
|
108
|
+
export type ExpressionFilter = FilterExpression;
|
|
@@ -20,6 +20,7 @@ import {
|
|
|
20
20
|
type ArgumentSpec,
|
|
21
21
|
type ArgumentConstraints,
|
|
22
22
|
} from "../arguments/argument-spec.js";
|
|
23
|
+
import { assertArrowType, assertArrowTypes } from "../arguments/argument-spec.js";
|
|
23
24
|
import type {
|
|
24
25
|
FunctionMeta,
|
|
25
26
|
VgiFunction,
|
|
@@ -63,6 +64,8 @@ export interface AggregateBindParams<TArgs = Record<string, any>> {
|
|
|
63
64
|
inputSchema: VgiSchema | null;
|
|
64
65
|
settings: Record<string, any>;
|
|
65
66
|
secrets: Record<string, Record<string, any>>;
|
|
67
|
+
/** Resolved names aligned with the complete logical argument order. */
|
|
68
|
+
argumentNames: (string | null)[] | null;
|
|
66
69
|
}
|
|
67
70
|
|
|
68
71
|
export interface AggregateUpdateParams<TArgs = Record<string, any>, TState = any> {
|
|
@@ -127,6 +130,8 @@ export interface AggregateFunctionConfig<TArgs = Record<string, any>, TState = a
|
|
|
127
130
|
onBind?: (params: AggregateBindParams<TArgs>) => VgiDataType | Promise<VgiDataType>;
|
|
128
131
|
/** Optional per-arg default values (positional only here). */
|
|
129
132
|
argDefaults?: Record<string, any>;
|
|
133
|
+
/** Authoritative typed defaults: one row, defaulted parameters only. */
|
|
134
|
+
parameterDefaultValues?: VgiBatch | null;
|
|
130
135
|
/**
|
|
131
136
|
* Per-argument discovery constraints (choices / ge / le / gt / lt / pattern),
|
|
132
137
|
* keyed by argument name. Surfaced via `vgi_function_arguments()` and enforced
|
|
@@ -333,6 +338,8 @@ export function deleteExecutionState(executionId: Uint8Array): void {
|
|
|
333
338
|
export function defineAggregate<TArgs = Record<string, any>, TState = any>(
|
|
334
339
|
config: AggregateFunctionConfig<TArgs, TState>,
|
|
335
340
|
): VgiFunction & { aggregateConfig: AggregateFunctionConfig<TArgs, TState> } {
|
|
341
|
+
assertArrowTypes((config as any).args, `defineAggregate("${config.name}"): args`);
|
|
342
|
+
|
|
336
343
|
const specs: ArgumentSpec[] = [];
|
|
337
344
|
let posIdx = 0;
|
|
338
345
|
const varargsSet = new Set(config.varargs ?? []);
|
|
@@ -353,7 +360,10 @@ export function defineAggregate<TArgs = Record<string, any>, TState = any>(
|
|
|
353
360
|
isAnyType,
|
|
354
361
|
isVarargs: varargsSet.has(name),
|
|
355
362
|
isConst,
|
|
356
|
-
...constraintSpecFields(
|
|
363
|
+
...constraintSpecFields(
|
|
364
|
+
config.argConstraints?.[name],
|
|
365
|
+
config.argDefaults?.[name],
|
|
366
|
+
),
|
|
357
367
|
});
|
|
358
368
|
}
|
|
359
369
|
}
|
|
@@ -364,6 +374,7 @@ export function defineAggregate<TArgs = Record<string, any>, TState = any>(
|
|
|
364
374
|
examples: config.examples,
|
|
365
375
|
categories: config.categories,
|
|
366
376
|
tags: config.tags,
|
|
377
|
+
parameterDefaultValues: config.parameterDefaultValues,
|
|
367
378
|
nullHandling: config.nullHandling as any,
|
|
368
379
|
requiredSecrets: config.requiredSecrets,
|
|
369
380
|
};
|
|
@@ -31,6 +31,7 @@ import type {
|
|
|
31
31
|
import type { VgiFunction, FunctionMeta, StreamHandlers, FunctionExample } from "./types.js";
|
|
32
32
|
import type { CopySecretLookup } from "./copy-to.js";
|
|
33
33
|
import type { ArgumentSpec } from "../arguments/argument-spec.js";
|
|
34
|
+
import { narrowArgValue } from "../arguments/argument-spec.js";
|
|
34
35
|
import { batchToScalarDict, batchToSecretDict, safeNumber } from "../util/arrow/index.js";
|
|
35
36
|
import { BoundStorage, storage as globalStorage } from "./storage.js";
|
|
36
37
|
import type { TableProcessParams } from "./table.js";
|
|
@@ -165,7 +166,7 @@ export function defineCopyFromFunction<TArgs = Record<string, unknown>>(
|
|
|
165
166
|
}
|
|
166
167
|
val = opt.default;
|
|
167
168
|
}
|
|
168
|
-
|
|
169
|
+
val = narrowArgValue(val);
|
|
169
170
|
if (opt.choices && !opt.choices.includes(val)) {
|
|
170
171
|
throw new Error(
|
|
171
172
|
`COPY (FORMAT ${config.format}): option '${name}' value ${JSON.stringify(val)} ` +
|
package/src/functions/copy-to.ts
CHANGED
|
@@ -38,6 +38,7 @@
|
|
|
38
38
|
import { type VgiSchema, schema, type VgiBatch, type VgiDataType } from "../arrow/index.js";
|
|
39
39
|
import type { BindRequest } from "../protocol/types.js";
|
|
40
40
|
import type { FunctionMeta, FunctionExample } from "./types.js";
|
|
41
|
+
import { narrowArgValue } from "../arguments/argument-spec.js";
|
|
41
42
|
import { safeNumber } from "../util/arrow/index.js";
|
|
42
43
|
import {
|
|
43
44
|
defineTableBufferingFunction,
|
|
@@ -200,7 +201,7 @@ export function defineCopyToFunction<TArgs = Record<string, unknown>>(
|
|
|
200
201
|
}
|
|
201
202
|
val = opt.default;
|
|
202
203
|
}
|
|
203
|
-
|
|
204
|
+
val = narrowArgValue(val);
|
|
204
205
|
if (opt.choices && !opt.choices.includes(val)) {
|
|
205
206
|
throw new Error(
|
|
206
207
|
`COPY (FORMAT ${config.format}): option '${name}' value ${JSON.stringify(val)} ` +
|
|
@@ -11,18 +11,19 @@ import type { VgiFunction } from "./types.js";
|
|
|
11
11
|
import type { Arguments } from "../arguments/arguments.js";
|
|
12
12
|
import type { ArgumentSpec } from "../arguments/argument-spec.js";
|
|
13
13
|
import { FunctionNotFoundError } from "../errors.js";
|
|
14
|
+
import { schemaPathDisplay, schemaPathKey } from "../schema-path.js";
|
|
14
15
|
|
|
15
16
|
export interface OverloadContext {
|
|
16
17
|
arguments?: Arguments;
|
|
17
18
|
inputSchema?: VgiSchema | null;
|
|
18
19
|
isScalar?: boolean;
|
|
19
20
|
/**
|
|
20
|
-
* Catalog schema that declares the function, from `BindRequest.
|
|
21
|
+
* Catalog schema that declares the function, from `BindRequest.schema_path`.
|
|
21
22
|
* A worker may register the same name in more than one schema, so the bare
|
|
22
23
|
* name is not a unique key. When set, resolution is scoped to that schema and
|
|
23
24
|
* never falls back to another one; when absent, every schema is searched.
|
|
24
25
|
*/
|
|
25
|
-
|
|
26
|
+
schemaPath?: string[] | null;
|
|
26
27
|
/**
|
|
27
28
|
* Catalog that owns the function, resolved from the bind's
|
|
28
29
|
* `attach_opaque_data`. Two catalogs served by one worker may each declare
|
|
@@ -205,13 +206,13 @@ function filterByArgumentTypes(
|
|
|
205
206
|
}
|
|
206
207
|
|
|
207
208
|
/** Key for the schema-scoped index: lowercased schema, then function name. */
|
|
208
|
-
function schemaKey(
|
|
209
|
-
return `${
|
|
209
|
+
function schemaKey(schemaPath: string[], functionName: string): string {
|
|
210
|
+
return `${schemaPathKey(schemaPath)}\u0001${functionName}`;
|
|
210
211
|
}
|
|
211
212
|
|
|
212
213
|
/** Key for the catalog-scoped index: lowercased catalog, then the schema key. */
|
|
213
|
-
function catalogKey(catalogName: string,
|
|
214
|
-
return `${catalogName.toLowerCase()}\u0000${schemaKey(
|
|
214
|
+
function catalogKey(catalogName: string, schemaPath: string[], functionName: string): string {
|
|
215
|
+
return `${catalogName.toLowerCase()}\u0000${schemaKey(schemaPath, functionName)}`;
|
|
215
216
|
}
|
|
216
217
|
|
|
217
218
|
export class FunctionRegistry {
|
|
@@ -234,12 +235,12 @@ export class FunctionRegistry {
|
|
|
234
235
|
}
|
|
235
236
|
|
|
236
237
|
/**
|
|
237
|
-
* Record that `func` is declared in `
|
|
238
|
+
* Record that `func` is declared in `schemaPath`, in addition to the flat
|
|
238
239
|
* by-name index. Idempotent, and independent of `register()` so a function
|
|
239
240
|
* reachable from several schemas (e.g. a scan function referenced by tables in
|
|
240
241
|
* more than one schema) resolves from any of them.
|
|
241
242
|
*/
|
|
242
|
-
registerInSchema(func: VgiFunction,
|
|
243
|
+
registerInSchema(func: VgiFunction, schemaPath: string[], catalogName?: string): void {
|
|
243
244
|
const add = (map: Map<string, VgiFunction[]>, key: string) => {
|
|
244
245
|
const bucket = map.get(key);
|
|
245
246
|
if (!bucket) {
|
|
@@ -248,20 +249,21 @@ export class FunctionRegistry {
|
|
|
248
249
|
}
|
|
249
250
|
if (!bucket.includes(func)) bucket.push(func);
|
|
250
251
|
};
|
|
251
|
-
add(this._bySchema, schemaKey(
|
|
252
|
+
add(this._bySchema, schemaKey(schemaPath, func.meta.name));
|
|
252
253
|
if (catalogName) {
|
|
253
|
-
add(this._byCatalog, catalogKey(catalogName,
|
|
254
|
+
add(this._byCatalog, catalogKey(catalogName, schemaPath, func.meta.name));
|
|
254
255
|
}
|
|
255
256
|
}
|
|
256
257
|
|
|
257
258
|
/** Schemas that declare `functionName`, sorted — for error messages. */
|
|
258
|
-
schemasFor(functionName: string): string[] {
|
|
259
|
-
const out: string[] = [];
|
|
259
|
+
schemasFor(functionName: string): string[][] {
|
|
260
|
+
const out: string[][] = [];
|
|
260
261
|
for (const key of this._bySchema.keys()) {
|
|
261
|
-
const
|
|
262
|
-
|
|
262
|
+
const separator = key.lastIndexOf("\u0001");
|
|
263
|
+
const name = key.slice(separator + 1);
|
|
264
|
+
if (name === functionName) out.push(key.slice(0, separator).split("\u0000"));
|
|
263
265
|
}
|
|
264
|
-
return out.sort();
|
|
266
|
+
return out.sort((left, right) => schemaPathKey(left).localeCompare(schemaPathKey(right)));
|
|
265
267
|
}
|
|
266
268
|
|
|
267
269
|
get(name: string, context?: OverloadContext): VgiFunction {
|
|
@@ -273,23 +275,23 @@ export class FunctionRegistry {
|
|
|
273
275
|
// Catalog first: it is the only key that separates two catalogs declaring
|
|
274
276
|
// the same schema and name.
|
|
275
277
|
const catalogName = context?.catalogName;
|
|
276
|
-
const
|
|
277
|
-
if (catalogName &&
|
|
278
|
-
const scoped = this._byCatalog.get(catalogKey(catalogName,
|
|
278
|
+
const schemaPath = context?.schemaPath;
|
|
279
|
+
if (catalogName && schemaPath) {
|
|
280
|
+
const scoped = this._byCatalog.get(catalogKey(catalogName, schemaPath, name));
|
|
279
281
|
if (scoped && scoped.length > 0) {
|
|
280
282
|
return this._disambiguate(name, scoped, context);
|
|
281
283
|
}
|
|
282
284
|
}
|
|
283
|
-
if (
|
|
284
|
-
const scoped = this._bySchema.get(schemaKey(
|
|
285
|
+
if (schemaPath) {
|
|
286
|
+
const scoped = this._bySchema.get(schemaKey(schemaPath, name));
|
|
285
287
|
if (scoped && scoped.length > 0) {
|
|
286
288
|
candidates = scoped;
|
|
287
289
|
} else if (candidates && candidates.length > 0) {
|
|
288
290
|
const schemas = this.schemasFor(name);
|
|
289
291
|
if (schemas.length > 0) {
|
|
290
292
|
throw new Error(
|
|
291
|
-
`Function '${name}' is not registered in schema
|
|
292
|
-
`It is available in: [${schemas.join(", ")}]`,
|
|
293
|
+
`Function '${name}' is not registered in schema ${schemaPathDisplay(schemaPath)}. ` +
|
|
294
|
+
`It is available in: [${schemas.map(schemaPathDisplay).join(", ")}]`,
|
|
293
295
|
);
|
|
294
296
|
}
|
|
295
297
|
}
|
package/src/functions/scalar.ts
CHANGED
|
@@ -21,6 +21,7 @@ import {
|
|
|
21
21
|
FunctionType,
|
|
22
22
|
FunctionStability,
|
|
23
23
|
NullHandling,
|
|
24
|
+
ArgumentMonotonicity,
|
|
24
25
|
DEFAULT_MAX_WORKERS,
|
|
25
26
|
} from "../types.js";
|
|
26
27
|
import { RowCountMismatchError } from "../errors.js";
|
|
@@ -41,6 +42,8 @@ import {
|
|
|
41
42
|
argumentSpecsToSchema,
|
|
42
43
|
constraintSpecFields,
|
|
43
44
|
validateConstConstraints,
|
|
45
|
+
assertArrowType,
|
|
46
|
+
assertArrowTypes,
|
|
44
47
|
type ArgumentConstraints,
|
|
45
48
|
} from "../arguments/argument-spec.js";
|
|
46
49
|
import { batchToScalarDict, batchToSecretDict, batchFromColumns, projectBatch, safeNumber } from "../util/arrow/index.js";
|
|
@@ -61,6 +64,8 @@ export interface ScalarBindParameters {
|
|
|
61
64
|
secrets: Record<string, Record<string, any>>;
|
|
62
65
|
/** Original bind request — exposes attach_opaque_data, transaction_opaque_data, function_type, etc. */
|
|
63
66
|
bindCall: BindRequest;
|
|
67
|
+
/** Resolved names aligned with the complete logical argument order. */
|
|
68
|
+
argumentNames: (string | null)[] | null;
|
|
64
69
|
}
|
|
65
70
|
|
|
66
71
|
// ============================================================================
|
|
@@ -176,9 +181,12 @@ export interface ScalarFunctionConfig<
|
|
|
176
181
|
// Metadata
|
|
177
182
|
stability?: FunctionStability;
|
|
178
183
|
nullHandling?: NullHandling;
|
|
184
|
+
/** One scalar monotonicity claim per ordered parameter declaration. */
|
|
185
|
+
argumentMonotonicity?: ArgumentMonotonicity[] | null;
|
|
179
186
|
examples?: FunctionExample[];
|
|
180
187
|
categories?: string[];
|
|
181
188
|
tags?: Record<string, string>;
|
|
189
|
+
parameterDefaultValues?: VgiBatch | null;
|
|
182
190
|
maxWorkers?: number;
|
|
183
191
|
requiredSettings?: string[];
|
|
184
192
|
requiredSecrets?: string[];
|
|
@@ -202,6 +210,15 @@ export function defineScalarFunction<
|
|
|
202
210
|
R extends VgiDataType = VgiDataType,
|
|
203
211
|
M extends Repr = "rich",
|
|
204
212
|
>(config: ScalarFunctionConfig<P, R, M>): VgiFunction {
|
|
213
|
+
assertArrowTypes(config.params, `defineScalarFunction("${config.name}"): params`);
|
|
214
|
+
assertArrowTypes(config.constParams, `defineScalarFunction("${config.name}"): constParams`);
|
|
215
|
+
if (config.returns !== undefined) {
|
|
216
|
+
assertArrowType(config.returns, `defineScalarFunction("${config.name}"): returns`);
|
|
217
|
+
}
|
|
218
|
+
for (const p of config.parameters ?? []) {
|
|
219
|
+
assertArrowType(p.type, `defineScalarFunction("${config.name}"): parameters["${p.name}"].type`);
|
|
220
|
+
}
|
|
221
|
+
|
|
205
222
|
const repr: Repr = config.repr ?? "rich";
|
|
206
223
|
// Build argument specs
|
|
207
224
|
const specs: ArgumentSpec[] = [];
|
|
@@ -251,6 +268,23 @@ export function defineScalarFunction<
|
|
|
251
268
|
}
|
|
252
269
|
}
|
|
253
270
|
|
|
271
|
+
if (config.argumentMonotonicity !== undefined && config.argumentMonotonicity !== null) {
|
|
272
|
+
if (config.argumentMonotonicity.length !== specs.length) {
|
|
273
|
+
throw new Error(
|
|
274
|
+
`defineScalarFunction("${config.name}"): argumentMonotonicity has ` +
|
|
275
|
+
`${config.argumentMonotonicity.length} entries, expected ${specs.length} declaration slots`,
|
|
276
|
+
);
|
|
277
|
+
}
|
|
278
|
+
const allowed = new Set<string>(Object.values(ArgumentMonotonicity));
|
|
279
|
+
for (const [index, value] of config.argumentMonotonicity.entries()) {
|
|
280
|
+
if (!allowed.has(value)) {
|
|
281
|
+
throw new Error(
|
|
282
|
+
`defineScalarFunction("${config.name}"): argumentMonotonicity[${index}] has unknown value ${String(value)}`,
|
|
283
|
+
);
|
|
284
|
+
}
|
|
285
|
+
}
|
|
286
|
+
}
|
|
287
|
+
|
|
254
288
|
// Raw const-arg constraints for bind-time enforcement, keyed by arg name.
|
|
255
289
|
// Only the ordered `parameters` API carries constraints (the legacy
|
|
256
290
|
// params/constParams path does not), so only its const params appear here.
|
|
@@ -276,9 +310,11 @@ export function defineScalarFunction<
|
|
|
276
310
|
description: config.description,
|
|
277
311
|
stability: config.stability,
|
|
278
312
|
nullHandling: config.nullHandling,
|
|
313
|
+
argumentMonotonicity: config.argumentMonotonicity,
|
|
279
314
|
examples: config.examples,
|
|
280
315
|
categories: config.categories,
|
|
281
316
|
tags: config.tags,
|
|
317
|
+
parameterDefaultValues: config.parameterDefaultValues,
|
|
282
318
|
maxWorkers: config.maxWorkers,
|
|
283
319
|
requiredSettings: config.requiredSettings,
|
|
284
320
|
requiredSecrets: config.requiredSecrets,
|
|
@@ -328,6 +364,7 @@ export function defineScalarFunction<
|
|
|
328
364
|
settings,
|
|
329
365
|
secrets,
|
|
330
366
|
bindCall: request,
|
|
367
|
+
argumentNames: request.argument_names ?? null,
|
|
331
368
|
});
|
|
332
369
|
} else if (config.returns) {
|
|
333
370
|
outputType = config.returns;
|
|
@@ -39,12 +39,17 @@ import type {
|
|
|
39
39
|
StreamHandlers,
|
|
40
40
|
FunctionExample,
|
|
41
41
|
} from "./types.js";
|
|
42
|
-
import type { ArgumentSpec } from "../arguments/argument-spec.js";
|
|
42
|
+
import type { ArgumentConstraints, ArgumentSpec } from "../arguments/argument-spec.js";
|
|
43
|
+
import {
|
|
44
|
+
assertArrowType,
|
|
45
|
+
assertArrowTypes,
|
|
46
|
+
constraintSpecFields,
|
|
47
|
+
validateConstConstraints,
|
|
48
|
+
} from "../arguments/argument-spec.js";
|
|
43
49
|
import { batchToScalarDict, batchToSecretDict, projectSchema } from "../util/arrow/index.js";
|
|
44
50
|
import { batchFromColumns, readCanonicalValue } from "../arrow/index.js";
|
|
45
51
|
import { codecFor } from "../arrow/codec/registry.js";
|
|
46
52
|
import {
|
|
47
|
-
buildJoinKeysLookup,
|
|
48
53
|
deserializeFilters,
|
|
49
54
|
FilteringOutputCollector,
|
|
50
55
|
type PushdownFilters,
|
|
@@ -70,6 +75,8 @@ export interface TableBufferingBindParams<TArgs = Record<string, any>> {
|
|
|
70
75
|
/** True on the second bind pass, after the connector resolved the secrets
|
|
71
76
|
* requested via the onBind `lookupSecret*` return fields on the first pass. */
|
|
72
77
|
resolvedSecretsProvided: boolean;
|
|
78
|
+
/** Resolved names aligned with the complete logical argument order. */
|
|
79
|
+
argumentNames?: (string | null)[] | null;
|
|
73
80
|
}
|
|
74
81
|
|
|
75
82
|
export interface TableBufferingParams<TArgs = Record<string, any>> {
|
|
@@ -106,6 +113,12 @@ export interface TableBufferingConfig<
|
|
|
106
113
|
args?: Record<string, VgiDataType>;
|
|
107
114
|
namedArgs?: Record<string, VgiDataType>;
|
|
108
115
|
argDefaults?: Record<string, any>;
|
|
116
|
+
/** Authoritative typed defaults: one row, defaulted parameters only. */
|
|
117
|
+
parameterDefaultValues?: VgiBatch | null;
|
|
118
|
+
/** Argument descriptions keyed by argument name. */
|
|
119
|
+
argDocs?: Record<string, string>;
|
|
120
|
+
/** Discovery constraints, also enforced for scalar arguments at bind. */
|
|
121
|
+
argConstraints?: Record<string, ArgumentConstraints>;
|
|
109
122
|
/** Bind: default passes through input schema. May be async. Return non-empty
|
|
110
123
|
* `lookupSecret*` lists on the first pass to request DuckDB secrets; the
|
|
111
124
|
* connector then re-binds with `resolvedSecretsProvided` and the resolved
|
|
@@ -182,6 +195,8 @@ export function defineTableBufferingFunction<
|
|
|
182
195
|
TArgs = Record<string, any>,
|
|
183
196
|
TState = any,
|
|
184
197
|
>(config: TableBufferingConfig<TArgs, TState>): TableBufferingVgiFunction {
|
|
198
|
+
assertArrowTypes((config as any).args, `defineTableBufferingFunction("${config.name}"): args`);
|
|
199
|
+
|
|
185
200
|
if (config.sinkOrderDependent && config.requiresInputBatchIndex) {
|
|
186
201
|
throw new Error(
|
|
187
202
|
`${config.name}: sinkOrderDependent and requiresInputBatchIndex are ` +
|
|
@@ -193,7 +208,16 @@ export function defineTableBufferingFunction<
|
|
|
193
208
|
let posIdx = 0;
|
|
194
209
|
if (config.args) {
|
|
195
210
|
for (const [name, type] of Object.entries(config.args)) {
|
|
196
|
-
specs.push({
|
|
211
|
+
specs.push({
|
|
212
|
+
name,
|
|
213
|
+
position: posIdx++,
|
|
214
|
+
arrowType: type,
|
|
215
|
+
doc: config.argDocs?.[name],
|
|
216
|
+
...constraintSpecFields(
|
|
217
|
+
config.argConstraints?.[name],
|
|
218
|
+
config.argDefaults?.[name],
|
|
219
|
+
),
|
|
220
|
+
});
|
|
197
221
|
}
|
|
198
222
|
}
|
|
199
223
|
specs.push({
|
|
@@ -204,7 +228,16 @@ export function defineTableBufferingFunction<
|
|
|
204
228
|
});
|
|
205
229
|
if (config.namedArgs) {
|
|
206
230
|
for (const [name, type] of Object.entries(config.namedArgs)) {
|
|
207
|
-
specs.push({
|
|
231
|
+
specs.push({
|
|
232
|
+
name,
|
|
233
|
+
position: name,
|
|
234
|
+
arrowType: type,
|
|
235
|
+
doc: config.argDocs?.[name],
|
|
236
|
+
...constraintSpecFields(
|
|
237
|
+
config.argConstraints?.[name],
|
|
238
|
+
config.argDefaults?.[name],
|
|
239
|
+
),
|
|
240
|
+
});
|
|
208
241
|
}
|
|
209
242
|
}
|
|
210
243
|
|
|
@@ -218,6 +251,7 @@ export function defineTableBufferingFunction<
|
|
|
218
251
|
examples: config.examples,
|
|
219
252
|
categories: config.categories,
|
|
220
253
|
tags: config.tags,
|
|
254
|
+
parameterDefaultValues: config.parameterDefaultValues,
|
|
221
255
|
maxWorkers: config.maxWorkers,
|
|
222
256
|
requiredSettings: config.requiredSettings,
|
|
223
257
|
requiredSecrets: config.requiredSecrets,
|
|
@@ -239,6 +273,8 @@ export function defineTableBufferingFunction<
|
|
|
239
273
|
: undefined;
|
|
240
274
|
let val = request.arguments.get(spec.position, defaultVal);
|
|
241
275
|
if (typeof val === "bigint") val = Number(val);
|
|
276
|
+
const constraints = config.argConstraints?.[spec.name];
|
|
277
|
+
if (constraints) validateConstConstraints(spec.name, constraints, val);
|
|
242
278
|
args[spec.name] = val;
|
|
243
279
|
}
|
|
244
280
|
return args as TArgs;
|
|
@@ -262,6 +298,7 @@ export function defineTableBufferingFunction<
|
|
|
262
298
|
settings,
|
|
263
299
|
secrets,
|
|
264
300
|
resolvedSecretsProvided: request.resolved_secrets_provided ?? false,
|
|
301
|
+
argumentNames: request.argument_names ?? null,
|
|
265
302
|
});
|
|
266
303
|
return {
|
|
267
304
|
output_schema: result.outputSchema,
|
|
@@ -323,9 +360,14 @@ export function defineTableBufferingFunction<
|
|
|
323
360
|
? projectSchema(projIds, request.output_schema)
|
|
324
361
|
: request.output_schema;
|
|
325
362
|
|
|
326
|
-
const joinKeysLookup = buildJoinKeysLookup(request.join_keys);
|
|
327
363
|
const pushdownFilters = request.pushdown_filters
|
|
328
|
-
? deserializeFilters(request.pushdown_filters,
|
|
364
|
+
? deserializeFilters(request.pushdown_filters, {
|
|
365
|
+
outputSchema: request.output_schema,
|
|
366
|
+
joinKeyBatches: request.join_keys,
|
|
367
|
+
extensionFunctions: meta.additionalFilterFunctions,
|
|
368
|
+
runtimeAlgorithms: meta.runtimeFilterAlgorithms,
|
|
369
|
+
evaluationContexts: meta.filterEvaluationContexts,
|
|
370
|
+
})
|
|
329
371
|
: undefined;
|
|
330
372
|
|
|
331
373
|
const bound = new BoundStorage(
|