@query-farm/vgi 0.28.0 → 0.29.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +52 -8
- package/dist/arguments/argument-spec.d.ts +57 -2
- package/dist/arrow/impl-arrowjs/index.d.ts +1 -1
- package/dist/arrow/impl-arrowjs/iterate.d.ts +10 -1
- package/dist/arrow/impl-arrowjs/schema.d.ts +2 -0
- package/dist/arrow/impl-flechette/index.d.ts +1 -1
- package/dist/arrow/impl-flechette/iterate.d.ts +10 -1
- package/dist/arrow/impl-flechette/schema.d.ts +2 -0
- package/dist/arrow/index.d.ts +1 -1
- package/dist/arrow/schema-types.d.ts +3 -1
- package/dist/arrow/types.d.ts +39 -3
- package/dist/catalog/composite.d.ts +13 -13
- package/dist/catalog/descriptors.d.ts +7 -3
- package/dist/catalog/interface.d.ts +108 -38
- package/dist/catalog/read-only.d.ts +32 -15
- package/dist/client/client.d.ts +37 -30
- package/dist/client/types.d.ts +2 -0
- package/dist/client-entry.d.ts +1 -1
- package/dist/client-entry.js +791 -447
- package/dist/client-entry.js.map +18 -17
- package/dist/filter-pushdown/capabilities.d.ts +3 -0
- package/dist/filter-pushdown/deserialize.d.ts +14 -15
- package/dist/filter-pushdown/evaluate.d.ts +14 -44
- package/dist/filter-pushdown/index.d.ts +2 -2
- package/dist/filter-pushdown/types.d.ts +120 -68
- package/dist/functions/aggregate.d.ts +4 -0
- package/dist/functions/registry.d.ts +5 -5
- package/dist/functions/scalar.d.ts +6 -1
- package/dist/functions/table-buffering.d.ts +9 -0
- package/dist/functions/table-in-out.d.ts +13 -0
- package/dist/functions/table.d.ts +72 -2
- package/dist/functions/types.d.ts +60 -2
- package/dist/generated/vgi-client.d.ts +114 -26
- package/dist/generated/vgi-protocol-schemas.d.ts +34 -0
- package/dist/http/fetch.d.ts +16 -0
- package/dist/index.core.d.ts +4 -3
- package/dist/index.js +3396 -2064
- package/dist/index.js.map +55 -49
- package/dist/metadata/types.d.ts +15 -2
- package/dist/protocol/handlers/catalog/shared.d.ts +0 -8
- package/dist/protocol/handlers/function.d.ts +11 -0
- package/dist/protocol/handlers/shared.d.ts +3 -1
- package/dist/protocol/serializers/bind.d.ts +1 -2
- package/dist/protocol/serializers/cardinality.d.ts +1 -2
- package/dist/protocol/serializers/splits.d.ts +126 -0
- package/dist/protocol/types.d.ts +36 -3
- package/dist/request-auth.d.ts +28 -0
- package/dist/schema-path.d.ts +7 -0
- package/dist/serve-entry.d.ts +25 -0
- package/dist/serve-entry.js +1129 -569
- package/dist/serve-entry.js.map +36 -32
- package/dist/split-token.d.ts +88 -0
- package/dist/types.d.ts +8 -0
- package/dist/worker-cf-entry.js +2928 -1689
- package/dist/worker-cf-entry.js.map +51 -44
- package/dist/worker.d.ts +1 -1
- package/package.json +3 -3
- package/src/arguments/argument-spec.ts +93 -4
- package/src/arguments/arguments.ts +0 -1
- package/src/arrow/impl-arrowjs/index.ts +1 -1
- package/src/arrow/impl-arrowjs/iterate.ts +21 -0
- package/src/arrow/impl-arrowjs/schema.ts +4 -0
- package/src/arrow/impl-flechette/index.ts +1 -1
- package/src/arrow/impl-flechette/iterate.ts +24 -1
- package/src/arrow/impl-flechette/schema.ts +4 -0
- package/src/arrow/index.ts +1 -1
- package/src/arrow/schema-types.ts +4 -0
- package/src/arrow/types.ts +39 -3
- package/src/catalog/composite.ts +26 -26
- package/src/catalog/descriptors.ts +12 -3
- package/src/catalog/interface.ts +217 -72
- package/src/catalog/read-only.ts +141 -84
- package/src/client/client.ts +97 -73
- package/src/client/types.ts +2 -0
- package/src/client-entry.ts +2 -1
- package/src/filter-pushdown/capabilities.ts +13 -0
- package/src/filter-pushdown/collector.ts +22 -35
- package/src/filter-pushdown/deserialize.ts +657 -221
- package/src/filter-pushdown/evaluate.ts +410 -499
- package/src/filter-pushdown/index.ts +16 -2
- package/src/filter-pushdown/types.ts +88 -71
- package/src/functions/aggregate.ts +12 -1
- package/src/functions/copy-from.ts +2 -1
- package/src/functions/copy-to.ts +2 -1
- package/src/functions/registry.ts +24 -22
- package/src/functions/scalar.ts +37 -0
- package/src/functions/table-buffering.ts +48 -6
- package/src/functions/table-in-out.ts +194 -33
- package/src/functions/table.ts +187 -22
- package/src/functions/types.ts +64 -1
- package/src/generated/vgi-client.ts +122 -28
- package/src/generated/vgi-protocol-schemas.ts +455 -58
- package/src/http/fetch.ts +129 -2
- package/src/index.core.ts +4 -2
- package/src/metadata/resolve.ts +43 -1
- package/src/metadata/serialize.ts +43 -5
- package/src/metadata/types.ts +17 -1
- package/src/protocol/dispatch.ts +1 -1
- package/src/protocol/handlers/aggregate.ts +23 -12
- package/src/protocol/handlers/catalog/admin.ts +47 -54
- package/src/protocol/handlers/catalog/index_methods.ts +6 -13
- package/src/protocol/handlers/catalog/macro.ts +12 -20
- package/src/protocol/handlers/catalog/shared.ts +12 -57
- package/src/protocol/handlers/catalog/table.ts +49 -112
- package/src/protocol/handlers/catalog/view.ts +18 -22
- package/src/protocol/handlers/function.ts +311 -18
- package/src/protocol/handlers/shared.ts +12 -3
- package/src/protocol/handlers/table-buffering.ts +11 -8
- package/src/protocol/serializers/bind.ts +69 -67
- package/src/protocol/serializers/cardinality.ts +7 -7
- package/src/protocol/serializers/init.ts +52 -10
- package/src/protocol/serializers/splits.ts +221 -0
- package/src/protocol/state-serializer.ts +55 -1
- package/src/protocol/types.ts +36 -3
- package/src/request-auth.ts +73 -0
- package/src/schema-path.ts +25 -0
- package/src/serve-entry.ts +69 -2
- package/src/split-token.ts +297 -0
- package/src/types.ts +9 -0
- package/src/worker-cf-entry.ts +2 -1
- package/src/worker.ts +79 -14
|
@@ -1,16 +1,15 @@
|
|
|
1
|
-
import { type VgiBatch } from "../arrow/index.js";
|
|
1
|
+
import { type VgiBatch, type VgiSchema } from "../arrow/index.js";
|
|
2
2
|
import { PushdownFilters } from "./evaluate.js";
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
export declare function deserializeFilters(batch: VgiBatch, getJoinKeysColumn?: (columnName: string) => any[] | null): PushdownFilters;
|
|
3
|
+
import { type FilterCapabilities } from "./types.js";
|
|
4
|
+
export declare class FilterV2Error extends Error {
|
|
5
|
+
constructor(message: string);
|
|
6
|
+
}
|
|
7
|
+
export interface DeserializeFilterOptions extends FilterCapabilities {
|
|
8
|
+
/** The authoritative, unprojected bind output schema. */
|
|
9
|
+
outputSchema: VgiSchema;
|
|
10
|
+
joinKeyBatches?: VgiBatch[];
|
|
11
|
+
}
|
|
12
|
+
export declare function deserializeFilters(batch: VgiBatch, options: DeserializeFilterOptions): PushdownFilters;
|
|
13
|
+
export declare function applyFilterDelta(prior: PushdownFilters, batch: VgiBatch): PushdownFilters;
|
|
14
|
+
/** @deprecated External-set references are positional in v2; pass joinKeyBatches instead. */
|
|
15
|
+
export declare function buildJoinKeysLookup(joinKeyBatches: VgiBatch[]): (columnName: string) => unknown[] | null;
|
|
@@ -1,51 +1,21 @@
|
|
|
1
1
|
import { type VgiBatch } from "../arrow/index.js";
|
|
2
|
-
import { type
|
|
2
|
+
import { type DeserializeFilterOptions } from "./deserialize.js";
|
|
3
|
+
import { type EvaluationContext, type FilterExpression, type FilterPredicate } from "./types.js";
|
|
3
4
|
export declare class PushdownFilters {
|
|
4
|
-
readonly
|
|
5
|
-
readonly
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
5
|
+
readonly predicates: FilterPredicate[];
|
|
6
|
+
readonly evaluationContext: EvaluationContext;
|
|
7
|
+
readonly options: DeserializeFilterOptions;
|
|
8
|
+
readonly revisions: Map<string, number>;
|
|
9
|
+
readonly requiredIds: Set<string>;
|
|
10
|
+
readonly version = "2";
|
|
11
|
+
constructor(predicates: FilterPredicate[], evaluationContext: EvaluationContext, options: DeserializeFilterOptions, revisions?: Map<string, number>, requiredIds?: Set<string>);
|
|
12
|
+
get filters(): FilterExpression[];
|
|
13
|
+
applyDelta(batch: VgiBatch): PushdownFilters;
|
|
10
14
|
evaluate(batch: VgiBatch): Uint8Array;
|
|
11
|
-
/** Apply filters to a batch, returning only passing rows. */
|
|
12
15
|
apply(batch: VgiBatch): VgiBatch;
|
|
13
|
-
/**
|
|
14
|
-
* The set of column names referenced by the pushed-down filters, sorted
|
|
15
|
-
* ascending. Descends one level into AndFilter children (DuckDB pushes
|
|
16
|
-
* `col IN (...) AND col>=min` as a single AndFilter), consistent with
|
|
17
|
-
* `getColumnValues` / `hasFilterForColumn`. Mirrors the cross-language
|
|
18
|
-
* `filtered_columns` accessor (vgi-python / vgi-go / vgi-rust).
|
|
19
|
-
*/
|
|
20
16
|
filteredColumns(): string[];
|
|
21
|
-
/**
|
|
22
|
-
* Whether any pushed-down filter references `columnName`. Mirrors the
|
|
23
|
-
* cross-language `has_filter_for_column` accessor.
|
|
24
|
-
*/
|
|
25
17
|
hasFilterForColumn(columnName: string): boolean;
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
* Useful for partition pruning (resolve the key set up front, fetch only
|
|
29
|
-
* those keys). Returns null when the predicate is not enumerable — no
|
|
30
|
-
* filter, a bare range, or an OR with a non-discrete branch.
|
|
31
|
-
*
|
|
32
|
-
* Mirrors vgi-python's `PushdownFilters.get_column_values`: descends one
|
|
33
|
-
* level into an AndFilter (DuckDB pushes `col IN (...) AND col>=min AND
|
|
34
|
-
* col<=max` as a single AndFilter from a semi-join), and unions OR branches
|
|
35
|
-
* only when every branch pins the column to discrete values.
|
|
36
|
-
*/
|
|
37
|
-
getColumnValues(columnName: string): any[] | null;
|
|
38
|
-
/**
|
|
39
|
-
* Union of discrete values for `columnName` across all OR branches, iff every
|
|
40
|
-
* child constrains it to discrete values (`=` or `IN`); otherwise null (the
|
|
41
|
-
* column could take any value through a non-discrete branch). Returning one
|
|
42
|
-
* branch's values would be an unsafe subset.
|
|
43
|
-
*/
|
|
44
|
-
private orDiscreteValues;
|
|
45
|
-
/**
|
|
46
|
-
* Collect filters for a column from the top level and direct AND children
|
|
47
|
-
* (one level of descent, consistent with the Python accessor). Deeply nested
|
|
48
|
-
* AND/AND is not traversed.
|
|
49
|
-
*/
|
|
50
|
-
private collectColumnFilters;
|
|
18
|
+
getColumnValues(columnName: string): unknown[] | null;
|
|
19
|
+
toSql(): string;
|
|
51
20
|
}
|
|
21
|
+
export declare function expressionToSql(expression: FilterExpression): string;
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
export { ComparisonOp, type AndFilter, type ConstantFilter, type ExpressionFilter, type ExprNode, type
|
|
1
|
+
export { FILTER_ENCODING, FILTER_VERSION, DUCKDB_STANDARD_V1, NO_EVALUATION_CONTEXT, DUCKDB_SESSION_CONTEXT, ComparisonOp, type ArithmeticOp, type EvaluationContext, type Filter, type AndFilter, type ConstantFilter, type ExpressionFilter, type ExprNode, type InFilter, type IsNotNullFilter, type IsNullFilter, type OrFilter, type StructFilter, type FilterExpression, type FilterPredicate, type FilterSet, type FunctionIdentity, type PredicateMode, type PredicateSource, type StandardFilterFunction, } from "./types.js";
|
|
2
2
|
export { PushdownFilters } from "./evaluate.js";
|
|
3
|
-
export { buildJoinKeysLookup, deserializeFilters } from "./deserialize.js";
|
|
3
|
+
export { FilterV2Error, applyFilterDelta, buildJoinKeysLookup, deserializeFilters } from "./deserialize.js";
|
|
4
4
|
export { FilteringOutputCollector, formatPushedFilters, reprPushedFilters, } from "./collector.js";
|
|
@@ -1,84 +1,136 @@
|
|
|
1
|
+
import type { VgiBatch, VgiDataType, VgiField } from "../arrow/index.js";
|
|
2
|
+
export declare const FILTER_ENCODING = "vgi.filters.v2";
|
|
3
|
+
export declare const FILTER_VERSION = "2";
|
|
4
|
+
export declare const DUCKDB_STANDARD_V1 = "vgi.duckdb.standard.v1";
|
|
5
|
+
export declare const NO_EVALUATION_CONTEXT = "vgi.none.v1";
|
|
6
|
+
export declare const DUCKDB_SESSION_CONTEXT = "vgi.duckdb.session.v1";
|
|
1
7
|
export declare enum ComparisonOp {
|
|
2
8
|
EQ = "eq",
|
|
3
9
|
NE = "ne",
|
|
10
|
+
LT = "lt",
|
|
11
|
+
LE = "le",
|
|
4
12
|
GT = "gt",
|
|
5
13
|
GE = "ge",
|
|
6
|
-
|
|
7
|
-
|
|
14
|
+
DISTINCT_FROM = "distinct_from",
|
|
15
|
+
NOT_DISTINCT_FROM = "not_distinct_from"
|
|
8
16
|
}
|
|
9
|
-
export
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
17
|
+
export type ArithmeticOp = "add" | "subtract" | "multiply" | "divide" | "modulo";
|
|
18
|
+
export type PredicateMode = "required" | "advisory";
|
|
19
|
+
export type PredicateSource = "query" | "join" | "top_n" | "split_refinement" | "other";
|
|
20
|
+
export type StandardFilterFunction = "starts_with" | "ends_with" | "contains" | "list_contains";
|
|
21
|
+
export interface FunctionIdentity {
|
|
22
|
+
namespace: string;
|
|
23
|
+
name: string;
|
|
24
|
+
version: number;
|
|
15
25
|
}
|
|
16
|
-
export interface
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
26
|
+
export interface EvaluationContext {
|
|
27
|
+
profile: string;
|
|
28
|
+
timeZone?: string;
|
|
29
|
+
calendar?: string;
|
|
30
|
+
defaultCollation?: string;
|
|
31
|
+
ieeeFloatingPointOps?: boolean;
|
|
32
|
+
integerDivision?: boolean;
|
|
33
|
+
providerFingerprint?: string;
|
|
20
34
|
}
|
|
21
|
-
export
|
|
22
|
-
|
|
23
|
-
columnName: string;
|
|
35
|
+
export type FilterExpression = {
|
|
36
|
+
node: "column_ref";
|
|
24
37
|
columnIndex: number;
|
|
25
|
-
}
|
|
26
|
-
export interface InFilter {
|
|
27
|
-
type: "in";
|
|
28
38
|
columnName: string;
|
|
29
|
-
|
|
30
|
-
values: Set<any>;
|
|
31
|
-
}
|
|
32
|
-
export interface AndFilter {
|
|
33
|
-
type: "and";
|
|
34
|
-
columnName: string;
|
|
35
|
-
columnIndex: number;
|
|
36
|
-
children: Filter[];
|
|
37
|
-
}
|
|
38
|
-
export interface OrFilter {
|
|
39
|
-
type: "or";
|
|
40
|
-
columnName: string;
|
|
41
|
-
columnIndex: number;
|
|
42
|
-
children: Filter[];
|
|
43
|
-
}
|
|
44
|
-
export interface StructFilter {
|
|
45
|
-
type: "struct";
|
|
46
|
-
columnName: string;
|
|
47
|
-
columnIndex: number;
|
|
48
|
-
childIndex: number;
|
|
49
|
-
childName: string;
|
|
50
|
-
childFilter: Filter;
|
|
51
|
-
}
|
|
52
|
-
export type Filter = ConstantFilter | IsNullFilter | IsNotNullFilter | InFilter | AndFilter | OrFilter | StructFilter | ExpressionFilter;
|
|
53
|
-
/**
|
|
54
|
-
* Expression-tree filter. The expr is a parsed AST; evaluation matches
|
|
55
|
-
* DuckDB's semantics for the function names the table function advertises in
|
|
56
|
-
* `supportedExpressionFilters` (e.g. list_contains, starts_with, contains,
|
|
57
|
-
* &&/st_intersects_extent for spatial).
|
|
58
|
-
*/
|
|
59
|
-
export interface ExpressionFilter {
|
|
60
|
-
type: "expression";
|
|
61
|
-
columnName: string;
|
|
62
|
-
columnIndex: number;
|
|
63
|
-
expr: ExprNode;
|
|
64
|
-
}
|
|
65
|
-
export type ExprNode = {
|
|
66
|
-
expr_type: "column_ref";
|
|
67
|
-
index: number;
|
|
39
|
+
dataType?: VgiDataType;
|
|
68
40
|
} | {
|
|
69
|
-
|
|
70
|
-
|
|
41
|
+
node: "field_ref";
|
|
42
|
+
expression: FilterExpression;
|
|
43
|
+
fieldIndex: number;
|
|
44
|
+
fieldName: string;
|
|
45
|
+
dataType: VgiDataType;
|
|
71
46
|
} | {
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
47
|
+
node: "literal";
|
|
48
|
+
valueRef: number;
|
|
49
|
+
field: VgiField;
|
|
50
|
+
value: unknown;
|
|
75
51
|
} | {
|
|
76
|
-
|
|
52
|
+
node: "comparison";
|
|
77
53
|
op: ComparisonOp;
|
|
78
|
-
left:
|
|
79
|
-
right:
|
|
54
|
+
left: FilterExpression;
|
|
55
|
+
right: FilterExpression;
|
|
56
|
+
} | {
|
|
57
|
+
node: "and" | "or";
|
|
58
|
+
children: FilterExpression[];
|
|
59
|
+
} | {
|
|
60
|
+
node: "not";
|
|
61
|
+
expression: FilterExpression;
|
|
62
|
+
} | {
|
|
63
|
+
node: "is_null";
|
|
64
|
+
expression: FilterExpression;
|
|
65
|
+
negated: boolean;
|
|
66
|
+
} | {
|
|
67
|
+
node: "in";
|
|
68
|
+
expression: FilterExpression;
|
|
69
|
+
set: FilterSet;
|
|
70
|
+
negated: boolean;
|
|
71
|
+
} | {
|
|
72
|
+
node: "cast";
|
|
73
|
+
expression: FilterExpression;
|
|
74
|
+
typeRef: number;
|
|
75
|
+
field: VgiField;
|
|
80
76
|
} | {
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
77
|
+
node: "arithmetic";
|
|
78
|
+
op: ArithmeticOp;
|
|
79
|
+
left: FilterExpression;
|
|
80
|
+
right: FilterExpression;
|
|
81
|
+
} | {
|
|
82
|
+
node: "negate";
|
|
83
|
+
expression: FilterExpression;
|
|
84
|
+
} | {
|
|
85
|
+
node: "call";
|
|
86
|
+
function: StandardFilterFunction | FunctionIdentity;
|
|
87
|
+
arguments: FilterExpression[];
|
|
88
|
+
options?: Record<string, unknown>;
|
|
89
|
+
} | {
|
|
90
|
+
node: "runtime_filter";
|
|
91
|
+
algorithm: FunctionIdentity;
|
|
92
|
+
input: FilterExpression;
|
|
93
|
+
artifactRef: number;
|
|
94
|
+
field: VgiField;
|
|
95
|
+
artifact: unknown;
|
|
96
|
+
nullHandling: "pass" | "reject";
|
|
97
|
+
supported: boolean;
|
|
84
98
|
};
|
|
99
|
+
export type FilterSet = {
|
|
100
|
+
kind: "literal";
|
|
101
|
+
valueRef: number;
|
|
102
|
+
field: VgiField;
|
|
103
|
+
values: unknown[];
|
|
104
|
+
} | {
|
|
105
|
+
kind: "external";
|
|
106
|
+
batchIndex: number;
|
|
107
|
+
columnIndex: number;
|
|
108
|
+
columnName: string;
|
|
109
|
+
values: unknown[];
|
|
110
|
+
batch: VgiBatch;
|
|
111
|
+
};
|
|
112
|
+
export interface FilterPredicate {
|
|
113
|
+
id: string;
|
|
114
|
+
revision: number;
|
|
115
|
+
mode: PredicateMode;
|
|
116
|
+
source: PredicateSource;
|
|
117
|
+
expression: FilterExpression;
|
|
118
|
+
}
|
|
119
|
+
export interface FilterCapabilities {
|
|
120
|
+
extensionFunctions?: ReadonlyArray<FunctionIdentity>;
|
|
121
|
+
runtimeAlgorithms?: ReadonlyArray<FunctionIdentity>;
|
|
122
|
+
evaluationContexts?: ReadonlyArray<{
|
|
123
|
+
profile: string;
|
|
124
|
+
providerFingerprint?: string | null;
|
|
125
|
+
}>;
|
|
126
|
+
}
|
|
127
|
+
export type Filter = FilterExpression;
|
|
128
|
+
export type ExprNode = FilterExpression;
|
|
129
|
+
export type ConstantFilter = FilterExpression;
|
|
130
|
+
export type IsNullFilter = FilterExpression;
|
|
131
|
+
export type IsNotNullFilter = FilterExpression;
|
|
132
|
+
export type InFilter = FilterExpression;
|
|
133
|
+
export type AndFilter = FilterExpression;
|
|
134
|
+
export type OrFilter = FilterExpression;
|
|
135
|
+
export type StructFilter = FilterExpression;
|
|
136
|
+
export type ExpressionFilter = FilterExpression;
|
|
@@ -10,6 +10,8 @@ export interface AggregateBindParams<TArgs = Record<string, any>> {
|
|
|
10
10
|
inputSchema: VgiSchema | null;
|
|
11
11
|
settings: Record<string, any>;
|
|
12
12
|
secrets: Record<string, Record<string, any>>;
|
|
13
|
+
/** Resolved names aligned with the complete logical argument order. */
|
|
14
|
+
argumentNames: (string | null)[] | null;
|
|
13
15
|
}
|
|
14
16
|
export interface AggregateUpdateParams<TArgs = Record<string, any>, TState = any> {
|
|
15
17
|
states: Map<bigint, TState>;
|
|
@@ -71,6 +73,8 @@ export interface AggregateFunctionConfig<TArgs = Record<string, any>, TState = a
|
|
|
71
73
|
onBind?: (params: AggregateBindParams<TArgs>) => VgiDataType | Promise<VgiDataType>;
|
|
72
74
|
/** Optional per-arg default values (positional only here). */
|
|
73
75
|
argDefaults?: Record<string, any>;
|
|
76
|
+
/** Authoritative typed defaults: one row, defaulted parameters only. */
|
|
77
|
+
parameterDefaultValues?: VgiBatch | null;
|
|
74
78
|
/**
|
|
75
79
|
* Per-argument discovery constraints (choices / ge / le / gt / lt / pattern),
|
|
76
80
|
* keyed by argument name. Surfaced via `vgi_function_arguments()` and enforced
|
|
@@ -6,12 +6,12 @@ export interface OverloadContext {
|
|
|
6
6
|
inputSchema?: VgiSchema | null;
|
|
7
7
|
isScalar?: boolean;
|
|
8
8
|
/**
|
|
9
|
-
* Catalog schema that declares the function, from `BindRequest.
|
|
9
|
+
* Catalog schema that declares the function, from `BindRequest.schema_path`.
|
|
10
10
|
* A worker may register the same name in more than one schema, so the bare
|
|
11
11
|
* name is not a unique key. When set, resolution is scoped to that schema and
|
|
12
12
|
* never falls back to another one; when absent, every schema is searched.
|
|
13
13
|
*/
|
|
14
|
-
|
|
14
|
+
schemaPath?: string[] | null;
|
|
15
15
|
/**
|
|
16
16
|
* Catalog that owns the function, resolved from the bind's
|
|
17
17
|
* `attach_opaque_data`. Two catalogs served by one worker may each declare
|
|
@@ -26,14 +26,14 @@ export declare class FunctionRegistry {
|
|
|
26
26
|
private _byCatalog;
|
|
27
27
|
register(func: VgiFunction): void;
|
|
28
28
|
/**
|
|
29
|
-
* Record that `func` is declared in `
|
|
29
|
+
* Record that `func` is declared in `schemaPath`, in addition to the flat
|
|
30
30
|
* by-name index. Idempotent, and independent of `register()` so a function
|
|
31
31
|
* reachable from several schemas (e.g. a scan function referenced by tables in
|
|
32
32
|
* more than one schema) resolves from any of them.
|
|
33
33
|
*/
|
|
34
|
-
registerInSchema(func: VgiFunction,
|
|
34
|
+
registerInSchema(func: VgiFunction, schemaPath: string[], catalogName?: string): void;
|
|
35
35
|
/** Schemas that declare `functionName`, sorted — for error messages. */
|
|
36
|
-
schemasFor(functionName: string): string[];
|
|
36
|
+
schemasFor(functionName: string): string[][];
|
|
37
37
|
get(name: string, context?: OverloadContext): VgiFunction;
|
|
38
38
|
/**
|
|
39
39
|
* Pick one function from an already-scoped candidate list, by argument shape
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { type VgiSchema, type VgiBatch, type VgiDataType, type ValueFor, type Repr } from "../arrow/index.js";
|
|
2
2
|
import type { AuthContext } from "@query-farm/vgi-rpc";
|
|
3
|
-
import { FunctionStability, NullHandling } from "../types.js";
|
|
3
|
+
import { FunctionStability, NullHandling, ArgumentMonotonicity } from "../types.js";
|
|
4
4
|
import type { BindRequest } from "../protocol/types.js";
|
|
5
5
|
import type { VgiFunction, FunctionExample } from "./types.js";
|
|
6
6
|
import { type CacheControl } from "../cache-control.js";
|
|
@@ -15,6 +15,8 @@ export interface ScalarBindParameters {
|
|
|
15
15
|
secrets: Record<string, Record<string, any>>;
|
|
16
16
|
/** Original bind request — exposes attach_opaque_data, transaction_opaque_data, function_type, etc. */
|
|
17
17
|
bindCall: BindRequest;
|
|
18
|
+
/** Resolved names aligned with the complete logical argument order. */
|
|
19
|
+
argumentNames: (string | null)[] | null;
|
|
18
20
|
}
|
|
19
21
|
/** Ordered parameter definition for scalar functions. */
|
|
20
22
|
export interface ScalarParameterDef {
|
|
@@ -103,9 +105,12 @@ export interface ScalarFunctionConfig<P extends Record<string, VgiDataType> = Re
|
|
|
103
105
|
}) => ScalarComputeResult<NoInfer<R>, NoInfer<M>>;
|
|
104
106
|
stability?: FunctionStability;
|
|
105
107
|
nullHandling?: NullHandling;
|
|
108
|
+
/** One scalar monotonicity claim per ordered parameter declaration. */
|
|
109
|
+
argumentMonotonicity?: ArgumentMonotonicity[] | null;
|
|
106
110
|
examples?: FunctionExample[];
|
|
107
111
|
categories?: string[];
|
|
108
112
|
tags?: Record<string, string>;
|
|
113
|
+
parameterDefaultValues?: VgiBatch | null;
|
|
109
114
|
maxWorkers?: number;
|
|
110
115
|
requiredSettings?: string[];
|
|
111
116
|
requiredSecrets?: string[];
|
|
@@ -3,6 +3,7 @@ import type { OutputCollector } from "@query-farm/vgi-rpc";
|
|
|
3
3
|
import type { BindRequest, InitRequest, GlobalInitResponse } from "../protocol/types.js";
|
|
4
4
|
import type { TableCardinality } from "../types.js";
|
|
5
5
|
import type { VgiFunction, FunctionExample } from "./types.js";
|
|
6
|
+
import type { ArgumentConstraints } from "../arguments/argument-spec.js";
|
|
6
7
|
import { FunctionStability } from "../types.js";
|
|
7
8
|
import { BoundStorage } from "./storage.js";
|
|
8
9
|
export interface TableBufferingBindParams<TArgs = Record<string, any>> {
|
|
@@ -13,6 +14,8 @@ export interface TableBufferingBindParams<TArgs = Record<string, any>> {
|
|
|
13
14
|
/** True on the second bind pass, after the connector resolved the secrets
|
|
14
15
|
* requested via the onBind `lookupSecret*` return fields on the first pass. */
|
|
15
16
|
resolvedSecretsProvided: boolean;
|
|
17
|
+
/** Resolved names aligned with the complete logical argument order. */
|
|
18
|
+
argumentNames?: (string | null)[] | null;
|
|
16
19
|
}
|
|
17
20
|
export interface TableBufferingParams<TArgs = Record<string, any>> {
|
|
18
21
|
args: TArgs;
|
|
@@ -40,6 +43,12 @@ export interface TableBufferingConfig<TArgs = Record<string, any>, TState = any>
|
|
|
40
43
|
args?: Record<string, VgiDataType>;
|
|
41
44
|
namedArgs?: Record<string, VgiDataType>;
|
|
42
45
|
argDefaults?: Record<string, any>;
|
|
46
|
+
/** Authoritative typed defaults: one row, defaulted parameters only. */
|
|
47
|
+
parameterDefaultValues?: VgiBatch | null;
|
|
48
|
+
/** Argument descriptions keyed by argument name. */
|
|
49
|
+
argDocs?: Record<string, string>;
|
|
50
|
+
/** Discovery constraints, also enforced for scalar arguments at bind. */
|
|
51
|
+
argConstraints?: Record<string, ArgumentConstraints>;
|
|
43
52
|
/** Bind: default passes through input schema. May be async. Return non-empty
|
|
44
53
|
* `lookupSecret*` lists on the first pass to request DuckDB secrets; the
|
|
45
54
|
* connector then re-binds with `resolvedSecretsProvided` and the resolved
|
|
@@ -2,6 +2,7 @@ import { type VgiSchema, type VgiDataType, type VgiBatch } from "../arrow/index.
|
|
|
2
2
|
import type { OutputCollector } from "@query-farm/vgi-rpc";
|
|
3
3
|
import type { BindRequest, InitRequest, GlobalInitResponse } from "../protocol/types.js";
|
|
4
4
|
import type { VgiFunction, FunctionExample } from "./types.js";
|
|
5
|
+
import type { ArgumentConstraints } from "../arguments/argument-spec.js";
|
|
5
6
|
import { type PushdownFilters } from "../filter-pushdown/index.js";
|
|
6
7
|
import { FunctionStability } from "../types.js";
|
|
7
8
|
import { BoundStorage } from "./storage.js";
|
|
@@ -10,6 +11,8 @@ export interface TableInOutBindParams<TArgs = Record<string, any>> {
|
|
|
10
11
|
bindCall: BindRequest;
|
|
11
12
|
settings: Record<string, any>;
|
|
12
13
|
secrets: Record<string, Record<string, any>>;
|
|
14
|
+
/** Resolved names aligned with the complete logical argument order. */
|
|
15
|
+
argumentNames?: (string | null)[] | null;
|
|
13
16
|
}
|
|
14
17
|
export interface TableInOutProcessParams<TArgs = Record<string, any>> {
|
|
15
18
|
args: TArgs;
|
|
@@ -52,6 +55,12 @@ export interface TableInOutConfig<TArgs = Record<string, any>, TState = null> {
|
|
|
52
55
|
namedArgs?: Record<string, VgiDataType>;
|
|
53
56
|
/** Argument defaults */
|
|
54
57
|
argDefaults?: Record<string, any>;
|
|
58
|
+
/** Authoritative typed defaults: one row, defaulted parameters only. */
|
|
59
|
+
parameterDefaultValues?: VgiBatch | null;
|
|
60
|
+
/** Argument descriptions keyed by argument name. */
|
|
61
|
+
argDocs?: Record<string, string>;
|
|
62
|
+
/** Discovery constraints, also enforced for scalar arguments at bind. */
|
|
63
|
+
argConstraints?: Record<string, ArgumentConstraints>;
|
|
55
64
|
/** Bind: default passes through input schema. May be async. */
|
|
56
65
|
onBind?: (params: TableInOutBindParams<TArgs>) => {
|
|
57
66
|
outputSchema: VgiSchema;
|
|
@@ -140,8 +149,12 @@ export interface RowTransformConfig<TArgs = Record<string, any>> {
|
|
|
140
149
|
/** Named (string-position) args stay bind-time scalars on `params.args`. */
|
|
141
150
|
namedArgs?: Record<string, VgiDataType>;
|
|
142
151
|
argDefaults?: Record<string, any>;
|
|
152
|
+
/** Authoritative typed defaults: one row, defaulted parameters only. */
|
|
153
|
+
parameterDefaultValues?: VgiBatch | null;
|
|
143
154
|
/** Per-argument descriptions keyed by arg name (surfaced as `vgi_doc`). */
|
|
144
155
|
argDocs?: Record<string, string>;
|
|
156
|
+
/** Discovery constraints, also enforced for named scalar arguments at bind. */
|
|
157
|
+
argConstraints?: Record<string, ArgumentConstraints>;
|
|
145
158
|
/** Bind: return the output schema. The input schema (the declared per-row
|
|
146
159
|
* columns, typed by the C++ bind) is on `params.bindCall.input_schema`. */
|
|
147
160
|
onBind: (params: TableInOutBindParams<TArgs>) => {
|
|
@@ -1,5 +1,6 @@
|
|
|
1
|
-
import { type VgiSchema, type VgiDataType } from "../arrow/index.js";
|
|
1
|
+
import { type VgiSchema, type VgiBatch, type VgiDataType } from "../arrow/index.js";
|
|
2
2
|
import type { OutputCollector } from "@query-farm/vgi-rpc";
|
|
3
|
+
import type { PlanResult, TableFunctionPlanRequest } from "../protocol/serializers/splits.js";
|
|
3
4
|
import type { BindRequest, InitRequest, GlobalInitResponse, TableCardinality } from "../protocol/types.js";
|
|
4
5
|
import type { VgiFunction, FunctionExample } from "./types.js";
|
|
5
6
|
import { type ArgumentConstraints } from "../arguments/argument-spec.js";
|
|
@@ -13,6 +14,8 @@ export interface TableBindParams<TArgs = Record<string, any>> {
|
|
|
13
14
|
settings: Record<string, any>;
|
|
14
15
|
secrets: Record<string, Record<string, any>>;
|
|
15
16
|
resolvedSecretsProvided: boolean;
|
|
17
|
+
/** Resolved names aligned with the complete logical argument order. */
|
|
18
|
+
argumentNames?: (string | null)[] | null;
|
|
16
19
|
}
|
|
17
20
|
export interface TableProcessParams<TArgs = Record<string, any>> {
|
|
18
21
|
args: TArgs;
|
|
@@ -22,6 +25,15 @@ export interface TableProcessParams<TArgs = Record<string, any>> {
|
|
|
22
25
|
settings: Record<string, any>;
|
|
23
26
|
secrets: Record<string, Record<string, any>>;
|
|
24
27
|
pushdownFilters?: PushdownFilters;
|
|
28
|
+
/**
|
|
29
|
+
* The verified payloads of the splits this init claimed, in the order given,
|
|
30
|
+
* or undefined when this is not a split scan.
|
|
31
|
+
*
|
|
32
|
+
* Undefined is meaningfully different from an empty array: undefined means
|
|
33
|
+
* the client did not plan (a split-only function should fail loudly rather
|
|
34
|
+
* than answer the same query differently), empty means a claim of no work.
|
|
35
|
+
*/
|
|
36
|
+
splitPayloads?: Uint8Array[];
|
|
25
37
|
storage?: BoundStorage;
|
|
26
38
|
/**
|
|
27
39
|
* AT (TIMESTAMP|VERSION) clause for this scan, or `undefined` when the scan
|
|
@@ -54,6 +66,8 @@ export interface TableFunctionConfig<TArgs = Record<string, any>, TState = null>
|
|
|
54
66
|
argDocs?: Record<string, string>;
|
|
55
67
|
/** Argument defaults */
|
|
56
68
|
argDefaults?: Record<string, any>;
|
|
69
|
+
/** Authoritative typed defaults: one row, defaulted parameters only. */
|
|
70
|
+
parameterDefaultValues?: VgiBatch | null;
|
|
57
71
|
/**
|
|
58
72
|
* Per-argument discovery constraints (choices / ge / le / gt / lt / pattern),
|
|
59
73
|
* keyed by argument name. Surfaced via `vgi_function_arguments()` for agent
|
|
@@ -91,6 +105,40 @@ export interface TableFunctionConfig<TArgs = Record<string, any>, TState = null>
|
|
|
91
105
|
process: (params: TableProcessParams<TArgs>, state: TState, out: OutputCollector) => void | Promise<void>;
|
|
92
106
|
/** Cardinality hints */
|
|
93
107
|
cardinality?: (params: TableBindParams<TArgs>) => TableCardinality | Promise<TableCardinality>;
|
|
108
|
+
/**
|
|
109
|
+
* Divide this scan into named, independently redeemable splits.
|
|
110
|
+
*
|
|
111
|
+
* Declaring it (together with `meta.supportsSplits`) is what opts a function
|
|
112
|
+
* into the split path; omitting it means the whole scan is one unit of work.
|
|
113
|
+
*
|
|
114
|
+
* A split *names* work rather than describing it. "These three files at
|
|
115
|
+
* version 47" survives a retry; "rows 0-999 of whatever this returns now"
|
|
116
|
+
* does not — and a distributed engine WILL retry, so the difference is
|
|
117
|
+
* correctness, not tidiness. The same split may also be redeemed more than
|
|
118
|
+
* once (recursive CTEs, re-collected DataFrames, task retry) and may be
|
|
119
|
+
* abandoned mid-stream (LIMIT, TopK, an empty join build side); neither is an
|
|
120
|
+
* error.
|
|
121
|
+
*
|
|
122
|
+
* Set only `payload` on each split. The framework stamps the consistency
|
|
123
|
+
* anchor, the bind fingerprint and (where a key exists) the seal.
|
|
124
|
+
*
|
|
125
|
+
* Size splits into comparable units of work and honour `targetSplitBytes`: a
|
|
126
|
+
* claiming client treats them as interchangeable because it cannot see
|
|
127
|
+
* per-split cost, so wildly uneven splits leave its makespan bounded by the
|
|
128
|
+
* largest one.
|
|
129
|
+
*/
|
|
130
|
+
plan?: (params: TableBindParams<TArgs>, request: TableFunctionPlanRequest) => PlanResult | Promise<PlanResult>;
|
|
131
|
+
/**
|
|
132
|
+
* Called on a split init with the VERIFIED payloads for the splits this
|
|
133
|
+
* connection claimed — the envelope is already opened and stripped, so
|
|
134
|
+
* unverified bytes never reach here.
|
|
135
|
+
*
|
|
136
|
+
* Any state carried from planning to reading must live in cross-process
|
|
137
|
+
* storage keyed by `execution_id`: the process that plans is, in the general
|
|
138
|
+
* case, not the process that reads — and under a distributed engine it is not
|
|
139
|
+
* even the same host.
|
|
140
|
+
*/
|
|
141
|
+
onSplit?: (payloads: Uint8Array[], params: TableBindParams<TArgs>) => void | Promise<void>;
|
|
94
142
|
/**
|
|
95
143
|
* Per-column statistics for the function's output. Returned to DuckDB via
|
|
96
144
|
* the `table_function_statistics` RPC; the optimizer uses min/max to
|
|
@@ -115,7 +163,10 @@ export interface TableFunctionConfig<TArgs = Record<string, any>, TState = null>
|
|
|
115
163
|
/** Opt in to DuckDB's late-materialization SEMI-join rewrite; requires a
|
|
116
164
|
* UNIQUE, snapshot-stable rowid column. FunctionInfo.late_materialization. */
|
|
117
165
|
lateMaterialization?: boolean;
|
|
118
|
-
|
|
166
|
+
filterSemanticProfiles?: string[];
|
|
167
|
+
additionalFilterFunctions?: import("./types.js").FilterFunctionCapability[];
|
|
168
|
+
runtimeFilterAlgorithms?: import("./types.js").FilterFunctionCapability[];
|
|
169
|
+
filterEvaluationContexts?: import("./types.js").EvaluationContextCapability[];
|
|
119
170
|
autoApplyFilters?: boolean;
|
|
120
171
|
stability?: FunctionStability;
|
|
121
172
|
examples?: FunctionExample[];
|
|
@@ -134,5 +185,24 @@ export interface TableFunctionConfig<TArgs = Record<string, any>, TState = null>
|
|
|
134
185
|
supportsBatchIndex?: boolean;
|
|
135
186
|
/** Hive-style partition-columns mode; FunctionInfo.partition_kind. */
|
|
136
187
|
partitionKind?: "NOT_PARTITIONED" | "SINGLE_VALUE_PARTITIONS" | "OVERLAPPING_PARTITIONS" | "DISJOINT_PARTITIONS";
|
|
188
|
+
/**
|
|
189
|
+
* This function divides its scan into named, independently redeemable splits;
|
|
190
|
+
* FunctionInfo.supports_splits. Declare it together with `plan` and `onSplit`.
|
|
191
|
+
*
|
|
192
|
+
* It is what a distributed engine reads to decide it can retry a task against
|
|
193
|
+
* this function, and what makes the client call `plan()` at all — without it a
|
|
194
|
+
* split-capable function is scanned as one undivided unit, silently.
|
|
195
|
+
*/
|
|
196
|
+
supportsSplits?: boolean;
|
|
197
|
+
/**
|
|
198
|
+
* How long a minted split token stays redeemable, or undefined for UNBOUNDED
|
|
199
|
+
* (NOT "expires immediately"); FunctionInfo.split_token_ttl_seconds.
|
|
200
|
+
*
|
|
201
|
+
* A client refuses a plan whose TTL is below its own scheduling horizon,
|
|
202
|
+
* because an expired token is a failed query rather than a degradation:
|
|
203
|
+
* nothing re-plans when one expires, since a distributed engine retries the
|
|
204
|
+
* serialized task it was handed and has no path back to the planner.
|
|
205
|
+
*/
|
|
206
|
+
splitTokenTtlSeconds?: number;
|
|
137
207
|
}
|
|
138
208
|
export declare function defineTableFunction<TArgs = Record<string, any>, TState = null>(config: TableFunctionConfig<TArgs, TState>): VgiFunction;
|