@query-farm/vgi 0.28.0 → 0.29.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +52 -8
- package/dist/arguments/argument-spec.d.ts +57 -2
- package/dist/arrow/impl-arrowjs/index.d.ts +1 -1
- package/dist/arrow/impl-arrowjs/iterate.d.ts +10 -1
- package/dist/arrow/impl-arrowjs/schema.d.ts +2 -0
- package/dist/arrow/impl-flechette/index.d.ts +1 -1
- package/dist/arrow/impl-flechette/iterate.d.ts +10 -1
- package/dist/arrow/impl-flechette/schema.d.ts +2 -0
- package/dist/arrow/index.d.ts +1 -1
- package/dist/arrow/schema-types.d.ts +3 -1
- package/dist/arrow/types.d.ts +39 -3
- package/dist/catalog/composite.d.ts +13 -13
- package/dist/catalog/descriptors.d.ts +7 -3
- package/dist/catalog/interface.d.ts +108 -38
- package/dist/catalog/read-only.d.ts +32 -15
- package/dist/client/client.d.ts +37 -30
- package/dist/client/types.d.ts +2 -0
- package/dist/client-entry.d.ts +1 -1
- package/dist/client-entry.js +791 -447
- package/dist/client-entry.js.map +18 -17
- package/dist/filter-pushdown/capabilities.d.ts +3 -0
- package/dist/filter-pushdown/deserialize.d.ts +14 -15
- package/dist/filter-pushdown/evaluate.d.ts +14 -44
- package/dist/filter-pushdown/index.d.ts +2 -2
- package/dist/filter-pushdown/types.d.ts +120 -68
- package/dist/functions/aggregate.d.ts +4 -0
- package/dist/functions/registry.d.ts +5 -5
- package/dist/functions/scalar.d.ts +6 -1
- package/dist/functions/table-buffering.d.ts +9 -0
- package/dist/functions/table-in-out.d.ts +13 -0
- package/dist/functions/table.d.ts +72 -2
- package/dist/functions/types.d.ts +60 -2
- package/dist/generated/vgi-client.d.ts +114 -26
- package/dist/generated/vgi-protocol-schemas.d.ts +34 -0
- package/dist/http/fetch.d.ts +16 -0
- package/dist/index.core.d.ts +4 -3
- package/dist/index.js +3396 -2064
- package/dist/index.js.map +55 -49
- package/dist/metadata/types.d.ts +15 -2
- package/dist/protocol/handlers/catalog/shared.d.ts +0 -8
- package/dist/protocol/handlers/function.d.ts +11 -0
- package/dist/protocol/handlers/shared.d.ts +3 -1
- package/dist/protocol/serializers/bind.d.ts +1 -2
- package/dist/protocol/serializers/cardinality.d.ts +1 -2
- package/dist/protocol/serializers/splits.d.ts +126 -0
- package/dist/protocol/types.d.ts +36 -3
- package/dist/request-auth.d.ts +28 -0
- package/dist/schema-path.d.ts +7 -0
- package/dist/serve-entry.d.ts +25 -0
- package/dist/serve-entry.js +1129 -569
- package/dist/serve-entry.js.map +36 -32
- package/dist/split-token.d.ts +88 -0
- package/dist/types.d.ts +8 -0
- package/dist/worker-cf-entry.js +2928 -1689
- package/dist/worker-cf-entry.js.map +51 -44
- package/dist/worker.d.ts +1 -1
- package/package.json +3 -3
- package/src/arguments/argument-spec.ts +93 -4
- package/src/arguments/arguments.ts +0 -1
- package/src/arrow/impl-arrowjs/index.ts +1 -1
- package/src/arrow/impl-arrowjs/iterate.ts +21 -0
- package/src/arrow/impl-arrowjs/schema.ts +4 -0
- package/src/arrow/impl-flechette/index.ts +1 -1
- package/src/arrow/impl-flechette/iterate.ts +24 -1
- package/src/arrow/impl-flechette/schema.ts +4 -0
- package/src/arrow/index.ts +1 -1
- package/src/arrow/schema-types.ts +4 -0
- package/src/arrow/types.ts +39 -3
- package/src/catalog/composite.ts +26 -26
- package/src/catalog/descriptors.ts +12 -3
- package/src/catalog/interface.ts +217 -72
- package/src/catalog/read-only.ts +141 -84
- package/src/client/client.ts +97 -73
- package/src/client/types.ts +2 -0
- package/src/client-entry.ts +2 -1
- package/src/filter-pushdown/capabilities.ts +13 -0
- package/src/filter-pushdown/collector.ts +22 -35
- package/src/filter-pushdown/deserialize.ts +657 -221
- package/src/filter-pushdown/evaluate.ts +410 -499
- package/src/filter-pushdown/index.ts +16 -2
- package/src/filter-pushdown/types.ts +88 -71
- package/src/functions/aggregate.ts +12 -1
- package/src/functions/copy-from.ts +2 -1
- package/src/functions/copy-to.ts +2 -1
- package/src/functions/registry.ts +24 -22
- package/src/functions/scalar.ts +37 -0
- package/src/functions/table-buffering.ts +48 -6
- package/src/functions/table-in-out.ts +194 -33
- package/src/functions/table.ts +187 -22
- package/src/functions/types.ts +64 -1
- package/src/generated/vgi-client.ts +122 -28
- package/src/generated/vgi-protocol-schemas.ts +455 -58
- package/src/http/fetch.ts +129 -2
- package/src/index.core.ts +4 -2
- package/src/metadata/resolve.ts +43 -1
- package/src/metadata/serialize.ts +43 -5
- package/src/metadata/types.ts +17 -1
- package/src/protocol/dispatch.ts +1 -1
- package/src/protocol/handlers/aggregate.ts +23 -12
- package/src/protocol/handlers/catalog/admin.ts +47 -54
- package/src/protocol/handlers/catalog/index_methods.ts +6 -13
- package/src/protocol/handlers/catalog/macro.ts +12 -20
- package/src/protocol/handlers/catalog/shared.ts +12 -57
- package/src/protocol/handlers/catalog/table.ts +49 -112
- package/src/protocol/handlers/catalog/view.ts +18 -22
- package/src/protocol/handlers/function.ts +311 -18
- package/src/protocol/handlers/shared.ts +12 -3
- package/src/protocol/handlers/table-buffering.ts +11 -8
- package/src/protocol/serializers/bind.ts +69 -67
- package/src/protocol/serializers/cardinality.ts +7 -7
- package/src/protocol/serializers/init.ts +52 -10
- package/src/protocol/serializers/splits.ts +221 -0
- package/src/protocol/state-serializer.ts +55 -1
- package/src/protocol/types.ts +36 -3
- package/src/request-auth.ts +73 -0
- package/src/schema-path.ts +25 -0
- package/src/serve-entry.ts +69 -2
- package/src/split-token.ts +297 -0
- package/src/types.ts +9 -0
- package/src/worker-cf-entry.ts +2 -1
- package/src/worker.ts +79 -14
package/src/http/fetch.ts
CHANGED
|
@@ -10,10 +10,23 @@
|
|
|
10
10
|
// `buildVgiProtocol` as `signingKey`, from one source. Passing different keys to
|
|
11
11
|
// those two seams produces a worker that mints tokens it cannot recover.
|
|
12
12
|
|
|
13
|
-
import {
|
|
13
|
+
import {
|
|
14
|
+
AuthContext,
|
|
15
|
+
createHttpHandler,
|
|
16
|
+
unpackStateToken,
|
|
17
|
+
type AuthenticateFn,
|
|
18
|
+
type DispatchHook,
|
|
19
|
+
type DispatchInfo,
|
|
20
|
+
type HookToken,
|
|
21
|
+
type PeerAuthenticationPolicy,
|
|
22
|
+
type PeerIdentityProvider,
|
|
23
|
+
type PeerResolutionOptions,
|
|
24
|
+
type Protocol,
|
|
25
|
+
} from "@query-farm/vgi-rpc";
|
|
14
26
|
import { arrowStateSerializer } from "../protocol/state-serializer.js";
|
|
15
27
|
import { buildVgiProtocol, type ProtocolConfig } from "../protocol/dispatch.js";
|
|
16
28
|
import { createLandingRoutes, type LandingInfo } from "./landing.js";
|
|
29
|
+
import { setRequestAuthScope, type RequestAuthHolder } from "../request-auth.js";
|
|
17
30
|
|
|
18
31
|
export interface VgiFetchOptions {
|
|
19
32
|
/** Wire-protocol config (registry + catalogInterface). */
|
|
@@ -57,8 +70,106 @@ export interface VgiFetchOptions {
|
|
|
57
70
|
* its required name/doc/version; the CF entry had to remember, and
|
|
58
71
|
* vgi-open-meteo shipped for weeks without it. */
|
|
59
72
|
landingInfo: LandingInfo;
|
|
73
|
+
/** Authenticates each incoming request, returning the caller's
|
|
74
|
+
* {@link AuthContext}. Forwarded verbatim to `createHttpHandler`; omit it
|
|
75
|
+
* and every caller is anonymous, which is what this package did before the
|
|
76
|
+
* option existed.
|
|
77
|
+
*
|
|
78
|
+
* Supplying it is what makes a worker able to serve more than one principal,
|
|
79
|
+
* and the identity it returns is the one the worker seals `attach_opaque_data`
|
|
80
|
+
* and split tokens under. */
|
|
81
|
+
authenticate?: AuthenticateFn;
|
|
82
|
+
/** Off-wire transport identities, including a trusted Iroh HTTP bridge. */
|
|
83
|
+
peerIdentityProviders?: readonly PeerIdentityProvider[];
|
|
84
|
+
/** Policy composing peer evidence with application authentication. */
|
|
85
|
+
peerAuthenticationPolicy?: PeerAuthenticationPolicy;
|
|
86
|
+
/** Host adapter supplying the physical peer and multiplicity-preserving headers. */
|
|
87
|
+
peerResolutionContext?: (request: Request) => PeerResolutionOptions | Promise<PeerResolutionOptions>;
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
// ---------------------------------------------------------------------------
|
|
91
|
+
// Request-scoped caller identity
|
|
92
|
+
//
|
|
93
|
+
// `@query-farm/vgi-rpc` types stream initialization as `(params) => state` —
|
|
94
|
+
// no CallContext — on both dispatchers, so the `init` handler cannot be handed
|
|
95
|
+
// the principal as an argument the way every unary handler is. But `init` is
|
|
96
|
+
// exactly where the identity-bound envelopes minted by `catalog_attach` and
|
|
97
|
+
// `table_function_plan` (both unary, both sealed under the REAL caller) are
|
|
98
|
+
// opened. Opening them under the anonymous identity works for anonymous
|
|
99
|
+
// callers and fails for everyone else.
|
|
100
|
+
//
|
|
101
|
+
// So the identity travels the way vgi-python's `current_auth()` ContextVar
|
|
102
|
+
// does: an ambient per-request scope. `AsyncLocalStorage.run` wraps the whole
|
|
103
|
+
// handler call, and a dispatch hook fills the holder in with whatever identity
|
|
104
|
+
// the framework actually dispatched under — the hook is read AFTER
|
|
105
|
+
// authentication, the PKCE cookie chain and bearer introspection have all had
|
|
106
|
+
// their say, so it reports the final principal rather than re-deriving one.
|
|
107
|
+
//
|
|
108
|
+
// A mutable holder rather than `enterWith`: `run` is the only form Cloudflare
|
|
109
|
+
// Workers supports, and the identity is not known until authentication has run
|
|
110
|
+
// inside the handler.
|
|
111
|
+
// ---------------------------------------------------------------------------
|
|
112
|
+
|
|
113
|
+
/** The `AsyncLocalStorage` surface used here, structurally typed so this file
|
|
114
|
+
* compiles where `node:async_hooks` has no types. */
|
|
115
|
+
interface AsyncLocalStorageLike<T> {
|
|
116
|
+
run<R>(store: T, fn: () => R): R;
|
|
117
|
+
getStore(): T | undefined;
|
|
60
118
|
}
|
|
61
119
|
|
|
120
|
+
let requestAuthStorage: AsyncLocalStorageLike<RequestAuthHolder> | null | undefined;
|
|
121
|
+
let requestAuthStoragePending: Promise<AsyncLocalStorageLike<RequestAuthHolder> | null> | undefined;
|
|
122
|
+
|
|
123
|
+
/**
|
|
124
|
+
* Resolve (once) the AsyncLocalStorage this process scopes requests with.
|
|
125
|
+
*
|
|
126
|
+
* `node:async_hooks` is loaded dynamically and defensively: it exists on Node
|
|
127
|
+
* and Bun, and on Cloudflare Workers only with the `nodejs_compat` /
|
|
128
|
+
* `nodejs_als` compatibility flag. Where it is missing this resolves to `null`
|
|
129
|
+
* and the worker keeps serving exactly as it did before — anonymous callers
|
|
130
|
+
* unaffected, authenticated ones still unable to open their sealed attach.
|
|
131
|
+
* Degrading is the right failure here: refusing to start would break every
|
|
132
|
+
* anonymous deployment to fix a case it does not have.
|
|
133
|
+
*/
|
|
134
|
+
function ensureRequestAuthStorage(): Promise<AsyncLocalStorageLike<RequestAuthHolder> | null> {
|
|
135
|
+
if (requestAuthStorage !== undefined) return Promise.resolve(requestAuthStorage);
|
|
136
|
+
requestAuthStoragePending ??= import("node:async_hooks")
|
|
137
|
+
.then((mod) => {
|
|
138
|
+
const storage = new mod.AsyncLocalStorage<RequestAuthHolder>() as AsyncLocalStorageLike<RequestAuthHolder>;
|
|
139
|
+
requestAuthStorage = storage;
|
|
140
|
+
setRequestAuthScope(storage);
|
|
141
|
+
return storage;
|
|
142
|
+
})
|
|
143
|
+
.catch(() => {
|
|
144
|
+
requestAuthStorage = null;
|
|
145
|
+
return null;
|
|
146
|
+
});
|
|
147
|
+
return requestAuthStoragePending;
|
|
148
|
+
}
|
|
149
|
+
|
|
150
|
+
/**
|
|
151
|
+
* Dispatch hook that records the dispatching principal on the open scope.
|
|
152
|
+
*
|
|
153
|
+
* Reads the identity off `DispatchInfo`, which the framework populates from the
|
|
154
|
+
* resolved auth context for every method — so this sees the same principal the
|
|
155
|
+
* access log and the unary handlers see, without this package having to know
|
|
156
|
+
* how authentication was configured.
|
|
157
|
+
*/
|
|
158
|
+
const REQUEST_AUTH_DISPATCH_HOOK: DispatchHook = {
|
|
159
|
+
onDispatchStart(info: DispatchInfo): HookToken {
|
|
160
|
+
const holder = requestAuthStorage?.getStore();
|
|
161
|
+
if (holder) {
|
|
162
|
+
holder.auth = info.authenticated
|
|
163
|
+
? new AuthContext(info.authDomain ?? "", true, info.principal ?? "", info.claims ?? {})
|
|
164
|
+
: AuthContext.anonymous();
|
|
165
|
+
}
|
|
166
|
+
return undefined;
|
|
167
|
+
},
|
|
168
|
+
onDispatchEnd() {
|
|
169
|
+
// Nothing to record; the scope ends when the request does.
|
|
170
|
+
},
|
|
171
|
+
};
|
|
172
|
+
|
|
62
173
|
/**
|
|
63
174
|
* Build a fetch handler suitable for `export default { fetch }` in a CF
|
|
64
175
|
* Worker module, or for `Bun.serve({ fetch })`. The returned handler is
|
|
@@ -100,6 +211,14 @@ export function createVgiFetch(opts: VgiFetchOptions): (req: Request) => Promise
|
|
|
100
211
|
tokenKey: opts.signingKey,
|
|
101
212
|
tokenTtl,
|
|
102
213
|
stateSerializer: arrowStateSerializer,
|
|
214
|
+
// Records the dispatching principal on the request scope opened below, so
|
|
215
|
+
// stream `init` — the one handler the framework passes no CallContext to —
|
|
216
|
+
// can still open the identity-bound attach and split envelopes.
|
|
217
|
+
dispatchHook: REQUEST_AUTH_DISPATCH_HOOK,
|
|
218
|
+
authenticate: opts.authenticate,
|
|
219
|
+
peerIdentityProviders: opts.peerIdentityProviders,
|
|
220
|
+
peerAuthenticationPolicy: opts.peerAuthenticationPolicy,
|
|
221
|
+
peerResolutionContext: opts.peerResolutionContext,
|
|
103
222
|
// `null` is the explicit opt-out; `undefined` (omitted) means "*".
|
|
104
223
|
corsOrigins: opts.corsOrigins === null ? undefined : (opts.corsOrigins ?? "*"),
|
|
105
224
|
repositoryUrl: opts.repositoryUrl,
|
|
@@ -110,5 +229,13 @@ export function createVgiFetch(opts: VgiFetchOptions): (req: Request) => Promise
|
|
|
110
229
|
extraRoutes: createLandingRoutes(opts.landingInfo),
|
|
111
230
|
enableLandingPage: false,
|
|
112
231
|
});
|
|
113
|
-
return async (req: Request) =>
|
|
232
|
+
return async (req: Request) => {
|
|
233
|
+
const storage = await ensureRequestAuthStorage();
|
|
234
|
+
// No AsyncLocalStorage on this host: serve exactly as before.
|
|
235
|
+
if (!storage) return handler(req);
|
|
236
|
+
// One holder per request. The dispatch hook fills it in once the framework
|
|
237
|
+
// has resolved the principal; `init` reads it back through
|
|
238
|
+
// `currentRequestAuth()`.
|
|
239
|
+
return storage.run({}, () => handler(req));
|
|
240
|
+
};
|
|
114
241
|
}
|
package/src/index.core.ts
CHANGED
|
@@ -77,7 +77,7 @@ export {
|
|
|
77
77
|
export type { VgiFunction, FunctionMeta, StreamHandlers, FunctionExample, HandlerState } from "./functions/types.js";
|
|
78
78
|
|
|
79
79
|
// Catalog
|
|
80
|
-
export { CatalogInterface, type CatalogAttachResult, type SchemaInfo, encodeSchemaInfo, decodeSchemaInfo, type TableInfo, encodeTableInfo, decodeTableInfo, type ViewInfo, encodeViewInfo, decodeViewInfo, type CatalogInfo, encodeCatalogInfo, decodeCatalogInfo, type FunctionInfo, encodeFunctionInfo, decodeFunctionInfo, type FunctionInfoOptions, type MacroInfo, encodeMacroInfo, decodeMacroInfo, type MacroType, type CopyFromFormatInfo, encodeCopyFromFormatInfo, type AttachOpaqueData, type TransactionOpaqueData, buildScanBranchesResult, type ScanBranchInput } from "./catalog/interface.js";
|
|
80
|
+
export { CatalogInterface, type CatalogAttachResult, type SchemaInfo, encodeSchemaInfo, decodeSchemaInfo, type TableInfo, encodeTableInfo, decodeTableInfo, type ViewInfo, encodeViewInfo, decodeViewInfo, type CatalogInfo, encodeCatalogInfo, decodeCatalogInfo, type FunctionInfo, encodeFunctionInfo, decodeFunctionInfo, type FunctionInfoOptions, type MacroInfo, encodeMacroInfo, decodeMacroInfo, type MacroType, type CopyFromFormatInfo, encodeCopyFromFormatInfo, type AttachOpaqueData, type TransactionOpaqueData, buildScanBranchesResult, encodeFormatOptions, type ScanBranchInput } from "./catalog/interface.js";
|
|
81
81
|
export type { CatalogDescriptor, SchemaDescriptor, TableDescriptor, ViewDescriptor, MacroDescriptor, SettingDescriptor, SecretTypeDescriptor, ForeignKeyDef, DefaultValue } from "./catalog/descriptors.js";
|
|
82
82
|
export { ReadOnlyCatalogInterface } from "./catalog/read-only.js";
|
|
83
83
|
export { CompositeCatalogInterface } from "./catalog/composite.js";
|
|
@@ -88,6 +88,7 @@ export {
|
|
|
88
88
|
FunctionType,
|
|
89
89
|
FunctionStability,
|
|
90
90
|
NullHandling,
|
|
91
|
+
ArgumentMonotonicity,
|
|
91
92
|
OrderPreservation,
|
|
92
93
|
OrderDependence,
|
|
93
94
|
DistinctDependence,
|
|
@@ -219,7 +220,7 @@ export {
|
|
|
219
220
|
int8, int16, int64,
|
|
220
221
|
uint8, uint16, uint32, uint64,
|
|
221
222
|
float16, float64,
|
|
222
|
-
utf8, binary, fixedSizeBinary,
|
|
223
|
+
utf8, largeUtf8, binary, largeBinary, fixedSizeBinary,
|
|
223
224
|
decimal, decimal128, decimal256,
|
|
224
225
|
date, dateDay, dateMillisecond,
|
|
225
226
|
time, timeSecond, timeMillisecond, timeMicrosecond, timeNanosecond,
|
|
@@ -307,3 +308,4 @@ export type {
|
|
|
307
308
|
export { str, bytes, int, int32, float, float32, bool, toSchema, OutputCollector, AuthContext } from "@query-farm/vgi-rpc";
|
|
308
309
|
export { subprocessConnect, httpConnect, tcpConnect } from "@query-farm/vgi-rpc";
|
|
309
310
|
export type { RpcClient, StreamSession, LogMessage } from "@query-farm/vgi-rpc";
|
|
311
|
+
export { normalizeSchemaPath, schemaPathDisplay, schemaPathKey, schemaPathsEqual, type SchemaPath } from "./schema-path.js";
|
package/src/metadata/resolve.ts
CHANGED
|
@@ -4,6 +4,7 @@
|
|
|
4
4
|
import {
|
|
5
5
|
FunctionStability,
|
|
6
6
|
NullHandling,
|
|
7
|
+
ArgumentMonotonicity,
|
|
7
8
|
OrderPreservation,
|
|
8
9
|
OrderDependence,
|
|
9
10
|
DistinctDependence,
|
|
@@ -11,6 +12,7 @@ import {
|
|
|
11
12
|
} from "../types.js";
|
|
12
13
|
import type { VgiFunction } from "../functions/types.js";
|
|
13
14
|
import type { ArgumentSpec } from "../arguments/argument-spec.js";
|
|
15
|
+
import { supportsExtensionFilterFunction } from "../filter-pushdown/capabilities.js";
|
|
14
16
|
import {
|
|
15
17
|
CatalogFunctionType,
|
|
16
18
|
type ResolvedMetadata,
|
|
@@ -55,6 +57,19 @@ function specToParameterInfo(spec: ArgumentSpec): ParameterInfo {
|
|
|
55
57
|
export function resolveMetadata(func: VgiFunction): ResolvedMetadata {
|
|
56
58
|
const meta = func.meta;
|
|
57
59
|
|
|
60
|
+
if (meta.runtimeFilterAlgorithms?.length) {
|
|
61
|
+
throw new Error("TypeScript SDK has no registered runtime-filter artifact evaluator to advertise");
|
|
62
|
+
}
|
|
63
|
+
if (meta.filterEvaluationContexts?.length) {
|
|
64
|
+
throw new Error("TypeScript SDK has no isolated DuckDB session-context evaluator to advertise");
|
|
65
|
+
}
|
|
66
|
+
if (meta.filterSemanticProfiles?.some((value) => value !== "vgi.duckdb.standard.v1")) {
|
|
67
|
+
throw new Error("TypeScript SDK supports only vgi.duckdb.standard.v1 filter semantics");
|
|
68
|
+
}
|
|
69
|
+
if (meta.additionalFilterFunctions?.some((value) => !supportsExtensionFilterFunction(value))) {
|
|
70
|
+
throw new Error("TypeScript SDK cannot advertise an extension-filter function without an evaluator");
|
|
71
|
+
}
|
|
72
|
+
|
|
58
73
|
let functionType: CatalogFunctionType;
|
|
59
74
|
switch (func.kind) {
|
|
60
75
|
case "scalar":
|
|
@@ -74,8 +89,28 @@ export function resolveMetadata(func: VgiFunction): ResolvedMetadata {
|
|
|
74
89
|
functionType = CatalogFunctionType.TABLE;
|
|
75
90
|
}
|
|
76
91
|
|
|
92
|
+
const argumentMonotonicity = meta.argumentMonotonicity ?? null;
|
|
93
|
+
if (argumentMonotonicity !== null && functionType !== CatalogFunctionType.SCALAR) {
|
|
94
|
+
throw new Error("argumentMonotonicity is only valid for scalar functions");
|
|
95
|
+
}
|
|
96
|
+
|
|
77
97
|
const parameters: ParameterInfo[] = func.argumentSpecs.map(specToParameterInfo);
|
|
78
98
|
|
|
99
|
+
if (argumentMonotonicity !== null) {
|
|
100
|
+
if (argumentMonotonicity.length !== parameters.length) {
|
|
101
|
+
throw new Error(
|
|
102
|
+
`argumentMonotonicity has ${argumentMonotonicity.length} entries, ` +
|
|
103
|
+
`expected ${parameters.length} declaration slots`,
|
|
104
|
+
);
|
|
105
|
+
}
|
|
106
|
+
const allowed = new Set<string>(Object.values(ArgumentMonotonicity));
|
|
107
|
+
for (const [index, value] of argumentMonotonicity.entries()) {
|
|
108
|
+
if (!allowed.has(value)) {
|
|
109
|
+
throw new Error(`argumentMonotonicity[${index}] has unknown value ${String(value)}`);
|
|
110
|
+
}
|
|
111
|
+
}
|
|
112
|
+
}
|
|
113
|
+
|
|
79
114
|
const examples: FunctionExample[] = (meta.examples ?? []).map((e) => ({
|
|
80
115
|
sql: e.sql,
|
|
81
116
|
description: e.description,
|
|
@@ -95,12 +130,19 @@ export function resolveMetadata(func: VgiFunction): ResolvedMetadata {
|
|
|
95
130
|
parameters,
|
|
96
131
|
stability: meta.stability ?? FunctionStability.CONSISTENT,
|
|
97
132
|
nullHandling: meta.nullHandling ?? NullHandling.DEFAULT,
|
|
133
|
+
argumentMonotonicity,
|
|
98
134
|
requiredSettings: meta.requiredSettings ?? [],
|
|
99
135
|
requiredSecrets: meta.requiredSecrets ?? [],
|
|
100
136
|
projectionPushdown: meta.projectionPushdown ?? false,
|
|
101
137
|
filterPushdown: meta.filterPushdown ?? false,
|
|
102
138
|
samplingPushdown: meta.samplingPushdown ?? false,
|
|
103
|
-
|
|
139
|
+
filterSemanticProfiles: meta.filterSemanticProfiles ?? (meta.filterPushdown ? ["vgi.duckdb.standard.v1"] : []),
|
|
140
|
+
additionalFilterFunctions: meta.additionalFilterFunctions ?? [],
|
|
141
|
+
runtimeFilterAlgorithms: meta.runtimeFilterAlgorithms ?? [],
|
|
142
|
+
filterEvaluationContexts: (meta.filterEvaluationContexts ?? []).map((value) => ({
|
|
143
|
+
profile: value.profile,
|
|
144
|
+
providerFingerprint: value.providerFingerprint ?? null,
|
|
145
|
+
})),
|
|
104
146
|
preservesOrder: meta.preservesOrder ?? OrderPreservation.NO_ORDER_GUARANTEE,
|
|
105
147
|
maxWorkers: maxWorkers === DEFAULT_MAX_WORKERS ? null : maxWorkers,
|
|
106
148
|
orderDependent: meta.orderDependent ?? OrderDependence.NOT_ORDER_DEPENDENT,
|
|
@@ -8,6 +8,7 @@ import {
|
|
|
8
8
|
field,
|
|
9
9
|
utf8,
|
|
10
10
|
int32,
|
|
11
|
+
uint64,
|
|
11
12
|
bool,
|
|
12
13
|
list,
|
|
13
14
|
struct as makeStruct,
|
|
@@ -16,6 +17,7 @@ import {
|
|
|
16
17
|
emptyBatch,
|
|
17
18
|
} from "../arrow/index.js";
|
|
18
19
|
import type { ResolvedMetadata, ParameterInfo, FunctionExample } from "./types.js";
|
|
20
|
+
import type { ArgumentMonotonicity } from "../types.js";
|
|
19
21
|
|
|
20
22
|
// ============================================================================
|
|
21
23
|
// Schema definitions matching Python's _METADATA_SCHEMA exactly
|
|
@@ -41,6 +43,17 @@ const PARAMETER_STRUCT = makeStruct([
|
|
|
41
43
|
field("is_const", bool(), false),
|
|
42
44
|
]);
|
|
43
45
|
|
|
46
|
+
const FILTER_IDENTITY_STRUCT = makeStruct([
|
|
47
|
+
field("namespace", utf8(), false),
|
|
48
|
+
field("name", utf8(), false),
|
|
49
|
+
field("version", uint64(), false),
|
|
50
|
+
]);
|
|
51
|
+
|
|
52
|
+
const FILTER_CONTEXT_STRUCT = makeStruct([
|
|
53
|
+
field("profile", utf8(), false),
|
|
54
|
+
field("provider_fingerprint", utf8(), true),
|
|
55
|
+
]);
|
|
56
|
+
|
|
44
57
|
const METADATA_SCHEMA = makeSchema([
|
|
45
58
|
field("name", utf8(), false),
|
|
46
59
|
field("class_name", utf8(), false),
|
|
@@ -52,10 +65,15 @@ const METADATA_SCHEMA = makeSchema([
|
|
|
52
65
|
field("parameters", list(field("item", PARAMETER_STRUCT, true)), false),
|
|
53
66
|
field("stability", utf8(), false),
|
|
54
67
|
field("null_handling", utf8(), false),
|
|
68
|
+
field("argument_monotonicity", list(field("item", utf8(), true)), true),
|
|
55
69
|
field("required_settings", list(field("item", utf8(), true)), false),
|
|
56
70
|
field("required_secrets", list(field("item", utf8(), true)), false),
|
|
57
71
|
field("projection_pushdown", bool(), false),
|
|
58
72
|
field("filter_pushdown", bool(), false),
|
|
73
|
+
field("filter_semantic_profiles", list(field("item", utf8(), true)), false),
|
|
74
|
+
field("additional_filter_functions", list(field("item", FILTER_IDENTITY_STRUCT, true)), false),
|
|
75
|
+
field("runtime_filter_algorithms", list(field("item", FILTER_IDENTITY_STRUCT, true)), false),
|
|
76
|
+
field("filter_evaluation_contexts", list(field("item", FILTER_CONTEXT_STRUCT, true)), false),
|
|
59
77
|
field("preserves_order", utf8(), false),
|
|
60
78
|
field("max_workers", int32(), true),
|
|
61
79
|
field("order_dependent", utf8(), false),
|
|
@@ -96,12 +114,23 @@ function metadataToRow(m: ResolvedMetadata): Record<string, any> {
|
|
|
96
114
|
})),
|
|
97
115
|
stability: m.stability,
|
|
98
116
|
null_handling: m.nullHandling,
|
|
117
|
+
argument_monotonicity: m.argumentMonotonicity,
|
|
99
118
|
required_settings: m.requiredSettings,
|
|
100
119
|
required_secrets: m.requiredSecrets,
|
|
101
120
|
projection_pushdown: m.projectionPushdown,
|
|
102
121
|
filter_pushdown: m.filterPushdown,
|
|
103
122
|
sampling_pushdown: m.samplingPushdown,
|
|
104
|
-
|
|
123
|
+
filter_semantic_profiles: m.filterSemanticProfiles,
|
|
124
|
+
additional_filter_functions: m.additionalFilterFunctions.map((value) => ({
|
|
125
|
+
...value, version: BigInt(value.version),
|
|
126
|
+
})),
|
|
127
|
+
runtime_filter_algorithms: m.runtimeFilterAlgorithms.map((value) => ({
|
|
128
|
+
...value, version: BigInt(value.version),
|
|
129
|
+
})),
|
|
130
|
+
filter_evaluation_contexts: m.filterEvaluationContexts.map((value) => ({
|
|
131
|
+
profile: value.profile,
|
|
132
|
+
provider_fingerprint: value.providerFingerprint,
|
|
133
|
+
})),
|
|
105
134
|
preserves_order: m.preservesOrder,
|
|
106
135
|
max_workers: m.maxWorkers,
|
|
107
136
|
order_dependent: m.orderDependent,
|
|
@@ -222,15 +251,24 @@ export function arrowToMetadatas(batch: VgiBatch): ResolvedMetadata[] {
|
|
|
222
251
|
parameters,
|
|
223
252
|
stability: (get("stability") as any) ?? "CONSISTENT",
|
|
224
253
|
nullHandling: (get("null_handling") as any) ?? "DEFAULT",
|
|
254
|
+
argumentMonotonicity: get("argument_monotonicity") == null
|
|
255
|
+
? null
|
|
256
|
+
: [...(get("argument_monotonicity") as any)].map(String) as ArgumentMonotonicity[],
|
|
225
257
|
requiredSettings,
|
|
226
258
|
requiredSecrets,
|
|
227
259
|
projectionPushdown: (get("projection_pushdown") as boolean) ?? false,
|
|
228
260
|
filterPushdown: (get("filter_pushdown") as boolean) ?? false,
|
|
229
261
|
samplingPushdown: (get("sampling_pushdown") as boolean) ?? false,
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
})
|
|
262
|
+
filterSemanticProfiles: [...((get("filter_semantic_profiles") as any) ?? [])].filter((s: any) => s != null).map(String),
|
|
263
|
+
additionalFilterFunctions: [...((get("additional_filter_functions") as any) ?? [])].filter((v: any) => v != null).map((v: any) => ({
|
|
264
|
+
namespace: String(v.namespace), name: String(v.name), version: Number(v.version),
|
|
265
|
+
})),
|
|
266
|
+
runtimeFilterAlgorithms: [...((get("runtime_filter_algorithms") as any) ?? [])].filter((v: any) => v != null).map((v: any) => ({
|
|
267
|
+
namespace: String(v.namespace), name: String(v.name), version: Number(v.version),
|
|
268
|
+
})),
|
|
269
|
+
filterEvaluationContexts: [...((get("filter_evaluation_contexts") as any) ?? [])].filter((v: any) => v != null).map((v: any) => ({
|
|
270
|
+
profile: String(v.profile), providerFingerprint: v.provider_fingerprint == null ? null : String(v.provider_fingerprint),
|
|
271
|
+
})),
|
|
234
272
|
preservesOrder: (get("preserves_order") as any) ?? "NO_ORDER_GUARANTEE",
|
|
235
273
|
maxWorkers: (get("max_workers") as number | null) ?? null,
|
|
236
274
|
orderDependent: (get("order_dependent") as any) ?? "NOT_ORDER_DEPENDENT",
|
package/src/metadata/types.ts
CHANGED
|
@@ -4,6 +4,7 @@
|
|
|
4
4
|
import {
|
|
5
5
|
FunctionStability,
|
|
6
6
|
NullHandling,
|
|
7
|
+
ArgumentMonotonicity,
|
|
7
8
|
OrderPreservation,
|
|
8
9
|
OrderDependence,
|
|
9
10
|
DistinctDependence,
|
|
@@ -36,6 +37,17 @@ export interface FunctionExample {
|
|
|
36
37
|
expectedOutput: string | null;
|
|
37
38
|
}
|
|
38
39
|
|
|
40
|
+
export interface FilterFunctionCapability {
|
|
41
|
+
namespace: string;
|
|
42
|
+
name: string;
|
|
43
|
+
version: number;
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
export interface EvaluationContextCapability {
|
|
47
|
+
profile: string;
|
|
48
|
+
providerFingerprint: string | null;
|
|
49
|
+
}
|
|
50
|
+
|
|
39
51
|
export interface ResolvedMetadata {
|
|
40
52
|
name: string;
|
|
41
53
|
className: string;
|
|
@@ -47,12 +59,16 @@ export interface ResolvedMetadata {
|
|
|
47
59
|
parameters: ParameterInfo[];
|
|
48
60
|
stability: FunctionStability;
|
|
49
61
|
nullHandling: NullHandling;
|
|
62
|
+
argumentMonotonicity: ArgumentMonotonicity[] | null;
|
|
50
63
|
requiredSettings: string[];
|
|
51
64
|
requiredSecrets: string[];
|
|
52
65
|
projectionPushdown: boolean;
|
|
53
66
|
filterPushdown: boolean;
|
|
54
67
|
samplingPushdown: boolean;
|
|
55
|
-
|
|
68
|
+
filterSemanticProfiles: string[];
|
|
69
|
+
additionalFilterFunctions: FilterFunctionCapability[];
|
|
70
|
+
runtimeFilterAlgorithms: FilterFunctionCapability[];
|
|
71
|
+
filterEvaluationContexts: EvaluationContextCapability[];
|
|
56
72
|
preservesOrder: OrderPreservation;
|
|
57
73
|
maxWorkers: number | null;
|
|
58
74
|
orderDependent: OrderDependence;
|
package/src/protocol/dispatch.ts
CHANGED
|
@@ -37,7 +37,7 @@ export interface ProtocolConfig {
|
|
|
37
37
|
}
|
|
38
38
|
|
|
39
39
|
export function buildVgiProtocol(config: ProtocolConfig): Protocol {
|
|
40
|
-
const protocol = new Protocol("vgi");
|
|
40
|
+
const protocol = new Protocol("vgi", { protocolVersion: "2.0.0" });
|
|
41
41
|
|
|
42
42
|
registerFunctionMethods(protocol, {
|
|
43
43
|
registry: config.registry,
|
|
@@ -20,8 +20,13 @@ import { deserializeArguments } from "../serialize.js";
|
|
|
20
20
|
import { deserializeSchema, serializeSchema, batchToSecretDict } from "../../util/arrow/index.js";
|
|
21
21
|
import { toUint8Array } from "../../util/bytes.js";
|
|
22
22
|
import {
|
|
23
|
+
AggregateBindParamsSchema,
|
|
23
24
|
AggregateBindResultSchema,
|
|
25
|
+
AggregateCombineParamsSchema,
|
|
26
|
+
AggregateDestructorParamsSchema,
|
|
27
|
+
AggregateFinalizeParamsSchema,
|
|
24
28
|
AggregateFinalizeResultSchema,
|
|
29
|
+
AggregateUpdateParamsSchema,
|
|
25
30
|
} from "../../generated/vgi-protocol-schemas.js";
|
|
26
31
|
import {
|
|
27
32
|
getExecutionState,
|
|
@@ -41,19 +46,19 @@ import {
|
|
|
41
46
|
} from "./shared.js";
|
|
42
47
|
|
|
43
48
|
export function registerAggregateMethods(protocol: Protocol, registry: FunctionRegistry): void {
|
|
44
|
-
// `
|
|
49
|
+
// `schemaPath` scopes resolution to the schema the caller named. Every
|
|
45
50
|
// aggregate RPC carries it (protocol 1.2.0), so an aggregate name declared in
|
|
46
51
|
// two schemas dispatches to the one the request names rather than colliding
|
|
47
52
|
// on the flat by-name index. `null` (no schema) keeps the cross-schema lookup.
|
|
48
53
|
function resolveAggregate(
|
|
49
54
|
name: string,
|
|
50
|
-
|
|
55
|
+
schemaPath?: string[] | null,
|
|
51
56
|
): AggregateFunctionConfig<any, any> {
|
|
52
57
|
const func = registry.get(name, {
|
|
53
58
|
arguments: new Arguments(),
|
|
54
59
|
inputSchema: null,
|
|
55
60
|
isScalar: false,
|
|
56
|
-
|
|
61
|
+
schemaPath: schemaPath ?? null,
|
|
57
62
|
}) as any;
|
|
58
63
|
if (!func || func.kind !== "aggregate") {
|
|
59
64
|
throw new Error(`Function '${name}' is not an aggregate function`);
|
|
@@ -65,12 +70,12 @@ export function registerAggregateMethods(protocol: Protocol, registry: FunctionR
|
|
|
65
70
|
// aggregate_bind — allocate execution_id, return output schema
|
|
66
71
|
// ------------------------------------------------------------------------
|
|
67
72
|
protocol.unary("aggregate_bind", {
|
|
68
|
-
params:
|
|
73
|
+
params: AggregateBindParamsSchema,
|
|
69
74
|
result: RESULT_BINARY_SCHEMA,
|
|
70
75
|
handler: async (params) => {
|
|
71
76
|
const innerParams = unwrapRequest(params.request);
|
|
72
77
|
const functionName: string = innerParams.function_name;
|
|
73
|
-
const cfg = resolveAggregate(functionName, innerParams.
|
|
78
|
+
const cfg = resolveAggregate(functionName, innerParams.schema_path ?? null);
|
|
74
79
|
const argBytes = toUint8Array(innerParams.arguments);
|
|
75
80
|
const args = deserializeArguments(argBytes);
|
|
76
81
|
const inputSchema = innerParams.input_schema
|
|
@@ -90,6 +95,12 @@ export function registerAggregateMethods(protocol: Protocol, registry: FunctionR
|
|
|
90
95
|
inputSchema,
|
|
91
96
|
settings: {},
|
|
92
97
|
secrets,
|
|
98
|
+
argumentNames: innerParams.argument_names
|
|
99
|
+
? (Array.isArray(innerParams.argument_names)
|
|
100
|
+
? innerParams.argument_names
|
|
101
|
+
: [...innerParams.argument_names])
|
|
102
|
+
.map((value: unknown) => value == null ? null : String(value))
|
|
103
|
+
: null,
|
|
93
104
|
};
|
|
94
105
|
const executionId = new Uint8Array(16);
|
|
95
106
|
crypto.getRandomValues(executionId);
|
|
@@ -115,13 +126,13 @@ export function registerAggregateMethods(protocol: Protocol, registry: FunctionR
|
|
|
115
126
|
// aggregate_update — fold the incoming batch into per-group state
|
|
116
127
|
// ------------------------------------------------------------------------
|
|
117
128
|
protocol.unary("aggregate_update", {
|
|
118
|
-
params:
|
|
129
|
+
params: AggregateUpdateParamsSchema,
|
|
119
130
|
result: RESULT_BINARY_SCHEMA,
|
|
120
131
|
handler: (params) => {
|
|
121
132
|
const innerParams = unwrapRequest(params.request);
|
|
122
133
|
const functionName: string = innerParams.function_name;
|
|
123
134
|
const executionId = toUint8Array(innerParams.execution_id);
|
|
124
|
-
const cfg = resolveAggregate(functionName, innerParams.
|
|
135
|
+
const cfg = resolveAggregate(functionName, innerParams.schema_path ?? null);
|
|
125
136
|
const batch = readSingleBatch(toUint8Array(innerParams.input_batch));
|
|
126
137
|
if (!batch || batch.numRows === 0) return wrapResult({}, makeSchema([]));
|
|
127
138
|
|
|
@@ -182,13 +193,13 @@ export function registerAggregateMethods(protocol: Protocol, registry: FunctionR
|
|
|
182
193
|
// aggregate_combine — merge (source_group_id, target_group_id) pairs
|
|
183
194
|
// ------------------------------------------------------------------------
|
|
184
195
|
protocol.unary("aggregate_combine", {
|
|
185
|
-
params:
|
|
196
|
+
params: AggregateCombineParamsSchema,
|
|
186
197
|
result: RESULT_BINARY_SCHEMA,
|
|
187
198
|
handler: (params) => {
|
|
188
199
|
const innerParams = unwrapRequest(params.request);
|
|
189
200
|
const functionName: string = innerParams.function_name;
|
|
190
201
|
const executionId = toUint8Array(innerParams.execution_id);
|
|
191
|
-
const cfg = resolveAggregate(functionName, innerParams.
|
|
202
|
+
const cfg = resolveAggregate(functionName, innerParams.schema_path ?? null);
|
|
192
203
|
const batch = readSingleBatch(toUint8Array(innerParams.merge_batch));
|
|
193
204
|
if (!batch || batch.numRows === 0) return wrapResult({}, makeSchema([]));
|
|
194
205
|
|
|
@@ -231,13 +242,13 @@ export function registerAggregateMethods(protocol: Protocol, registry: FunctionR
|
|
|
231
242
|
// aggregate_finalize — emit one result row per requested group_id
|
|
232
243
|
// ------------------------------------------------------------------------
|
|
233
244
|
protocol.unary("aggregate_finalize", {
|
|
234
|
-
params:
|
|
245
|
+
params: AggregateFinalizeParamsSchema,
|
|
235
246
|
result: RESULT_BINARY_SCHEMA,
|
|
236
247
|
handler: (params) => {
|
|
237
248
|
const innerParams = unwrapRequest(params.request);
|
|
238
249
|
const functionName: string = innerParams.function_name;
|
|
239
250
|
const executionId = toUint8Array(innerParams.execution_id);
|
|
240
|
-
const cfg = resolveAggregate(functionName, innerParams.
|
|
251
|
+
const cfg = resolveAggregate(functionName, innerParams.schema_path ?? null);
|
|
241
252
|
const outputSchema = deserializeSchema(toUint8Array(innerParams.output_schema));
|
|
242
253
|
const gidBatch = readSingleBatch(toUint8Array(innerParams.group_ids_batch));
|
|
243
254
|
const groupIds: bigint[] = [];
|
|
@@ -281,7 +292,7 @@ export function registerAggregateMethods(protocol: Protocol, registry: FunctionR
|
|
|
281
292
|
// up when the worker process exits).
|
|
282
293
|
// ------------------------------------------------------------------------
|
|
283
294
|
protocol.unary("aggregate_destructor", {
|
|
284
|
-
params:
|
|
295
|
+
params: AggregateDestructorParamsSchema,
|
|
285
296
|
result: RESULT_BINARY_SCHEMA,
|
|
286
297
|
handler: (params) => {
|
|
287
298
|
const innerParams = unwrapRequest(params.request);
|