@query-farm/vgi 0.28.0 → 0.29.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (121) hide show
  1. package/README.md +52 -8
  2. package/dist/arguments/argument-spec.d.ts +57 -2
  3. package/dist/arrow/impl-arrowjs/index.d.ts +1 -1
  4. package/dist/arrow/impl-arrowjs/iterate.d.ts +10 -1
  5. package/dist/arrow/impl-arrowjs/schema.d.ts +2 -0
  6. package/dist/arrow/impl-flechette/index.d.ts +1 -1
  7. package/dist/arrow/impl-flechette/iterate.d.ts +10 -1
  8. package/dist/arrow/impl-flechette/schema.d.ts +2 -0
  9. package/dist/arrow/index.d.ts +1 -1
  10. package/dist/arrow/schema-types.d.ts +3 -1
  11. package/dist/arrow/types.d.ts +39 -3
  12. package/dist/catalog/composite.d.ts +13 -13
  13. package/dist/catalog/descriptors.d.ts +7 -3
  14. package/dist/catalog/interface.d.ts +108 -38
  15. package/dist/catalog/read-only.d.ts +32 -15
  16. package/dist/client/client.d.ts +37 -30
  17. package/dist/client/types.d.ts +2 -0
  18. package/dist/client-entry.d.ts +1 -1
  19. package/dist/client-entry.js +791 -447
  20. package/dist/client-entry.js.map +18 -17
  21. package/dist/filter-pushdown/capabilities.d.ts +3 -0
  22. package/dist/filter-pushdown/deserialize.d.ts +14 -15
  23. package/dist/filter-pushdown/evaluate.d.ts +14 -44
  24. package/dist/filter-pushdown/index.d.ts +2 -2
  25. package/dist/filter-pushdown/types.d.ts +120 -68
  26. package/dist/functions/aggregate.d.ts +4 -0
  27. package/dist/functions/registry.d.ts +5 -5
  28. package/dist/functions/scalar.d.ts +6 -1
  29. package/dist/functions/table-buffering.d.ts +9 -0
  30. package/dist/functions/table-in-out.d.ts +13 -0
  31. package/dist/functions/table.d.ts +72 -2
  32. package/dist/functions/types.d.ts +60 -2
  33. package/dist/generated/vgi-client.d.ts +114 -26
  34. package/dist/generated/vgi-protocol-schemas.d.ts +34 -0
  35. package/dist/http/fetch.d.ts +16 -0
  36. package/dist/index.core.d.ts +4 -3
  37. package/dist/index.js +3396 -2064
  38. package/dist/index.js.map +55 -49
  39. package/dist/metadata/types.d.ts +15 -2
  40. package/dist/protocol/handlers/catalog/shared.d.ts +0 -8
  41. package/dist/protocol/handlers/function.d.ts +11 -0
  42. package/dist/protocol/handlers/shared.d.ts +3 -1
  43. package/dist/protocol/serializers/bind.d.ts +1 -2
  44. package/dist/protocol/serializers/cardinality.d.ts +1 -2
  45. package/dist/protocol/serializers/splits.d.ts +126 -0
  46. package/dist/protocol/types.d.ts +36 -3
  47. package/dist/request-auth.d.ts +28 -0
  48. package/dist/schema-path.d.ts +7 -0
  49. package/dist/serve-entry.d.ts +25 -0
  50. package/dist/serve-entry.js +1129 -569
  51. package/dist/serve-entry.js.map +36 -32
  52. package/dist/split-token.d.ts +88 -0
  53. package/dist/types.d.ts +8 -0
  54. package/dist/worker-cf-entry.js +2928 -1689
  55. package/dist/worker-cf-entry.js.map +51 -44
  56. package/dist/worker.d.ts +1 -1
  57. package/package.json +3 -3
  58. package/src/arguments/argument-spec.ts +93 -4
  59. package/src/arguments/arguments.ts +0 -1
  60. package/src/arrow/impl-arrowjs/index.ts +1 -1
  61. package/src/arrow/impl-arrowjs/iterate.ts +21 -0
  62. package/src/arrow/impl-arrowjs/schema.ts +4 -0
  63. package/src/arrow/impl-flechette/index.ts +1 -1
  64. package/src/arrow/impl-flechette/iterate.ts +24 -1
  65. package/src/arrow/impl-flechette/schema.ts +4 -0
  66. package/src/arrow/index.ts +1 -1
  67. package/src/arrow/schema-types.ts +4 -0
  68. package/src/arrow/types.ts +39 -3
  69. package/src/catalog/composite.ts +26 -26
  70. package/src/catalog/descriptors.ts +12 -3
  71. package/src/catalog/interface.ts +217 -72
  72. package/src/catalog/read-only.ts +141 -84
  73. package/src/client/client.ts +97 -73
  74. package/src/client/types.ts +2 -0
  75. package/src/client-entry.ts +2 -1
  76. package/src/filter-pushdown/capabilities.ts +13 -0
  77. package/src/filter-pushdown/collector.ts +22 -35
  78. package/src/filter-pushdown/deserialize.ts +657 -221
  79. package/src/filter-pushdown/evaluate.ts +410 -499
  80. package/src/filter-pushdown/index.ts +16 -2
  81. package/src/filter-pushdown/types.ts +88 -71
  82. package/src/functions/aggregate.ts +12 -1
  83. package/src/functions/copy-from.ts +2 -1
  84. package/src/functions/copy-to.ts +2 -1
  85. package/src/functions/registry.ts +24 -22
  86. package/src/functions/scalar.ts +37 -0
  87. package/src/functions/table-buffering.ts +48 -6
  88. package/src/functions/table-in-out.ts +194 -33
  89. package/src/functions/table.ts +187 -22
  90. package/src/functions/types.ts +64 -1
  91. package/src/generated/vgi-client.ts +122 -28
  92. package/src/generated/vgi-protocol-schemas.ts +455 -58
  93. package/src/http/fetch.ts +129 -2
  94. package/src/index.core.ts +4 -2
  95. package/src/metadata/resolve.ts +43 -1
  96. package/src/metadata/serialize.ts +43 -5
  97. package/src/metadata/types.ts +17 -1
  98. package/src/protocol/dispatch.ts +1 -1
  99. package/src/protocol/handlers/aggregate.ts +23 -12
  100. package/src/protocol/handlers/catalog/admin.ts +47 -54
  101. package/src/protocol/handlers/catalog/index_methods.ts +6 -13
  102. package/src/protocol/handlers/catalog/macro.ts +12 -20
  103. package/src/protocol/handlers/catalog/shared.ts +12 -57
  104. package/src/protocol/handlers/catalog/table.ts +49 -112
  105. package/src/protocol/handlers/catalog/view.ts +18 -22
  106. package/src/protocol/handlers/function.ts +311 -18
  107. package/src/protocol/handlers/shared.ts +12 -3
  108. package/src/protocol/handlers/table-buffering.ts +11 -8
  109. package/src/protocol/serializers/bind.ts +69 -67
  110. package/src/protocol/serializers/cardinality.ts +7 -7
  111. package/src/protocol/serializers/init.ts +52 -10
  112. package/src/protocol/serializers/splits.ts +221 -0
  113. package/src/protocol/state-serializer.ts +55 -1
  114. package/src/protocol/types.ts +36 -3
  115. package/src/request-auth.ts +73 -0
  116. package/src/schema-path.ts +25 -0
  117. package/src/serve-entry.ts +69 -2
  118. package/src/split-token.ts +297 -0
  119. package/src/types.ts +9 -0
  120. package/src/worker-cf-entry.ts +2 -1
  121. package/src/worker.ts +79 -14
package/src/http/fetch.ts CHANGED
@@ -10,10 +10,23 @@
10
10
  // `buildVgiProtocol` as `signingKey`, from one source. Passing different keys to
11
11
  // those two seams produces a worker that mints tokens it cannot recover.
12
12
 
13
- import { createHttpHandler, unpackStateToken, type Protocol } from "@query-farm/vgi-rpc";
13
+ import {
14
+ AuthContext,
15
+ createHttpHandler,
16
+ unpackStateToken,
17
+ type AuthenticateFn,
18
+ type DispatchHook,
19
+ type DispatchInfo,
20
+ type HookToken,
21
+ type PeerAuthenticationPolicy,
22
+ type PeerIdentityProvider,
23
+ type PeerResolutionOptions,
24
+ type Protocol,
25
+ } from "@query-farm/vgi-rpc";
14
26
  import { arrowStateSerializer } from "../protocol/state-serializer.js";
15
27
  import { buildVgiProtocol, type ProtocolConfig } from "../protocol/dispatch.js";
16
28
  import { createLandingRoutes, type LandingInfo } from "./landing.js";
29
+ import { setRequestAuthScope, type RequestAuthHolder } from "../request-auth.js";
17
30
 
18
31
  export interface VgiFetchOptions {
19
32
  /** Wire-protocol config (registry + catalogInterface). */
@@ -57,8 +70,106 @@ export interface VgiFetchOptions {
57
70
  * its required name/doc/version; the CF entry had to remember, and
58
71
  * vgi-open-meteo shipped for weeks without it. */
59
72
  landingInfo: LandingInfo;
73
+ /** Authenticates each incoming request, returning the caller's
74
+ * {@link AuthContext}. Forwarded verbatim to `createHttpHandler`; omit it
75
+ * and every caller is anonymous, which is what this package did before the
76
+ * option existed.
77
+ *
78
+ * Supplying it is what makes a worker able to serve more than one principal,
79
+ * and the identity it returns is the one the worker seals `attach_opaque_data`
80
+ * and split tokens under. */
81
+ authenticate?: AuthenticateFn;
82
+ /** Off-wire transport identities, including a trusted Iroh HTTP bridge. */
83
+ peerIdentityProviders?: readonly PeerIdentityProvider[];
84
+ /** Policy composing peer evidence with application authentication. */
85
+ peerAuthenticationPolicy?: PeerAuthenticationPolicy;
86
+ /** Host adapter supplying the physical peer and multiplicity-preserving headers. */
87
+ peerResolutionContext?: (request: Request) => PeerResolutionOptions | Promise<PeerResolutionOptions>;
88
+ }
89
+
90
+ // ---------------------------------------------------------------------------
91
+ // Request-scoped caller identity
92
+ //
93
+ // `@query-farm/vgi-rpc` types stream initialization as `(params) => state` —
94
+ // no CallContext — on both dispatchers, so the `init` handler cannot be handed
95
+ // the principal as an argument the way every unary handler is. But `init` is
96
+ // exactly where the identity-bound envelopes minted by `catalog_attach` and
97
+ // `table_function_plan` (both unary, both sealed under the REAL caller) are
98
+ // opened. Opening them under the anonymous identity works for anonymous
99
+ // callers and fails for everyone else.
100
+ //
101
+ // So the identity travels the way vgi-python's `current_auth()` ContextVar
102
+ // does: an ambient per-request scope. `AsyncLocalStorage.run` wraps the whole
103
+ // handler call, and a dispatch hook fills the holder in with whatever identity
104
+ // the framework actually dispatched under — the hook is read AFTER
105
+ // authentication, the PKCE cookie chain and bearer introspection have all had
106
+ // their say, so it reports the final principal rather than re-deriving one.
107
+ //
108
+ // A mutable holder rather than `enterWith`: `run` is the only form Cloudflare
109
+ // Workers supports, and the identity is not known until authentication has run
110
+ // inside the handler.
111
+ // ---------------------------------------------------------------------------
112
+
113
+ /** The `AsyncLocalStorage` surface used here, structurally typed so this file
114
+ * compiles where `node:async_hooks` has no types. */
115
+ interface AsyncLocalStorageLike<T> {
116
+ run<R>(store: T, fn: () => R): R;
117
+ getStore(): T | undefined;
60
118
  }
61
119
 
120
+ let requestAuthStorage: AsyncLocalStorageLike<RequestAuthHolder> | null | undefined;
121
+ let requestAuthStoragePending: Promise<AsyncLocalStorageLike<RequestAuthHolder> | null> | undefined;
122
+
123
+ /**
124
+ * Resolve (once) the AsyncLocalStorage this process scopes requests with.
125
+ *
126
+ * `node:async_hooks` is loaded dynamically and defensively: it exists on Node
127
+ * and Bun, and on Cloudflare Workers only with the `nodejs_compat` /
128
+ * `nodejs_als` compatibility flag. Where it is missing this resolves to `null`
129
+ * and the worker keeps serving exactly as it did before — anonymous callers
130
+ * unaffected, authenticated ones still unable to open their sealed attach.
131
+ * Degrading is the right failure here: refusing to start would break every
132
+ * anonymous deployment to fix a case it does not have.
133
+ */
134
+ function ensureRequestAuthStorage(): Promise<AsyncLocalStorageLike<RequestAuthHolder> | null> {
135
+ if (requestAuthStorage !== undefined) return Promise.resolve(requestAuthStorage);
136
+ requestAuthStoragePending ??= import("node:async_hooks")
137
+ .then((mod) => {
138
+ const storage = new mod.AsyncLocalStorage<RequestAuthHolder>() as AsyncLocalStorageLike<RequestAuthHolder>;
139
+ requestAuthStorage = storage;
140
+ setRequestAuthScope(storage);
141
+ return storage;
142
+ })
143
+ .catch(() => {
144
+ requestAuthStorage = null;
145
+ return null;
146
+ });
147
+ return requestAuthStoragePending;
148
+ }
149
+
150
+ /**
151
+ * Dispatch hook that records the dispatching principal on the open scope.
152
+ *
153
+ * Reads the identity off `DispatchInfo`, which the framework populates from the
154
+ * resolved auth context for every method — so this sees the same principal the
155
+ * access log and the unary handlers see, without this package having to know
156
+ * how authentication was configured.
157
+ */
158
+ const REQUEST_AUTH_DISPATCH_HOOK: DispatchHook = {
159
+ onDispatchStart(info: DispatchInfo): HookToken {
160
+ const holder = requestAuthStorage?.getStore();
161
+ if (holder) {
162
+ holder.auth = info.authenticated
163
+ ? new AuthContext(info.authDomain ?? "", true, info.principal ?? "", info.claims ?? {})
164
+ : AuthContext.anonymous();
165
+ }
166
+ return undefined;
167
+ },
168
+ onDispatchEnd() {
169
+ // Nothing to record; the scope ends when the request does.
170
+ },
171
+ };
172
+
62
173
  /**
63
174
  * Build a fetch handler suitable for `export default { fetch }` in a CF
64
175
  * Worker module, or for `Bun.serve({ fetch })`. The returned handler is
@@ -100,6 +211,14 @@ export function createVgiFetch(opts: VgiFetchOptions): (req: Request) => Promise
100
211
  tokenKey: opts.signingKey,
101
212
  tokenTtl,
102
213
  stateSerializer: arrowStateSerializer,
214
+ // Records the dispatching principal on the request scope opened below, so
215
+ // stream `init` — the one handler the framework passes no CallContext to —
216
+ // can still open the identity-bound attach and split envelopes.
217
+ dispatchHook: REQUEST_AUTH_DISPATCH_HOOK,
218
+ authenticate: opts.authenticate,
219
+ peerIdentityProviders: opts.peerIdentityProviders,
220
+ peerAuthenticationPolicy: opts.peerAuthenticationPolicy,
221
+ peerResolutionContext: opts.peerResolutionContext,
103
222
  // `null` is the explicit opt-out; `undefined` (omitted) means "*".
104
223
  corsOrigins: opts.corsOrigins === null ? undefined : (opts.corsOrigins ?? "*"),
105
224
  repositoryUrl: opts.repositoryUrl,
@@ -110,5 +229,13 @@ export function createVgiFetch(opts: VgiFetchOptions): (req: Request) => Promise
110
229
  extraRoutes: createLandingRoutes(opts.landingInfo),
111
230
  enableLandingPage: false,
112
231
  });
113
- return async (req: Request) => handler(req);
232
+ return async (req: Request) => {
233
+ const storage = await ensureRequestAuthStorage();
234
+ // No AsyncLocalStorage on this host: serve exactly as before.
235
+ if (!storage) return handler(req);
236
+ // One holder per request. The dispatch hook fills it in once the framework
237
+ // has resolved the principal; `init` reads it back through
238
+ // `currentRequestAuth()`.
239
+ return storage.run({}, () => handler(req));
240
+ };
114
241
  }
package/src/index.core.ts CHANGED
@@ -77,7 +77,7 @@ export {
77
77
  export type { VgiFunction, FunctionMeta, StreamHandlers, FunctionExample, HandlerState } from "./functions/types.js";
78
78
 
79
79
  // Catalog
80
- export { CatalogInterface, type CatalogAttachResult, type SchemaInfo, encodeSchemaInfo, decodeSchemaInfo, type TableInfo, encodeTableInfo, decodeTableInfo, type ViewInfo, encodeViewInfo, decodeViewInfo, type CatalogInfo, encodeCatalogInfo, decodeCatalogInfo, type FunctionInfo, encodeFunctionInfo, decodeFunctionInfo, type FunctionInfoOptions, type MacroInfo, encodeMacroInfo, decodeMacroInfo, type MacroType, type CopyFromFormatInfo, encodeCopyFromFormatInfo, type AttachOpaqueData, type TransactionOpaqueData, buildScanBranchesResult, type ScanBranchInput } from "./catalog/interface.js";
80
+ export { CatalogInterface, type CatalogAttachResult, type SchemaInfo, encodeSchemaInfo, decodeSchemaInfo, type TableInfo, encodeTableInfo, decodeTableInfo, type ViewInfo, encodeViewInfo, decodeViewInfo, type CatalogInfo, encodeCatalogInfo, decodeCatalogInfo, type FunctionInfo, encodeFunctionInfo, decodeFunctionInfo, type FunctionInfoOptions, type MacroInfo, encodeMacroInfo, decodeMacroInfo, type MacroType, type CopyFromFormatInfo, encodeCopyFromFormatInfo, type AttachOpaqueData, type TransactionOpaqueData, buildScanBranchesResult, encodeFormatOptions, type ScanBranchInput } from "./catalog/interface.js";
81
81
  export type { CatalogDescriptor, SchemaDescriptor, TableDescriptor, ViewDescriptor, MacroDescriptor, SettingDescriptor, SecretTypeDescriptor, ForeignKeyDef, DefaultValue } from "./catalog/descriptors.js";
82
82
  export { ReadOnlyCatalogInterface } from "./catalog/read-only.js";
83
83
  export { CompositeCatalogInterface } from "./catalog/composite.js";
@@ -88,6 +88,7 @@ export {
88
88
  FunctionType,
89
89
  FunctionStability,
90
90
  NullHandling,
91
+ ArgumentMonotonicity,
91
92
  OrderPreservation,
92
93
  OrderDependence,
93
94
  DistinctDependence,
@@ -219,7 +220,7 @@ export {
219
220
  int8, int16, int64,
220
221
  uint8, uint16, uint32, uint64,
221
222
  float16, float64,
222
- utf8, binary, fixedSizeBinary,
223
+ utf8, largeUtf8, binary, largeBinary, fixedSizeBinary,
223
224
  decimal, decimal128, decimal256,
224
225
  date, dateDay, dateMillisecond,
225
226
  time, timeSecond, timeMillisecond, timeMicrosecond, timeNanosecond,
@@ -307,3 +308,4 @@ export type {
307
308
  export { str, bytes, int, int32, float, float32, bool, toSchema, OutputCollector, AuthContext } from "@query-farm/vgi-rpc";
308
309
  export { subprocessConnect, httpConnect, tcpConnect } from "@query-farm/vgi-rpc";
309
310
  export type { RpcClient, StreamSession, LogMessage } from "@query-farm/vgi-rpc";
311
+ export { normalizeSchemaPath, schemaPathDisplay, schemaPathKey, schemaPathsEqual, type SchemaPath } from "./schema-path.js";
@@ -4,6 +4,7 @@
4
4
  import {
5
5
  FunctionStability,
6
6
  NullHandling,
7
+ ArgumentMonotonicity,
7
8
  OrderPreservation,
8
9
  OrderDependence,
9
10
  DistinctDependence,
@@ -11,6 +12,7 @@ import {
11
12
  } from "../types.js";
12
13
  import type { VgiFunction } from "../functions/types.js";
13
14
  import type { ArgumentSpec } from "../arguments/argument-spec.js";
15
+ import { supportsExtensionFilterFunction } from "../filter-pushdown/capabilities.js";
14
16
  import {
15
17
  CatalogFunctionType,
16
18
  type ResolvedMetadata,
@@ -55,6 +57,19 @@ function specToParameterInfo(spec: ArgumentSpec): ParameterInfo {
55
57
  export function resolveMetadata(func: VgiFunction): ResolvedMetadata {
56
58
  const meta = func.meta;
57
59
 
60
+ if (meta.runtimeFilterAlgorithms?.length) {
61
+ throw new Error("TypeScript SDK has no registered runtime-filter artifact evaluator to advertise");
62
+ }
63
+ if (meta.filterEvaluationContexts?.length) {
64
+ throw new Error("TypeScript SDK has no isolated DuckDB session-context evaluator to advertise");
65
+ }
66
+ if (meta.filterSemanticProfiles?.some((value) => value !== "vgi.duckdb.standard.v1")) {
67
+ throw new Error("TypeScript SDK supports only vgi.duckdb.standard.v1 filter semantics");
68
+ }
69
+ if (meta.additionalFilterFunctions?.some((value) => !supportsExtensionFilterFunction(value))) {
70
+ throw new Error("TypeScript SDK cannot advertise an extension-filter function without an evaluator");
71
+ }
72
+
58
73
  let functionType: CatalogFunctionType;
59
74
  switch (func.kind) {
60
75
  case "scalar":
@@ -74,8 +89,28 @@ export function resolveMetadata(func: VgiFunction): ResolvedMetadata {
74
89
  functionType = CatalogFunctionType.TABLE;
75
90
  }
76
91
 
92
+ const argumentMonotonicity = meta.argumentMonotonicity ?? null;
93
+ if (argumentMonotonicity !== null && functionType !== CatalogFunctionType.SCALAR) {
94
+ throw new Error("argumentMonotonicity is only valid for scalar functions");
95
+ }
96
+
77
97
  const parameters: ParameterInfo[] = func.argumentSpecs.map(specToParameterInfo);
78
98
 
99
+ if (argumentMonotonicity !== null) {
100
+ if (argumentMonotonicity.length !== parameters.length) {
101
+ throw new Error(
102
+ `argumentMonotonicity has ${argumentMonotonicity.length} entries, ` +
103
+ `expected ${parameters.length} declaration slots`,
104
+ );
105
+ }
106
+ const allowed = new Set<string>(Object.values(ArgumentMonotonicity));
107
+ for (const [index, value] of argumentMonotonicity.entries()) {
108
+ if (!allowed.has(value)) {
109
+ throw new Error(`argumentMonotonicity[${index}] has unknown value ${String(value)}`);
110
+ }
111
+ }
112
+ }
113
+
79
114
  const examples: FunctionExample[] = (meta.examples ?? []).map((e) => ({
80
115
  sql: e.sql,
81
116
  description: e.description,
@@ -95,12 +130,19 @@ export function resolveMetadata(func: VgiFunction): ResolvedMetadata {
95
130
  parameters,
96
131
  stability: meta.stability ?? FunctionStability.CONSISTENT,
97
132
  nullHandling: meta.nullHandling ?? NullHandling.DEFAULT,
133
+ argumentMonotonicity,
98
134
  requiredSettings: meta.requiredSettings ?? [],
99
135
  requiredSecrets: meta.requiredSecrets ?? [],
100
136
  projectionPushdown: meta.projectionPushdown ?? false,
101
137
  filterPushdown: meta.filterPushdown ?? false,
102
138
  samplingPushdown: meta.samplingPushdown ?? false,
103
- supportedExpressionFilters: meta.supportedExpressionFilters ?? [],
139
+ filterSemanticProfiles: meta.filterSemanticProfiles ?? (meta.filterPushdown ? ["vgi.duckdb.standard.v1"] : []),
140
+ additionalFilterFunctions: meta.additionalFilterFunctions ?? [],
141
+ runtimeFilterAlgorithms: meta.runtimeFilterAlgorithms ?? [],
142
+ filterEvaluationContexts: (meta.filterEvaluationContexts ?? []).map((value) => ({
143
+ profile: value.profile,
144
+ providerFingerprint: value.providerFingerprint ?? null,
145
+ })),
104
146
  preservesOrder: meta.preservesOrder ?? OrderPreservation.NO_ORDER_GUARANTEE,
105
147
  maxWorkers: maxWorkers === DEFAULT_MAX_WORKERS ? null : maxWorkers,
106
148
  orderDependent: meta.orderDependent ?? OrderDependence.NOT_ORDER_DEPENDENT,
@@ -8,6 +8,7 @@ import {
8
8
  field,
9
9
  utf8,
10
10
  int32,
11
+ uint64,
11
12
  bool,
12
13
  list,
13
14
  struct as makeStruct,
@@ -16,6 +17,7 @@ import {
16
17
  emptyBatch,
17
18
  } from "../arrow/index.js";
18
19
  import type { ResolvedMetadata, ParameterInfo, FunctionExample } from "./types.js";
20
+ import type { ArgumentMonotonicity } from "../types.js";
19
21
 
20
22
  // ============================================================================
21
23
  // Schema definitions matching Python's _METADATA_SCHEMA exactly
@@ -41,6 +43,17 @@ const PARAMETER_STRUCT = makeStruct([
41
43
  field("is_const", bool(), false),
42
44
  ]);
43
45
 
46
+ const FILTER_IDENTITY_STRUCT = makeStruct([
47
+ field("namespace", utf8(), false),
48
+ field("name", utf8(), false),
49
+ field("version", uint64(), false),
50
+ ]);
51
+
52
+ const FILTER_CONTEXT_STRUCT = makeStruct([
53
+ field("profile", utf8(), false),
54
+ field("provider_fingerprint", utf8(), true),
55
+ ]);
56
+
44
57
  const METADATA_SCHEMA = makeSchema([
45
58
  field("name", utf8(), false),
46
59
  field("class_name", utf8(), false),
@@ -52,10 +65,15 @@ const METADATA_SCHEMA = makeSchema([
52
65
  field("parameters", list(field("item", PARAMETER_STRUCT, true)), false),
53
66
  field("stability", utf8(), false),
54
67
  field("null_handling", utf8(), false),
68
+ field("argument_monotonicity", list(field("item", utf8(), true)), true),
55
69
  field("required_settings", list(field("item", utf8(), true)), false),
56
70
  field("required_secrets", list(field("item", utf8(), true)), false),
57
71
  field("projection_pushdown", bool(), false),
58
72
  field("filter_pushdown", bool(), false),
73
+ field("filter_semantic_profiles", list(field("item", utf8(), true)), false),
74
+ field("additional_filter_functions", list(field("item", FILTER_IDENTITY_STRUCT, true)), false),
75
+ field("runtime_filter_algorithms", list(field("item", FILTER_IDENTITY_STRUCT, true)), false),
76
+ field("filter_evaluation_contexts", list(field("item", FILTER_CONTEXT_STRUCT, true)), false),
59
77
  field("preserves_order", utf8(), false),
60
78
  field("max_workers", int32(), true),
61
79
  field("order_dependent", utf8(), false),
@@ -96,12 +114,23 @@ function metadataToRow(m: ResolvedMetadata): Record<string, any> {
96
114
  })),
97
115
  stability: m.stability,
98
116
  null_handling: m.nullHandling,
117
+ argument_monotonicity: m.argumentMonotonicity,
99
118
  required_settings: m.requiredSettings,
100
119
  required_secrets: m.requiredSecrets,
101
120
  projection_pushdown: m.projectionPushdown,
102
121
  filter_pushdown: m.filterPushdown,
103
122
  sampling_pushdown: m.samplingPushdown,
104
- supported_expression_filters: m.supportedExpressionFilters,
123
+ filter_semantic_profiles: m.filterSemanticProfiles,
124
+ additional_filter_functions: m.additionalFilterFunctions.map((value) => ({
125
+ ...value, version: BigInt(value.version),
126
+ })),
127
+ runtime_filter_algorithms: m.runtimeFilterAlgorithms.map((value) => ({
128
+ ...value, version: BigInt(value.version),
129
+ })),
130
+ filter_evaluation_contexts: m.filterEvaluationContexts.map((value) => ({
131
+ profile: value.profile,
132
+ provider_fingerprint: value.providerFingerprint,
133
+ })),
105
134
  preserves_order: m.preservesOrder,
106
135
  max_workers: m.maxWorkers,
107
136
  order_dependent: m.orderDependent,
@@ -222,15 +251,24 @@ export function arrowToMetadatas(batch: VgiBatch): ResolvedMetadata[] {
222
251
  parameters,
223
252
  stability: (get("stability") as any) ?? "CONSISTENT",
224
253
  nullHandling: (get("null_handling") as any) ?? "DEFAULT",
254
+ argumentMonotonicity: get("argument_monotonicity") == null
255
+ ? null
256
+ : [...(get("argument_monotonicity") as any)].map(String) as ArgumentMonotonicity[],
225
257
  requiredSettings,
226
258
  requiredSecrets,
227
259
  projectionPushdown: (get("projection_pushdown") as boolean) ?? false,
228
260
  filterPushdown: (get("filter_pushdown") as boolean) ?? false,
229
261
  samplingPushdown: (get("sampling_pushdown") as boolean) ?? false,
230
- supportedExpressionFilters: (() => {
231
- const raw = get("supported_expression_filters") as any ?? [];
232
- return raw ? [...raw].filter((s: any) => s != null).map(String) : [];
233
- })(),
262
+ filterSemanticProfiles: [...((get("filter_semantic_profiles") as any) ?? [])].filter((s: any) => s != null).map(String),
263
+ additionalFilterFunctions: [...((get("additional_filter_functions") as any) ?? [])].filter((v: any) => v != null).map((v: any) => ({
264
+ namespace: String(v.namespace), name: String(v.name), version: Number(v.version),
265
+ })),
266
+ runtimeFilterAlgorithms: [...((get("runtime_filter_algorithms") as any) ?? [])].filter((v: any) => v != null).map((v: any) => ({
267
+ namespace: String(v.namespace), name: String(v.name), version: Number(v.version),
268
+ })),
269
+ filterEvaluationContexts: [...((get("filter_evaluation_contexts") as any) ?? [])].filter((v: any) => v != null).map((v: any) => ({
270
+ profile: String(v.profile), providerFingerprint: v.provider_fingerprint == null ? null : String(v.provider_fingerprint),
271
+ })),
234
272
  preservesOrder: (get("preserves_order") as any) ?? "NO_ORDER_GUARANTEE",
235
273
  maxWorkers: (get("max_workers") as number | null) ?? null,
236
274
  orderDependent: (get("order_dependent") as any) ?? "NOT_ORDER_DEPENDENT",
@@ -4,6 +4,7 @@
4
4
  import {
5
5
  FunctionStability,
6
6
  NullHandling,
7
+ ArgumentMonotonicity,
7
8
  OrderPreservation,
8
9
  OrderDependence,
9
10
  DistinctDependence,
@@ -36,6 +37,17 @@ export interface FunctionExample {
36
37
  expectedOutput: string | null;
37
38
  }
38
39
 
40
+ export interface FilterFunctionCapability {
41
+ namespace: string;
42
+ name: string;
43
+ version: number;
44
+ }
45
+
46
+ export interface EvaluationContextCapability {
47
+ profile: string;
48
+ providerFingerprint: string | null;
49
+ }
50
+
39
51
  export interface ResolvedMetadata {
40
52
  name: string;
41
53
  className: string;
@@ -47,12 +59,16 @@ export interface ResolvedMetadata {
47
59
  parameters: ParameterInfo[];
48
60
  stability: FunctionStability;
49
61
  nullHandling: NullHandling;
62
+ argumentMonotonicity: ArgumentMonotonicity[] | null;
50
63
  requiredSettings: string[];
51
64
  requiredSecrets: string[];
52
65
  projectionPushdown: boolean;
53
66
  filterPushdown: boolean;
54
67
  samplingPushdown: boolean;
55
- supportedExpressionFilters: string[];
68
+ filterSemanticProfiles: string[];
69
+ additionalFilterFunctions: FilterFunctionCapability[];
70
+ runtimeFilterAlgorithms: FilterFunctionCapability[];
71
+ filterEvaluationContexts: EvaluationContextCapability[];
56
72
  preservesOrder: OrderPreservation;
57
73
  maxWorkers: number | null;
58
74
  orderDependent: OrderDependence;
@@ -37,7 +37,7 @@ export interface ProtocolConfig {
37
37
  }
38
38
 
39
39
  export function buildVgiProtocol(config: ProtocolConfig): Protocol {
40
- const protocol = new Protocol("vgi");
40
+ const protocol = new Protocol("vgi", { protocolVersion: "2.0.0" });
41
41
 
42
42
  registerFunctionMethods(protocol, {
43
43
  registry: config.registry,
@@ -20,8 +20,13 @@ import { deserializeArguments } from "../serialize.js";
20
20
  import { deserializeSchema, serializeSchema, batchToSecretDict } from "../../util/arrow/index.js";
21
21
  import { toUint8Array } from "../../util/bytes.js";
22
22
  import {
23
+ AggregateBindParamsSchema,
23
24
  AggregateBindResultSchema,
25
+ AggregateCombineParamsSchema,
26
+ AggregateDestructorParamsSchema,
27
+ AggregateFinalizeParamsSchema,
24
28
  AggregateFinalizeResultSchema,
29
+ AggregateUpdateParamsSchema,
25
30
  } from "../../generated/vgi-protocol-schemas.js";
26
31
  import {
27
32
  getExecutionState,
@@ -41,19 +46,19 @@ import {
41
46
  } from "./shared.js";
42
47
 
43
48
  export function registerAggregateMethods(protocol: Protocol, registry: FunctionRegistry): void {
44
- // `schemaName` scopes resolution to the schema the caller named. Every
49
+ // `schemaPath` scopes resolution to the schema the caller named. Every
45
50
  // aggregate RPC carries it (protocol 1.2.0), so an aggregate name declared in
46
51
  // two schemas dispatches to the one the request names rather than colliding
47
52
  // on the flat by-name index. `null` (no schema) keeps the cross-schema lookup.
48
53
  function resolveAggregate(
49
54
  name: string,
50
- schemaName?: string | null,
55
+ schemaPath?: string[] | null,
51
56
  ): AggregateFunctionConfig<any, any> {
52
57
  const func = registry.get(name, {
53
58
  arguments: new Arguments(),
54
59
  inputSchema: null,
55
60
  isScalar: false,
56
- schemaName: schemaName ?? null,
61
+ schemaPath: schemaPath ?? null,
57
62
  }) as any;
58
63
  if (!func || func.kind !== "aggregate") {
59
64
  throw new Error(`Function '${name}' is not an aggregate function`);
@@ -65,12 +70,12 @@ export function registerAggregateMethods(protocol: Protocol, registry: FunctionR
65
70
  // aggregate_bind — allocate execution_id, return output schema
66
71
  // ------------------------------------------------------------------------
67
72
  protocol.unary("aggregate_bind", {
68
- params: REQUEST_PARAMS_SCHEMA,
73
+ params: AggregateBindParamsSchema,
69
74
  result: RESULT_BINARY_SCHEMA,
70
75
  handler: async (params) => {
71
76
  const innerParams = unwrapRequest(params.request);
72
77
  const functionName: string = innerParams.function_name;
73
- const cfg = resolveAggregate(functionName, innerParams.schema_name ?? null);
78
+ const cfg = resolveAggregate(functionName, innerParams.schema_path ?? null);
74
79
  const argBytes = toUint8Array(innerParams.arguments);
75
80
  const args = deserializeArguments(argBytes);
76
81
  const inputSchema = innerParams.input_schema
@@ -90,6 +95,12 @@ export function registerAggregateMethods(protocol: Protocol, registry: FunctionR
90
95
  inputSchema,
91
96
  settings: {},
92
97
  secrets,
98
+ argumentNames: innerParams.argument_names
99
+ ? (Array.isArray(innerParams.argument_names)
100
+ ? innerParams.argument_names
101
+ : [...innerParams.argument_names])
102
+ .map((value: unknown) => value == null ? null : String(value))
103
+ : null,
93
104
  };
94
105
  const executionId = new Uint8Array(16);
95
106
  crypto.getRandomValues(executionId);
@@ -115,13 +126,13 @@ export function registerAggregateMethods(protocol: Protocol, registry: FunctionR
115
126
  // aggregate_update — fold the incoming batch into per-group state
116
127
  // ------------------------------------------------------------------------
117
128
  protocol.unary("aggregate_update", {
118
- params: REQUEST_PARAMS_SCHEMA,
129
+ params: AggregateUpdateParamsSchema,
119
130
  result: RESULT_BINARY_SCHEMA,
120
131
  handler: (params) => {
121
132
  const innerParams = unwrapRequest(params.request);
122
133
  const functionName: string = innerParams.function_name;
123
134
  const executionId = toUint8Array(innerParams.execution_id);
124
- const cfg = resolveAggregate(functionName, innerParams.schema_name ?? null);
135
+ const cfg = resolveAggregate(functionName, innerParams.schema_path ?? null);
125
136
  const batch = readSingleBatch(toUint8Array(innerParams.input_batch));
126
137
  if (!batch || batch.numRows === 0) return wrapResult({}, makeSchema([]));
127
138
 
@@ -182,13 +193,13 @@ export function registerAggregateMethods(protocol: Protocol, registry: FunctionR
182
193
  // aggregate_combine — merge (source_group_id, target_group_id) pairs
183
194
  // ------------------------------------------------------------------------
184
195
  protocol.unary("aggregate_combine", {
185
- params: REQUEST_PARAMS_SCHEMA,
196
+ params: AggregateCombineParamsSchema,
186
197
  result: RESULT_BINARY_SCHEMA,
187
198
  handler: (params) => {
188
199
  const innerParams = unwrapRequest(params.request);
189
200
  const functionName: string = innerParams.function_name;
190
201
  const executionId = toUint8Array(innerParams.execution_id);
191
- const cfg = resolveAggregate(functionName, innerParams.schema_name ?? null);
202
+ const cfg = resolveAggregate(functionName, innerParams.schema_path ?? null);
192
203
  const batch = readSingleBatch(toUint8Array(innerParams.merge_batch));
193
204
  if (!batch || batch.numRows === 0) return wrapResult({}, makeSchema([]));
194
205
 
@@ -231,13 +242,13 @@ export function registerAggregateMethods(protocol: Protocol, registry: FunctionR
231
242
  // aggregate_finalize — emit one result row per requested group_id
232
243
  // ------------------------------------------------------------------------
233
244
  protocol.unary("aggregate_finalize", {
234
- params: REQUEST_PARAMS_SCHEMA,
245
+ params: AggregateFinalizeParamsSchema,
235
246
  result: RESULT_BINARY_SCHEMA,
236
247
  handler: (params) => {
237
248
  const innerParams = unwrapRequest(params.request);
238
249
  const functionName: string = innerParams.function_name;
239
250
  const executionId = toUint8Array(innerParams.execution_id);
240
- const cfg = resolveAggregate(functionName, innerParams.schema_name ?? null);
251
+ const cfg = resolveAggregate(functionName, innerParams.schema_path ?? null);
241
252
  const outputSchema = deserializeSchema(toUint8Array(innerParams.output_schema));
242
253
  const gidBatch = readSingleBatch(toUint8Array(innerParams.group_ids_batch));
243
254
  const groupIds: bigint[] = [];
@@ -281,7 +292,7 @@ export function registerAggregateMethods(protocol: Protocol, registry: FunctionR
281
292
  // up when the worker process exits).
282
293
  // ------------------------------------------------------------------------
283
294
  protocol.unary("aggregate_destructor", {
284
- params: REQUEST_PARAMS_SCHEMA,
295
+ params: AggregateDestructorParamsSchema,
285
296
  result: RESULT_BINARY_SCHEMA,
286
297
  handler: (params) => {
287
298
  const innerParams = unwrapRequest(params.request);