@query-farm/vgi 0.15.0 → 0.18.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +53 -0
- package/dist/cache-control.d.ts +90 -0
- package/dist/catalog/descriptors.d.ts +13 -7
- package/dist/client-entry.js +3862 -1
- package/dist/client-entry.js.map +34 -3
- package/dist/functions/table.d.ts +12 -0
- package/dist/generated/vgi-client.d.ts +1 -1
- package/dist/http/fetch.d.ts +35 -0
- package/dist/index.core.d.ts +1 -0
- package/dist/index.js +225 -11
- package/dist/index.js.map +11 -9
- package/dist/serve-entry.d.ts +84 -0
- package/dist/serve-entry.js +6428 -0
- package/dist/serve-entry.js.map +62 -0
- package/dist/worker-cf-entry.d.ts +4 -22
- package/dist/worker-cf-entry.js +466 -34
- package/dist/worker-cf-entry.js.map +12 -9
- package/package.json +18 -6
- package/src/cache-control.ts +133 -0
- package/src/catalog/descriptors.ts +13 -7
- package/src/catalog/read-only.ts +22 -14
- package/src/functions/table.ts +25 -1
- package/src/generated/vgi-client.ts +2 -2
- package/src/generated/vgi-protocol-schemas.ts +2 -2
- package/src/http/fetch.ts +80 -0
- package/src/index.core.ts +21 -0
- package/src/serve-entry.ts +233 -0
- package/src/worker-cf-entry.ts +7 -51
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@query-farm/vgi",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.18.0",
|
|
4
4
|
"description": "TypeScript framework for extending DuckDB with user-defined functions via the Vector Gateway Interface (VGI) over Arrow IPC.",
|
|
5
5
|
"license": "SEE LICENSE IN LICENSE",
|
|
6
6
|
"author": "Query Farm LLC (https://query.farm)",
|
|
@@ -29,7 +29,13 @@
|
|
|
29
29
|
"node": ">=22.15"
|
|
30
30
|
},
|
|
31
31
|
"type": "module",
|
|
32
|
-
"sideEffects": false,
|
|
32
|
+
"//sideEffects": "Entry modules are listed so the bundler cannot tree-shake them away. Under a bare `sideEffects: false`, an entry consisting only of re-exports builds to a dangling export list with no imports and no definitions \u2014 invalid ESM that throws on first import. Modules not listed stay side-effect-free, so consumers still tree-shake normally.",
|
|
33
|
+
"sideEffects": [
|
|
34
|
+
"./src/index.ts",
|
|
35
|
+
"./src/client-entry.ts",
|
|
36
|
+
"./src/worker-cf-entry.ts",
|
|
37
|
+
"./src/serve-entry.ts"
|
|
38
|
+
],
|
|
33
39
|
"main": "dist/index.js",
|
|
34
40
|
"types": "dist/index.d.ts",
|
|
35
41
|
"exports": {
|
|
@@ -53,6 +59,11 @@
|
|
|
53
59
|
"browser": "./src/worker-cf-entry.ts",
|
|
54
60
|
"bun": "./src/worker-cf-entry.ts",
|
|
55
61
|
"import": "./dist/worker-cf-entry.js"
|
|
62
|
+
},
|
|
63
|
+
"./serve": {
|
|
64
|
+
"types": "./dist/serve-entry.d.ts",
|
|
65
|
+
"bun": "./src/serve-entry.ts",
|
|
66
|
+
"import": "./dist/serve-entry.js"
|
|
56
67
|
}
|
|
57
68
|
},
|
|
58
69
|
"files": [
|
|
@@ -75,20 +86,21 @@
|
|
|
75
86
|
},
|
|
76
87
|
"peerDependencies": {
|
|
77
88
|
"@query-farm/apache-arrow": "^21.1.1",
|
|
78
|
-
"@query-farm/vgi-rpc": "^0.
|
|
89
|
+
"@query-farm/vgi-rpc": "^0.13.0"
|
|
79
90
|
},
|
|
80
91
|
"devDependencies": {
|
|
81
92
|
"@duckdb/node-api": "^1.5.2-r.1",
|
|
82
93
|
"@query-farm/apache-arrow": "^21.1.1",
|
|
83
|
-
"@query-farm/vgi-rpc": "^0.
|
|
94
|
+
"@query-farm/vgi-rpc": "^0.13.0",
|
|
84
95
|
"@types/bun": "latest",
|
|
85
96
|
"typescript": "^6.0.3"
|
|
86
97
|
},
|
|
87
98
|
"scripts": {
|
|
88
|
-
"build": "bun run build:types && bun run build:js",
|
|
99
|
+
"build": "bun run build:types && bun run build:js && bun run check:bundles",
|
|
100
|
+
"check:bundles": "bun run scripts/check-bundles.ts",
|
|
89
101
|
"prepack": "bun run build",
|
|
90
102
|
"build:types": "bunx tsc -p tsconfig.build.json",
|
|
91
|
-
"build:js": "bun build ./src/index.ts --outdir dist --target node --format esm --sourcemap=external --external @query-farm/apache-arrow --external @query-farm/vgi-rpc && bun build ./src/client-entry.ts --outdir dist --target browser --format esm --sourcemap=external --external @query-farm/apache-arrow --external @query-farm/vgi-rpc && bun build ./src/worker-cf-entry.ts --outdir dist --target node --format esm --sourcemap=external --external @query-farm/apache-arrow --external @query-farm/vgi-rpc",
|
|
103
|
+
"build:js": "bun build ./src/index.ts --outdir dist --target node --format esm --sourcemap=external --external @query-farm/apache-arrow --external @query-farm/vgi-rpc && bun build ./src/client-entry.ts --outdir dist --target browser --format esm --sourcemap=external --external @query-farm/apache-arrow --external @query-farm/vgi-rpc && bun build ./src/worker-cf-entry.ts --outdir dist --target node --format esm --sourcemap=external --external @query-farm/apache-arrow --external @query-farm/vgi-rpc && bun build ./src/serve-entry.ts --outdir dist --target bun --format esm --sourcemap=external --external @query-farm/apache-arrow --external @query-farm/vgi-rpc",
|
|
92
104
|
"postinstall": "node scripts/postinstall.cjs",
|
|
93
105
|
"build:browser-test": "bun build examples/browser/main.ts --outdir examples/browser/dist --target browser --format esm --sourcemap=external",
|
|
94
106
|
"test": "bun test"
|
|
@@ -0,0 +1,133 @@
|
|
|
1
|
+
// Copyright 2025, 2026 Query Farm LLC - https://query.farm
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* Result-cache control metadata (`vgi.cache.*`).
|
|
5
|
+
*
|
|
6
|
+
* A table function can advertise that its result is cacheable by the client
|
|
7
|
+
* (the DuckDB extension) by attaching `vgi.cache.*` metadata to the **first**
|
|
8
|
+
* data batch it emits. The vocabulary mirrors HTTP caching (RFC 9111/9110): a
|
|
9
|
+
* freshness lifetime (`ttl` / `expires`), a reuse `scope`, validators
|
|
10
|
+
* (ETag / Last-Modified) for conditional revalidation, and stale-serving grace
|
|
11
|
+
* windows.
|
|
12
|
+
*
|
|
13
|
+
* The key strings are the single source of truth shared with the C++ extension
|
|
14
|
+
* (which reads them by string). {@link cacheControlMetadata} renders a
|
|
15
|
+
* {@link CacheControl} to the `Map<string, string>` of `vgi.cache.*` keys that
|
|
16
|
+
* rides on batch metadata:
|
|
17
|
+
*
|
|
18
|
+
* ```ts
|
|
19
|
+
* out.emit(firstBatch, cacheControlMetadata({ ttl: 300 }));
|
|
20
|
+
* ```
|
|
21
|
+
*
|
|
22
|
+
* Pass `extra` to merge in metadata a function already emits per batch (for
|
|
23
|
+
* example `vgi_batch_index`); the rendered cache keys win on collision.
|
|
24
|
+
*
|
|
25
|
+
* Booleans render as `"1"` and are omitted when false; timestamps are RFC 3339
|
|
26
|
+
* UTC strings; durations are whole seconds.
|
|
27
|
+
*
|
|
28
|
+
* Ports `vgi/cache_control.py` from vgi-python.
|
|
29
|
+
*/
|
|
30
|
+
|
|
31
|
+
// --- Response-side metadata keys (worker -> client) -------------------------
|
|
32
|
+
// Defined once here; the C++ extension reads these exact strings.
|
|
33
|
+
|
|
34
|
+
export const CACHE_TTL_KEY = "vgi.cache.ttl";
|
|
35
|
+
export const CACHE_EXPIRES_KEY = "vgi.cache.expires";
|
|
36
|
+
export const CACHE_NO_STORE_KEY = "vgi.cache.no_store";
|
|
37
|
+
export const CACHE_SCOPE_KEY = "vgi.cache.scope";
|
|
38
|
+
export const CACHE_ETAG_KEY = "vgi.cache.etag";
|
|
39
|
+
export const CACHE_LAST_MODIFIED_KEY = "vgi.cache.last_modified";
|
|
40
|
+
export const CACHE_REVALIDATABLE_KEY = "vgi.cache.revalidatable";
|
|
41
|
+
export const CACHE_STALE_WHILE_REVALIDATE_KEY = "vgi.cache.stale_while_revalidate";
|
|
42
|
+
export const CACHE_STALE_IF_ERROR_KEY = "vgi.cache.stale_if_error";
|
|
43
|
+
export const CACHE_NOT_MODIFIED_KEY = "vgi.cache.not_modified";
|
|
44
|
+
|
|
45
|
+
// --- Request-side metadata keys (client -> worker) --------------------------
|
|
46
|
+
// Ride on the first producer tick; surfaced as `TableProcessParams.ifNoneMatch`
|
|
47
|
+
// / `.ifModifiedSince`.
|
|
48
|
+
|
|
49
|
+
export const CACHE_IF_NONE_MATCH_KEY = "vgi.cache.if_none_match";
|
|
50
|
+
export const CACHE_IF_MODIFIED_SINCE_KEY = "vgi.cache.if_modified_since";
|
|
51
|
+
|
|
52
|
+
// --- Reuse-scope values -----------------------------------------------------
|
|
53
|
+
|
|
54
|
+
export const CACHE_SCOPE_CATALOG = "catalog";
|
|
55
|
+
export const CACHE_SCOPE_TRANSACTION = "transaction";
|
|
56
|
+
|
|
57
|
+
/** Reuse scope for a cacheable result. */
|
|
58
|
+
export type CacheScope = typeof CACHE_SCOPE_CATALOG | typeof CACHE_SCOPE_TRANSACTION;
|
|
59
|
+
|
|
60
|
+
/**
|
|
61
|
+
* Cacheability advertised by a table function on its first result batch.
|
|
62
|
+
*
|
|
63
|
+
* Presence of `ttl` **or** `expires` is what makes a result cacheable;
|
|
64
|
+
* `noStore` overrides any freshness key.
|
|
65
|
+
*/
|
|
66
|
+
export interface CacheControl {
|
|
67
|
+
/** Freshness lifetime in whole seconds, relative to full-result receipt
|
|
68
|
+
* (skew-immune; wins over `expires`). */
|
|
69
|
+
ttl?: number;
|
|
70
|
+
/** Absolute RFC 3339 UTC deadline. Lifetime is `expires - now` at receipt. */
|
|
71
|
+
expires?: string;
|
|
72
|
+
/** Reuse scope — `"catalog"` (default; reusable across transactions within
|
|
73
|
+
* the calling catalog identity) or `"transaction"` (reused only within the
|
|
74
|
+
* same transaction). */
|
|
75
|
+
scope?: CacheScope;
|
|
76
|
+
/** Explicit "never cache"; overrides any freshness key. */
|
|
77
|
+
noStore?: boolean;
|
|
78
|
+
/** Strong validator (opaque quoted string) for conditional revalidation. */
|
|
79
|
+
etag?: string;
|
|
80
|
+
/** Weaker RFC 3339 UTC validator; fallback when no ETag. */
|
|
81
|
+
lastModified?: string;
|
|
82
|
+
/** The worker can check freshness cheaply without recomputing; gates whether
|
|
83
|
+
* the client ever sends a conditional request. */
|
|
84
|
+
revalidatable?: boolean;
|
|
85
|
+
/** Grace window (seconds) to serve stale immediately while revalidating in
|
|
86
|
+
* the background. */
|
|
87
|
+
staleWhileRevalidate?: number;
|
|
88
|
+
/** Grace window (seconds) to serve stale if a revalidation RPC fails. */
|
|
89
|
+
staleIfError?: number;
|
|
90
|
+
/** 304-equivalent — set on a 0-row batch in reply to a conditional request to
|
|
91
|
+
* assert the client's stored payload is still fresh (the client reuses it
|
|
92
|
+
* instead of re-streaming). */
|
|
93
|
+
notModified?: boolean;
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
const VALID_SCOPES: readonly string[] = [CACHE_SCOPE_CATALOG, CACHE_SCOPE_TRANSACTION];
|
|
97
|
+
|
|
98
|
+
/**
|
|
99
|
+
* Render a {@link CacheControl} to the `vgi.cache.*` batch-metadata map.
|
|
100
|
+
*
|
|
101
|
+
* `scope` is always emitted so the client never has to infer the default.
|
|
102
|
+
* Unset optional fields are omitted; false booleans are omitted. Entries from
|
|
103
|
+
* `extra` are written first, so a rendered cache key wins on collision.
|
|
104
|
+
*
|
|
105
|
+
* Throws when `scope` is unrecognized or a duration is negative — a silently
|
|
106
|
+
* ignored advertisement is far harder to debug than a stack trace at the
|
|
107
|
+
* offending `emit()`.
|
|
108
|
+
*/
|
|
109
|
+
export function cacheControlMetadata(cc: CacheControl, extra?: Map<string, string>): Map<string, string> {
|
|
110
|
+
const scope = cc.scope ?? CACHE_SCOPE_CATALOG;
|
|
111
|
+
if (!VALID_SCOPES.includes(scope)) {
|
|
112
|
+
throw new Error(`CacheControl.scope must be one of ${VALID_SCOPES.join(", ")}, got ${JSON.stringify(scope)}`);
|
|
113
|
+
}
|
|
114
|
+
for (const name of ["ttl", "staleWhileRevalidate", "staleIfError"] as const) {
|
|
115
|
+
const value = cc[name];
|
|
116
|
+
if (value !== undefined && value < 0) {
|
|
117
|
+
throw new Error(`CacheControl.${name} must be >= 0, got ${value}`);
|
|
118
|
+
}
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
const md = new Map<string, string>(extra ?? []);
|
|
122
|
+
if (cc.ttl !== undefined) md.set(CACHE_TTL_KEY, String(cc.ttl));
|
|
123
|
+
if (cc.expires !== undefined) md.set(CACHE_EXPIRES_KEY, cc.expires);
|
|
124
|
+
if (cc.noStore) md.set(CACHE_NO_STORE_KEY, "1");
|
|
125
|
+
md.set(CACHE_SCOPE_KEY, scope);
|
|
126
|
+
if (cc.etag !== undefined) md.set(CACHE_ETAG_KEY, cc.etag);
|
|
127
|
+
if (cc.lastModified !== undefined) md.set(CACHE_LAST_MODIFIED_KEY, cc.lastModified);
|
|
128
|
+
if (cc.revalidatable) md.set(CACHE_REVALIDATABLE_KEY, "1");
|
|
129
|
+
if (cc.staleWhileRevalidate !== undefined) md.set(CACHE_STALE_WHILE_REVALIDATE_KEY, String(cc.staleWhileRevalidate));
|
|
130
|
+
if (cc.staleIfError !== undefined) md.set(CACHE_STALE_IF_ERROR_KEY, String(cc.staleIfError));
|
|
131
|
+
if (cc.notModified) md.set(CACHE_NOT_MODIFIED_KEY, "1");
|
|
132
|
+
return md;
|
|
133
|
+
}
|
|
@@ -60,18 +60,24 @@ export interface TableDescriptor {
|
|
|
60
60
|
*/
|
|
61
61
|
statisticsCacheMaxAgeSeconds?: number | null;
|
|
62
62
|
/**
|
|
63
|
-
*
|
|
64
|
-
*
|
|
65
|
-
*
|
|
66
|
-
*
|
|
63
|
+
* Required WHERE-filter groups in conjunctive normal form — an AND (outer
|
|
64
|
+
* list) of OR-groups (inner lists) of dotted-path column references that MUST
|
|
65
|
+
* appear in a WHERE expression for any scan of this table. A group is
|
|
66
|
+
* satisfied when any one of its paths has a filter; every group must be
|
|
67
|
+
* satisfied. So `[["accession_number"], ["ticker", "cik"]]` means
|
|
68
|
+
* "accession_number AND one of (ticker, cik)"; a single-path group
|
|
69
|
+
* `[["country"]]` is a plain mandatory filter. Paths are top-level names
|
|
70
|
+
* (`"country"`) or struct subfields (`"bbox.xmin"`, `"nested.outer.inner"`).
|
|
71
|
+
* Empty/undefined (default) means no enforcement — the zero-cost fast path
|
|
72
|
+
* for every existing table.
|
|
67
73
|
*
|
|
68
74
|
* Satisfaction is prefix-based: a present filter on a shorter path satisfies
|
|
69
75
|
* any required path it is a prefix of. So a whole-struct filter on `bbox`
|
|
70
76
|
* satisfies all of `bbox.xmin` / `.xmax` / `.ymin` / `.ymax`. The VGI DuckDB
|
|
71
|
-
* extension's optimizer pass consults this
|
|
72
|
-
* `BinderException` listing any unsatisfied
|
|
77
|
+
* extension's optimizer pass consults this at bind time and throws a
|
|
78
|
+
* `BinderException` listing any unsatisfied groups.
|
|
73
79
|
*/
|
|
74
|
-
|
|
80
|
+
requiredFilters?: string[][];
|
|
75
81
|
supportsTimeTravel?: boolean;
|
|
76
82
|
/**
|
|
77
83
|
* Inline cardinality estimate/max surfaced through TableInfo. When set,
|
package/src/catalog/read-only.ts
CHANGED
|
@@ -235,26 +235,34 @@ export class ReadOnlyCatalogInterface extends CatalogInterface {
|
|
|
235
235
|
const pkIndices = resolveIndexGroups(colSchema, t.primaryKey);
|
|
236
236
|
const fkBytes = serializeForeignKeys(t.foreignKey, name);
|
|
237
237
|
|
|
238
|
-
// Validate
|
|
239
|
-
//
|
|
238
|
+
// Validate required_filters: an AND (outer list) of OR-groups (inner
|
|
239
|
+
// lists). Each OR-group must be non-empty, and the leading dotted segment
|
|
240
|
+
// of each path must be a real column on this table. Struct subfield
|
|
240
241
|
// validity is not checked here — DuckDB's binder catches typos at scan
|
|
241
242
|
// time, and the descriptor doesn't unpack STRUCT subfields. (Mirrors
|
|
242
243
|
// vgi-python's Table descriptor validation.)
|
|
243
|
-
const
|
|
244
|
-
if (
|
|
244
|
+
const requiredFilters = t.requiredFilters ?? [];
|
|
245
|
+
if (requiredFilters.length > 0) {
|
|
245
246
|
const columnNames = new Set(colSchema.fields.map((f) => f.name));
|
|
246
|
-
for (const
|
|
247
|
-
if (!
|
|
247
|
+
for (const group of requiredFilters) {
|
|
248
|
+
if (!group || group.length === 0) {
|
|
248
249
|
throw new Error(
|
|
249
|
-
`Table '${t.name}':
|
|
250
|
+
`Table '${t.name}': requiredFilters must not contain empty groups`
|
|
250
251
|
);
|
|
251
252
|
}
|
|
252
|
-
const
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
253
|
+
for (const path of group) {
|
|
254
|
+
if (!path) {
|
|
255
|
+
throw new Error(
|
|
256
|
+
`Table '${t.name}': requiredFilters must not contain empty strings`
|
|
257
|
+
);
|
|
258
|
+
}
|
|
259
|
+
const head = path.split(".", 1)[0];
|
|
260
|
+
if (!columnNames.has(head)) {
|
|
261
|
+
throw new Error(
|
|
262
|
+
`Table '${t.name}': requiredFilters path '${path}' references ` +
|
|
263
|
+
`unknown column '${head}'. Available columns: ${[...columnNames].sort().join(", ")}`
|
|
264
|
+
);
|
|
265
|
+
}
|
|
258
266
|
}
|
|
259
267
|
}
|
|
260
268
|
}
|
|
@@ -292,7 +300,7 @@ export class ReadOnlyCatalogInterface extends CatalogInterface {
|
|
|
292
300
|
cardinality_max: t.inlinedCardinality
|
|
293
301
|
? Number(t.inlinedCardinality.max)
|
|
294
302
|
: null,
|
|
295
|
-
|
|
303
|
+
required_filters: requiredFilters,
|
|
296
304
|
};
|
|
297
305
|
}));
|
|
298
306
|
}
|
package/src/functions/table.ts
CHANGED
|
@@ -51,6 +51,7 @@ import {
|
|
|
51
51
|
type DistinctDependence,
|
|
52
52
|
} from "../types.js";
|
|
53
53
|
import { BoundStorage, storage as globalStorage } from "./storage.js";
|
|
54
|
+
import { CACHE_IF_MODIFIED_SINCE_KEY, CACHE_IF_NONE_MATCH_KEY } from "../cache-control.js";
|
|
54
55
|
|
|
55
56
|
// Base64-decode a string into raw bytes. Used to unpack the dynamic filter
|
|
56
57
|
// update DuckDB attaches to each tick batch's custom metadata.
|
|
@@ -153,6 +154,18 @@ export interface TableProcessParams<TArgs = Record<string, any>> {
|
|
|
153
154
|
*/
|
|
154
155
|
atUnit?: string;
|
|
155
156
|
atValue?: string;
|
|
157
|
+
/**
|
|
158
|
+
* Conditional-revalidation validator (the client's stored ETag). Set when the
|
|
159
|
+
* client holds a stale-but-revalidatable cached result and asks the worker to
|
|
160
|
+
* confirm freshness cheaply; a worker that advertised `revalidatable` compares
|
|
161
|
+
* it and, if unchanged, emits a 0-row batch tagged
|
|
162
|
+
* `cacheControlMetadata({ notModified: true })` instead of re-streaming.
|
|
163
|
+
* Undefined on a normal call.
|
|
164
|
+
*/
|
|
165
|
+
ifNoneMatch?: string;
|
|
166
|
+
/** Conditional-revalidation validator (the client's stored Last-Modified).
|
|
167
|
+
* Companion to {@link ifNoneMatch}. Undefined on a normal call. */
|
|
168
|
+
ifModifiedSince?: string;
|
|
156
169
|
}
|
|
157
170
|
|
|
158
171
|
// ============================================================================
|
|
@@ -443,12 +456,23 @@ export function defineTableFunction<
|
|
|
443
456
|
pState: { state: TState; processParams: TableProcessParams<TArgs> },
|
|
444
457
|
tickMetadata: Map<string, string> | undefined,
|
|
445
458
|
) => {
|
|
459
|
+
if (!tickMetadata) return;
|
|
460
|
+
|
|
461
|
+
// Conditional-revalidation validators: the client holds a stale
|
|
462
|
+
// cached result and asks the worker to confirm freshness cheaply.
|
|
463
|
+
// Over subprocess these ride the first producer tick; over HTTP the
|
|
464
|
+
// first tick folds into the /init request, whose metadata vgi-rpc
|
|
465
|
+
// surfaces here.
|
|
466
|
+
const ifNoneMatch = tickMetadata.get(CACHE_IF_NONE_MATCH_KEY);
|
|
467
|
+
const ifModifiedSince = tickMetadata.get(CACHE_IF_MODIFIED_SINCE_KEY);
|
|
468
|
+
if (ifNoneMatch !== undefined) pState.processParams.ifNoneMatch = ifNoneMatch;
|
|
469
|
+
if (ifModifiedSince !== undefined) pState.processParams.ifModifiedSince = ifModifiedSince;
|
|
470
|
+
|
|
446
471
|
// Dynamic filter pushdown: DuckDB's Top-N optimizer tightens filters
|
|
447
472
|
// between ticks and serializes the current filter into the tick
|
|
448
473
|
// batch's custom metadata under `vgi_pushdown_filters` (base64 of a
|
|
449
474
|
// filter IPC stream). Decode and overwrite the current pushdown
|
|
450
475
|
// filters so process() sees the updated value.
|
|
451
|
-
if (!tickMetadata) return;
|
|
452
476
|
const encoded = tickMetadata.get("vgi_pushdown_filters");
|
|
453
477
|
if (!encoded) return;
|
|
454
478
|
try {
|
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
// GENERATED by vgi.codegen.ts_client. DO NOT EDIT BY HAND.
|
|
3
3
|
//
|
|
4
4
|
// Generator: vgi-gen-ts-client v1
|
|
5
|
-
// Content hash:
|
|
5
|
+
// Content hash: f3ac87321a0c
|
|
6
6
|
//
|
|
7
7
|
// To regenerate:
|
|
8
8
|
// uv run --project ~/Development/vgi-python vgi-gen-ts-client \
|
|
@@ -84,7 +84,7 @@ export interface TableInfo {
|
|
|
84
84
|
cardinality_max?: number | null;
|
|
85
85
|
column_statistics?: Uint8Array | null;
|
|
86
86
|
bind_result?: Uint8Array | null;
|
|
87
|
-
|
|
87
|
+
required_filters: string[][];
|
|
88
88
|
}
|
|
89
89
|
|
|
90
90
|
export interface ViewInfo {
|
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
// GENERATED by vgi.codegen.ts_schemas. DO NOT EDIT BY HAND.
|
|
3
3
|
//
|
|
4
4
|
// Generator: vgi-gen-ts-schemas v2
|
|
5
|
-
// Content hash:
|
|
5
|
+
// Content hash: b3c99cc5fac8
|
|
6
6
|
//
|
|
7
7
|
// To regenerate:
|
|
8
8
|
// uv run --project ~/Development/vgi-python vgi-gen-ts-schemas \
|
|
@@ -70,7 +70,7 @@ export const TableInfoSchema = schema([
|
|
|
70
70
|
field("cardinality_max", int64(), false),
|
|
71
71
|
field("column_statistics", binary(), false),
|
|
72
72
|
field("bind_result", binary(), false),
|
|
73
|
-
field("
|
|
73
|
+
field("required_filters", list(field("item", list(field("item", utf8(), true)), true)), false),
|
|
74
74
|
]);
|
|
75
75
|
|
|
76
76
|
// Origin: ViewInfo
|
|
@@ -0,0 +1,80 @@
|
|
|
1
|
+
// Copyright 2025, 2026 Query Farm LLC - https://query.farm
|
|
2
|
+
// Shared HTTP fetch-handler factory for VGI workers.
|
|
3
|
+
//
|
|
4
|
+
// Both HTTP entrypoints funnel through here:
|
|
5
|
+
// - `vgi/worker-cf` -> createVgiFetch, exported as `default.fetch` for workerd
|
|
6
|
+
// - `vgi/serve` -> serveVgiWorker, which hands the handler to Bun.serve
|
|
7
|
+
//
|
|
8
|
+
// Centralising the wiring means the state-token key reaches `createHttpHandler`
|
|
9
|
+
// as `tokenKey` (the option the handler actually reads) and reaches
|
|
10
|
+
// `buildVgiProtocol` as `signingKey`, from one source. Passing different keys to
|
|
11
|
+
// those two seams produces a worker that mints tokens it cannot recover.
|
|
12
|
+
|
|
13
|
+
import { createHttpHandler, unpackStateToken, type Protocol } from "@query-farm/vgi-rpc";
|
|
14
|
+
import { arrowStateSerializer } from "../protocol/state-serializer.js";
|
|
15
|
+
import { buildVgiProtocol, type ProtocolConfig } from "../protocol/dispatch.js";
|
|
16
|
+
import type { LandingDescribeProvider } from "./describe-json.js";
|
|
17
|
+
|
|
18
|
+
export interface VgiFetchOptions {
|
|
19
|
+
/** Wire-protocol config (registry + catalogInterface). */
|
|
20
|
+
protocol: Omit<ProtocolConfig, "recoverExchangeState">;
|
|
21
|
+
/** HMAC key for state-token signing. Pass a stable, secret 32-byte key
|
|
22
|
+
* (e.g. derived from a Wrangler secret). Required because Workers don't
|
|
23
|
+
* preserve in-memory state across requests/instances. */
|
|
24
|
+
signingKey: Uint8Array;
|
|
25
|
+
/** State-token TTL in seconds (default 3600). */
|
|
26
|
+
tokenTtl?: number;
|
|
27
|
+
/** URL path prefix for VGI requests (default "/vgi"). Pass "" to mount at
|
|
28
|
+
* the root, which is what a dedicated single-worker HTTP server wants. */
|
|
29
|
+
prefix?: string;
|
|
30
|
+
/** Server ID for state-token attribution (default "vgi-cf"). */
|
|
31
|
+
serverId?: string;
|
|
32
|
+
/** CORS allowed origins. When set, CORS headers are added to all responses,
|
|
33
|
+
* and the preflight `OPTIONS` that browser clients (e.g. the hosted Cupola
|
|
34
|
+
* UI) send before `__describe__` will succeed. Omit to disable CORS. */
|
|
35
|
+
corsOrigins?: string;
|
|
36
|
+
/** Public source-repository URL, surfaced on the landing page. */
|
|
37
|
+
repositoryUrl?: string;
|
|
38
|
+
/** Enables the standardized VGI landing surface: `GET /` serves the shared
|
|
39
|
+
* landing.html, `GET /describe.json` the catalog contract, and
|
|
40
|
+
* `GET /describe/{catalog}/{schema}/{table}.json` lazy per-object columns.
|
|
41
|
+
* Build one with `createLandingDescribe(catalogInterface, workerInfo)`. */
|
|
42
|
+
landingDescribe?: LandingDescribeProvider;
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
/**
|
|
46
|
+
* Build a fetch handler suitable for `export default { fetch }` in a CF
|
|
47
|
+
* Worker module, or for `Bun.serve({ fetch })`. The returned handler is
|
|
48
|
+
* async-safe across Workers' isolate-per-request execution model — all state
|
|
49
|
+
* round-trips through the signed state token created here.
|
|
50
|
+
*/
|
|
51
|
+
export function createVgiFetch(opts: VgiFetchOptions): (req: Request) => Promise<Response> {
|
|
52
|
+
const tokenTtl = opts.tokenTtl ?? 3600;
|
|
53
|
+
// `?? "/vgi"` and not `|| "/vgi"`: an explicit "" means "mount at root".
|
|
54
|
+
const prefix = opts.prefix ?? "/vgi";
|
|
55
|
+
const serverId = opts.serverId ?? "vgi-cf";
|
|
56
|
+
|
|
57
|
+
const protocol: Protocol = buildVgiProtocol({
|
|
58
|
+
signingKey: opts.signingKey,
|
|
59
|
+
...opts.protocol,
|
|
60
|
+
recoverExchangeState: async (opaqueData: Uint8Array) => {
|
|
61
|
+
const tokenString = new TextDecoder().decode(opaqueData);
|
|
62
|
+
// principal binding is enforced by the HTTP handler on the request that
|
|
63
|
+
// carried this token; the recovery path itself is not principal-scoped.
|
|
64
|
+
const unpacked = await unpackStateToken(tokenString, opts.signingKey, tokenTtl, undefined);
|
|
65
|
+
return arrowStateSerializer.deserialize(unpacked.stateBytes);
|
|
66
|
+
},
|
|
67
|
+
});
|
|
68
|
+
|
|
69
|
+
const handler = createHttpHandler(protocol, {
|
|
70
|
+
prefix,
|
|
71
|
+
serverId,
|
|
72
|
+
tokenKey: opts.signingKey,
|
|
73
|
+
tokenTtl,
|
|
74
|
+
stateSerializer: arrowStateSerializer,
|
|
75
|
+
corsOrigins: opts.corsOrigins,
|
|
76
|
+
repositoryUrl: opts.repositoryUrl,
|
|
77
|
+
landingDescribe: opts.landingDescribe,
|
|
78
|
+
});
|
|
79
|
+
return async (req: Request) => handler(req);
|
|
80
|
+
}
|
package/src/index.core.ts
CHANGED
|
@@ -7,6 +7,27 @@
|
|
|
7
7
|
// for Cloudflare Workers / browsers via worker-cf-entry.ts. The node/bun barrel
|
|
8
8
|
// (`./index.ts`) re-exports this plus `Worker`.
|
|
9
9
|
|
|
10
|
+
// Result-cache control vocabulary (`vgi.cache.*`)
|
|
11
|
+
export {
|
|
12
|
+
CACHE_ETAG_KEY,
|
|
13
|
+
CACHE_EXPIRES_KEY,
|
|
14
|
+
CACHE_IF_MODIFIED_SINCE_KEY,
|
|
15
|
+
CACHE_IF_NONE_MATCH_KEY,
|
|
16
|
+
CACHE_LAST_MODIFIED_KEY,
|
|
17
|
+
CACHE_NO_STORE_KEY,
|
|
18
|
+
CACHE_NOT_MODIFIED_KEY,
|
|
19
|
+
CACHE_REVALIDATABLE_KEY,
|
|
20
|
+
CACHE_SCOPE_CATALOG,
|
|
21
|
+
CACHE_SCOPE_KEY,
|
|
22
|
+
CACHE_SCOPE_TRANSACTION,
|
|
23
|
+
CACHE_STALE_IF_ERROR_KEY,
|
|
24
|
+
CACHE_STALE_WHILE_REVALIDATE_KEY,
|
|
25
|
+
CACHE_TTL_KEY,
|
|
26
|
+
type CacheControl,
|
|
27
|
+
cacheControlMetadata,
|
|
28
|
+
type CacheScope,
|
|
29
|
+
} from "./cache-control.js";
|
|
30
|
+
|
|
10
31
|
// Function factories (primary API)
|
|
11
32
|
export { defineScalarFunction, type ScalarFunctionConfig, type ScalarBindParameters, type ScalarParameterDef } from "./functions/scalar.js";
|
|
12
33
|
export { defineTableFunction, type TableFunctionConfig, type TableBindParams, type TableProcessParams } from "./functions/table.js";
|