@query-farm/vgi 0.15.0 → 0.18.0

package/README.md

@@ -441,6 +441,59 @@ A worker serves the same functions over any of:
 
 Transport handling lives in [`@query-farm/vgi-rpc`](https://www.npmjs.com/package/@query-farm/vgi-rpc).
 
+### Serving a worker over HTTP
+
+Don't hand-roll the HTTP wiring. `@query-farm/vgi/serve` assembles the protocol, the
+signed state-token key, CORS, and the standardized landing surface, then binds the port:
+
+```ts
+// scripts/serve.ts
+import { serveVgiWorker } from "@query-farm/vgi/serve";
+import { makeWorkerParts } from "../src/parts.js";
+
+const { registry, catalogInterface } = makeWorkerParts();
+
+serveVgiWorker({
+  name: "ishares",
+  doc: "iShares (BlackRock) US fund data.",
+  version: "0.1.0",
+  repositoryUrl: "https://github.com/Query-farm/vgi-etf-ishares",
+  registry,
+  catalogInterface,
+});
+```
+
+```console
+$ PORT=8787 bun run scripts/serve.ts
+```
+
+Mounted at the origin root, so DuckDB attaches the bare URL:
+
+```sql
+ATTACH 'ishares' AS ishares (TYPE vgi, LOCATION 'http://localhost:8787');
+```
+
+It also serves `GET /` (landing page), `GET /describe.json` (catalog contract), and
+`GET /health`.
+
+| Variable          | Default | Meaning                                                     |
+| ----------------- | ------- | ----------------------------------------------------------- |
+| `PORT`            | `8787`  | Listen port. `0` binds an ephemeral port.                    |
+| `VGI_SIGNING_KEY` | random  | State-token HMAC key, **64 hex chars** (`openssl rand -hex 32`). |
+| `VGI_TOKEN_TTL`   | `3600`  | State-token lifetime in seconds.                             |
+| `CORS_ORIGINS`    | `*`     | Allowed origins. Pass `corsOrigins: null` to disable CORS.   |
+
+Every variable has an explicit option that takes precedence over it.
+
+With no `VGI_SIGNING_KEY`, a random key is generated and a warning is printed: state
+tokens then stop validating after a restart, and never validate across two instances
+behind a load balancer. Set it for any real deployment. A key that isn't exactly 64 hex
+characters is rejected rather than silently truncated.
+
+To mount the VGI routes inside a server you already own, use `createVgiWorkerFetch`,
+which returns the `fetch` handler without binding a port. On Cloudflare Workers use
+`createVgiFetch` from `@query-farm/vgi/worker-cf`; both share one implementation.
+
 ## Runtimes & entry points
 
 The package ships a backend-agnostic Arrow facade and selects an implementation at

package/dist/cache-control.d.ts

@@ -0,0 +1,90 @@
+/**
+ * Result-cache control metadata (`vgi.cache.*`).
+ *
+ * A table function can advertise that its result is cacheable by the client
+ * (the DuckDB extension) by attaching `vgi.cache.*` metadata to the **first**
+ * data batch it emits. The vocabulary mirrors HTTP caching (RFC 9111/9110): a
+ * freshness lifetime (`ttl` / `expires`), a reuse `scope`, validators
+ * (ETag / Last-Modified) for conditional revalidation, and stale-serving grace
+ * windows.
+ *
+ * The key strings are the single source of truth shared with the C++ extension
+ * (which reads them by string). {@link cacheControlMetadata} renders a
+ * {@link CacheControl} to the `Map<string, string>` of `vgi.cache.*` keys that
+ * rides on batch metadata:
+ *
+ * ```ts
+ * out.emit(firstBatch, cacheControlMetadata({ ttl: 300 }));
+ * ```
+ *
+ * Pass `extra` to merge in metadata a function already emits per batch (for
+ * example `vgi_batch_index`); the rendered cache keys win on collision.
+ *
+ * Booleans render as `"1"` and are omitted when false; timestamps are RFC 3339
+ * UTC strings; durations are whole seconds.
+ *
+ * Ports `vgi/cache_control.py` from vgi-python.
+ */
+export declare const CACHE_TTL_KEY = "vgi.cache.ttl";
+export declare const CACHE_EXPIRES_KEY = "vgi.cache.expires";
+export declare const CACHE_NO_STORE_KEY = "vgi.cache.no_store";
+export declare const CACHE_SCOPE_KEY = "vgi.cache.scope";
+export declare const CACHE_ETAG_KEY = "vgi.cache.etag";
+export declare const CACHE_LAST_MODIFIED_KEY = "vgi.cache.last_modified";
+export declare const CACHE_REVALIDATABLE_KEY = "vgi.cache.revalidatable";
+export declare const CACHE_STALE_WHILE_REVALIDATE_KEY = "vgi.cache.stale_while_revalidate";
+export declare const CACHE_STALE_IF_ERROR_KEY = "vgi.cache.stale_if_error";
+export declare const CACHE_NOT_MODIFIED_KEY = "vgi.cache.not_modified";
+export declare const CACHE_IF_NONE_MATCH_KEY = "vgi.cache.if_none_match";
+export declare const CACHE_IF_MODIFIED_SINCE_KEY = "vgi.cache.if_modified_since";
+export declare const CACHE_SCOPE_CATALOG = "catalog";
+export declare const CACHE_SCOPE_TRANSACTION = "transaction";
+/** Reuse scope for a cacheable result. */
+export type CacheScope = typeof CACHE_SCOPE_CATALOG | typeof CACHE_SCOPE_TRANSACTION;
+/**
+ * Cacheability advertised by a table function on its first result batch.
+ *
+ * Presence of `ttl` **or** `expires` is what makes a result cacheable;
+ * `noStore` overrides any freshness key.
+ */
+export interface CacheControl {
+    /** Freshness lifetime in whole seconds, relative to full-result receipt
+     *  (skew-immune; wins over `expires`). */
+    ttl?: number;
+    /** Absolute RFC 3339 UTC deadline. Lifetime is `expires - now` at receipt. */
+    expires?: string;
+    /** Reuse scope — `"catalog"` (default; reusable across transactions within
+     *  the calling catalog identity) or `"transaction"` (reused only within the
+     *  same transaction). */
+    scope?: CacheScope;
+    /** Explicit "never cache"; overrides any freshness key. */
+    noStore?: boolean;
+    /** Strong validator (opaque quoted string) for conditional revalidation. */
+    etag?: string;
+    /** Weaker RFC 3339 UTC validator; fallback when no ETag. */
+    lastModified?: string;
+    /** The worker can check freshness cheaply without recomputing; gates whether
+     *  the client ever sends a conditional request. */
+    revalidatable?: boolean;
+    /** Grace window (seconds) to serve stale immediately while revalidating in
+     *  the background. */
+    staleWhileRevalidate?: number;
+    /** Grace window (seconds) to serve stale if a revalidation RPC fails. */
+    staleIfError?: number;
+    /** 304-equivalent — set on a 0-row batch in reply to a conditional request to
+     *  assert the client's stored payload is still fresh (the client reuses it
+     *  instead of re-streaming). */
+    notModified?: boolean;
+}
+/**
+ * Render a {@link CacheControl} to the `vgi.cache.*` batch-metadata map.
+ *
+ * `scope` is always emitted so the client never has to infer the default.
+ * Unset optional fields are omitted; false booleans are omitted. Entries from
+ * `extra` are written first, so a rendered cache key wins on collision.
+ *
+ * Throws when `scope` is unrecognized or a duration is negative — a silently
+ * ignored advertisement is far harder to debug than a stack trace at the
+ * offending `emit()`.
+ */
+export declare function cacheControlMetadata(cc: CacheControl, extra?: Map<string, string>): Map<string, string>;

package/dist/catalog/descriptors.d.ts

@@ -53,18 +53,24 @@ export interface TableDescriptor {
      */
     statisticsCacheMaxAgeSeconds?: number | null;
     /**
-     * Dotted-path column references that MUST appear in a WHERE expression for
-     * any scan of this table. Top-level names (`"country"`) or struct subfields
-     * (`"bbox.xmin"`, `"nested.outer.inner"`). Empty/undefined (default) means
-     * no enforcement — the zero-cost fast path for every existing table.
+     * Required WHERE-filter groups in conjunctive normal form — an AND (outer
+     * list) of OR-groups (inner lists) of dotted-path column references that MUST
+     * appear in a WHERE expression for any scan of this table. A group is
+     * satisfied when any one of its paths has a filter; every group must be
+     * satisfied. So `[["accession_number"], ["ticker", "cik"]]` means
+     * "accession_number AND one of (ticker, cik)"; a single-path group
+     * `[["country"]]` is a plain mandatory filter. Paths are top-level names
+     * (`"country"`) or struct subfields (`"bbox.xmin"`, `"nested.outer.inner"`).
+     * Empty/undefined (default) means no enforcement — the zero-cost fast path
+     * for every existing table.
      *
      * Satisfaction is prefix-based: a present filter on a shorter path satisfies
      * any required path it is a prefix of. So a whole-struct filter on `bbox`
      * satisfies all of `bbox.xmin` / `.xmax` / `.ymin` / `.ymax`. The VGI DuckDB
-     * extension's optimizer pass consults this list at bind time and throws a
-     * `BinderException` listing any unsatisfied paths.
+     * extension's optimizer pass consults this at bind time and throws a
+     * `BinderException` listing any unsatisfied groups.
      */
-    requiredFieldFilterPaths?: string[];
+    requiredFilters?: string[][];
     supportsTimeTravel?: boolean;
     /**
      * Inline cardinality estimate/max surfaced through TableInfo. When set,