@query-farm/vgi 0.14.0 → 0.17.0

package/dist/functions/table.d.ts

@@ -32,6 +32,18 @@ export interface TableProcessParams<TArgs = Record<string, any>> {
      */
     atUnit?: string;
     atValue?: string;
+    /**
+     * Conditional-revalidation validator (the client's stored ETag). Set when the
+     * client holds a stale-but-revalidatable cached result and asks the worker to
+     * confirm freshness cheaply; a worker that advertised `revalidatable` compares
+     * it and, if unchanged, emits a 0-row batch tagged
+     * `cacheControlMetadata({ notModified: true })` instead of re-streaming.
+     * Undefined on a normal call.
+     */
+    ifNoneMatch?: string;
+    /** Conditional-revalidation validator (the client's stored Last-Modified).
+     *  Companion to {@link ifNoneMatch}. Undefined on a normal call. */
+    ifModifiedSince?: string;
 }
 export interface TableFunctionConfig<TArgs = Record<string, any>, TState = null> {
     name: string;

package/dist/http/fetch.d.ts

@@ -0,0 +1,35 @@
+import { type ProtocolConfig } from "../protocol/dispatch.js";
+import type { LandingDescribeProvider } from "./describe-json.js";
+export interface VgiFetchOptions {
+    /** Wire-protocol config (registry + catalogInterface). */
+    protocol: Omit<ProtocolConfig, "recoverExchangeState">;
+    /** HMAC key for state-token signing. Pass a stable, secret 32-byte key
+     *  (e.g. derived from a Wrangler secret). Required because Workers don't
+     *  preserve in-memory state across requests/instances. */
+    signingKey: Uint8Array;
+    /** State-token TTL in seconds (default 3600). */
+    tokenTtl?: number;
+    /** URL path prefix for VGI requests (default "/vgi"). Pass "" to mount at
+     *  the root, which is what a dedicated single-worker HTTP server wants. */
+    prefix?: string;
+    /** Server ID for state-token attribution (default "vgi-cf"). */
+    serverId?: string;
+    /** CORS allowed origins. When set, CORS headers are added to all responses,
+     *  and the preflight `OPTIONS` that browser clients (e.g. the hosted Cupola
+     *  UI) send before `__describe__` will succeed. Omit to disable CORS. */
+    corsOrigins?: string;
+    /** Public source-repository URL, surfaced on the landing page. */
+    repositoryUrl?: string;
+    /** Enables the standardized VGI landing surface: `GET /` serves the shared
+     *  landing.html, `GET /describe.json` the catalog contract, and
+     *  `GET /describe/{catalog}/{schema}/{table}.json` lazy per-object columns.
+     *  Build one with `createLandingDescribe(catalogInterface, workerInfo)`. */
+    landingDescribe?: LandingDescribeProvider;
+}
+/**
+ * Build a fetch handler suitable for `export default { fetch }` in a CF
+ * Worker module, or for `Bun.serve({ fetch })`. The returned handler is
+ * async-safe across Workers' isolate-per-request execution model — all state
+ * round-trips through the signed state token created here.
+ */
+export declare function createVgiFetch(opts: VgiFetchOptions): (req: Request) => Promise<Response>;

package/dist/index.core.d.ts

@@ -1,3 +1,4 @@
+export { CACHE_ETAG_KEY, CACHE_EXPIRES_KEY, CACHE_IF_MODIFIED_SINCE_KEY, CACHE_IF_NONE_MATCH_KEY, CACHE_LAST_MODIFIED_KEY, CACHE_NO_STORE_KEY, CACHE_NOT_MODIFIED_KEY, CACHE_REVALIDATABLE_KEY, CACHE_SCOPE_CATALOG, CACHE_SCOPE_KEY, CACHE_SCOPE_TRANSACTION, CACHE_STALE_IF_ERROR_KEY, CACHE_STALE_WHILE_REVALIDATE_KEY, CACHE_TTL_KEY, type CacheControl, cacheControlMetadata, type CacheScope, } from "./cache-control.js";
 export { defineScalarFunction, type ScalarFunctionConfig, type ScalarBindParameters, type ScalarParameterDef } from "./functions/scalar.js";
 export { defineTableFunction, type TableFunctionConfig, type TableBindParams, type TableProcessParams } from "./functions/table.js";
 export { defineAggregate, GROUP_COLUMN_NAME, type AggregateFunctionConfig, type AggregateBindParams, type AggregateUpdateParams, type AggregateFinalizeParams, } from "./functions/aggregate.js";

package/dist/index.js

@@ -2546,6 +2546,55 @@ var init_storage = __esm(() => {
   });
 });
 
+// src/cache-control.ts
+var CACHE_TTL_KEY = "vgi.cache.ttl";
+var CACHE_EXPIRES_KEY = "vgi.cache.expires";
+var CACHE_NO_STORE_KEY = "vgi.cache.no_store";
+var CACHE_SCOPE_KEY = "vgi.cache.scope";
+var CACHE_ETAG_KEY = "vgi.cache.etag";
+var CACHE_LAST_MODIFIED_KEY = "vgi.cache.last_modified";
+var CACHE_REVALIDATABLE_KEY = "vgi.cache.revalidatable";
+var CACHE_STALE_WHILE_REVALIDATE_KEY = "vgi.cache.stale_while_revalidate";
+var CACHE_STALE_IF_ERROR_KEY = "vgi.cache.stale_if_error";
+var CACHE_NOT_MODIFIED_KEY = "vgi.cache.not_modified";
+var CACHE_IF_NONE_MATCH_KEY = "vgi.cache.if_none_match";
+var CACHE_IF_MODIFIED_SINCE_KEY = "vgi.cache.if_modified_since";
+var CACHE_SCOPE_CATALOG = "catalog";
+var CACHE_SCOPE_TRANSACTION = "transaction";
+var VALID_SCOPES = [CACHE_SCOPE_CATALOG, CACHE_SCOPE_TRANSACTION];
+function cacheControlMetadata(cc, extra) {
+  const scope = cc.scope ?? CACHE_SCOPE_CATALOG;
+  if (!VALID_SCOPES.includes(scope)) {
+    throw new Error(`CacheControl.scope must be one of ${VALID_SCOPES.join(", ")}, got ${JSON.stringify(scope)}`);
+  }
+  for (const name of ["ttl", "staleWhileRevalidate", "staleIfError"]) {
+    const value = cc[name];
+    if (value !== undefined && value < 0) {
+      throw new Error(`CacheControl.${name} must be >= 0, got ${value}`);
+    }
+  }
+  const md = new Map(extra ?? []);
+  if (cc.ttl !== undefined)
+    md.set(CACHE_TTL_KEY, String(cc.ttl));
+  if (cc.expires !== undefined)
+    md.set(CACHE_EXPIRES_KEY, cc.expires);
+  if (cc.noStore)
+    md.set(CACHE_NO_STORE_KEY, "1");
+  md.set(CACHE_SCOPE_KEY, scope);
+  if (cc.etag !== undefined)
+    md.set(CACHE_ETAG_KEY, cc.etag);
+  if (cc.lastModified !== undefined)
+    md.set(CACHE_LAST_MODIFIED_KEY, cc.lastModified);
+  if (cc.revalidatable)
+    md.set(CACHE_REVALIDATABLE_KEY, "1");
+  if (cc.staleWhileRevalidate !== undefined)
+    md.set(CACHE_STALE_WHILE_REVALIDATE_KEY, String(cc.staleWhileRevalidate));
+  if (cc.staleIfError !== undefined)
+    md.set(CACHE_STALE_IF_ERROR_KEY, String(cc.staleIfError));
+  if (cc.notModified)
+    md.set(CACHE_NOT_MODIFIED_KEY, "1");
+  return md;
+}
 // src/functions/scalar.ts
 init_arrow();
 
@@ -3989,6 +4038,12 @@ function defineTableFunction(config) {
         onTick: (pState, tickMetadata) => {
           if (!tickMetadata)
             return;
+          const ifNoneMatch = tickMetadata.get(CACHE_IF_NONE_MATCH_KEY);
+          const ifModifiedSince = tickMetadata.get(CACHE_IF_MODIFIED_SINCE_KEY);
+          if (ifNoneMatch !== undefined)
+            pState.processParams.ifNoneMatch = ifNoneMatch;
+          if (ifModifiedSince !== undefined)
+            pState.processParams.ifModifiedSince = ifModifiedSince;
           const encoded = tickMetadata.get("vgi_pushdown_filters");
           if (!encoded)
             return;
@@ -10172,7 +10227,13 @@ async function buildSchemas(iface, aod) {
     });
     functions.push(...macrosRaw.map((m) => macroToObject(m)));
     functions.sort((a, b) => String(a.type).localeCompare(String(b.type)) || String(a.name).localeCompare(String(b.name)));
-    schemas.push({ name: si.name, tables, views, functions });
+    const schemaObj = { name: si.name };
+    if (si.comment)
+      schemaObj.doc = si.comment;
+    schemaObj.tables = tables;
+    schemaObj.views = views;
+    schemaObj.functions = functions;
+    schemas.push(schemaObj);
     counts.schemas += 1;
     counts.tables += tables.length;
     counts.views += views.length;
@@ -11261,6 +11322,145 @@ class VgiClient {
 // src/index.core.ts
 import { str, bytes, int as int3, int32 as int323, float, float32 as float323, bool as bool3, toSchema, OutputCollector, AuthContext } from "@query-farm/vgi-rpc";
 import { subprocessConnect, httpConnect, tcpConnect } from "@query-farm/vgi-rpc";
+// src/worker.ts
+import { VgiRpcServer, serveUnix, serveTcp, serveStream as serveStreamRpc } from "@query-farm/vgi-rpc";
+function parseTcpSpec(spec, out) {
+  const idx = spec.lastIndexOf(":");
+  if (idx >= 0) {
+    const h = spec.slice(0, idx);
+    out.tcpHost = h === "" ? "127.0.0.1" : h;
+    out.tcpPort = Number(spec.slice(idx + 1));
+  } else {
+    out.tcpHost = "127.0.0.1";
+    out.tcpPort = Number(spec);
+  }
+}
+function parseLauncherArgs(argv) {
+  const out = {};
+  for (let i = 0;i < argv.length; i++) {
+    const a = argv[i];
+    if (a === "--unix" && i + 1 < argv.length) {
+      out.unixPath = argv[++i];
+    } else if (a.startsWith("--unix=")) {
+      out.unixPath = a.slice("--unix=".length);
+    } else if (a === "--tcp" && i + 1 < argv.length) {
+      parseTcpSpec(argv[++i], out);
+    } else if (a.startsWith("--tcp=")) {
+      parseTcpSpec(a.slice("--tcp=".length), out);
+    } else if (a === "--idle-timeout" && i + 1 < argv.length) {
+      out.idleTimeout = Number(argv[++i]);
+    } else if (a.startsWith("--idle-timeout=")) {
+      out.idleTimeout = Number(a.slice("--idle-timeout=".length));
+    }
+  }
+  return out;
+}
+
+class Worker {
+  _registry;
+  _catalogInterface;
+  _catalogName;
+  constructor(config) {
+    this._registry = new FunctionRegistry;
+    this._catalogName = config.catalogName;
+    if (config.functions) {
+      for (const func of config.functions) {
+        this._registry.register(func);
+      }
+    }
+    if (config.catalog) {
+      for (const schema3 of config.catalog.schemas) {
+        if (schema3.functions) {
+          for (const func of schema3.functions) {
+            this._registry.register(func);
+          }
+        }
+      }
+    }
+    if (config.catalogInterface) {
+      this._catalogInterface = config.catalogInterface;
+    } else if (config.catalog) {
+      const base = new ReadOnlyCatalogInterface(config.catalog, this._registry);
+      this._catalogInterface = config.catalogInterfaceFactory ? config.catalogInterfaceFactory(base) : base;
+    }
+  }
+  run(argv = process.argv.slice(2)) {
+    process.stderr.write(`[worker] starting, pid=${process.pid}
+`);
+    try {
+      const protocol = this.buildProtocol();
+      process.stderr.write(`[worker] protocol built
+`);
+      const launcher = parseLauncherArgs(argv);
+      if (launcher.unixPath !== undefined) {
+        const envOverride = process.env.VGI_WORKER_IDLE_TIMEOUT;
+        const idleTimeout = envOverride !== undefined && envOverride !== "" ? Number(envOverride) : launcher.idleTimeout;
+        process.stderr.write(`[worker] AF_UNIX mode: ${launcher.unixPath} idle=${idleTimeout ?? 300}s${envOverride !== undefined && envOverride !== "" ? " (env override)" : ""}
+`);
+        serveUnix(protocol, {
+          unixPath: launcher.unixPath,
+          idleTimeout
+        }).then((handle) => handle.done).then(() => {
+          process.stderr.write(`[worker] serveUnix done (idle shutdown)
+`);
+        }).catch((err2) => {
+          process.stderr.write(`Worker error: ${err2.message}
+${err2.stack}
+`);
+          process.exit(1);
+        });
+        return;
+      }
+      if (launcher.tcpPort !== undefined) {
+        const envOverride = process.env.VGI_WORKER_IDLE_TIMEOUT;
+        const idleTimeout = envOverride !== undefined && envOverride !== "" ? Number(envOverride) : launcher.idleTimeout;
+        process.stderr.write(`[worker] TCP mode: ${launcher.tcpHost}:${launcher.tcpPort} idle=${idleTimeout ?? 300}s${envOverride !== undefined && envOverride !== "" ? " (env override)" : ""}
+`);
+        serveTcp(protocol, {
+          host: launcher.tcpHost,
+          port: launcher.tcpPort,
+          idleTimeout
+        }).then((handle) => handle.done).then(() => {
+          process.stderr.write(`[worker] serveTcp done (idle shutdown)
+`);
+        }).catch((err2) => {
+          process.stderr.write(`Worker error: ${err2.message}
+${err2.stack}
+`);
+          process.exit(1);
+        });
+        return;
+      }
+      const server = new VgiRpcServer(protocol);
+      process.stderr.write(`[worker] server created, calling run()
+`);
+      server.run().then(() => {
+        process.stderr.write(`[worker] server.run() resolved cleanly
+`);
+      }).catch((err2) => {
+        process.stderr.write(`Worker error: ${err2.message}
+${err2.stack}
+`);
+        process.exit(1);
+      });
+    } catch (err2) {
+      process.stderr.write(`Worker init error: ${err2.message}
+${err2.stack}
+`);
+      process.exit(1);
+    }
+  }
+  buildProtocol() {
+    return buildVgiProtocol({
+      registry: this._registry,
+      catalogInterface: this._catalogInterface,
+      catalogName: this._catalogName
+    });
+  }
+  async serveStream(readable, writable) {
+    await serveStreamRpc(this.buildProtocol(), { readable, writable });
+  }
+}
 export {
   utf82 as utf8,
   union2 as union,
@@ -11396,6 +11596,7 @@ export {
   constraintSpecFields,
   columnFromArray,
   codecFor,
+  cacheControlMetadata,
   bytes,
   buildVgiProtocol,
   buildScanBranchesResult,
@@ -11469,6 +11670,20 @@ export {
   CatalogFunctionType,
   CatalogAlreadyExistsError,
   CUPOLA_BASE,
+  CACHE_TTL_KEY,
+  CACHE_STALE_WHILE_REVALIDATE_KEY,
+  CACHE_STALE_IF_ERROR_KEY,
+  CACHE_SCOPE_TRANSACTION,
+  CACHE_SCOPE_KEY,
+  CACHE_SCOPE_CATALOG,
+  CACHE_REVALIDATABLE_KEY,
+  CACHE_NO_STORE_KEY,
+  CACHE_NOT_MODIFIED_KEY,
+  CACHE_LAST_MODIFIED_KEY,
+  CACHE_IF_NONE_MATCH_KEY,
+  CACHE_IF_MODIFIED_SINCE_KEY,
+  CACHE_EXPIRES_KEY,
+  CACHE_ETAG_KEY,
   BoundStorage,
   BindResultSchema,
   AuthContext,
@@ -11476,4 +11691,4 @@ export {
   ArgumentValidationError
 };
 
-//# debugId=55E5F7DD831BB4DD64756E2164756E21
+//# debugId=0510FF19793B06C764756E2164756E21