@malloy-publisher/server 0.0.198 → 0.0.200

package/src/package_load/protocol.ts

@@ -0,0 +1,336 @@
+/**
+ * Wire protocol between the main thread (`PackageLoadPool`) and a
+ * package-load worker thread.
+ *
+ * Boundary
+ * --------
+ * The worker performs the **CPU-bound bulk of `Package.create`** off
+ * the main event loop:
+ *
+ *   1. Read `publisher.config.json` (cheap, but already on the worker
+ *      side of the boundary so the main thread isn't blocked).
+ *   2. Compile every `.malloy` / `.malloynb` (the Malloy parser,
+ *      type-checker, and IR-builder — the dominant CPU cost).
+ *   3. Return a structured-clonable POJO carrying every `modelDef`,
+ *      `sourceInfos`, dataStyles, etc. that the main thread needs to
+ *      reconstitute a live `Package`.
+ *
+ * Embedded database probing (`.parquet` / `.csv` schema + row count)
+ * stays on the main thread — it reuses the package's existing DuckDB
+ * connection (PR #772) and the probe queries are async-IO-bound, not
+ * CPU-bound. Keeping all native-DB handles on the main thread also
+ * sidesteps Bun crash 0x20131 where duckdb-native cannot be loaded
+ * into more than one isolate of the same process.
+ *
+ * The main thread reconstitutes by:
+ *   - Building a fresh `MalloyConfig` against its own connection pool
+ *     (live native handles can't cross the worker boundary).
+ *   - Lazy-hydrating each model's `ModelMaterializer` from `modelDef`
+ *     via `Runtime._loadModelFromModelDef` on first query — NO
+ *     recompile. This is what closes the loop on PR #767's original
+ *     "first-query recompile on main thread" gap.
+ *
+ * Per-model compile failures are returned in-band on
+ * `SerializedModel.compilationError` so a single bad model doesn't
+ * abort the rest of the package load. The main thread decides
+ * whether/when to surface as a fatal `Package.create` error (today
+ * it throws on the first error; `Package.reloadAllModels` keeps the
+ * failed models as placeholders in the package's model map).
+ *
+ * Whole-package failures (manifest missing, FS errors, worker
+ * crashes) come back as `LoadPackageError`. The pool main-thread
+ * half (`PackageLoadPool.loadPackage`) rejects with a deserialised
+ * Error; `Package.loadViaWorker` then rewraps any non-compile
+ * failure as `ServiceUnavailableError` so the HTTP layer responds
+ * 503 (transient, retryable) — there is no in-process fallback.
+ *
+ * Direction summary
+ * -----------------
+ *   main  ──▶ worker:  LoadPackageRequest         (start)
+ *   worker ──▶ main:   LoadPackageResult          (success)
+ *   worker ──▶ main:   LoadPackageError           (whole-package failure)
+ *
+ *   worker ──▶ main:   ConnectionMetadataRequest  (proxy non-duckdb lookups)
+ *   worker ──▶ main:   SchemaForTablesRequest     (proxy schema fetch)
+ *   worker ──▶ main:   SchemaForSqlRequest        (proxy SQL block schema)
+ *   worker ──▶ main:   ReadUrlRequest             (proxy non-file URL reads)
+ *   main  ──▶ worker:  *Response                  (correlated by requestId)
+ *
+ *   main  ──▶ worker:  ShutdownRequest            (graceful drain)
+ *   worker ──▶ main:   ReadyMessage               (post-init handshake)
+ *
+ * The protocol uses plain structured-clonable POJOs so the
+ * `postMessage` transfer goes through V8's structured clone — much
+ * cheaper than `JSON.stringify` for the multi-MB modelDef payloads.
+ */
+
+import type {
+   Annotation,
+   SQLSourceDef,
+   TableSourceDef,
+} from "@malloydata/malloy";
+
+// ──────────────────────────────────────────────────────────────────────
+// Direction: main ──▶ worker (load-package job)
+// ──────────────────────────────────────────────────────────────────────
+
+/**
+ * Connection metadata the worker needs to construct a stub
+ * `InfoConnection`. Resolved lazily — the worker asks the main thread
+ * on the first `lookupConnection(name)` call (see
+ * {@link ConnectionMetadataRequest}). We don't ship the full list
+ * upfront because Malloy only references connections by name as it
+ * encounters `<connection>.table('...')` / `<connection>.sql('...')`
+ * inside the model.
+ */
+export interface ConnectionMetadata {
+   name: string;
+   dialectName: string;
+   digest: string;
+}
+
+export interface LoadPackageRequest {
+   type: "load-package";
+   requestId: string;
+   /** Absolute path to the package directory on disk. */
+   packagePath: string;
+   /** Logical package name (used in metric labels + log fields). */
+   packageName: string;
+   /**
+    * Default connection name (passed verbatim to the worker; today
+    * always `"duckdb"` for embedded packages, but kept configurable
+    * to mirror Malloy's own surface).
+    */
+   defaultConnectionName: string | null;
+   /** Optional row-build manifest passed through to Malloy Runtime. */
+   buildManifest?: unknown;
+}
+
+// ──────────────────────────────────────────────────────────────────────
+// Direction: worker ──▶ main (load-package result)
+// ──────────────────────────────────────────────────────────────────────
+
+/**
+ * Wire shape for one compiled model in the package. Mirrors the
+ * data a main-thread `Model` constructor needs without holding a
+ * `ModelMaterializer` reference (that binds to live native
+ * connection handles and can't cross the worker boundary).
+ *
+ * `compilationError` is set when this single model failed to
+ * compile but the rest of the package is fine; the main thread
+ * decides whether to abort `Package.create`.
+ */
+export interface SerializedModel {
+   /** Path relative to the package root, forward-slash normalized. */
+   modelPath: string;
+   modelType: "model" | "notebook";
+   /** Set when the model compiled successfully. Wire-typed as
+    *  `unknown` so the protocol module doesn't drag in the full
+    *  Malloy type surface; cast to `ModelDef` on receipt. */
+   modelDef?: unknown;
+   /**
+    * Precomputed `modelDefToModelInfo(modelDef)`. Shipped from the
+    * worker so the main-thread `Model` constructor doesn't pay the
+    * derivation cost on every package load and every subsequent
+    * `getModel()` / `getNotebook()` API hit can stringify a cached
+    * object instead of recomputing.
+    */
+   modelInfo?: unknown;
+   sourceInfos?: unknown[];
+   sources?: unknown[];
+   queries?: unknown[];
+   filterMap?: Array<[string, unknown[]]>;
+   givens?: unknown[];
+   /** Notebook (.malloynb) only — per-cell pre-extracted info. */
+   notebookCells?: SerializedNotebookCell[];
+   /** Accumulated dataStyles from sibling `.styles.json` files. */
+   dataStyles?: unknown;
+   /** Wall-clock ms spent compiling this single model in the worker. */
+   compileDurationMs?: number;
+   /** Set when the model failed to compile. */
+   compilationError?: SerializedError;
+}
+
+export interface SerializedNotebookCell {
+   type: "code" | "markdown";
+   /** Raw cell text. */
+   text: string;
+   /**
+    * Per-cell ModelDef captured at the cell's point in the
+    * `extendModel` chain. The main thread hydrates a per-cell
+    * `ModelMaterializer` from this via
+    * `Runtime._loadModelFromModelDef`, so cell-level filter
+    * refinement can compile new queries against the correct scope
+    * without ever recompiling the .malloynb itself.
+    */
+   cellModelDef?: unknown;
+   /**
+    * The final-query QueryDef for this cell, captured during the
+    * worker's compile. Main thread hydrates a `QueryMaterializer`
+    * via `ModelMaterializer._loadQueryFromQueryDef` — no recompile.
+    */
+   cellQueryDef?: unknown;
+   newSources?: unknown[];
+   queryInfo?: unknown;
+}
+
+export interface LoadPackageResult {
+   type: "load-package-result";
+   requestId: string;
+   packageMetadata: { name?: string; description?: string };
+   models: SerializedModel[];
+   /** Wall-clock ms inside the worker for the full package load. */
+   loadDurationMs: number;
+}
+
+export interface LoadPackageError {
+   type: "load-package-error";
+   requestId: string;
+   error: SerializedError;
+}
+
+/**
+ * Error wire-shape. We cannot transfer `Error` instances directly
+ * across `postMessage` cleanly (Bun/Node behaviour diverges on stack
+ * propagation), so we ship a structured payload and reconstitute on
+ * the main thread.
+ */
+export interface SerializedError {
+   name: string;
+   message: string;
+   stack?: string;
+   /** Set when the error originated as a Malloy `MalloyError`. */
+   malloyProblems?: unknown[];
+   /** Set when the error originated as `ModelCompilationError`. */
+   isCompilationError?: boolean;
+}
+
+// ──────────────────────────────────────────────────────────────────────
+// Direction: worker ──▶ main (proxy connection metadata)
+// ──────────────────────────────────────────────────────────────────────
+
+export interface ConnectionMetadataRequest {
+   type: "connection-metadata";
+   requestId: string;
+   jobId: string;
+   connectionName: string;
+}
+
+export interface ConnectionMetadataResponse {
+   type: "connection-metadata-response";
+   requestId: string;
+   ok: true;
+   metadata: ConnectionMetadata;
+}
+
+// ──────────────────────────────────────────────────────────────────────
+// Direction: worker ──▶ main (proxy schema fetches for non-duckdb)
+// ──────────────────────────────────────────────────────────────────────
+
+export interface SchemaForTablesRequest {
+   type: "schema-for-tables";
+   requestId: string;
+   /** Job this RPC belongs to (so main routes to the right config). */
+   jobId: string;
+   connectionName: string;
+   tables: Record<string, string>;
+   options: {
+      refreshTimestamp?: number;
+      modelAnnotation?: Annotation;
+   };
+}
+
+export interface SchemaForTablesResponse {
+   type: "schema-for-tables-response";
+   requestId: string;
+   ok: true;
+   schemas: Record<string, TableSourceDef>;
+   errors: Record<string, string>;
+}
+
+export interface SchemaForSqlRequest {
+   type: "schema-for-sql";
+   requestId: string;
+   jobId: string;
+   connectionName: string;
+   sentence: unknown;
+   options: {
+      refreshTimestamp?: number;
+      modelAnnotation?: Annotation;
+   };
+}
+
+export interface SchemaForSqlResponse {
+   type: "schema-for-sql-response";
+   requestId: string;
+   ok: true;
+   structDef?: SQLSourceDef;
+   error?: string;
+}
+
+export interface RpcErrorResponse {
+   type: "rpc-error";
+   requestId: string;
+   ok: false;
+   error: SerializedError;
+}
+
+// ──────────────────────────────────────────────────────────────────────
+// Direction: worker ──▶ main (file read for non-file URLs)
+// ──────────────────────────────────────────────────────────────────────
+
+/**
+ * Workers read most files directly via `fs` (they share the host's
+ * filesystem namespace). This RPC exists for the rare case where the
+ * package URL reader has host-specific behaviour (e.g. virtual files,
+ * remote URLs) — we delegate back to the main thread's URL reader so
+ * compile semantics stay identical to the in-process path.
+ */
+export interface ReadUrlRequest {
+   type: "read-url";
+   requestId: string;
+   jobId: string;
+   url: string;
+}
+
+export interface ReadUrlResponse {
+   type: "read-url-response";
+   requestId: string;
+   ok: true;
+   contents: string;
+   invalidationKey?: string | number | null;
+}
+
+// ──────────────────────────────────────────────────────────────────────
+// Lifecycle
+// ──────────────────────────────────────────────────────────────────────
+
+export interface ShutdownRequest {
+   type: "shutdown";
+}
+
+export interface ReadyMessage {
+   type: "ready";
+}
+
+// ──────────────────────────────────────────────────────────────────────
+// Union types for routing
+// ──────────────────────────────────────────────────────────────────────
+
+export type MainToWorkerMessage =
+   | LoadPackageRequest
+   | ConnectionMetadataResponse
+   | SchemaForTablesResponse
+   | SchemaForSqlResponse
+   | ReadUrlResponse
+   | RpcErrorResponse
+   | ShutdownRequest;
+
+export type WorkerToMainMessage =
+   | LoadPackageResult
+   | LoadPackageError
+   | ConnectionMetadataRequest
+   | SchemaForTablesRequest
+   | SchemaForSqlRequest
+   | ReadUrlRequest
+   | ReadyMessage;

package/src/path_safety.ts

@@ -44,7 +44,9 @@ const MAX_ENVIRONMENT_PATH_LEN = 4096;
  * production package name we've seen fits within it, and tightening
  * here costs nothing.
  */
-export function assertSafePackageName(packageName: unknown): void {
+export function assertSafePackageName(
+   packageName: unknown,
+): asserts packageName is string {
    if (typeof packageName !== "string" || !SAFE_NAME_RE.test(packageName)) {
       throw new BadRequestError(
          `Invalid package name: must be 1-255 characters of letters, digits, "-", "_", or "." and must not start with "."`,
@@ -58,7 +60,9 @@ export function assertSafePackageName(packageName: unknown): void {
  * live in subdirectories like `models/foo.malloy`); backslashes,
  * absolute paths, NUL bytes, and `..` / `.` segments are not.
  */
-export function assertSafeRelativeModelPath(modelPath: unknown): void {
+export function assertSafeRelativeModelPath(
+   modelPath: unknown,
+): asserts modelPath is string {
    if (
       typeof modelPath !== "string" ||
       modelPath.length === 0 ||
@@ -90,7 +94,9 @@ export function assertSafeRelativeModelPath(modelPath: unknown): void {
  * sanitizer-barrier pattern CodeQL's `js/path-injection` query
  * recognises.
  */
-export function assertSafeEnvironmentPath(environmentPath: unknown): void {
+export function assertSafeEnvironmentPath(
+   environmentPath: unknown,
+): asserts environmentPath is string {
    if (typeof environmentPath !== "string") {
       throw new BadRequestError(`Invalid environment path: must be a string`);
    }

package/src/query_cap_metrics.spec.ts

@@ -0,0 +1,89 @@
+import { afterEach, beforeEach, describe, expect, it } from "bun:test";
+
+import {
+   recordQueryCapExceeded,
+   resetQueryCapTelemetryForTesting,
+} from "./query_cap_metrics";
+import {
+   startMetricsHarness,
+   type MetricsHarness,
+} from "./test_helpers/metrics_harness";
+
+describe("query_cap_metrics", () => {
+   let harness: MetricsHarness;
+
+   beforeEach(async () => {
+      harness = await startMetricsHarness();
+      // Drop cached instruments so they re-init against the new
+      // provider; otherwise this test's writes go to a counter
+      // bound to the previous provider's reader.
+      resetQueryCapTelemetryForTesting();
+   });
+
+   afterEach(async () => {
+      delete process.env.PUBLISHER_MAX_QUERY_ROWS;
+      delete process.env.PUBLISHER_MAX_RESPONSE_BYTES;
+      resetQueryCapTelemetryForTesting();
+      await harness.shutdown();
+   });
+
+   it("publisher_query_cap_exceeded_total ticks per call, labeled by cap_type and source", async () => {
+      recordQueryCapExceeded("rows", "connection_sql");
+      recordQueryCapExceeded("rows", "connection_sql");
+      recordQueryCapExceeded("bytes", "model_query");
+      recordQueryCapExceeded("rows", "notebook_cell");
+
+      expect(
+         await harness.collectCounter("publisher_query_cap_exceeded_total", {
+            cap_type: "rows",
+            source: "connection_sql",
+         }),
+      ).toBe(2);
+      expect(
+         await harness.collectCounter("publisher_query_cap_exceeded_total", {
+            cap_type: "bytes",
+            source: "model_query",
+         }),
+      ).toBe(1);
+      expect(
+         await harness.collectCounter("publisher_query_cap_exceeded_total", {
+            cap_type: "rows",
+            source: "notebook_cell",
+         }),
+      ).toBe(1);
+   });
+
+   it("publisher_max_query_rows gauge reports the live env-var value", async () => {
+      process.env.PUBLISHER_MAX_QUERY_ROWS = "12345";
+      // Prime telemetry — the gauges install on the first
+      // counter-emitting call (`recordQueryCapExceeded`); in
+      // production that's the first 413, in tests we trigger it
+      // explicitly so the gauge is observable without a 413.
+      recordQueryCapExceeded("rows", "connection_sql");
+      expect(await harness.collectGauge("publisher_max_query_rows")).toBe(
+         12345,
+      );
+   });
+
+   it("publisher_max_response_bytes gauge reports the live env-var value", async () => {
+      process.env.PUBLISHER_MAX_RESPONSE_BYTES = "9876543";
+      recordQueryCapExceeded("bytes", "connection_sql");
+      expect(await harness.collectGauge("publisher_max_response_bytes")).toBe(
+         9876543,
+      );
+   });
+
+   it("publisher_max_query_rows gauge reports 0 when the cap is opted out", async () => {
+      process.env.PUBLISHER_MAX_QUERY_ROWS = "0";
+      recordQueryCapExceeded("rows", "connection_sql");
+      expect(await harness.collectGauge("publisher_max_query_rows")).toBe(0);
+   });
+
+   it("publisher_max_query_rows gauge reports -1 on misconfig so dashboards reveal the bad value", async () => {
+      process.env.PUBLISHER_MAX_QUERY_ROWS = "not-a-number";
+      // Misconfig must not crash the scrape; -1 is the agreed
+      // signal mirroring `publisher_query_timeout_ms`.
+      recordQueryCapExceeded("rows", "connection_sql");
+      expect(await harness.collectGauge("publisher_max_query_rows")).toBe(-1);
+   });
+});

package/src/query_cap_metrics.ts

@@ -0,0 +1,115 @@
+/**
+ * Centralized telemetry for row-cap / byte-cap rejections (HTTP 413).
+ *
+ * Without this, operators tuning {@link PUBLISHER_MAX_QUERY_ROWS} /
+ * {@link PUBLISHER_MAX_RESPONSE_BYTES} can only see undifferentiated
+ * `http_server_requests_total{status_code="413"}` — they can't tell
+ * which cap is firing or which query surface is hottest. The counter
+ * here carries `cap_type` (`rows` / `bytes`) and `source`
+ * (`connection_sql` / `model_query` / `notebook_cell`) so a single
+ * dashboard panel can answer "what should I tune and on which
+ * endpoint?".
+ *
+ * Observable gauges expose the current effective caps so dashboards
+ * can render `actual_rows_returned / max_rows` utilization without
+ * a separate config feed — same pattern the memory governor uses
+ * for high/low water bytes and the concurrency middleware uses for
+ * its slot cap.
+ *
+ * Lazy init for the same reason as `query_timeout.ts` /
+ * `query_concurrency.ts`: instruments created before
+ * `setGlobalMeterProvider` bind to a NoOp meter
+ * (https://github.com/open-telemetry/opentelemetry-js/issues/3505).
+ * The first throw site initializes the counter; the gauges are
+ * installed alongside on the same call so the production hot path
+ * is one boolean check after the first 413.
+ */
+
+import { metrics, type Counter } from "@opentelemetry/api";
+
+import { getMaxQueryRows, getMaxResponseBytes } from "./config";
+
+export type QueryCapType = "rows" | "bytes";
+export type QueryCapSource = "connection_sql" | "model_query" | "notebook_cell";
+
+let capExceededCounter: Counter | null = null;
+let configGaugesInstalled = false;
+
+function ensureCapTelemetry(): Counter {
+   if (capExceededCounter && configGaugesInstalled) {
+      return capExceededCounter;
+   }
+   const meter = metrics.getMeter("publisher");
+   if (!capExceededCounter) {
+      capExceededCounter = meter.createCounter(
+         "publisher_query_cap_exceeded_total",
+         {
+            description:
+               "Queries rejected with 413 because the row or byte cap was exceeded. Labels: cap_type ('rows'|'bytes'), source ('connection_sql'|'model_query'|'notebook_cell').",
+         },
+      );
+   }
+   if (!configGaugesInstalled) {
+      // Live config readouts so dashboards can render
+      // "actual / max" utilization for the row and byte caps the
+      // same way `publisher_memory_*_bytes` does for the governor.
+      // Read on every scrape so a runtime env-var change is
+      // visible without a restart; an env-var parse failure
+      // reports -1 so misconfig is visible rather than silently
+      // dropped (mirrors `publisher_query_timeout_ms`).
+      meter
+         .createObservableGauge("publisher_max_query_rows", {
+            description:
+               "Current effective PUBLISHER_MAX_QUERY_ROWS cap (0 = disabled, -1 = misconfigured)",
+         })
+         .addCallback((observation) => {
+            try {
+               observation.observe(getMaxQueryRows());
+            } catch {
+               observation.observe(-1);
+            }
+         });
+      meter
+         .createObservableGauge("publisher_max_response_bytes", {
+            description:
+               "Current effective PUBLISHER_MAX_RESPONSE_BYTES cap (0 = disabled, -1 = misconfigured)",
+            unit: "By",
+         })
+         .addCallback((observation) => {
+            try {
+               observation.observe(getMaxResponseBytes());
+            } catch {
+               observation.observe(-1);
+            }
+         });
+      configGaugesInstalled = true;
+   }
+   return capExceededCounter;
+}
+
+/**
+ * Record a single 413 cap-exceeded event. Call BEFORE throwing
+ * `PayloadTooLargeError` so the metric ticks even if a downstream
+ * `catch` swallows the error (MCP tools surface failures as content
+ * payloads rather than letting them bubble to the HTTP error
+ * mapper).
+ *
+ * `cap_type` must be one of `rows` / `bytes`; `source` identifies
+ * the query surface that detected the overflow.
+ */
+export function recordQueryCapExceeded(
+   capType: QueryCapType,
+   source: QueryCapSource,
+): void {
+   ensureCapTelemetry().add(1, { cap_type: capType, source });
+}
+
+/**
+ * Visible for tests. Drops the cached instruments so a fresh
+ * `MeterProvider` (installed via `startMetricsHarness`) can capture
+ * future emissions. Do NOT call from production code.
+ */
+export function resetQueryCapTelemetryForTesting(): void {
+   capExceededCounter = null;
+   configGaugesInstalled = false;
+}