@malloy-publisher/server 0.0.235 → 0.0.236

package/src/package_load/protocol.ts

@@ -1,364 +0,0 @@
-/**
- * Wire protocol between the main thread (`PackageLoadPool`) and a
- * package-load worker thread.
- *
- * Boundary
- * --------
- * The worker performs the **CPU-bound bulk of `Package.create`** off
- * the main event loop:
- *
- *   1. Read `publisher.config.json` (cheap, but already on the worker
- *      side of the boundary so the main thread isn't blocked).
- *   2. Compile every `.malloy` / `.malloynb` (the Malloy parser,
- *      type-checker, and IR-builder — the dominant CPU cost).
- *   3. Return a structured-clonable POJO carrying every `modelDef`,
- *      `sourceInfos`, dataStyles, etc. that the main thread needs to
- *      reconstitute a live `Package`.
- *
- * Embedded database probing (`.parquet` / `.csv` schema + row count)
- * stays on the main thread — it reuses the package's existing DuckDB
- * connection and the probe queries are async-IO-bound, not
- * CPU-bound. Keeping all native-DB handles on the main thread also
- * sidesteps Bun crash 0x20131 where duckdb-native cannot be loaded
- * into more than one isolate of the same process.
- *
- * The main thread reconstitutes by:
- *   - Building a fresh `MalloyConfig` against its own connection pool
- *     (live native handles can't cross the worker boundary).
- *   - Lazy-hydrating each model's `ModelMaterializer` from `modelDef`
- *     via `Runtime._loadModelFromModelDef` on first query — NO
- *     recompile, so the first query never re-runs the compiler on
- *     the main thread.
- *
- * Per-model compile failures are returned in-band on
- * `SerializedModel.compilationError` so a single bad model doesn't
- * abort the rest of the package load. The main thread decides
- * whether/when to surface as a fatal `Package.create` error (today
- * it throws on the first error; `Package.reloadAllModels` keeps the
- * failed models as placeholders in the package's model map).
- *
- * Whole-package failures (manifest missing, FS errors, worker
- * crashes) come back as `LoadPackageError`. The pool main-thread
- * half (`PackageLoadPool.loadPackage`) rejects with a deserialised
- * Error; `Package.loadViaWorker` then rewraps any non-compile
- * failure as `ServiceUnavailableError` so the HTTP layer responds
- * 503 (transient, retryable) — there is no in-process fallback.
- *
- * Direction summary
- * -----------------
- *   main  ──▶ worker:  LoadPackageRequest         (start)
- *   worker ──▶ main:   LoadPackageResult          (success)
- *   worker ──▶ main:   LoadPackageError           (whole-package failure)
- *
- *   worker ──▶ main:   ConnectionMetadataRequest  (proxy non-duckdb lookups)
- *   worker ──▶ main:   SchemaForTablesRequest     (proxy schema fetch)
- *   worker ──▶ main:   SchemaForSqlRequest        (proxy SQL block schema)
- *   worker ──▶ main:   ReadUrlRequest             (proxy non-file URL reads)
- *   main  ──▶ worker:  *Response                  (correlated by requestId)
- *
- *   main  ──▶ worker:  ShutdownRequest            (graceful drain)
- *   worker ──▶ main:   ReadyMessage               (post-init handshake)
- *
- * The protocol uses plain structured-clonable POJOs so the
- * `postMessage` transfer goes through V8's structured clone — much
- * cheaper than `JSON.stringify` for the multi-MB modelDef payloads.
- */
-
-import type { SQLSourceDef, TableSourceDef } from "@malloydata/malloy";
-import type {
-   PackageMaterializationConfig,
-   PackageScope,
-} from "../service/package_manifest";
-
-// ──────────────────────────────────────────────────────────────────────
-// Direction: main ──▶ worker (load-package job)
-// ──────────────────────────────────────────────────────────────────────
-
-/**
- * Connection metadata the worker needs to construct a stub
- * `InfoConnection`. Resolved lazily — the worker asks the main thread
- * on the first `lookupConnection(name)` call (see
- * {@link ConnectionMetadataRequest}). We don't ship the full list
- * upfront because Malloy only references connections by name as it
- * encounters `<connection>.table('...')` / `<connection>.sql('...')`
- * inside the model.
- */
-export interface ConnectionMetadata {
-   name: string;
-   dialectName: string;
-   digest: string;
-}
-
-export interface LoadPackageRequest {
-   type: "load-package";
-   requestId: string;
-   /** Absolute path to the package directory on disk. */
-   packagePath: string;
-   /** Logical package name (used in metric labels + log fields). */
-   packageName: string;
-   /**
-    * Default connection name (passed verbatim to the worker; today
-    * always `"duckdb"` for embedded packages, but kept configurable
-    * to mirror Malloy's own surface).
-    */
-   defaultConnectionName: string | null;
-   /** Optional row-build manifest passed through to Malloy Runtime. */
-   buildManifest?: unknown;
-}
-
-// ──────────────────────────────────────────────────────────────────────
-// Direction: worker ──▶ main (load-package result)
-// ──────────────────────────────────────────────────────────────────────
-
-/**
- * Wire shape for one compiled model in the package. Mirrors the
- * data a main-thread `Model` constructor needs without holding a
- * `ModelMaterializer` reference (that binds to live native
- * connection handles and can't cross the worker boundary).
- *
- * `compilationError` is set when this single model failed to
- * compile but the rest of the package is fine; the main thread
- * decides whether to abort `Package.create`.
- */
-export interface SerializedModel {
-   /** Path relative to the package root, forward-slash normalized. */
-   modelPath: string;
-   modelType: "model" | "notebook";
-   /** Set when the model compiled successfully. Wire-typed as
-    *  `unknown` so the protocol module doesn't drag in the full
-    *  Malloy type surface; cast to `ModelDef` on receipt. */
-   modelDef?: unknown;
-   /**
-    * Precomputed `modelDefToModelInfo(modelDef)`. Shipped from the
-    * worker so the main-thread `Model` constructor doesn't pay the
-    * derivation cost on every package load and every subsequent
-    * `getModel()` / `getNotebook()` API hit can stringify a cached
-    * object instead of recomputing.
-    */
-   modelInfo?: unknown;
-   sourceInfos?: unknown[];
-   sources?: unknown[];
-   queries?: unknown[];
-   filterMap?: Array<[string, unknown[]]>;
-   givens?: unknown[];
-   /** Notebook (.malloynb) only — per-cell pre-extracted info. */
-   notebookCells?: SerializedNotebookCell[];
-   /** Accumulated dataStyles from sibling `.styles.json` files. */
-   dataStyles?: unknown;
-   /** Wall-clock ms spent compiling this single model in the worker. */
-   compileDurationMs?: number;
-   /** Set when the model failed to compile. */
-   compilationError?: SerializedError;
-}
-
-export interface SerializedNotebookCell {
-   type: "code" | "markdown";
-   /** Raw cell text. */
-   text: string;
-   /**
-    * Per-cell ModelDef captured at the cell's point in the
-    * `extendModel` chain. The main thread hydrates a per-cell
-    * `ModelMaterializer` from this via
-    * `Runtime._loadModelFromModelDef`, so cell-level filter
-    * refinement can compile new queries against the correct scope
-    * without ever recompiling the .malloynb itself.
-    */
-   cellModelDef?: unknown;
-   /**
-    * The final-query QueryDef for this cell, captured during the
-    * worker's compile. Main thread hydrates a `QueryMaterializer`
-    * via `ModelMaterializer._loadQueryFromQueryDef` — no recompile.
-    */
-   cellQueryDef?: unknown;
-   newSources?: unknown[];
-   queryInfo?: unknown;
-}
-
-export interface LoadPackageResult {
-   type: "load-package-result";
-   requestId: string;
-   packageMetadata: {
-      name?: string;
-      description?: string;
-      explores?: string[];
-      queryableSources?: "declared" | "all";
-      manifestLocation?: string | null;
-      materialization?: PackageMaterializationConfig | null;
-      scope?: PackageScope;
-      /**
-       * Manifest-shape deprecations the load tolerated (e.g. a root-level
-       * `scope`). Surfaced with the package's other policy warnings so a
-       * still-parsing-but-outdated manifest is visible without failing a load.
-       */
-      manifestWarnings?: string[];
-   };
-   models: SerializedModel[];
-   /** Wall-clock ms inside the worker for the full package load. */
-   loadDurationMs: number;
-   /**
-    * Per-load timing breakdown (subset of {@link loadDurationMs}; the
-    * remainder is worker setup + post-compile extraction). See
-    * `rpc_wait_accountant.ts` for how compile time is separated from I/O.
-    */
-   timings: {
-      /**
-       * Compile-region wall-clock minus time awaiting proxied schema fetches —
-       * a conservative ceiling on Malloy compile CPU.
-       */
-      compileDurationMs: number;
-      /** Wall-clock awaiting proxied connection schema fetches (union). */
-      schemaFetchDurationMs: number;
-      /** Number of proxied connection schema fetches the load drove. */
-      schemaFetchCount: number;
-   };
-}
-
-export interface LoadPackageError {
-   type: "load-package-error";
-   requestId: string;
-   error: SerializedError;
-}
-
-/**
- * Error wire-shape. We cannot transfer `Error` instances directly
- * across `postMessage` cleanly (Bun/Node behaviour diverges on stack
- * propagation), so we ship a structured payload and reconstitute on
- * the main thread.
- */
-export interface SerializedError {
-   name: string;
-   message: string;
-   stack?: string;
-   /** Set when the error originated as a Malloy `MalloyError`. */
-   malloyProblems?: unknown[];
-   /** Set when the error originated as `ModelCompilationError`. */
-   isCompilationError?: boolean;
-}
-
-// ──────────────────────────────────────────────────────────────────────
-// Direction: worker ──▶ main (proxy connection metadata)
-// ──────────────────────────────────────────────────────────────────────
-
-export interface ConnectionMetadataRequest {
-   type: "connection-metadata";
-   requestId: string;
-   jobId: string;
-   connectionName: string;
-}
-
-export interface ConnectionMetadataResponse {
-   type: "connection-metadata-response";
-   requestId: string;
-   ok: true;
-   metadata: ConnectionMetadata;
-}
-
-// ──────────────────────────────────────────────────────────────────────
-// Direction: worker ──▶ main (proxy schema fetches for non-duckdb)
-// ──────────────────────────────────────────────────────────────────────
-
-export interface SchemaForTablesRequest {
-   type: "schema-for-tables";
-   requestId: string;
-   /** Job this RPC belongs to (so main routes to the right config). */
-   jobId: string;
-   connectionName: string;
-   tables: Record<string, string>;
-   options: {
-      refreshTimestamp?: number;
-   };
-}
-
-export interface SchemaForTablesResponse {
-   type: "schema-for-tables-response";
-   requestId: string;
-   ok: true;
-   schemas: Record<string, TableSourceDef>;
-   errors: Record<string, string>;
-}
-
-export interface SchemaForSqlRequest {
-   type: "schema-for-sql";
-   requestId: string;
-   jobId: string;
-   connectionName: string;
-   sentence: unknown;
-   options: {
-      refreshTimestamp?: number;
-   };
-}
-
-export interface SchemaForSqlResponse {
-   type: "schema-for-sql-response";
-   requestId: string;
-   ok: true;
-   structDef?: SQLSourceDef;
-   error?: string;
-}
-
-export interface RpcErrorResponse {
-   type: "rpc-error";
-   requestId: string;
-   ok: false;
-   error: SerializedError;
-}
-
-// ──────────────────────────────────────────────────────────────────────
-// Direction: worker ──▶ main (file read for non-file URLs)
-// ──────────────────────────────────────────────────────────────────────
-
-/**
- * Workers read most files directly via `fs` (they share the host's
- * filesystem namespace). This RPC exists for the rare case where the
- * package URL reader has host-specific behaviour (e.g. virtual files,
- * remote URLs) — we delegate back to the main thread's URL reader so
- * compile semantics stay identical to the in-process path.
- */
-export interface ReadUrlRequest {
-   type: "read-url";
-   requestId: string;
-   jobId: string;
-   url: string;
-}
-
-export interface ReadUrlResponse {
-   type: "read-url-response";
-   requestId: string;
-   ok: true;
-   contents: string;
-   invalidationKey?: string | number | null;
-}
-
-// ──────────────────────────────────────────────────────────────────────
-// Lifecycle
-// ──────────────────────────────────────────────────────────────────────
-
-export interface ShutdownRequest {
-   type: "shutdown";
-}
-
-export interface ReadyMessage {
-   type: "ready";
-}
-
-// ──────────────────────────────────────────────────────────────────────
-// Union types for routing
-// ──────────────────────────────────────────────────────────────────────
-
-export type MainToWorkerMessage =
-   | LoadPackageRequest
-   | ConnectionMetadataResponse
-   | SchemaForTablesResponse
-   | SchemaForSqlResponse
-   | ReadUrlResponse
-   | RpcErrorResponse
-   | ShutdownRequest;
-
-export type WorkerToMainMessage =
-   | LoadPackageResult
-   | LoadPackageError
-   | ConnectionMetadataRequest
-   | SchemaForTablesRequest
-   | SchemaForSqlRequest
-   | ReadUrlRequest
-   | ReadyMessage;

package/src/package_load/rpc_wait_accountant.spec.ts

@@ -1,109 +0,0 @@
-import { describe, expect, it } from "bun:test";
-
-import { RpcWaitAccountant } from "./rpc_wait_accountant";
-
-/** Accountant driven by a settable fake clock. */
-function withClock(): { acc: RpcWaitAccountant; at: (ms: number) => void } {
-   let t = 0;
-   const acc = new RpcWaitAccountant(() => t);
-   return { acc, at: (ms) => (t = ms) };
-}
-
-describe("RpcWaitAccountant", () => {
-   it("brackets a single fetch's wait", () => {
-      const { acc, at } = withClock();
-      at(0);
-      acc.begin("A");
-      at(10);
-      acc.noteStart("A");
-      at(60);
-      acc.noteSettle("A");
-      expect(acc.waitMs).toBe(50);
-   });
-
-   it("counts overlapping fetches as their union, not the sum", () => {
-      const { acc, at } = withClock();
-      at(0);
-      acc.begin("A");
-      at(0);
-      acc.noteStart("A"); // window opens
-      at(10);
-      acc.noteStart("A"); // second fetch, window stays open
-      at(40);
-      acc.noteSettle("A"); // one still outstanding
-      at(60);
-      acc.noteSettle("A"); // window closes
-      expect(acc.waitMs).toBe(60); // union [0,60], not 30+50
-   });
-
-   it("accumulates disjoint fetch windows", () => {
-      const { acc, at } = withClock();
-      at(0);
-      acc.begin("A");
-      at(0);
-      acc.noteStart("A");
-      at(20);
-      acc.noteSettle("A");
-      at(50);
-      acc.noteStart("A");
-      at(70);
-      acc.noteSettle("A");
-      expect(acc.waitMs).toBe(40);
-   });
-
-   it("ignores a straggler fetch from a prior reused-worker load", () => {
-      const { acc, at } = withClock();
-      at(0);
-      acc.begin("A");
-      at(0);
-      acc.noteStart("A"); // load A opens a fetch and never settles it (failed)
-
-      at(100);
-      acc.begin("B"); // worker reused for load B
-      at(110);
-      acc.noteStart("B");
-      at(130);
-      acc.noteSettle("B");
-
-      at(140);
-      acc.noteSettle("A"); // A's late response — must be a no-op
-
-      expect(acc.waitMs).toBe(20); // only B's [110,130]
-   });
-
-   it("ignores a stray start from a non-active load", () => {
-      const { acc, at } = withClock();
-      at(0);
-      acc.begin("B");
-      at(10);
-      acc.noteStart("A"); // wrong load
-      at(50);
-      acc.noteSettle("A");
-      expect(acc.waitMs).toBe(0);
-   });
-
-   it("counts active-load fetches only", () => {
-      const { acc, at } = withClock();
-      at(0);
-      acc.begin("B");
-      acc.noteStart("B");
-      acc.noteSettle("B");
-      acc.noteStart("B");
-      acc.noteStart("A"); // straggler — not counted
-      expect(acc.fetches).toBe(2);
-   });
-
-   it("begin() discards prior-load state", () => {
-      const { acc, at } = withClock();
-      at(0);
-      acc.begin("A");
-      at(0);
-      acc.noteStart("A");
-      at(50);
-      acc.noteSettle("A");
-      expect(acc.waitMs).toBe(50);
-      at(60);
-      acc.begin("B");
-      expect(acc.waitMs).toBe(0);
-   });
-});

package/src/package_load/rpc_wait_accountant.ts

@@ -1,76 +0,0 @@
-/**
- * Per-load "wait" accounting for the package-load worker.
- *
- * A package load spends its time in two very different ways: CPU-bound Malloy
- * compilation, and I/O waiting on connection schema fetches that are proxied
- * back to the main thread. Today only the *total* load time is observable, so
- * an operator seeing slow loads can't tell which half dominates. This tracks
- * the wall-clock during which at least one proxied schema fetch is outstanding,
- * so the load handler can report `compile = compileRegion - schemaFetchWait`
- * alongside the fetch time itself.
- *
- * Why wall-minus-wait and not a CPU counter: worker threads share the host
- * process, so `process.cpuUsage()` is process-wide and would fold in every
- * other worker's CPU — useless for a per-load number. Wall-minus-wait is the
- * honest per-load proxy. Treat the compute figure as a conservative *ceiling*
- * on compile CPU: only proxied waits are subtracted, so any non-fetch stall
- * (GC, scheduler preemption) also lands in "compute".
- *
- * Overlapping fetches are counted as their union (the wall-time with >=1
- * outstanding), never the sum, so concurrent fetches during one compile don't
- * over-subtract.
- *
- * Scoped by `jobId`: the pool runs one load per worker at a time, but reuses a
- * worker across loads, and a load that fails can leave a proxied fetch in
- * flight. Gating every start/settle on the active job makes such a straggler's
- * late response a no-op instead of corrupting the next load's numbers. (If the
- * one-load-per-worker rule ever changes, this stays correct per job but the
- * compute ceiling would then also fold in a sibling load's CPU.)
- */
-export class RpcWaitAccountant {
-   private inFlight = 0;
-   private waitStartMs = 0;
-   private accumMs = 0;
-   private startedCount = 0;
-   private activeJobId: string | undefined;
-
-   constructor(private readonly now: () => number = () => performance.now()) {}
-
-   /** Begin accounting for `jobId`, discarding any prior-load state. */
-   begin(jobId: string): void {
-      this.activeJobId = jobId;
-      this.inFlight = 0;
-      this.waitStartMs = 0;
-      this.accumMs = 0;
-      this.startedCount = 0;
-   }
-
-   /** A proxied fetch for `jobId` was issued. Ignored if `jobId` isn't active. */
-   noteStart(jobId: string): void {
-      if (jobId !== this.activeJobId) return;
-      if (this.inFlight === 0) this.waitStartMs = this.now();
-      this.inFlight += 1;
-      this.startedCount += 1;
-   }
-
-   /**
-    * A proxied fetch for `jobId` settled (resolved or rejected). Ignored if
-    * `jobId` isn't active, so a straggler from a prior reused-worker load
-    * can't drive the counter negative or close the active load's bracket.
-    */
-   noteSettle(jobId: string): void {
-      if (jobId !== this.activeJobId) return;
-      this.inFlight -= 1;
-      if (this.inFlight === 0) this.accumMs += this.now() - this.waitStartMs;
-   }
-
-   /** Accumulated wall-time with >=1 active-load fetch outstanding (union). */
-   get waitMs(): number {
-      return this.accumMs;
-   }
-
-   /** Number of proxied fetches started for the active load. */
-   get fetches(): number {
-      return this.startedCount;
-   }
-}

package/src/package_load_metrics.spec.ts

@@ -1,114 +0,0 @@
-import { afterEach, beforeEach, describe, expect, it } from "bun:test";
-
-import {
-   recordPackageLoadPhases,
-   resetPackageLoadMetricsForTesting,
-} from "./package_load_metrics";
-import {
-   startMetricsHarness,
-   type MetricsHarness,
-} from "./test_helpers/metrics_harness";
-
-describe("package_load_metrics", () => {
-   let harness: MetricsHarness;
-
-   beforeEach(async () => {
-      harness = await startMetricsHarness();
-      resetPackageLoadMetricsForTesting();
-   });
-
-   afterEach(async () => {
-      resetPackageLoadMetricsForTesting();
-      await harness.shutdown();
-   });
-
-   it("records the three phase histograms, labeled by status", async () => {
-      recordPackageLoadPhases(
-         {
-            compileDurationMs: 120,
-            schemaFetchDurationMs: 80,
-            schemaFetchCount: 4,
-         },
-         "success",
-      );
-      recordPackageLoadPhases(
-         {
-            compileDurationMs: 40,
-            schemaFetchDurationMs: 10,
-            schemaFetchCount: 1,
-         },
-         "success",
-      );
-
-      const compile = await harness.collectHistogram(
-         "malloy_package_load_compile_duration",
-         { status: "success" },
-      );
-      expect(compile.count).toBe(2);
-      expect(compile.sum).toBe(160);
-
-      const fetchDuration = await harness.collectHistogram(
-         "malloy_package_load_schema_fetch_duration",
-         { status: "success" },
-      );
-      expect(fetchDuration.count).toBe(2);
-      expect(fetchDuration.sum).toBe(90);
-
-      const fetches = await harness.collectHistogram(
-         "malloy_package_load_schema_fetches",
-         { status: "success" },
-      );
-      expect(fetches.count).toBe(2);
-      expect(fetches.sum).toBe(5);
-   });
-
-   it("labels phases by terminal status, so failures are separable from successes", async () => {
-      recordPackageLoadPhases(
-         {
-            compileDurationMs: 30,
-            schemaFetchDurationMs: 5,
-            schemaFetchCount: 1,
-         },
-         "success",
-      );
-      recordPackageLoadPhases(
-         {
-            compileDurationMs: 12,
-            schemaFetchDurationMs: 3,
-            schemaFetchCount: 1,
-         },
-         "compilation_error",
-      );
-
-      const ok = await harness.collectHistogram(
-         "malloy_package_load_compile_duration",
-         { status: "success" },
-      );
-      const failed = await harness.collectHistogram(
-         "malloy_package_load_compile_duration",
-         { status: "compilation_error" },
-      );
-      expect(ok.count).toBe(1);
-      expect(ok.sum).toBe(30);
-      expect(failed.count).toBe(1);
-      expect(failed.sum).toBe(12);
-   });
-
-   it("resolves the slow-load tail past OTel's default 10s cap", async () => {
-      // A 90s load would fall in the +Inf bucket under OTel's default
-      // boundaries (top = 10000ms); the explicit buckets must extend further.
-      recordPackageLoadPhases(
-         {
-            compileDurationMs: 90_000,
-            schemaFetchDurationMs: 0,
-            schemaFetchCount: 0,
-         },
-         "success",
-      );
-      const compile = await harness.collectHistogram(
-         "malloy_package_load_compile_duration",
-      );
-      expect(compile.boundaries.some((b) => b > 10_000)).toBe(true);
-      expect(compile.boundaries[compile.boundaries.length - 1]).toBe(300_000);
-   });
-});