@malloy-publisher/server 0.0.198-dev3 → 0.0.198-dev6

package/src/service/package_worker_path.spec.ts

@@ -0,0 +1,196 @@
+/**
+ * Integration test: exercise `Package.create` with the package-load
+ * worker pool enabled (PACKAGE_LOAD_WORKERS=1).
+ *
+ * Validates that the worker-load path:
+ *   - reads the manifest, probes embedded databases, and compiles
+ *     every model in a single off-thread job
+ *   - produces a live `Package` whose `Model`s have populated
+ *     `modelDef` / `sources` / `queries`
+ *   - hydrates the `ModelMaterializer` from `modelDef` on first
+ *     query (no recompile) — verified end-to-end by running a
+ *     query through the resulting Model and getting a result
+ *
+ * Kept separate from `package.spec.ts` so the existing tests keep
+ * running on the in-process path without paying worker startup cost.
+ *
+ * Pool reuse strategy: one `PackageLoadPool` shared across all
+ * cases in this file. Spawning a fresh worker per test crashes Bun
+ * (segfault) because DuckDB's native bindings don't tolerate being
+ * loaded concurrently into multiple worker isolates of the same Bun
+ * process. Production uses one pool; this matches.
+ */
+import {
+   afterAll,
+   afterEach,
+   beforeAll,
+   beforeEach,
+   describe,
+   expect,
+   it,
+} from "bun:test";
+import * as fs from "fs";
+import * as os from "os";
+import * as path from "path";
+import {
+   PackageLoadPool,
+   __setPackageLoadPoolForTests,
+} from "../package_load/package_load_pool";
+import { Package } from "./package";
+
+const ORIGINAL_ENV = process.env.PACKAGE_LOAD_WORKERS;
+
+describe("Package.create via worker pool", () => {
+   let tempDir: string;
+   let pool: PackageLoadPool;
+
+   beforeAll(async () => {
+      process.env.PACKAGE_LOAD_WORKERS = "1";
+      pool = new PackageLoadPool(1);
+      // Wire our pool into the module-level singleton so Package.create
+      // picks it up via getPackageLoadPool().
+      await __setPackageLoadPoolForTests(pool);
+   });
+
+   afterAll(async () => {
+      await __setPackageLoadPoolForTests(null);
+      if (ORIGINAL_ENV === undefined) {
+         delete process.env.PACKAGE_LOAD_WORKERS;
+      } else {
+         process.env.PACKAGE_LOAD_WORKERS = ORIGINAL_ENV;
+      }
+   });
+
+   beforeEach(() => {
+      tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "publisher-pkg-worker-"));
+   });
+
+   afterEach(() => {
+      if (tempDir) {
+         // tempDir gets wiped by Package.create on failure (it's the
+         // staging-cleanup path); ignore ENOENT here.
+         try {
+            fs.rmSync(tempDir, { recursive: true, force: true });
+         } catch {
+            /* already gone */
+         }
+         tempDir = "";
+      }
+   });
+
+   async function makeMalloyConfig(): Promise<{
+      malloyConfig: import("@malloydata/malloy").MalloyConfig;
+      duckdb: { close: () => Promise<void> };
+   }> {
+      const { MalloyConfig, FixedConnectionMap } = await import(
+         "@malloydata/malloy"
+      );
+      const { DuckDBConnection } = await import("@malloydata/db-duckdb");
+      const duckdb = new DuckDBConnection("duckdb", ":memory:");
+      const connections = new FixedConnectionMap(
+         new Map([["duckdb", duckdb]]),
+         "duckdb",
+      );
+      const malloyConfig = new MalloyConfig({ connections: {} });
+      malloyConfig.wrapConnections(() => connections);
+      return { malloyConfig, duckdb };
+   }
+
+   function writeManifest(): void {
+      fs.writeFileSync(
+         path.join(tempDir, "publisher.json"),
+         JSON.stringify({ name: "pkg", description: "test package" }),
+      );
+   }
+
+   it("loads a package end-to-end and serves a query through the hydrated materializer", async () => {
+      writeManifest();
+      // Define `total_v` as a *view* on the source so the query
+      // builder's `run: nums -> total_v` form resolves. (Top-level
+      // queries take the `run: total_q` form — orthogonal path that
+      // the in-process tests already cover.)
+      fs.writeFileSync(
+         path.join(tempDir, "trivial.malloy"),
+         `source: nums is duckdb.sql("select 1 as a, 2 as b") extend {
+  measure: total is a.sum()
+  view: total_v is { aggregate: total }
+}`,
+      );
+
+      const { malloyConfig, duckdb } = await makeMalloyConfig();
+      try {
+         const pkg = await Package.create("env", "pkg", tempDir, malloyConfig);
+         expect(pkg).toBeInstanceOf(Package);
+         expect(pkg.getModelPaths()).toEqual(["trivial.malloy"]);
+
+         const model = pkg.getModel("trivial.malloy");
+         expect(model).toBeDefined();
+         const apiModel = (await model!.getModel()) as {
+            modelDef?: string;
+            sources?: { name?: string }[];
+         };
+         expect(apiModel.modelDef).toBeDefined();
+         expect(apiModel.sources?.[0]?.name).toBe("nums");
+
+         // First query against the package — hydrates the
+         // ModelMaterializer from the worker's modelDef without a
+         // recompile, then runs the SQL against the *main thread's*
+         // DuckDB connection (the only one with the in-memory `nums`
+         // source loaded via duckdb.sql()).
+         const { result } = await model!.getQueryResults(
+            "nums",
+            "total_v",
+            undefined,
+         );
+         expect(result.data).toBeDefined();
+      } finally {
+         await duckdb.close();
+      }
+   });
+
+   it("propagates a per-model compile failure as a thrown error from Package.create", async () => {
+      writeManifest();
+      fs.writeFileSync(
+         path.join(tempDir, "broken.malloy"),
+         `source: bad is duckdb.sql("select 1 as a") extend {
+  measure: oops is THIS_FUNC_DOES_NOT_EXIST(a)
+}`,
+      );
+
+      const { malloyConfig, duckdb } = await makeMalloyConfig();
+      try {
+         await expect(
+            Package.create("env", "pkg", tempDir, malloyConfig),
+         ).rejects.toBeInstanceOf(Error);
+      } finally {
+         await duckdb.close();
+      }
+   });
+
+   // NB: kept last in this describe — swapping the singleton for a
+   // pre-shutdown pool also tears down the shared `pool` (the swap
+   // implementation shuts down the outgoing singleton). Subsequent
+   // tests in this describe would see a dead pool. afterAll only
+   // resets the singleton to null, so this is safe at the tail.
+   it("rewraps pool-infrastructure failures as ServiceUnavailableError (HTTP 503)", async () => {
+      writeManifest();
+      fs.writeFileSync(
+         path.join(tempDir, "trivial.malloy"),
+         `source: nums is duckdb.sql("select 1 as a")`,
+      );
+
+      const deadPool = new PackageLoadPool(1);
+      await deadPool.shutdown();
+      await __setPackageLoadPoolForTests(deadPool);
+
+      const { ServiceUnavailableError } = await import("../errors");
+      const { malloyConfig, duckdb } = await makeMalloyConfig();
+      try {
+         await expect(
+            Package.create("env", "pkg", tempDir, malloyConfig),
+         ).rejects.toBeInstanceOf(ServiceUnavailableError);
+      } finally {
+         await duckdb.close();
+      }
+   });
+});

package/dist/service/schema_worker.mjs

@@ -1,61 +0,0 @@
-// src/service/schema_worker.ts
-import { DuckDBConnection } from "@malloydata/db-duckdb";
-import"@malloydata/db-duckdb/native";
-import {
-  ConnectionRuntime,
-  EmptyURLReader
-} from "@malloydata/malloy";
-import * as path from "path";
-import { parentPort } from "worker_threads";
-if (!parentPort) {
-  throw new Error("schema_worker.ts loaded outside a worker thread");
-}
-var connection = new DuckDBConnection({
-  name: "duckdb",
-  databasePath: ":memory:",
-  threads: 1,
-  memoryLimit: "256MB"
-});
-async function handleRequest(req) {
-  try {
-    const fullPath = path.join(req.packagePath, req.databasePath);
-    const normalizedPath = fullPath.replace(/\\/g, "/");
-    const runtime = new ConnectionRuntime({
-      urlReader: new EmptyURLReader,
-      connections: [connection]
-    });
-    const model = runtime.loadModel(`source: temp is duckdb.table('${normalizedPath}')`);
-    const modelDef = await model.getModel();
-    const fields = modelDef._modelDef.contents["temp"].fields;
-    const columns = fields.map((field) => ({
-      type: String(field.type),
-      name: field.name
-    }));
-    const runner = model.loadQuery("run: temp->{aggregate: row_count is count()}");
-    const result = await runner.run();
-    const rowCount = result.data.value[0].row_count?.valueOf();
-    return {
-      id: req.id,
-      ok: true,
-      result: { name: req.databasePath, rowCount, columns }
-    };
-  } catch (err) {
-    const error = err instanceof Error ? err : new Error(String(err));
-    return {
-      id: req.id,
-      ok: false,
-      error: { message: error.message, stack: error.stack }
-    };
-  }
-}
-parentPort.on("message", async (msg) => {
-  const response = await handleRequest(msg);
-  parentPort.postMessage(response);
-});
-var shutdown = async () => {
-  try {
-    await connection.close();
-  } catch {}
-  process.exit(0);
-};
-parentPort.on("close", () => void shutdown());

package/src/service/process_stats_reporter.ts

@@ -1,169 +0,0 @@
-import * as fs from "fs";
-
-import { logger } from "../logger";
-import type { PackageMemoryGovernor } from "./package_memory_governor";
-
-const DEFAULT_INTERVAL_MS = 30_000;
-
-interface LinuxProcStatus {
-   threads?: number;
-   vmRssBytes?: number;
-   vmSizeBytes?: number;
-   vmPeakBytes?: number;
-   vmDataBytes?: number;
-   voluntaryCtxSwitches?: number;
-   nonvoluntaryCtxSwitches?: number;
-}
-
-/**
- * Parse the subset of `/proc/self/status` that matters for diagnosing
- * thread / virtual-memory leaks. The file is small (<5KB), so reading
- * it synchronously here is cheap and avoids fs-promise queueing.
- *
- * Format is `Key:\t<value> [unit]` per line. Sizes are reported in kB;
- * we normalize to bytes so log output matches `process.memoryUsage()`.
- */
-function readLinuxProcStatus(): LinuxProcStatus | null {
-   try {
-      const raw = fs.readFileSync("/proc/self/status", "utf8");
-      const out: LinuxProcStatus = {};
-      for (const line of raw.split("\n")) {
-         const [keyRaw, valueRaw] = line.split(":");
-         if (!keyRaw || !valueRaw) continue;
-         const key = keyRaw.trim();
-         const value = valueRaw.trim();
-         switch (key) {
-            case "Threads":
-               out.threads = Number(value);
-               break;
-            case "VmRSS":
-               out.vmRssBytes = kBToBytes(value);
-               break;
-            case "VmSize":
-               out.vmSizeBytes = kBToBytes(value);
-               break;
-            case "VmPeak":
-               out.vmPeakBytes = kBToBytes(value);
-               break;
-            case "VmData":
-               out.vmDataBytes = kBToBytes(value);
-               break;
-            case "voluntary_ctxt_switches":
-               out.voluntaryCtxSwitches = Number(value);
-               break;
-            case "nonvoluntary_ctxt_switches":
-               out.nonvoluntaryCtxSwitches = Number(value);
-               break;
-         }
-      }
-      return out;
-   } catch {
-      return null;
-   }
-}
-
-function kBToBytes(value: string): number | undefined {
-   const num = Number(value.replace(/\s*kB$/, ""));
-   if (!Number.isFinite(num)) return undefined;
-   return num * 1024;
-}
-
-/**
- * Bun exposes JSC heap stats via the `bun:jsc` builtin. Optional —
- * absent under plain Node — and best-effort: failures are swallowed
- * so the reporter never crashes the process.
- */
-async function readBunJscStats(): Promise<Record<string, number> | null> {
-   if (typeof (globalThis as { Bun?: unknown }).Bun === "undefined") {
-      return null;
-   }
-   try {
-      // Dynamic import so Node builds don't fail at parse time.
-      const jsc = (await import("bun:jsc")) as unknown as {
-         heapStats?: () => Record<string, number>;
-         memoryUsage?: () => Record<string, number>;
-      };
-      const heap = jsc.heapStats?.();
-      const mem = jsc.memoryUsage?.();
-      if (!heap && !mem) return null;
-      return { ...(heap ?? {}), ...(mem ?? {}) };
-   } catch {
-      return null;
-   }
-}
-
-/**
- * Periodically logs process memory and thread counts to give ops a
- * cheap, always-on signal for the leak classes that have OOM-killed
- * prod (DuckDB connection thread pools, libuv worker pool, Malloy
- * compile heap, etc.).
- *
- * Logs at `info` so it shows up without flipping `LOG_LEVEL`. Volume
- * is low (~2 lines/minute by default). Pulls the memory governor's
- * snapshot too so RSS/back-pressure state appears in the same line as
- * Node/Bun heap.
- */
-export class ProcessStatsReporter {
-   private timer: ReturnType<typeof setInterval> | null = null;
-   private readonly intervalMs: number;
-   private readonly memoryGovernor: PackageMemoryGovernor | null;
-
-   constructor(
-      memoryGovernor: PackageMemoryGovernor | null,
-      intervalMs: number = DEFAULT_INTERVAL_MS,
-   ) {
-      this.memoryGovernor = memoryGovernor;
-      this.intervalMs = intervalMs;
-   }
-
-   public start(): void {
-      if (this.timer !== null) return;
-      // Immediate first sample so a freshly-started pod logs its
-      // baseline before the first 30s has elapsed.
-      void this.tick();
-      this.timer = setInterval(() => void this.tick(), this.intervalMs);
-      // Don't keep the event loop alive on our account — if everything
-      // else has shut down, the reporter shouldn't block exit.
-      (
-         this.timer as ReturnType<typeof setInterval> & {
-            unref?: () => void;
-         }
-      ).unref?.();
-      logger.info(
-         `ProcessStatsReporter started (intervalMs=${this.intervalMs})`,
-      );
-   }
-
-   public stop(): void {
-      if (this.timer !== null) {
-         clearInterval(this.timer);
-         this.timer = null;
-      }
-   }
-
-   private async tick(): Promise<void> {
-      try {
-         const mem = process.memoryUsage();
-         const proc =
-            process.platform === "linux" ? readLinuxProcStatus() : null;
-         const bun = await readBunJscStats();
-         const governor = this.memoryGovernor?.getStatus() ?? null;
-
-         logger.info("process stats", {
-            uptimeSeconds: Math.round(process.uptime()),
-            nodeMemory: {
-               rssBytes: mem.rss,
-               heapTotalBytes: mem.heapTotal,
-               heapUsedBytes: mem.heapUsed,
-               externalBytes: mem.external,
-               arrayBuffersBytes: mem.arrayBuffers,
-            },
-            linux: proc,
-            bunJsc: bun,
-            memoryGovernor: governor,
-         });
-      } catch (err) {
-         logger.warn("ProcessStatsReporter tick failed", { error: err });
-      }
-   }
-}

package/src/service/schema_worker.ts

@@ -1,123 +0,0 @@
-/**
- * Worker thread that owns one capped DuckDB connection and answers
- * schema-introspection requests for parquet/csv files. Running this
- * off the main thread isolates the native DuckDB thread pool — when
- * the worker exits, its threads die with it, which puts a hard
- * ceiling on the leak class that OOM-killed prod
- * (worker-76b49bdb89-8bsv4: 466 leaked Bun Pool threads).
- *
- * Protocol (parent ↔ worker):
- *   parent → worker:  { id, packagePath, databasePath }
- *   worker → parent:  { id, ok: true,  result: SchemaResult }
- *                  |  { id, ok: false, error: { message, stack? } }
- *
- * One request at a time per worker — the pool in the parent
- * (`schema_worker_pool.ts`) handles fan-out. Keeping the worker
- * single-threaded from the JS side matches DuckDB's behavior on a
- * single connection and avoids head-of-line blocking inside the
- * worker itself.
- */
-import { DuckDBConnection } from "@malloydata/db-duckdb";
-import "@malloydata/db-duckdb/native";
-import {
-   ConnectionRuntime,
-   EmptyURLReader,
-   SourceDef,
-} from "@malloydata/malloy";
-import * as path from "path";
-import { parentPort } from "worker_threads";
-
-export interface SchemaRequest {
-   id: number;
-   packagePath: string;
-   databasePath: string;
-}
-
-export interface SchemaResponse {
-   id: number;
-   ok: boolean;
-   result?: {
-      name: string;
-      rowCount: number;
-      columns: Array<{ type: string; name: string }>;
-   };
-   error?: { message: string; stack?: string };
-}
-
-if (!parentPort) {
-   // Defensive: schema_worker.ts must only be loaded as a worker. If
-   // someone accidentally imports it from the main thread the
-   // connection below would still allocate its native pool there,
-   // recreating the exact leak this file exists to fix.
-   throw new Error("schema_worker.ts loaded outside a worker thread");
-}
-
-// One DuckDB connection per worker, capped tight. Schema introspection
-// reads parquet footers / csv headers — it does not need parallelism
-// or a large memory arena. The cap is what keeps the per-worker
-// native-thread cost bounded.
-const connection = new DuckDBConnection({
-   name: "duckdb",
-   databasePath: ":memory:",
-   threads: 1,
-   memoryLimit: "256MB",
-});
-
-async function handleRequest(req: SchemaRequest): Promise<SchemaResponse> {
-   try {
-      const fullPath = path.join(req.packagePath, req.databasePath);
-      // DuckDB on Windows supports forward slashes, and this avoids
-      // escaping issues in the inline SQL below.
-      const normalizedPath = fullPath.replace(/\\/g, "/");
-
-      const runtime = new ConnectionRuntime({
-         urlReader: new EmptyURLReader(),
-         connections: [connection],
-      });
-      const model = runtime.loadModel(
-         `source: temp is duckdb.table('${normalizedPath}')`,
-      );
-      const modelDef = await model.getModel();
-      const fields = (modelDef._modelDef.contents["temp"] as SourceDef).fields;
-      const columns = fields.map((field) => ({
-         type: String(field.type),
-         name: field.name,
-      }));
-
-      const runner = model.loadQuery(
-         "run: temp->{aggregate: row_count is count()}",
-      );
-      const result = await runner.run();
-      const rowCount = result.data.value[0].row_count?.valueOf() as number;
-
-      return {
-         id: req.id,
-         ok: true,
-         result: { name: req.databasePath, rowCount, columns },
-      };
-   } catch (err) {
-      const error = err instanceof Error ? err : new Error(String(err));
-      return {
-         id: req.id,
-         ok: false,
-         error: { message: error.message, stack: error.stack },
-      };
-   }
-}
-
-parentPort.on("message", async (msg: SchemaRequest) => {
-   const response = await handleRequest(msg);
-   parentPort!.postMessage(response);
-});
-
-// On any termination signal, close the connection so DuckDB releases
-// its native threads cleanly instead of leaking them past worker exit.
-const shutdown = async () => {
-   try {
-      await connection.close();
-   } catch {
-      // best effort
-   }
-   process.exit(0);
-};
-parentPort.on("close", () => void shutdown());