@malloy-publisher/server 0.0.198-dev → 0.0.198-dev2

package/src/service/process_stats_reporter.ts

@@ -0,0 +1,169 @@
+import * as fs from "fs";
+
+import { logger } from "../logger";
+import type { PackageMemoryGovernor } from "./package_memory_governor";
+
+const DEFAULT_INTERVAL_MS = 30_000;
+
+interface LinuxProcStatus {
+   threads?: number;
+   vmRssBytes?: number;
+   vmSizeBytes?: number;
+   vmPeakBytes?: number;
+   vmDataBytes?: number;
+   voluntaryCtxSwitches?: number;
+   nonvoluntaryCtxSwitches?: number;
+}
+
+/**
+ * Parse the subset of `/proc/self/status` that matters for diagnosing
+ * thread / virtual-memory leaks. The file is small (<5KB), so reading
+ * it synchronously here is cheap and avoids fs-promise queueing.
+ *
+ * Format is `Key:\t<value> [unit]` per line. Sizes are reported in kB;
+ * we normalize to bytes so log output matches `process.memoryUsage()`.
+ */
+function readLinuxProcStatus(): LinuxProcStatus | null {
+   try {
+      const raw = fs.readFileSync("/proc/self/status", "utf8");
+      const out: LinuxProcStatus = {};
+      for (const line of raw.split("\n")) {
+         const [keyRaw, valueRaw] = line.split(":");
+         if (!keyRaw || !valueRaw) continue;
+         const key = keyRaw.trim();
+         const value = valueRaw.trim();
+         switch (key) {
+            case "Threads":
+               out.threads = Number(value);
+               break;
+            case "VmRSS":
+               out.vmRssBytes = kBToBytes(value);
+               break;
+            case "VmSize":
+               out.vmSizeBytes = kBToBytes(value);
+               break;
+            case "VmPeak":
+               out.vmPeakBytes = kBToBytes(value);
+               break;
+            case "VmData":
+               out.vmDataBytes = kBToBytes(value);
+               break;
+            case "voluntary_ctxt_switches":
+               out.voluntaryCtxSwitches = Number(value);
+               break;
+            case "nonvoluntary_ctxt_switches":
+               out.nonvoluntaryCtxSwitches = Number(value);
+               break;
+         }
+      }
+      return out;
+   } catch {
+      return null;
+   }
+}
+
+function kBToBytes(value: string): number | undefined {
+   const num = Number(value.replace(/\s*kB$/, ""));
+   if (!Number.isFinite(num)) return undefined;
+   return num * 1024;
+}
+
+/**
+ * Bun exposes JSC heap stats via the `bun:jsc` builtin. Optional —
+ * absent under plain Node — and best-effort: failures are swallowed
+ * so the reporter never crashes the process.
+ */
+async function readBunJscStats(): Promise<Record<string, number> | null> {
+   if (typeof (globalThis as { Bun?: unknown }).Bun === "undefined") {
+      return null;
+   }
+   try {
+      // Dynamic import so Node builds don't fail at parse time.
+      const jsc = (await import("bun:jsc")) as unknown as {
+         heapStats?: () => Record<string, number>;
+         memoryUsage?: () => Record<string, number>;
+      };
+      const heap = jsc.heapStats?.();
+      const mem = jsc.memoryUsage?.();
+      if (!heap && !mem) return null;
+      return { ...(heap ?? {}), ...(mem ?? {}) };
+   } catch {
+      return null;
+   }
+}
+
+/**
+ * Periodically logs process memory and thread counts to give ops a
+ * cheap, always-on signal for the leak classes that have OOM-killed
+ * prod (DuckDB connection thread pools, libuv worker pool, Malloy
+ * compile heap, etc.).
+ *
+ * Logs at `info` so it shows up without flipping `LOG_LEVEL`. Volume
+ * is low (~2 lines/minute by default). Pulls the memory governor's
+ * snapshot too so RSS/back-pressure state appears in the same line as
+ * Node/Bun heap.
+ */
+export class ProcessStatsReporter {
+   private timer: ReturnType<typeof setInterval> | null = null;
+   private readonly intervalMs: number;
+   private readonly memoryGovernor: PackageMemoryGovernor | null;
+
+   constructor(
+      memoryGovernor: PackageMemoryGovernor | null,
+      intervalMs: number = DEFAULT_INTERVAL_MS,
+   ) {
+      this.memoryGovernor = memoryGovernor;
+      this.intervalMs = intervalMs;
+   }
+
+   public start(): void {
+      if (this.timer !== null) return;
+      // Immediate first sample so a freshly-started pod logs its
+      // baseline before the first 30s has elapsed.
+      void this.tick();
+      this.timer = setInterval(() => void this.tick(), this.intervalMs);
+      // Don't keep the event loop alive on our account — if everything
+      // else has shut down, the reporter shouldn't block exit.
+      (
+         this.timer as ReturnType<typeof setInterval> & {
+            unref?: () => void;
+         }
+      ).unref?.();
+      logger.info(
+         `ProcessStatsReporter started (intervalMs=${this.intervalMs})`,
+      );
+   }
+
+   public stop(): void {
+      if (this.timer !== null) {
+         clearInterval(this.timer);
+         this.timer = null;
+      }
+   }
+
+   private async tick(): Promise<void> {
+      try {
+         const mem = process.memoryUsage();
+         const proc =
+            process.platform === "linux" ? readLinuxProcStatus() : null;
+         const bun = await readBunJscStats();
+         const governor = this.memoryGovernor?.getStatus() ?? null;
+
+         logger.info("process stats", {
+            uptimeSeconds: Math.round(process.uptime()),
+            nodeMemory: {
+               rssBytes: mem.rss,
+               heapTotalBytes: mem.heapTotal,
+               heapUsedBytes: mem.heapUsed,
+               externalBytes: mem.external,
+               arrayBuffersBytes: mem.arrayBuffers,
+            },
+            linux: proc,
+            bunJsc: bun,
+            memoryGovernor: governor,
+         });
+      } catch (err) {
+         logger.warn("ProcessStatsReporter tick failed", { error: err });
+      }
+   }
+}

package/src/service/schema_worker.ts

@@ -0,0 +1,123 @@
+/**
+ * Worker thread that owns one capped DuckDB connection and answers
+ * schema-introspection requests for parquet/csv files. Running this
+ * off the main thread isolates the native DuckDB thread pool — when
+ * the worker exits, its threads die with it, which puts a hard
+ * ceiling on the leak class that OOM-killed prod
+ * (worker-76b49bdb89-8bsv4: 466 leaked Bun Pool threads).
+ *
+ * Protocol (parent ↔ worker):
+ *   parent → worker:  { id, packagePath, databasePath }
+ *   worker → parent:  { id, ok: true,  result: SchemaResult }
+ *                  |  { id, ok: false, error: { message, stack? } }
+ *
+ * One request at a time per worker — the pool in the parent
+ * (`schema_worker_pool.ts`) handles fan-out. Keeping the worker
+ * single-threaded from the JS side matches DuckDB's behavior on a
+ * single connection and avoids head-of-line blocking inside the
+ * worker itself.
+ */
+import { DuckDBConnection } from "@malloydata/db-duckdb";
+import "@malloydata/db-duckdb/native";
+import {
+   ConnectionRuntime,
+   EmptyURLReader,
+   SourceDef,
+} from "@malloydata/malloy";
+import * as path from "path";
+import { parentPort } from "worker_threads";
+
+export interface SchemaRequest {
+   id: number;
+   packagePath: string;
+   databasePath: string;
+}
+
+export interface SchemaResponse {
+   id: number;
+   ok: boolean;
+   result?: {
+      name: string;
+      rowCount: number;
+      columns: Array<{ type: string; name: string }>;
+   };
+   error?: { message: string; stack?: string };
+}
+
+if (!parentPort) {
+   // Defensive: schema_worker.ts must only be loaded as a worker. If
+   // someone accidentally imports it from the main thread the
+   // connection below would still allocate its native pool there,
+   // recreating the exact leak this file exists to fix.
+   throw new Error("schema_worker.ts loaded outside a worker thread");
+}
+
+// One DuckDB connection per worker, capped tight. Schema introspection
+// reads parquet footers / csv headers — it does not need parallelism
+// or a large memory arena. The cap is what keeps the per-worker
+// native-thread cost bounded.
+const connection = new DuckDBConnection({
+   name: "duckdb",
+   databasePath: ":memory:",
+   threads: 1,
+   memoryLimit: "256MB",
+});
+
+async function handleRequest(req: SchemaRequest): Promise<SchemaResponse> {
+   try {
+      const fullPath = path.join(req.packagePath, req.databasePath);
+      // DuckDB on Windows supports forward slashes, and this avoids
+      // escaping issues in the inline SQL below.
+      const normalizedPath = fullPath.replace(/\\/g, "/");
+
+      const runtime = new ConnectionRuntime({
+         urlReader: new EmptyURLReader(),
+         connections: [connection],
+      });
+      const model = runtime.loadModel(
+         `source: temp is duckdb.table('${normalizedPath}')`,
+      );
+      const modelDef = await model.getModel();
+      const fields = (modelDef._modelDef.contents["temp"] as SourceDef).fields;
+      const columns = fields.map((field) => ({
+         type: String(field.type),
+         name: field.name,
+      }));
+
+      const runner = model.loadQuery(
+         "run: temp->{aggregate: row_count is count()}",
+      );
+      const result = await runner.run();
+      const rowCount = result.data.value[0].row_count?.valueOf() as number;
+
+      return {
+         id: req.id,
+         ok: true,
+         result: { name: req.databasePath, rowCount, columns },
+      };
+   } catch (err) {
+      const error = err instanceof Error ? err : new Error(String(err));
+      return {
+         id: req.id,
+         ok: false,
+         error: { message: error.message, stack: error.stack },
+      };
+   }
+}
+
+parentPort.on("message", async (msg: SchemaRequest) => {
+   const response = await handleRequest(msg);
+   parentPort!.postMessage(response);
+});
+
+// On any termination signal, close the connection so DuckDB releases
+// its native threads cleanly instead of leaking them past worker exit.
+const shutdown = async () => {
+   try {
+      await connection.close();
+   } catch {
+      // best effort
+   }
+   process.exit(0);
+};
+parentPort.on("close", () => void shutdown());

package/src/service/schema_worker_pool.ts

@@ -0,0 +1,278 @@
+/**
+ * Long-lived pool of {@link Worker} threads that perform DuckDB schema
+ * introspection off the main event loop.
+ *
+ * Why a dedicated pool (not the libuv worker pool, not setImmediate):
+ *
+ *  - `@malloydata/db-duckdb` opens DuckDB via a native addon. Every
+ *    DuckDBConnection allocates its own native thread pool sized to
+ *    the host's CPU count (not the cgroup's CPU share). Concurrent
+ *    schema introspection on the main thread compounded into the
+ *    466-leaked-Bun-Pool-threads / 90GB-VmSize OOM signature seen on
+ *    `worker-76b49bdb89-8bsv4`.
+ *
+ *  - Owning the DuckDBConnection inside a worker isolates the native
+ *    pool to *that* worker. Per-pool sizing → predictable thread
+ *    budget. Worker exit → native threads die with it. No leak across
+ *    package loads.
+ *
+ *  - The schema-introspection case is uniquely suited to workers:
+ *    inputs and outputs are plain JSON (structured-cloneable), and
+ *    the work touches no environment connections, so we don't need
+ *    cross-thread IPC for live Snowflake/BigQuery handles. This is
+ *    why we tackle schema first — model compile (which *does* need
+ *    live env connections) is a much bigger lift, tracked separately.
+ */
+import { Worker } from "worker_threads";
+
+import { logger } from "../logger";
+
+type ColumnInfo = { type: string; name: string };
+
+/**
+ * Public-facing schema-row shape. Mirrors the original synchronous
+ * `getDatabaseInfo` return so callers in `package.ts` are unchanged.
+ */
+export interface SchemaResult {
+   name: string;
+   rowCount: number;
+   columns: ColumnInfo[];
+}
+
+interface WorkerSlot {
+   worker: Worker;
+   /** Whether the worker is currently handling a request. */
+   busy: boolean;
+}
+
+interface PendingRequest {
+   id: number;
+   packagePath: string;
+   databasePath: string;
+   resolve: (value: SchemaResult) => void;
+   reject: (reason: Error) => void;
+}
+
+const DEFAULT_POOL_SIZE = 2;
+
+export class SchemaWorkerPool {
+   private readonly workers: WorkerSlot[] = [];
+   private readonly queue: PendingRequest[] = [];
+   /** id → pending request currently executing. */
+   private readonly inFlight = new Map<number, PendingRequest>();
+   /** Maps a worker index to the id of the request it's running. */
+   private readonly workerCurrentId = new Map<number, number>();
+   private nextId = 1;
+   private stopped = false;
+
+   constructor(
+      private readonly workerUrl: URL,
+      private readonly size: number = DEFAULT_POOL_SIZE,
+   ) {}
+
+   public start(): void {
+      if (this.workers.length > 0) return;
+      for (let i = 0; i < this.size; i++) {
+         this.workers.push(this.spawn(i));
+      }
+      logger.info(`SchemaWorkerPool started (size=${this.size})`);
+   }
+
+   public async stop(): Promise<void> {
+      this.stopped = true;
+      // Fail any queued/in-flight work so callers don't hang on shutdown.
+      const shutdownError = new Error("SchemaWorkerPool stopped");
+      for (const req of this.queue.splice(0)) req.reject(shutdownError);
+      for (const req of this.inFlight.values()) req.reject(shutdownError);
+      this.inFlight.clear();
+      await Promise.all(
+         this.workers.map(async (slot) => {
+            try {
+               await slot.worker.terminate();
+            } catch {
+               // Best-effort: terminate failures shouldn't block shutdown.
+            }
+         }),
+      );
+      this.workers.length = 0;
+   }
+
+   /**
+    * Submit one schema-introspection job. Resolves with the schema
+    * description; rejects if the worker returns an error or crashes.
+    *
+    * Concurrent calls beyond the pool size are queued FIFO; once a
+    * worker frees up the next queued request is dispatched.
+    */
+   public submit(
+      packagePath: string,
+      databasePath: string,
+   ): Promise<SchemaResult> {
+      if (this.stopped) {
+         return Promise.reject(new Error("SchemaWorkerPool stopped"));
+      }
+      if (this.workers.length === 0) {
+         return Promise.reject(
+            new Error("SchemaWorkerPool.submit called before start()"),
+         );
+      }
+      return new Promise<SchemaResult>((resolve, reject) => {
+         const req: PendingRequest = {
+            id: this.nextId++,
+            packagePath,
+            databasePath,
+            resolve,
+            reject,
+         };
+         this.queue.push(req);
+         this.drain();
+      });
+   }
+
+   /**
+    * Try to assign queued requests to idle workers. Cheap; called
+    * after every enqueue and after every worker completes a request.
+    */
+   private drain(): void {
+      for (let i = 0; i < this.workers.length; i++) {
+         if (this.queue.length === 0) return;
+         const slot = this.workers[i];
+         if (slot.busy) continue;
+         const req = this.queue.shift()!;
+         slot.busy = true;
+         this.inFlight.set(req.id, req);
+         this.workerCurrentId.set(i, req.id);
+         slot.worker.postMessage({
+            id: req.id,
+            packagePath: req.packagePath,
+            databasePath: req.databasePath,
+         });
+      }
+   }
+
+   private spawn(index: number): WorkerSlot {
+      const worker = new Worker(this.workerUrl);
+      const slot: WorkerSlot = { worker, busy: false };
+
+      worker.on(
+         "message",
+         (msg: {
+            id: number;
+            ok: boolean;
+            result?: SchemaResult;
+            error?: { message: string; stack?: string };
+         }) => {
+            const req = this.inFlight.get(msg.id);
+            if (!req) {
+               logger.warn("SchemaWorkerPool: response for unknown request", {
+                  id: msg.id,
+                  workerIndex: index,
+               });
+               return;
+            }
+            this.inFlight.delete(msg.id);
+            this.workerCurrentId.delete(index);
+            slot.busy = false;
+            if (msg.ok && msg.result) {
+               req.resolve(msg.result);
+            } else {
+               const err = new Error(msg.error?.message ?? "Unknown error");
+               if (msg.error?.stack) err.stack = msg.error.stack;
+               req.reject(err);
+            }
+            this.drain();
+         },
+      );
+
+      worker.on("error", (err) => {
+         // A native crash inside the worker — fail the in-flight request
+         // attributed to this slot, then respawn the worker so the pool
+         // self-heals. Without respawn, one crash silently shrinks
+         // capacity and concurrent loads would queue forever.
+         const inFlightId = this.workerCurrentId.get(index);
+         if (inFlightId !== undefined) {
+            const req = this.inFlight.get(inFlightId);
+            if (req) {
+               this.inFlight.delete(inFlightId);
+               req.reject(err);
+            }
+            this.workerCurrentId.delete(index);
+         }
+         logger.error("SchemaWorkerPool: worker errored, respawning", {
+            workerIndex: index,
+            error: err,
+         });
+         if (!this.stopped) {
+            this.workers[index] = this.spawn(index);
+            this.drain();
+         }
+      });
+
+      worker.on("exit", (code) => {
+         if (this.stopped) return;
+         if (code !== 0) {
+            logger.warn("SchemaWorkerPool: worker exited unexpectedly", {
+               workerIndex: index,
+               code,
+            });
+            // Treat unexpected exit like an error: respawn so the pool
+            // doesn't silently lose capacity.
+            const inFlightId = this.workerCurrentId.get(index);
+            if (inFlightId !== undefined) {
+               const req = this.inFlight.get(inFlightId);
+               if (req) {
+                  this.inFlight.delete(inFlightId);
+                  req.reject(
+                     new Error(`SchemaWorker exited with code ${code}`),
+                  );
+               }
+               this.workerCurrentId.delete(index);
+            }
+            this.workers[index] = this.spawn(index);
+            this.drain();
+         }
+      });
+
+      return slot;
+   }
+}
+
+/**
+ * Process-wide singleton. Constructed lazily so importing this module
+ * doesn't spawn workers in test environments that never call
+ * `getSchemaWorkerPool()`.
+ *
+ * The worker URL is resolved from `import.meta.url`, which lets Bun
+ * load `schema_worker.ts` directly in dev and the bundled
+ * `schema_worker.mjs` in prod (see `build.ts`).
+ */
+let singleton: SchemaWorkerPool | null = null;
+
+export function getSchemaWorkerPool(): SchemaWorkerPool {
+   if (!singleton) {
+      const url = resolveWorkerUrl();
+      const size = Number(process.env.PUBLISHER_SCHEMA_WORKER_POOL_SIZE) || 2;
+      singleton = new SchemaWorkerPool(url, size);
+      singleton.start();
+   }
+   return singleton;
+}
+
+function resolveWorkerUrl(): URL {
+   // In dev (`bun --watch src/server.ts`), import.meta.url points at
+   // `.../src/service/schema_worker_pool.ts` and the worker is the
+   // sibling `.ts` file.
+   //
+   // In prod, this module gets inlined into `dist/server.mjs`, so
+   // `import.meta.url` resolves to `dist/server.mjs`. Bun's bundler
+   // nests outputs by their path relative to the common entrypoint
+   // root (./src), so schema_worker lands at
+   // `dist/service/schema_worker.mjs` — one directory below
+   // server.mjs.
+   const base = new URL(import.meta.url);
+   const isBundled = base.pathname.endsWith(".mjs");
+   return new URL(
+      isBundled ? "./service/schema_worker.mjs" : "./schema_worker.ts",
+      base,
+   );
+}

package/src/storage/StorageManager.ts

@@ -1,5 +1,13 @@
+import { Mutex } from "async-mutex";
 import * as crypto from "crypto";
+import { ConnectionAuthError } from "../errors";
 import { logger } from "../logger";
+import {
+   handlePgAttachError,
+   pgConnectTimeoutSeconds,
+   redactPgSecrets,
+   withPgConnectTimeout,
+} from "../pg_helpers";
 import {
    DatabaseConnection,
    ManifestStore,
@@ -78,6 +86,13 @@ export class StorageManager {
     */
    private attachedCatalogs = new Map<string, string>();
 
+   // Serializes DuckLake catalog attaches. Concurrent POST /environments calls
+   // hitting the same DuckDB connection would otherwise race on extension
+   // autoload (httpfs/azure/etc.), where multiple connections download the
+   // extension to `.tmp-<uuid>` files in parallel; only one wins the rename
+   // and the rest crash with "Could not remove file ... No such file or directory".
+   private duckLakeAttachMutex: Mutex = new Mutex();
+
    private config: StorageConfig;
 
    constructor(config: StorageConfig) {
@@ -141,14 +156,18 @@ export class StorageManager {
       }
 
       const key = configKey(config);
-      let catalogName = this.attachedCatalogs.get(key);
-      if (!catalogName) {
-         // Catalog name derived from the config so multiple configs can coexist as
-         // separate ATTACHments without colliding on the name.
-         catalogName = catalogNameForConfig(config);
-         await this.attachDuckLakeCatalog(config, catalogName);
-         this.attachedCatalogs.set(key, catalogName);
-      }
+      const catalogName = await this.duckLakeAttachMutex.runExclusive(
+         async () => {
+            const existing = this.attachedCatalogs.get(key);
+            if (existing) return existing;
+            // Catalog name derived from the config so multiple configs can coexist as
+            // separate ATTACHments without colliding on the name.
+            const name = catalogNameForConfig(config);
+            await this.attachDuckLakeCatalog(config, name);
+            this.attachedCatalogs.set(key, name);
+            return name;
+         },
+      );
 
       const store = new DuckLakeManifestStore(
          this.duckDbConnection,
@@ -178,12 +197,31 @@ export class StorageManager {
          await connection.run("INSTALL postgres; LOAD postgres;");
       }
 
-      const escapedCatalogUrl = escapeSQL(config.catalogUrl);
+      // For PG-backed catalogs, inject connect_timeout so a wedged libpq
+      // handshake fails the caller in seconds rather than hanging the
+      // worker until the K8s liveness probe trips (the 2026-05 incident).
+      // Non-PG catalogs (e.g. SQLite, MySQL) pass through unchanged.
+      const catalogUrl = isPostgres
+         ? withPgConnectTimeout(config.catalogUrl, pgConnectTimeoutSeconds())
+         : config.catalogUrl;
+
+      const escapedCatalogUrl = escapeSQL(catalogUrl);
       const escapedDataPath = escapeSQL(config.dataPath);
       const isCloudStorage =
          config.dataPath.startsWith("gs://") ||
          config.dataPath.startsWith("s3://");
 
+      // Pre-install httpfs explicitly so the ATTACH below doesn't trigger
+      // DuckDB's autoloader. The autoloader downloads extensions to
+      // `<ext>.tmp-<uuid>` and races when multiple connections within the
+      // same process hit it concurrently — losers fail with
+      // "Could not remove file ... No such file or directory" on cleanup
+      // of their .tmp file. INSTALL/LOAD here is idempotent and serialized
+      // by the caller's mutex.
+      if (isCloudStorage) {
+         await connection.run("INSTALL httpfs; LOAD httpfs;");
+      }
+
       let attachCmd = `ATTACH 'ducklake:${escapedCatalogUrl}' AS ${catalogName}`;
       const attachOpts: string[] = [
          `DATA_PATH '${escapedDataPath}'`,
@@ -193,13 +231,35 @@ export class StorageManager {
          // sidestepping object-storage auth issues entirely for this path.
          "DATA_INLINING_ROW_LIMIT 100000",
       ];
+
       if (isCloudStorage) {
          attachOpts.push("OVERRIDE_DATA_PATH true");
       }
       attachCmd += ` (${attachOpts.join(", ")});`;
 
-      logger.info(`Attaching DuckLake manifest catalog: ${attachCmd}`);
-      await connection.run(attachCmd);
+      logger.info(
+         `Attaching DuckLake manifest catalog: ${redactPgSecrets(attachCmd)}`,
+      );
+      try {
+         await connection.run(attachCmd);
+      } catch (error) {
+         const outcome = handlePgAttachError(
+            error,
+            `DuckLake catalog credentials rejected for ${catalogName}`,
+         );
+         if (outcome.action === "swallow") {
+            logger.info(
+               `DuckLake catalog ${catalogName} is already attached, skipping`,
+            );
+            return;
+         }
+         if (outcome.error instanceof ConnectionAuthError) {
+            logger.warn("DuckLake catalog credentials rejected", {
+               catalogName,
+            });
+         }
+         throw outcome.error;
+      }
    }
 
    getRepository(): ResourceRepository {