@malloy-publisher/server 0.0.198-dev2 → 0.0.198-dev4

package/src/service/schema_worker_pool.ts

@@ -1,278 +0,0 @@
-/**
- * Long-lived pool of {@link Worker} threads that perform DuckDB schema
- * introspection off the main event loop.
- *
- * Why a dedicated pool (not the libuv worker pool, not setImmediate):
- *
- *  - `@malloydata/db-duckdb` opens DuckDB via a native addon. Every
- *    DuckDBConnection allocates its own native thread pool sized to
- *    the host's CPU count (not the cgroup's CPU share). Concurrent
- *    schema introspection on the main thread compounded into the
- *    466-leaked-Bun-Pool-threads / 90GB-VmSize OOM signature seen on
- *    `worker-76b49bdb89-8bsv4`.
- *
- *  - Owning the DuckDBConnection inside a worker isolates the native
- *    pool to *that* worker. Per-pool sizing → predictable thread
- *    budget. Worker exit → native threads die with it. No leak across
- *    package loads.
- *
- *  - The schema-introspection case is uniquely suited to workers:
- *    inputs and outputs are plain JSON (structured-cloneable), and
- *    the work touches no environment connections, so we don't need
- *    cross-thread IPC for live Snowflake/BigQuery handles. This is
- *    why we tackle schema first — model compile (which *does* need
- *    live env connections) is a much bigger lift, tracked separately.
- */
-import { Worker } from "worker_threads";
-
-import { logger } from "../logger";
-
-type ColumnInfo = { type: string; name: string };
-
-/**
- * Public-facing schema-row shape. Mirrors the original synchronous
- * `getDatabaseInfo` return so callers in `package.ts` are unchanged.
- */
-export interface SchemaResult {
-   name: string;
-   rowCount: number;
-   columns: ColumnInfo[];
-}
-
-interface WorkerSlot {
-   worker: Worker;
-   /** Whether the worker is currently handling a request. */
-   busy: boolean;
-}
-
-interface PendingRequest {
-   id: number;
-   packagePath: string;
-   databasePath: string;
-   resolve: (value: SchemaResult) => void;
-   reject: (reason: Error) => void;
-}
-
-const DEFAULT_POOL_SIZE = 2;
-
-export class SchemaWorkerPool {
-   private readonly workers: WorkerSlot[] = [];
-   private readonly queue: PendingRequest[] = [];
-   /** id → pending request currently executing. */
-   private readonly inFlight = new Map<number, PendingRequest>();
-   /** Maps a worker index to the id of the request it's running. */
-   private readonly workerCurrentId = new Map<number, number>();
-   private nextId = 1;
-   private stopped = false;
-
-   constructor(
-      private readonly workerUrl: URL,
-      private readonly size: number = DEFAULT_POOL_SIZE,
-   ) {}
-
-   public start(): void {
-      if (this.workers.length > 0) return;
-      for (let i = 0; i < this.size; i++) {
-         this.workers.push(this.spawn(i));
-      }
-      logger.info(`SchemaWorkerPool started (size=${this.size})`);
-   }
-
-   public async stop(): Promise<void> {
-      this.stopped = true;
-      // Fail any queued/in-flight work so callers don't hang on shutdown.
-      const shutdownError = new Error("SchemaWorkerPool stopped");
-      for (const req of this.queue.splice(0)) req.reject(shutdownError);
-      for (const req of this.inFlight.values()) req.reject(shutdownError);
-      this.inFlight.clear();
-      await Promise.all(
-         this.workers.map(async (slot) => {
-            try {
-               await slot.worker.terminate();
-            } catch {
-               // Best-effort: terminate failures shouldn't block shutdown.
-            }
-         }),
-      );
-      this.workers.length = 0;
-   }
-
-   /**
-    * Submit one schema-introspection job. Resolves with the schema
-    * description; rejects if the worker returns an error or crashes.
-    *
-    * Concurrent calls beyond the pool size are queued FIFO; once a
-    * worker frees up the next queued request is dispatched.
-    */
-   public submit(
-      packagePath: string,
-      databasePath: string,
-   ): Promise<SchemaResult> {
-      if (this.stopped) {
-         return Promise.reject(new Error("SchemaWorkerPool stopped"));
-      }
-      if (this.workers.length === 0) {
-         return Promise.reject(
-            new Error("SchemaWorkerPool.submit called before start()"),
-         );
-      }
-      return new Promise<SchemaResult>((resolve, reject) => {
-         const req: PendingRequest = {
-            id: this.nextId++,
-            packagePath,
-            databasePath,
-            resolve,
-            reject,
-         };
-         this.queue.push(req);
-         this.drain();
-      });
-   }
-
-   /**
-    * Try to assign queued requests to idle workers. Cheap; called
-    * after every enqueue and after every worker completes a request.
-    */
-   private drain(): void {
-      for (let i = 0; i < this.workers.length; i++) {
-         if (this.queue.length === 0) return;
-         const slot = this.workers[i];
-         if (slot.busy) continue;
-         const req = this.queue.shift()!;
-         slot.busy = true;
-         this.inFlight.set(req.id, req);
-         this.workerCurrentId.set(i, req.id);
-         slot.worker.postMessage({
-            id: req.id,
-            packagePath: req.packagePath,
-            databasePath: req.databasePath,
-         });
-      }
-   }
-
-   private spawn(index: number): WorkerSlot {
-      const worker = new Worker(this.workerUrl);
-      const slot: WorkerSlot = { worker, busy: false };
-
-      worker.on(
-         "message",
-         (msg: {
-            id: number;
-            ok: boolean;
-            result?: SchemaResult;
-            error?: { message: string; stack?: string };
-         }) => {
-            const req = this.inFlight.get(msg.id);
-            if (!req) {
-               logger.warn("SchemaWorkerPool: response for unknown request", {
-                  id: msg.id,
-                  workerIndex: index,
-               });
-               return;
-            }
-            this.inFlight.delete(msg.id);
-            this.workerCurrentId.delete(index);
-            slot.busy = false;
-            if (msg.ok && msg.result) {
-               req.resolve(msg.result);
-            } else {
-               const err = new Error(msg.error?.message ?? "Unknown error");
-               if (msg.error?.stack) err.stack = msg.error.stack;
-               req.reject(err);
-            }
-            this.drain();
-         },
-      );
-
-      worker.on("error", (err) => {
-         // A native crash inside the worker — fail the in-flight request
-         // attributed to this slot, then respawn the worker so the pool
-         // self-heals. Without respawn, one crash silently shrinks
-         // capacity and concurrent loads would queue forever.
-         const inFlightId = this.workerCurrentId.get(index);
-         if (inFlightId !== undefined) {
-            const req = this.inFlight.get(inFlightId);
-            if (req) {
-               this.inFlight.delete(inFlightId);
-               req.reject(err);
-            }
-            this.workerCurrentId.delete(index);
-         }
-         logger.error("SchemaWorkerPool: worker errored, respawning", {
-            workerIndex: index,
-            error: err,
-         });
-         if (!this.stopped) {
-            this.workers[index] = this.spawn(index);
-            this.drain();
-         }
-      });
-
-      worker.on("exit", (code) => {
-         if (this.stopped) return;
-         if (code !== 0) {
-            logger.warn("SchemaWorkerPool: worker exited unexpectedly", {
-               workerIndex: index,
-               code,
-            });
-            // Treat unexpected exit like an error: respawn so the pool
-            // doesn't silently lose capacity.
-            const inFlightId = this.workerCurrentId.get(index);
-            if (inFlightId !== undefined) {
-               const req = this.inFlight.get(inFlightId);
-               if (req) {
-                  this.inFlight.delete(inFlightId);
-                  req.reject(
-                     new Error(`SchemaWorker exited with code ${code}`),
-                  );
-               }
-               this.workerCurrentId.delete(index);
-            }
-            this.workers[index] = this.spawn(index);
-            this.drain();
-         }
-      });
-
-      return slot;
-   }
-}
-
-/**
- * Process-wide singleton. Constructed lazily so importing this module
- * doesn't spawn workers in test environments that never call
- * `getSchemaWorkerPool()`.
- *
- * The worker URL is resolved from `import.meta.url`, which lets Bun
- * load `schema_worker.ts` directly in dev and the bundled
- * `schema_worker.mjs` in prod (see `build.ts`).
- */
-let singleton: SchemaWorkerPool | null = null;
-
-export function getSchemaWorkerPool(): SchemaWorkerPool {
-   if (!singleton) {
-      const url = resolveWorkerUrl();
-      const size = Number(process.env.PUBLISHER_SCHEMA_WORKER_POOL_SIZE) || 2;
-      singleton = new SchemaWorkerPool(url, size);
-      singleton.start();
-   }
-   return singleton;
-}
-
-function resolveWorkerUrl(): URL {
-   // In dev (`bun --watch src/server.ts`), import.meta.url points at
-   // `.../src/service/schema_worker_pool.ts` and the worker is the
-   // sibling `.ts` file.
-   //
-   // In prod, this module gets inlined into `dist/server.mjs`, so
-   // `import.meta.url` resolves to `dist/server.mjs`. Bun's bundler
-   // nests outputs by their path relative to the common entrypoint
-   // root (./src), so schema_worker lands at
-   // `dist/service/schema_worker.mjs` — one directory below
-   // server.mjs.
-   const base = new URL(import.meta.url);
-   const isBundled = base.pathname.endsWith(".mjs");
-   return new URL(
-      isBundled ? "./service/schema_worker.mjs" : "./schema_worker.ts",
-      base,
-   );
-}

package/tests/integration/concurrent_environment/concurrent_environment.integration.spec.ts

@@ -1,235 +0,0 @@
-/// <reference types="bun-types" />
-
-/**
- * Regression test for per-environment load/scaffold races.
- *
- * Before fix: concurrent GET (especially ?reload=true), POST, and PATCH against
- * the same environment name could enter `getEnvironment` / `addEnvironment` in
- * parallel. Multiple callers would scaffold or re-load the same directory
- * concurrently, leaving publisher.db and on-disk state inconsistent. Lazy loads
- * then failed with `Environment "…" could not be resolved to a path.`
- *
- * After fix: environment operations serialize on a per-environment mutex in
- * `EnvironmentStore.getEnvironment`, so concurrent callers share one load path.
- * All N requests must succeed and the environment must remain usable afterwards.
- */
-
-import { afterAll, beforeAll, describe, expect, it } from "bun:test";
-import { RestE2EEnv, startRestE2E } from "../../harness/rest_e2e";
-
-const ENV_NAME = `concurrent-environment-test-env-${Date.now()}`;
-const PACKAGE_NAME = "gcs_faa";
-const FIXTURE_LOCATION = "gs://publisher_test_packages/gcs_faa.zip";
-const CONCURRENCY = 12;
-
-const FORBIDDEN_ERROR_FRAGMENTS = [
-   "could not be resolved to a path",
-   "Package manifest for",
-   "does not exist",
-   "compiling model path not found",
-   "model path not found",
-];
-
-function findForbiddenError(body: unknown): string | undefined {
-   const text = typeof body === "string" ? body : JSON.stringify(body ?? "");
-   return FORBIDDEN_ERROR_FRAGMENTS.find((frag) => text.includes(frag));
-}
-
-function environmentPayload(description?: string) {
-   return {
-      name: ENV_NAME,
-      packages: [{ name: PACKAGE_NAME, location: FIXTURE_LOCATION }],
-      connections: [],
-      ...(description !== undefined ? { description } : {}),
-   };
-}
-
-async function waitForPackageReady(
-   baseUrl: string,
-   deadlineMs = 30_000,
-): Promise<void> {
-   const deadline = Date.now() + deadlineMs;
-   while (Date.now() < deadline) {
-      try {
-         const res = await fetch(
-            `${baseUrl}/api/v0/environments/${ENV_NAME}/packages/${PACKAGE_NAME}`,
-         );
-         if (res.ok) {
-            return;
-         }
-      } catch {
-         // not ready yet
-      }
-      await new Promise((r) => setTimeout(r, 250));
-   }
-   throw new Error("Seeded package did not become available in time");
-}
-
-describe("Concurrent environment operations (E2E)", () => {
-   let env: (RestE2EEnv & { stop(): Promise<void> }) | null = null;
-   let baseUrl: string;
-
-   beforeAll(async () => {
-      env = await startRestE2E();
-      baseUrl = env.baseUrl;
-
-      const createRes = await fetch(`${baseUrl}/api/v0/environments`, {
-         method: "POST",
-         headers: { "Content-Type": "application/json" },
-         body: JSON.stringify(environmentPayload()),
-      });
-      if (!createRes.ok) {
-         const body = await createRes.text();
-         throw new Error(
-            `Failed to seed test environment (${createRes.status}): ${body}`,
-         );
-      }
-      await waitForPackageReady(baseUrl);
-   });
-
-   afterAll(async () => {
-      if (baseUrl) {
-         try {
-            await fetch(`${baseUrl}/api/v0/environments/${ENV_NAME}`, {
-               method: "DELETE",
-            });
-         } catch {
-            // best-effort cleanup
-         }
-      }
-      await env?.stop();
-      env = null;
-   });
-
-   it("concurrent POST /environments for the same name all succeed", async () => {
-      const requests = Array.from({ length: CONCURRENCY }, () =>
-         fetch(`${baseUrl}/api/v0/environments`, {
-            method: "POST",
-            headers: { "Content-Type": "application/json" },
-            body: JSON.stringify(environmentPayload()),
-         }),
-      );
-
-      const responses = await Promise.all(requests);
-      const bodies = await Promise.all(
-         responses.map(async (r) => ({
-            status: r.status,
-            body: await r.json().catch(() => null),
-         })),
-      );
-
-      for (const { status, body } of bodies) {
-         expect(status).toBe(200);
-         const forbidden = findForbiddenError(body);
-         expect(forbidden).toBeUndefined();
-         const meta = body as { name?: string };
-         expect(meta.name).toBe(ENV_NAME);
-      }
-
-      await waitForPackageReady(baseUrl);
-
-      const getRes = await fetch(`${baseUrl}/api/v0/environments/${ENV_NAME}`);
-      expect(getRes.status).toBe(200);
-      const forbidden = findForbiddenError(
-         await getRes.json().catch(() => null),
-      );
-      expect(forbidden).toBeUndefined();
-   });
-
-   it("concurrent GET /environments/:name?reload=true all succeed", async () => {
-      const requests = Array.from({ length: CONCURRENCY }, () =>
-         fetch(`${baseUrl}/api/v0/environments/${ENV_NAME}?reload=true`),
-      );
-      const responses = await Promise.all(requests);
-      const bodies = await Promise.all(
-         responses.map(async (r) => ({
-            status: r.status,
-            body: await r.json().catch(() => null),
-         })),
-      );
-
-      for (const { status, body } of bodies) {
-         expect(status).toBe(200);
-         const forbidden = findForbiddenError(body);
-         expect(forbidden).toBeUndefined();
-         const meta = body as { name?: string };
-         expect(meta.name).toBe(ENV_NAME);
-      }
-
-      await waitForPackageReady(baseUrl);
-   });
-
-   it("simultaneous POST + PATCH for the same environment serialize cleanly", async () => {
-      const newReadme = `concurrent-env-update-${Date.now()}`;
-      const [postRes, patchRes] = await Promise.all([
-         fetch(`${baseUrl}/api/v0/environments`, {
-            method: "POST",
-            headers: { "Content-Type": "application/json" },
-            body: JSON.stringify(environmentPayload()),
-         }),
-         fetch(`${baseUrl}/api/v0/environments/${ENV_NAME}`, {
-            method: "PATCH",
-            headers: { "Content-Type": "application/json" },
-            body: JSON.stringify({
-               name: ENV_NAME,
-               readme: newReadme,
-            }),
-         }),
-      ]);
-
-      expect(postRes.status).toBe(200);
-      expect(
-         findForbiddenError(await postRes.json().catch(() => null)),
-      ).toBeUndefined();
-
-      const patchBody = await patchRes.json().catch(() => null);
-      expect(findForbiddenError(patchBody)).toBeUndefined();
-      expect([200, 404]).toContain(patchRes.status);
-
-      const getRes = await fetch(`${baseUrl}/api/v0/environments/${ENV_NAME}`);
-      expect(getRes.status).toBe(200);
-      const meta = (await getRes.json()) as { name?: string; readme?: string };
-      expect(meta.name).toBe(ENV_NAME);
-   });
-
-   it("interleaved POST + GET-reload + package list never surface path errors", async () => {
-      const work: Array<Promise<{ status: number; body: unknown }>> = [];
-      for (let i = 0; i < CONCURRENCY; i++) {
-         work.push(
-            fetch(`${baseUrl}/api/v0/environments`, {
-               method: "POST",
-               headers: { "Content-Type": "application/json" },
-               body: JSON.stringify(environmentPayload()),
-            }).then(async (r) => ({
-               status: r.status,
-               body: await r.json().catch(() => null),
-            })),
-         );
-         work.push(
-            fetch(
-               `${baseUrl}/api/v0/environments/${ENV_NAME}?reload=true`,
-            ).then(async (r) => ({
-               status: r.status,
-               body: await r.json().catch(() => null),
-            })),
-         );
-         work.push(
-            fetch(`${baseUrl}/api/v0/environments/${ENV_NAME}/packages`).then(
-               async (r) => ({
-                  status: r.status,
-                  body: await r.json().catch(() => null),
-               }),
-            ),
-         );
-      }
-
-      const results = await Promise.all(work);
-      for (const { status, body } of results) {
-         const forbidden = findForbiddenError(body);
-         expect(forbidden).toBeUndefined();
-         expect(status).toBeLessThan(500);
-      }
-
-      await waitForPackageReady(baseUrl);
-   });
-});