@malloy-publisher/server 0.0.197 → 0.0.198-dev1

package/src/service/package_race.spec.ts

@@ -0,0 +1,208 @@
+import { afterEach, beforeEach, describe, expect, it } from "bun:test";
+import * as fs from "fs/promises";
+import * as os from "os";
+import * as path from "path";
+import { Environment } from "./environment";
+
+/**
+ * Race-condition regression tests for the package-directory pipeline.
+ *
+ * Three tests, all deterministic without timing-based flakiness:
+ *
+ *  1. **Behavioral race repro** — concurrently install (rewrite the
+ *     package directory) and read (`getModelFileText`); assert no
+ *     `ENOENT` is observed. On the pre-fix code, the read would fail
+ *     mid-rewrite. With the per-package mutex now covering both paths,
+ *     all reads succeed.
+ *
+ *  2. **Mutex coverage** — manually hold `withPackageLock` and assert
+ *     that a concurrent reader is pending until released. Pins the
+ *     invariant that readers actually take the lock.
+ *
+ *  3. **Download does not block compile** — start an `installPackage`
+ *     whose downloader never resolves on its own, then assert that
+ *     `getModelFileText` resolves promptly. This pins the Phase 1 /
+ *     Phase 2 split — if a future regression accidentally moves the
+ *     download inside the lock, this test fails.
+ */
+describe("package directory race", () => {
+   let rootDir: string;
+   let envPath: string;
+   let fixtureDir: string;
+
+   const PUBLISHER_JSON = JSON.stringify({
+      name: "pkg",
+      description: "race-test fixture",
+   });
+   const MODEL_MALLOY = `source: ones is duckdb.sql("SELECT 1 as x")\n`;
+
+   async function writeFixture(targetDir: string): Promise<void> {
+      await fs.mkdir(targetDir, { recursive: true });
+      await fs.writeFile(
+         path.join(targetDir, "publisher.json"),
+         PUBLISHER_JSON,
+      );
+      await fs.writeFile(path.join(targetDir, "model.malloy"), MODEL_MALLOY);
+   }
+
+   async function copyDir(src: string, dst: string): Promise<void> {
+      await fs.mkdir(dst, { recursive: true });
+      await fs.cp(src, dst, { recursive: true });
+   }
+
+   beforeEach(async () => {
+      rootDir = await fs.mkdtemp(path.join(os.tmpdir(), "publisher-race-"));
+      envPath = path.join(rootDir, "env");
+      fixtureDir = path.join(rootDir, "fixture");
+      await fs.mkdir(envPath, { recursive: true });
+      await writeFixture(fixtureDir);
+   });
+
+   afterEach(async () => {
+      await fs.rm(rootDir, { recursive: true, force: true }).catch(() => {});
+   });
+
+   it("(A) concurrent installs and reads never observe a half-rewritten tree", async () => {
+      const env = await Environment.create("testEnv", envPath, []);
+
+      // Initial install to populate the canonical path.
+      await env.installPackage("pkg", (stagingPath) =>
+         copyDir(fixtureDir, stagingPath),
+      );
+
+      const ITERATIONS = 30;
+      const errors: unknown[] = [];
+      let mutatorDone = false;
+
+      // Mutator loop: re-install the package over and over. Each iteration
+      // exercises the full Phase 1 (no-lock) + Phase 2 (locked) swap.
+      const mutator = (async () => {
+         try {
+            for (let i = 0; i < ITERATIONS; i++) {
+               try {
+                  await env.installPackage("pkg", (stagingPath) =>
+                     copyDir(fixtureDir, stagingPath),
+                  );
+               } catch (err) {
+                  errors.push({ kind: "install", err });
+               }
+            }
+         } finally {
+            mutatorDone = true;
+         }
+      })();
+
+      // Reader loop: hammer `getModelFileText` while installs run. On the
+      // pre-fix code (no lock on reads), the read would sometimes hit ENOENT
+      // because the canonical dir was momentarily missing during the rename
+      // window. With the per-package mutex covering reads as well, this
+      // window is never observable.
+      const reader = (async () => {
+         while (!mutatorDone) {
+            try {
+               const text = await env.getModelFileText("pkg", "model.malloy");
+               expect(text).toBe(MODEL_MALLOY);
+            } catch (err) {
+               errors.push({ kind: "read", err });
+            }
+         }
+      })();
+
+      await mutator;
+      await reader;
+
+      // Any error here means the lock wasn't actually covering one of the
+      // sides — that's the regression we're guarding against.
+      if (errors.length > 0) {
+         throw new Error(
+            `Observed ${errors.length} race-window error(s): ${JSON.stringify(
+               errors.slice(0, 3),
+               (_k, v) => (v instanceof Error ? `${v.name}: ${v.message}` : v),
+            )}`,
+         );
+      }
+   }, 60_000);
+
+   it("(B) compile-time disk reads queue behind withPackageLock", async () => {
+      const env = await Environment.create("testEnv", envPath, []);
+      await env.installPackage("pkg", (stagingPath) =>
+         copyDir(fixtureDir, stagingPath),
+      );
+
+      const lockEntered = defer<void>();
+      const releaseLock = defer<void>();
+
+      // Hold the per-package mutex from "outside" — simulates a mutator
+      // (install / delete / writePackageManifest) being in flight.
+      const lockHolder = env.withPackageLock("pkg", async () => {
+         lockEntered.resolve();
+         await releaseLock.promise;
+      });
+
+      await lockEntered.promise;
+
+      // While the lock is held, the reader must NOT make progress.
+      const readPromise = env.getModelFileText("pkg", "model.malloy");
+      const TIMEOUT_SENTINEL = Symbol("timeout");
+      const raced = await Promise.race([
+         readPromise,
+         new Promise<typeof TIMEOUT_SENTINEL>((resolve) =>
+            setTimeout(() => resolve(TIMEOUT_SENTINEL), 50),
+         ),
+      ]);
+      expect(raced).toBe(TIMEOUT_SENTINEL);
+
+      // Release the lock; the reader must now complete.
+      releaseLock.resolve();
+      await lockHolder;
+      const text = await readPromise;
+      expect(text).toBe(MODEL_MALLOY);
+   }, 15_000);
+
+   it("(C) a slow download does not block concurrent reads", async () => {
+      const env = await Environment.create("testEnv", envPath, []);
+      // Initial install to make the package present.
+      await env.installPackage("pkg", (stagingPath) =>
+         copyDir(fixtureDir, stagingPath),
+      );
+
+      const downloadGate = defer<void>();
+
+      // Kick off an install whose Phase 1 downloader stalls until we open
+      // the gate. Phase 2 (the brief locked swap) cannot run until then.
+      const slowInstall = env.installPackage("pkg", async (stagingPath) => {
+         await downloadGate.promise;
+         await copyDir(fixtureDir, stagingPath);
+      });
+
+      // The reader must resolve well before we open the gate, proving the
+      // per-package mutex is NOT held during Phase 1.
+      const readStart = Date.now();
+      const text = await env.getModelFileText("pkg", "model.malloy");
+      const readElapsedMs = Date.now() - readStart;
+
+      expect(text).toBe(MODEL_MALLOY);
+      // 1s is generous; in practice this resolves in single-digit ms.
+      expect(readElapsedMs).toBeLessThan(1_000);
+
+      // Now open the gate and let the install complete.
+      downloadGate.resolve();
+      await slowInstall;
+   }, 15_000);
+});
+
+interface Deferred<T> {
+   promise: Promise<T>;
+   resolve: (value: T) => void;
+   reject: (reason?: unknown) => void;
+}
+
+function defer<T>(): Deferred<T> {
+   let resolve!: (value: T) => void;
+   let reject!: (reason?: unknown) => void;
+   const promise = new Promise<T>((res, rej) => {
+      resolve = res;
+      reject = rej;
+   });
+   return { promise, resolve, reject };
+}

package/tests/integration/concurrent_package/concurrent_package.integration.spec.ts

@@ -0,0 +1,280 @@
+/// <reference types="bun-types" />
+
+/**
+ * Regression test for the per-package download/unzip race.
+ *
+ * Before fix: concurrent POST/PATCH/GET-with-reload against the same package
+ * name landed in `downloadPackage` *before* acquiring the per-package mutex.
+ * Multiple callers would `rm -rf <targetPath>` and then `cp -r` / `extractAllTo`
+ * in parallel, leaving the directory in an inconsistent state. Subsequent
+ * reads failed with "Package manifest for ... does not exist" and any
+ * in-flight model compilation would 500 with "compiling model path not found".
+ *
+ * After fix: download is run inside the per-package mutex, so concurrent
+ * callers serialize on the same target path. All N requests must succeed
+ * and the package must be usable afterwards.
+ */
+
+import { afterAll, beforeAll, describe, expect, it } from "bun:test";
+import path from "path";
+import { fileURLToPath } from "url";
+import { RestE2EEnv, startRestE2E } from "../../harness/rest_e2e";
+
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = path.dirname(__filename);
+
+// Use a unique env name per run so stale state from a previous run (env
+// directory, persisted env metadata in publisher.db) can't poison startup.
+const ENV_NAME = `concurrent-package-test-env-${Date.now()}`;
+const PACKAGE_NAME = "gcs_faa";
+const CONCURRENCY = 12;
+
+const FORBIDDEN_ERROR_FRAGMENTS = [
+   "Package manifest for",
+   "does not exist",
+   "compiling model path not found",
+   "model path not found",
+];
+
+function findForbiddenError(body: unknown): string | undefined {
+   const text = typeof body === "string" ? body : JSON.stringify(body ?? "");
+   return FORBIDDEN_ERROR_FRAGMENTS.find((frag) => text.includes(frag));
+}
+
+describe("Concurrent package operations (E2E)", () => {
+   let env: (RestE2EEnv & { stop(): Promise<void> }) | null = null;
+   let baseUrl: string;
+   let fixtureDir: string;
+
+   beforeAll(async () => {
+      env = await startRestE2E();
+      baseUrl = env.baseUrl;
+      fixtureDir = "gs://publisher_test_packages/gcs_faa.zip";
+
+      // Seed the environment with the package once so the env exists on disk.
+      // addEnvironment requires at least one package.
+      const createRes = await fetch(`${baseUrl}/api/v0/environments`, {
+         method: "POST",
+         headers: { "Content-Type": "application/json" },
+         body: JSON.stringify({
+            name: ENV_NAME,
+            packages: [{ name: PACKAGE_NAME, location: fixtureDir }],
+            connections: [],
+         }),
+      });
+      if (!createRes.ok) {
+         const body = await createRes.text();
+         throw new Error(
+            `Failed to seed test environment (${createRes.status}): ${body}`,
+         );
+      }
+
+      // Wait for the seeded package to be loadable.
+      const deadline = Date.now() + 30_000;
+      let ready = false;
+      while (!ready && Date.now() < deadline) {
+         try {
+            const res = await fetch(
+               `${baseUrl}/api/v0/environments/${ENV_NAME}/packages/${PACKAGE_NAME}`,
+            );
+            if (res.ok) {
+               ready = true;
+               break;
+            }
+         } catch {
+            // not ready yet
+         }
+         await new Promise((r) => setTimeout(r, 250));
+      }
+      if (!ready) {
+         throw new Error("Seeded package did not become available in time");
+      }
+   });
+
+   afterAll(async () => {
+      if (baseUrl) {
+         try {
+            await fetch(`${baseUrl}/api/v0/environments/${ENV_NAME}`, {
+               method: "DELETE",
+            });
+         } catch {
+            // best-effort cleanup
+         }
+      }
+      await env?.stop();
+      env = null;
+   });
+
+   it("concurrent POST /packages for the same name all succeed", async () => {
+      const requests = Array.from({ length: CONCURRENCY }, () =>
+         fetch(`${baseUrl}/api/v0/environments/${ENV_NAME}/packages`, {
+            method: "POST",
+            headers: { "Content-Type": "application/json" },
+            body: JSON.stringify({
+               name: PACKAGE_NAME,
+               location: fixtureDir,
+            }),
+         }),
+      );
+
+      const responses = await Promise.all(requests);
+      const bodies = await Promise.all(
+         responses.map(async (r) => ({
+            status: r.status,
+            body: await r.json().catch(() => null),
+         })),
+      );
+
+      for (const { status, body } of bodies) {
+         expect(status).toBe(200);
+         const forbidden = findForbiddenError(body);
+         expect(forbidden).toBeUndefined();
+      }
+
+      // Package must be loadable after the storm.
+      const res = await fetch(
+         `${baseUrl}/api/v0/environments/${ENV_NAME}/packages/${PACKAGE_NAME}`,
+      );
+      expect(res.status).toBe(200);
+      const meta = (await res.json()) as { name?: string };
+      expect(meta.name).toBe(PACKAGE_NAME);
+
+      // Models must be listable — proves the model files survived the unzip
+      // race and Package.create read a consistent directory.
+      const modelsRes = await fetch(
+         `${baseUrl}/api/v0/environments/${ENV_NAME}/packages/${PACKAGE_NAME}/models`,
+      );
+      expect(modelsRes.status).toBe(200);
+      const models = (await modelsRes.json()) as Array<{ path?: string }>;
+      expect(Array.isArray(models)).toBe(true);
+      expect(models.length).toBeGreaterThan(0);
+   });
+
+   it("concurrent GET /packages/:name?reload=true all succeed", async () => {
+      const requests = Array.from({ length: CONCURRENCY }, () =>
+         fetch(
+            `${baseUrl}/api/v0/environments/${ENV_NAME}/packages/${PACKAGE_NAME}?reload=true`,
+         ),
+      );
+      const responses = await Promise.all(requests);
+      const bodies = await Promise.all(
+         responses.map(async (r) => ({
+            status: r.status,
+            body: await r.json().catch(() => null),
+         })),
+      );
+      for (const { status, body } of bodies) {
+         expect(status).toBe(200);
+         const forbidden = findForbiddenError(body);
+         expect(forbidden).toBeUndefined();
+      }
+
+      // Models must still list cleanly.
+      const modelsRes = await fetch(
+         `${baseUrl}/api/v0/environments/${ENV_NAME}/packages/${PACKAGE_NAME}/models`,
+      );
+      expect(modelsRes.status).toBe(200);
+   });
+
+   it("simultaneous POST + PATCH for the same package serialize cleanly", async () => {
+      // Fire create and update at the same time. Both should land under the
+      // same per-package mutex. Whichever wins the mutex first runs first;
+      // the loser sees a coherent post-condition. After both settle the
+      // package must be loadable and the description must reflect the PATCH
+      // when the PATCH ran *after* a successful POST.
+      const newDescription = `concurrent-update-${Date.now()}`;
+      const [postRes, patchRes] = await Promise.all([
+         fetch(`${baseUrl}/api/v0/environments/${ENV_NAME}/packages`, {
+            method: "POST",
+            headers: { "Content-Type": "application/json" },
+            body: JSON.stringify({
+               name: PACKAGE_NAME,
+               location: fixtureDir,
+            }),
+         }),
+         fetch(
+            `${baseUrl}/api/v0/environments/${ENV_NAME}/packages/${PACKAGE_NAME}`,
+            {
+               method: "PATCH",
+               headers: { "Content-Type": "application/json" },
+               body: JSON.stringify({
+                  name: PACKAGE_NAME,
+                  description: newDescription,
+               }),
+            },
+         ),
+      ]);
+
+      // POST is idempotent here — always succeeds.
+      expect(postRes.status).toBe(200);
+      const postBody = await postRes.json().catch(() => null);
+      expect(findForbiddenError(postBody)).toBeUndefined();
+
+      // PATCH either succeeds (ran after POST loaded the package) or 404s
+      // (ran before POST populated `this.packages`). It must NOT silently
+      // rewrite disk and then 404, and must never surface the forbidden
+      // error fragments.
+      const patchBody = await patchRes.json().catch(() => null);
+      expect(findForbiddenError(patchBody)).toBeUndefined();
+      expect([200, 404]).toContain(patchRes.status);
+
+      // Whatever happened, the package must remain loadable.
+      const getRes = await fetch(
+         `${baseUrl}/api/v0/environments/${ENV_NAME}/packages/${PACKAGE_NAME}`,
+      );
+      expect(getRes.status).toBe(200);
+      const meta = (await getRes.json()) as {
+         name?: string;
+         description?: string;
+      };
+      expect(meta.name).toBe(PACKAGE_NAME);
+   });
+
+   it("interleaved POST + GET-reload + model list never surface stale-dir errors", async () => {
+      // Worst-case interleave: writers and readers hammering the same
+      // package. None of them should observe a missing manifest or a
+      // half-extracted directory.
+      const work: Array<Promise<{ status: number; body: unknown }>> = [];
+      for (let i = 0; i < CONCURRENCY; i++) {
+         work.push(
+            fetch(`${baseUrl}/api/v0/environments/${ENV_NAME}/packages`, {
+               method: "POST",
+               headers: { "Content-Type": "application/json" },
+               body: JSON.stringify({
+                  name: PACKAGE_NAME,
+                  location: fixtureDir,
+               }),
+            }).then(async (r) => ({
+               status: r.status,
+               body: await r.json().catch(() => null),
+            })),
+         );
+         work.push(
+            fetch(
+               `${baseUrl}/api/v0/environments/${ENV_NAME}/packages/${PACKAGE_NAME}?reload=true`,
+            ).then(async (r) => ({
+               status: r.status,
+               body: await r.json().catch(() => null),
+            })),
+         );
+         work.push(
+            fetch(
+               `${baseUrl}/api/v0/environments/${ENV_NAME}/packages/${PACKAGE_NAME}/models`,
+            ).then(async (r) => ({
+               status: r.status,
+               body: await r.json().catch(() => null),
+            })),
+         );
+      }
+
+      const results = await Promise.all(work);
+      for (const { status, body } of results) {
+         const forbidden = findForbiddenError(body);
+         expect(forbidden).toBeUndefined();
+         // Model list while the package is being rewritten can transiently
+         // return 404 (the package is unloaded mid-rewrite), but never 5xx,
+         // and never with the forbidden error fragments.
+         expect(status).toBeLessThan(500);
+      }
+   });
+});