@malloy-publisher/server 0.0.198-dev → 0.0.198-dev1

package/tests/integration/concurrent_package/concurrent_package.integration.spec.ts

@@ -0,0 +1,280 @@
+/// <reference types="bun-types" />
+
+/**
+ * Regression test for the per-package download/unzip race.
+ *
+ * Before fix: concurrent POST/PATCH/GET-with-reload against the same package
+ * name landed in `downloadPackage` *before* acquiring the per-package mutex.
+ * Multiple callers would `rm -rf <targetPath>` and then `cp -r` / `extractAllTo`
+ * in parallel, leaving the directory in an inconsistent state. Subsequent
+ * reads failed with "Package manifest for ... does not exist" and any
+ * in-flight model compilation would 500 with "compiling model path not found".
+ *
+ * After fix: download is run inside the per-package mutex, so concurrent
+ * callers serialize on the same target path. All N requests must succeed
+ * and the package must be usable afterwards.
+ */
+
+import { afterAll, beforeAll, describe, expect, it } from "bun:test";
+import path from "path";
+import { fileURLToPath } from "url";
+import { RestE2EEnv, startRestE2E } from "../../harness/rest_e2e";
+
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = path.dirname(__filename);
+
+// Use a unique env name per run so stale state from a previous run (env
+// directory, persisted env metadata in publisher.db) can't poison startup.
+const ENV_NAME = `concurrent-package-test-env-${Date.now()}`;
+const PACKAGE_NAME = "gcs_faa";
+const CONCURRENCY = 12;
+
+const FORBIDDEN_ERROR_FRAGMENTS = [
+   "Package manifest for",
+   "does not exist",
+   "compiling model path not found",
+   "model path not found",
+];
+
+function findForbiddenError(body: unknown): string | undefined {
+   const text = typeof body === "string" ? body : JSON.stringify(body ?? "");
+   return FORBIDDEN_ERROR_FRAGMENTS.find((frag) => text.includes(frag));
+}
+
+describe("Concurrent package operations (E2E)", () => {
+   let env: (RestE2EEnv & { stop(): Promise<void> }) | null = null;
+   let baseUrl: string;
+   let fixtureDir: string;
+
+   beforeAll(async () => {
+      env = await startRestE2E();
+      baseUrl = env.baseUrl;
+      fixtureDir = "gs://publisher_test_packages/gcs_faa.zip";
+
+      // Seed the environment with the package once so the env exists on disk.
+      // addEnvironment requires at least one package.
+      const createRes = await fetch(`${baseUrl}/api/v0/environments`, {
+         method: "POST",
+         headers: { "Content-Type": "application/json" },
+         body: JSON.stringify({
+            name: ENV_NAME,
+            packages: [{ name: PACKAGE_NAME, location: fixtureDir }],
+            connections: [],
+         }),
+      });
+      if (!createRes.ok) {
+         const body = await createRes.text();
+         throw new Error(
+            `Failed to seed test environment (${createRes.status}): ${body}`,
+         );
+      }
+
+      // Wait for the seeded package to be loadable.
+      const deadline = Date.now() + 30_000;
+      let ready = false;
+      while (!ready && Date.now() < deadline) {
+         try {
+            const res = await fetch(
+               `${baseUrl}/api/v0/environments/${ENV_NAME}/packages/${PACKAGE_NAME}`,
+            );
+            if (res.ok) {
+               ready = true;
+               break;
+            }
+         } catch {
+            // not ready yet
+         }
+         await new Promise((r) => setTimeout(r, 250));
+      }
+      if (!ready) {
+         throw new Error("Seeded package did not become available in time");
+      }
+   });
+
+   afterAll(async () => {
+      if (baseUrl) {
+         try {
+            await fetch(`${baseUrl}/api/v0/environments/${ENV_NAME}`, {
+               method: "DELETE",
+            });
+         } catch {
+            // best-effort cleanup
+         }
+      }
+      await env?.stop();
+      env = null;
+   });
+
+   it("concurrent POST /packages for the same name all succeed", async () => {
+      const requests = Array.from({ length: CONCURRENCY }, () =>
+         fetch(`${baseUrl}/api/v0/environments/${ENV_NAME}/packages`, {
+            method: "POST",
+            headers: { "Content-Type": "application/json" },
+            body: JSON.stringify({
+               name: PACKAGE_NAME,
+               location: fixtureDir,
+            }),
+         }),
+      );
+
+      const responses = await Promise.all(requests);
+      const bodies = await Promise.all(
+         responses.map(async (r) => ({
+            status: r.status,
+            body: await r.json().catch(() => null),
+         })),
+      );
+
+      for (const { status, body } of bodies) {
+         expect(status).toBe(200);
+         const forbidden = findForbiddenError(body);
+         expect(forbidden).toBeUndefined();
+      }
+
+      // Package must be loadable after the storm.
+      const res = await fetch(
+         `${baseUrl}/api/v0/environments/${ENV_NAME}/packages/${PACKAGE_NAME}`,
+      );
+      expect(res.status).toBe(200);
+      const meta = (await res.json()) as { name?: string };
+      expect(meta.name).toBe(PACKAGE_NAME);
+
+      // Models must be listable — proves the model files survived the unzip
+      // race and Package.create read a consistent directory.
+      const modelsRes = await fetch(
+         `${baseUrl}/api/v0/environments/${ENV_NAME}/packages/${PACKAGE_NAME}/models`,
+      );
+      expect(modelsRes.status).toBe(200);
+      const models = (await modelsRes.json()) as Array<{ path?: string }>;
+      expect(Array.isArray(models)).toBe(true);
+      expect(models.length).toBeGreaterThan(0);
+   });
+
+   it("concurrent GET /packages/:name?reload=true all succeed", async () => {
+      const requests = Array.from({ length: CONCURRENCY }, () =>
+         fetch(
+            `${baseUrl}/api/v0/environments/${ENV_NAME}/packages/${PACKAGE_NAME}?reload=true`,
+         ),
+      );
+      const responses = await Promise.all(requests);
+      const bodies = await Promise.all(
+         responses.map(async (r) => ({
+            status: r.status,
+            body: await r.json().catch(() => null),
+         })),
+      );
+      for (const { status, body } of bodies) {
+         expect(status).toBe(200);
+         const forbidden = findForbiddenError(body);
+         expect(forbidden).toBeUndefined();
+      }
+
+      // Models must still list cleanly.
+      const modelsRes = await fetch(
+         `${baseUrl}/api/v0/environments/${ENV_NAME}/packages/${PACKAGE_NAME}/models`,
+      );
+      expect(modelsRes.status).toBe(200);
+   });
+
+   it("simultaneous POST + PATCH for the same package serialize cleanly", async () => {
+      // Fire create and update at the same time. Both should land under the
+      // same per-package mutex. Whichever wins the mutex first runs first;
+      // the loser sees a coherent post-condition. After both settle the
+      // package must be loadable and the description must reflect the PATCH
+      // when the PATCH ran *after* a successful POST.
+      const newDescription = `concurrent-update-${Date.now()}`;
+      const [postRes, patchRes] = await Promise.all([
+         fetch(`${baseUrl}/api/v0/environments/${ENV_NAME}/packages`, {
+            method: "POST",
+            headers: { "Content-Type": "application/json" },
+            body: JSON.stringify({
+               name: PACKAGE_NAME,
+               location: fixtureDir,
+            }),
+         }),
+         fetch(
+            `${baseUrl}/api/v0/environments/${ENV_NAME}/packages/${PACKAGE_NAME}`,
+            {
+               method: "PATCH",
+               headers: { "Content-Type": "application/json" },
+               body: JSON.stringify({
+                  name: PACKAGE_NAME,
+                  description: newDescription,
+               }),
+            },
+         ),
+      ]);
+
+      // POST is idempotent here — always succeeds.
+      expect(postRes.status).toBe(200);
+      const postBody = await postRes.json().catch(() => null);
+      expect(findForbiddenError(postBody)).toBeUndefined();
+
+      // PATCH either succeeds (ran after POST loaded the package) or 404s
+      // (ran before POST populated `this.packages`). It must NOT silently
+      // rewrite disk and then 404, and must never surface the forbidden
+      // error fragments.
+      const patchBody = await patchRes.json().catch(() => null);
+      expect(findForbiddenError(patchBody)).toBeUndefined();
+      expect([200, 404]).toContain(patchRes.status);
+
+      // Whatever happened, the package must remain loadable.
+      const getRes = await fetch(
+         `${baseUrl}/api/v0/environments/${ENV_NAME}/packages/${PACKAGE_NAME}`,
+      );
+      expect(getRes.status).toBe(200);
+      const meta = (await getRes.json()) as {
+         name?: string;
+         description?: string;
+      };
+      expect(meta.name).toBe(PACKAGE_NAME);
+   });
+
+   it("interleaved POST + GET-reload + model list never surface stale-dir errors", async () => {
+      // Worst-case interleave: writers and readers hammering the same
+      // package. None of them should observe a missing manifest or a
+      // half-extracted directory.
+      const work: Array<Promise<{ status: number; body: unknown }>> = [];
+      for (let i = 0; i < CONCURRENCY; i++) {
+         work.push(
+            fetch(`${baseUrl}/api/v0/environments/${ENV_NAME}/packages`, {
+               method: "POST",
+               headers: { "Content-Type": "application/json" },
+               body: JSON.stringify({
+                  name: PACKAGE_NAME,
+                  location: fixtureDir,
+               }),
+            }).then(async (r) => ({
+               status: r.status,
+               body: await r.json().catch(() => null),
+            })),
+         );
+         work.push(
+            fetch(
+               `${baseUrl}/api/v0/environments/${ENV_NAME}/packages/${PACKAGE_NAME}?reload=true`,
+            ).then(async (r) => ({
+               status: r.status,
+               body: await r.json().catch(() => null),
+            })),
+         );
+         work.push(
+            fetch(
+               `${baseUrl}/api/v0/environments/${ENV_NAME}/packages/${PACKAGE_NAME}/models`,
+            ).then(async (r) => ({
+               status: r.status,
+               body: await r.json().catch(() => null),
+            })),
+         );
+      }
+
+      const results = await Promise.all(work);
+      for (const { status, body } of results) {
+         const forbidden = findForbiddenError(body);
+         expect(forbidden).toBeUndefined();
+         // Model list while the package is being rewritten can transiently
+         // return 404 (the package is unloaded mid-rewrite), but never 5xx,
+         // and never with the forbidden error fragments.
+         expect(status).toBeLessThan(500);
+      }
+   });
+});

package/tests/integration/legacy_routes/legacy_routes.integration.spec.ts

@@ -0,0 +1,259 @@
+/// <reference types="bun-types" />
+
+// TODO: Remove this during projects cleanup
+/**
+ * Smoke tests for the legacy `/api/v0/projects/...` REST surface registered
+ * by `server-old.ts`. These routes exist purely to keep pre-rename SDK
+ * clients (e.g. `@malloydata/db-publisher`) working after the
+ * projects→environments rename.
+ *
+ * One test per route group: projects CRUD, packages, connections, models,
+ * notebooks, databases, queries, materializations, manifest. The
+ * materialization test additionally asserts the response field rename
+ * (`projectId` not `environmentId`). `/status` is no longer in the legacy
+ * surface — both old and new clients hit the single `/api/v0/status`
+ * handler in server.ts, which returns `environments`.
+ *
+ * This file is intentionally separate from the regular integration suite so
+ * it can be deleted in one motion when legacy support is dropped.
+ */
+
+import { afterAll, beforeAll, describe, expect, it } from "bun:test";
+import path from "path";
+import { fileURLToPath } from "url";
+import { RestE2EEnv, startRestE2E } from "../../harness/rest_e2e";
+
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = path.dirname(__filename);
+
+// Use a distinct project name so this suite doesn't collide with the
+// materialization integration suite (which also uses "test-project") if
+// they ever run in the same DB instance.
+const PROJECT_NAME = "legacy-routes-test-project";
+const PACKAGE_NAME = "persist-test";
+
+describe("Legacy /api/v0/projects/* REST routes (E2E)", () => {
+   let env: (RestE2EEnv & { stop(): Promise<void> }) | null = null;
+   let baseUrl: string;
+
+   beforeAll(async () => {
+      env = await startRestE2E();
+      baseUrl = env.baseUrl;
+
+      // Create the test environment via the LEGACY route — proves POST
+      // /projects works end-to-end.
+      const fixtureDir = path.resolve(__dirname, "../../fixtures/persist-test");
+      const createRes = await fetch(`${baseUrl}/api/v0/projects`, {
+         method: "POST",
+         headers: { "Content-Type": "application/json" },
+         body: JSON.stringify({
+            name: PROJECT_NAME,
+            packages: [{ name: PACKAGE_NAME, location: fixtureDir }],
+            connections: [],
+         }),
+      });
+      if (!createRes.ok) {
+         const body = await createRes.text();
+         throw new Error(
+            `Failed to create test project via legacy route (${createRes.status}): ${body}`,
+         );
+      }
+
+      // Wait for the package to finish loading via the legacy GET path.
+      const deadline = Date.now() + 30_000;
+      let pkgReady = false;
+      while (!pkgReady && Date.now() < deadline) {
+         try {
+            const res = await fetch(
+               `${baseUrl}/api/v0/projects/${PROJECT_NAME}/packages/${PACKAGE_NAME}`,
+            );
+            if (res.ok) {
+               pkgReady = true;
+               break;
+            }
+         } catch {
+            // not ready yet
+         }
+         await new Promise((r) => setTimeout(r, 500));
+      }
+      if (!pkgReady) {
+         throw new Error("Test package did not become available in time");
+      }
+   });
+
+   afterAll(async () => {
+      if (baseUrl) {
+         try {
+            // Clean up via the legacy DELETE route.
+            await fetch(`${baseUrl}/api/v0/projects/${PROJECT_NAME}`, {
+               method: "DELETE",
+            });
+         } catch {
+            // best-effort cleanup
+         }
+      }
+      await env?.stop();
+      env = null;
+   });
+
+   describe("projects CRUD", () => {
+      it("GET /projects lists environments under the legacy URL", async () => {
+         const res = await fetch(`${baseUrl}/api/v0/projects`);
+         expect(res.status).toBe(200);
+         const body = (await res.json()) as Array<{ name?: string }>;
+         expect(Array.isArray(body)).toBe(true);
+         expect(body.some((e) => e.name === PROJECT_NAME)).toBe(true);
+      });
+
+      it("GET /projects/:projectName returns the project", async () => {
+         const res = await fetch(`${baseUrl}/api/v0/projects/${PROJECT_NAME}`);
+         expect(res.status).toBe(200);
+         const body = (await res.json()) as { name?: string };
+         expect(body.name).toBe(PROJECT_NAME);
+      });
+   });
+
+   describe("packages", () => {
+      it("GET /projects/:projectName/packages returns the package list", async () => {
+         const res = await fetch(
+            `${baseUrl}/api/v0/projects/${PROJECT_NAME}/packages`,
+         );
+         expect(res.status).toBe(200);
+         const body = (await res.json()) as Array<{ name?: string }>;
+         expect(Array.isArray(body)).toBe(true);
+         expect(body.some((p) => p.name === PACKAGE_NAME)).toBe(true);
+      });
+
+      it("GET /projects/:projectName/packages/:packageName returns the package", async () => {
+         const res = await fetch(
+            `${baseUrl}/api/v0/projects/${PROJECT_NAME}/packages/${PACKAGE_NAME}`,
+         );
+         expect(res.status).toBe(200);
+         const body = (await res.json()) as { name?: string };
+         expect(body.name).toBe(PACKAGE_NAME);
+      });
+   });
+
+   describe("connections", () => {
+      it("GET /projects/:projectName/connections returns 200 (may be empty)", async () => {
+         const res = await fetch(
+            `${baseUrl}/api/v0/projects/${PROJECT_NAME}/connections`,
+         );
+         expect(res.status).toBe(200);
+         const body = await res.json();
+         expect(Array.isArray(body)).toBe(true);
+      });
+   });
+
+   describe("models", () => {
+      it("GET /projects/:projectName/packages/:packageName/models returns the model list", async () => {
+         const res = await fetch(
+            `${baseUrl}/api/v0/projects/${PROJECT_NAME}/packages/${PACKAGE_NAME}/models`,
+         );
+         expect(res.status).toBe(200);
+         const body = await res.json();
+         expect(Array.isArray(body)).toBe(true);
+      });
+   });
+
+   describe("notebooks", () => {
+      it("GET /projects/:projectName/packages/:packageName/notebooks returns 200", async () => {
+         const res = await fetch(
+            `${baseUrl}/api/v0/projects/${PROJECT_NAME}/packages/${PACKAGE_NAME}/notebooks`,
+         );
+         expect(res.status).toBe(200);
+         const body = await res.json();
+         expect(Array.isArray(body)).toBe(true);
+      });
+   });
+
+   describe("databases", () => {
+      it("GET /projects/:projectName/packages/:packageName/databases returns 200", async () => {
+         const res = await fetch(
+            `${baseUrl}/api/v0/projects/${PROJECT_NAME}/packages/${PACKAGE_NAME}/databases`,
+         );
+         expect(res.status).toBe(200);
+         const body = await res.json();
+         expect(Array.isArray(body)).toBe(true);
+      });
+   });
+
+   describe("queries", () => {
+      it("POST /projects/:projectName/packages/:packageName/models/.../query reaches the handler", async () => {
+         // Hit the route with a bogus model name. We only need to prove the
+         // legacy URL is wired up to the controller — a structured JSON
+         // error (not Express's HTML fall-through 404) is sufficient signal.
+         const res = await fetch(
+            `${baseUrl}/api/v0/projects/${PROJECT_NAME}/packages/${PACKAGE_NAME}/models/does-not-exist.malloy/query`,
+            {
+               method: "POST",
+               headers: { "Content-Type": "application/json" },
+               body: JSON.stringify({ query: "run: nothing" }),
+            },
+         );
+         expect(res.status).toBeGreaterThanOrEqual(400);
+         expect(res.status).toBeLessThan(600);
+         // Controller errors come back as JSON with a `message` field.
+         // An unhandled Express 404 returns HTML — that would fail here.
+         const body = (await res.json()) as Record<string, unknown>;
+         expect(typeof body.message).toBe("string");
+      });
+   });
+
+   describe("materializations", () => {
+      it("GET list and POST create return 'projectId' (not 'environmentId') under the legacy URL", async () => {
+         const listRes = await fetch(
+            `${baseUrl}/api/v0/projects/${PROJECT_NAME}/packages/${PACKAGE_NAME}/materializations`,
+         );
+         expect(listRes.status).toBe(200);
+         const list = (await listRes.json()) as unknown;
+         expect(Array.isArray(list)).toBe(true);
+
+         // Create one so we can assert the field rename on a populated payload.
+         const createRes = await fetch(
+            `${baseUrl}/api/v0/projects/${PROJECT_NAME}/packages/${PACKAGE_NAME}/materializations`,
+            {
+               method: "POST",
+               headers: { "Content-Type": "application/json" },
+               body: JSON.stringify({ autoLoadManifest: true }),
+            },
+         );
+         expect(createRes.status).toBe(201);
+         const created = (await createRes.json()) as Record<string, unknown>;
+
+         // Legacy contract: materialization payloads expose `projectId`, not
+         // `environmentId`. This is the response remapper in server-old.ts.
+         expect(created).toHaveProperty("projectId");
+         expect(created).not.toHaveProperty("environmentId");
+
+         const id = created.id as string;
+         // Best-effort cleanup so we don't leak a PENDING materialization
+         // into other tests. We don't poll-to-terminal; the suite teardown
+         // of the project will mop up.
+         try {
+            await fetch(
+               `${baseUrl}/api/v0/projects/${PROJECT_NAME}/packages/${PACKAGE_NAME}/materializations/${id}?action=stop`,
+               { method: "POST" },
+            );
+            await fetch(
+               `${baseUrl}/api/v0/projects/${PROJECT_NAME}/packages/${PACKAGE_NAME}/materializations/${id}`,
+               { method: "DELETE" },
+            );
+         } catch {
+            // ignore
+         }
+      });
+   });
+
+   describe("manifest", () => {
+      it("GET /projects/:projectName/packages/:packageName/manifest returns 200 or a structured 4xx", async () => {
+         const res = await fetch(
+            `${baseUrl}/api/v0/projects/${PROJECT_NAME}/packages/${PACKAGE_NAME}/manifest`,
+         );
+         // Without a built materialization the manifest may be empty/404 —
+         // we only assert the legacy URL reaches the handler, not 404 from
+         // Express's catch-all.
+         expect([200, 400, 404]).toContain(res.status);
+      });
+   });
+});

package/tests/unit/duckdb/attached_databases.test.ts

@@ -855,7 +855,7 @@ describe("createEnvironmentConnections - DuckDB", () => {
 
          await expect(
             createEnvironmentConnections(connections, PROJECT_TEST_DIR),
-         ).rejects.toThrow("DuckDB connection name cannot be 'duckdb'");
+         ).rejects.toThrow(/'duckdb' is reserved/);
       });
 
       it("should throw when DuckDB connection name is 'duckdb' with attached databases", async () => {
@@ -885,10 +885,12 @@ describe("createEnvironmentConnections - DuckDB", () => {
 
          await expect(
             createEnvironmentConnections(connections, PROJECT_TEST_DIR),
-         ).rejects.toThrow("DuckDB connection name cannot be 'duckdb'");
+         ).rejects.toThrow(/'duckdb' is reserved/);
       });
 
       it("should throw when DuckDB connection has no attached databases", async () => {
+         // Env-level DuckDB requires at least one attached foreign db;
+         // the per-package "duckdb" sandbox covers the plain-in-memory case.
          const connections: ApiConnection[] = [
             {
                name: "no_attached_db",
@@ -901,9 +903,7 @@ describe("createEnvironmentConnections - DuckDB", () => {
 
          await expect(
             createEnvironmentConnections(connections, PROJECT_TEST_DIR),
-         ).rejects.toThrow(
-            "DuckDB connection must have at least one attached database",
-         );
+         ).rejects.toThrow(/has no attached databases/);
       });
 
       it("should throw on unsupported connection type", async () => {