@malloy-publisher/server 0.0.235 → 0.0.236

package/tests/integration/concurrent_package/concurrent_package.integration.spec.ts

@@ -1,280 +0,0 @@
-/// <reference types="bun-types" />
-
-/**
- * Regression test for the per-package download/unzip race.
- *
- * Before fix: concurrent POST/PATCH/GET-with-reload against the same package
- * name landed in `downloadPackage` *before* acquiring the per-package mutex.
- * Multiple callers would `rm -rf <targetPath>` and then `cp -r` / `extractAllTo`
- * in parallel, leaving the directory in an inconsistent state. Subsequent
- * reads failed with "Package manifest for ... does not exist" and any
- * in-flight model compilation would 500 with "compiling model path not found".
- *
- * After fix: download is run inside the per-package mutex, so concurrent
- * callers serialize on the same target path. All N requests must succeed
- * and the package must be usable afterwards.
- */
-
-import { afterAll, beforeAll, describe, expect, it } from "bun:test";
-import path from "path";
-import { fileURLToPath } from "url";
-import { RestE2EEnv, startRestE2E } from "../../harness/rest_e2e";
-
-const __filename = fileURLToPath(import.meta.url);
-const __dirname = path.dirname(__filename);
-
-// Use a unique env name per run so stale state from a previous run (env
-// directory, persisted env metadata in publisher.db) can't poison startup.
-const ENV_NAME = `concurrent-package-test-env-${Date.now()}`;
-const PACKAGE_NAME = "gcs_faa";
-const CONCURRENCY = 12;
-
-const FORBIDDEN_ERROR_FRAGMENTS = [
-   "Package manifest for",
-   "does not exist",
-   "compiling model path not found",
-   "model path not found",
-];
-
-function findForbiddenError(body: unknown): string | undefined {
-   const text = typeof body === "string" ? body : JSON.stringify(body ?? "");
-   return FORBIDDEN_ERROR_FRAGMENTS.find((frag) => text.includes(frag));
-}
-
-describe("Concurrent package operations (E2E)", () => {
-   let env: (RestE2EEnv & { stop(): Promise<void> }) | null = null;
-   let baseUrl: string;
-   let fixtureDir: string;
-
-   beforeAll(async () => {
-      env = await startRestE2E();
-      baseUrl = env.baseUrl;
-      fixtureDir = "gs://publisher_test_packages/gcs_faa.zip";
-
-      // Seed the environment with the package once so the env exists on disk.
-      // addEnvironment requires at least one package.
-      const createRes = await fetch(`${baseUrl}/api/v0/environments`, {
-         method: "POST",
-         headers: { "Content-Type": "application/json" },
-         body: JSON.stringify({
-            name: ENV_NAME,
-            packages: [{ name: PACKAGE_NAME, location: fixtureDir }],
-            connections: [],
-         }),
-      });
-      if (!createRes.ok) {
-         const body = await createRes.text();
-         throw new Error(
-            `Failed to seed test environment (${createRes.status}): ${body}`,
-         );
-      }
-
-      // Wait for the seeded package to be loadable.
-      const deadline = Date.now() + 30_000;
-      let ready = false;
-      while (!ready && Date.now() < deadline) {
-         try {
-            const res = await fetch(
-               `${baseUrl}/api/v0/environments/${ENV_NAME}/packages/${PACKAGE_NAME}`,
-            );
-            if (res.ok) {
-               ready = true;
-               break;
-            }
-         } catch {
-            // not ready yet
-         }
-         await new Promise((r) => setTimeout(r, 250));
-      }
-      if (!ready) {
-         throw new Error("Seeded package did not become available in time");
-      }
-   });
-
-   afterAll(async () => {
-      if (baseUrl) {
-         try {
-            await fetch(`${baseUrl}/api/v0/environments/${ENV_NAME}`, {
-               method: "DELETE",
-            });
-         } catch {
-            // best-effort cleanup
-         }
-      }
-      await env?.stop();
-      env = null;
-   });
-
-   it("concurrent POST /packages for the same name all succeed", async () => {
-      const requests = Array.from({ length: CONCURRENCY }, () =>
-         fetch(`${baseUrl}/api/v0/environments/${ENV_NAME}/packages`, {
-            method: "POST",
-            headers: { "Content-Type": "application/json" },
-            body: JSON.stringify({
-               name: PACKAGE_NAME,
-               location: fixtureDir,
-            }),
-         }),
-      );
-
-      const responses = await Promise.all(requests);
-      const bodies = await Promise.all(
-         responses.map(async (r) => ({
-            status: r.status,
-            body: await r.json().catch(() => null),
-         })),
-      );
-
-      for (const { status, body } of bodies) {
-         expect(status).toBe(200);
-         const forbidden = findForbiddenError(body);
-         expect(forbidden).toBeUndefined();
-      }
-
-      // Package must be loadable after the storm.
-      const res = await fetch(
-         `${baseUrl}/api/v0/environments/${ENV_NAME}/packages/${PACKAGE_NAME}`,
-      );
-      expect(res.status).toBe(200);
-      const meta = (await res.json()) as { name?: string };
-      expect(meta.name).toBe(PACKAGE_NAME);
-
-      // Models must be listable — proves the model files survived the unzip
-      // race and Package.create read a consistent directory.
-      const modelsRes = await fetch(
-         `${baseUrl}/api/v0/environments/${ENV_NAME}/packages/${PACKAGE_NAME}/models`,
-      );
-      expect(modelsRes.status).toBe(200);
-      const models = (await modelsRes.json()) as Array<{ path?: string }>;
-      expect(Array.isArray(models)).toBe(true);
-      expect(models.length).toBeGreaterThan(0);
-   });
-
-   it("concurrent GET /packages/:name?reload=true all succeed", async () => {
-      const requests = Array.from({ length: CONCURRENCY }, () =>
-         fetch(
-            `${baseUrl}/api/v0/environments/${ENV_NAME}/packages/${PACKAGE_NAME}?reload=true`,
-         ),
-      );
-      const responses = await Promise.all(requests);
-      const bodies = await Promise.all(
-         responses.map(async (r) => ({
-            status: r.status,
-            body: await r.json().catch(() => null),
-         })),
-      );
-      for (const { status, body } of bodies) {
-         expect(status).toBe(200);
-         const forbidden = findForbiddenError(body);
-         expect(forbidden).toBeUndefined();
-      }
-
-      // Models must still list cleanly.
-      const modelsRes = await fetch(
-         `${baseUrl}/api/v0/environments/${ENV_NAME}/packages/${PACKAGE_NAME}/models`,
-      );
-      expect(modelsRes.status).toBe(200);
-   });
-
-   it("simultaneous POST + PATCH for the same package serialize cleanly", async () => {
-      // Fire create and update at the same time. Both should land under the
-      // same per-package mutex. Whichever wins the mutex first runs first;
-      // the loser sees a coherent post-condition. After both settle the
-      // package must be loadable and the description must reflect the PATCH
-      // when the PATCH ran *after* a successful POST.
-      const newDescription = `concurrent-update-${Date.now()}`;
-      const [postRes, patchRes] = await Promise.all([
-         fetch(`${baseUrl}/api/v0/environments/${ENV_NAME}/packages`, {
-            method: "POST",
-            headers: { "Content-Type": "application/json" },
-            body: JSON.stringify({
-               name: PACKAGE_NAME,
-               location: fixtureDir,
-            }),
-         }),
-         fetch(
-            `${baseUrl}/api/v0/environments/${ENV_NAME}/packages/${PACKAGE_NAME}`,
-            {
-               method: "PATCH",
-               headers: { "Content-Type": "application/json" },
-               body: JSON.stringify({
-                  name: PACKAGE_NAME,
-                  description: newDescription,
-               }),
-            },
-         ),
-      ]);
-
-      // POST is idempotent here — always succeeds.
-      expect(postRes.status).toBe(200);
-      const postBody = await postRes.json().catch(() => null);
-      expect(findForbiddenError(postBody)).toBeUndefined();
-
-      // PATCH either succeeds (ran after POST loaded the package) or 404s
-      // (ran before POST populated `this.packages`). It must NOT silently
-      // rewrite disk and then 404, and must never surface the forbidden
-      // error fragments.
-      const patchBody = await patchRes.json().catch(() => null);
-      expect(findForbiddenError(patchBody)).toBeUndefined();
-      expect([200, 404]).toContain(patchRes.status);
-
-      // Whatever happened, the package must remain loadable.
-      const getRes = await fetch(
-         `${baseUrl}/api/v0/environments/${ENV_NAME}/packages/${PACKAGE_NAME}`,
-      );
-      expect(getRes.status).toBe(200);
-      const meta = (await getRes.json()) as {
-         name?: string;
-         description?: string;
-      };
-      expect(meta.name).toBe(PACKAGE_NAME);
-   });
-
-   it("interleaved POST + GET-reload + model list never surface stale-dir errors", async () => {
-      // Worst-case interleave: writers and readers hammering the same
-      // package. None of them should observe a missing manifest or a
-      // half-extracted directory.
-      const work: Array<Promise<{ status: number; body: unknown }>> = [];
-      for (let i = 0; i < CONCURRENCY; i++) {
-         work.push(
-            fetch(`${baseUrl}/api/v0/environments/${ENV_NAME}/packages`, {
-               method: "POST",
-               headers: { "Content-Type": "application/json" },
-               body: JSON.stringify({
-                  name: PACKAGE_NAME,
-                  location: fixtureDir,
-               }),
-            }).then(async (r) => ({
-               status: r.status,
-               body: await r.json().catch(() => null),
-            })),
-         );
-         work.push(
-            fetch(
-               `${baseUrl}/api/v0/environments/${ENV_NAME}/packages/${PACKAGE_NAME}?reload=true`,
-            ).then(async (r) => ({
-               status: r.status,
-               body: await r.json().catch(() => null),
-            })),
-         );
-         work.push(
-            fetch(
-               `${baseUrl}/api/v0/environments/${ENV_NAME}/packages/${PACKAGE_NAME}/models`,
-            ).then(async (r) => ({
-               status: r.status,
-               body: await r.json().catch(() => null),
-            })),
-         );
-      }
-
-      const results = await Promise.all(work);
-      for (const { status, body } of results) {
-         const forbidden = findForbiddenError(body);
-         expect(forbidden).toBeUndefined();
-         // Model list while the package is being rewritten can transiently
-         // return 404 (the package is unloaded mid-rewrite), but never 5xx,
-         // and never with the forbidden error fragments.
-         expect(status).toBeLessThan(500);
-      }
-   });
-});

package/tests/integration/duckdb_storage/duckdb_storage.integration.spec.ts

@@ -1,138 +0,0 @@
-/// <reference types="bun-types" />
-
-// Integration test for the DuckDB storage stack end-to-end:
-//   StorageManager -> ResourceRepository -> DuckDBConnection (@duckdb/node-api)
-//   -> a real on-disk publisher.db
-//
-// Unlike the repository unit tests (which construct a single repo against a bare
-// connection) this drives the same path the server uses -- StorageManager.initialize()
-// wires the schema + repositories over the migrated connection, and getRepository()
-// is what the service layer calls. It also reopens a second StorageManager against
-// the same file to prove data persists across a restart, which is the real
-// end-to-end guarantee for the migration.
-
-import { afterEach, beforeEach, describe, expect, it } from "bun:test";
-import fs from "fs/promises";
-import os from "os";
-import path from "path";
-import { StorageManager } from "../../../src/storage/StorageManager";
-
-describe("DuckDB storage stack (integration)", () => {
-   let dbDir: string;
-   let dbPath: string;
-   let sm: StorageManager;
-
-   beforeEach(async () => {
-      dbDir = await fs.mkdtemp(path.join(os.tmpdir(), "duckdb-storage-int-"));
-      dbPath = path.join(dbDir, "publisher.db");
-      sm = new StorageManager({ type: "duckdb", duckdb: { path: dbPath } });
-      await sm.initialize();
-   });
-
-   afterEach(async () => {
-      try {
-         await sm.close();
-      } catch {
-         // ignore
-      }
-      await fs.rm(dbDir, { recursive: true, force: true });
-   });
-
-   it("initializes the schema and reports ready", () => {
-      expect(sm.isInitialized()).toBe(true);
-   });
-
-   it("creates an environment with a package and connection through the repository", async () => {
-      const repo = sm.getRepository();
-
-      const env = await repo.createEnvironment({
-         name: "prod",
-         path: "/srv/prod",
-         metadata: { owner: "platform" },
-      });
-      expect(env.id).toBeTruthy();
-
-      const pkg = await repo.createPackage({
-         environmentId: env.id,
-         name: "sales",
-         manifestPath: "/srv/prod/sales/publisher.json",
-      });
-
-      const conn = await repo.createConnection({
-         environmentId: env.id,
-         name: "warehouse",
-         type: "snowflake",
-         config: { account: "acct", warehouse: "wh" },
-      });
-
-      // Read everything back through the same repository.
-      expect((await repo.getEnvironmentByName("prod"))?.id).toBe(env.id);
-      expect((await repo.listPackages(env.id)).map((p) => p.name)).toEqual([
-         "sales",
-      ]);
-      const conns = await repo.listConnections(env.id);
-      expect(conns.map((c) => c.name)).toEqual(["warehouse"]);
-      expect(conns[0].config).toEqual({ account: "acct", warehouse: "wh" });
-      expect(pkg.environmentId).toBe(env.id);
-   });
-
-   it("persists data across a StorageManager restart (reopen the same file)", async () => {
-      const repo = sm.getRepository();
-      const env = await repo.createEnvironment({
-         name: "persisted",
-         path: "/srv/persisted",
-         metadata: { keep: true },
-      });
-      await repo.createConnection({
-         environmentId: env.id,
-         name: "pg",
-         type: "postgres",
-         config: { host: "db.internal", port: 5432 },
-      });
-
-      // Close the manager (releases the file), then open a fresh one on the
-      // same path -- the data must still be there.
-      await sm.close();
-
-      const reopened = new StorageManager({
-         type: "duckdb",
-         duckdb: { path: dbPath },
-      });
-      await reopened.initialize();
-      try {
-         const repo2 = reopened.getRepository();
-         const env2 = await repo2.getEnvironmentByName("persisted");
-         expect(env2).not.toBeNull();
-         expect(env2!.metadata).toEqual({ keep: true });
-         expect(env2!.createdAt).toBeInstanceOf(Date);
-
-         const conns = await repo2.listConnections(env2!.id);
-         expect(conns.map((c) => c.name)).toEqual(["pg"]);
-         expect(conns[0].config).toEqual({ host: "db.internal", port: 5432 });
-      } finally {
-         await reopened.close();
-      }
-
-      // Point afterEach's close() at the already-closed original without error.
-      sm = reopened;
-   });
-
-   it("deleteEnvironment also removes the environment's connections", async () => {
-      const repo = sm.getRepository();
-      const env = await repo.createEnvironment({
-         name: "to-delete",
-         path: "/srv/del",
-      });
-      await repo.createConnection({
-         environmentId: env.id,
-         name: "c1",
-         type: "duckdb",
-         config: {},
-      });
-
-      await repo.deleteEnvironment(env.id);
-
-      expect(await repo.getEnvironmentById(env.id)).toBeNull();
-      expect(await repo.listConnections(env.id)).toEqual([]);
-   });
-});

package/tests/integration/first_boot/readiness_line.integration.spec.ts

@@ -1,177 +0,0 @@
-/// <reference types="bun-types" />
-
-import { afterAll, beforeAll, describe, expect, it } from "bun:test";
-import { type ChildProcess, spawn } from "child_process";
-import fs from "fs";
-import net from "net";
-import os from "os";
-import path from "path";
-import { fileURLToPath } from "url";
-
-/**
- * End-to-end coverage for the PUBLISHER_READY line: the contract scripts grep
- * for instead of polling /api/v0/status. It must appear on stderr exactly
- * once, only when the server is actually serving, and carry the environment,
- * package, and load-error counts. That contract spans process boundaries
- * (a real stderr stream from a real boot), so it needs a dedicated server
- * subprocess rather than the in-process REST harness.
- */
-
-const __dirname = path.dirname(fileURLToPath(import.meta.url));
-// tests/integration/first_boot -> packages/server
-const SERVER_DIR = path.resolve(__dirname, "../../..");
-
-const ENV_NAME = "ready-env";
-const PKG_NAME = "ready-pkg";
-
-/** Allocate an OS-assigned free TCP port (avoids fixed-port collisions). */
-async function getFreePort(): Promise<number> {
-   return new Promise<number>((resolve, reject) => {
-      const srv = net.createServer();
-      srv.on("error", reject);
-      srv.listen(0, "127.0.0.1", () => {
-         const addr = srv.address();
-         const found = typeof addr === "object" && addr ? addr.port : 0;
-         srv.close(() =>
-            found ? resolve(found) : reject(new Error("no free port")),
-         );
-      });
-   });
-}
-
-/** Returns true once `predicate` does, or false if `timeoutMs` elapses first. */
-async function poll(
-   predicate: () => Promise<boolean>,
-   timeoutMs: number,
-   intervalMs = 300,
-): Promise<boolean> {
-   const deadline = Date.now() + timeoutMs;
-   while (Date.now() < deadline) {
-      if (await predicate()) return true;
-      await new Promise((r) => setTimeout(r, intervalMs));
-   }
-   return false;
-}
-
-describe("first-boot readiness line", () => {
-   let proc: ChildProcess | undefined;
-   let exited = false;
-   let stderrLog = "";
-   let combinedLog = "";
-   let srcDir = "";
-   let serverRoot = "";
-   let port = 0;
-
-   const readyLines = () =>
-      stderrLog
-         .split("\n")
-         .filter((line) => line.startsWith("PUBLISHER_READY"));
-
-   beforeAll(async () => {
-      srcDir = fs.mkdtempSync(path.join(os.tmpdir(), "ready-src-"));
-      fs.writeFileSync(
-         path.join(srcDir, "publisher.json"),
-         JSON.stringify({ name: PKG_NAME }),
-      );
-      fs.writeFileSync(
-         path.join(srcDir, "first.malloy"),
-         'source: first_source is duckdb.sql("SELECT 1 as n")\n',
-      );
-
-      serverRoot = fs.mkdtempSync(path.join(os.tmpdir(), "ready-root-"));
-      fs.writeFileSync(
-         path.join(serverRoot, "publisher.config.json"),
-         JSON.stringify({
-            frozenConfig: false,
-            environments: [
-               {
-                  name: ENV_NAME,
-                  packages: [{ name: PKG_NAME, location: srcDir }],
-                  connections: [],
-               },
-            ],
-         }),
-      );
-
-      port = await getFreePort();
-      let mcpPort = await getFreePort();
-      while (mcpPort === port) mcpPort = await getFreePort();
-
-      proc = spawn("bun", ["src/server.ts"], {
-         cwd: SERVER_DIR,
-         env: {
-            ...process.env,
-            SERVER_ROOT: serverRoot,
-            PUBLISHER_HOST: "127.0.0.1",
-            PUBLISHER_PORT: String(port),
-            MCP_PORT: String(mcpPort),
-         },
-         stdio: ["ignore", "pipe", "pipe"],
-      });
-      proc.stdout?.on("data", (d: Buffer) => {
-         combinedLog = (combinedLog + d.toString()).slice(-8000);
-      });
-      proc.stderr?.on("data", (d: Buffer) => {
-         stderrLog += d.toString();
-         combinedLog = (combinedLog + d.toString()).slice(-8000);
-      });
-      proc.on("exit", () => {
-         exited = true;
-      });
-
-      const serving = await poll(async () => {
-         if (exited) throw new Error("server exited before becoming ready");
-         try {
-            const res = await fetch(`http://127.0.0.1:${port}/api/v0/status`);
-            if (!res.ok) return false;
-            const body = (await res.json()) as { operationalState?: string };
-            return body.operationalState === "serving";
-         } catch {
-            return false;
-         }
-      }, 120_000);
-      if (!serving) {
-         throw new Error(
-            `server did not reach serving within 120s\n--- log tail ---\n${combinedLog}`,
-         );
-      }
-      // The line is written in the same tick that flips the status to
-      // serving, but the pipe delivery can trail the HTTP response.
-      await poll(async () => readyLines().length > 0, 10_000, 100);
-   }, 150_000);
-
-   afterAll(async () => {
-      if (proc && !exited) {
-         await new Promise<void>((resolve) => {
-            // Backstop in case SIGTERM is ignored; cleared once the process exits.
-            const backstop = setTimeout(() => proc?.kill("SIGKILL"), 5_000);
-            proc?.on("exit", () => {
-               clearTimeout(backstop);
-               resolve();
-            });
-            proc?.kill("SIGTERM");
-         });
-      }
-      for (const dir of [srcDir, serverRoot]) {
-         try {
-            fs.rmSync(dir, { recursive: true, force: true });
-         } catch {
-            // best-effort cleanup
-         }
-      }
-   });
-
-   it("prints one PUBLISHER_READY line on stderr once serving", () => {
-      const lines = readyLines();
-      expect(lines).toHaveLength(1);
-      // The URL host/port are the ones the server was actually started with,
-      // so a script can dial what the line says.
-      expect(lines[0]).toMatch(
-         new RegExp(
-            `^PUBLISHER_READY url=http://127\\.0\\.0\\.1:${port} ` +
-               `mcp=http://127\\.0\\.0\\.1:\\d+ ` +
-               `environments=1 packages=1 load_errors=0$`,
-         ),
-      );
-   });
-});