@mycadet/assistant 0.12.10 → 0.13.0

package/src/__tests__/migration-export-to-gcs.test.ts

@@ -1,706 +0,0 @@
-/**
- * Integration tests for POST /v1/migrations/export-to-gcs.
- *
- * Covered:
- * - Happy path: the handler returns 202 + job_id, the background job
- *   PUTs a valid .vbundle to the fixture server, and the final job
- *   record carries a matching result payload.
- * - Concurrency: a second export-to-gcs while the first is still
- *   pending/running returns 409 `export_in_progress`.
- * - Upload failure: when the PUT server responds 500, the job ends
- *   `failed` with `error.code === "upload_failed"` and
- *   `upstreamStatus: 500`.
- * - Invalid URL: non-https, wrong host, and path-traversal URLs are
- *   rejected at the handler (400) with the validator's reason code
- *   in `error.reason`.
- *
- * The .vbundle bytes observed by the fixture server are cross-checked
- * against `validateVBundle` to prove the upload body matches what
- * `streamExportVBundle()` would produce.
- */
-
-import { mkdirSync, writeFileSync } from "node:fs";
-import { createServer, type Server } from "node:http";
-import { join } from "node:path";
-import {
-  afterAll,
-  afterEach,
-  beforeAll,
-  describe,
-  expect,
-  mock,
-  test,
-} from "bun:test";
-
-const testDir = process.env.CADET_WORKSPACE_DIR!;
-const testDbDir = join(testDir, "data", "db");
-const testDbPath = join(testDbDir, "assistant.db");
-const testConfigPath = join(testDir, "config.json");
-
-// The gateway's debug export arrives over IPC as a base64 tar.gz. Stand in
-// for the socket so the debug profile can be exercised without a gateway.
-const FAKE_GATEWAY_ARCHIVE = Buffer.from("not really a tar.gz");
-const gatewayDebugExportMock = mock(async () => ({
-  ok: true as const,
-  archive_base64: FAKE_GATEWAY_ARCHIVE.toString("base64"),
-  size_bytes: FAKE_GATEWAY_ARCHIVE.length,
-}));
-mock.module("../ipc/gateway-client.js", () => ({
-  ipcCallPersistent: (method: string) => {
-    if (method !== "gateway_debug_export") {
-      throw new Error(`unexpected IPC method ${method}`);
-    }
-    return gatewayDebugExportMock();
-  },
-}));
-
-// Force the credential collector onto the unreachable branch so the
-// handler treats it as "export without credentials" (fast, no CES).
-// The REAL secure-keys module pinned to a faultable memory backend with the
-// whole vault faulted: the handler treats it as "export without credentials"
-// (fast, no CES).
-import { useFaultableSecureKeyMap } from "./helpers/secure-keys.js";
-
-const { setAllUnreachable } = useFaultableSecureKeyMap();
-setAllUnreachable(true);
-
-// ---------------------------------------------------------------------------
-// Imports (after mocks so module-level code picks up the stubs)
-// ---------------------------------------------------------------------------
-
-import { migrationJobs } from "../runtime/migrations/job-registry.js";
-import { validateVBundle } from "../runtime/migrations/vbundle-validator.js";
-import {
-  _setUrlImportValidatorOptionsForTests,
-  handleMigrationExportToGcs,
-} from "../runtime/routes/migration-routes.js";
-import { callHandler } from "./helpers/call-route-handler.js";
-
-// ---------------------------------------------------------------------------
-// Fixture workspace: write a minimal SQLite file + config so the exporter
-// has real data to walk.
-// ---------------------------------------------------------------------------
-
-const SQLITE_HEADER = new Uint8Array([
-  0x53, 0x51, 0x4c, 0x69, 0x74, 0x65, 0x20, 0x66, 0x6f, 0x72, 0x6d, 0x61, 0x74,
-  0x20, 0x33, 0x00,
-]);
-const TEST_CONFIG = { provider: "anthropic", model: "test-model" };
-
-beforeAll(() => {
-  mkdirSync(testDbDir, { recursive: true });
-  writeFileSync(testDbPath, SQLITE_HEADER);
-  writeFileSync(testConfigPath, JSON.stringify(TEST_CONFIG, null, 2));
-
-  // Widen the URL validator allowlist so the handler accepts a URL
-  // pointing at the local fixture server.
-  _setUrlImportValidatorOptionsForTests({
-    allowedHosts: ["127.0.0.1", "storage.googleapis.com"],
-  });
-});
-
-afterAll(() => {
-  _setUrlImportValidatorOptionsForTests(undefined);
-});
-
-// Drop stale job records between tests so concurrency assertions are
-// deterministic (a `failed` job from a previous test must not collide
-// with the next test's `startJob("export", …)` call — the registry only
-// rejects while the prior job is still pending/running, but evicting the
-// entry defensively keeps log output clean too).
-afterEach(async () => {
-  // Wait a tick so any in-flight runners can finalize their state.
-  await new Promise((resolve) => setTimeout(resolve, 0));
-  // Force-evict any remaining terminal records by tightening the TTL.
-  const savedTtl = migrationJobs.completedJobTtlMs;
-  migrationJobs.completedJobTtlMs = 0;
-  migrationJobs.sweep();
-  migrationJobs.completedJobTtlMs = savedTtl;
-});
-
-// ---------------------------------------------------------------------------
-// Local http fixture server that captures PUT bodies
-// ---------------------------------------------------------------------------
-
-interface FixtureServer {
-  server: Server;
-  port: number;
-  close: () => Promise<void>;
-}
-
-async function startFixtureServer(
-  handler: (
-    req: import("node:http").IncomingMessage,
-    res: import("node:http").ServerResponse,
-  ) => void,
-): Promise<FixtureServer> {
-  const server = createServer(handler);
-  await new Promise<void>((resolve) => server.listen(0, "127.0.0.1", resolve));
-  const address = server.address();
-  if (!address || typeof address === "string") {
-    throw new Error("fixture server bound to unexpected address");
-  }
-  return {
-    server,
-    port: address.port,
-    close: async () => {
-      await new Promise<void>((resolve) => server.close(() => resolve()));
-    },
-  };
-}
-
-function makeFakeSignedUploadUrl(port: number): string {
-  // `validateGcsSignedUrl` requires a signature query param; pin a dummy.
-  return `http://127.0.0.1:${port}/upload/bundle?X-Goog-Signature=fake`;
-}
-
-/** Read the full body of an incoming request into a Buffer. */
-function collectBody(
-  req: import("node:http").IncomingMessage,
-): Promise<Buffer> {
-  return new Promise((resolve, reject) => {
-    const chunks: Buffer[] = [];
-    req.on("data", (chunk: Buffer) => chunks.push(chunk));
-    req.on("end", () => resolve(Buffer.concat(chunks)));
-    req.on("error", reject);
-  });
-}
-
-/** Poll `migrationJobs.getJob(id)` until the job reaches a terminal state. */
-async function waitForJobTerminal(
-  id: string,
-  { timeoutMs = 10_000 }: { timeoutMs?: number } = {},
-): Promise<NonNullable<ReturnType<typeof migrationJobs.getJob>>> {
-  const deadline = Date.now() + timeoutMs;
-  while (Date.now() < deadline) {
-    const job = migrationJobs.getJob(id);
-    if (job && (job.status === "complete" || job.status === "failed")) {
-      return job;
-    }
-    await new Promise((resolve) => setTimeout(resolve, 10));
-  }
-  throw new Error(`Job ${id} did not reach a terminal state in time`);
-}
-
-// ---------------------------------------------------------------------------
-// Response shape types
-// ---------------------------------------------------------------------------
-
-interface AcceptedResponse {
-  job_id: string;
-  status: "pending";
-  type: "export";
-}
-
-interface ErrorEnvelope {
-  error: {
-    code: string;
-    reason?: string;
-    job_id?: string;
-    message?: string;
-  };
-}
-
-// ---------------------------------------------------------------------------
-// Tests
-// ---------------------------------------------------------------------------
-
-describe("handleMigrationExportToGcs — happy path", () => {
-  test("202 + job completes with a valid .vbundle PUT to the upload URL", async () => {
-    let capturedBody: Buffer | undefined;
-    let capturedContentType: string | undefined;
-
-    const fixture = await startFixtureServer(async (req, res) => {
-      if (req.method !== "PUT") {
-        res.writeHead(405);
-        res.end();
-        return;
-      }
-      capturedContentType = req.headers["content-type"];
-      capturedBody = await collectBody(req);
-      res.writeHead(200);
-      res.end();
-    });
-
-    try {
-      const req = new Request("http://localhost/v1/migrations/export-to-gcs", {
-        method: "POST",
-        headers: { "Content-Type": "application/json" },
-        body: JSON.stringify({
-          upload_url: makeFakeSignedUploadUrl(fixture.port),
-          description: "happy-path export",
-        }),
-      });
-
-      const res = await callHandler(
-        handleMigrationExportToGcs,
-        req,
-        undefined,
-        202,
-      );
-      expect(res.status).toBe(202);
-
-      const body = (await res.json()) as AcceptedResponse;
-      expect(body.status).toBe("pending");
-      expect(body.type).toBe("export");
-      expect(body.job_id).toBeDefined();
-      expect(typeof body.job_id).toBe("string");
-
-      const terminal = await waitForJobTerminal(body.job_id);
-      expect(terminal.status).toBe("complete");
-      expect(terminal.error).toBeUndefined();
-
-      const result = terminal.result as {
-        size: number;
-        sha256: string;
-        schemaVersion: string;
-        credentialsIncluded: number;
-      };
-      expect(result.size).toBeGreaterThan(0);
-      expect(typeof result.sha256).toBe("string");
-      expect(result.sha256.length).toBeGreaterThan(0);
-      expect(result.schemaVersion).toBeDefined();
-      expect(result.credentialsIncluded).toBe(0);
-
-      expect(capturedContentType).toBe("application/octet-stream");
-      expect(capturedBody).toBeDefined();
-      expect(capturedBody!.byteLength).toBe(result.size);
-
-      // The uploaded body is a real .vbundle — validate it.
-      const validation = validateVBundle(new Uint8Array(capturedBody!));
-      expect(validation.is_valid).toBe(true);
-      expect(validation.errors).toHaveLength(0);
-      expect(validation.manifest?.checksum).toBe(result.sha256);
-    } finally {
-      await fixture.close();
-    }
-  }, 15_000);
-
-  test("managed origin skips credential collection and redacts secrets", async () => {
-    // The secure-keys mock reports the store unreachable, which normally
-    // forces `secrets_redacted: false` (we can't prove the bundle is clean).
-    // In managed mode the handler must skip collection entirely — the
-    // manifest schema refuses managed bundles carrying secrets — so the
-    // manifest claims a truthful `secrets_redacted: true` regardless of
-    // credential-store state, and the bundle passes validation with
-    // `origin.mode: "managed"`.
-    const originalIsPlatform = process.env.IS_CLOUD;
-    process.env.IS_CLOUD = "true";
-
-    let capturedBody: Buffer | undefined;
-    const fixture = await startFixtureServer(async (req, res) => {
-      if (req.method !== "PUT") {
-        res.writeHead(405);
-        res.end();
-        return;
-      }
-      capturedBody = await collectBody(req);
-      res.writeHead(200);
-      res.end();
-    });
-
-    try {
-      const req = new Request("http://localhost/v1/migrations/export-to-gcs", {
-        method: "POST",
-        headers: { "Content-Type": "application/json" },
-        body: JSON.stringify({
-          upload_url: makeFakeSignedUploadUrl(fixture.port),
-          description: "managed export",
-        }),
-      });
-
-      const res = await callHandler(
-        handleMigrationExportToGcs,
-        req,
-        undefined,
-        202,
-      );
-      const body = (await res.json()) as AcceptedResponse;
-      const terminal = await waitForJobTerminal(body.job_id);
-      expect(terminal.status).toBe("complete");
-
-      const result = terminal.result as { credentialsIncluded: number };
-      expect(result.credentialsIncluded).toBe(0);
-
-      const validation = validateVBundle(new Uint8Array(capturedBody!));
-      expect(validation.is_valid).toBe(true);
-      expect(validation.manifest?.origin.mode).toBe("managed");
-      expect(validation.manifest?.secrets_redacted).toBe(true);
-    } finally {
-      if (originalIsPlatform === undefined) {
-        delete process.env.IS_CLOUD;
-      } else {
-        process.env.IS_CLOUD = originalIsPlatform;
-      }
-      await fixture.close();
-    }
-  }, 15_000);
-});
-
-describe("handleMigrationExportToGcs — debug profile", () => {
-  test("adds the gateway's archive under gateway/ and marks the manifest", async () => {
-    gatewayDebugExportMock.mockClear();
-    let capturedBody: Buffer | undefined;
-    const fixture = await startFixtureServer(async (req, res) => {
-      capturedBody = await collectBody(req);
-      res.writeHead(200);
-      res.end();
-    });
-
-    try {
-      const req = new Request("http://localhost/v1/migrations/export-to-gcs", {
-        method: "POST",
-        headers: { "Content-Type": "application/json" },
-        body: JSON.stringify({
-          upload_url: makeFakeSignedUploadUrl(fixture.port),
-          profile: "debug",
-        }),
-      });
-      const res = await callHandler(
-        handleMigrationExportToGcs,
-        req,
-        undefined,
-        202,
-      );
-      const body = (await res.json()) as AcceptedResponse;
-      const terminal = await waitForJobTerminal(body.job_id);
-      expect(terminal.status).toBe("complete");
-
-      expect(gatewayDebugExportMock).toHaveBeenCalledTimes(1);
-      const validation = validateVBundle(new Uint8Array(capturedBody!));
-      expect(validation.is_valid).toBe(true);
-      const manifest = validation.manifest!;
-      expect(manifest.export_options.include_gateway).toBe(true);
-      const gatewayEntry = manifest.contents.find(
-        (f) => f.path === "gateway/export.tar.gz",
-      );
-      expect(gatewayEntry?.size_bytes).toBe(FAKE_GATEWAY_ARCHIVE.length);
-      // Staff never receive credentials. The store mock reports itself
-      // unreachable, which would force `secrets_redacted: false` had the
-      // handler tried to collect them.
-      expect(manifest.secrets_redacted).toBe(true);
-      expect(
-        manifest.contents.some((f) => f.path.startsWith("credentials/")),
-      ).toBe(false);
-    } finally {
-      await fixture.close();
-    }
-  }, 15_000);
-
-  test("the default profile never asks the gateway", async () => {
-    gatewayDebugExportMock.mockClear();
-    let capturedBody: Buffer | undefined;
-    const fixture = await startFixtureServer(async (req, res) => {
-      capturedBody = await collectBody(req);
-      res.writeHead(200);
-      res.end();
-    });
-
-    try {
-      const req = new Request("http://localhost/v1/migrations/export-to-gcs", {
-        method: "POST",
-        headers: { "Content-Type": "application/json" },
-        body: JSON.stringify({
-          upload_url: makeFakeSignedUploadUrl(fixture.port),
-        }),
-      });
-      const res = await callHandler(
-        handleMigrationExportToGcs,
-        req,
-        undefined,
-        202,
-      );
-      const body = (await res.json()) as AcceptedResponse;
-      const terminal = await waitForJobTerminal(body.job_id);
-      expect(terminal.status).toBe("complete");
-
-      expect(gatewayDebugExportMock).not.toHaveBeenCalled();
-      const manifest = validateVBundle(new Uint8Array(capturedBody!)).manifest!;
-      expect(manifest.export_options.include_gateway).toBeUndefined();
-      expect(manifest.contents.some((f) => f.path.startsWith("gateway/"))).toBe(
-        false,
-      );
-    } finally {
-      await fixture.close();
-    }
-  }, 15_000);
-
-  test("a debug bundle is not sent at all if the gateway cannot export", async () => {
-    gatewayDebugExportMock.mockImplementationOnce(async () => {
-      throw new Error("gateway socket missing");
-    });
-    let putCount = 0;
-    const fixture = await startFixtureServer(async (_req, res) => {
-      putCount += 1;
-      res.writeHead(200);
-      res.end();
-    });
-
-    try {
-      const req = new Request("http://localhost/v1/migrations/export-to-gcs", {
-        method: "POST",
-        headers: { "Content-Type": "application/json" },
-        body: JSON.stringify({
-          upload_url: makeFakeSignedUploadUrl(fixture.port),
-          profile: "debug",
-        }),
-      });
-      const res = await callHandler(
-        handleMigrationExportToGcs,
-        req,
-        undefined,
-        202,
-      );
-      const body = (await res.json()) as { job_id: string };
-      const terminal = await waitForJobTerminal(body.job_id);
-      expect(terminal.status).toBe("failed");
-      expect(terminal.error?.code).toBe("gateway_debug_export_failed");
-      expect(putCount).toBe(0);
-    } finally {
-      await fixture.close();
-    }
-  }, 15_000);
-});
-
-describe("handleMigrationExportToGcs — concurrency", () => {
-  test("second export while first is running returns 409 export_in_progress", async () => {
-    // First fixture: deliberately slow so the first job stays pending long
-    // enough for the second request to collide with it.
-    let release: () => void = () => {};
-    const released = new Promise<void>((resolve) => {
-      release = resolve;
-    });
-
-    const slow = await startFixtureServer(async (req, res) => {
-      if (req.method !== "PUT") {
-        res.writeHead(405);
-        res.end();
-        return;
-      }
-      // Consume the body but defer the response until the test says so.
-      await collectBody(req);
-      await released;
-      res.writeHead(200);
-      res.end();
-    });
-
-    // Second fixture: never hit in this test — but we still need a valid
-    // URL shape so the handler gets past URL validation on the second
-    // call. We point the second request at the same fixture URL; the
-    // handler rejects before the URL is fetched.
-    try {
-      const firstReq = new Request(
-        "http://localhost/v1/migrations/export-to-gcs",
-        {
-          method: "POST",
-          headers: { "Content-Type": "application/json" },
-          body: JSON.stringify({
-            upload_url: makeFakeSignedUploadUrl(slow.port),
-          }),
-        },
-      );
-      const firstRes = await callHandler(
-        handleMigrationExportToGcs,
-        firstReq,
-        undefined,
-        202,
-      );
-      expect(firstRes.status).toBe(202);
-      const firstBody = (await firstRes.json()) as AcceptedResponse;
-
-      // Yield to the microtask queue so the first job's runner actually
-      // transitions from pending → running before we try to start a
-      // second one.
-      await new Promise((resolve) => setTimeout(resolve, 50));
-
-      const secondReq = new Request(
-        "http://localhost/v1/migrations/export-to-gcs",
-        {
-          method: "POST",
-          headers: { "Content-Type": "application/json" },
-          body: JSON.stringify({
-            upload_url: makeFakeSignedUploadUrl(slow.port),
-          }),
-        },
-      );
-      const secondRes = await callHandler(
-        handleMigrationExportToGcs,
-        secondReq,
-        undefined,
-        202,
-      );
-      expect(secondRes.status).toBe(409);
-      const secondBody = (await secondRes.json()) as ErrorEnvelope;
-      expect(secondBody.error.code).toBe("export_in_progress");
-      expect(secondBody.error.message).toContain(firstBody.job_id);
-
-      // Unblock the first fixture so the first job can finalize and the
-      // afterEach cleanup sweeps it out.
-      release();
-      await waitForJobTerminal(firstBody.job_id);
-    } finally {
-      release();
-      await slow.close();
-    }
-  }, 20_000);
-});
-
-describe("handleMigrationExportToGcs — upload failure", () => {
-  test("PUT server responds 500 → job failed with upload_failed + upstreamStatus", async () => {
-    const fixture = await startFixtureServer(async (req, res) => {
-      // Drain the body so the client side completes its write before we
-      // send the 500 response. Without this, fetch may surface the 500 as
-      // a transport-level error instead of a non-2xx response.
-      await collectBody(req);
-      res.writeHead(500);
-      res.end("upload rejected");
-    });
-
-    try {
-      const req = new Request("http://localhost/v1/migrations/export-to-gcs", {
-        method: "POST",
-        headers: { "Content-Type": "application/json" },
-        body: JSON.stringify({
-          upload_url: makeFakeSignedUploadUrl(fixture.port),
-        }),
-      });
-
-      const res = await callHandler(
-        handleMigrationExportToGcs,
-        req,
-        undefined,
-        202,
-      );
-      expect(res.status).toBe(202);
-      const body = (await res.json()) as AcceptedResponse;
-
-      const terminal = await waitForJobTerminal(body.job_id);
-      expect(terminal.status).toBe("failed");
-      expect(terminal.error).toBeDefined();
-      expect(terminal.error?.code).toBe("upload_failed");
-      expect(terminal.error?.upstreamStatus).toBe(500);
-    } finally {
-      await fixture.close();
-    }
-  }, 15_000);
-});
-
-describe("handleMigrationExportToGcs — redirect handling", () => {
-  test("PUT server responds with a 302 redirect → job failed (no SSRF follow)", async () => {
-    // If the upstream "storage.googleapis.com" responded with a 3xx to an
-    // attacker-controlled host, default fetch would follow and PUT bytes
-    // there. `redirect: "error"` on the fetch must surface the redirect
-    // as a transport failure instead — the job must end `failed` rather
-    // than `complete`, and no bytes must reach the redirect target.
-    let redirectTargetHit = false;
-    const redirectTarget = await startFixtureServer(async (_req, res) => {
-      redirectTargetHit = true;
-      res.writeHead(200);
-      res.end();
-    });
-    const fixture = await startFixtureServer(async (_req, res) => {
-      res.writeHead(302, {
-        Location: `http://127.0.0.1:${redirectTarget.port}/attacker`,
-      });
-      res.end();
-    });
-
-    try {
-      const req = new Request("http://localhost/v1/migrations/export-to-gcs", {
-        method: "POST",
-        headers: { "Content-Type": "application/json" },
-        body: JSON.stringify({
-          upload_url: makeFakeSignedUploadUrl(fixture.port),
-        }),
-      });
-
-      const res = await callHandler(
-        handleMigrationExportToGcs,
-        req,
-        undefined,
-        202,
-      );
-      expect(res.status).toBe(202);
-      const body = (await res.json()) as AcceptedResponse;
-
-      const terminal = await waitForJobTerminal(body.job_id);
-      expect(terminal.status).toBe("failed");
-      expect(redirectTargetHit).toBe(false);
-    } finally {
-      await fixture.close();
-      await redirectTarget.close();
-    }
-  }, 15_000);
-});
-
-describe("handleMigrationExportToGcs — URL validation", () => {
-  test("rejects non-https scheme when allowlist is default (400)", async () => {
-    // Temporarily reset the validator to production defaults so this
-    // assertion exercises the strict scheme check.
-    _setUrlImportValidatorOptionsForTests(undefined);
-    try {
-      const req = new Request("http://localhost/v1/migrations/export-to-gcs", {
-        method: "POST",
-        headers: { "Content-Type": "application/json" },
-        body: JSON.stringify({
-          upload_url: "http://storage.googleapis.com/b/o?X-Goog-Signature=fake",
-        }),
-      });
-      const res = await callHandler(
-        handleMigrationExportToGcs,
-        req,
-        undefined,
-        202,
-      );
-      expect(res.status).toBe(400);
-      const body = (await res.json()) as ErrorEnvelope;
-      expect(body.error.code).toBe("invalid_upload_url");
-      expect(body.error.message).toContain("scheme");
-    } finally {
-      _setUrlImportValidatorOptionsForTests({
-        allowedHosts: ["127.0.0.1", "storage.googleapis.com"],
-      });
-    }
-  });
-
-  test("rejects wrong host (400)", async () => {
-    const req = new Request("http://localhost/v1/migrations/export-to-gcs", {
-      method: "POST",
-      headers: { "Content-Type": "application/json" },
-      body: JSON.stringify({
-        upload_url: "https://evil.example.com/bucket/obj?X-Goog-Signature=fake",
-      }),
-    });
-    const res = await callHandler(
-      handleMigrationExportToGcs,
-      req,
-      undefined,
-      202,
-    );
-    expect(res.status).toBe(400);
-    const body = (await res.json()) as ErrorEnvelope;
-    expect(body.error.code).toBe("invalid_upload_url");
-    expect(body.error.message).toContain("host");
-  });
-
-  test("rejects path traversal (400)", async () => {
-    const req = new Request("http://localhost/v1/migrations/export-to-gcs", {
-      method: "POST",
-      headers: { "Content-Type": "application/json" },
-      body: JSON.stringify({
-        upload_url:
-          "https://storage.googleapis.com/bucket/..%2Fother?X-Goog-Signature=fake",
-      }),
-    });
-    const res = await callHandler(
-      handleMigrationExportToGcs,
-      req,
-      undefined,
-      202,
-    );
-    expect(res.status).toBe(400);
-    const body = (await res.json()) as ErrorEnvelope;
-    expect(body.error.code).toBe("invalid_upload_url");
-    expect(body.error.message).toContain("path_traversal");
-  });
-});