@desplega.ai/agent-swarm 2.7.4 → 2.7.6

package/src/tests/runtime-providers-docker.test.ts

@@ -61,6 +61,259 @@ afterEach(async () => {
 });
 
 describe("Docker runtime provider", () => {
+  for (const failureAt of ["create", "start"] as const) {
+    test(`cache mount failure during container ${failureAt} retries once without cache`, async () => {
+      const bodies: Array<{ HostConfig: { Mounts?: unknown }; Env: string[] }> = [];
+      const secret = "mount-failure-private-token";
+      handler = (url, init) => {
+        if (url.pathname.endsWith("/volumes/create"))
+          return Response.json(JSON.parse(String(init.body)));
+        if (url.pathname.endsWith("/containers/create")) {
+          bodies.push(JSON.parse(String(init.body)));
+          if (bodies.length === 1 && failureAt === "create")
+            return Response.json({ message: `invalid mount config ${secret}` }, { status: 500 });
+          return Response.json({ Id: `container-${bodies.length}` });
+        }
+        if (url.pathname.endsWith("/container-1/start") && failureAt === "start")
+          return Response.json(
+            { message: `error while mounting volume ${secret}` },
+            { status: 500 },
+          );
+        return new Response(null, { status: 204 });
+      };
+      const result = await provider.provision(target, {
+        ...request,
+        cache: { key: "swarm-cache-pool-1", mountPath: "/cache" },
+        env: { API_KEY: secret },
+      });
+      expect(result.externalId).toBe("container-2");
+      expect(result.meta?.cacheWarning).toContain("mount");
+      expect(result.meta?.cacheWarning).not.toContain(secret);
+      expect(bodies).toHaveLength(2);
+      expect(bodies[0]?.HostConfig.Mounts).toBeDefined();
+      expect(bodies[1]?.HostConfig.Mounts).toBeUndefined();
+      expect(bodies[1]?.Env).toEqual([`API_KEY=${secret}`]);
+      const operations = calls.map(({ url, init }) => `${init.method} ${url.pathname}`);
+      if (failureAt === "start")
+        expect(operations.indexOf("DELETE /v1.47/containers/container-1")).toBeLessThan(
+          operations.lastIndexOf("POST /v1.47/containers/create"),
+        );
+      expect(operations.at(-1)).toBe("POST /v1.47/containers/container-2/start");
+    });
+  }
+
+  test("unrelated container errors never trigger a cache fallback", async () => {
+    handler = (url, init) =>
+      url.pathname.endsWith("/volumes/create")
+        ? Response.json(JSON.parse(String(init.body)))
+        : Response.json({ message: "network not found" }, { status: 500 });
+    await expect(
+      provider.provision(target, {
+        ...request,
+        cache: { key: "swarm-cache-pool-1", mountPath: "/cache" },
+      }),
+    ).rejects.toThrow("network not found");
+    expect(calls.filter(({ url }) => url.pathname.endsWith("/containers/create"))).toHaveLength(1);
+  });
+
+  test("a failed cleanup after a mount start error prevents a duplicate container attempt", async () => {
+    handler = (url, init) => {
+      if (url.pathname.endsWith("/volumes/create"))
+        return Response.json(JSON.parse(String(init.body)));
+      if (url.pathname.endsWith("/containers/create"))
+        return Response.json({ Id: handle.externalId });
+      return Response.json(
+        { message: init.method === "DELETE" ? "cleanup unavailable" : "volume mount failed" },
+        { status: 500 },
+      );
+    };
+    await expect(
+      provider.provision(target, {
+        ...request,
+        cache: { key: "swarm-cache-pool-1", mountPath: "/cache" },
+      }),
+    ).rejects.toThrow("failed-start cleanup failed");
+    expect(calls.filter(({ url }) => url.pathname.endsWith("/containers/create"))).toHaveLength(1);
+  });
+
+  test("cache-free retry errors propagate without a third attempt", async () => {
+    let creates = 0;
+    handler = (url, init) =>
+      url.pathname.endsWith("/volumes/create")
+        ? Response.json(JSON.parse(String(init.body)))
+        : Response.json(
+            { message: ++creates === 1 ? "volume mount failed" : "daemon unavailable" },
+            { status: 500 },
+          );
+    await expect(
+      provider.provision(target, {
+        ...request,
+        cache: { key: "swarm-cache-pool-1", mountPath: "/cache" },
+      }),
+    ).rejects.toThrow("daemon unavailable");
+    expect(creates).toBe(2);
+  });
+
+  test("cache release verifies ownership and never forces removal", async () => {
+    const cache = { key: "swarm-cache-pool-1", poolId: request.poolId, ownerId: request.ownerId };
+    handler = (_url, init) =>
+      init.method === "GET"
+        ? Response.json({
+            Name: cache.key,
+            Labels: {
+              [RUNTIME_LABELS.owner]: cache.ownerId,
+              [RUNTIME_LABELS.poolId]: cache.poolId,
+              "swarm.cache": "true",
+            },
+          })
+        : new Response(null, { status: 204 });
+    await provider.releaseCache(target, cache);
+    expect(calls.map(({ init }) => init.method)).toEqual(["GET", "DELETE"]);
+    expect(calls[1]?.url.searchParams.has("force")).toBe(false);
+    calls = [];
+    handler = () => Response.json({ Name: cache.key, Labels: {} });
+    await expect(provider.releaseCache(target, cache)).rejects.toThrow("not owned");
+    expect(calls).toHaveLength(1);
+  });
+
+  test("cache release tolerates missing volumes and defers in-use volumes", async () => {
+    const cache = { key: "swarm-cache-pool-1", poolId: request.poolId, ownerId: request.ownerId };
+    handler = () => new Response("missing", { status: 404 });
+    await provider.releaseCache(target, cache);
+    handler = (_url, init) =>
+      init.method === "GET"
+        ? Response.json({
+            Name: cache.key,
+            Labels: {
+              [RUNTIME_LABELS.owner]: cache.ownerId,
+              [RUNTIME_LABELS.poolId]: cache.poolId,
+              "swarm.cache": "true",
+            },
+          })
+        : new Response("volume is in use", { status: 409 });
+    await expect(provider.releaseCache(target, cache)).rejects.toThrow("volume is in use");
+  });
+
+  test("cache volumes are shared across runtimes and survive container termination", async () => {
+    const volumes = new Map<
+      string,
+      { Name: string; Labels: Record<string, string>; content: string }
+    >();
+    const mounts: unknown[] = [];
+    handler = (url, init) => {
+      const body = init.body ? JSON.parse(String(init.body)) : undefined;
+      if (url.pathname.endsWith("/volumes/create")) {
+        if (!volumes.has(body.Name)) volumes.set(body.Name, { ...body, content: "" });
+        return Response.json(volumes.get(body.Name));
+      }
+      if (url.pathname.endsWith("/containers/create")) {
+        mounts.push(body.HostConfig.Mounts);
+        return Response.json({ Id: crypto.randomUUID() });
+      }
+      return new Response(null, { status: 204 });
+    };
+    const cache = { key: "swarm-cache-pool-1", mountPath: "/cache" };
+    const first = await provider.provision(target, { ...request, cache });
+    volumes.get(cache.key)!.content = "cached git objects";
+    await provider.terminate(target, first);
+    const second = await provider.provision(target, {
+      ...request,
+      runtimeId: crypto.randomUUID(),
+      cache,
+    });
+    await provider.terminate(target, second);
+    expect(volumes.size).toBe(1);
+    expect(volumes.get(cache.key)?.content).toBe("cached git objects");
+    expect(volumes.get(cache.key)?.Labels).toEqual({
+      [RUNTIME_LABELS.owner]: request.ownerId,
+      [RUNTIME_LABELS.poolId]: request.poolId,
+      "swarm.cache": "true",
+    });
+    expect(mounts).toEqual(
+      Array(2).fill([{ Type: "volume", Source: cache.key, Target: "/cache" }]),
+    );
+    expect(
+      calls
+        .filter(({ init }) => init.method === "DELETE")
+        .every(
+          ({ url }) =>
+            url.pathname.includes("/containers/") && url.searchParams.get("v") === "true",
+        ),
+    ).toBe(true);
+    expect(provider.traits).toMatchObject({ persistentCache: true, ephemeralDisk: true });
+  });
+
+  test("cache provisioning fails open with a scrubbed warning and no mount", async () => {
+    const secret = "cache-volume-short-secret";
+    handler = (url) =>
+      url.pathname.endsWith("/volumes/create")
+        ? new Response(`volume unavailable ${secret}`, { status: 500 })
+        : url.pathname.endsWith("/containers/create")
+          ? Response.json({ Id: handle.externalId })
+          : new Response(null, { status: 204 });
+    const result = await provider.provision(target, {
+      ...request,
+      cache: { key: "swarm-cache-pool-1", mountPath: "/cache" },
+      env: { API_KEY: secret },
+    });
+    expect(result.meta?.cacheWarning).toContain("volume unavailable");
+    expect(result.meta?.cacheWarning).not.toContain(secret);
+    const body = JSON.parse(
+      String(calls.find(({ url }) => url.pathname.endsWith("/containers/create"))?.init.body),
+    );
+    expect(body.HostConfig.Mounts).toBeUndefined();
+    expect(calls.at(-1)?.url.pathname).toEndWith("/containers/container-1/start");
+  });
+
+  for (const mismatched of ["owner", "poolId", "cache"] as const) {
+    test(`cache volumes with a mismatched ${mismatched} label are never mounted or relabeled`, async () => {
+      const labels = {
+        [RUNTIME_LABELS.owner]: request.ownerId,
+        [RUNTIME_LABELS.poolId]: request.poolId,
+        "swarm.cache": "true",
+      };
+      labels[mismatched === "cache" ? "swarm.cache" : RUNTIME_LABELS[mismatched]] = "foreign";
+      handler = (url) =>
+        url.pathname.endsWith("/volumes/create")
+          ? Response.json({ Name: "swarm-cache-pool-1", Labels: labels })
+          : url.pathname.endsWith("/containers/create")
+            ? Response.json({ Id: handle.externalId })
+            : new Response(null, { status: 204 });
+      const result = await provider.provision(target, {
+        ...request,
+        cache: { key: "swarm-cache-pool-1", mountPath: "/cache" },
+      });
+      expect(result.meta?.cacheWarning).toContain("not owned");
+      expect(JSON.parse(String(calls[1]?.init.body)).HostConfig.Mounts).toBeUndefined();
+      expect(calls.some(({ init }) => init.method === "DELETE")).toBe(false);
+    });
+  }
+
+  test("cache mounts survive the missing-image retry", async () => {
+    let creates = 0;
+    handler = (url, init) => {
+      if (url.pathname.endsWith("/volumes/create"))
+        return Response.json(JSON.parse(String(init.body)));
+      if (url.pathname.endsWith("/containers/create")) {
+        expect(JSON.parse(String(init.body)).HostConfig.Mounts).toEqual([
+          { Type: "volume", Source: "swarm-cache-pool-1", Target: "/cache" },
+        ]);
+        return ++creates === 1
+          ? new Response("No such image", { status: 404 })
+          : Response.json({ Id: handle.externalId });
+      }
+      return url.pathname.endsWith("/images/create")
+        ? new Response('{"status":"complete"}\n')
+        : new Response(null, { status: 204 });
+    };
+    await provider.provision(target, {
+      ...request,
+      cache: { key: "swarm-cache-pool-1", mountPath: "/cache" },
+    });
+    expect(creates).toBe(2);
+    expect(calls.filter(({ url }) => url.pathname.endsWith("/volumes/create"))).toHaveLength(1);
+  });
+
   for (const message of [
     "client version 1.44 is too old. Minimum supported API version is 1.47",
     "API version is unsupported",

package/src/tests/runtime-remote-controller.test.ts

@@ -17,6 +17,7 @@ import {
   markRuntimeTerminated,
   upsertRuntimeInstance,
 } from "../be/multi-runtime";
+import { reconcileCacheReleases } from "../be/runtime-pools/cache-cleanup";
 import {
   clearRuntimeOwnerIdCache,
   ensureRuntimeOwnerId,
@@ -30,6 +31,7 @@ import {
   getRuntimeController,
   getRuntimePool,
   updateRuntimeController,
+  updateRuntimePool,
 } from "../be/runtime-pools/db";
 import { remoteControllerSession } from "../be/runtime-pools/remote-controller";
 import {
@@ -50,15 +52,29 @@ import type {
   RuntimeTargetConfig,
 } from "../runtime-providers";
 import { ControllerOfflineError } from "../runtime-providers/controller";
-import { RUNTIME_LABELS } from "../runtime-providers/types";
+import { RUNTIME_LABELS, type RuntimeCacheReference } from "../runtime-providers/types";
 import { listenOnFreePort } from "./test-net";
 import { CHILD_PROCESS_TEST_BUDGET_MS, runChild } from "./test-proc";
 
 class FakeProvider implements RuntimeProvider {
   readonly kind = "docker" as const;
-  readonly traits = { canProvision: true, ephemeralDisk: true, supportsPause: false };
+  readonly traits: RuntimeProvider["traits"] = {
+    canProvision: true,
+    ephemeralDisk: true,
+    supportsPause: false,
+  };
   requests: ProvisionRequest[] = [];
   terminated: string[] = [];
+  releasedCaches: RuntimeCacheReference[] = [];
+  caches = new Set<string>();
+  cacheReleaseGate?: Promise<void>;
+  completedReleases = 0;
+  async releaseCache(_target: RuntimeTargetConfig, cache: RuntimeCacheReference) {
+    this.releasedCaches.push(cache);
+    await this.cacheReleaseGate;
+    this.caches.delete(cache.key);
+    this.completedReleases++;
+  }
   owners: string[] = [];
   runtimes = new Map<string, { handle: RuntimeHandle; ownerId: string; status: RuntimeStatus }>();
   gate?: Promise<void>;
@@ -67,6 +83,7 @@ class FakeProvider implements RuntimeProvider {
   }
   async provision(_target: RuntimeTargetConfig, req: ProvisionRequest) {
     this.requests.push(req);
+    if (req.cache) this.caches.add(req.cache.key);
     await this.gate;
     const handle: RuntimeHandle = {
       kind: this.kind,
@@ -133,6 +150,7 @@ beforeEach(async () => {
   process.env.RUNTIME_CIRCUIT_THRESHOLD = "3";
   process.env.RUNTIME_PROVISION_MAX_ATTEMPTS = "1";
   for (const table of [
+    "runtime_cache_releases",
     "active_sessions",
     "runtime_instances",
     "session_tokens",
@@ -205,22 +223,68 @@ function request(ownerId: string): ProvisionRequest {
 }
 
 test("hello sets online and all controller commands make round trips through the CLI client", async () => {
+  fake.traits.persistentCache = true;
   await connect();
   const row = await getRuntimeController(controllerId);
   expect(row?.lastSeenAt).toBeTruthy();
   const commands = await getControllerCommands(controllerId);
   expect(commands.workerApiUrl).toBe("http://remote-worker.invalid");
   const req = request(await ensureRuntimeOwnerId());
+  expect(commands.traits.persistentCache).toBe(true);
+  req.cache = { key: `swarm-cache-${req.poolId}`, mountPath: "/cache" };
   const handle = await commands.provision(req);
+  expect(fake.requests[0]?.cache).toEqual(req.cache);
   expect(handle.controllerId).toBe(controllerId);
   expect(await commands.status([handle])).toEqual({ [handle.externalId]: "running" });
   expect(await commands.list(req.ownerId)).toEqual([handle]);
   expect(await commands.list("another-swarm")).toEqual([]);
   await commands.terminate(handle, "test complete");
+  const cache = { key: req.cache.key, poolId: req.poolId, ownerId: req.ownerId };
+  await commands.releaseCache!(cache);
+  expect(fake.releasedCaches).toEqual([cache]);
   expect(await commands.status([handle])).toEqual({ [handle.externalId]: "exited" });
   expect(JSON.stringify(logSpy.mock.calls)).not.toContain(token);
 });
 
+test("a timed-out remote cache release cannot delete a re-enabled cache", async () => {
+  fake.traits.persistentCache = true;
+  await connect();
+  const pool = await createRuntimePool({
+    name: "cache-timeout",
+    kind: "docker",
+    placements: [controllerId],
+    image: "worker:test",
+    cache: { mountPath: "/cache" },
+  });
+  const agent = await createPoolAgent(pool.id, { name: "cache-worker" });
+  const commands = await getControllerCommands(controllerId);
+  const owner = await ensureRuntimeOwnerId();
+  await provisionRuntime(pool, agent, commands, owner);
+  const oldKey = fake.requests[0]!.cache!.key;
+  const runtime = (await listRuntimeInstancesForPool(pool.id))[0]!;
+  await commands.terminate(runtime.externalHandle as unknown as RuntimeHandle, "test");
+  await markRuntimeTerminated(runtime.id, "stopped");
+  await updateRuntimePool(pool.id, { cache: null });
+  const finish = Promise.withResolvers<void>();
+  fake.cacheReleaseGate = finish.promise;
+  try {
+    await reconcileCacheReleases(async () => commands, new AbortController().signal);
+    expect(fake.completedReleases).toBe(0);
+    const enabled = await updateRuntimePool(pool.id, { cache: { mountPath: "/cache" } });
+    await provisionRuntime(enabled, agent, commands, owner);
+    const newKey = fake.requests.at(-1)!.cache!.key;
+    finish.resolve();
+    await until(async () => fake.completedReleases === 1);
+    expect(fake.caches.has(oldKey)).toBe(false);
+    expect(fake.caches.has(newKey)).toBe(true);
+    expect(newKey).not.toBe(oldKey);
+    await reconcileCacheReleases(async () => commands, new AbortController().signal);
+  } finally {
+    finish.resolve();
+    await until(async () => fake.completedReleases >= 1);
+  }
+});
+
 test("remote E2B provisioning, renewal, pause, and resume use the controller session", async () => {
   await updateRuntimeController(controllerId, {
     kind: "e2b",

package/src/tests/runtime-supervisor.test.ts

@@ -31,6 +31,7 @@ import {
   getRuntimePool,
 } from "../be/runtime-pools/db";
 import {
+  provisionRuntime,
   reconcileControllerOrphans,
   reconcileOnce,
   runtimeSupervisorStatus,
@@ -53,13 +54,19 @@ import { registerVolatileSecret, scrubSecrets } from "../utils/secret-scrubber";
 
 class FakeRuntimeProvider implements RuntimeProvider {
   readonly kind = "docker" as const;
-  readonly traits = { canProvision: true, ephemeralDisk: true, supportsPause: false };
+  readonly traits: RuntimeProvider["traits"] = {
+    canProvision: true,
+    ephemeralDisk: true,
+    supportsPause: false,
+    persistentCache: true,
+  };
   requests: ProvisionRequest[] = [];
   targets: RuntimeTargetConfig[] = [];
   terminated: string[] = [];
   owners: string[] = [];
   runtimes = new Map<string, { handle: RuntimeHandle; owner: string; status: RuntimeStatus }>();
   failure: string | null = null;
+  cacheWarning: string | null = null;
   gate: Promise<void> | null = null;
   terminateFailures = new Map<string, string>();
   statusFailures = new Set<string>();
@@ -74,7 +81,11 @@ class FakeRuntimeProvider implements RuntimeProvider {
     if (this.failure) throw new Error(this.failure);
     if (this.provisionFailures.has(req.agentId))
       throw new Error(this.provisionFailures.get(req.agentId));
-    const handle: RuntimeHandle = { kind: "docker", externalId: crypto.randomUUID() };
+    const handle: RuntimeHandle = {
+      kind: "docker",
+      externalId: crypto.randomUUID(),
+      ...(this.cacheWarning ? { meta: { cacheWarning: this.cacheWarning } } : {}),
+    };
     this.runtimes.set(handle.externalId, { handle, owner: req.ownerId, status: "running" });
     return handle;
   }
@@ -184,6 +195,59 @@ async function activate(runtimeId: string, agentId: string, reportedSlots = 1) {
   await getDbClient().run("UPDATE agents SET status = 'idle' WHERE id = ?", [agentId]);
 }
 
+test("cache identity is stable across provisions of one pool and scoped away from other pools", async () => {
+  const { pool, agent } = await wake({ cache: { mountPath: "/cache" } });
+  const controller = await getControllerCommands(controllerId);
+  await provisionRuntime(pool, agent, controller, await ensureRuntimeOwnerId());
+  const other = await setup({ cache: { mountPath: "/cache" } });
+  await provisionRuntime(other.pool, other.agent, controller, await ensureRuntimeOwnerId());
+  const [first, second, third] = fake.requests.map((req) => req.cache!);
+  expect(first).toEqual({
+    key: expect.stringContaining(`swarm-cache-${pool.id}-`),
+    mountPath: "/cache",
+  });
+  expect(second).toEqual(first);
+  expect(third.key).toStartWith(`swarm-cache-${other.pool.id}-`);
+  expect(third.key).not.toBe(first.key);
+});
+
+test("an older controller boots without cache and surfaces a warning without failing the provision", async () => {
+  const { pool, agent } = await setup({ cache: { mountPath: "/cache" } });
+  delete fake.traits.persistentCache;
+  await provisionRuntime(
+    pool,
+    agent,
+    await getControllerCommands(controllerId),
+    await ensureRuntimeOwnerId(),
+  );
+  expect(fake.requests[0]?.cache).toBeUndefined();
+  expect(await getRuntimePool(pool.id)).toMatchObject({
+    consecutiveFailures: 0,
+    lastError: expect.stringContaining("Controller does not support persistent caches"),
+  });
+  expect((await getAgentById(agent.id))?.runtimeProvisionAttempts).toBe(0);
+});
+
+test("cache failure warnings clear after recovery without hiding unrelated provision errors", async () => {
+  fake.cacheWarning = "cache unavailable";
+  const { pool, agent } = await wake({ cache: { mountPath: "/cache" } });
+  expect(await getRuntimePool(pool.id)).toMatchObject({
+    consecutiveFailures: 0,
+    lastError: "Persistent cache: cache unavailable",
+  });
+  fake.cacheWarning = null;
+  const commands = await getControllerCommands(controllerId);
+  await provisionRuntime(pool, agent, commands, await ensureRuntimeOwnerId());
+  expect((await getRuntimePool(pool.id))?.lastError).toBeNull();
+  await getDbClient().run(
+    "UPDATE runtime_pools SET last_error = 'Existing provision error' WHERE id = ?",
+    [pool.id],
+  );
+  fake.cacheWarning = "another cache failure";
+  await provisionRuntime(pool, agent, commands, await ensureRuntimeOwnerId());
+  expect((await getRuntimePool(pool.id))?.lastError).toBe("Existing provision error");
+});
+
 test("pending task provisions once while registration is pending, with scoped env and owner labels", async () => {
   const { pool, agent, runtime } = await wake({
     env: {

package/src/tests/script-connections.test.ts

@@ -665,6 +665,19 @@ describe("script connections", () => {
       markMigrationApplied(database, "254_query_audit_indexes.sql");
       // 256 alters agents, which this migration-112-only fixture does not create.
       markMigrationApplied(database, "256_agent_slack_icon.sql");
+      // 259 alters session_summaries, which 253 (skipped above) creates.
+      markMigrationApplied(database, "259_session_summaries_open_tasks.sql");
+      // 260 alters runtime_pools and creates runtime_cache_releases. This
+      // migration-112-only fixture has neither runtime_pools nor runtime_controllers.
+      markMigrationApplied(database, "260_runtime_pool_cache.sql");
+      // 262 indexes agent_tasks columns that this fixture's id-only agent_tasks lacks.
+      markMigrationApplied(database, "262_session_visibility_indexes.sql");
+      // 263 indexes agent_tasks columns that this fixture's id-only agent_tasks lacks.
+      markMigrationApplied(database, "263_task_list_filter_index.sql");
+      // 264 indexes agent_tasks columns that this fixture's id-only agent_tasks lacks.
+      markMigrationApplied(database, "264_slack_thread_cover_index.sql");
+      // 266 alters script_runs, which this migration-112-only fixture does not create.
+      markMigrationApplied(database, "266_script_runs_source_task.sql");
 
       const id = crypto.randomUUID();
       const now = new Date().toISOString();