@desplega.ai/agent-swarm 2.3.0 → 2.4.1

package/src/tests/used-by-route.test.ts

@@ -5,6 +5,7 @@ import { closeDb, createAgent, getSwarmConfigs, initDb, upsertSwarmConfig } from
 import { fillOrgLogoFromWebsite } from "../be/org-profile";
 import { handleUsedBy } from "../http/used-by";
 import { getPathSegments, parseQueryParams } from "../http/utils";
+import { _resetTelemetryStateForTests, initTelemetry } from "../telemetry";
 import type { HttpRequestAuth } from "../utils/request-auth-context";
 import { setRequestAuth } from "../utils/request-auth-context";
 
@@ -94,26 +95,155 @@ describe("POST /api/used-by", () => {
   });
 
   test("a non-lead agent gets 403 and nothing reaches the proxy", async () => {
-    const result = await post({ granted: true }, null, WORKER_ID);
+    const result = await post({ granted: true, contactEmail: "admin@acme.com" }, null, WORKER_ID);
     expect(result.statusCode).toBe(403);
     expect(proxied).toEqual([]);
   });
 
+  test("consent telemetry reports quote and function presence without their contents", async () => {
+    const savedFlag = process.env.ANONYMIZED_TELEMETRY;
+    try {
+      process.env.ANONYMIZED_TELEMETRY = "true";
+      _resetTelemetryStateForTests();
+      await initTelemetry("api-server", configValue, async () => {}, { generateIfMissing: true });
+      await post(
+        {
+          granted: true,
+          contactEmail: "admin@acme.com",
+          contactFunction: "founder",
+          quote: "Private quote",
+        },
+        OPERATOR,
+      );
+      await post({ granted: false }, OPERATOR);
+      await post({ granted: true, contactEmail: "admin@acme.com" }, OPERATOR);
+      const events = proxied.filter((request) => request.body.event === "used_by.consent_changed");
+      expect(events).toHaveLength(3);
+      expect(events.map((event) => event.body.properties)).toMatchObject([
+        { granted: true, has_quote: true, has_function: true },
+        { granted: false, has_quote: false, has_function: false },
+        { granted: true, has_quote: false, has_function: false },
+      ]);
+      for (const event of events) {
+        expect(JSON.stringify(event.body)).not.toContain("Private quote");
+        expect(JSON.stringify(event.body)).not.toContain("admin@acme.com");
+      }
+    } finally {
+      if (savedFlag === undefined) delete process.env.ANONYMIZED_TELEMETRY;
+      else process.env.ANONYMIZED_TELEMETRY = savedFlag;
+      _resetTelemetryStateForTests();
+    }
+  });
+
   test("grant and revoke both reach the proxy and are stored", async () => {
-    const grant = await post({ granted: true }, OPERATOR);
+    const grant = await post(
+      {
+        granted: true,
+        contactEmail: "admin@acme.com",
+        contactFunction: "founder",
+        quote: "Our swarm ships features.",
+      },
+      OPERATOR,
+    );
     expect(grant.statusCode).toBe(200);
     expect(proxied[0]?.url).toBe("https://proxy.desplega.sh/v1/used-by");
     expect(proxied[0]?.body).toMatchObject({
       org_name: "Acme",
       org_website: "acme.com",
       granted: true,
+      contact_email: "admin@acme.com",
+      contact_function: "founder",
+      quote: "Our swarm ships features.",
     });
     expect(await configValue("SWARM_USED_BY_CONSENT")).toBe("true");
+    expect(await configValue("SWARM_USED_BY_CONTACT_EMAIL")).toBe("admin@acme.com");
+    expect(await configValue("SWARM_USED_BY_CONTACT_FUNCTION")).toBe("founder");
+    expect(await configValue("SWARM_USED_BY_QUOTE")).toBe("Our swarm ships features.");
+    // Contact details never reach agent env, and readers see the email masked.
+    const rows = await getSwarmConfigs({ scope: "global" });
+    const row = (key: string) => rows.find((entry) => entry.key === key);
+    for (const key of [
+      "SWARM_USED_BY_CONTACT_EMAIL",
+      "SWARM_USED_BY_CONTACT_FUNCTION",
+      "SWARM_USED_BY_QUOTE",
+    ]) {
+      expect(row(key)?.exportToEnv).toBe(false);
+    }
+    expect(row("SWARM_USED_BY_CONTACT_EMAIL")?.isSecret).toBe(true);
 
     const revoke = await post({ granted: false }, OPERATOR);
     expect(revoke.statusCode).toBe(200);
     expect(proxied[1]?.body).toMatchObject({ granted: false });
     expect(await configValue("SWARM_USED_BY_CONSENT")).toBe("false");
+    expect(await configValue("SWARM_USED_BY_CONTACT_EMAIL")).toBe("");
+    expect(await configValue("SWARM_USED_BY_CONTACT_FUNCTION")).toBe("");
+    expect(await configValue("SWARM_USED_BY_QUOTE")).toBe("");
+    expect(proxied[1]?.body).not.toHaveProperty("contact_email");
+    expect(proxied[1]?.body).not.toHaveProperty("contact_function");
+    expect(proxied[1]?.body).not.toHaveProperty("quote");
+  });
+
+  test("a grant requires valid contact details and a quote of at most 280 characters", async () => {
+    for (const details of [
+      {},
+      { contactEmail: "" },
+      { contactEmail: "invalid" },
+      { contactEmail: "admin@acme.com", contactFunction: "CEO" },
+      { contactEmail: "admin@acme.com", quote: "x".repeat(281) },
+    ]) {
+      const result = await post({ granted: true, ...details }, OPERATOR);
+      expect(result.statusCode).toBe(400);
+      expect(result.body).toContain("Validation error");
+    }
+    expect(proxied).toEqual([]);
+    expect((await post({ granted: false, contactEmail: "invalid" }, OPERATOR)).statusCode).toBe(
+      400,
+    );
+    expect(proxied).toEqual([]);
+  });
+
+  test("contact email accepts 320 characters and rejects longer values", async () => {
+    const contactEmail = `${"a".repeat(64)}@${Array(4).fill("b".repeat(63)).join(".")}`;
+    expect(contactEmail).toHaveLength(320);
+    expect((await post({ granted: true, contactEmail }, OPERATOR)).statusCode).toBe(200);
+    proxied = [];
+    const result = await post({ granted: true, contactEmail: `a${contactEmail}` }, OPERATOR);
+    expect(result.statusCode).toBe(400);
+    expect(result.body).toContain("Contact email must be at most 320 characters");
+    expect(proxied).toEqual([]);
+  });
+
+  test("an empty stored name overrides the deployment name", async () => {
+    process.env.SWARM_ORG_NAME = "Deployment name";
+    try {
+      await upsertSwarmConfig({ scope: "global", key: "SWARM_ORG_NAME", value: "" });
+      const result = await post({ granted: true, contactEmail: "admin@acme.com" }, OPERATOR);
+      expect(result.statusCode).toBe(400);
+      expect(result.body).toContain("Set the organization name and website first");
+      expect(proxied).toEqual([]);
+    } finally {
+      delete process.env.SWARM_ORG_NAME;
+    }
+  });
+
+  test("a new grant clears omitted optional fields from the previous grant", async () => {
+    const result = await post(
+      {
+        granted: true,
+        contactEmail: "admin@acme.com",
+        contactFunction: "engineer",
+        quote: "x".repeat(280),
+      },
+      OPERATOR,
+    );
+    expect(result.statusCode).toBe(200);
+    expect(await configValue("SWARM_USED_BY_QUOTE")).toHaveLength(280);
+    expect((await post({ granted: true, contactEmail: "new@acme.com" }, OPERATOR)).statusCode).toBe(
+      200,
+    );
+    expect(await configValue("SWARM_USED_BY_CONTACT_EMAIL")).toBe("new@acme.com");
+    expect(await configValue("SWARM_USED_BY_CONTACT_FUNCTION")).toBe("");
+    expect(await configValue("SWARM_USED_BY_QUOTE")).toBe("");
   });
 
   test("with telemetry off the install ID stays home", async () => {
@@ -124,9 +254,9 @@ describe("POST /api/used-by", () => {
     });
     const saved = process.env.ANONYMIZED_TELEMETRY;
     process.env.ANONYMIZED_TELEMETRY = "true";
-    await post({ granted: true }, OPERATOR);
+    await post({ granted: true, contactEmail: "admin@acme.com" }, OPERATOR);
     process.env.ANONYMIZED_TELEMETRY = "false";
-    await post({ granted: true }, OPERATOR);
+    await post({ granted: true, contactEmail: "admin@acme.com" }, OPERATOR);
     if (saved === undefined) delete process.env.ANONYMIZED_TELEMETRY;
     else process.env.ANONYMIZED_TELEMETRY = saved;
     expect(proxied[0]?.body.install_id).toBe("install_0123456789abcdef");
@@ -139,7 +269,7 @@ describe("POST /api/used-by", () => {
       key: "feedback_endpoint",
       value: "http://stub.test/v1/feedback",
     });
-    await post({ granted: true }, OPERATOR);
+    await post({ granted: true, contactEmail: "admin@acme.com" }, OPERATOR);
     expect(proxied[0]?.url).toBe("http://stub.test/v1/used-by");
     await upsertSwarmConfig({
       scope: "global",
@@ -151,13 +281,15 @@ describe("POST /api/used-by", () => {
   test("a proxy failure is a 502 and leaves the stored consent alone", async () => {
     await upsertSwarmConfig({ scope: "global", key: "SWARM_USED_BY_CONSENT", value: "false" });
     proxyStatus = 500;
-    const result = await post({ granted: true }, OPERATOR);
+    const result = await post({ granted: true, contactEmail: "admin@acme.com" }, OPERATOR);
     expect(result.statusCode).toBe(502);
     expect(await configValue("SWARM_USED_BY_CONSENT")).toBe("false");
   });
 
   test("a revoke after the website is cleared or changed uses the granted identity", async () => {
-    expect((await post({ granted: true }, OPERATOR)).statusCode).toBe(200);
+    expect(
+      (await post({ granted: true, contactEmail: "admin@acme.com" }, OPERATOR)).statusCode,
+    ).toBe(200);
     await upsertSwarmConfig({ scope: "global", key: "SWARM_ORG_WEBSITE", value: "" });
     const revoke = await post({ granted: false }, OPERATOR);
     expect(revoke.statusCode).toBe(200);
@@ -173,7 +305,9 @@ describe("POST /api/used-by", () => {
     expect(proxied[2]?.body).toMatchObject({ org_name: "Acme", org_website: "acme.com" });
 
     // A new grant sends, and from then on revokes with, the current identity.
-    expect((await post({ granted: true }, OPERATOR)).statusCode).toBe(200);
+    expect(
+      (await post({ granted: true, contactEmail: "admin@acme.com" }, OPERATOR)).statusCode,
+    ).toBe(200);
     expect(proxied[3]?.body).toMatchObject({ org_name: "Acme Labs", org_website: "acme.io" });
     await upsertSwarmConfig({ scope: "global", key: "SWARM_ORG_WEBSITE", value: "" });
     await post({ granted: false }, OPERATOR);
@@ -182,10 +316,20 @@ describe("POST /api/used-by", () => {
 
   test("without a website it is a 400", async () => {
     await upsertSwarmConfig({ scope: "global", key: "SWARM_ORG_WEBSITE", value: "" });
-    const result = await post({ granted: true }, OPERATOR);
+    const result = await post({ granted: true, contactEmail: "admin@acme.com" }, OPERATOR);
     expect(result.statusCode).toBe(400);
+    expect(JSON.parse(result.body)).toEqual({
+      error: "Set the organization name and website first",
+    });
     expect(proxied).toEqual([]);
   });
+
+  test("a placeholder name still grants: the ranking's placeholder rule does not apply", async () => {
+    await upsertSwarmConfig({ scope: "global", key: "SWARM_ORG_NAME", value: "Swarm" });
+    const result = await post({ granted: true, contactEmail: "admin@acme.com" }, OPERATOR);
+    expect(result.statusCode).toBe(200);
+    expect(proxied[0]?.body).toMatchObject({ org_name: "Swarm", org_website: "acme.com" });
+  });
 });
 
 describe("fillOrgLogoFromWebsite", () => {

package/src/tests/workflow-agent-task-retry.test.ts

@@ -13,8 +13,14 @@ import {
   getWorkflowRun,
   getWorkflowRunStepsByRunId,
   initDb,
+  updateWorkflowRunStep,
 } from "../be/db";
-import type { AgentTask, WorkflowDefinition, WorkflowRunStep } from "../types";
+import {
+  type AgentTask,
+  SUPERSEDED_WORKFLOW_TASK_REASON,
+  type WorkflowDefinition,
+  type WorkflowRunStep,
+} from "../types";
 import { startWorkflowExecution } from "../workflows/engine";
 import { workflowEventBus } from "../workflows/event-bus";
 import { AgentTaskExecutor } from "../workflows/executors/agent-task";
@@ -27,6 +33,7 @@ import { ExecutorRegistry } from "../workflows/executors/registry";
 import { recoverIncompleteRuns } from "../workflows/recovery";
 import { setupWorkflowResumeListener } from "../workflows/resume";
 import { startRetryPoller, stopRetryPoller } from "../workflows/retry-poller";
+import { SUPERSEDE_RETRY_POLICY } from "../workflows/task-step-routing";
 import { interpolate } from "../workflows/template";
 
 const TEST_DB_PATH = "./test-workflow-agent-task-retry.sqlite";
@@ -217,18 +224,18 @@ describe("agent-task node.retry on task failure", () => {
     expect((await getTaskByWorkflowRunStepId(step.id))!.id).toBe(second.id);
   });
 
-  test("a node without retry fails the run exactly as before", async () => {
+  test("a node without retry fails the run exactly as before on a non-supersede failure", async () => {
     const runId = await startRun({});
     const step = await synthesizeStep(runId);
     const first = (await getTaskByWorkflowRunStepId(step.id))!;
 
-    await failTask(first.id, "superseded_workflow_task");
+    await failTask(first.id, "agent crashed");
 
     const run = await waitFor(async () => {
       const r = await getWorkflowRun(runId);
       return r?.status === "failed" ? r : null;
     }, "run failed");
-    expect(run.error).toContain("superseded_workflow_task");
+    expect(run.error).toContain("agent crashed");
     const failedStep = await synthesizeStep(runId);
     expect(failedStep.status).toBe("failed");
     expect(failedStep.retryCount).toBe(0);
@@ -281,6 +288,125 @@ describe("agent-task node.retry on task failure", () => {
   });
 });
 
+/** Wait for the step to be queued for the poller, then make it due now. */
+async function waitForQueuedRetry(runId: string): Promise<WorkflowRunStep> {
+  const queued = await waitFor(async () => {
+    const s = await synthesizeStep(runId);
+    return s.status === "failed" && s.nextRetryAt != null ? s : null;
+  }, "step queued for retry");
+  await updateWorkflowRunStep(queued.id, { nextRetryAt: new Date(Date.now() - 1).toISOString() });
+  return queued;
+}
+
+describe("implicit retry for a superseded task on a node without retry", () => {
+  test("a cut is redriven once after the static delay; a second cut follows onNodeFailure", async () => {
+    const runId = await startRun({});
+    const step = await synthesizeStep(runId);
+    const first = (await getTaskByWorkflowRunStepId(step.id))!;
+
+    const cutAt = Date.now();
+    await failTask(first.id, SUPERSEDED_WORKFLOW_TASK_REASON);
+
+    const queued = await waitFor(async () => {
+      const s = await synthesizeStep(runId);
+      return s.status === "failed" && s.nextRetryAt != null ? s : null;
+    }, "step queued for retry");
+    expect(queued.retryCount).toBe(1);
+    expect(queued.maxRetries).toBe(1);
+    expect(queued.error).toBe(SUPERSEDED_WORKFLOW_TASK_REASON);
+    const delayMs = new Date(queued.nextRetryAt!).getTime() - cutAt;
+    expect(delayMs).toBeGreaterThanOrEqual(SUPERSEDE_RETRY_POLICY.baseDelayMs - 1000);
+    expect(delayMs).toBeLessThanOrEqual(SUPERSEDE_RETRY_POLICY.baseDelayMs + 5000);
+    expect((await getWorkflowRun(runId))!.status).toBe("waiting");
+    expect((await getTaskById(first.id))!.workflowRunStepId).toBeFalsy();
+
+    await waitForQueuedRetry(runId);
+    startRetryPoller(registry, 10);
+    const second = await waitForRedrive(step.id, first);
+    await waitFor(async () => {
+      const s = await synthesizeStep(runId);
+      return s.status === "waiting" ? s : null;
+    }, "step waiting on redriven task");
+    stopRetryPoller();
+
+    await failTask(second.id, SUPERSEDED_WORKFLOW_TASK_REASON);
+    const run = await waitFor(async () => {
+      const r = await getWorkflowRun(runId);
+      return r?.status === "failed" ? r : null;
+    }, "run failed via onNodeFailure:fail");
+    expect(run.error).toContain(SUPERSEDED_WORKFLOW_TASK_REASON);
+    const finalStep = await synthesizeStep(runId);
+    expect(finalStep.status).toBe("failed");
+    expect(finalStep.retryCount).toBe(1);
+    expect(finalStep.nextRetryAt).toBeFalsy();
+    expect((await getTaskByWorkflowRunStepId(step.id))!.id).toBe(second.id);
+  });
+
+  test("the redriven task completing continues the run", async () => {
+    const runId = await startRun({ onNodeFailure: "continue" });
+    const step = await synthesizeStep(runId);
+    const first = (await getTaskByWorkflowRunStepId(step.id))!;
+
+    await failTask(first.id, SUPERSEDED_WORKFLOW_TASK_REASON);
+    await waitForQueuedRetry(runId);
+    startRetryPoller(registry, 10);
+    const second = await waitForRedrive(step.id, first);
+    await waitFor(async () => {
+      const s = await synthesizeStep(runId);
+      return s.status === "waiting" ? s : null;
+    }, "step waiting on redriven task");
+    stopRetryPoller();
+
+    await completeTask(second.id, "synthesis done");
+    const run = await waitFor(async () => {
+      const r = await getWorkflowRun(runId);
+      return r?.status === "completed" ? r : null;
+    }, "run completed");
+    expect(run.status).toBe("completed");
+    const finalStep = await synthesizeStep(runId);
+    expect(JSON.stringify(finalStep.output)).not.toContain("[FAILED:");
+  });
+
+  test("explicit retry {maxRetries:0} opts out: the cut fails the run", async () => {
+    const runId = await startRun({ retry: { ...RETRY_ONCE, maxRetries: 0 } });
+    const step = await synthesizeStep(runId);
+    const first = (await getTaskByWorkflowRunStepId(step.id))!;
+
+    await failTask(first.id, SUPERSEDED_WORKFLOW_TASK_REASON);
+
+    const run = await waitFor(async () => {
+      const r = await getWorkflowRun(runId);
+      return r?.status === "failed" ? r : null;
+    }, "run failed");
+    expect(run.error).toContain(SUPERSEDED_WORKFLOW_TASK_REASON);
+    const failedStep = await synthesizeStep(runId);
+    expect(failedStep.retryCount).toBe(0);
+    expect(failedStep.nextRetryAt).toBeFalsy();
+    expect((await getTaskByWorkflowRunStepId(step.id))!.id).toBe(first.id);
+  });
+
+  test("the recovery sweep reads the cut from the task row", async () => {
+    const runId = await startRun({});
+    const step = await synthesizeStep(runId);
+    const first = (await getTaskByWorkflowRunStepId(step.id))!;
+
+    teardownResumeListener?.();
+    try {
+      await failTask(first.id, SUPERSEDED_WORKFLOW_TASK_REASON);
+      await new Promise((r) => setTimeout(r, 20));
+      await recoverIncompleteRuns(registry);
+    } finally {
+      teardownResumeListener = setupWorkflowResumeListener(workflowEventBus, registry);
+    }
+
+    const queued = await synthesizeStep(runId);
+    expect(queued.status).toBe("failed");
+    expect(queued.nextRetryAt).toBeTruthy();
+    expect(queued.retryCount).toBe(1);
+    expect((await getWorkflowRun(runId))!.status).toBe("waiting");
+  });
+});
+
 describe("recovery sweep holding a stale failed-task snapshot", () => {
   /**
    * Recovery snapshots task A as failed, then pauses. Meanwhile the live

package/src/tests/workflow-wait-recovery.test.ts

@@ -1,4 +1,4 @@
-import { afterAll, beforeAll, describe, expect, test } from "bun:test";
+import { afterAll, beforeAll, describe, expect, spyOn, test } from "bun:test";
 import { unlink } from "node:fs/promises";
 import * as db from "../be/db";
 import {
@@ -13,6 +13,7 @@ import {
   getWorkflowRunStep,
   getWorkflowRunStepsByRunId,
   initDb,
+  resolveWaitState,
   updateWorkflowRun,
   updateWorkflowRunStep,
 } from "../be/db";
@@ -23,6 +24,10 @@ import { createExecutorRegistry } from "../workflows/executors/registry";
 import { recoverIncompleteRuns } from "../workflows/recovery";
 
 const TEST_DB_PATH = "./test-workflow-wait-recovery.sqlite";
+const templateGlobals = globalThis as typeof globalThis & {
+  __testMigrationTemplate?: Uint8Array;
+};
+let savedTemplate: Uint8Array | undefined;
 
 const eventBus = new InProcessEventBus();
 const deps: ExecutorDependencies = {
@@ -42,7 +47,52 @@ async function makeWorkflow(name: string, def: WorkflowDefinition): Promise<Work
   return wf;
 }
 
+async function makeResolvedWait(
+  mode: "time" | "event",
+  status: "fired" | "timeout",
+  nextPort: string,
+) {
+  const wf = await makeWorkflow("wait-recovery-resolved", {
+    nodes: [
+      {
+        id: "w1",
+        type: "wait",
+        config:
+          mode === "time"
+            ? { mode, durationMs: 60_000 }
+            : { mode, eventName: "saved.signal", timeoutMs: 60_000 },
+        next: { [nextPort]: "done" },
+      },
+      { id: "done", type: "notify", config: { channel: "swarm", template: "recovered" } },
+    ],
+  });
+  const run = await createWorkflowRun({ id: crypto.randomUUID(), workflowId: wf.id });
+  await updateWorkflowRun(run.id, { status: "waiting" });
+  const step = await createWorkflowRunStep({
+    id: crypto.randomUUID(),
+    runId: run.id,
+    nodeId: "w1",
+    nodeType: "wait",
+  });
+  await updateWorkflowRunStep(step.id, { status: "waiting" });
+  const future = new Date(Date.now() + 60_000).toISOString();
+  const wait = await createWaitState({
+    id: crypto.randomUUID(),
+    workflowRunId: run.id,
+    workflowRunStepId: step.id,
+    mode,
+    ...(mode === "time" ? { wakeUpAt: future } : { eventName: "saved.signal", expiresAt: future }),
+  });
+  const payload = mode === "event" && status === "fired" ? { signal: "saved", ok: true } : null;
+  const resolution = await resolveWaitState(wait.id, { status, firedPayload: payload });
+  expect(resolution.updated).toBe(true);
+  return { run, step, wait, payload, resolution };
+}
+
 beforeAll(async () => {
+  // Use a file-backed DB so restart cases retain their committed wait resolutions.
+  savedTemplate = templateGlobals.__testMigrationTemplate;
+  templateGlobals.__testMigrationTemplate = undefined;
   try {
     await unlink(TEST_DB_PATH);
   } catch {
@@ -60,12 +110,76 @@ afterAll(async () => {
     }
   }
   closeDb();
+  templateGlobals.__testMigrationTemplate = savedTemplate;
   for (const suffix of ["", "-wal", "-shm"]) {
     await unlink(`${TEST_DB_PATH}${suffix}`).catch(() => {});
   }
 });
 
 describe("WaitExecutor — recovery on startup", () => {
+  test.each([
+    ["time", "fired", "default"],
+    ["event", "fired", "event"],
+    ["event", "timeout", "timeout"],
+  ] as const)("recovers a persisted %s/%s resolution after restart via '%s' exactly once", async (mode, status, nextPort) => {
+    const registry = createExecutorRegistry(deps);
+    const { run, step, wait, payload, resolution } = await makeResolvedWait(mode, status, nextPort);
+
+    // Reopen after the resolution commits, before the waiting step is checkpointed.
+    closeDb();
+    initDb(TEST_DB_PATH);
+    expect((await getWorkflowRunStep(step.id))?.status).toBe("waiting");
+    expect((await getWaitStateByStepId(step.id))?.status).toBe(status);
+    await Promise.all([recoverIncompleteRuns(registry), recoverIncompleteRuns(registry)]);
+
+    const recoveredStep = await getWorkflowRunStep(step.id);
+    expect(recoveredStep?.status).toBe("completed");
+    expect(recoveredStep?.nextPort).toBe(nextPort);
+    expect(recoveredStep?.output).toEqual({
+      waitId: wait.id,
+      mode,
+      firedAt: resolution.row!.resolvedAt,
+      payload,
+    });
+    expect(await getWaitStateByStepId(step.id)).toEqual(resolution.row);
+    expect((await getWorkflowRun(run.id))?.status).toBe("completed");
+    const doneSteps = (await getWorkflowRunStepsByRunId(run.id)).filter((s) => s.nodeId === "done");
+    expect(doneSteps).toHaveLength(1);
+    expect(doneSteps[0]?.status).toBe("completed");
+
+    await recoverIncompleteRuns(registry);
+    expect((await getWorkflowRunStepsByRunId(run.id)).filter((s) => s.nodeId === "done")).toEqual(
+      doneSteps,
+    );
+    expect((await getWorkflowRun(run.id))?.status).toBe("completed");
+  });
+
+  test.each([
+    "completed",
+    "failed",
+    "cancelled",
+  ] as const)("does not reopen a %s run that becomes terminal during wait recovery", async (status) => {
+    const registry = createExecutorRegistry(deps);
+    const { run, step } = await makeResolvedWait("event", "fired", "event");
+    const getWorkflow = db.getWorkflow;
+    const snapshot = spyOn(db, "getWorkflow").mockImplementationOnce(async (id) => {
+      const workflow = await getWorkflow(id);
+      expect((await getWorkflowRun(run.id))?.status).toBe("waiting");
+      // A sibling can fail the run after the resume read, leaving this step waiting.
+      await updateWorkflowRun(run.id, { status });
+      return workflow;
+    });
+    try {
+      await recoverIncompleteRuns(registry);
+      expect(snapshot).toHaveBeenCalledTimes(1);
+      expect((await getWorkflowRun(run.id))?.status).toBe(status);
+      expect((await getWorkflowRunStep(step.id))?.status).toBe("waiting");
+      expect(await getWorkflowRunStepsByRunId(run.id)).toHaveLength(1);
+    } finally {
+      snapshot.mockRestore();
+    }
+  });
+
   test("server died while waiting: overdue time-mode wait recovers via 'default' port", async () => {
     const registry = createExecutorRegistry(deps);
 
@@ -170,9 +284,7 @@ describe("WaitExecutor — recovery on startup", () => {
     const doneCount1 = stepsAfter1.filter((s) => s.nodeId === "done").length;
     expect(doneCount1).toBe(1);
 
-    // Second recovery pass — wait_state is now 'fired' (case a in the
-    // recovery query). resumeWaitState's atomic update returns updated=false,
-    // so the step shouldn't re-run.
+    // Completed runs and steps are no longer eligible for recovery.
     await recoverIncompleteRuns(registry);
     const stepsAfter2 = await getWorkflowRunStepsByRunId(run.id);
     expect(stepsAfter2.filter((s) => s.nodeId === "done").length).toBe(1);

package/src/tools/app-action-run.ts

@@ -0,0 +1,63 @@
+import type { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
+import * as z from "zod";
+import { runAppAction } from "../apps/action-runner";
+import { AppNameSchema } from "../apps/definition";
+import { appDefinitionNeedsRepair, getApp } from "../apps/store";
+import { getAgentById } from "../be/db";
+import { can } from "../rbac";
+import { createToolRegistrar, swarmToolOutputSchema, toolErr, toolOk } from "./utils";
+
+export function registerAppActionRunTool(server: McpServer) {
+  createToolRegistrar(server)(
+    "app-action-run",
+    {
+      title: "Run an app action",
+      description:
+        "Run a declared app action as an agent. Scripts and extensions cannot invoke this tool.",
+      // An action runs a script, a sync, or a task: it can write rows and reach external sources.
+      annotations: { readOnlyHint: false, destructiveHint: true, openWorldHint: true },
+      rbac: { permission: "app.use" },
+      inputSchema: z.object({
+        appId: z.string().min(1),
+        action: AppNameSchema,
+        input: z.record(z.string(), z.unknown()).optional(),
+      }),
+      outputSchema: swarmToolOutputSchema({
+        ok: z.boolean().optional(),
+        result: z.unknown().optional(),
+        stdout: z.string().optional(),
+        error: z.string().optional(),
+        durationMs: z.number().optional(),
+        taskId: z.string().optional(),
+        status: z.string().optional(),
+      }),
+    },
+    async ({ appId, action, input }, info) => {
+      if (info.callOrigin === "script-sdk" || info.callOrigin === "extension")
+        return toolErr("Scripts and extensions cannot run app actions.");
+      if (!info.agentId) return toolErr('Agent ID not found. Set the "X-Agent-ID" header.');
+      const agent = await getAgentById(info.agentId);
+      const decision = can({
+        principal: { kind: "agent", agentId: info.agentId, isLead: agent?.isLead ?? false },
+        verb: "app.use",
+        resource: { kind: "app", appId },
+        source: "mcp",
+      });
+      if (!decision.allow) return toolErr(decision.reason);
+      const app = await getApp(appId);
+      if (!app) return toolErr("App not found.");
+      if (appDefinitionNeedsRepair(app))
+        return toolErr("App definition needs repair.", { data: { issues: app.definitionError } });
+      const output = await runAppAction({
+        app,
+        name: action,
+        input,
+        actor: `agent:${info.agentId}`,
+        caller: { kind: "agent", id: info.agentId, ...(agent?.name ? { name: agent.name } : {}) },
+      });
+      if (output.status !== 200)
+        return toolErr(output.body.error ?? "Action failed.", { data: output.body });
+      return toolOk("Action invoked.", { data: output.body });
+    },
+  );
+}