@desplega.ai/agent-swarm 2.6.0 → 2.7.0

package/src/tests/store-progress-learning.test.ts

@@ -0,0 +1,288 @@
+import { afterAll, beforeAll, describe, expect, test } from "bun:test";
+import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
+import type { ZodType } from "zod";
+import {
+  closeDb,
+  createAgent,
+  createTaskExtended,
+  getDb,
+  getTaskById,
+  initDb,
+  startTask,
+} from "../be/db";
+import { registerStoreProgressTool } from "../tools/store-progress";
+
+interface ToolResult {
+  isError?: boolean;
+  structuredContent?: Record<string, unknown>;
+}
+
+const agentId = "dddd0000-0000-4000-8000-000000000b01";
+
+const server = new McpServer({ name: "store-progress-learning", version: "1.0.0" });
+registerStoreProgressTool(server);
+const tool = (
+  server as unknown as {
+    _registeredTools: Record<
+      string,
+      {
+        inputSchema: ZodType;
+        handler: (args: unknown, meta: unknown) => Promise<ToolResult>;
+      }
+    >;
+  }
+)._registeredTools["store-progress"]!;
+
+async function call(args: Record<string, unknown>) {
+  return tool.handler(tool.inputSchema.parse(args), {
+    sessionId: "store-progress-learning-session",
+    requestInfo: { headers: { "x-agent-id": agentId } },
+  });
+}
+
+async function task(vcsRepo?: string) {
+  const row = await createTaskExtended("learning capture", {
+    agentId,
+    source: "system",
+    followUpConfig: { disabled: true },
+    ...(vcsRepo ? { vcsRepo } : {}),
+  });
+  await startTask(row.id);
+  return row;
+}
+
+interface MemoryRow {
+  id: string;
+  agentId: string | null;
+  key: string;
+  name: string;
+  scope: string;
+  source: string;
+  sourceTaskId: string;
+  tags: string;
+  content: string;
+}
+
+function learningRows(taskId: string): MemoryRow[] {
+  return getDb()
+    .query<MemoryRow, [string]>(
+      "SELECT id, agentId, key, name, scope, source, sourceTaskId, tags, content FROM agent_memory WHERE sourceTaskId = ? AND key LIKE '/learnings/%' ORDER BY createdAt",
+    )
+    .all(taskId);
+}
+
+function learningData(result: ToolResult) {
+  return result.structuredContent?.learning as
+    | { status: string; key?: string; error?: string }
+    | undefined;
+}
+
+function message(result: ToolResult) {
+  return String(result.structuredContent?.message);
+}
+
+const originalEmbeddingKey = process.env.EMBEDDING_API_KEY;
+const originalOpenAiKey = process.env.OPENAI_API_KEY;
+beforeAll(async () => {
+  process.env.EMBEDDING_API_KEY = "";
+  process.env.OPENAI_API_KEY = "";
+  initDb(":memory:");
+  await createAgent({ id: agentId, name: "Learner", isLead: false, status: "idle" });
+});
+afterAll(() => {
+  closeDb();
+  if (originalEmbeddingKey === undefined) delete process.env.EMBEDDING_API_KEY;
+  else process.env.EMBEDDING_API_KEY = originalEmbeddingKey;
+  if (originalOpenAiKey === undefined) delete process.env.OPENAI_API_KEY;
+  else process.env.OPENAI_API_KEY = originalOpenAiKey;
+});
+
+describe("store-progress learning", () => {
+  test("stores a swarm manual memory under the explicit workstream", async () => {
+    const t = await task("desplega-ai/agent-swarm");
+    const result = await call({
+      taskId: t.id,
+      progress: "found the cause",
+      learning: {
+        text: "SQLite BUSY needs BEGIN IMMEDIATE\nA deferred BEGIN that reads first fails with BUSY_SNAPSHOT.",
+        workstream: "DB Runtime",
+        tags: ["SQLite", "gotcha", "learning"],
+      },
+    });
+
+    expect(result.isError).toBeFalsy();
+    expect(learningData(result)?.status).toBe("stored");
+    expect(message(result)).toContain(
+      "Learning stored at /learnings/db-runtime/sqlite-busy-needs-begin-immediate.",
+    );
+    const [row, ...rest] = learningRows(t.id);
+    expect(rest).toHaveLength(0);
+    expect(row).toMatchObject({
+      agentId,
+      key: "/learnings/db-runtime/sqlite-busy-needs-begin-immediate",
+      name: "SQLite BUSY needs BEGIN IMMEDIATE",
+      scope: "swarm",
+      source: "manual",
+      sourceTaskId: t.id,
+    });
+    expect(JSON.parse(row!.tags)).toEqual(["learning", "sqlite", "gotcha"]);
+    expect((await getTaskById(t.id))?.progress).toBe("found the cause");
+  });
+
+  test("string form falls back to the repo name, then to general", async () => {
+    const withRepo = await task("desplega-ai/agent-swarm");
+    await call({ taskId: withRepo.id, learning: "Run lint before tsc" });
+    expect(learningRows(withRepo.id)[0]?.key).toBe("/learnings/agent-swarm/run-lint-before-tsc");
+
+    const withoutRepo = await task();
+    const result = await call({ taskId: withoutRepo.id, learning: { text: "Prefer scripts" } });
+    expect(learningData(result)).toMatchObject({ status: "stored", workstream: "general" });
+    expect(learningRows(withoutRepo.id)[0]?.key).toBe("/learnings/general/prefer-scripts");
+  });
+
+  test("a taken key gets a short suffix instead of failing", async () => {
+    const first = await task();
+    const second = await task();
+    await call({ taskId: first.id, learning: "Same title\nfirst body" });
+    const result = await call({ taskId: second.id, learning: "Same title\nsecond body" });
+
+    expect(learningData(result)?.status).toBe("stored");
+    expect(learningRows(first.id)[0]?.key).toBe("/learnings/general/same-title");
+    expect(learningRows(second.id)[0]?.key).toMatch(
+      /^\/learnings\/general\/same-title-[0-9a-f]{6}$/,
+    );
+  });
+
+  test("an exact replay on the same task is skipped", async () => {
+    const t = await task();
+    await call({ taskId: t.id, learning: "Replay me" });
+    const replay = await call({ taskId: t.id, progress: "again", learning: "Replay me" });
+
+    expect(learningData(replay)).toMatchObject({
+      status: "replayed",
+      key: "/learnings/general/replay-me",
+    });
+    expect(learningRows(t.id)).toHaveLength(1);
+  });
+
+  test("overlapping calls with the same learning store it once", async () => {
+    const t = await task();
+    const learning = "Concurrent replay\nKeep exactly one copy of this lesson.";
+    const results = await Promise.all([
+      call({ taskId: t.id, learning }),
+      call({ taskId: t.id, learning }),
+    ]);
+
+    expect(learningRows(t.id)).toHaveLength(1);
+    expect(results.map((result) => learningData(result)?.status).sort()).toEqual([
+      "replayed",
+      "stored",
+    ]);
+  });
+
+  // Synthetic token, built at runtime so no literal credential sits in the repo.
+  const token = `ghp_${"abcd1234".repeat(5)}`;
+  const kebabToken = token.replace("_", "-");
+
+  test("a secret in the title, workstream or tags never reaches the key or tags", async () => {
+    const t = await task();
+    const result = await call({
+      taskId: t.id,
+      learning: {
+        text: `${token}\nKeep credentials out of memory.`,
+        workstream: token,
+        tags: [token],
+      },
+    });
+
+    expect(learningData(result)?.status).toBe("stored");
+    const [row] = learningRows(t.id);
+    for (const field of [row!.key, row!.name, row!.tags, row!.content, learningData(result)?.key]) {
+      expect(field).not.toContain(token);
+      expect(field).not.toContain(kebabToken);
+    }
+    expect(row!.content).toContain("[REDACTED:");
+  });
+
+  test("a replay of a secret-bearing learning is still detected", async () => {
+    const t = await task();
+    const learning = `Redact diagnostic tokens\nExample token: ${token}`;
+    await call({ taskId: t.id, learning });
+    const replay = await call({ taskId: t.id, learning });
+
+    expect(learningData(replay)?.status).toBe("replayed");
+    expect(learningRows(t.id)).toHaveLength(1);
+  });
+
+  test("a failed learning write leaves the progress write intact", async () => {
+    const t = await task();
+    getDb().run(
+      "CREATE TEMP TRIGGER fail_learning BEFORE INSERT ON agent_memory WHEN NEW.key LIKE '/learnings/%' BEGIN SELECT RAISE(ABORT, 'learning insert refused'); END",
+    );
+    try {
+      const result = await call({
+        taskId: t.id,
+        progress: "progress survives",
+        status: "completed",
+        output: "done",
+        learning: "This write fails",
+      });
+      expect(result.isError).toBeFalsy();
+      expect(learningData(result)).toMatchObject({ status: "failed" });
+      expect(learningData(result)?.error).toContain("learning insert refused");
+      expect(message(result)).toContain("Learning not stored:");
+    } finally {
+      getDb().run("DROP TRIGGER fail_learning");
+    }
+    const stored = await getTaskById(t.id);
+    expect(stored?.progress).toBe("progress survives");
+    expect(stored?.status).toBe("completed");
+    expect(learningRows(t.id)).toHaveLength(0);
+  });
+
+  test("invalid learnings are reported without blocking progress", async () => {
+    const t = await task();
+    const tooManyTags = await call({
+      taskId: t.id,
+      progress: "still stored",
+      learning: { text: "Tagged", tags: ["a", "b", "c", "d", "e", "f"] },
+    });
+    expect(learningData(tooManyTags)).toMatchObject({
+      status: "failed",
+      error: "learning takes at most 5 tags",
+    });
+    const tooLong = await call({ taskId: t.id, learning: "x".repeat(2001) });
+    expect(learningData(tooLong)?.status).toBe("failed");
+    expect((await getTaskById(t.id))?.progress).toBe("still stored");
+    expect(learningRows(t.id)).toHaveLength(0);
+  });
+
+  test("a learning on the completing call is stored", async () => {
+    const t = await task();
+    const result = await call({
+      taskId: t.id,
+      status: "completed",
+      output: "done",
+      learning: "Ship on the completing call",
+    });
+    expect(learningData(result)?.status).toBe("stored");
+    expect((await getTaskById(t.id))?.status).toBe("completed");
+  });
+
+  test("a call on a finished task stores nothing", async () => {
+    const t = await task();
+    await call({ taskId: t.id, status: "completed", output: "done" });
+    const late = await call({
+      taskId: t.id,
+      status: "completed",
+      output: "done",
+      learning: "Too late",
+    });
+    expect(late.structuredContent).toMatchObject({ wasNoOp: true });
+    expect(learningData(late)?.status).toBe("failed");
+
+    const progressOnly = await call({ taskId: t.id, learning: "Still too late" });
+    expect(learningData(progressOnly)?.status).toBe("failed");
+    expect(learningRows(t.id)).toHaveLength(0);
+  });
+});

package/src/tests/ui-logs-parser.test.ts

@@ -1,10 +1,12 @@
 import { describe, expect, test } from "bun:test";
 import { normalizeSessionLogs as normalizeEvalLogs } from "../../apps/evals/ui/src/logs-parser";
 import {
+  cacheHitRatio,
   extractSubagentRuns,
   normalizeSessionLogs,
   parseSessionLogs,
   type SessionLogRecord,
+  sumItemCallUsage,
   unwrapResult,
 } from "../../apps/ui/src/logs-parser";
 
@@ -1696,3 +1698,181 @@ describe.each([
     expect(item?.meta).toMatchObject({ type: "codex_error", isError: true });
   });
 });
+
+describe("per-call usage", () => {
+  const claudeUsage = {
+    input_tokens: 6,
+    cache_read_input_tokens: 90_000,
+    cache_creation_input_tokens: 684,
+    output_tokens: 120,
+  };
+  function claudeLine(
+    id: string,
+    line: number,
+    messageId: string,
+    block: unknown,
+    extra: Record<string, unknown> = {},
+  ) {
+    return log(id, "claude", line, {
+      type: "assistant",
+      message: { id: messageId, role: "assistant", content: [block], usage: claudeUsage },
+      ...extra,
+    });
+  }
+  const usages = (logs: SessionLogRecord[]) =>
+    normalizeSessionLogs(logs)
+      .items.filter((item) => item.kind === "call_usage")
+      .map((item) => item.usage);
+
+  test("claude: one call_usage per message.id, not per content-block line", () => {
+    const result = usages([
+      claudeLine("a1", 1, "msg_1", { type: "thinking", thinking: "" }),
+      claudeLine("a2", 2, "msg_1", { type: "text", text: "Looking." }),
+      claudeLine("a3", 3, "msg_1", { type: "tool_use", id: "t1", name: "Read", input: {} }),
+      claudeLine(
+        "s1",
+        4,
+        "msg_2",
+        { type: "text", text: "sub" },
+        {
+          parent_tool_use_id: "toolu_agent",
+        },
+      ),
+    ]);
+    expect(result).toHaveLength(2);
+    expect(result[0]).toMatchObject({
+      callId: "msg_1",
+      promptTokens: 90_690,
+      cacheReadTokens: 90_000,
+      cacheWriteTokens: 684,
+      uncachedInputTokens: 6,
+      outputTokens: 120,
+      sidechain: false,
+      harness: "claude",
+      reportedAt: "start",
+    });
+    expect(result[1]).toMatchObject({ callId: "msg_2", sidechain: true });
+    expect(cacheHitRatio(result[0]!)).toBeCloseTo(90_000 / 90_690);
+  });
+
+  test("claude: an assistant line without usage emits no call_usage", () => {
+    const result = usages([
+      log("a1", "claude", 1, {
+        type: "assistant",
+        message: { id: "msg_1", role: "assistant", content: [{ type: "text", text: "hi" }] },
+      }),
+    ]);
+    expect(result).toEqual([]);
+  });
+
+  test("the call_usage block shares the call's message, ahead of its content", () => {
+    const messages = parseSessionLogs([
+      claudeLine("a1", 1, "msg_1", { type: "text", text: "Looking." }),
+    ]);
+    expect(messages).toHaveLength(1);
+    expect(messages[0]?.role).toBe("assistant");
+    expect(messages[0]?.content.map((b) => b.type)).toEqual(["provider_meta", "text"]);
+    expect(messages[0]?.content[0]).toMatchObject({
+      kind: "helper",
+      data: { helperType: "call_usage" },
+    });
+  });
+
+  test("codex: context.updated input_tokens already includes cached tokens", () => {
+    const result = usages([
+      log("c1", "codex", 1, {
+        type: "context.updated",
+        usage: {
+          input_tokens: 49_300,
+          cached_input_tokens: 12_000,
+          cache_write_input_tokens: 0,
+          output_tokens: 300,
+          reasoning_output_tokens: 40,
+        },
+        contextWindow: 272_000,
+      }),
+    ]);
+    expect(result).toEqual([
+      {
+        callId: "c1",
+        promptTokens: 49_300,
+        cacheReadTokens: 12_000,
+        cacheWriteTokens: 0,
+        uncachedInputTokens: 37_300,
+        outputTokens: 300,
+        reasoningTokens: 40,
+        sidechain: false,
+        harness: "codex",
+        reportedAt: "end",
+      },
+    ]);
+    expect(cacheHitRatio(result[0]!)).toBeCloseTo(12_000 / 49_300);
+  });
+
+  test("opencode: session.step.ended input excludes cache", () => {
+    const result = usages([
+      log("o1", "opencode", 1, {
+        type: "session.step.ended",
+        data: {
+          sessionID: "ses_1",
+          tokens: { input: 200, output: 50, reasoning: 5, cache: { read: 9_000, write: 800 } },
+        },
+      }),
+    ]);
+    expect(result).toHaveLength(1);
+    expect(result[0]).toMatchObject({
+      promptTokens: 10_000,
+      cacheReadTokens: 9_000,
+      cacheWriteTokens: 800,
+      uncachedInputTokens: 200,
+      harness: "opencode",
+      reportedAt: "end",
+    });
+    expect(cacheHitRatio(result[0]!)).toBeCloseTo(0.9);
+  });
+
+  test("pi: the adapter's call_usage line", () => {
+    const result = usages([
+      log("p1", "pi", 1, {
+        type: "call_usage",
+        harness: "pi",
+        usage: { input: 100, cacheRead: 300, cacheWrite: 100, output: 20, reasoning: 0 },
+      }),
+    ]);
+    expect(result).toHaveLength(1);
+    expect(result[0]).toMatchObject({
+      promptTokens: 500,
+      cacheReadTokens: 300,
+      harness: "pi",
+      reportedAt: "start",
+    });
+    expect(cacheHitRatio(result[0]!)).toBeCloseTo(0.6);
+  });
+
+  test("session totals: Σ read / Σ prompt, and no-cache sessions say so", () => {
+    const totals = sumItemCallUsage(
+      normalizeSessionLogs([
+        claudeLine("a1", 1, "msg_1", { type: "text", text: "one" }),
+        claudeLine("a2", 2, "msg_2", { type: "text", text: "two" }),
+      ]).items,
+    );
+    expect(totals).toMatchObject({
+      calls: 2,
+      promptTokens: 181_380,
+      cacheReadTokens: 180_000,
+      cacheReported: true,
+    });
+    expect(totals?.ratio).toBeCloseTo(180_000 / 181_380);
+
+    const uncached = sumItemCallUsage(
+      normalizeSessionLogs([
+        log("o1", "opencode", 1, {
+          type: "session.step.ended",
+          data: { tokens: { input: 500, output: 5, reasoning: 0, cache: { read: 0, write: 0 } } },
+        }),
+      ]).items,
+    );
+    expect(uncached).toMatchObject({ calls: 1, cacheReported: false, ratio: 0 });
+    expect(sumItemCallUsage([])).toBeNull();
+  });
+});

package/src/tests/workflow-engine-v2.test.ts

@@ -21,7 +21,7 @@ import {
   getWorkflowRunStepsByRunId,
   initDb,
 } from "../be/db";
-import type { Workflow, WorkflowDefinition } from "../types";
+import type { ExecutorMeta, Workflow, WorkflowDefinition } from "../types";
 import { shouldSkipCooldown } from "../workflows/cooldown";
 import {
   findReadyNodes,
@@ -143,6 +143,45 @@ class SlowExecutor extends BaseExecutor<typeof SlowExecutor.schema, typeof SlowE
   }
 }
 
+/** What the last `signal-probe` step saw: its abort signal and whether it got to its side effect. */
+const signalProbe: { signal?: AbortSignal; sideEffect: boolean } = { sideEffect: false };
+
+class SignalProbeExecutor extends BaseExecutor<
+  typeof SignalProbeExecutor.schema,
+  typeof SignalProbeExecutor.outSchema
+> {
+  static readonly schema = z.object({ delayMs: z.number() });
+  static readonly outSchema = z.object({ finished: z.boolean() });
+
+  readonly type = "signal-probe";
+  readonly mode = "instant" as const;
+  readonly configSchema = SignalProbeExecutor.schema;
+  readonly outputSchema = SignalProbeExecutor.outSchema;
+
+  protected async execute(
+    config: z.infer<typeof SignalProbeExecutor.schema>,
+    _context: Readonly<Record<string, unknown>>,
+    meta: ExecutorMeta,
+  ): Promise<ExecutorResult<z.infer<typeof SignalProbeExecutor.outSchema>>> {
+    const signal = meta.signal;
+    signalProbe.signal = signal;
+    await new Promise<void>((resolve) => {
+      const timer = setTimeout(resolve, config.delayMs);
+      signal?.addEventListener(
+        "abort",
+        () => {
+          clearTimeout(timer);
+          resolve();
+        },
+        { once: true },
+      );
+    });
+    // A one-shot action, like the review trigger a claim script fires.
+    if (!signal?.aborted) signalProbe.sideEffect = true;
+    return { status: "success", output: { finished: true } };
+  }
+}
+
 class FailingExecutor extends BaseExecutor<
   typeof FailingExecutor.schema,
   typeof FailingExecutor.outSchema
@@ -216,6 +255,7 @@ function createTestRegistry(): ExecutorRegistry {
   registry.register(new SlowExecutor(mockDeps));
   registry.register(new FailingExecutor(mockDeps));
   registry.register(new ValidatePassExecutor(mockDeps));
+  registry.register(new SignalProbeExecutor(mockDeps));
   return registry;
 }
 
@@ -561,6 +601,41 @@ describe("Workflow Engine v2 (Phase 3)", () => {
       expect(run!.error).toContain("timed out");
     }, 5_000); // Allow test up to 5s
 
+    test("a timed-out step aborts its executor, so the side effect never happens", async () => {
+      signalProbe.signal = undefined;
+      signalProbe.sideEffect = false;
+      const registry = createTestRegistry();
+      const workflow = await makeWorkflow({
+        nodes: [{ id: "probe", type: "signal-probe", config: { delayMs: 600, timeoutMs: 100 } }],
+      });
+
+      const runId = await startWorkflowExecution(workflow, {}, registry);
+
+      const run = await getWorkflowRun(runId);
+      expect(run!.status).toBe("failed");
+      expect(run!.error).toContain("Step timed out after 100ms");
+      expect(signalProbe.signal?.aborted).toBe(true);
+      // Past the executor's own delay: without the abort it would have fired.
+      await Bun.sleep(700);
+      expect(signalProbe.sideEffect).toBe(false);
+    }, 5_000);
+
+    test("a step that finishes in budget is never aborted afterwards", async () => {
+      signalProbe.signal = undefined;
+      signalProbe.sideEffect = false;
+      const registry = createTestRegistry();
+      const workflow = await makeWorkflow({
+        nodes: [{ id: "probe", type: "signal-probe", config: { delayMs: 10, timeoutMs: 150 } }],
+      });
+
+      const runId = await startWorkflowExecution(workflow, {}, registry);
+
+      expect((await getWorkflowRun(runId))!.status).toBe("completed");
+      expect(signalProbe.sideEffect).toBe(true);
+      await Bun.sleep(250);
+      expect(signalProbe.signal?.aborted).toBe(false);
+    }, 5_000);
+
     // The inline runtime's ulimit preamble fails on macOS (no usable
     // RLIMIT_AS), so the script dies before the watchdog behavior under test.
     // Linux CI is the enforcing environment; the skip only unblocks local
@@ -598,6 +673,46 @@ describe("Workflow Engine v2 (Phase 3)", () => {
 
   // ─── Validation ───────────────────────────────────────────
 
+  describe("Detached start", () => {
+    test("detach returns once the run is committed and walks its instant steps in the background", async () => {
+      const registry = createTestRegistry();
+      const workflow = await makeWorkflow({
+        nodes: [
+          { id: "slow", type: "slow", config: { delayMs: 400 }, next: "after" },
+          { id: "after", type: "echo", config: { message: "done" } },
+        ],
+      });
+
+      const started = performance.now();
+      const runId = await startWorkflowExecution(workflow, {}, registry, { detach: true });
+      const returnedAfterMs = performance.now() - started;
+
+      expect(returnedAfterMs).toBeLessThan(300);
+      expect((await getWorkflowRun(runId))!.status).toBe("running");
+
+      const deadline = Date.now() + 3_000;
+      while ((await getWorkflowRun(runId))!.status === "running" && Date.now() < deadline) {
+        await Bun.sleep(25);
+      }
+      expect((await getWorkflowRun(runId))!.status).toBe("completed");
+      const steps = await getWorkflowRunStepsByRunId(runId);
+      expect(steps.map((step) => step.nodeId).sort()).toEqual(["after", "slow"]);
+    }, 5_000);
+
+    test("without detach the start resolves only after the instant steps ran", async () => {
+      const registry = createTestRegistry();
+      const workflow = await makeWorkflow({
+        nodes: [{ id: "slow", type: "slow", config: { delayMs: 300 } }],
+      });
+
+      const started = performance.now();
+      const runId = await startWorkflowExecution(workflow, {}, registry);
+
+      expect(performance.now() - started).toBeGreaterThanOrEqual(290);
+      expect((await getWorkflowRun(runId))!.status).toBe("completed");
+    }, 5_000);
+  });
+
   describe("Validation", () => {
     test("validation pass allows step to complete", async () => {
       const registry = createTestRegistry();