@cohortapp/agent-sdk 2.5.1 → 2.6.0

package/scripts/daemon/assurance-e2e.test.mjs

@@ -0,0 +1,421 @@
+/**
+ * assurance-e2e.test.mjs — the owner's four scenarios, end to end, through the
+ * REAL daemon.
+ *
+ * Run:            node --test scripts/daemon/assurance-e2e.test.mjs
+ * Read the story: ASSURANCE_TRANSCRIPT=1 node --test scripts/daemon/assurance-e2e.test.mjs
+ *
+ * This drives `agent-daemon.processItem` itself — the real classifier gate, the
+ * real acknowledgement decision, the real obligation ledger, the real
+ * settle-on-close logic, the real sweep. Only the two things that would cost
+ * money or touch a live workspace are injected fakes:
+ *
+ *   * `dispatch`  — instead of spawning a `claude` child, the fake invokes
+ *                   `onClose` with the exit code the scenario is about.
+ *   * transport   — instead of posting to Cohort/Slack, it appends to a
+ *                   transcript so the test can assert on the exact words a
+ *                   human would read.
+ *
+ * NOTHING here posts to the live workspace, spawns a model, or writes to the
+ * database. The four scenarios are the four the brief names: a fast ask, a slow
+ * ask, a timing-out session, and a crashing session — plus the one the old code
+ * could not even detect, a session that exits 0 having said nothing.
+ */
+
+import { test, before, after } from "node:test";
+import assert from "node:assert/strict";
+import { register } from "node:module";
+import { mkdtempSync, mkdirSync, writeFileSync, rmSync } from "node:fs";
+import { join } from "node:path";
+import { tmpdir } from "node:os";
+
+// The daemon statically pulls in the agent-only `imapflow` dep via the gmail
+// poller; stub it so the real daemon module can be imported in CI.
+const loaderSrc = `
+export async function resolve(specifier, context, next) {
+  if (specifier === "imapflow") return { url: "imapflow-stub:main", shortCircuit: true };
+  return next(specifier, context);
+}
+export async function load(url, context, next) {
+  if (url === "imapflow-stub:main") return { format: "module", shortCircuit: true, source: "export class ImapFlow {}" };
+  return next(url, context);
+}
+`;
+register(`data:text/javascript,${encodeURIComponent(loaderSrc)}`);
+
+const AGENT_DIR = mkdtempSync(join(tmpdir(), "maestro-assurance-e2e-"));
+process.env.AGENT_DIR = AGENT_DIR;
+process.env.AGENT_ROOT = AGENT_DIR;
+process.env.MAESTRO_DAEMON_NO_AUTOSTART = "1";
+
+let daemon, assurance, receipts;
+
+before(async () => {
+  mkdirSync(join(AGENT_DIR, "state", "inbox", "cohort"), { recursive: true });
+  mkdirSync(join(AGENT_DIR, "state", "sessions", "locks"), { recursive: true });
+  daemon = await import("./agent-daemon.mjs");
+  assurance = await import("./assurance.mjs");
+  receipts = await import("../../lib/comms/receipts.mjs");
+});
+
+after(() => { try { rmSync(AGENT_DIR, { recursive: true, force: true }); } catch { /* */ } });
+
+// ---------------------------------------------------------------------------
+// Transcript: what the human sees, and when
+// ---------------------------------------------------------------------------
+
+const SHOW = process.env.ASSURANCE_TRANSCRIPT === "1";
+let transcript = [];
+let t0 = 0;
+
+function beginScenario(title, ask) {
+  transcript = [];
+  t0 = Date.now();
+  if (SHOW) {
+    console.log(`\n${"═".repeat(78)}\n  ${title}\n${"═".repeat(78)}`);
+    console.log(`  [   0.00s]  OWNER  →  ${ask}`);
+  }
+}
+
+/** Record a message the human would receive, at a stated virtual offset. */
+function heard(kind, text, virtualMs = null) {
+  const realMs = Date.now() - t0;
+  transcript.push({ kind, text, ms: realMs, virtualMs });
+  if (SHOW) {
+    const stamp = virtualMs != null
+      ? `+${(virtualMs / 60000).toFixed(1)}min`
+      : `${(realMs / 1000).toFixed(2)}s`;
+    console.log(`  [${String(stamp).padStart(8)}]  ISLA   →  ${text.replace(/\n/g, "\n                        ")}`);
+  }
+}
+
+function note(text, virtualMs = null) {
+  if (!SHOW) return;
+  const stamp = virtualMs != null ? `+${(virtualMs / 60000).toFixed(1)}min` : `${((Date.now() - t0) / 1000).toFixed(2)}s`;
+  console.log(`  [${String(stamp).padStart(8)}]  ·····     (${text})`);
+}
+
+/** A transport fake that feeds the transcript. Never touches a network. */
+function transport(virtualClock = () => null) {
+  return async (item, text, o = {}) => {
+    heard(o.kind || "reply", text, virtualClock());
+    return { sent: true, via: "fake", channel: item.channel_id || item.channel };
+  };
+}
+
+/** The daemon's ack seam, shaped exactly like responder.sendHoldingMessage. */
+function ackSeam() {
+  return async (item, classResult) => {
+    const text = assurance.composeAck(item, classResult);
+    heard("ack", text);
+    return { sent: true, holdingText: text, via: "fake" };
+  };
+}
+
+/**
+ * Each scenario gets its OWN channel id. The daemon's per-thread lock and
+ * request-claim would otherwise (correctly) suppress the second and subsequent
+ * asks as duplicates of a conversation already in flight — real behaviour, but
+ * orthogonal to what these scenarios are proving.
+ */
+function makeItem(id, content) {
+  const channelId = `CDM-${id}`;
+  const item = {
+    id,
+    raw_ref: `cohort:${channelId}:${id}`,
+    message_id: id,
+    service: "cohort",
+    channel: "dm/owner",
+    channel_id: channelId,
+    is_dm: true,
+    sender: "the owner",
+    content,
+  };
+  // The poll filter expects a live inbox file for markProcessed to act on.
+  writeFileSync(join(AGENT_DIR, "state", "inbox", "cohort", `${id}.yaml`), `id: ${id}\n`);
+  return item;
+}
+
+const COMPLEX = {
+  priority: "critical", action: "respond", model: "opus", category: "action_required",
+  directed_at_agent: true, summary: "Fix the issues you just noted and push the fixes to git",
+};
+
+// ===========================================================================
+// SCENARIO 1 — A FAST ASK. Answered directly. No "let me look into it".
+// ===========================================================================
+
+test("SCENARIO 1 — a fast ask gets a direct answer, with no pointless holding message", async () => {
+  const ask = "what time is the board call on Thursday?";
+  beginScenario("SCENARIO 1 — FAST ASK", ask);
+  const item = makeItem("FAST-1", ask);
+
+  let holdingCalls = 0;
+  await daemon.processItem(item, "cohort", {
+    classify: async () => ({ priority: "normal", action: "respond", model: "sonnet", category: "question", directed_at_agent: true, summary: "board call time" }),
+    isQuickReply: () => true,
+    sendQuickResponse: async (it) => {
+      heard("reply", "Thursday's board call is at 14:00 UK — calendar invite went out last week.");
+      return { sent: true, text: "…", channel: it.channel_id };
+    },
+    sendHoldingMessage: async () => { holdingCalls++; return { sent: true, holdingText: "x" }; },
+    dispatch: () => { throw new Error("a fast ask must not spawn a session"); },
+  });
+
+  assert.equal(holdingCalls, 0, "a simple ask must NOT be told 'I'll look into it' before its answer");
+  assert.equal(transcript.length, 1, "exactly one message: the answer");
+  assert.equal(transcript[0].kind, "reply");
+  assert.ok(transcript[0].ms < 1000, `answered in ${transcript[0].ms}ms`);
+  assert.equal(assurance.openObligations().length, 0, "nothing is owed — the ask is already discharged");
+  note("no obligation opened: the answer already went out");
+});
+
+// ===========================================================================
+// SCENARIO 2 — A SLOW ASK. Acknowledged in milliseconds, updated, then answered.
+// ===========================================================================
+
+test("SCENARIO 2 — a slow ask is acknowledged at once, updated while it runs, then answered", async () => {
+  const ask = "with that in mind, fix these issues you just noted please and push their fixes to git";
+  beginScenario("SCENARIO 2 — SLOW ASK (the owner's actual message)", ask);
+  const item = makeItem("SLOW-1", ask);
+
+  let onCloseHook = null;
+  await daemon.processItem(item, "cohort", {
+    classify: async () => COMPLEX,
+    isQuickReply: () => false,
+    sendHoldingMessage: ackSeam(),
+    dispatch: (_p, _it, _cr, _s, opts) => { onCloseHook = opts.onClose; },
+  });
+
+  const ack = transcript.find((m) => m.kind === "ack");
+  assert.ok(ack, "the acknowledgement must go out");
+  assert.ok(ack.ms < 1000, `acknowledged in ${ack.ms}ms — the old path budgeted 60,000ms and missed 2 times in 3`);
+  assert.match(ack.text, /either way/i);
+
+  const key = assurance.openObligations()[0].key;
+  note("obligation opened and acknowledged; session running");
+
+  // ── 6 minutes later: still working. The sweep speaks rather than let the
+  //    human sit behind a typing indicator wondering.
+  let virtual = 6 * 60_000;
+  const clock = () => virtual;
+  const deps = { deliverImpl: transport(clock), spokeSinceImpl: () => false };
+  let stats = await assurance.sweepObligations({ now: Date.now() + virtual, deps });
+  assert.equal(stats.progressed, 1, "long work must produce an interim update");
+
+  // ── 17 minutes: the session finally answers the human itself.
+  virtual = 17 * 60_000;
+  receipts.recordOutbound({ service: "cohort", channel: "CDM-SLOW-1", kind: "session", via: "messaging_send", agentRoot: AGENT_DIR });
+  heard("reply", "Done — all three issues fixed and pushed (commit a41f0c9). The flaky test was a race in the poller; I've pinned it and re-run CI twice, green both times.", virtual);
+  onCloseHook({ ok: true, code: 0, sessionId: "s-e2e-2", stdout: "" });
+  await new Promise((r) => setImmediate(r));
+
+  const rec = assurance.readObligation(key);
+  assert.equal(rec.state, "answered", "the debt is discharged by evidence of a delivered reply");
+  const progress = transcript.filter((m) => m.kind === "progress");
+  assert.equal(progress.length, 1);
+  assert.equal(transcript.filter((m) => m.kind === "ack").length, 1, "acknowledge once, never twice");
+  note("obligation discharged by receipt — the daemon added nothing on top of the session's own answer", virtual);
+});
+
+// ===========================================================================
+// SCENARIO 3 — A TIMING-OUT SESSION. SIGTERM at the 45-minute cap.
+// ===========================================================================
+
+test("SCENARIO 3 — a timing-out session tells the human what happened and that a retry is running", async () => {
+  const ask = "can you reconcile the July spend against the forecast and tell me where we're over?";
+  beginScenario("SCENARIO 3 — SESSION TIMES OUT (exit 143 / SIGTERM)", ask);
+  const item = makeItem("TIMEOUT-1", ask);
+
+  let onCloseHook = null;
+  await daemon.processItem(item, "cohort", {
+    classify: async () => ({ ...COMPLEX, summary: "Reconcile July spend against forecast" }),
+    isQuickReply: () => false,
+    sendHoldingMessage: ackSeam(),
+    dispatch: (_p, _it, _cr, _s, opts) => { onCloseHook = opts.onClose; },
+  });
+  const key = assurance.openObligations().find((o) => o.key === item.raw_ref).key;
+  assert.ok(transcript.find((m) => m.kind === "ack"));
+
+  // 45 minutes in, the dispatcher's SIGTERM lands and the session closes 143.
+  const virtual = { v: 45 * 60_000 };
+  const r = await assurance.settleSession({
+    key, ok: false, code: 143, error: "session_close exit 143", sessionId: "s-e2e-3",
+    deps: { deliverImpl: transport(() => virtual.v), spokeSinceImpl: () => false },
+  });
+
+  assert.equal(r.willRetry, true, "a timeout is transient — it earns one retry");
+  const failure = transcript.filter((m) => m.kind === "failure").at(-1);
+  assert.ok(failure, "the human MUST be told; the old code emitted telemetry and nothing else");
+  assert.match(failure.text, /ran past its time limit/i);
+  assert.match(failure.text, /retrying it now/i);
+  assert.equal(assurance.readObligation(key).state, "open", "still owed until the retry resolves");
+  note("item left un-processed for redelivery; the debt stays open so the retry is covered too", virtual.v);
+});
+
+// ===========================================================================
+// SCENARIO 4 — A CRASHING SESSION. Nothing to retry; say so and escalate.
+// ===========================================================================
+
+test("SCENARIO 4 — a crashing session tells the human it failed, stops, and escalates", async () => {
+  const ask = "pull the Q3 pipeline and draft the investor update";
+  beginScenario("SCENARIO 4 — SESSION CRASHES (unrecoverable)", ask);
+  const item = makeItem("CRASH-1", ask);
+
+  let onCloseHook = null;
+  await daemon.processItem(item, "cohort", {
+    classify: async () => ({ ...COMPLEX, summary: "Draft the Q3 investor update" }),
+    isQuickReply: () => false,
+    sendHoldingMessage: ackSeam(),
+    dispatch: (_p, _it, _cr, _s, opts) => { onCloseHook = opts.onClose; },
+  });
+  const key = item.raw_ref;
+
+  const r = await assurance.settleSession({
+    key, ok: false, code: 1, error: "claude CLI spawn error: ENOENT", sessionId: "s-e2e-4",
+    deps: { deliverImpl: transport(() => 90_000), spokeSinceImpl: () => false },
+  });
+
+  assert.equal(r.willRetry, false, "a spawn failure is permanent — retrying burns another 45 minutes for the same outcome");
+  const failure = transcript.filter((m) => m.kind === "failure").at(-1);
+  assert.match(failure.text, /couldn't get this done/i);
+  assert.match(failure.text, /stopped retrying/i);
+  assert.match(failure.text, /flagged it so it isn't lost/i);
+  assert.match(failure.text, /\?\s*$/, "a dead end ends in a question, so the human has a move");
+  assert.equal(assurance.readObligation(key).state, "failed");
+  note("durable needs-attention record written to state/obligations/needs-attention/", 90_000);
+});
+
+// ===========================================================================
+// SCENARIO 5 — EXIT ZERO, NOTHING SAID. The silent death the old code called
+// a success, marked processed forever, and emitted as `sent`.
+// ===========================================================================
+
+test("SCENARIO 5 — a session that exits 0 without replying no longer passes as a success", async () => {
+  const ask = "did the migration land cleanly?";
+  beginScenario("SCENARIO 5 — SESSION EXITS 0 HAVING SAID NOTHING", ask);
+  const item = makeItem("SILENT-1", ask);
+
+  let onCloseHook = null;
+  await daemon.processItem(item, "cohort", {
+    classify: async () => ({ ...COMPLEX, summary: "Check whether the migration landed cleanly" }),
+    isQuickReply: () => false,
+    sendHoldingMessage: ackSeam(),
+    dispatch: (_p, _it, _cr, _s, opts) => { onCloseHook = opts.onClose; },
+  });
+  const key = item.raw_ref;
+
+  // The real shape: exit 0, real work done, final text admitting nothing was sent.
+  const stdout = JSON.stringify({
+    type: "result",
+    result: "Checked the migration: it applied cleanly on the primary, but the read replica is 4 minutes behind and still catching up. No action needed unless it stalls. Nothing was sent.",
+  });
+  const r = await assurance.settleSession({
+    key, ok: true, code: 0, stdout, sessionId: "s-e2e-5",
+    deps: { deliverImpl: transport(() => 4 * 60_000), spokeSinceImpl: () => false },
+  });
+
+  assert.equal(r.verdict, "silent-success");
+  const rescued = transcript.at(-1);
+  assert.match(rescued.text, /didn't get a reply out to you/i);
+  assert.match(rescued.text, /read replica is 4 minutes behind/, "the work the session actually did reaches the human");
+  note("previously: marked processed, emitted `sent`, human heard nothing — forever", 4 * 60_000);
+});
+
+// ===========================================================================
+// SCENARIO 6 — THE DAEMON ITSELF DIES MID-SESSION.
+// ===========================================================================
+
+test("SCENARIO 6 — a debt orphaned by a daemon restart is spoken to on the next start", async () => {
+  const ask = "sort out the DNS on the staging domain";
+  beginScenario("SCENARIO 6 — THE DAEMON DIES MID-SESSION", ask);
+  const item = makeItem("ORPHAN-1", ask);
+
+  await daemon.processItem(item, "cohort", {
+    classify: async () => ({ ...COMPLEX, summary: "Fix DNS on the staging domain" }),
+    isQuickReply: () => false,
+    sendHoldingMessage: ackSeam(),
+    dispatch: () => { /* the daemon dies here — onClose is never called */ },
+  });
+
+  // Rewrite the record as a previous process's, exactly as a restart would find it.
+  const rec = assurance.readObligation(item.raw_ref);
+  rec.daemonPid = process.pid + 12345;
+  writeFileSync(join(AGENT_DIR, "state", "obligations", `${assurance.sanitiseKey(item.raw_ref)}.json`), JSON.stringify(rec));
+  note("daemon process dies; obligation file survives on disk");
+
+  // The sweep must PROVE the owning process is gone before it says this — a bare
+  // pid mismatch is also what a second daemon on the same AGENT_DIR looks like,
+  // and telling a requester their running work was interrupted is its own bug.
+  const stats = await assurance.sweepObligations({
+    now: Date.now() + assurance.INTERRUPT_GRACE_MS + 30_000,
+    deps: { deliverImpl: transport(() => 30_000), spokeSinceImpl: () => false, isPidAliveImpl: () => false },
+  });
+  assert.equal(stats.interrupted, 1);
+  assert.match(transcript.at(-1).text, /was interrupted before it finished/i);
+  note("previously: the typing dot stopped and that was the last the human ever heard", 30_000);
+});
+
+// ===========================================================================
+// SCENARIO 7 — TWO ASKS, ONE ROOM. The owner sends a second thing before the
+// first is done. One session answers; the other exits 0 in silence.
+// ===========================================================================
+
+test("SCENARIO 7 — a second ask in the same DM is not discharged by the first one's answer", async () => {
+  const askA = "what's our runway at the current burn?";
+  const askB = "also please fix the flaky poller test and push it";
+  beginScenario("SCENARIO 7 — TWO ASKS IN ONE DM", `${askA}\n  [   0.00s]  OWNER  →  ${askB}`);
+
+  // ONE room, two messages — which is simply what a DM with the owner is.
+  const room = "CDM-BOTH";
+  const mk = (id, content) => {
+    const item = {
+      id, raw_ref: `cohort:${room}:${id}`, message_id: id, service: "cohort",
+      channel: "dm/owner", channel_id: room, is_dm: true, sender: "the owner", content,
+    };
+    writeFileSync(join(AGENT_DIR, "state", "inbox", "cohort", `${id}.yaml`), `id: ${id}\n`);
+    return item;
+  };
+  const a = mk("BOTH-A", askA);
+  const b = mk("BOTH-B", askB);
+
+  const openA = await assurance.openAndAcknowledge({
+    item: a, classResult: { ...COMPLEX, summary: "work out the runway" },
+    deps: { ackSender: ackSeam() },
+  });
+  const openB = await assurance.openAndAcknowledge({
+    item: b, classResult: { ...COMPLEX, summary: "fix the flaky poller test" },
+    deps: { ackSender: ackSeam() },
+  });
+  assurance.noteSession(openA.key, "s-both-A");
+  assurance.noteSession(openB.key, "s-both-B");
+  note("two debts open, one room");
+
+  // Session A answers, through the CLI lane, stamped with the debt it was
+  // spawned for — exactly what the dispatcher's env now makes possible.
+  receipts.recordOutbound({
+    service: "cohort", channel: room, kind: "session", via: "slack-send.sh",
+    obligationKey: openA.key, sessionId: "s-both-A", agentRoot: AGENT_DIR,
+  });
+  heard("reply", "Runway is 14 months at the current burn — the July infra overage is the only thing moving it.", 8 * 60_000);
+
+  const settledA = await assurance.settleSession({
+    key: openA.key, ok: true, code: 0, sessionId: "s-both-A",
+    deps: { deliverImpl: transport(() => 8 * 60_000) },
+  });
+  assert.equal(settledA.verdict, "answered-by-session");
+  assert.equal(settledA.basis, "obligation-keyed", "the receipt names the debt it discharged");
+
+  // Session B exits 0 having said NOTHING. Keyed on the room alone — which is
+  // what shipped — A's receipt closed B too, and this ask vanished in silence.
+  const settledB = await assurance.settleSession({
+    key: openB.key, ok: true, code: 0, sessionId: "s-both-B",
+    stdout: JSON.stringify({ type: "result", result: "The flake is a race between the poller's lock release and the next tick. I've pinned the ordering but not pushed it yet." }),
+    deps: { deliverImpl: transport(() => 9 * 60_000) },
+  });
+  assert.equal(settledB.verdict, "silent-success", "the unanswered ask must NOT ride on the other one's receipt");
+  const rescue = transcript.at(-1);
+  assert.match(rescue.text, /didn't get a reply out to you/i);
+  assert.match(rescue.text, /race between the poller's lock release/, "and B's actual work reaches the human");
+  note("previously: B closed as `answered` on A's receipt — zero messages, ask gone");
+});