@cohortapp/agent-sdk 2.14.0 → 2.16.0

package/scripts/daemon/assurance-e2e.test.mjs

@@ -207,10 +207,42 @@ test("SCENARIO 1 — a fast ask gets a direct answer, with no pointless holding
 });
 
 // ===========================================================================
-// SCENARIO 2 — A SLOW ASK. Acknowledged in milliseconds, updated, then answered.
+// SCENARIO 2 — A SLOW ASK. Silent for ninety seconds, one line, then answered.
+//
+// WHY THIS SCENARIO CHANGED ON 2026-09-12, AND WHY THE PROGRESS PING IS GONE
+//
+// It used to assert an acknowledgement at t≈0 and EXACTLY ONE progress ping at
+// six minutes. Both assertions were faithful to the code and both encoded the
+// defect that code had become.
+//
+// Measured over fourteen days in org_default_adaptic: 10,667 agent messages, of
+// which 3,069 (28.8%) were opening acknowledgements and 961 (9.0%) were progress
+// nags — 37.8% of everything this fleet said carried no content at all. 272 of
+// those acknowledgements were never followed by a substantive reply inside an
+// hour, i.e. the promise in them ("I'll come back as soon as I've got
+// something") was simply not kept. Agent-to-human volume went from 2:1 to 158:1
+// in a fortnight, and 9,187 of ~11,400 messages in 21 days landed in ONE
+// channel from eight agents, ~45% of them exact duplicates. Sampled live at
+// 06:32Z: "Still on this — 5 minutes in" beside "Still going — 15 minutes in"
+// at the SAME timestamp, two independent per-obligation timers narrating one
+// piece of overlapping work.
+//
+// The progress ping could not have been better written. `composeProgress` was a
+// pure function of `now - openedAt`; nothing read the session's stdout, its
+// tool calls or its partial findings, so the message could not contain anything
+// a reader could not already see on the clock. A message that says only what
+// the timestamp says is not an update. It is deleted, and this scenario now
+// asserts that NONE is emitted — not because the assertion was wrong about the
+// code, but because the behaviour it pinned was the bug.
+//
+// What replaces it: silence below ACK_AFTER_MS (the typing indicator is the
+// acknowledgement), then AT MOST ONE bespoke line, and only if the room has not
+// already heard one inside its budget window. Beyond that the agent either says
+// something with content in it — the plan the session itself stated — or it says
+// nothing and lets the answer be the answer.
 // ===========================================================================
 
-test("SCENARIO 2 — a slow ask is acknowledged at once, updated while it runs, then answered", async () => {
+test("SCENARIO 2 — a slow ask is silent, then gets ONE line, then the answer — and no progress nag", async () => {
   const ask = "with that in mind, fix these issues you just noted please and push their fixes to git";
   beginScenario("SCENARIO 2 — SLOW ASK (the owner's actual message)", ask);
   const item = makeItem("SLOW-1", ask);
@@ -224,23 +256,45 @@ test("SCENARIO 2 — a slow ask is acknowledged at once, updated while it runs,
     dispatch: (_p, _it, _cr, _s, opts) => { onCloseHook = opts.onClose; },
   });
 
-  const ack = transcript.find((m) => m.kind === "ack");
-  assert.ok(ack, "the acknowledgement must go out");
-  assert.ok(ack.ms < 1000, `acknowledged in ${ack.ms}ms`);
-  assert.match(ack.text, /git fixes/i, "the ack is bespoke to the actual ask, not a rotation pick");
-  assert.doesNotMatch(ack.text, /^(on it|looking now)[.!—]?\s*$/i, "never the retired canned lines");
-  assert.ok(ack.text.length <= 120, "the ack is one short human line, not a templated paragraph");
-
+  assert.equal(transcript.length, 0, "NOTHING at t=0 — the reflex ack is what 3,069 content-free messages were");
   const key = assurance.openObligations()[0].key;
-  note("obligation opened and acknowledged; session running");
+  assert.equal(assurance.readObligation(key).state, "open", "the DEBT is opened all the same: silence about timing is not silence about outcome");
+  note("obligation opened, nothing said; session running");
 
-  // ── 6 minutes later: still working. The sweep speaks rather than let the
-  //    human sit behind a typing indicator wondering.
-  let virtual = 6 * 60_000;
+  // ── Thirty seconds in: still nothing. Below ACK_AFTER_MS the typing
+  //    indicator is the acknowledgement.
+  let virtual = 30_000;
   const clock = () => virtual;
-  const deps = { deliverImpl: transport(clock), spokeSinceImpl: () => false };
-  let stats = await assurance.sweepObligations({ now: Date.now() + virtual, deps });
-  assert.equal(stats.progressed, 1, "long work must produce an interim update");
+  const deps = () => ({
+    deliverImpl: transport(clock),
+    spokeSinceImpl: () => false,
+    generateAckImpl: async () => "Taking the git fixes now — push coming.",
+  });
+  let stats = await assurance.sweepObligations({ now: Date.now() + virtual, deps: deps() });
+  assert.equal(stats.acked, 0);
+  assert.equal(transcript.length, 0, "under ninety seconds, silence is the right answer");
+
+  // ── Two minutes in: ONE line, specific to the ask.
+  virtual = 2 * 60_000;
+  stats = await assurance.sweepObligations({ now: Date.now() + virtual, deps: deps() });
+  assert.equal(stats.acked, 1, "past ACK_AFTER_MS the human gets exactly one line");
+  const ack = transcript.find((m) => m.kind === "ack");
+  assert.ok(ack, "…and it is that line");
+  assert.match(ack.text, /git fixes/i, "bespoke to the actual ask, not a rotation pick");
+  assert.doesNotMatch(ack.text, /^(on it|looking now|checking|one moment|got it|will do)\b/i, "never a generic opener — the sanitiser now enforces the prompt");
+  assert.ok(ack.text.length <= 120, "one short human line, not a templated paragraph");
+
+  // ── Six minutes in — where the progress ping used to land. NOTHING.
+  virtual = 6 * 60_000;
+  stats = await assurance.sweepObligations({ now: Date.now() + virtual, deps: deps() });
+  assert.equal(stats.acked, 0);
+  const progressish = transcript.filter((m) => /still (on this|going)/i.test(m.text));
+  assert.equal(progressish.length, 0, "a message that is a pure function of elapsed minutes cannot carry progress");
+
+  // ── Sixteen minutes: and still nothing, however long it runs.
+  virtual = 16 * 60_000;
+  await assurance.sweepObligations({ now: Date.now() + virtual, deps: deps() });
+  assert.equal(transcript.length, 1, `the whole wait costs the human ONE message, got ${transcript.length}`);
 
   // ── 17 minutes: the session finally answers the human itself.
   virtual = 17 * 60_000;
@@ -251,9 +305,8 @@ test("SCENARIO 2 — a slow ask is acknowledged at once, updated while it runs,
 
   const rec = assurance.readObligation(key);
   assert.equal(rec.state, "answered", "the debt is discharged by evidence of a delivered reply");
-  const progress = transcript.filter((m) => m.kind === "progress");
-  assert.equal(progress.length, 1);
   assert.equal(transcript.filter((m) => m.kind === "ack").length, 1, "acknowledge once, never twice");
+  assert.equal(transcript.filter((m) => m.kind === "progress").length, 0, "and never a progress ping — the branch that emitted them is deleted");
   note("obligation discharged by receipt — the daemon added nothing on top of the session's own answer", virtual);
 });
 
@@ -275,7 +328,10 @@ test("SCENARIO 3 — a timing-out session tells the human what happened and that
     dispatch: (_p, _it, _cr, _s, opts) => { onCloseHook = opts.onClose; },
   });
   const key = assurance.openObligations().find((o) => o.key === item.raw_ref).key;
-  assert.ok(transcript.find((m) => m.kind === "ack"));
+  // Nothing has been said yet — the ask is 45 minutes old in VIRTUAL time only,
+  // and the sweep has not run. What matters for this scenario is that the debt
+  // exists, so the failure has somewhere to be reported from.
+  assert.equal(transcript.length, 0, "silence is the default; the failure notice below is the first thing this human hears");
 
   // 45 minutes in, the dispatcher's SIGTERM lands and the session closes 143.
   const virtual = { v: 45 * 60_000 };