@cohortapp/agent-sdk 2.17.0 → 2.18.5

package/scripts/daemon/execution-ladder.test.mjs

@@ -1,470 +0,0 @@
-/**
- * scripts/daemon/execution-ladder.test.mjs — the daemon actually ENTERS
- * lib/execution/**.
- *
- * Run: node --test scripts/daemon/execution-ladder.test.mjs
- *
- * `lib/execution/**` had exactly one production importer (lib/goals/loop.mjs,
- * for `routeRung`); `processItem` went from ITEM_RECEIVED straight to the Haiku
- * classifier and never entered the ladder. These tests drive the daemon's OWN
- * entry point — `processItem` — with the network/subprocess collaborators
- * injected, and assert on what lands in `state/execution/journal.jsonl`,
- * `state/queues/inbound.yaml` and the fakes. Nothing about intake, match, route,
- * disposition, drive or journal is faked.
- *
- * The two regression tests at the bottom encode bugs a live run of this wiring
- * actually produced:
- *   - with no `need` estimate every DM routed to rung 3, which vetoed the quick
- *     reply and turned a 4-second answer into a spawned Claude session;
- *   - with the rule-based directedness override applied to ALL ignore reasons,
- *     a DM could never be `duplicate` (a DM is always "directed"), so the agent
- *     answered every redelivery twice.
- */
-import { test, before, beforeEach } from "node:test";
-import assert from "node:assert/strict";
-import { register } from "node:module";
-import { mkdtempSync, mkdirSync, writeFileSync, readFileSync, existsSync, rmSync, readdirSync } from "node:fs";
-import { join } from "node:path";
-import { tmpdir } from "node:os";
-
-// --- stub the agent-only `imapflow` dep so the daemon module can be imported ---
-register(
-  "data:text/javascript," +
-    encodeURIComponent(`
-export async function resolve(s, c, next) {
-  if (s === "imapflow") return { url: "imapflow-stub:main", shortCircuit: true };
-  return next(s, c);
-}
-export async function load(u, c, next) {
-  if (u === "imapflow-stub:main") return { format: "module", shortCircuit: true, source: "export class ImapFlow {}" };
-  return next(u, c);
-}
-`),
-);
-
-const AGENT_DIR = mkdtempSync(join(tmpdir(), "maestro-exec-ladder-"));
-process.env.AGENT_DIR = AGENT_DIR;
-process.env.MAESTRO_DAEMON_NO_AUTOSTART = "1";
-process.env.COHORT_AGENT_ID = "mem_self_0001";
-process.env.ANTHROPIC_API_KEY = "";
-
-const PLAN = `version: 1
-enforcement: enforce
-obligations:
-  - key: react.message.dm
-    kind: REACT
-    status: active
-    trigger: { topic: message, kind: null, predicate: null }
-    uses: []
-    allowed_tools: []
-    action_classes: []
-    offline_safe: true
-  - key: react.task.assigned
-    kind: REACT
-    status: active
-    trigger: { topic: task, kind: assigned, predicate: null }
-    uses: []
-    allowed_tools: []
-    action_classes: []
-    offline_safe: true
-`;
-
-let daemon;
-let responder;
-
-before(async () => {
-  mkdirSync(join(AGENT_DIR, "config"), { recursive: true });
-  writeFileSync(
-    join(AGENT_DIR, "config", "agent.json"),
-    JSON.stringify({ firstName: "Nova", lastName: "Vale", slackMemberId: "U09NOVA" }),
-  );
-  writeFileSync(join(AGENT_DIR, "config", "plan.yaml"), PLAN);
-  daemon = await import("./agent-daemon.mjs");
-  responder = await import("./responder.mjs");
-});
-
-beforeEach(() => {
-  for (const rel of [["state", "execution"], ["state", "queues"], ["state", "inbox"], ["state", "locks"], ["logs", "daemon"]]) {
-    rmSync(join(AGENT_DIR, ...rel), { recursive: true, force: true });
-  }
-  if (daemon) daemon._resetLadderCaches();
-  delete process.env.DAEMON_EXECUTION_LADDER;
-});
-
-// --- helpers ---------------------------------------------------------------
-
-function writeLive(item, service) {
-  const dir = join(AGENT_DIR, "state", "inbox", service);
-  mkdirSync(dir, { recursive: true });
-  writeFileSync(join(dir, `${item.id}.json`), JSON.stringify(item));
-}
-
-/** Fakes for exactly the network / subprocess boundary. Nothing else. */
-function spy(classResult) {
-  const seen = { classify: 0, quick: [], dispatch: [], holding: 0 };
-  return {
-    seen,
-    deps: {
-      classify: async () => {
-        seen.classify += 1;
-        return classResult;
-      },
-      sendQuickResponse: async (item, _cr, routed) => {
-        seen.quick.push({ id: item.id, rung: routed ? routed.rung : null });
-        return { sent: true, text: "ack", via: "fake" };
-      },
-      sendHoldingMessage: async () => {
-        seen.holding += 1;
-        return { sent: false, holdingText: null };
-      },
-      dispatch: (_p, item, _cr, _s, opts) => {
-        seen.dispatch.push({ id: item.id, execution: item.execution || null });
-        opts.onClose({ ok: true, code: 0 });
-      },
-    },
-  };
-}
-
-function journal() {
-  const p = join(AGENT_DIR, "state", "execution", "journal.jsonl");
-  if (!existsSync(p)) return [];
-  return readFileSync(p, "utf-8").trim().split("\n").filter(Boolean).map((l) => JSON.parse(l));
-}
-
-const decisions = () => journal().filter((r) => r.event === "decision");
-const outcomes = () => journal().filter((r) => r.event === "outcome");
-
-function dm(over = {}) {
-  return {
-    id: "MSG-DM",
-    raw_ref: "slack:D77:1",
-    channel: "dm/priya",
-    channel_id: "D77",
-    sender: "Priya Raman",
-    sender_id: "U_PRIYA",
-    // A work-request ("review") so this shared fixture takes the full classify →
-    // route path — the fast-path (fastPathClassify) deliberately skips the
-    // classifier only for PLAIN short questions; that path has its own test below.
-    content: "can you review and confirm the deadline for the launch plan?",
-    timestamp: new Date().toISOString(),
-    ...over,
-  };
-}
-
-const RESPOND = {
-  priority: "normal",
-  action: "respond",
-  model: "sonnet",
-  summary: "deadline question",
-  category: "action_required",
-  directed_at_agent: true,
-};
-
-// ---------------------------------------------------------------------------
-// 1) A DM goes THROUGH the ladder — and is still answered exactly as before.
-// ---------------------------------------------------------------------------
-
-test("LADDER: a DM is journalled intake→match→route→outcome AND still quick-replies", async () => {
-  const item = dm();
-  writeLive(item, "slack");
-  const s = spy(RESPOND);
-  await daemon.processItem(item, "slack", s.deps);
-
-  const d = decisions();
-  assert.equal(d.length, 1, "one decision row");
-  assert.equal(d[0].surface, "dm");
-  assert.equal(d[0].disposition, "react_now");
-  assert.equal(d[0].reason, "directed_now");
-  assert.equal(d[0].obligationKey, "react.message.dm", "match bound the DM to the plan's REACT obligation");
-  assert.equal(d[0].rung, 0, "route chose rung 0 — one protocol method with the params in hand");
-  assert.equal(d[0].mechanism, "tool-surface");
-  assert.ok(d[0].why.some((w) => w.includes("single method messaging.send")), "the why records the rung reasoning");
-
-  const o = outcomes();
-  assert.equal(o.length, 1);
-  assert.equal(o[0].effect, "react", "the react effect ran");
-  assert.equal(o[0].ok, true);
-  assert.equal(o[0].spoke, true, "the agent spoke, so the ping-pong chain deepens");
-  assert.equal(o[0].ref.path, "quick_reply", "the effect drove the daemon's real quick-reply path");
-
-  // NO REGRESSION: the classifier ran and the reply was sent.
-  assert.equal(s.seen.classify, 1, "the Haiku classifier still ran for a directed DM");
-  assert.deepEqual(s.seen.quick, [{ id: "MSG-DM", rung: 0 }], "the quick reply was sent, carrying its rung");
-  assert.equal(s.seen.dispatch.length, 0, "no session spawned for a rung-0 reply");
-});
-
-test("LADDER: a plain short question DM FAST-PATHS — quick-replies WITHOUT a classifier call", async () => {
-  const item = dm({ id: "MSG-DM-FAST", raw_ref: "slack:D91:1", channel_id: "D91", content: "what's the deadline?" });
-  writeLive(item, "slack");
-  const s = spy(RESPOND);
-  await daemon.processItem(item, "slack", s.deps);
-
-  // The whole point: the ~5-8s Haiku classify call is SKIPPED for a plain question.
-  assert.equal(s.seen.classify, 0, "fast-path: a plain directed question skips the classifier");
-  // ...but the DM is still answered via the quick-reply path, still journalled.
-  assert.deepEqual(s.seen.quick, [{ id: "MSG-DM-FAST", rung: 0 }], "the quick reply still went out at rung 0");
-  assert.equal(s.seen.dispatch.length, 0, "no session spawned");
-  assert.ok(decisions().some((r) => r.surface === "dm" && r.disposition === "react_now"), "still journalled intake→route→outcome");
-});
-
-// ---------------------------------------------------------------------------
-// 2) A board task is WORK: it queues, and never reaches the classifier.
-// ---------------------------------------------------------------------------
-
-test("LADDER: an assigned board task is queued to state/queues/inbound.yaml, not answered", async () => {
-  const task = {
-    id: "cohort-task_assigned-149",
-    raw_ref: "cohort:task_assigned:cmql3j5j301jwhhv2ipp8gmir:149",
-    service: "cohort",
-    kind: "task_assigned",
-    sender: "Priya Raman",
-    sender_id: "mem_priya_0002",
-    content: "Ship the Q3 revenue reconciliation",
-    channel_id: "board_main",
-    timestamp: new Date().toISOString(),
-  };
-  writeLive(task, "cohort");
-  const s = spy(RESPOND);
-  await daemon.processItem(task, "cohort", s.deps);
-
-  const d = decisions()[0];
-  assert.equal(d.surface, "task_assigned");
-  assert.equal(d.disposition, "schedule");
-  assert.equal(d.reason, "not_respondable", "a board assignment is work, not conversation");
-  assert.equal(d.obligationKey, "react.task.assigned");
-  assert.equal(d.rung, 3, "no lower predicate holds for open work → a bounded session");
-
-  const o = outcomes()[0];
-  assert.equal(o.effect, "schedule");
-  assert.equal(o.ok, true);
-
-  // The queue row is real, and it is the file `sweepBacklog` already reads.
-  const q = readFileSync(join(AGENT_DIR, "state", "queues", "inbound.yaml"), "utf-8");
-  assert.match(q, /id: "inb-board\.task_assigned#149"/);
-  assert.match(q, /rung: 3/);
-  assert.match(q, /obligation_key: "react\.task\.assigned"/);
-  assert.match(q, /why: ".*not a conversational surface/, "the reasoning travels with the work");
-
-  assert.equal(s.seen.classify, 0, "no Haiku call — the ladder placed it without one");
-  assert.equal(s.seen.quick.length, 0);
-  assert.equal(s.seen.dispatch.length, 0);
-});
-
-// ---------------------------------------------------------------------------
-// 3) `ignore` is first-class: journalled, cheap, and it does not call Haiku.
-// ---------------------------------------------------------------------------
-
-test("LADDER: ambient channel chatter is IGNORED, with a journalled reason, and costs no classifier call", async () => {
-  const ambient = {
-    id: "MSG-AMBIENT",
-    raw_ref: "slack:C90:1",
-    channel: "#engineering",
-    channel_id: "C90",
-    sender: "Tomas Lind",
-    sender_id: "U_TOMAS",
-    content: "lunch anyone",
-    timestamp: new Date().toISOString(),
-  };
-  writeLive(ambient, "slack");
-  const s = spy(RESPOND);
-  await daemon.processItem(ambient, "slack", s.deps);
-
-  const d = decisions()[0];
-  assert.equal(d.disposition, "ignore");
-  assert.equal(d.reason, "ambient_channel");
-  assert.ok(d.why.some((w) => w.includes("park for the ambient sweep")), "the ignore explains itself");
-
-  const o = outcomes()[0];
-  assert.equal(o.effect, "none", "an ignore is a completed decision with no effect");
-  assert.equal(o.ok, true, "an ignore is a SUCCESS, not a failure");
-  assert.equal(o.spoke, false);
-
-  assert.equal(s.seen.classify, 0, "an ambient message no longer costs a Haiku call");
-  assert.equal(s.seen.quick.length, 0);
-  assert.equal(s.seen.dispatch.length, 0);
-
-  // It is finished with, not left to be re-polled forever.
-  const files = readdirSync(join(AGENT_DIR, "state", "inbox", "slack"));
-  assert.deepEqual(files, ["MSG-AMBIENT.json.processed"]);
-});
-
-// ---------------------------------------------------------------------------
-// 4) REQUIREMENT: an event no obligation covers still reaches today's path.
-// ---------------------------------------------------------------------------
-
-test("LADDER: an event no obligation covers is still answered, capped at rung ≤1, and proposes the missing obligation", async () => {
-  rmSync(join(AGENT_DIR, "config", "plan.yaml"), { force: true });
-  daemon._resetLadderCaches();
-  try {
-    const item = dm({ id: "MSG-DM-UNCOVERED", raw_ref: "slack:D78:1", channel_id: "D78" });
-    writeLive(item, "slack");
-    const s = spy(RESPOND);
-    await daemon.processItem(item, "slack", s.deps);
-
-    const d = decisions()[0];
-    assert.equal(d.disposition, "react_now", "an uncovered event is still answered");
-    assert.equal(d.obligationKey, null, "…but it is recorded as uncovered");
-    assert.ok(d.rung <= 1, `uncovered work is capped at rung ≤1, got ${d.rung}`);
-    assert.ok(
-      d.why.some((w) => w.includes("no obligation in the compiled plan covers this event")),
-      "the plan gap is stated in the reasoning",
-    );
-    assert.equal(s.seen.quick.length, 1, "the sender still got an answer");
-
-    // §6.1's other half: the gap becomes a proposal, not just a complaint.
-    const proposed = join(AGENT_DIR, "state", "plan", "proposed.jsonl");
-    assert.ok(existsSync(proposed), "an obligation was proposed to cover the gap");
-    assert.match(readFileSync(proposed, "utf-8"), /react\.message\./);
-  } finally {
-    writeFileSync(join(AGENT_DIR, "config", "plan.yaml"), PLAN);
-    daemon._resetLadderCaches();
-  }
-});
-
-// ---------------------------------------------------------------------------
-// 5) REGRESSION (live run): the rule override must not defeat the GUARDS.
-// ---------------------------------------------------------------------------
-
-test("LADDER REGRESSION: a redelivered DM is dropped as `duplicate` and is NOT answered twice", async () => {
-  const item = dm({ id: "MSG-DM-DUP", raw_ref: "slack:D79:1", channel_id: "D79" });
-  writeLive(item, "slack");
-
-  const first = spy(RESPOND);
-  await daemon.processItem(item, "slack", first.deps);
-  assert.equal(first.seen.quick.length, 1, "the first delivery is answered");
-
-  // The SSE push and the poll backstop converge — that convergence is the design.
-  const again = { ...item };
-  writeLive(again, "slack");
-  const second = spy(RESPOND);
-  await daemon.processItem(again, "slack", second.deps);
-
-  const d = decisions();
-  assert.equal(d.length, 2);
-  assert.equal(d[1].disposition, "ignore");
-  assert.equal(d[1].reason, "duplicate", "the dedupe guard fired");
-
-  // The bug this encodes: `isDirectedAtAgent` returns true for ANY DM, so
-  // consulting it on every ignore reason made `duplicate` unreachable and the
-  // agent answered both deliveries.
-  assert.equal(second.seen.quick.length, 0, "the redelivery was NOT answered");
-  assert.equal(second.seen.classify, 0, "…and did not even reach the classifier");
-});
-
-// ---------------------------------------------------------------------------
-// 6) …but a directedness MISS must still fall through. Silence is the expensive
-//    failure, so the ladder never has the last word on "is this mine".
-// ---------------------------------------------------------------------------
-
-test("LADDER: a directedness disagreement falls through to the classifier path rather than gagging the agent", async () => {
-  // Ambient by intake's rules (no mention flag, not a DM, not a thread reply),
-  // but the agent's own name is in the text — which `isDirectedAtAgent` catches
-  // and intake does not.
-  const item = {
-    id: "MSG-NAMED",
-    raw_ref: "slack:C91:1",
-    channel: "#engineering",
-    channel_id: "C91",
-    sender: "Tomas Lind",
-    sender_id: "U_TOMAS",
-    content: "nova can you take the reconciliation?",
-    timestamp: new Date().toISOString(),
-  };
-  writeLive(item, "slack");
-  const s = spy(RESPOND);
-  const gate = await daemon.runExecutionLadder(item, "slack", item.raw_ref, "trace-x", s.deps);
-
-  assert.equal(gate.handled, false, "the ladder did not take responsibility");
-  assert.equal(gate.reason, "rule_override");
-  assert.equal(decisions()[0].disposition, "ignore", "the ignore is still journalled — the disagreement is on record");
-});
-
-// ---------------------------------------------------------------------------
-// 7) The rung is a VETO on the quick path, never a promotion.
-// ---------------------------------------------------------------------------
-
-test("RUNG: the quick-reply responder accepts rungs 0-1 and refuses 2+", async () => {
-  assert.equal(responder.rungPermitsQuickReply(null), true, "no routing decision → today's behaviour");
-  assert.equal(responder.rungPermitsQuickReply({ rung: 0 }), true);
-  assert.equal(responder.rungPermitsQuickReply({ rung: 1 }), true);
-  assert.equal(responder.rungPermitsQuickReply({ rung: 2 }), false);
-  assert.equal(responder.rungPermitsQuickReply({ rung: 3 }), false);
-  assert.equal(responder.rungPermitsQuickReply({ rung: 5 }), false, "no subagent-fanout mechanism exists here");
-
-  // A veto, not a promotion: rung 0 does not make research quick-repliable.
-  assert.equal(responder.isQuickReply({ action: "research", priority: "normal", model: "sonnet" }, { rung: 0 }), false);
-  assert.equal(responder.isQuickReply({ action: "respond", priority: "normal", model: "sonnet" }, { rung: 0 }), true);
-  assert.equal(
-    responder.isQuickReply({ action: "respond", priority: "normal", model: "sonnet" }, { rung: 3 }),
-    false,
-    "a rung-3 answer needs a session even when the classifier called it simple",
-  );
-});
-
-test("ANSWERABLE: a directly-answerable question is quick-replied, not sessioned + acked", () => {
-  // "what machine are you running on?" from the CEO classifies critical/opus,
-  // which would otherwise force a session and a robotic holding ack. Flagged
-  // answerable, it is answered in the turn instead — the defect being fixed.
-  assert.equal(
-    responder.isQuickReply({ action: "respond", priority: "critical", model: "opus", answerable: true }),
-    true,
-    "an answerable question skips the session even at critical/opus",
-  );
-  // The flag only fast-paths a `respond` — a draft/research item stays on the
-  // session path even if the classifier also (contradictorily) flags it.
-  assert.equal(responder.isQuickReply({ action: "draft", priority: "high", model: "opus", answerable: true }), false);
-  assert.equal(responder.isQuickReply({ action: "research", priority: "normal", model: "sonnet", answerable: true }), false);
-  // And the rung veto still wins over answerable.
-  assert.equal(
-    responder.isQuickReply({ action: "respond", priority: "critical", model: "opus", answerable: true }, { rung: 3 }),
-    false,
-    "a rung-3 answer needs a session even when the classifier called it answerable",
-  );
-  // Without the flag, a critical/opus item still takes the session path.
-  assert.equal(responder.isQuickReply({ action: "respond", priority: "critical", model: "opus" }), false);
-});
-
-// ---------------------------------------------------------------------------
-// 8) REGRESSION (live run): with no need estimate every DM routed to rung 3,
-//    which vetoed the quick reply and spawned a session for a two-line message.
-// ---------------------------------------------------------------------------
-
-test("LADDER REGRESSION: the need estimate marks a resolvable reply target as params-known", () => {
-  assert.equal(daemon.ladderNeed({ channel_id: "D1" }).paramsKnown, true, "a resolvable channel → rung 0 is available");
-  assert.equal(daemon.ladderNeed({ raw_ref: "cohort:dm:x:1" }).paramsKnown, true);
-  assert.equal(
-    daemon.ladderNeed({ sender: "someone" }).paramsKnown,
-    false,
-    "no resolvable target → params are NOT known and the ladder walks up to a session that can find it",
-  );
-  assert.equal(daemon.ladderNeed({ channel_id: "D1" }).steps, 1);
-});
-
-// ---------------------------------------------------------------------------
-// 9) The escape hatch preserves the old behaviour exactly — and says so.
-// ---------------------------------------------------------------------------
-
-test("LADDER: DAEMON_EXECUTION_LADDER=0 restores the pre-ladder path and warns about what goes dark", async () => {
-  process.env.DAEMON_EXECUTION_LADDER = "0";
-  daemon._resetLadderCaches();
-  const warnings = [];
-  const realWarn = console.warn;
-  console.warn = (...a) => warnings.push(a.join(" "));
-  try {
-    const item = dm({ id: "MSG-DM-OFF", raw_ref: "slack:D80:1", channel_id: "D80" });
-    writeLive(item, "slack");
-    const s = spy(RESPOND);
-    await daemon.processItem(item, "slack", s.deps);
-
-    assert.equal(journal().length, 0, "nothing is journalled with the ladder off");
-    assert.equal(s.seen.classify, 1, "the classifier path is untouched");
-    assert.equal(s.seen.quick.length, 1, "the DM is still answered");
-    assert.ok(
-      warnings.some((w) => w.includes("DAEMON_EXECUTION_LADDER=0") && w.includes("no record of what was ignored")),
-      "turning it off is never silent",
-    );
-  } finally {
-    console.warn = realWarn;
-    delete process.env.DAEMON_EXECUTION_LADDER;
-  }
-});