@cohortapp/agent-sdk 2.17.0 → 2.18.5

package/scripts/daemon/cadence-consumer-governance.test.mjs

@@ -1,276 +0,0 @@
-/**
- * cadence-consumer-governance.test.mjs — WS4 coverage for the consumer's
- * resource-governor / 429-breaker admission gate on the escalation path.
- *
- * Proves the two invariants for the cadence path:
- *   (1) A governor DEFER / open breaker REQUEUES the tick (decision:"deferred")
- *       and does NOT call failTick — the event stays in inbox/ with its attempt
- *       count UNCHANGED, so a busy box never DLQs a perfectly good cadence.
- *   (2) When pressure clears, the same event escalates normally — no work lost.
- */
-
-import { test } from "node:test";
-import assert from "node:assert/strict";
-import { promises as fsp } from "node:fs";
-import { writeFileSync, mkdirSync, readFileSync, existsSync } from "node:fs";
-import { tmpdir } from "node:os";
-import { join, dirname } from "node:path";
-
-import { enqueueTick, busDepth, listInbox, getBusPaths } from "../../lib/cadence-bus.mjs";
-import { startConsumer } from "./cadence-consumer.mjs";
-import { obligationAllowedUnderPosture } from "../../lib/plan/compile.mjs";
-import { postureForBand } from "../../lib/budget-guard.mjs";
-import { isHumanLaneCadence } from "../../lib/cadences.mjs";
-
-async function makeAgentRoot() {
-  const path = join(tmpdir(), `consumer-gov-${process.pid}-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`);
-  await fsp.mkdir(path, { recursive: true });
-  return path;
-}
-async function rmRoot(path) { try { await fsp.rm(path, { recursive: true, force: true }); } catch { /* */ } }
-function plantPrompt(root, cadence) {
-  const p = join(root, "schedules/triggers", `${cadence}.md`);
-  mkdirSync(dirname(p), { recursive: true });
-  writeFileSync(p, `# ${cadence}`);
-}
-async function waitFor(pred, { timeoutMs = 5_000, intervalMs = 20 } = {}) {
-  const deadline = Date.now() + timeoutMs;
-  while (Date.now() < deadline) { if (await pred()) return true; await new Promise((r) => setTimeout(r, intervalMs)); }
-  return false;
-}
-
-const DEFER_GOV = { admit: () => ({ decision: "DEFER", reason: "low free memory (test)", snapshot: {} }), defaultDeps: () => ({}) };
-const ADMIT_GOV = { admit: () => ({ decision: "ADMIT", reason: "ok", snapshot: {} }), defaultDeps: () => ({}) };
-const RATE_OK = { checkRateLimit: () => ({ allowed: true, retryAt: null }), classifyStderr: () => false, recordRateLimit: () => {}, recordSuccess: () => {} };
-const RATE_OPEN = { checkRateLimit: () => ({ allowed: false, retryAt: Date.now() + 60_000 }), classifyStderr: () => false, recordRateLimit: () => {}, recordSuccess: () => {} };
-const BUDGET_OK = { dailyStatus: () => ({ essentialOnly: false }) };
-
-// ---------------------------------------------------------------------------
-// governor DEFER → requeue, never failTick, attempts unchanged
-// ---------------------------------------------------------------------------
-
-test("governor DEFER requeues the tick without spawning or burning retry budget", async () => {
-  const root = await makeAgentRoot();
-  plantPrompt(root, "weekly-strategic-memo");
-  let spawnCalls = 0;
-  const consumer = startConsumer({
-    agentRoot: root,
-    pollMs: 30,
-    governor: DEFER_GOV,
-    rateGuard: RATE_OK,
-    budgetGuard: BUDGET_OK,
-    spawnSession: async () => { spawnCalls++; return { ok: true, exit_code: 0 }; },
-  });
-  try {
-    enqueueTick({ cadence: "weekly-strategic-memo", source: "launchd", agentRoot: root });
-    // Let several poll cycles run — under DEFER the event must keep being
-    // requeued, never spawned, never DLQ'd.
-    await waitFor(() => consumer.getStats().retries >= 1);
-    await new Promise((r) => setTimeout(r, 200));
-    const stats = consumer.getStats();
-    assert.equal(spawnCalls, 0, "DEFER must not spawn a sub-session");
-    assert.equal(stats.dlq, 0, "DEFER must never DLQ the cadence");
-    assert.equal(stats.spawn_failures, 0, "DEFER is not a spawn failure");
-    assert.equal(stats.last_decision, "deferred");
-    // The event is back in inbox/ with attempts STILL 0 (no retry budget burned).
-    assert.equal(busDepth(root).inbox, 1, "event requeued to inbox");
-    const id = listInbox(root)[0];
-    const ev = JSON.parse(readFileSync(join(getBusPaths(root).inbox, `${id}.json`), "utf-8"));
-    assert.equal(ev.attempts, 0, "DEFER must not increment attempts (failTick never called)");
-  } finally { await consumer.stop(); await rmRoot(root); }
-});
-
-test("open 429 breaker defers the cadence the same way", async () => {
-  const root = await makeAgentRoot();
-  plantPrompt(root, "weekly-strategic-memo");
-  let spawnCalls = 0;
-  const consumer = startConsumer({
-    agentRoot: root,
-    pollMs: 30,
-    governor: ADMIT_GOV,     // governor would admit…
-    rateGuard: RATE_OPEN,    // …but the shared breaker is open.
-    budgetGuard: BUDGET_OK,
-    spawnSession: async () => { spawnCalls++; return { ok: true, exit_code: 0 }; },
-  });
-  try {
-    enqueueTick({ cadence: "weekly-strategic-memo", source: "launchd", agentRoot: root });
-    await waitFor(() => consumer.getStats().retries >= 1);
-    await new Promise((r) => setTimeout(r, 150));
-    assert.equal(spawnCalls, 0, "rate-limited cadence must not spawn");
-    assert.equal(consumer.getStats().dlq, 0);
-    assert.equal(busDepth(root).inbox, 1);
-  } finally { await consumer.stop(); await rmRoot(root); }
-});
-
-// ---------------------------------------------------------------------------
-// recovery: once pressure clears, the same event escalates (no work lost)
-// ---------------------------------------------------------------------------
-
-test("a deferred tick escalates once the governor admits again (work not lost)", async () => {
-  const root = await makeAgentRoot();
-  plantPrompt(root, "weekly-strategic-memo");
-  let admitNow = false;
-  const flipGov = { admit: () => ({ decision: admitNow ? "ADMIT" : "DEFER", reason: "flip", snapshot: {} }), defaultDeps: () => ({}) };
-  const spawns = [];
-  const consumer = startConsumer({
-    agentRoot: root,
-    pollMs: 30,
-    governor: flipGov,
-    rateGuard: RATE_OK,
-    budgetGuard: BUDGET_OK,
-    spawnSession: async (o) => { spawns.push(o.cadence); return { ok: true, exit_code: 0 }; },
-  });
-  try {
-    enqueueTick({ cadence: "weekly-strategic-memo", source: "launchd", agentRoot: root });
-    // Initially deferred.
-    await waitFor(() => consumer.getStats().retries >= 1);
-    assert.equal(spawns.length, 0);
-    // Pressure clears → next poll escalates the SAME event.
-    admitNow = true;
-    const ran = await waitFor(() => consumer.getStats().escalated >= 1);
-    assert.ok(ran, "the deferred tick must escalate once admitted");
-    assert.equal(spawns[0], "weekly-strategic-memo");
-    assert.equal(busDepth(root).inbox, 0);
-  } finally { await consumer.stop(); await rmRoot(root); }
-});
-
-// ---------------------------------------------------------------------------
-// 429 from the sub-session itself → open breaker + requeue (not failTick)
-// ---------------------------------------------------------------------------
-
-test("a 429 in sub-session stderr opens the breaker and requeues (does not DLQ)", async () => {
-  const root = await makeAgentRoot();
-  plantPrompt(root, "weekly-strategic-memo");
-  let opened = 0;
-  const rate429 = {
-    checkRateLimit: () => ({ allowed: true, retryAt: null }), // breaker closed at gate time
-    classifyStderr: (t) => /429|rate limit|overloaded/i.test(t || ""),
-    recordRateLimit: () => { opened++; return { openUntil: Date.now() + 60_000, consecutive429: 1 }; },
-    recordSuccess: () => {},
-  };
-  const consumer = startConsumer({
-    agentRoot: root,
-    pollMs: 30,
-    maxAttempts: 1, // would DLQ on first failTick — proves we DON'T failTick here
-    governor: ADMIT_GOV,
-    rateGuard: rate429,
-    budgetGuard: BUDGET_OK,
-    spawnSession: async () => ({ ok: false, exit_code: 1, stderr_tail: "HTTP 429 rate limit exceeded", duration_ms: 1 }),
-  });
-  try {
-    enqueueTick({ cadence: "weekly-strategic-memo", source: "launchd", agentRoot: root });
-    await waitFor(() => opened >= 1);
-    await new Promise((r) => setTimeout(r, 120));
-    assert.ok(opened >= 1, "429 stderr must open the shared breaker");
-    assert.equal(consumer.getStats().dlq, 0, "a 429 must NOT DLQ the cadence even at maxAttempts=1");
-    // Event requeued, attempts not bumped (no failTick on the rate path).
-    assert.equal(busDepth(root).inbox, 1);
-    const id = listInbox(root)[0];
-    const ev = JSON.parse(readFileSync(join(getBusPaths(root).inbox, `${id}.json`), "utf-8"));
-    assert.equal(ev.attempts, 0, "rate-limit requeue must not burn retry budget");
-  } finally { await consumer.stop(); await rmRoot(root); }
-});
-
-// ---------------------------------------------------------------------------
-// L2 — a 429 is a SHARED rate-limit signal, NOT evidence this cadence is
-// broken. It must requeue WITHOUT tripping the per-cadence circuit, so a good
-// cadence isn't left circuit-open after the shared breaker clears.
-// ---------------------------------------------------------------------------
-
-test("L2: a 429 sub-session failure does NOT trip the per-cadence circuit", async () => {
-  const root = await makeAgentRoot();
-  plantPrompt(root, "weekly-strategic-memo");
-  let recordedRateLimit = 0;
-  const rate429 = {
-    checkRateLimit: () => ({ allowed: true, retryAt: null }),
-    classifyStderr: (t) => /429|rate limit|overloaded/i.test(t || ""),
-    recordRateLimit: () => { recordedRateLimit++; return { openUntil: Date.now() + 60_000, consecutive429: 1 }; },
-    recordSuccess: () => {},
-  };
-  const consumer = startConsumer({
-    agentRoot: root,
-    pollMs: 30,
-    maxAttempts: 5,
-    // circuitThreshold:1 means a SINGLE recordSubsessionFailure would open the
-    // circuit. The fix must therefore NOT call it on the 429 path — if it did,
-    // one rate-limit would wrongly circuit-open a healthy cadence.
-    circuitThreshold: 1,
-    circuitDurationMs: 60_000,
-    governor: ADMIT_GOV,
-    rateGuard: rate429,
-    budgetGuard: BUDGET_OK,
-    spawnSession: async () => ({ ok: false, exit_code: 1, stderr_tail: "HTTP 429 rate limit exceeded", duration_ms: 1 }),
-  });
-  try {
-    enqueueTick({ cadence: "weekly-strategic-memo", source: "launchd", agentRoot: root });
-    await waitFor(() => recordedRateLimit >= 1);
-    // Give the consumer a beat to (not) write the circuit file.
-    await new Promise((r) => setTimeout(r, 120));
-
-    const stats = consumer.getStats();
-    assert.ok(recordedRateLimit >= 1, "the shared breaker was opened (429 throttles all cadences)");
-    assert.equal(stats.skipped_circuit_open, 0, "the per-cadence circuit must NOT block this cadence");
-    // The SoT the operator/doctor reads: circuit-open.json is written only when a
-    // cadence circuit actually opens. A 429 must not create it.
-    assert.equal(
-      existsSync(join(root, "state/cadence-bus/circuit-open.json")), false,
-      "no per-cadence circuit may open from a shared rate-limit signal",
-    );
-    // And it was requeued (deferred), not DLQ'd.
-    assert.equal(stats.dlq, 0, "429 never DLQs");
-    assert.ok(stats.retries >= 1, "the tick was requeued");
-  } finally { await consumer.stop(); await rmRoot(root); }
-});
-
-test("band 150 does not silence the inbox at EITHER gate", async () => {
-  // Two gates sit between a cadence tick and a spawn, and the 150% rung has to
-  // agree with itself across both:
-  //   1. `obligationAllowedUnderPosture` — was `kind !== "REACT"`, so every
-  //      cadence (all SCHEDULEs) was refused, including `messaging-inbound`, the
-  //      only puller of org DMs into the inbox pipeline.
-  //   2. `resource-governor.admit` — treats only `source:"inbox"` as a human
-  //      reply, and every cadence tick arrives as `source:"cadence"`, so fixing
-  //      gate 1 alone just moves the silence one hop down.
-  const posture = postureForBand(150);
-  const gov = await import("../../lib/resource-governor.mjs");
-  // An IDLE, roomy machine, injected. `defaultDeps` reads real freemem/loadavg/
-  // live-process counts, so asserting against it makes the test a measure of
-  // whatever else is running — this test is about the BUDGET rung, nothing else.
-  const deps = {
-    totalmem: 64 * 1024 ** 3,
-    freemem: 48 * 1024 ** 3,
-    cpus: 10,
-    loadavg: [0.1],
-    liveClaude: { count: 0, rssMB: 0 },
-    throttleCeiling: Infinity,
-  };
-
-  for (const id of ["inbox-processor", "messaging-inbound"]) {
-    assert.equal(
-      obligationAllowedUnderPosture(
-        { kind: "SCHEDULE", mode: "guarded", status: "active", human_lane: isHumanLaneCadence(id) },
-        posture
-      ).allowed,
-      true,
-      `${id} passes the obligation gate`
-    );
-    assert.notEqual(
-      gov.admit({ source: "cadence", mode: posture.mode, humanReply: isHumanLaneCadence(id) }, deps).decision,
-      "DEFER",
-      `${id} is not deferred by the spawn governor either`
-    );
-  }
-
-  // And discretionary cadences ARE still deferred, at both gates.
-  assert.equal(
-    obligationAllowedUnderPosture(
-      { kind: "SCHEDULE", mode: "guarded", status: "active", human_lane: isHumanLaneCadence("backlog-executor") },
-      posture
-    ).allowed,
-    false
-  );
-  assert.equal(
-    gov.admit({ source: "cadence", mode: posture.mode, humanReply: isHumanLaneCadence("backlog-executor") }, deps).decision,
-    "DEFER"
-  );
-});