@cohortapp/agent-sdk 2.11.13 → 2.11.14

package/lib/budget-guard.mjs

@@ -1086,3 +1086,99 @@ function formatNotice(band, status) {
   }
   return `Seat spend at ${status.pct}% ($${usd} / $${cap} today) — heads-up only, nothing is degraded yet.${envelope}`;
 }
+
+// ---------------------------------------------------------------------------
+// Rolling usage window (subscription seats: TOKEN VOLUME, not dollars)
+// ---------------------------------------------------------------------------
+//
+// A Max seat has no dollar meter; what it can exhaust is a rolling token window
+// (≈5-hour session + weekly). The exact caps are UNPUBLISHED, so pacing is
+// DEFAULT-OFF: with GOV_WINDOW_5H_TOKENS / GOV_WEEK_TOKENS unset (0),
+// windowBand() is always "ok" and nothing paces — but the rolling volume is
+// still COMPUTED and surfaced so an operator can watch usage climb and calibrate
+// the caps. Reads the SAME cost-tracking ledger dailyStatus uses; "tokens" here
+// is input+output+cache summed over the window. The hard safety net remains
+// rate-guard's usage-limit recognition (recordUsageLimit holds the breaker until
+// reset); this is the proactive, opt-in layer above it.
+
+const WINDOW_5H_MS = 5 * 60 * 60 * 1000;
+const WINDOW_WEEK_MS = 7 * 24 * 60 * 60 * 1000;
+export const WINDOW_5H_TOKEN_CAP = Math.max(0, parseInt(process.env.GOV_WINDOW_5H_TOKENS || "0", 10) || 0);
+export const WINDOW_WEEK_TOKEN_CAP = Math.max(0, parseInt(process.env.GOV_WEEK_TOKENS || "0", 10) || 0);
+export const WINDOW_APPROACH_FRACTION = (() => {
+  const f = parseFloat(process.env.GOV_WINDOW_APPROACH || "0.8");
+  return f > 0 && f < 1 ? f : 0.8;
+})();
+
+function rowTokens(r) {
+  return (
+    (Number(r.input_tokens) || 0) +
+    (Number(r.output_tokens) || 0) +
+    (Number(r.cache_read_input_tokens) || 0) +
+    (Number(r.cache_creation_input_tokens) || 0)
+  );
+}
+
+/**
+ * Rolling token volume + session count over the last 5 hours and 7 days, from
+ * the cost-tracking ledger. Never throws (a missing/unreadable ledger → zeros).
+ */
+export function rollingWindowUsage(deps) {
+  const now = clock(deps)();
+  const dir = ledgerDir(deps);
+  let t5 = 0, s5 = 0, t7 = 0, s7 = 0;
+  let files = [];
+  try {
+    files = readdirSync(dir).filter((f) => /^\d{4}-\d{2}-\d{2}\.jsonl$/.test(f)).sort().slice(-8);
+  } catch {
+    return { window5hTokens: 0, window5hSessions: 0, week7dTokens: 0, week7dSessions: 0 };
+  }
+  for (const f of files) {
+    let body;
+    try { body = readFileSync(join(dir, f), "utf-8"); } catch { continue; }
+    for (const line of body.split("\n")) {
+      if (!line.trim()) continue;
+      let r;
+      try { r = JSON.parse(line); } catch { continue; }
+      const at = r && r.ts ? Date.parse(r.ts) : NaN;
+      if (!Number.isFinite(at)) continue;
+      const dt = now - at;
+      if (dt < 0 || dt > WINDOW_WEEK_MS) continue;
+      const tok = rowTokens(r);
+      t7 += tok; s7 += 1;
+      if (dt <= WINDOW_5H_MS) { t5 += tok; s5 += 1; }
+    }
+  }
+  return { window5hTokens: t5, window5hSessions: s5, week7dTokens: t7, week7dSessions: s7 };
+}
+
+/**
+ * The usage-window band: "ok" | "approaching" | "at". DEFAULT-OFF — an unset
+ * (0) cap yields "ok" for that horizon, so the fleet paces on the window only
+ * once an operator has calibrated GOV_WINDOW_5H_TOKENS / GOV_WEEK_TOKENS. The
+ * band is the WORSE of the 5h and weekly horizons.
+ */
+export function windowBand(deps) {
+  const u = rollingWindowUsage(deps);
+  // Caps come from deps first (tests / a per-seat override), else the env consts.
+  const cap5 = deps && deps.cap5h != null ? deps.cap5h : WINDOW_5H_TOKEN_CAP;
+  const cap7 = deps && deps.capWeek != null ? deps.capWeek : WINDOW_WEEK_TOKEN_CAP;
+  const frac = deps && deps.approachFraction != null ? deps.approachFraction : WINDOW_APPROACH_FRACTION;
+  const bandFor = (used, cap) => {
+    if (!cap || cap <= 0) return "ok";
+    if (used >= cap) return "at";
+    if (used >= cap * frac) return "approaching";
+    return "ok";
+  };
+  const rank = { ok: 0, approaching: 1, at: 2 };
+  const b5 = bandFor(u.window5hTokens, cap5);
+  const b7 = bandFor(u.week7dTokens, cap7);
+  const band = rank[b5] >= rank[b7] ? b5 : b7;
+  return {
+    band,
+    ...u,
+    cap5h: cap5 || null,
+    capWeek: cap7 || null,
+    active: cap5 > 0 || cap7 > 0,
+  };
+}

package/lib/budget-guard.test.mjs

@@ -18,6 +18,8 @@ import {
   readNotified,
   BANDS,
   DEFAULT_DAILY_SPEND_CAP_USD,
+  rollingWindowUsage,
+  windowBand,
 } from "./budget-guard.mjs";
 
 const FIXED = Date.UTC(2026, 5, 9, 12, 0, 0); // 2026-06-09T12:00:00Z
@@ -363,3 +365,63 @@ test("dailyStatus: a subscription-auth seat STILL bands on session volume (a rea
     assert.equal(sub.mode, "degraded");
   } finally { await rm(dir); }
 });
+
+// ---------------------------------------------------------------------------
+// rolling usage window (token volume) + windowBand (default-off pacing)
+// ---------------------------------------------------------------------------
+
+function seedWindowLedger(dir, rows) {
+  // rows: [{ minsAgo, tokens }] written to today's + yesterday's files by ts.
+  const now = FIXED;
+  const byFile = {};
+  for (const r of rows) {
+    const at = new Date(now - r.minsAgo * 60_000);
+    const day = at.toISOString().slice(0, 10);
+    (byFile[day] ||= []).push(JSON.stringify({
+      ts: at.toISOString(),
+      total_cost_usd: 0.1, model: "sonnet", cadence: "x",
+      input_tokens: r.tokens, output_tokens: 0,
+    }));
+  }
+  for (const [day, lines] of Object.entries(byFile)) {
+    writeFileSync(join(dir, `${day}.jsonl`), lines.join("\n") + "\n");
+  }
+}
+
+test("rollingWindowUsage sums tokens/sessions in the 5h and 7d windows", async () => {
+  const dir = await makeLedgerDir();
+  try {
+    seedWindowLedger(dir, [
+      { minsAgo: 10, tokens: 1000 },   // in 5h + 7d
+      { minsAgo: 60, tokens: 2000 },   // in 5h + 7d
+      { minsAgo: 400, tokens: 500 },   // >5h (400m), in 7d
+      { minsAgo: 60 * 24 * 8, tokens: 9999 }, // >7d — excluded
+    ]);
+    const u = rollingWindowUsage({ ledgerDir: dir, now: clk });
+    assert.equal(u.window5hTokens, 3000);
+    assert.equal(u.window5hSessions, 2);
+    assert.equal(u.week7dTokens, 3500);
+    assert.equal(u.week7dSessions, 3);
+  } finally { await rm(dir); }
+});
+
+test("windowBand is DEFAULT-OFF: no caps → always 'ok' however high usage climbs", async () => {
+  const dir = await makeLedgerDir();
+  try {
+    seedWindowLedger(dir, [{ minsAgo: 5, tokens: 10_000_000 }]);
+    const b = windowBand({ ledgerDir: dir, now: clk }); // no caps
+    assert.equal(b.band, "ok");
+    assert.equal(b.active, false);
+    assert.equal(b.window5hTokens, 10_000_000);
+  } finally { await rm(dir); }
+});
+
+test("windowBand bands 'approaching' then 'at' once a 5h cap is calibrated", async () => {
+  const dir = await makeLedgerDir();
+  try {
+    seedWindowLedger(dir, [{ minsAgo: 5, tokens: 8500 }]);
+    assert.equal(windowBand({ ledgerDir: dir, now: clk, cap5h: 10_000 }).band, "approaching"); // 85% of 10k
+    assert.equal(windowBand({ ledgerDir: dir, now: clk, cap5h: 8000 }).band, "at");             // over 8k
+    assert.equal(windowBand({ ledgerDir: dir, now: clk, cap5h: 100_000 }).band, "ok");          // well under
+  } finally { await rm(dir); }
+});

package/lib/resource-governor.mjs

@@ -504,6 +504,16 @@ export function admit(req = {}, deps) {
   if (mode === "suspended" && (source === "backlog" || source === "cadence")) {
     return decide(DEFER, "budget suspended (>=125% of the seat envelope): deferring non-inbox work");
   }
+  // USAGE-WINDOW PACING (default-off; only active once GOV_WINDOW_* caps are
+  // calibrated — see budget-guard.windowBand). At the rolling window's limit,
+  // pause self-directed work so a burst spreads across windows instead of
+  // exhausting one; a direct human reply is never paced. "approaching" is folded
+  // into the concurrency ceiling below (throttle, don't stop). `windowBand`
+  // defaults to "ok" so an uncalibrated seat behaves exactly as before.
+  const windowBand = req.windowBand || "ok";
+  if (windowBand === "at" && !humanReply) {
+    return decide(DEFER, `usage window at limit: deferring ${source} work so it spreads across windows; direct human replies continue`);
+  }
   // NOTE: `degraded` (>=100% / unfunded) no longer hard-stops backlog. Owner
   // instruction (2026-08-25): THROTTLE, don't stop — the seat keeps a thin
   // trickle of self-directed work so autonomous progress never fully halts,
@@ -524,10 +534,14 @@ export function admit(req = {}, deps) {
   // ABOVE the REACTIVE_RESERVE math: reserve keeps steady-state headroom; this
   // clamps to a trickle for the seconds a human is actually waiting.
   const reactiveBusy = numField(d.reactiveInFlight, 0) > 0;
+  // "approaching" the usage window → throttle self-directed work toward a trickle
+  // (like a reactive turn) so the seat glides into the window rather than slamming
+  // it. Default-off: windowBand is "ok" unless GOV_WINDOW_* caps are set.
+  const windowApproaching = windowBand === "approaching";
   let sourceCeiling;
   if (humanReply) {
     sourceCeiling = effectiveMax;
-  } else if (reactiveBusy) {
+  } else if (reactiveBusy || windowApproaching) {
     sourceCeiling = Math.min(effectiveMax, REACTIVE_INFLIGHT_BACKLOG_MAX);
     if (mode === "degraded" && source === "backlog") {
       sourceCeiling = Math.min(sourceCeiling, DEGRADED_BACKLOG_MAX);
@@ -540,6 +554,8 @@ export function admit(req = {}, deps) {
   if (liveCount >= sourceCeiling) {
     const why = reactiveBusy
       ? `reactive turn in flight: self-directed ${source} yields the subscription, capped to ${sourceCeiling} (${liveCount} live)`
+      : windowApproaching
+      ? `usage window approaching: self-directed ${source} throttled to ${sourceCeiling} to spread across windows (${liveCount} live)`
       : mode === "degraded" && source === "backlog"
         ? `budget degraded: self-directed backlog throttled to ${sourceCeiling} (${liveCount} live; inbox unaffected)`
         : throttleCeiling < dMax

package/lib/resource-governor.test.mjs

@@ -180,6 +180,33 @@ test("admit: one backlog session is still allowed alongside a reply (cap 1, not
   assert.equal(r.decision, DECISIONS.ADMIT);
 });
 
+// ---------------------------------------------------------------------------
+// Usage-window pacing (default-off; driven by req.windowBand from budget-guard)
+// ---------------------------------------------------------------------------
+
+test("admit: windowBand 'at' DEFERS self-directed work; a human reply is never paced", () => {
+  assert.equal(admit({ source: "backlog", windowBand: "at" }, deps()).decision, DECISIONS.DEFER);
+  assert.equal(admit({ source: "cadence", windowBand: "at" }, deps()).decision, DECISIONS.DEFER);
+  // inbox / humanReply continue at the window limit
+  assert.equal(admit({ source: "inbox", windowBand: "at" }, deps()).decision, DECISIONS.ADMIT);
+  assert.equal(admit({ source: "backlog", humanReply: true, windowBand: "at" }, deps()).decision, DECISIONS.ADMIT);
+});
+
+test("admit: windowBand 'approaching' throttles backlog to a trickle (QUEUE), inbox unaffected", () => {
+  // 1 already live → backlog capped to REACTIVE_INFLIGHT_BACKLOG_MAX(1) → QUEUE
+  const r = admit({ source: "backlog", windowBand: "approaching" }, deps({ liveClaude: { count: 1, rssMB: 300 } }));
+  assert.equal(r.decision, DECISIONS.QUEUE);
+  assert.match(r.reason, /usage window approaching/);
+  // inbox still admits at the same live count
+  assert.equal(admit({ source: "inbox", windowBand: "approaching" }, deps({ liveClaude: { count: 1, rssMB: 300 } })).decision, DECISIONS.ADMIT);
+});
+
+test("admit: windowBand default (undefined/'ok') changes nothing — pacing is off by default", () => {
+  // 3 live, no window signal → backlog still ADMITs (reserve leaves 5)
+  assert.equal(admit({ source: "backlog" }, deps({ liveClaude: { count: 3, rssMB: 900 } })).decision, DECISIONS.ADMIT);
+  assert.equal(admit({ source: "backlog", windowBand: "ok" }, deps({ liveClaude: { count: 3, rssMB: 900 } })).decision, DECISIONS.ADMIT);
+});
+
 test("admit: throttle ceiling clamps effectiveMax below dynamicMax → QUEUE earlier", () => {
   // dynamicMax 8 but soft-throttle pinned the ceiling to 3; 3 live → QUEUE.
   const r = admit({ source: "backlog" }, deps({ throttleCeiling: 3, liveClaude: { count: 3, rssMB: 1000 } }));

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@cohortapp/agent-sdk",
-  "version": "2.11.13",
+  "version": "2.11.14",
   "description": "Cohort Agent SDK — autonomous AI colleague runtime. Deploy senior AI colleagues on dedicated Mac minis, wired to the Cohort operating surface.",
   "type": "module",
   "bin": {

package/scripts/daemon/cadence-consumer.mjs

@@ -601,8 +601,10 @@ export function startConsumer(opts = {}) {
       // as a human reply, and every cadence tick arrives as `source:"cadence"`.
       // Two gates, one predicate, or the rung silences the inbox on the second
       // hop instead of the first.
+      let windowBand;
+      try { windowBand = budgetGuardModule.windowBand?.({ agentRoot })?.band; } catch { /* default-off */ }
       const adm = governor.admit(
-        { source: "cadence", mode, humanReply: cadence ? isHumanLaneCadence(cadence) : false },
+        { source: "cadence", mode, humanReply: cadence ? isHumanLaneCadence(cadence) : false, windowBand },
         governor.defaultDeps({ agentRoot })
       );
       if (adm.decision !== "ADMIT") return { admit: false, reason: adm.reason || adm.decision };

package/scripts/daemon/dispatcher-governance.test.mjs

@@ -191,11 +191,45 @@ test("reconcile blocks a marker that is older than the freshness window", async
   } finally { await cleanup(dir); }
 });
 
+test("resume reconcile DEFERS (never re-storms) when the backlog budget is 0 (reactive-only seat)", async () => {
+  // DAEMON_MAX_CONCURRENT=3 → backlogCap = 3 - RESERVED_INBOX_SLOTS(3) = 0, so a
+  // reactive-only seat must resume NOTHING on boot — the fresh-seat storm root.
+  process.env.DAEMON_MAX_CONCURRENT = "3";
+  const { mod, dir } = await freshDispatcher();
+  try {
+    plantMarker(dir, "s-a", { itemId: "BL-A", sourceFile: "q.yaml", startedAt: Date.now() });
+    plantMarker(dir, "s-b", { itemId: "BL-B", sourceFile: "q.yaml", startedAt: Date.now() });
+    let spawned = 0;
+    const stats = mod.reconcileResumePending({ spawnResume: () => { spawned++; } });
+    assert.equal(spawned, 0, "no resume spawns on a budget-0 seat");
+    assert.equal(stats.resumed, 0);
+    assert.equal(stats.deferred, 2, "both live markers deferred, not blocked/expired");
+    // markers are LEFT on disk (deferred, not retired) for a later reconcile
+    const left = readdirSync(join(dir, "state/sessions/resume-pending")).filter((f) => f.endsWith(".json"));
+    assert.equal(left.length, 2, "deferred markers persist");
+  } finally { delete process.env.DAEMON_MAX_CONCURRENT; await cleanup(dir); }
+});
+
+test("resume reconcile is GATED (resumes nothing) while the shared breaker is open", async () => {
+  const { mod, dir } = await freshDispatcher();
+  try {
+    mod.setGovernanceForTests({ rateGuard: { checkRateLimit: () => ({ allowed: false, retryAt: Date.now() + 60_000 }) } });
+    plantMarker(dir, "s-x", { itemId: "BL-X", sourceFile: "q.yaml", startedAt: Date.now() });
+    let spawned = 0;
+    const stats = mod.reconcileResumePending({ spawnResume: () => { spawned++; } });
+    assert.equal(spawned, 0, "an open breaker (e.g. a usage-limit hold) blocks all resume");
+    assert.equal(stats.resumed, 0);
+    assert.equal(stats.scanned, 0, "returns before scanning when the breaker is open");
+    // the marker is untouched, ready for the next reconcile after the breaker clears
+    assert.equal(readdirSync(join(dir, "state/sessions/resume-pending")).filter((f) => f.endsWith(".json")).length, 1);
+  } finally { await cleanup(dir); }
+});
+
 test("reconcile on an empty resume-pending dir is a no-op", async () => {
   const { mod, dir } = await freshDispatcher();
   try {
     const stats = mod.reconcileResumePending({ spawnResume: () => { throw new Error("should not spawn"); } });
-    assert.deepEqual(stats, { scanned: 0, resumed: 0, blocked: 0, expired: 0 });
+    assert.deepEqual(stats, { scanned: 0, resumed: 0, blocked: 0, expired: 0, deferred: 0 });
   } finally { await cleanup(dir); }
 });
 

package/scripts/daemon/dispatcher.mjs

@@ -75,7 +75,11 @@ function admitFor(source, priority) {
         console.warn(`[dispatcher] budget ${st.mode}: ${st.pct}% of $${st.capUSD}/day (${st.capSource}) — gating ${source} work`);
       }
     } catch { /* budget read best-effort */ }
-    return governor.admit({ source, priority, mode }, governor.defaultDeps({ agentRoot: AGENT_REPO_DIR }));
+    // Usage-window band (default-off unless GOV_WINDOW_* caps set) → the governor
+    // paces self-directed work as the rolling window fills. Best-effort.
+    let windowBand;
+    try { windowBand = budgetGuard.windowBand?.({ agentRoot: AGENT_REPO_DIR })?.band; } catch { /* */ }
+    return governor.admit({ source, priority, mode, windowBand }, governor.defaultDeps({ agentRoot: AGENT_REPO_DIR }));
   } catch {
     return { decision: "ADMIT", reason: "governor-error-fail-open", snapshot: {} };
   }
@@ -499,13 +503,32 @@ export function resetActiveSessions(opts = {}) {
  */
 export function reconcileResumePending(opts = {}) {
   const now = (typeof opts.now === "function" ? opts.now : Date.now)();
-  const stats = { scanned: 0, resumed: 0, blocked: 0, expired: 0 };
+  const stats = { scanned: 0, resumed: 0, blocked: 0, expired: 0, deferred: 0 };
   let files = [];
   try {
     if (!existsSync(RESUME_PENDING_DIR)) return stats;
     files = readdirSync(RESUME_PENDING_DIR).filter((f) => f.endsWith(".json") && !f.endsWith(".tmp"));
   } catch { return stats; }
 
+  // ADMISSION GATE — resume must honour the same throttle as any backlog spawn,
+  // or a stormed seat's leftover markers bypass it and re-storm on every boot
+  // (the fresh-seat backlog-storm root). Two guards:
+  //  (1) if the shared breaker is open (a usage/rate limit is holding spawns),
+  //      resume NOTHING — resuming would hammer a drained pool the moment it
+  //      re-opens. Markers stay for the next reconcile after the breaker clears.
+  //  (2) cap resumes this pass to the BACKLOG concurrency (in-memory, race-free
+  //      — unlike the governor's ps count). DAEMON_MAX_CONCURRENT=3 →
+  //      RESERVED_INBOX_SLOTS(3) → budget 0 → a reactive-only seat resumes
+  //      nothing on boot (the item stays re-deliverable / the marker persists).
+  try {
+    const rb = rateGuard.checkRateLimit(RATE_PROVIDER, { agentRoot: AGENT_REPO_DIR });
+    if (rb && rb.allowed === false) {
+      logSession({ event: "resume_reconcile_gated", reason: "breaker_open", retry_at: rb.retryAt || null, markers: files.length });
+      return stats;
+    }
+  } catch { /* fail-open — a breaker read error must not strand recovery */ }
+  const resumeBudget = Math.max(0, MAX_CONCURRENT - RESERVED_INBOX_SLOTS - activeSessions.size);
+
   for (const file of files) {
     const path = join(RESUME_PENDING_DIR, file);
     stats.scanned++;
@@ -554,6 +577,15 @@ export function reconcileResumePending(opts = {}) {
       continue;
     }
 
+    // Over the backlog budget for this pass → DEFER (leave the marker, do NOT
+    // bump attempts — a deferral is not a try). The next reconcile picks it up
+    // once slots free / the throttle lifts. This is what stops the re-storm.
+    if (stats.resumed >= resumeBudget) {
+      logSession({ event: "resume_deferred_throttle", sessionId: marker.sessionId, item_id: marker.itemId, budget: resumeBudget });
+      stats.deferred++;
+      continue;
+    }
+
     // Bump the attempt count on the marker BEFORE re-dispatch so a crash mid-
     // resume still advances toward the 3-strike cap (no infinite loop).
     marker.recoveryAttempts = attempts + 1;
@@ -573,7 +605,7 @@ export function reconcileResumePending(opts = {}) {
     }
   }
   if (stats.scanned > 0) {
-    console.log(`[dispatcher] resume reconcile: ${stats.resumed} resumed, ${stats.blocked} blocked, ${stats.expired} expired (of ${stats.scanned})`);
+    console.log(`[dispatcher] resume reconcile: ${stats.resumed} resumed, ${stats.deferred} deferred, ${stats.blocked} blocked, ${stats.expired} expired (of ${stats.scanned})`);
   }
   return stats;
 }