@cohortapp/agent-sdk 2.5.0 → 2.6.0

package/bin/maestro.mjs

@@ -30,6 +30,7 @@ import { execFileSync, spawnSync } from "node:child_process";
 import { createHash } from "node:crypto";
 import { homedir } from "node:os";
 import { checkOwnershipSet } from "../lib/fs-ownership.mjs";
+import { summariseRows as summariseCostRows } from "../lib/cost/ledger-row.mjs";
 import { resolveArchetype, migrateLegacyArchetype } from "../lib/archetype.mjs";
 import { runAudit, applyFixes, buildAttestation } from "../lib/security/audit-engine.mjs";
 import { selectProvider } from "../lib/secrets/providers.mjs";
@@ -1448,70 +1449,132 @@ Per-file behaviour:
 // in isolation from disk + the doctor's side-effecting ok/warn/fail output.
 
 /**
- * Sum estimated_usd and count rows from already-parsed ledger rows.
- * Tolerant of missing/NaN estimated_usd (treated as 0) and ignores non-objects.
+ * Fold already-parsed ledger rows into the billable total plus the measurement
+ * provenance the tripwire needs.
+ *
+ * Bills against lib/cost/ledger-row.mjs — the CLI's authoritative
+ * `total_cost_usd` where present, the cache-aware token estimate as a logged
+ * fallback. It used to sum `estimated_usd` blindly, which both under-counted
+ * (no cache tier) and treated an unmeasured session as a free one.
+ *
  * @param {Array<object>} rows
- * @returns {{ rowCount: number, totalUsd: number }}
+ * @returns {{ rowCount:number, totalUsd:number, measured:number, unmeasured:number, nonLlmRows:number, degradations:string[] }}
  */
 function summariseLedgerRows(rows) {
-  let rowCount = 0;
-  let totalUsd = 0;
-  for (const row of rows || []) {
-    if (!row || typeof row !== "object") continue;
-    rowCount++;
-    const usd = Number(row.estimated_usd);
-    if (Number.isFinite(usd)) totalUsd += usd;
-  }
-  return { rowCount, totalUsd: +totalUsd.toFixed(6) };
+  const s = summariseCostRows(rows || []);
+  return {
+    // rowCount stays "LLM session rows" — attribution rows for zero-model work
+    // are not sessions and must not arm a blindness alarm on their own.
+    rowCount: s.sessions,
+    totalUsd: s.measuredUsd,
+    measured: s.measured,
+    unmeasured: s.unmeasured,
+    // WHICH failure produced `unmeasured`: usage unread, or usage read and
+    // unpriceable. Different bugs, different fixes, different sentences.
+    noTokenRows: s.noTokenRows,
+    unpricedRows: s.unpricedRows,
+    nonLlmRows: s.nonLlmRows,
+    degradations: s.degradations,
+  };
 }
 
 /**
- * Decide the cost-telemetry posture from today's ledger rows and a count of
- * sessions known to have run today (rows themselves OR a daemon heartbeat that
- * proves activity). The contract the spec pins down:
+ * Decide the cost-telemetry posture from today's ledger rows and the number of
+ * sessions the daemon logged today.
+ *
+ * The contract, restated so it detects the real defect instead of the clock:
  *
- *   RED   — sessions ran today (rows exist OR heartbeat active) but summed
- *           estimated_usd === 0 → "spend telemetry is blind".
- *   GREEN — no sessions ran (nothing to measure) OR spend > 0.
+ *   RED — sessions ran and we cannot price them. Three distinct ways:
+ *         (a) unmeasured rows exist   — a caller recorded measurement:"unknown"
+ *         (b) sessions ran with NO ledger row at all — spawned but never recorded
+ *         (c) measured rows exist but bill to $0 — pricing itself is broken
+ *   GREEN — no sessions ran (nothing to measure), or every session that ran is
+ *         measured and bills above $0.
+ *
+ * `sessionCount` MUST be evidence that a session actually ran (a logged session
+ * start/completion), NOT a liveness signal. The previous version accepted a
+ * fresh daemon heartbeat, so it went RED every UTC day between midnight and the
+ * first session — an empty ledger plus a live daemon — and printed a fix that
+ * had already shipped. A loud wrong alarm trains the operator to ignore the one
+ * channel meant to be trustworthy, so it is as much a defect as a quiet bug.
  *
  * @param {{ rows?: Array<object>, sessionCount?: number }} input
- *   rows         — parsed ledger rows for today (may be empty)
- *   sessionCount — sessions evidenced today from outside the ledger (e.g. a
- *                  fresh daemon heartbeat). Activity = (rows.length>0) || (sessionCount>0).
- * @returns {{ red: boolean, status: "red"|"green", reason: string, rowCount: number, totalUsd: number, sessionCount: number }}
+ * @returns {{ red:boolean, status:"red"|"green", reason:string, fix:string|null, rowCount:number, totalUsd:number, sessionCount:number, unmeasured:number, unrecorded:number }}
  */
-function evaluateCostTripwire({ rows = [], sessionCount = 0 } = {}) {
-  const { rowCount, totalUsd } = summariseLedgerRows(rows);
+function evaluateCostTripwire({ rows = [], sessionCount = 0, sessionSource = "dispatcher" } = {}) {
+  const { rowCount, totalUsd, measured, unmeasured, noTokenRows, unpricedRows, nonLlmRows } = summariseLedgerRows(rows);
   const sessions = Math.max(0, Number(sessionCount) || 0);
-  const hadActivity = rowCount > 0 || sessions > 0;
 
-  if (!hadActivity) {
+  // LIKE FOR LIKE. `sessionCount` comes from logs/daemon/<date>-sessions.jsonl,
+  // which ONLY the dispatcher writes (`responder` and the cadence consumer have
+  // no logSession). Comparing it against rows from EVERY writer meant the
+  // responder rows this changeset added would mask a dispatcher that had stopped
+  // recording entirely: 35 dispatcher sessions unpriced + 40 responder rows read
+  // as `unrecorded: 0`, status green. Count only the rows from the writer the
+  // session log belongs to.
+  const sourceRows = rows.filter((r) => r && (r.source ?? null) === sessionSource);
+  const sourceSessions = summariseLedgerRows(sourceRows).rowCount;
+  const unrecorded = Math.max(0, sessions - sourceSessions);
+  const base = { rowCount, totalUsd, sessionCount: sessions, unmeasured, unrecorded, sourceRowCount: sourceSessions };
+
+  if (rowCount === 0 && sessions === 0) {
     return {
+      ...base,
       red: false,
       status: "green",
-      reason: "no sessions ran today — nothing to measure",
-      rowCount,
-      totalUsd,
-      sessionCount: sessions,
+      fix: null,
+      reason: nonLlmRows > 0
+        ? `no LLM sessions ran today — nothing to measure (${nonLlmRows} zero-model attribution row(s))`
+        : "no sessions ran today — nothing to measure",
     };
   }
-  if (totalUsd > 0) {
+
+  if (noTokenRows > 0) {
     return {
-      red: false,
-      status: "green",
-      reason: `spend telemetry live ($${totalUsd} across ${rowCount} ledger row(s))`,
-      rowCount,
-      totalUsd,
-      sessionCount: sessions,
+      ...base,
+      red: true,
+      status: "red",
+      reason: `spend telemetry is partially blind — ${noTokenRows}/${rowCount} session row(s) recorded NO token counts (measurement:"unknown"). Their cost is unknown, not zero.`,
+      fix: "check logs/daemon for cost_usage_parse_failed — a caller ran a session but could not read usage from the `claude --output-format json` envelope. Each row's unmeasured_reason names the parse failure.",
+    };
+  }
+
+  if (unpricedRows > 0) {
+    return {
+      ...base,
+      red: true,
+      status: "red",
+      reason: `spend telemetry is blind — ${unpricedRows}/${rowCount} session row(s) bill to $0 despite carrying usage. Pricing is broken, not the day.`,
+      fix: "callers must pass real --input-tokens/--output-tokens (and --cache-read-tokens/--cache-creation-tokens/--total-cost-usd) to scripts/cost/track-claude-usage.mjs, lifted from the `claude --output-format json` envelope. A model missing from the router catalog also lands here.",
+    };
+  }
+
+  if (unrecorded > 0) {
+    return {
+      ...base,
+      red: true,
+      status: "red",
+      reason: `spend telemetry is incomplete — the ${sessionSource} logged ${sessions} session(s) today but only ${sourceSessions} ${sessionSource} row(s) reached the cost ledger; ${unrecorded} ran unpriced.`,
+      fix: "a spawn path is not calling scripts/cost/track-claude-usage.mjs at all. Every path that spawns `claude` must record a row — measured, or explicitly --tokens-unknown.",
     };
   }
+
+  if (measured > 0 && totalUsd <= 0) {
+    return {
+      ...base,
+      red: true,
+      status: "red",
+      reason: `spend telemetry is blind — ${measured} measured session(s) bill to $0. Pricing is broken, not the day.`,
+      fix: "callers must pass real --input-tokens/--output-tokens (and --cache-read-tokens/--cache-creation-tokens/--total-cost-usd) to scripts/cost/track-claude-usage.mjs, lifted from the `claude --output-format json` envelope.",
+    };
+  }
+
   return {
-    red: true,
-    status: "red",
-    reason: `spend telemetry is blind — ${hadActivity ? "sessions ran today" : ""} but ledger sums $0 (${rowCount} row(s)${sessions ? `, heartbeat-active` : ""}). Callers are recording 0 tokens; budget governor can never trip on spend.`,
-    rowCount,
-    totalUsd,
-    sessionCount: sessions,
+    ...base,
+    red: false,
+    status: "green",
+    fix: null,
+    reason: `spend telemetry live ($${totalUsd} across ${measured} measured session(s) today${unmeasured ? `, ${unmeasured} unmeasured` : ""})`,
   };
 }
 
@@ -1942,13 +2005,34 @@ async function doctor() {
       }
     } catch { /* breaker read best-effort */ }
 
-    // Today's budget band.
+    // Today's budget band + the enforcement posture it implies.
+    //
+    // The band is measured against the seat's hq-FUNDED envelope
+    // (Employee.payBasis.meteredBudget → mandate body seatBudgetCents), with the
+    // local cap as a safety net. An unfunded seat is reported as exactly that —
+    // "no ceiling is being enforced" is a finding, not a clean bill of health.
     try {
       const { dailyStatus } = await import(join(MAESTRO_ROOT, "lib", "budget-guard.mjs"));
       const b = dailyStatus({ agentRoot: cwd });
-      if (b.essentialOnly) { warn(`Daily budget cap reached ($${b.spentUSD}/$${b.capUSD}) — essential-only (inbox replies continue; backlog+cadence deferred)`); issues++; }
-      else if (b.band >= 75) { warn(`Daily budget at ${b.band}% ($${b.spentUSD}/$${b.capUSD})`); }
-      else ok(`Daily budget band ${b.band}% ($${b.spentUSD}/$${b.capUSD})`);
+      const where = `$${b.spentUSD.toFixed(2)}/$${b.capUSD.toFixed(2)} today (${b.pct}%, ${b.capSource})`;
+      const monthly = b.envelope && b.envelope.monthUSD != null
+        ? `; envelope $${b.envelope.monthUSD.toFixed(2)}/mo, MTD $${b.month.spentUSD.toFixed(2)} (${b.monthPct}%)`
+        : "";
+      if (b.mode === "refused") { fail(`Budget REFUSED — ${where}${monthly}: spawns other than direct human replies are deferred`); issues++; }
+      else if (b.mode === "suspended") { warn(`Budget SUSPENDED — ${where}${monthly}: OUTCOME obligations suspended; inbox + offline-safe cadences continue`); issues++; }
+      else if (b.mode === "degraded") { warn(`Budget DEGRADED — ${where}${monthly}: cheapest model class, fan-out 1, self-directed work stopped`); issues++; }
+      else if (b.band >= 80) { warn(`Budget at ${b.band}% — ${where}${monthly} (supervisor notified; nothing degraded yet)`); }
+      else ok(`Budget band ${b.band}% — ${where}${monthly}`);
+      if (!b.funded) {
+        warn(
+          "Seat is UNFUNDED: hq published no seatBudgetCents, so the org's ceiling is not being enforced — " +
+          "the number above is this seat's own config/recovery.yaml. Set Employee.payBasis.meteredBudget for this member in Cohort."
+        );
+        issues++;
+      }
+      if (!b.supervisorMemberId) {
+        warn("No supervisor edge on the mandate — budget escalations have nowhere to go, and are NOT sent to the owner by design");
+      }
     } catch { /* ledger read best-effort */ }
 
     // Cost-telemetry tripwire (observability F2). Read today's cost ledger and
@@ -1967,25 +2051,36 @@ async function doctor() {
           try { rows.push(JSON.parse(line)); } catch { /* skip malformed row */ }
         }
       }
-      // Heartbeat as out-of-ledger session evidence: a fresh consumer heartbeat
-      // (<5m) means the daemon is live and spawning, so $0 spend is suspicious
-      // even before the first ledger row lands.
-      let heartbeatActive = 0;
+      // Out-of-ledger session evidence: sessions the daemon LOGGED as completed
+      // today. This deliberately replaced a liveness heartbeat. A fresh
+      // heartbeat only proves the daemon is up, so "heartbeat + empty ledger"
+      // fired RED every UTC day between midnight and the first session and
+      // printed a fix that had already shipped — an alarm that cries wolf daily
+      // is worse than no alarm, because the operator learns to skip it.
+      // A logged completion is proof a session actually ran and therefore that
+      // a ledger row is genuinely owed.
+      let loggedSessions = 0;
       try {
-        const health = JSON.parse(readFileSync(join(cwd, "state/cadence-bus/health.json"), "utf-8"));
-        const ageMs = Date.now() - new Date(health.ts).getTime();
-        if (Number.isFinite(ageMs) && ageMs >= 0 && ageMs < 5 * 60_000) heartbeatActive = 1;
-      } catch { /* no heartbeat — rely on ledger rows alone */ }
+        const sessionLog = join(cwd, "logs/daemon", `${ledgerDate}-sessions.jsonl`);
+        if (existsSync(sessionLog)) {
+          for (const line of readFileSync(sessionLog, "utf-8").split("\n")) {
+            if (!line.trim()) continue;
+            try {
+              if (JSON.parse(line).event === "completed") loggedSessions++;
+            } catch { /* skip malformed line */ }
+          }
+        }
+      } catch { /* no session log — rely on ledger rows alone */ }
 
-      const tripwire = evaluateCostTripwire({ rows, sessionCount: heartbeatActive });
+      // `sessionSource` names the writer the session log belongs to, so the two
+      // populations being compared are the same population.
+      const tripwire = evaluateCostTripwire({ rows, sessionCount: loggedSessions, sessionSource: "dispatcher" });
       if (tripwire.red) {
         fail(`Cost telemetry BLIND — ${tripwire.reason}`);
-        fail(`  Fix: callers must pass real --input-tokens/--output-tokens to scripts/cost/track-claude-usage.mjs (lift usage from the claude --output-format json envelope).`);
+        if (tripwire.fix) fail(`  Fix: ${tripwire.fix}`);
         issues++;
-      } else if (tripwire.rowCount === 0 && heartbeatActive === 0) {
-        ok("Cost telemetry: no sessions recorded today (nothing to measure)");
       } else {
-        ok(`Cost telemetry live ($${tripwire.totalUsd} across ${tripwire.rowCount} ledger row(s) today)`);
+        ok(`Cost telemetry: ${tripwire.reason}`);
       }
     } catch { /* cost ledger read best-effort */ }
 
@@ -2325,35 +2420,37 @@ async function doctor() {
     }
   }
 
-  // ── Backup configured + fresh (DR-config, gaps-enterprise-ops G16) ──────
-  // RED when backup is unconfigured: at fleet scale a machine with no DR target
-  // is a silent data-loss risk. Previously doctor exited 0 on a missing backup
-  // target; now it FAILs so the gap surfaces. GREEN only when a target is
-  // configured AND a recent (<25h) successful run is recorded.
-  const backupCfg = join(cwd, ".maestro/backup-config.yaml");
-  if (existsSync(backupCfg)) {
-    const cfg = readFileSync(backupCfg, "utf-8");
-    if (/enabled:\s*true/.test(cfg)) {
-      const lastBackup = join(cwd, ".maestro/last-backup.json");
-      if (existsSync(lastBackup)) {
-        try {
-          const lb = JSON.parse(readFileSync(lastBackup, "utf-8"));
-          const age = Date.now() - new Date(lb.completed_at).getTime();
-          if (age < 25 * 60 * 60 * 1000) ok(`Backup configured + fresh: last run ${Math.round(age / 3600000)}h ago (${lb.provider}://${lb.bucket})`);
-          else { warn(`Last backup ${Math.round(age / 3600000)}h ago — investigate scripts/maintenance/backup-to-cloud.sh`); issues++; }
-        } catch { warn("last-backup.json is malformed"); }
-      } else {
-        warn("Backup enabled but no successful run yet — schedule scripts/maintenance/backup-to-cloud.sh");
-        issues++;
-      }
-    } else {
-      fail("Backup target present but disabled (.maestro/backup-config.yaml: enabled is not true).");
-      console.log("        fix: set enabled: true and schedule scripts/maintenance/backup-to-cloud.sh");
-      issues++;
+  // ── Backup posture (DR, gaps-enterprise-ops G16) ────────────────────────
+  // The verdict is NOT computed here — lib/backup/policy.mjs owns it, shared
+  // with the nightly-backup cadence and the runner, because three call sites
+  // each deriving their own idea of "configured" is how they drifted apart.
+  //
+  // The ladder it applies (see backupVerdict):
+  //   fail — no config / unparseable / disabled / nothing would be archived
+  //   fail — enabled but no run has EVER succeeded (there is no restore point)
+  //   warn — restore points are stale, OR they are local-only (a dead machine
+  //          still loses everything — named in words, not left implied)
+  //   ok   — fresh AND off-machine
+  //
+  // "local-only" is a WARN and not a FAIL on purpose: a check that is red on
+  // every machine in the fleet is a check nobody reads, and "no restore point
+  // at all" is a genuinely worse state than "restore points that would not
+  // survive a house fire". Doctor now tells you which one you have.
+  try {
+    const { resolveBackupPlan, backupVerdict } = await import("../lib/backup/policy.mjs");
+    const { backupFreshness } = await import("../lib/diagnostics/backup-freshness.mjs");
+    const plan = resolveBackupPlan({ agentRoot: cwd });
+    const verdict = backupVerdict(plan, backupFreshness({ agentRoot: cwd }));
+    if (verdict.level === "ok") ok(verdict.msg);
+    else if (verdict.level === "warn") { warn(verdict.msg); issues++; }
+    else { fail(verdict.msg); issues++; }
+    if (verdict.fix) console.log(`        fix: ${verdict.fix}`);
+    for (const v of plan.violations || []) {
+      warn(`  backup include path "${v.path}" is DROPPED by the credential deny-list (${v.pattern}) — it will never be archived.`);
     }
-  } else {
-    fail("No backup target configured — fleet machines need a DR backup (.maestro/backup-config.yaml missing).");
-    console.log("        fix: create .maestro/backup-config.yaml (enabled: true, provider, bucket) and schedule scripts/maintenance/backup-to-cloud.sh");
+  } catch (err) {
+    // Fail-open, never silent: a doctor probe must not take doctor down.
+    warn(`Backup posture could not be evaluated (${err && err.message}) — treat as unbacked until this resolves.`);
     issues++;
   }