@cohortapp/agent-sdk 2.17.0 → 2.18.4

package/lib/diagnostics/alerts.mjs

@@ -91,6 +91,16 @@ export const DEFAULT_THRESHOLDS = Object.freeze({
     warnSec: 120, // 2 min (heartbeat refreshes ~15s)
     critSec: 300, // 5 min
   }),
+  cadenceOutputMissing: Object.freeze({
+    // A scheduled workflow finished without leaving its output artefacts behind.
+    // This is the ONLY rule here not derived from the cadence bus, and that is
+    // the point: the evening wrap missed 5 of 9 days in Aug/Sep 2026 and the bus
+    // logged one of those misses as stage="processed". A rule reading the bus's
+    // own decision is green during that outage. See
+    // lib/diagnostics/cadence-output-freshness.mjs.
+    warnCount: 1, // one missing artefact is already a real miss
+    critCount: 2, // both artefacts gone = the workflow did not run at all
+  }),
 });
 
 const SEVERITY_RANK = { critical: 0, warning: 1 };
@@ -243,6 +253,9 @@ function round2(n) {
  *                                   optionally with `perSessionUSD: number[]`
  * @param {object} [args.presence]   `{ heartbeatAgeSec?, daemonHealthAgeSec?,
  *                                       daemonDown? }`
+ * @param {object} [args.outputs]    cadence output verdict from
+ *                                   lib/diagnostics/cadence-output-freshness.mjs
+ *                                   `{ ok, days, checked, missing[] }`
  * @param {object} [args.thresholds] merged thresholds (default DEFAULT_THRESHOLDS)
  * @param {number} [args.now]        clock in epoch ms (reserved; rules are
  *                                   age-based via injected `*AgeSec`)
@@ -254,6 +267,7 @@ export function evaluateAlerts(args = {}) {
     const snapshot = args.snapshot && typeof args.snapshot === "object" ? args.snapshot : {};
     const ledger = args.ledger && typeof args.ledger === "object" ? args.ledger : {};
     const presence = args.presence && typeof args.presence === "object" ? args.presence : {};
+    const outputs = args.outputs && typeof args.outputs === "object" ? args.outputs : {};
     const t = mergeThresholds(DEFAULT_THRESHOLDS, args.thresholds || {});
 
     push(alerts, refusalSpikeRule(snapshot, t.refusalSpike));
@@ -261,6 +275,7 @@ export function evaluateAlerts(args = {}) {
     push(alerts, costPerSessionRule(ledger, t.costPerSessionP99));
     push(alerts, absentAgentRule(presence, t.absentAgent));
     push(alerts, daemonDownRule(presence, t.daemonDown));
+    push(alerts, cadenceOutputMissingRule(outputs, t.cadenceOutputMissing));
 
     // Deterministic order: critical first, then by id, so identical inputs always
     // produce identical output (a de-dupe upstream can key on id).
@@ -365,6 +380,40 @@ function daemonDownRule(presence, cfg = {}) {
   };
 }
 
+/**
+ * A scheduled workflow completed without producing its output artefacts.
+ *
+ * Unlike every other rule here, this one is NOT derived from the lane it judges.
+ * It reads files on disk that the workflow's own steps write, so it stays true
+ * when the cadence bus is wrong about itself — which is the failure this exists
+ * to catch (a timed-out evening wrap recorded as stage="processed", 29 Aug 2026).
+ *
+ * `checked === 0` means nothing was in scope (e.g. every candidate day predates
+ * its cadence). That is silence, not health, so it does not fire.
+ */
+function cadenceOutputMissingRule(outputs, cfg = {}) {
+  const missing = Array.isArray(outputs.missing) ? outputs.missing : null;
+  if (!missing || !num(outputs.checked)) return null; // nothing in scope
+  const count = missing.length;
+  const sev = severityFor(count, num(cfg.warnCount, Infinity), num(cfg.critCount, Infinity));
+  if (!sev) return null;
+
+  // Name the days and the workflow — an alert saying only "2 artefacts missing"
+  // sends the reader back to the bus log, which is exactly where the answer is not.
+  const days = [...new Set(missing.map((m) => m.day))].sort();
+  const labels = [...new Set(missing.map((m) => m.label || m.cadence))].sort();
+  const detail = missing
+    .slice(0, 4)
+    .map((m) => `${m.path} (${m.reason})`)
+    .join("; ");
+  const more = count > 4 ? ` +${count - 4} more` : "";
+  return {
+    id: "cadence_output_missing",
+    severity: sev,
+    detail: `${labels.join(", ")} produced no usable output for ${days.join(", ")}: ${detail}${more}. Checked on disk, independently of the cadence bus — the bus may still report this run as processed.`,
+  };
+}
+
 // ---------------------------------------------------------------------------
 // emitAlerts
 // ---------------------------------------------------------------------------

package/lib/diagnostics/cadence-output-freshness.mjs

@@ -0,0 +1,288 @@
+/**
+ * lib/diagnostics/cadence-output-freshness.mjs — did the cadence actually PRODUCE
+ * anything, or did it merely finish?
+ *
+ * WHY THIS EXISTS
+ *
+ * The daily evening wrap — which its own trigger calls the most important
+ * maintenance workflow — ran on 4 of the 9 days from 26 Aug to 3 Sep 2026. It
+ * missed 25, 29, 30, 31 Aug and 1 Sep, and NOTHING NOTICED, because the five
+ * misses failed four different ways and only two of them look like failure from
+ * the cadence bus:
+ *
+ *   29 Aug  enqueued, claimed 17x, escalated 2x, subsession_timeout,
+ *           subsession_failed, retry_requeued, 15x skipped_backoff — and then
+ *           stage "processed". THE BUS RECORDED SUCCESS FOR A RUN THAT PRODUCED
+ *           NO BRIEF AND NO WORKFLOW LOG.
+ *   30 Aug  attempts=2, routed to the DLQ. Honest — but nothing reads the DLQ.
+ *   31 Aug  ditto.
+ *    1 Sep  one line in the bus log, `enqueued:1`, and nothing further. Never
+ *           claimed at all.
+ *
+ * That first mode is the one that makes this module necessary. Any alert keyed
+ * on the bus's own decision is GREEN during the outage, because the bus cannot
+ * distinguish a completed wrap from a timed-out one — `stage=processed` fired on
+ * a run that wrote nothing. A guard that reads the same field the lane writes
+ * agrees with that lane forever.
+ *
+ * So this check is deliberately derived from a DIFFERENT SOURCE than the lane it
+ * judges: the OUTPUT ARTEFACTS on disk. It asks one question — for the day in
+ * question, do the files this workflow is supposed to leave behind exist, and are
+ * they non-trivial? That was the only signal that caught all four modes, and it
+ * is independent by construction: the bus can say whatever it likes, a brief that
+ * was never written is still absent.
+ *
+ * WHAT "NON-TRIVIAL" MEANS, AND WHY IT IS NOT `run_end`
+ *
+ * The tempting discriminator is a terminal event — assert the workflow log ends
+ * with `{"event":"run_end","status":"completed"}`. MEASURED, THAT IS WRONG: of
+ * the five wraps that genuinely ran and produced full briefs, only ONE
+ * (2026-09-02) carries a `run_end` line at all. Pinning on it would have fired on
+ * 4 of the 5 healthy days — an 80% false-positive rate on the exact signal this
+ * module exists to make trustworthy. A checker that cries wolf on most good days
+ * gets muted, and then it is worth less than nothing.
+ *
+ * The same trap caught a second candidate. Pinning the JSONL on its `workflow`
+ * FIELD VALUE (`workflow == "daily-evening-wrap"`) looks like the careful choice —
+ * it reads a value rather than grepping a token. Replayed over the real corpus it
+ * failed on THREE of the five healthy days: 26 Aug writes `"workflow":
+ * "evening-wrap"` and 27-28 Aug carry no `workflow` key at all, only `step`. The
+ * workflow log is hand-authored per run and HAS NO STABLE INTERNAL SCHEMA. No
+ * assertion about a named event, status or cadence field survives the corpus, so
+ * none of them belongs in a guard.
+ *
+ * What does survive, measured across all five logs: every one of the 61 lines
+ * parses as a JSON object and every one carries a `ts`. So the honest bar is size
+ * plus that schema-agnostic shape — the file must be a JSONL of timestamped
+ * records, not merely a file. Floors are calibrated against what real runs
+ * actually produced (briefs 5115-10906 bytes, logs 8-17 lines / 2722-5459 bytes)
+ * with roughly an order of magnitude of headroom beneath the observed minimum. A
+ * healthy run clears it comfortably; an absent file, an empty file, a truncated
+ * file or a plain-text stub from a session that died mid-run does not.
+ *
+ * The general rule, learned twice here: VALIDATE A CANDIDATE DISCRIMINATOR AGAINST
+ * THE KNOWN-GOOD DAYS BEFORE SHIPPING IT. A check is only as good as its false
+ * positive rate on healthy input, and both of the discriminators that read like
+ * the rigorous choice would have fired on most of the days the workflow worked.
+ *
+ * WHICH DAY, AND WHY UTC
+ *
+ * Artefact filenames are keyed by UTC date. The wrap fires at 18:00 GST = 14:00
+ * UTC, so a run always writes the file for its own UTC day with ~10h of clearance
+ * on either side of the boundary — there is no hour at which "yesterday, UTC" is
+ * racing a run in progress. Checking YESTERDAY rather than today is what keeps
+ * this quiet: today's wrap has not happened yet for most of the day, and its
+ * absence at 09:00 is not a fault.
+ *
+ * `notBefore` exists so a widened lookback cannot claim a miss for a day on which
+ * the cadence did not yet exist. It is a declared constant, NOT derived from the
+ * artefacts — deriving it from the oldest file present would make the check agree
+ * with itself.
+ *
+ * Invariant: NEVER throws. A missing directory, an unreadable file, a bad clock
+ * all degrade to a verdict. Diagnostics must not be able to crash the path that
+ * observes them. But the fail-open direction is chosen per case: when a file
+ * cannot be read at all we report it MISSING, because the bug being fixed is
+ * silence, and the worst case of a wrong alert is one redundant message.
+ *
+ * Constraints (CLAUDE.md): ESM, Node built-ins only, injectable clock + paths so
+ * tests are hermetic and need no real outputs/ or logs/ on disk.
+ *
+ * @module lib/diagnostics/cadence-output-freshness
+ */
+
+import { existsSync, readFileSync, statSync } from "node:fs";
+import { join, resolve } from "node:path";
+
+const MS_PER_DAY = 86_400_000;
+
+/**
+ * What each watched cadence must leave on disk for a day it ran.
+ *
+ * Floors are calibrated against observed healthy runs with ~10x headroom (see the
+ * module header). `jsonLineField` makes the check read a FIELD VALUE rather than
+ * grep for a token, so an empty or corrupt log cannot pass on a coincidental
+ * substring.
+ *
+ * @type {Readonly<object>}
+ */
+export const DEFAULT_OUTPUT_CONTRACTS = Object.freeze({
+  "daily-evening-wrap": Object.freeze({
+    label: "evening wrap",
+    // The cadence's first completed run. Declared, not derived from disk.
+    notBefore: "2026-08-26",
+    artefacts: Object.freeze([
+      Object.freeze({
+        // Written by the brief-writing step.
+        path: "outputs/briefs/{date}-evening-wrap.md",
+        minBytes: 500,
+        minLines: 5,
+      }),
+      Object.freeze({
+        // Written by the run-logging step — a DIFFERENT step, so the two
+        // artefacts corroborate rather than restate one another.
+        path: "logs/workflows/{date}-evening-wrap.jsonl",
+        minBytes: 200,
+        minLines: 3,
+        // Schema-agnostic on purpose: assert only that it IS a JSONL of
+        // timestamped records. Every field richer than this varies run to run
+        // (see header) and would fire on healthy days.
+        minJsonRecords: Object.freeze({ count: 3, requireField: "ts" }),
+      }),
+    ]),
+  }),
+});
+
+/** How many days back to check. 1 = yesterday only (see header: today is not yet due). */
+export const DEFAULT_LOOKBACK_DAYS = 1;
+
+/**
+ * Resolve the agent root the same way every other maestro module does.
+ * @param {object} [cfg]
+ * @returns {string}
+ */
+function agentRoot(cfg = {}) {
+  return resolve(
+    cfg.agentRoot ||
+      process.env.AGENT_ROOT ||
+      process.env.AGENT_DIR ||
+      process.cwd()
+  );
+}
+
+/**
+ * UTC day key `YYYY-MM-DD`, `back` days before `now`. UTC because artefact
+ * filenames are UTC-keyed (module header).
+ * @param {number} now epoch ms
+ * @param {number} back days back
+ * @returns {string}
+ */
+export function dayKeyUTC(now, back = 0) {
+  return new Date(now - back * MS_PER_DAY).toISOString().slice(0, 10);
+}
+
+/**
+ * Inspect one artefact. Returns `null` when it satisfies its contract, else a
+ * reason record. Never throws: an unreadable file is reported as a fault, not
+ * swallowed.
+ * @param {string} root
+ * @param {object} spec
+ * @param {string} day
+ * @returns {null|{path:string, reason:string, bytes:number|null, lines:number|null}}
+ */
+function inspectArtefact(root, spec, day) {
+  const rel = String(spec.path || "").replace(/\{date\}/g, day);
+  const abs = join(root, rel);
+  const fault = (reason, bytes = null, lines = null) => ({ path: rel, reason, bytes, lines });
+
+  if (!existsSync(abs)) return fault("absent");
+
+  let bytes;
+  try {
+    bytes = statSync(abs).size;
+  } catch (err) {
+    return fault(`unstatable: ${err && err.message ? err.message : String(err)}`);
+  }
+
+  const minBytes = Number.isFinite(spec.minBytes) ? spec.minBytes : 0;
+  if (bytes < minBytes) return fault(`stub: ${bytes} bytes < ${minBytes} floor`, bytes);
+
+  let body;
+  try {
+    body = readFileSync(abs, "utf-8");
+  } catch (err) {
+    return fault(`unreadable: ${err && err.message ? err.message : String(err)}`, bytes);
+  }
+
+  const lines = body.split("\n").filter((l) => l.trim().length > 0);
+  const minLines = Number.isFinite(spec.minLines) ? spec.minLines : 0;
+  if (lines.length < minLines) {
+    return fault(`stub: ${lines.length} non-empty lines < ${minLines} floor`, bytes, lines.length);
+  }
+
+  // Structural bar for JSONL: enough PARSEABLE objects each carrying the required
+  // field. Tests parseability and a field's PRESENCE, never a field's name as a
+  // substring and never a value that varies between healthy runs.
+  const jr = spec.minJsonRecords;
+  if (jr && Number.isFinite(jr.count)) {
+    let records = 0;
+    for (const l of lines) {
+      let o;
+      try { o = JSON.parse(l); } catch { continue; }
+      if (!o || typeof o !== "object" || Array.isArray(o)) continue;
+      if (jr.requireField && o[jr.requireField] === undefined) continue;
+      records++;
+    }
+    if (records < jr.count) {
+      const q = jr.requireField ? ` carrying "${jr.requireField}"` : "";
+      return fault(
+        `only ${records} parseable JSON record(s)${q} < ${jr.count} floor`,
+        bytes,
+        lines.length
+      );
+    }
+  }
+
+  return null;
+}
+
+/**
+ * Did the watched cadences produce their output artefacts?
+ *
+ * @param {object}  [cfg]
+ * @param {string}  [cfg.agentRoot]      agent root (default resolved as usual)
+ * @param {object}  [cfg.contracts]      override DEFAULT_OUTPUT_CONTRACTS
+ * @param {number}  [cfg.lookbackDays]   days back to check (default 1 = yesterday)
+ * @param {number}  [cfg.now]            clock in epoch ms (default Date.now())
+ * @returns {{ok:boolean, days:string[], checked:number,
+ *            missing:Array<{cadence:string,label:string,day:string,path:string,reason:string}>}}
+ */
+export function cadenceOutputFreshness(cfg = {}) {
+  const missing = [];
+  const days = [];
+  let checked = 0;
+  try {
+    const root = agentRoot(cfg);
+    const contracts = cfg.contracts && typeof cfg.contracts === "object"
+      ? cfg.contracts
+      : DEFAULT_OUTPUT_CONTRACTS;
+    const lookback = Number.isFinite(cfg.lookbackDays) && cfg.lookbackDays > 0
+      ? Math.floor(cfg.lookbackDays)
+      : DEFAULT_LOOKBACK_DAYS;
+    const now = Number.isFinite(cfg.now) ? cfg.now : Date.now();
+
+    for (let back = 1; back <= lookback; back++) days.push(dayKeyUTC(now, back));
+
+    for (const [cadence, contract] of Object.entries(contracts)) {
+      if (!contract || !Array.isArray(contract.artefacts)) continue;
+      for (const day of days) {
+        // A day before the cadence existed cannot be a miss.
+        if (contract.notBefore && day < contract.notBefore) continue;
+        for (const spec of contract.artefacts) {
+          checked++;
+          const fault = inspectArtefact(root, spec, day);
+          if (fault) {
+            missing.push({
+              cadence,
+              label: contract.label || cadence,
+              day,
+              path: fault.path,
+              reason: fault.reason,
+            });
+          }
+        }
+      }
+    }
+  } catch {
+    // Never throw. Whatever was accumulated before the failure is still valid.
+  }
+
+  // Deterministic order so an upstream de-dupe can key on the list.
+  missing.sort((a, b) =>
+    a.day < b.day ? -1 : a.day > b.day ? 1 : a.path < b.path ? -1 : a.path > b.path ? 1 : 0
+  );
+
+  return { ok: missing.length === 0, days, checked, missing };
+}
+
+export default { cadenceOutputFreshness, dayKeyUTC, DEFAULT_OUTPUT_CONTRACTS, DEFAULT_LOOKBACK_DAYS };