@open-everywhere/core 0.1.0

package/agent/routines/fire-audit.ts

@@ -0,0 +1,823 @@
+// Routine FORENSICS — the durable answer to "did my 03:00 routine actually run, and what did it
+// do?" (OPE-145). Before this module the entire dispatcher trail was `console.log`, and the only
+// durable traces a fire could leave were a `reply.post` row IF it posted (no routine id) and a
+// `usage_events` row (no routine id) — so a fire that stayed silent, was declined, or was lost in
+// delivery left ZERO rows, and one that DID post could only be attributed by correlating surface +
+// timestamp by hand.
+//
+// WHY THIS IS ALLOWED TO WRITE TO AN ACTIONS-ONLY LOG. The audit table is deliberately
+// ACTIONS-ONLY (audit/audit.ts auditSilencesEnabled — the "audit every silence" proposal was
+// superseded), and a fire is arguably a mechanism rather than an action. It gets rows anyway on one
+// narrow ground: an autonomous fire is the ONE class of work with no human in the room, so there is
+// no conversation to reconstruct it from — the row is not a record OF an action, it is the only
+// thing that makes "did it run?" answerable at all. Everything else here is kept to that bar:
+//
+//   • The 10-minutely tick writes NOTHING when nothing is due (schedules/dispatch.ts returns
+//     early). The tick is NOT the volume driver — the operator's own routines are, and every
+//     `routine.fire` row corresponds to a real model turn whose token cost dwarfs the row.
+//   • A DEFERRED routine (over the per-tick concurrency cap) writes nothing: it keeps its
+//     nextRunAt, stays due, and fires on a later tick. Not firing YET is not a skip.
+//   • Every "it didn't run" row goes through {@link auditRoutineSkip}, which is THROTTLED to one
+//     row per (routine, reason) per {@link SKIP_WINDOW_MS} with a carried `suppressed` count —
+//     the same shape (and the same rationale) as webhooks/reject-audit.ts. That matters most on
+//     the EVENT leg, which is driven by inbound webhook deliveries rather than by our own clock:
+//     an unbounded delivery stream must not become an unbounded stream of chained, per-org-
+//     serialised audit inserts. It also stops a permanently-misconfigured routine from writing
+//     144 identical rows a day off the clock leg.
+//
+// RECONCILED VOCABULARY. OPE-143 landed `routine.event_skipped` for the event fan-out's over-cap
+// drop. That is the same species of fact as the clock leg's over-budget decline and either leg's
+// unresolvable target — "a routine that was due/matched was NOT dispatched, for a reason" — so
+// there is ONE action, `routine.skip`, carrying `reason`, rather than a second skip vocabulary
+// alongside the first. The old name is gone; `reason: "fanout_cap"` is the event-cap case.
+//
+// The actions this module writes, in the existing `<subject>.<verb>` shape (audit vocabulary:
+// routine.create, routine.cancel, routine.complete, routine.expire):
+//
+//   routine.fire       a fired session ENDED. The ONLY row that can carry BOTH the routine id and
+//                      the session id — the dispatcher never learns the session id (`receive()`
+//                      does not return one), so this is written from hooks/routine-outcome.ts, the
+//                      one place holding the marker AND `ctx.session.id`. THIS IS THE JOIN: it is
+//                      what makes GET /control/v1/session/:id/transcript reachable from a routine.
+//   routine.skip       due/matched, NOT dispatched, with a machine-readable `reason`. Throttled.
+//   routine.fire_lost  dispatched, but `receive()` REJECTED — no session ever started, so there is
+//                      no session id to carry and the row says so. Without it this case is
+//                      indistinguishable from "never due".
+//   routine.disabled   the routine self-disabled (consecutive-failure threshold). Rare by
+//                      construction — at most once per routine, ever.
+//   routine.stalled    dispatched, a session ran, and it was REFUSED a consequential tool by a
+//                      configured spend checkpoint (OPE-253). Neither a fire that failed nor a fire
+//                      that worked, and previously the ONLY fire outcome that wrote nothing anywhere
+//                      after the first one (`checkpoint.trip` is written once per session, so a
+//                      durable job's 2nd..200th blocked fire left no trace at all).
+//   routine.eligibility_unresolved
+//                      a creator-eligibility check could NOT answer, so it did not block this fire
+//                      (OPE-1810). Throttled. Says the fire was not CHECKED, not that it ran — a
+//                      later gate on the tick can still decline it. The row exists because a
+//                      fail-open and a real pass otherwise leave the same trace, which is none.
+
+import { appendAudit, readAudit, type AuditInput } from "../audit/audit.js";
+import type { TeamsMembershipUnresolved } from "../channels/teams-bridge.js";
+import { getOperationalStore } from "../operational/provider.js";
+import { usageSummary } from "../spend/usage-events.js";
+import { isSilentRoutine, type Routine } from "./routine.js";
+
+/** The routine-forensics audit actions. Values are the stable dotted keys readers join on. */
+export const ROUTINE_AUDIT = {
+  /** A fired session ended — carries routineId + sessionId + outcome (THE forensic join). */
+  fire: "routine.fire",
+  /** Due/matched but NOT dispatched, with a reason. Throttled. */
+  skip: "routine.skip",
+  /** Dispatch attempted, delivery rejected — no session started, so no sessionId exists. */
+  fireLost: "routine.fire_lost",
+  /** Self-disabled after the consecutive-failure threshold. */
+  disabled: "routine.disabled",
+  /**
+   * OPE-253: this fire was BLOCKED — a configured session cost checkpoint refused it a consequential
+   * tool (spend/checkpoint.ts requireSessionContinuation). One row per stalled FIRE, written the
+   * moment it happens rather than folded onto that fire's `routine.fire` row at session end, for two
+   * reasons: a job that cannot act is exactly the one whose session may not end cleanly, and the row
+   * is the only immediate signal in a story whose whole defect is silence.
+   *
+   * NOT throttled, unlike {@link ROUTINE_AUDIT.skip}: it is already at most once per fire (the
+   * dispatcher clears `lastStallAt` at fire, and the recorder writes only on the first refusal of a
+   * fire), and fires are at least DISPATCH_TICK_MINUTES apart — so the natural rate is already the
+   * one a throttle would impose, and one row per blocked run is exactly the resolution a reader
+   * asking "how long has this been stuck?" needs.
+   */
+  stalled: "routine.stalled",
+  /**
+   * OPE-147: a human asked for an OFF-SCHEDULE run (`run_routine_now`). Written at REQUEST time,
+   * by the requesting turn, so it is the one row that carries the REQUESTER — `routine.fire` is
+   * stamped with the synthetic {@link ROUTINE_DISPATCHER_PRINCIPAL} and structurally cannot say
+   * who asked. The run itself still produces its ordinary `routine.fire` row (carrying
+   * `manual: true`) when the fired session ends, so `fires7d` counts runs, not requests, and a
+   * request that never becomes a run — cancelled, over budget, target unresolvable — is visibly a
+   * request with no fire rather than a silently-missing one.
+   */
+  runNow: "routine.run_now",
+  /**
+   * OPE-151: one or more of this fire's declared PRE-RUN COLLECTORS did not collect — refused for a
+   * permission the routine does not hold, unknown to this deployment's config, or broken at the read
+   * (routines/collector.ts). The fire STILL HAPPENED, with the failure named in its own message, so
+   * this row is what makes "the run saw nothing" and "the run could not look" different facts after
+   * the event as well as during it.
+   *
+   * THROTTLED, exactly like {@link ROUTINE_AUDIT.skip} and for the same reason. It is at most once
+   * per FIRE, and on the CLOCK leg fires are at least DISPATCH_TICK_MINUTES apart — but the pre-fire
+   * step runs on the EVENT fan-out too (routines/pre-fire.ts), once per matched delivery, and a
+   * delivery stream has no such spacing. A routine whose collector is permanently refused would
+   * otherwise turn a 200-delivery push burst into 200 chained audit inserts.
+   *
+   * The detail carries collector NAMES and closed-set REASONS, plus the recorded CAUSE of a read
+   * that threw — external text, capped by routines/collector.ts COLLECTOR_DETAIL_MAX_CHARS, which
+   * reaches this row and no model. Never a byte of a collected BODY.
+   */
+  collectFailed: "routine.collect_failed",
+  /**
+   * OPE-1810: this fire's CREATOR-ELIGIBILITY room check could not answer, so the fire proceeded on
+   * the ruled fail-open. The row names which check and why it could not answer.
+   *
+   * 🔴 IT EXISTS BECAUSE A FAIL-OPEN AND A PASS OTHERWISE LEAVE THE SAME TRACE — none. The Teams
+   * room check addressed Graph with an id Graph does not take, so every fire took the fail-open arm
+   * silently and the leg read as a healthy, enforcing check for its whole life. An operator asking
+   * "is the creator check working?" now gets an answer from the rows THEMSELVES: a row means that
+   * fire was NOT checked, and its `suppressed` and `windowMs` fields say how many further fires in
+   * the window it stands for.
+   *
+   * 🔴 READ THE PRESENCE, NEVER THE ABSENCE. A fire with no row was not necessarily checked, and
+   * three arms make that so: this row is THROTTLED (below), so a standing unresolvable check writes
+   * one row an hour and the fires between it write none; the write is `.catch`-ed at the call site,
+   * so a store failure drops the row and keeps the fire; and schedules/dispatch.ts resolves a
+   * throwing eligibility probe to `{ eligible: true }`, which fires with no row at all.
+   * `docs/deploy/surfaces/teams.md` states the same direction for the operator.
+   *
+   * THROTTLED, exactly like {@link ROUTINE_AUDIT.skip} and for the same reason: the honest shape of
+   * this row is a standing condition — a missing Graph consent, a team whose group id will not
+   * resolve — which would otherwise write one row per fire, forever, saying the same thing.
+   */
+  eligibilityUnresolved: "routine.eligibility_unresolved",
+} as const;
+
+/** Why a due/matched routine was not dispatched. Closed set — readers switch on it. */
+export type RoutineSkipReason =
+  /** Event fan-out only: past the per-delivery fan-out cap, and an event has no "later". */
+  | "fanout_cap"
+  /**
+   * Either leg (OPE-275): the scope's spend cap is reached (spend/check.ts scopeOverBudget).
+   * On the clock leg the schedule still advances, so the routine retries at its own cadence; on
+   * the event fan-out the delivery is simply skipped — an event has no "later" — and the routine
+   * fires again the moment its scope is back under budget. A skip, never a strike: being over
+   * budget is the deployment's state, not the routine's mistake.
+   */
+  | "over_budget"
+  /** Either leg: no wired channel/target for the routine's surface (dry-run, fail-closed). */
+  | "unresolved_target"
+  /**
+   * Event fan-out only (OPE-163): the STORED filter no longer validates against its source's live
+   * normalizer — the honest case is a `matchKeys` shrink across a deploy. Treated as NO MATCH, and
+   * deliberately NOT as a routine failure: config drift is not the routine's fault, and a strike
+   * here would let one config change 3-strike-disable a whole fleet via lifecycle.ts.
+   */
+  | "filter_invalid"
+  /**
+   * Event fan-out only (OPE-163): the filter matched but the READ GRANT said no — this partition's
+   * effective access does not grant read on the scope the delivery proved (routines/event-authz.ts).
+   * Also covers the fail-closed cases: no grant adapter for the source, an empty scope path, and an
+   * unresolvable config. Audited rather than silent because "matches but never fires" is otherwise
+   * indistinguishable from "never matches", and both look like a dead routine.
+   */
+  | "grant_refused"
+  /**
+   * Event fan-out only (OPE-178): OUR grant said yes and the VENDOR said no — a 401/403 hydrating
+   * the entity the delivery pointed at. That is GRANT DRIFT: access changed after the grant was
+   * written, so the two authorities now disagree and only the vendor's answer is authoritative.
+   *
+   * 🔴 IT IS A SKIP, NOT A FAILURE, AND THAT IS LOAD-BEARING. Routed here rather than through
+   * {@link auditRoutineFireLost} precisely because a skip can never fold onto the routine: if a
+   * hydration 403 counted toward lifecycle.ts's three-strike self-disable, a repo the agent lost
+   * access to would silently disable every routine watching it — a permissions change becoming
+   * permanent, invisible loss of automation, blamed on the routine. Losing access is the
+   * DEPLOYMENT's state change, not the routine's mistake, so the routine stays armed and starts
+   * working again by itself the moment access returns.
+   */
+  | "grant_drift"
+  /**
+   * Event fan-out only (OPE-166): the read grant said yes, and the ROOM said no — the routine's own
+   * channel resolved to an OBSERVED `guest` audience floor, so org-internal trigger-sourced content
+   * may not be disclosed into it (routines/fire-disclosure.ts). "May the domain read this" and "is
+   * this room a fit audience for it" are different questions; this row is the second one's answer.
+   *
+   * A SKIP, not a failure, for the same load-bearing reason as `grant_drift` above: a channel
+   * becoming externally shared is the DEPLOYMENT's state change, not the routine's mistake, and
+   * folding it onto the routine would let one Slack Connect invite three-strike-disable every
+   * routine in the room. The routine stays armed and resumes by itself when the room is internal
+   * again.
+   *
+   * Uniquely among these reasons it is ALSO announced in the room (a content-free, host-composed
+   * line), because a silent withhold on an UNATTENDED path is exactly the "why didn't it tell me"
+   * failure the audit wedge exists to prevent. The announcement rides THIS row's throttle — it is
+   * posted only when {@link auditRoutineSkip} actually emitted — so there is one visibility
+   * mechanism with one rate limit, not two.
+   */
+  | "audience_floor"
+  /**
+   * Clock leg only (OPE-506, routines/fire-eligibility.ts): the routine's CREATOR is no longer a
+   * member of the room the fire posts into — Tag's own policy, adopted verbatim ("stop firing if
+   * the creator is removed from the channel"). A SKIP, never a failure, and deliberately different
+   * from the creator LEAVING THE ORG (which does not stop the routine at all — it fires on the
+   * room's own authority): this is the narrower, room-scoped removal.
+   */
+  | "creator_removed_from_room"
+  /**
+   * Clock leg only (OPE-506): the routine's CREATOR's account is deactivated/deleted outright
+   * (Slack `users.info.deleted`) — a stronger signal than losing internal-member standing, checked
+   * independently of room membership. A SKIP: the room itself is untouched, so a later fire (once
+   * a different creator's standing question no longer applies, or the account is restored) can
+   * still succeed.
+   */
+  | "creator_account_inactive"
+  /**
+   * Either leg (OPE-1602): a BREAK-GLASS HALT covers this routine's address, so the fire is skipped
+   * (routines/fire-halt.ts). A SKIP, never a failure, for the same load-bearing reason `over_budget`
+   * is one: a halt is the DEPLOYMENT's posture, not the routine's mistake, and folding it onto the
+   * routine would let one incident three-strike-disable every routine in the org. The routine stays
+   * armed and fires again by itself once an admin runs `resume`.
+   */
+  | "halted"
+  /**
+   * Either leg (OPE-151): the routine's declared CONDITION GATE said no — the predicate read this
+   * run's pre-run collection and the condition it guards is not true, so no session was created and
+   * nothing was spent (routines/gate.ts). The row carries the gate's name and the one line the
+   * predicate gave, which is the only place a human learns WHY a routine that looks healthy has not
+   * run.
+   *
+   * A SKIP, never a failure, and here that is the mechanism rather than a mercy: a gate exists
+   * precisely so a routine can decline to run, so folding a decline onto `consecutiveFailures` would
+   * three-strike-disable every routine that uses one correctly. It is also the reason an
+   * UNRESOLVABLE gate lands here — the failing-closed arms (an unknown name, a throwing predicate)
+   * write this same row, so a broken gate reports itself hourly instead of stopping a routine in
+   * silence.
+   *
+   * Throttled with every other skip reason ({@link SKIP_WINDOW_MS}): a routine whose condition is
+   * false all day writes one row an hour carrying a `suppressed` count, not one per tick.
+   */
+  | "condition_gate";
+
+/** At most ~one skip row per (routine, reason) per window. An hour: long enough that a standing
+ *  misconfiguration reports hourly rather than every tick/delivery, short enough that the count
+ *  stays legible. */
+export const SKIP_WINDOW_MS = 60 * 60_000;
+
+/**
+ * The routine's own partition segments, for an audit row about it. A routine's surfaceKey is
+ * `platform/workspace/surface` — the same split schedules/dispatch.ts partitionOf uses. (This
+ * replaces the local copy github-fanout.ts carried: one shape, one place.)
+ */
+export function routineAuditScope(r: Pick<Routine, "surfaceKey">): { platform: string; workspace: string; surface: string } {
+  const [platform = "", workspace = "", ...rest] = r.surfaceKey.split("/");
+  return { platform, workspace, surface: rest.join("/") };
+}
+
+/** Principal stamped on a routine-lifecycle row. A fire has NO acting human — the dispatcher is
+ *  the actor — so this is a synthetic, low-cardinality service identity, never the creator (who
+ *  did not act at fire time). The event leg passes its own (`github:webhook`). */
+export const ROUTINE_DISPATCHER_PRINCIPAL = "routine:dispatcher";
+
+/** Injectable edges — live defaults are appendAudit + the operational store's markEventSeen. */
+export interface RoutineAuditDeps {
+  readonly audit?: (entry: AuditInput) => Promise<void>;
+  readonly markSeen?: (org: string, key: string, atIso: string, ttlMs: number) => Promise<boolean>;
+  readonly now?: () => Date;
+  /** Overrides the synthetic dispatcher principal (the event leg stamps `github:webhook`). */
+  readonly principalId?: string;
+}
+
+/** Suppressed-since-last-row counts, per (routine, reason). In-process by design, exactly as
+ *  webhooks/reject-audit.ts argues: a durable counter would reintroduce the per-event write the
+ *  throttle exists to remove, and each serverless instance reporting its own share is the same
+ *  granularity the throttle itself has. */
+const suppressed = new Map<string, number>();
+
+/** Test seam — drop all in-process suppression counters. */
+export function resetRoutineSkipThrottle(): void {
+  suppressed.clear();
+}
+
+/** The throttle identity. Both parts are OE-minted (`r-xxxxxxxx` from routines/store.ts
+ *  newRoutineId, and a closed reason set) — never a caller-supplied string, so a hostile delivery
+ *  stream cannot mint fresh keys to defeat the throttle (webhooks/reject-audit.ts §"THE THROTTLE
+ *  KEY MUST NOT CONTAIN AN ATTACKER-CHOSEN STRING"). */
+export function skipThrottleKey(routineId: string, reason: RoutineSkipReason): string {
+  return `routineskip:${routineId}:${reason}`;
+}
+
+/** The throttle identity for {@link auditRoutineEligibilityUnresolved} (OPE-1810). Same
+ *  construction as every other key here: an OE-minted routine id plus a closed-set check name, so no
+ *  caller-supplied string can mint fresh keys and defeat the throttle. Keyed on the CHECK rather
+ *  than the cause, so a probe that alternates between two causes still reports once per window
+ *  instead of twice. */
+function eligibilityUnresolvedThrottleKey(routineId: string, check: EligibilityCheckName): string {
+  return `routineeligibility:${routineId}:${check}`;
+}
+
+/** The throttle identity for {@link auditCollectorFailures} (OPE-151). Same construction and the
+ *  same reason: the only variable part is an OE-minted routine id, so a hostile delivery stream
+ *  cannot mint fresh keys to defeat the throttle. */
+export function collectFailedThrottleKey(routineId: string): string {
+  return `routinecollect:${routineId}`;
+}
+
+/**
+ * Has this key already emitted a row inside {@link SKIP_WINDOW_MS}? Returns true when a row SHOULD
+ * be written, and counts a suppressed one onto the next emitted row otherwise.
+ *
+ * FAIL-OPEN on a throttle-store error — emit the row. The throttle and the audit write hit the SAME
+ * store, so a markEventSeen failure means the append is about to fail harmlessly anyway.
+ *
+ * Extracted so the skip row and the collector-failure row cannot drift into two throttles with two
+ * window lengths and two suppression counters (OPE-151).
+ */
+async function admitThrottled(org: string, key: string, nowIso: string, deps: RoutineAuditDeps): Promise<number | null> {
+  let first = true;
+  try {
+    const markSeen =
+      deps.markSeen ?? ((o: string, k: string, at: string, ttl: number) => getOperationalStore().markEventSeen(o, k, at, ttl));
+    first = await markSeen(org, key, nowIso, SKIP_WINDOW_MS);
+  } catch {
+    first = true;
+  }
+  if (!first) {
+    suppressed.set(key, (suppressed.get(key) ?? 0) + 1);
+    return null;
+  }
+  const carried = suppressed.get(key) ?? 0;
+  suppressed.delete(key);
+  return carried;
+}
+
+async function write(entry: AuditInput, deps: RoutineAuditDeps): Promise<void> {
+  // Best-effort by contract (appendAudit never throws), but an injected audit in a test might —
+  // and a forensics row must never break the fire it is recording.
+  await (deps.audit ?? appendAudit)(entry).catch(() => undefined);
+}
+
+/**
+ * A due/matched routine was NOT dispatched. THROTTLED: returns true when a row was emitted, false
+ * when it was suppressed (counted onto the next emitted row's `suppressed` field, so the throttle
+ * never hides its own existence).
+ *
+ * FAIL-OPEN on a throttle-store error — emit the row. The throttle and the audit write hit the
+ * SAME store, so a markEventSeen failure means the append is about to fail harmlessly anyway;
+ * failing closed would instead silence genuine skip auditing on any transient blip.
+ */
+export async function auditRoutineSkip(
+  r: Routine,
+  reason: RoutineSkipReason,
+  extra: Readonly<Record<string, unknown>> = {},
+  deps: RoutineAuditDeps = {},
+): Promise<boolean> {
+  const nowIso = (deps.now?.() ?? new Date()).toISOString();
+  const carried = await admitThrottled(r.org, skipThrottleKey(r.id, reason), nowIso, deps);
+  if (carried === null) return false;
+  await write(
+    {
+      org: r.org,
+      ...routineAuditScope(r),
+      principalId: deps.principalId ?? ROUTINE_DISPATCHER_PRINCIPAL,
+      action: ROUTINE_AUDIT.skip,
+      detail: {
+        routineId: r.id,
+        reason,
+        ...extra,
+        // Present only when this row stands in for others — its absence means "exactly one".
+        ...(carried > 0 ? { suppressed: carried, windowMs: SKIP_WINDOW_MS } : {}),
+      },
+    },
+    deps,
+  );
+  return true;
+}
+
+/** Which creator-eligibility check could not answer. A closed set — readers switch on it, and it is
+ *  what keeps one platform's fail-open distinguishable from another's. */
+export type EligibilityCheckName = "teams_room_membership";
+
+/** Why it could not answer. Typed as the PROBE's own closed set rather than as `string`, so the
+ *  vocabulary the docs promise a reader is the vocabulary the seam actually admits — a later
+ *  platform leg adds its causes to a union here, never free text at the call site. */
+export type EligibilityUnresolvedCause = TeamsMembershipUnresolved;
+
+/**
+ * THE CREATOR CHECK DID NOT BLOCK THIS FIRE (OPE-1810): a creator-eligibility check could not
+ * answer, and the ruling is that eligibility does not stop the fire on that arm. The row says the
+ * fire was not really CHECKED, never that it ran — a later gate on the same tick (the spend cap)
+ * can still decline it, and that decline writes its own `routine.skip`. `cause` is the probe's own closed-set reason (the Teams
+ * leg's `TeamsMembershipUnresolved`), so an operator reads WHY from the row — a missing Graph
+ * consent and a roster too large to finish are different problems with different remedies.
+ *
+ * Written only when a check was actually ATTEMPTED and came back unresolved. A fire whose creator
+ * was confirmed present writes nothing, and neither does a platform with no check at all, so the
+ * row's presence means "this fire was not really checked" rather than "this fire ran".
+ *
+ * Fail-soft and throttled like every other row here: recording that a check went unanswered must
+ * never break the fire it is recording, and a standing misconfiguration must not write a row a tick.
+ */
+export async function auditRoutineEligibilityUnresolved(
+  r: Routine,
+  detail: { readonly check: EligibilityCheckName; readonly cause: EligibilityUnresolvedCause },
+  deps: RoutineAuditDeps = {},
+): Promise<boolean> {
+  const nowIso = (deps.now?.() ?? new Date()).toISOString();
+  const carried = await admitThrottled(r.org, eligibilityUnresolvedThrottleKey(r.id, detail.check), nowIso, deps);
+  if (carried === null) return false;
+  await write(
+    {
+      org: r.org,
+      ...routineAuditScope(r),
+      principalId: deps.principalId ?? ROUTINE_DISPATCHER_PRINCIPAL,
+      action: ROUTINE_AUDIT.eligibilityUnresolved,
+      detail: {
+        routineId: r.id,
+        check: detail.check,
+        cause: detail.cause,
+        // Present only when this row stands in for others — its absence means "exactly one".
+        ...(carried > 0 ? { suppressed: carried, windowMs: SKIP_WINDOW_MS } : {}),
+      },
+    },
+    deps,
+  );
+  return true;
+}
+
+/**
+ * A fire's PRE-RUN COLLECTORS did not all collect (OPE-151). Written once per fire, before the
+ * delivery, and only when at least one collector failed — a fire whose reads all worked writes
+ * nothing, so this row's existence is the signal.
+ *
+ * 🔴 NAMES AND REASONS ONLY, NEVER A BODY. A collected body is external content; the failure reason
+ * is host-composed prose about a read that did not happen (routines/collector.ts
+ * describeCollectorFailure), so it carries and the body never could. The same names/ids-only
+ * discipline `routine.fire_attrs_sent` and `turn.tool_gated` keep.
+ *
+ * Fail-soft like every other row here: recording that a read broke must never break the fire.
+ */
+export async function auditCollectorFailures(
+  r: Routine,
+  failures: readonly { readonly name: string; readonly failure: string; readonly detail?: string }[],
+  deps: RoutineAuditDeps = {},
+): Promise<boolean> {
+  if (failures.length === 0) return false;
+  const nowIso = (deps.now?.() ?? new Date()).toISOString();
+  const carried = await admitThrottled(r.org, collectFailedThrottleKey(r.id), nowIso, deps);
+  if (carried === null) return false;
+  await write(
+    {
+      org: r.org,
+      ...routineAuditScope(r),
+      principalId: deps.principalId ?? ROUTINE_DISPATCHER_PRINCIPAL,
+      action: ROUTINE_AUDIT.collectFailed,
+      detail: {
+        routineId: r.id,
+        failed: failures.map((f) => ({ collector: f.name, reason: f.failure, ...(f.detail ? { cause: f.detail } : {}) })),
+        count: failures.length,
+        // Present only when this row stands in for others — its absence means "exactly one".
+        ...(carried > 0 ? { suppressed: carried, windowMs: SKIP_WINDOW_MS } : {}),
+      },
+    },
+    deps,
+  );
+  return true;
+}
+
+/** THE JOIN. A fired session ended: this row names the routine AND the session, so a human (or the
+ *  agent itself, via list_routines) can go from "r-1a2b3c4d" to that fire's transcript.
+ *
+ *  `silent` (OPE-149) is stamped for a `report:{kind:"silent"}` routine, and it is what makes this
+ *  row LOAD-BEARING rather than merely useful. A silent routine posts nothing by design, so it can
+ *  never appear in the `lastPostedAt` join below — without the flag, a healthy silent fire and a
+ *  posting routine that has gone quiet look identical in the forensics, and "did the nightly job
+ *  run this week?" has no answer at all. Positive-only, so a posting routine's row is unchanged. */
+export async function auditRoutineFire(
+  r: Routine,
+  sessionId: string,
+  outcome: string,
+  extra: Readonly<Record<string, unknown>> = {},
+  deps: RoutineAuditDeps = {},
+): Promise<void> {
+  await write(
+    {
+      org: r.org,
+      ...routineAuditScope(r),
+      principalId: deps.principalId ?? ROUTINE_DISPATCHER_PRINCIPAL,
+      action: ROUTINE_AUDIT.fire,
+      detail: { routineId: r.id, sessionId, outcome, ...(isSilentRoutine(r) ? { silent: true, reportKind: "silent" } : {}), ...extra },
+    },
+    deps,
+  );
+}
+
+/** A fire whose `receive()` was REJECTED. No session started, so there is no session id — the row
+ *  says so explicitly rather than omitting the field and reading as "we forgot to record it". */
+export async function auditRoutineFireLost(r: Routine, error: string, deps: RoutineAuditDeps = {}): Promise<void> {
+  await write(
+    {
+      org: r.org,
+      ...routineAuditScope(r),
+      principalId: deps.principalId ?? ROUTINE_DISPATCHER_PRINCIPAL,
+      action: ROUTINE_AUDIT.fireLost,
+      detail: {
+        routineId: r.id,
+        // Bounded: the rejection text comes from an adapter/vendor, so it is not ours to trust as
+        // short. Ids and counts only otherwise — no prompt, no message text.
+        error: error.slice(0, 300),
+        sessionId: null,
+        note: "delivery rejected — no session started, so no sessionId exists for this fire",
+      },
+    },
+    deps,
+  );
+}
+
+/**
+ * THIS FIRE WAS BLOCKED (OPE-253) — a configured session cost checkpoint refused it a consequential
+ * tool. `tool` is the OE tool name the guard was called with (host-supplied, from a closed set of
+ * authored tool slugs — never model text); `reason` is the checkpoint's own host-composed line
+ * ("this session reached its cost checkpoint (…tokens)"), and `stalledFires` is the run this fire
+ * extends, so the row answers "is this the first time, or the third?" without a second read.
+ *
+ * There is no `sessionId` argument on purpose. The guard runs at a tool execute, which is inside
+ * `turnStep` — the same process that has `ctx.session.id` — but wiring it in would make the recorder's
+ * signature depend on the guard's ctx shape, and the `routine.fire` join row written at session end
+ * already carries the (routineId, sessionId) pair for this same fire. One join, one row that owns it.
+ */
+export async function auditRoutineStalled(
+  r: Routine,
+  detail: { readonly tool: string; readonly reason: string; readonly stalledFires: number },
+  deps: RoutineAuditDeps = {},
+): Promise<void> {
+  await write(
+    {
+      org: r.org,
+      ...routineAuditScope(r),
+      principalId: deps.principalId ?? ROUTINE_DISPATCHER_PRINCIPAL,
+      action: ROUTINE_AUDIT.stalled,
+      detail: {
+        routineId: r.id,
+        tool: detail.tool,
+        // Host-composed by spend/checkpoint.ts from the session's own numbers, but bounded anyway on
+        // the standing rule that a forensic row is not the place to discover a surprisingly long string.
+        reason: detail.reason.slice(0, 300),
+        stalledFires: detail.stalledFires,
+      },
+    },
+    deps,
+  );
+}
+
+/**
+ * A human asked for an off-schedule run (OPE-147). `requestedBy` is a REAL principal — the whole
+ * reason this row exists separately from `routine.fire`, which can only ever name the dispatcher.
+ * `nextRunAt` is the instant the request moved the routine to, so the row also answers "when will
+ * it actually happen" without a second lookup.
+ */
+export async function auditRoutineRunNow(
+  r: Routine,
+  requestedBy: string,
+  nextRunAt: string,
+  deps: RoutineAuditDeps = {},
+): Promise<void> {
+  await write(
+    {
+      org: r.org,
+      ...routineAuditScope(r),
+      principalId: requestedBy,
+      action: ROUTINE_AUDIT.runNow,
+      detail: { routineId: r.id, requestedBy, nextRunAt, trigger: r.trigger.kind },
+    },
+    deps,
+  );
+}
+
+/** The routine self-disabled (consecutive-failure threshold). At most once per routine, ever.
+ *
+ *  `reason` is the HOST-composed `disabledReason` ("auto-disabled after 3 consecutive failures") —
+ *  it says the routine hit the threshold, not what was wrong. `extra` is where the answer to the
+ *  second question goes: OPE-177 passes `declaredFailure`, the final run's own words for why the
+ *  work did not happen, so the one row a human reads after "why did this stop?" carries a diagnosis
+ *  rather than only a count. Kept as a separate field, never merged into `reason`, because `reason`
+ *  is also what routines/disable-notice.ts POSTS INTO THE ROOM and it must stay host-composed. */
+export async function auditRoutineDisabled(
+  r: Routine,
+  reason: string,
+  consecutiveFailures: number,
+  sessionId: string | undefined,
+  extra: Readonly<Record<string, unknown>> = {},
+  deps: RoutineAuditDeps = {},
+): Promise<void> {
+  await write(
+    {
+      org: r.org,
+      ...routineAuditScope(r),
+      principalId: deps.principalId ?? ROUTINE_DISPATCHER_PRINCIPAL,
+      action: ROUTINE_AUDIT.disabled,
+      detail: { routineId: r.id, reason, consecutiveFailures, ...(sessionId ? { sessionId } : {}), ...extra },
+    },
+    deps,
+  );
+}
+
+// --- the READ side: turning those rows back into an answer -----------------------------------
+// Writing the rows is only half of OPE-145. The other half is that the CONVERSATIONAL view
+// (tools/list_routines.ts) was poorer than both the operator CLI and the MCP `routines_list`,
+// which already showed consecutiveFailures/disabledReason — backwards for a product whose whole
+// thesis is "the control plane is where people already are". This reader is what lets the agent
+// answer "did the 03:00 one run, what did it cost, and where's the transcript?" in the room.
+
+/** The seven-day forensic window every routine-history read uses. Long enough to cover a daily
+ *  routine's week and a weekly routine's last run; short enough that the underlying scans stay
+ *  cheap and well inside the usage_events retention floor (90 days). */
+export const FORENSIC_WINDOW_MS = 7 * 24 * 60 * 60_000;
+
+/** Row ceiling on each audit scan behind {@link readRoutineForensics}. Counts are therefore "over
+ *  at most the newest N rows in the window" — stated, not silently approximated. Generous enough
+ *  that a realistic surface never reaches it (a 15-minute routine fires 672 times a week). */
+export const FORENSIC_SCAN_LIMIT = 5000;
+
+/** What the durable rows can now say about one routine. Every field is null/0 when the routine
+ *  simply has not run in the window — which is itself the answer to "did it run?". */
+export interface RoutineForensics {
+  /** Session id of the most recent COMPLETED fire — the handle for
+   *  `GET /control/v1/session/:id/transcript`. Null if none in the window. */
+  readonly lastSessionId: string | null;
+  /** When that fire's session ended (ISO). */
+  readonly lastFireAt: string | null;
+  /** When a fire of this routine most recently POSTED anything (ISO). Distinguishes "ran and had
+   *  nothing to say" from "ran and told you" — joined through `reply.post`'s sessionId, which is
+   *  the only link that row carries. */
+  readonly lastPostedAt: string | null;
+  /** Completed fires in the window. Zero, with a nextRunAt already in the past, is the shape of
+   *  "it never ran" — the question that had no answer before this ticket. */
+  readonly fires7d: number;
+  /** Every token attributed to this routine in the window (input + output + both cache legs) —
+   *  the full model cost, not a billable subset. */
+  readonly tokens7d: number;
+}
+
+const EMPTY_FORENSICS: RoutineForensics = { lastSessionId: null, lastFireAt: null, lastPostedAt: null, fires7d: 0, tokens7d: 0 };
+
+/** The zero reading — a routine with no durable trace in the window. */
+export function emptyRoutineForensics(): RoutineForensics {
+  return EMPTY_FORENSICS;
+}
+
+function detailString(detail: Readonly<Record<string, unknown>> | undefined, key: string): string | undefined {
+  const v = detail?.[key];
+  return typeof v === "string" && v.length > 0 ? v : undefined;
+}
+
+/**
+ * Read the durable trace for every routine on ONE surface, keyed by routine id.
+ *
+ * SCOPING: every read is pinned to the caller's own (org, platform, workspace, surface) — the
+ * partition the caller already resolved. This function never widens, so it cannot be used to see
+ * another surface's or another org's routines. That is why extending list_routines with it does not
+ * touch the cross-org exposure OPE-146 item 3 tracks: that one is `routines_list` in
+ * control/server.ts over `listAllRoutines()`, a different code path with a different reader.
+ *
+ * FAIL-SOFT: any store error degrades to an EMPTY map rather than throwing — a forensics read must
+ * never break the tool that is merely decorating its output with it.
+ */
+export async function readRoutineForensics(
+  scope: {
+    readonly org: string;
+    readonly platform: string;
+    readonly workspace: string;
+    readonly surface: string;
+    readonly surfaceKey: string;
+  },
+  now: Date = new Date(),
+): Promise<Map<string, RoutineForensics>> {
+  const out = new Map<string, RoutineForensics>();
+  const since = new Date(now.getTime() - FORENSIC_WINDOW_MS).toISOString();
+  const base = {
+    org: scope.org,
+    platform: scope.platform,
+    workspace: scope.workspace,
+    surface: scope.surface,
+    since,
+    limit: FORENSIC_SCAN_LIMIT,
+  };
+  try {
+    // 1. The fires themselves (newest-first, per readAudit's contract).
+    const fires = await readAudit({ ...base, action: ROUTINE_AUDIT.fire });
+    const acc = new Map<string, { lastSessionId: string | null; lastFireAt: string | null; fires: number; sessions: Set<string> }>();
+    for (const row of fires) {
+      const id = detailString(row.detail, "routineId");
+      if (id === undefined) continue;
+      let a = acc.get(id);
+      if (!a) {
+        a = { lastSessionId: null, lastFireAt: null, fires: 0, sessions: new Set<string>() };
+        acc.set(id, a);
+      }
+      a.fires++;
+      const sid = detailString(row.detail, "sessionId");
+      if (sid !== undefined) a.sessions.add(sid);
+      // Newest-first ⇒ the FIRST row seen for an id is its latest fire.
+      if (a.lastFireAt === null) {
+        a.lastFireAt = row.ts;
+        a.lastSessionId = sid ?? null;
+      }
+    }
+
+    // 2. "Did it actually say anything?" — `reply.post` carries a sessionId but NO routine id
+    //    (channels/reply-post.ts ReplyAudit), so the join runs the other way: newest-first, the
+    //    first post whose session belongs to one of a routine's fires is that routine's last post.
+    const posted = new Map<string, string>();
+    if (acc.size > 0) {
+      const posts = await readAudit({ ...base, action: "reply.post" });
+      for (const row of posts) {
+        const sid = detailString(row.detail, "sessionId");
+        if (sid === undefined) continue;
+        for (const [id, a] of acc) {
+          if (!posted.has(id) && a.sessions.has(sid)) posted.set(id, row.ts);
+        }
+      }
+    }
+
+    // 3. Cost, off the routine_id dimension this same ticket added to usage_events.
+    const tokens = new Map<string, number>();
+    const rollup = await usageSummary(scope.org, { surfaceKeyPrefix: scope.surfaceKey, sinceTs: since });
+    for (const g of rollup) {
+      if (g.routineId === undefined) continue;
+      tokens.set(g.routineId, (tokens.get(g.routineId) ?? 0) + g.inputTokens + g.outputTokens + g.cacheReadTokens + g.cacheWriteTokens);
+    }
+
+    for (const id of new Set([...acc.keys(), ...tokens.keys()])) {
+      const a = acc.get(id);
+      out.set(id, {
+        lastSessionId: a?.lastSessionId ?? null,
+        lastFireAt: a?.lastFireAt ?? null,
+        lastPostedAt: posted.get(id) ?? null,
+        fires7d: a?.fires ?? 0,
+        tokens7d: tokens.get(id) ?? 0,
+      });
+    }
+  } catch (err) {
+    console.warn(`[open-everywhere:routines] forensics read skipped for ${scope.surfaceKey}: ${String(err)}`);
+    return new Map();
+  }
+  return out;
+}
+
+/** One `routine.fire` row, flattened — the per-fire view {@link readRoutineForensics} collapses into
+ *  a single "latest" slot. `sessionId` is non-null by construction: a fire row without one is a
+ *  malformed row and is dropped by the reader rather than surfaced as an unreadable fire. */
+export interface RoutineFireRow {
+  readonly routineId: string;
+  readonly sessionId: string;
+  /** When the fired session ENDED (the row's own timestamp). */
+  readonly at: string;
+  readonly outcome: string | null;
+  /** A `report:{kind:"silent"}` routine's fire — it posted nothing BY DESIGN. */
+  readonly silent: boolean;
+}
+
+/**
+ * The fires of ONE routine on ONE surface, newest-first — the resolver behind OPE-153's
+ * same-channel transcript read.
+ *
+ * 🔴 THIS FUNCTION IS AN AUTHORIZATION ORACLE, not a convenience reader, and two properties make it
+ * one:
+ *
+ *  1. IT IS PARTITION-PINNED, LIKE {@link readRoutineForensics}. Every fire row was written with the
+ *     firing routine's own (org, platform, workspace, surface) — so a routineId belonging to another
+ *     channel simply RESOLVES TO NOTHING here. The same-channel boundary is therefore the SCOPE OF
+ *     THE LOOKUP, not a comparison a caller could forget to make. That is deliberate: a check that
+ *     can be skipped eventually is, and a caller cannot skip the shape of a query.
+ *
+ *  2. IT FAILS CLOSED, WHERE ITS SIBLING FAILS SOFT. readRoutineForensics swallows store errors and
+ *     degrades to an empty map, because it only DECORATES a listing. This one gates a read, so a
+ *     store error must not be indistinguishable from "that routine never fired here" — it throws,
+ *     and the caller refuses. Returning `[]` on an error would be fail-closed in effect but would
+ *     tell the operator the wrong story about why.
+ *
+ * Window + scan ceiling are the shared forensic ones ({@link FORENSIC_WINDOW_MS} /
+ * {@link FORENSIC_SCAN_LIMIT}), so "what a transcript read can reach" is exactly "what the run
+ * history in `list_routines` already showed" — the tool can never see further back than the listing
+ * that told the model the routine exists.
+ */
+export async function listRoutineFires(
+  scope: {
+    readonly org: string;
+    readonly platform: string;
+    readonly workspace: string;
+    readonly surface: string;
+  },
+  routineId: string,
+  now: Date = new Date(),
+): Promise<RoutineFireRow[]> {
+  const rows = await readAudit({
+    org: scope.org,
+    platform: scope.platform,
+    workspace: scope.workspace,
+    surface: scope.surface,
+    action: ROUTINE_AUDIT.fire,
+    since: new Date(now.getTime() - FORENSIC_WINDOW_MS).toISOString(),
+    limit: FORENSIC_SCAN_LIMIT,
+  });
+  const out: RoutineFireRow[] = [];
+  for (const row of rows) {
+    if (detailString(row.detail, "routineId") !== routineId) continue;
+    const sessionId = detailString(row.detail, "sessionId");
+    if (sessionId === undefined) continue; // malformed row — never surface a fire with no transcript
+    out.push({
+      routineId,
+      sessionId,
+      at: row.ts,
+      outcome: detailString(row.detail, "outcome") ?? null,
+      silent: row.detail?.silent === true,
+    });
+  }
+  return out; // readAudit's contract is newest-first; the filter preserves order
+}