@ouro.bot/cli 0.1.0-alpha.721 → 0.1.0-alpha.723

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -7,9 +7,10 @@
7
7
  * "fail" check and the remaining categories still run.
8
8
  */
9
9
  Object.defineProperty(exports, "__esModule", { value: true });
10
- exports.KNOWN_DOCTOR_CATEGORIES = void 0;
10
+ exports.KNOWN_DOCTOR_CATEGORIES = exports.DEFAULT_SENSE_DELIVERY_THRESHOLDS = exports.DEFAULT_MAIL_INGEST_THRESHOLDS = void 0;
11
11
  exports.checkCliPath = checkCliPath;
12
12
  exports.checkDaemon = checkDaemon;
13
+ exports.senseInboundDeliveryCheck = senseInboundDeliveryCheck;
13
14
  exports.checkAgents = checkAgents;
14
15
  exports.checkSenses = checkSenses;
15
16
  exports.checkHabits = checkHabits;
@@ -23,6 +24,7 @@ exports.checkLifecycle = checkLifecycle;
23
24
  exports.runDoctorChecks = runDoctorChecks;
24
25
  const runtime_1 = require("../../nerves/runtime");
25
26
  const bluebubbles_health_diagnostics_1 = require("./bluebubbles-health-diagnostics");
27
+ const freshness_1 = require("./freshness");
26
28
  const ouro_path_installer_1 = require("../versioning/ouro-path-installer");
27
29
  const runtime_credentials_1 = require("../runtime-credentials");
28
30
  const machine_identity_1 = require("../machine-identity");
@@ -115,6 +117,102 @@ const SENSITIVE_CONFIG_KEYS = ["apiKey", "token", "secret", "password"];
115
117
  function credentialKeyLeaks(raw) {
116
118
  return SENSITIVE_CONFIG_KEYS.filter((key) => raw.includes(`"${key}"`));
117
119
  }
120
+ // ── Pipeline liveness (dead-pipe detection) ──
121
+ //
122
+ // Configuration checks answer "is this wired up?". These answer "is anything
123
+ // actually flowing?". Both are needed: the 77-day mail outage and the
124
+ // multi-day BlueBubbles inbound outage were both invisible to configuration
125
+ // and connection checks that stayed green throughout.
126
+ /**
127
+ * Mail ingest thresholds.
128
+ *
129
+ * A delegated mailbox forwards a human's real inbox, so a full calendar day
130
+ * with zero inbound mail is already anomalous — that is the warn line. Fail
131
+ * waits until 72h so a genuinely quiet weekend on a low-traffic mailbox does
132
+ * not hard-fail doctor; the 2026-05-10 outage would have failed on day four
133
+ * instead of going unnoticed for seventy-seven. Override per agent with
134
+ * `senses.mail.freshness.{warnAfterHours,failAfterHours}` in `agent.json`.
135
+ */
136
+ exports.DEFAULT_MAIL_INGEST_THRESHOLDS = {
137
+ warnAfterMs: 24 * freshness_1.FRESHNESS_HOUR_MS,
138
+ failAfterMs: 72 * freshness_1.FRESHNESS_HOUR_MS,
139
+ };
140
+ /**
141
+ * Inbound sense-delivery thresholds.
142
+ *
143
+ * Human-driven chat surfaces are bursty — nobody texts an agent every day —
144
+ * so the warn line sits at 72h and failure waits a full week. Seven days
145
+ * without a single inbound message on an attached chat sense is not plausible
146
+ * quiet; it is a dead pipe. Override per agent with
147
+ * `senses.<sense>.freshness.{warnAfterHours,failAfterHours}` in `agent.json`.
148
+ */
149
+ exports.DEFAULT_SENSE_DELIVERY_THRESHOLDS = {
150
+ warnAfterMs: 72 * freshness_1.FRESHNESS_HOUR_MS,
151
+ failAfterMs: 7 * freshness_1.FRESHNESS_DAY_MS,
152
+ };
153
+ /** Hard bound on how many per-conversation inbound logs a sense probe stats. */
154
+ const SENSE_DELIVERY_MAX_LOG_SCAN = 500;
155
+ function pipelineLivenessCheck(input) {
156
+ const result = (0, freshness_1.evaluateFreshness)({
157
+ activity: input.activity,
158
+ unit: input.unit,
159
+ observation: input.probe.observation,
160
+ provenance: input.probe.provenance,
161
+ nowMs: input.nowMs,
162
+ thresholds: input.thresholds,
163
+ remediation: input.remediation,
164
+ configuredSinceMs: input.configuredSinceMs,
165
+ context: input.context,
166
+ });
167
+ return { id: input.id, label: input.label, status: result.status, detail: result.detail };
168
+ }
169
+ /** Best-effort mtime used as a "this pipe has been wired up since" floor. */
170
+ function pathMtimeMs(deps, filePath) {
171
+ if (!deps.existsSync(filePath))
172
+ return null;
173
+ try {
174
+ return deps.statSync(filePath).mtimeMs;
175
+ }
176
+ catch {
177
+ return null;
178
+ }
179
+ }
180
+ /** Read `senses.<sense>.freshness` from `agent.json`, if present. */
181
+ function senseFreshnessOverride(deps, agentDir, sense) {
182
+ const configPath = `${deps.bundlesRoot}/${agentDir}/agent.json`;
183
+ if (!deps.existsSync(configPath))
184
+ return null;
185
+ try {
186
+ const config = JSON.parse(deps.readFileSync(configPath));
187
+ return asRecord(asRecord(config.senses)?.[sense])?.freshness ?? null;
188
+ }
189
+ catch {
190
+ return null;
191
+ }
192
+ }
193
+ /**
194
+ * Inbound-delivery liveness for a sense that records what it received.
195
+ *
196
+ * This asserts on the *inbound* log, deliberately not on the upstream health
197
+ * probe: BlueBubbles reported `upstreamStatus: ok` for days while inbound
198
+ * delivery was dead because the server was POSTing to a stale webhook port.
199
+ * A sense opts in by calling this with its inbound log directory.
200
+ */
201
+ function senseInboundDeliveryCheck(input) {
202
+ const suffix = input.logSuffix ?? ".ndjson";
203
+ return pipelineLivenessCheck({
204
+ id: `senses.${input.sense}.inbound_liveness`,
205
+ label: `${input.agentDir} ${input.sense} inbound delivery`,
206
+ activity: `${input.sense} inbound delivery`,
207
+ unit: "inbound message",
208
+ probe: (0, freshness_1.observeAppendPerEventStore)(input.inboundDir, input.deps, { suffix, maxEntries: SENSE_DELIVERY_MAX_LOG_SCAN }),
209
+ thresholds: (0, freshness_1.resolveFreshnessThresholds)(exports.DEFAULT_SENSE_DELIVERY_THRESHOLDS, senseFreshnessOverride(input.deps, input.agentDir, input.sense)),
210
+ remediation: input.remediation,
211
+ configuredSinceMs: input.configuredSincePath ? pathMtimeMs(input.deps, input.configuredSincePath) : null,
212
+ context: input.context,
213
+ nowMs: input.nowMs,
214
+ });
215
+ }
118
216
  function checkAgents(deps) {
119
217
  const checks = [];
120
218
  if (!deps.existsSync(deps.bundlesRoot)) {
@@ -255,6 +353,7 @@ async function checkSenses(deps) {
255
353
  status: "pass",
256
354
  detail: serverUrl,
257
355
  });
356
+ let upstreamContext = "upstream probe not run in this pass";
258
357
  if (deps.fetchImpl) {
259
358
  const probe = await (0, bluebubbles_health_diagnostics_1.probeBlueBubblesHealth)({
260
359
  serverUrl,
@@ -267,7 +366,21 @@ async function checkSenses(deps) {
267
366
  status: probe.ok ? "pass" : "fail",
268
367
  detail: probe.detail,
269
368
  });
369
+ upstreamContext = probe.ok
370
+ ? "upstream probe reachable — which does not prove inbound delivery"
371
+ : "upstream probe failing";
270
372
  }
373
+ const bluebubblesStateRoot = `${deps.bundlesRoot}/${agentDir}/state/senses/bluebubbles`;
374
+ checks.push(senseInboundDeliveryCheck({
375
+ deps,
376
+ agentDir,
377
+ sense: "bluebubbles",
378
+ inboundDir: `${bluebubblesStateRoot}/inbound`,
379
+ configuredSincePath: bluebubblesStateRoot,
380
+ context: upstreamContext,
381
+ remediation: "no iMessage is reaching this agent — confirm the BlueBubbles server's configured webhook URL/port matches the port this daemon is listening on (a stale webhook port is the known cause), then re-attach with `ouro connect bluebubbles --agent <agent>`",
382
+ nowMs: Date.now(),
383
+ }));
271
384
  }
272
385
  if (sense === "mail" && senseObj.enabled === true) {
273
386
  const runtimeConfig = await (0, runtime_credentials_1.refreshRuntimeCredentialConfig)(agentName, { preserveCachedOnFailure: true });
@@ -695,7 +808,7 @@ function tripStoresDiffer(deps, durableRoot, legacyRoot) {
695
808
  }
696
809
  return false;
697
810
  }
698
- function checkMailroom(deps) {
811
+ async function checkMailroom(deps) {
699
812
  const checks = [];
700
813
  const agents = discoverAgents(deps);
701
814
  if (agents.length === 0) {
@@ -739,26 +852,131 @@ function checkMailroom(deps) {
739
852
  checks.push({ label: `${agentDir} mailroom`, status: "warn", detail: "registry.json has no mailboxes — provision via `ouro connect mail`" });
740
853
  continue;
741
854
  }
742
- let messagesCount = 0;
743
855
  const messagesDir = `${mailroomRoot}/messages`;
744
- /* v8 ignore start -- defensive: messages-dir presence + readdir error + pluralization branches depend on filesystem-state fixtures not exhaustively covered @preserve */
745
- if (deps.existsSync(messagesDir)) {
746
- try {
747
- messagesCount = deps.readdirSync(messagesDir).filter((name) => name.endsWith(".json")).length;
748
- }
749
- catch {
750
- // ignore — pass detail just won't include the message count
751
- }
752
- }
856
+ const listing = listJsonDocuments(deps, messagesDir);
857
+ /* v8 ignore start -- defensive: pluralization branches depend on filesystem-state fixtures not exhaustively covered @preserve */
753
858
  checks.push({
754
859
  label: `${agentDir} mailroom`,
755
860
  status: "pass",
756
- detail: `${mailboxes.length} mailbox${mailboxes.length === 1 ? "" : "es"}, ${sourceGrants.length} source grant${sourceGrants.length === 1 ? "" : "s"}, ${messagesCount} message${messagesCount === 1 ? "" : "s"}`,
861
+ detail: `${mailboxes.length} mailbox${mailboxes.length === 1 ? "" : "es"}, ${sourceGrants.length} source grant${sourceGrants.length === 1 ? "" : "s"}, ${listing.count} message${listing.count === 1 ? "" : "s"} stored (cumulative — not a liveness signal)`,
757
862
  });
758
863
  /* v8 ignore stop */
864
+ checks.push(mailIngestLivenessCheck(deps, agentDir, mailroomRoot, messagesDir, listing, await resolveMailStore(agentDir)));
759
865
  }
760
866
  return { name: "Mailroom", checks };
761
867
  }
868
+ function listJsonDocuments(deps, dir) {
869
+ if (!deps.existsSync(dir))
870
+ return { count: 0, readable: true };
871
+ try {
872
+ return { count: deps.readdirSync(dir).filter((name) => name.endsWith(".json")).length, readable: true };
873
+ }
874
+ catch {
875
+ return { count: 0, readable: false };
876
+ }
877
+ }
878
+ async function resolveMailStore(agentDir) {
879
+ const agentName = agentDir.replace(/\.ouro$/, "");
880
+ // An unreadable vault means the mode is unknown, so this falls back to the
881
+ // reader's own default — the local file store. `mail config` in checkSenses
882
+ // is where an unavailable runtime config is reported, and it fails hard.
883
+ const runtimeConfig = await (0, runtime_credentials_1.refreshRuntimeCredentialConfig)(agentName, { preserveCachedOnFailure: true });
884
+ if (!runtimeConfig.ok)
885
+ return { hosted: false };
886
+ const mailroom = asRecord(runtimeConfig.config.mailroom);
887
+ const azureAccountUrl = textField(mailroom, "azureAccountUrl");
888
+ if (!azureAccountUrl)
889
+ return { hosted: false };
890
+ const azureContainer = textField(mailroom, "azureContainer") || "mailroom";
891
+ return { hosted: true, label: `hosted azure-blob ${azureAccountUrl}/${azureContainer}` };
892
+ }
893
+ /**
894
+ * Local-file Mailroom liveness signal.
895
+ *
896
+ * The `messages/` directory mtime is both the cheapest and the most correct
897
+ * option. Cheapest: one stat regardless of store size, and the production store
898
+ * holds ~45k message files. Most correct: a message's `receivedAt` is when the
899
+ * mail was *sent* (an mbox backfill writes month-old values today), so parsing
900
+ * message bodies would answer a different question than "did this machine
901
+ * ingest anything recently?".
902
+ */
903
+ function localMailStoreProbe(deps, messagesDir, listing) {
904
+ if (!listing.readable) {
905
+ return {
906
+ observation: { kind: "unknown", reason: `${messagesDir} exists but could not be listed` },
907
+ provenance: `attempted directory listing of ${messagesDir}`,
908
+ };
909
+ }
910
+ return (0, freshness_1.observeCreatePerEventStore)(messagesDir, deps, { hasEntries: listing.count > 0 });
911
+ }
912
+ /**
913
+ * Hosted Mailroom liveness signal.
914
+ *
915
+ * Once an agent is cut over to the hosted store, messages live in the Blob
916
+ * container and nothing writes to `state/mailroom/messages` any more, so its
917
+ * mtime freezes at the cutover. Reading it in hosted mode is how this check
918
+ * reported "no mail ingested in 77 days" on 2026-07-27 while the container was
919
+ * taking mail that same morning — a false failure, which trains operators to
920
+ * ignore the check and so undoes the reason it exists.
921
+ *
922
+ * The local artifact that does still move is the hosted reader's search cache:
923
+ * `AzureBlobMailroomStore` writes `state/mail-search/<messageId>.json` for
924
+ * every message it decrypts (see `mailroom/reader.ts`, which hands the hosted
925
+ * store that cache directory). Message ids are content hashes, so a *new* file
926
+ * appears exactly when this machine sees a message it has never seen before;
927
+ * re-reading old mail rewrites existing files and leaves the directory mtime
928
+ * untouched. Same create-per-event shape as the local store, same O(1) stat.
929
+ *
930
+ * It is deliberately one step downstream of ingest: it proves hosted mail
931
+ * reached this machine, which is the strongest claim available without network
932
+ * access or Azure credentials — a health check must need neither. The activity
933
+ * wording, provenance and remediation all say so rather than implying doctor
934
+ * measured the container itself, and an empty mirror reports `unknown` rather
935
+ * than the hosted store being empty.
936
+ */
937
+ function hostedMailMirrorProbe(deps, agentDir, storeLabel) {
938
+ const mirrorDir = `${deps.bundlesRoot}/${agentDir}/state/mail-search`;
939
+ const listing = listJsonDocuments(deps, mirrorDir);
940
+ if (!listing.readable) {
941
+ return {
942
+ observation: { kind: "unknown", reason: `${mirrorDir} exists but could not be listed` },
943
+ provenance: `attempted directory listing of ${mirrorDir}`,
944
+ };
945
+ }
946
+ return (0, freshness_1.observeMirroredStore)(mirrorDir, deps, { hasEntries: listing.count > 0, remote: storeLabel });
947
+ }
948
+ /**
949
+ * Mail-ingest liveness — "is mail still arriving?", as opposed to the check
950
+ * above, which only says "is a mailbox configured?".
951
+ *
952
+ * This is the check that would have caught the 2026-05-10 outage: the mailbox,
953
+ * its config, and a cumulative count of 45,479 messages all reported ✔ for 77
954
+ * days while zero mail was ingested.
955
+ *
956
+ * Which signal is honest depends on where the store lives, so the probe, the
957
+ * activity wording and the remediation are all chosen per store kind. Reusing
958
+ * the local path in hosted mode measures a directory nothing writes to.
959
+ */
960
+ function mailIngestLivenessCheck(deps, agentDir, mailroomRoot, messagesDir, listing, store) {
961
+ const hosted = store.hosted ? store : null;
962
+ const probe = hosted
963
+ ? hostedMailMirrorProbe(deps, agentDir, hosted.label)
964
+ : localMailStoreProbe(deps, messagesDir, listing);
965
+ return pipelineLivenessCheck({
966
+ id: "mail.ingest_liveness",
967
+ label: `${agentDir} mail ingest liveness`,
968
+ activity: hosted ? "hosted mail observed locally" : "mail ingested",
969
+ unit: "message",
970
+ probe,
971
+ thresholds: (0, freshness_1.resolveFreshnessThresholds)(exports.DEFAULT_MAIL_INGEST_THRESHOLDS, senseFreshnessOverride(deps, agentDir, "mail")),
972
+ remediation: hosted
973
+ ? `doctor measures hosted mail only through this machine's local mirror and makes no network calls — confirm the container itself by listing \`messages/\` blobs in ${hosted.label} by Last-Modified, or read the mailbox directly (\`ouro mailbox\`, or the agent's \`mail_recent\` tool); if the mirror is empty or stale while the container is current, the hosted reader on this machine is not running`
974
+ : "mail is configured but nothing is arriving — re-check the mailbox grant and keyIds against the vault (a server-side key rotation silently orphans ingestion), run `ouro connect mail --agent <agent>`, and inspect the mailroom ingress logs",
975
+ configuredSinceMs: pathMtimeMs(deps, `${mailroomRoot}/registry.json`),
976
+ context: hosted ? `mailbox configured; ${hosted.label}` : "mailbox configured",
977
+ nowMs: Date.now(),
978
+ });
979
+ }
762
980
  function checkFriends(deps) {
763
981
  const checks = [];
764
982
  const agents = discoverAgents(deps);
@@ -0,0 +1,343 @@
1
+ "use strict";
2
+ /**
3
+ * Pipeline freshness — "is this pipe still moving?"
4
+ *
5
+ * Every outage this module exists to prevent had the same shape: a health
6
+ * check verified that something was *configured* or *connected*, and reported
7
+ * a green tick next to a pipe that had been dead for weeks.
8
+ *
9
+ * - mail: `mail enabled` / `mail config ok` / `45479 messages` all passed
10
+ * while zero mail had been ingested for 77 days after a vault key rotation
11
+ * changed the mailbox keyIds. The message count is cumulative, so it can
12
+ * only ever go up — it is not a liveness signal.
13
+ * - bluebubbles: the upstream HTTP probe stayed `ok` for days while inbound
14
+ * delivery was dead, because the server was POSTing its webhook to a
15
+ * stale port. A reachable upstream does not prove inbound delivery.
16
+ * - launchd: checks asserted a plist *file* existed rather than that the
17
+ * job was loaded.
18
+ *
19
+ * The fix is to assert on *observed data flow*, with three rules baked into
20
+ * this primitive:
21
+ *
22
+ * 1. Age is always stated in the message, in human units, with the
23
+ * timestamp it was derived from.
24
+ * 2. "Never observed any activity" and "could not observe" are distinct
25
+ * states from "activity, but stale" — and none of them can read as
26
+ * `pass`. An unobservable pipe is unverified, not healthy.
27
+ * 3. Every non-`pass` result carries a concrete remediation hint.
28
+ *
29
+ * Thresholds are always supplied by the caller so freshness policy lives with
30
+ * the pipe it describes, not in this file.
31
+ */
32
+ Object.defineProperty(exports, "__esModule", { value: true });
33
+ exports.DEFAULT_CLOCK_SKEW_TOLERANCE_MS = exports.FRESHNESS_DAY_MS = exports.FRESHNESS_HOUR_MS = exports.FRESHNESS_MINUTE_MS = void 0;
34
+ exports.formatFreshnessAge = formatFreshnessAge;
35
+ exports.formatFreshnessTimestamp = formatFreshnessTimestamp;
36
+ exports.evaluateFreshness = evaluateFreshness;
37
+ exports.observeCreatePerEventStore = observeCreatePerEventStore;
38
+ exports.observeMirroredStore = observeMirroredStore;
39
+ exports.observeAppendPerEventStore = observeAppendPerEventStore;
40
+ exports.resolveFreshnessThresholds = resolveFreshnessThresholds;
41
+ const runtime_1 = require("../../nerves/runtime");
42
+ exports.FRESHNESS_MINUTE_MS = 60 * 1000;
43
+ exports.FRESHNESS_HOUR_MS = 60 * exports.FRESHNESS_MINUTE_MS;
44
+ exports.FRESHNESS_DAY_MS = 24 * exports.FRESHNESS_HOUR_MS;
45
+ /** Small tolerance before a future timestamp is treated as clock skew. */
46
+ exports.DEFAULT_CLOCK_SKEW_TOLERANCE_MS = exports.FRESHNESS_MINUTE_MS;
47
+ /**
48
+ * Human-readable duration. Always rounds down so the age is never overstated:
49
+ * "77 days" means at least 77 days have elapsed.
50
+ */
51
+ function formatFreshnessAge(ms) {
52
+ const clamped = Math.max(0, ms);
53
+ const seconds = Math.floor(clamped / 1000);
54
+ if (seconds < 60)
55
+ return `${seconds} second${seconds === 1 ? "" : "s"}`;
56
+ const minutes = Math.floor(clamped / exports.FRESHNESS_MINUTE_MS);
57
+ if (minutes < 60)
58
+ return `${minutes} minute${minutes === 1 ? "" : "s"}`;
59
+ const hours = Math.floor(clamped / exports.FRESHNESS_HOUR_MS);
60
+ if (hours < 48)
61
+ return `${hours} hour${hours === 1 ? "" : "s"}`;
62
+ // The day bucket only starts at 48h, so it is always plural.
63
+ return `${Math.floor(clamped / exports.FRESHNESS_DAY_MS)} days`;
64
+ }
65
+ /** Minute-precision ISO instant — keeps the incident date visible and unambiguous. */
66
+ function formatFreshnessTimestamp(ms) {
67
+ return new Date(ms).toISOString().replace(/:\d{2}\.\d{3}Z$/, "Z");
68
+ }
69
+ function joinDetail(parts) {
70
+ return parts.filter((part) => !!part && part.trim().length > 0).join("; ");
71
+ }
72
+ function statusForAge(ageMs, thresholds) {
73
+ if (ageMs >= thresholds.failAfterMs)
74
+ return "fail";
75
+ if (ageMs >= thresholds.warnAfterMs)
76
+ return "warn";
77
+ return "pass";
78
+ }
79
+ /**
80
+ * Severity for "readable store, nothing ever recorded".
81
+ *
82
+ * Floors at `warn` on purpose: a configured-but-never-delivered pipe is never
83
+ * healthy, but a pipe wired up minutes ago does not deserve a hard failure.
84
+ */
85
+ function statusForNever(input) {
86
+ const since = input.configuredSinceMs;
87
+ if (typeof since !== "number" || !Number.isFinite(since)) {
88
+ return { status: "fail", configuredForMs: null };
89
+ }
90
+ const configuredForMs = Math.max(0, input.nowMs - since);
91
+ const escalated = statusForAge(configuredForMs, input.thresholds);
92
+ return { status: escalated === "pass" ? "warn" : escalated, configuredForMs };
93
+ }
94
+ function evaluateActivity(input, atMs) {
95
+ const tolerance = input.clockSkewToleranceMs ?? exports.DEFAULT_CLOCK_SKEW_TOLERANCE_MS;
96
+ const skewMs = atMs - input.nowMs;
97
+ const timestamp = formatFreshnessTimestamp(atMs);
98
+ if (skewMs > tolerance) {
99
+ return {
100
+ status: "warn",
101
+ state: "future",
102
+ ageMs: 0,
103
+ detail: joinDetail([
104
+ input.context,
105
+ `last ${input.unit} is timestamped ${timestamp}, ${formatFreshnessAge(skewMs)} in the future — clock skew means freshness cannot be trusted`,
106
+ input.provenance,
107
+ `fix: ${input.remediation}`,
108
+ ]),
109
+ };
110
+ }
111
+ const ageMs = Math.max(0, -skewMs);
112
+ const status = statusForAge(ageMs, input.thresholds);
113
+ if (status === "pass") {
114
+ return {
115
+ status,
116
+ state: "fresh",
117
+ ageMs,
118
+ detail: joinDetail([
119
+ input.context,
120
+ `${input.activity} ${formatFreshnessAge(ageMs)} ago (last ${input.unit} ${timestamp})`,
121
+ input.provenance,
122
+ ]),
123
+ };
124
+ }
125
+ return {
126
+ status,
127
+ state: "stale",
128
+ ageMs,
129
+ detail: joinDetail([
130
+ input.context,
131
+ `no ${input.activity} in ${formatFreshnessAge(ageMs)} — last ${input.unit} ${timestamp}`,
132
+ input.provenance,
133
+ `fix: ${input.remediation}`,
134
+ ]),
135
+ };
136
+ }
137
+ /**
138
+ * Turn a single observation into a status + actionable detail.
139
+ *
140
+ * The only path that can return `pass` is an observed activity timestamp
141
+ * inside the warn threshold. Everything else — no activity, unreadable store,
142
+ * skewed clock — is surfaced, never swallowed.
143
+ */
144
+ function evaluateFreshness(input) {
145
+ const observation = input.observation;
146
+ if (observation.kind === "unknown") {
147
+ return {
148
+ status: "fail",
149
+ state: "unknown",
150
+ ageMs: null,
151
+ detail: joinDetail([
152
+ input.context,
153
+ `${input.activity}: last-activity time could not be determined — ${observation.reason}; unverified, not healthy`,
154
+ input.provenance,
155
+ `fix: ${input.remediation}`,
156
+ ]),
157
+ };
158
+ }
159
+ if (observation.kind === "none") {
160
+ const { status, configuredForMs } = statusForNever(input);
161
+ return {
162
+ status,
163
+ state: "never",
164
+ ageMs: null,
165
+ detail: joinDetail([
166
+ input.context,
167
+ configuredForMs === null
168
+ ? `no ${input.activity} ever — the store is readable and empty`
169
+ : `no ${input.activity} ever — the store is readable and empty, and this pipe has been configured for ${formatFreshnessAge(configuredForMs)}`,
170
+ input.provenance,
171
+ `fix: ${input.remediation}`,
172
+ ]),
173
+ };
174
+ }
175
+ if (!Number.isFinite(observation.atMs)) {
176
+ return evaluateFreshness({
177
+ ...input,
178
+ observation: { kind: "unknown", reason: `observed timestamp is not a finite epoch-ms value (${String(observation.atMs)})` },
179
+ });
180
+ }
181
+ return evaluateActivity(input, observation.atMs);
182
+ }
183
+ /**
184
+ * Last-write probe for **create-per-event** stores — one new file per event,
185
+ * such as the mailroom's `messages/<id>.json`.
186
+ *
187
+ * A directory's mtime advances whenever an entry is created or removed, so it
188
+ * is an exact answer to "when was a file last added here" for O(1) cost, with
189
+ * no per-entry stat. That matters: the production mailroom holds ~45k message
190
+ * files and must never be stat-walked by a health check.
191
+ *
192
+ * It is also the semantically *correct* signal here. A message's `receivedAt`
193
+ * is when the mail was sent, not when this machine ingested it — an mbox
194
+ * backfill writes month-old `receivedAt` values today — so parsing message
195
+ * bodies would answer a different question than "is the pipe moving?".
196
+ *
197
+ * `hasEntries` is supplied by the caller (which typically already lists the
198
+ * directory for a count) so an empty directory reports `none` rather than
199
+ * reporting its own creation time as activity.
200
+ */
201
+ function observeCreatePerEventStore(dir, deps, options) {
202
+ const provenance = `derived from ${dir} directory mtime (O(1), no per-entry scan)`;
203
+ if (!deps.existsSync(dir)) {
204
+ return { observation: { kind: "none" }, provenance: `${dir} does not exist` };
205
+ }
206
+ if (!options.hasEntries) {
207
+ return { observation: { kind: "none" }, provenance: `${dir} is empty` };
208
+ }
209
+ try {
210
+ return { observation: { kind: "activity", atMs: deps.statSync(dir).mtimeMs }, provenance };
211
+ }
212
+ catch (error) {
213
+ return {
214
+ observation: { kind: "unknown", reason: `${dir} could not be stat'd: ${error instanceof Error ? error.message : String(error)}` },
215
+ provenance,
216
+ };
217
+ }
218
+ }
219
+ /**
220
+ * Last-observation probe for a **local mirror of a remote store** — the cache
221
+ * this machine writes as it reads a store that lives somewhere else, such as
222
+ * the hosted Mailroom's `state/mail-search/<messageId>.json` documents.
223
+ *
224
+ * Mechanically this is `observeCreatePerEventStore`: one file per event, named
225
+ * by a content-addressed id, so the directory mtime answers "when did this
226
+ * machine last record something it had never seen before" for O(1) cost.
227
+ *
228
+ * One rule differs, and it is the entire reason this exists: an absent or empty
229
+ * mirror is `unknown`, never `none`. For a local store the directory *is* the
230
+ * store, so empty means "nothing was ever delivered". For a mirror the
231
+ * authoritative store is remote and deliberately not read here — a health check
232
+ * must not need network access or credentials — so empty means only "this
233
+ * machine holds no record". That is absence of evidence, and rendering it as a
234
+ * confident "never delivered" would be the same crying-wolf failure as reading
235
+ * a local directory that nothing writes to any more.
236
+ */
237
+ function observeMirroredStore(dir, deps, options) {
238
+ const provenance = `derived from the local mirror at ${dir} (directory mtime, O(1), no per-entry scan); the authoritative store is ${options.remote}, which doctor does not read`;
239
+ const exists = deps.existsSync(dir);
240
+ if (!exists || !options.hasEntries) {
241
+ return {
242
+ observation: {
243
+ kind: "unknown",
244
+ reason: `the local mirror at ${dir} is ${exists ? "empty" : "absent"}, and ${options.remote} is not read by doctor (no network calls, no credentials), so recency cannot be measured on this machine`,
245
+ },
246
+ provenance,
247
+ };
248
+ }
249
+ try {
250
+ return { observation: { kind: "activity", atMs: deps.statSync(dir).mtimeMs }, provenance };
251
+ }
252
+ catch (error) {
253
+ return {
254
+ observation: { kind: "unknown", reason: `${dir} could not be stat'd: ${error instanceof Error ? error.message : String(error)}` },
255
+ provenance,
256
+ };
257
+ }
258
+ }
259
+ /**
260
+ * Last-write probe for **append-per-event** stores — a small set of long-lived
261
+ * logs appended in place, such as the BlueBubbles inbound ndjson (one file per
262
+ * chat).
263
+ *
264
+ * Directory mtime is the wrong signal here: appending to an existing file does
265
+ * not touch its parent directory, so a busy-but-not-growing log set would look
266
+ * frozen. This stats each matching entry and takes the newest.
267
+ *
268
+ * The scan is bounded by `maxEntries`. When the bound is hit the provenance
269
+ * says so explicitly — a bounded scan must never masquerade as exhaustive.
270
+ */
271
+ function observeAppendPerEventStore(dir, deps, options) {
272
+ if (!deps.existsSync(dir)) {
273
+ return { observation: { kind: "none" }, provenance: `${dir} does not exist` };
274
+ }
275
+ let entries;
276
+ try {
277
+ entries = deps.readdirSync(dir).filter((name) => name.endsWith(options.suffix));
278
+ }
279
+ catch (error) {
280
+ return {
281
+ observation: { kind: "unknown", reason: `${dir} could not be listed: ${error instanceof Error ? error.message : String(error)}` },
282
+ provenance: `attempted directory listing of ${dir}`,
283
+ };
284
+ }
285
+ if (entries.length === 0) {
286
+ return { observation: { kind: "none" }, provenance: `${dir} holds no ${options.suffix} logs` };
287
+ }
288
+ const scanned = entries.slice(0, options.maxEntries);
289
+ const truncated = entries.length > scanned.length;
290
+ const provenance = truncated
291
+ ? `newest mtime across the first ${scanned.length} of ${entries.length} ${options.suffix} logs in ${dir} — scan bounded, newer activity may exist in the unscanned ${entries.length - scanned.length}`
292
+ : `newest mtime across ${scanned.length} ${options.suffix} log${scanned.length === 1 ? "" : "s"} in ${dir}`;
293
+ let newest = Number.NEGATIVE_INFINITY;
294
+ const failures = [];
295
+ for (const name of scanned) {
296
+ try {
297
+ const mtimeMs = deps.statSync(`${dir}/${name}`).mtimeMs;
298
+ if (mtimeMs > newest)
299
+ newest = mtimeMs;
300
+ }
301
+ catch (error) {
302
+ failures.push(`${name}: ${error instanceof Error ? error.message : String(error)}`);
303
+ }
304
+ }
305
+ if (newest === Number.NEGATIVE_INFINITY) {
306
+ return {
307
+ observation: { kind: "unknown", reason: `none of the ${scanned.length} log(s) in ${dir} could be stat'd: ${failures.slice(0, 3).join("; ")}` },
308
+ provenance,
309
+ };
310
+ }
311
+ return {
312
+ observation: { kind: "activity", atMs: newest },
313
+ provenance: failures.length > 0 ? `${provenance} (${failures.length} unreadable and skipped)` : provenance,
314
+ };
315
+ }
316
+ /**
317
+ * Merge an `agent.json` override onto in-repo defaults.
318
+ *
319
+ * Configuration lives in a committed config file rather than an environment
320
+ * variable, per the repo configuration policy. Non-positive and non-finite
321
+ * values are ignored so a malformed override can never disable the check.
322
+ */
323
+ function resolveFreshnessThresholds(defaults, override) {
324
+ const record = override && typeof override === "object" && !Array.isArray(override)
325
+ ? override
326
+ : null;
327
+ const hours = (key) => {
328
+ const value = record?.[key];
329
+ return typeof value === "number" && Number.isFinite(value) && value > 0 ? value * exports.FRESHNESS_HOUR_MS : null;
330
+ };
331
+ return {
332
+ warnAfterMs: hours("warnAfterHours") ?? defaults.warnAfterMs,
333
+ failAfterMs: hours("failAfterHours") ?? defaults.failAfterMs,
334
+ };
335
+ }
336
+ /* v8 ignore start -- module load observability event */
337
+ (0, runtime_1.emitNervesEvent)({
338
+ component: "daemon",
339
+ event: "daemon.pipeline_freshness_loaded",
340
+ message: "pipeline freshness primitive loaded",
341
+ meta: {},
342
+ });
343
+ /* v8 ignore stop */