@ouro.bot/cli 0.1.0-alpha.721 → 0.1.0-alpha.722
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/changelog.json +6 -0
- package/dist/heart/daemon/doctor.js +163 -12
- package/dist/heart/daemon/freshness.js +302 -0
- package/package.json +1 -1
package/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Arye Mendelow
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
package/changelog.json
CHANGED
|
@@ -1,6 +1,12 @@
|
|
|
1
1
|
{
|
|
2
2
|
"_note": "This changelog is maintained as part of the PR/version-bump workflow. Agent-curated, not auto-generated. Agents read this file directly via read_file to understand what changed between versions.",
|
|
3
3
|
"versions": [
|
|
4
|
+
{
|
|
5
|
+
"version": "0.1.0-alpha.722",
|
|
6
|
+
"changes": [
|
|
7
|
+
"ouro doctor now asserts pipeline liveness (mail ingest age, sense inbound-delivery age) so a configured-but-dead pipe fails loudly instead of reporting healthy."
|
|
8
|
+
]
|
|
9
|
+
},
|
|
4
10
|
{
|
|
5
11
|
"version": "0.1.0-alpha.721",
|
|
6
12
|
"changes": [
|
|
@@ -7,9 +7,10 @@
|
|
|
7
7
|
* "fail" check and the remaining categories still run.
|
|
8
8
|
*/
|
|
9
9
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
10
|
-
exports.KNOWN_DOCTOR_CATEGORIES = void 0;
|
|
10
|
+
exports.KNOWN_DOCTOR_CATEGORIES = exports.DEFAULT_SENSE_DELIVERY_THRESHOLDS = exports.DEFAULT_MAIL_INGEST_THRESHOLDS = void 0;
|
|
11
11
|
exports.checkCliPath = checkCliPath;
|
|
12
12
|
exports.checkDaemon = checkDaemon;
|
|
13
|
+
exports.senseInboundDeliveryCheck = senseInboundDeliveryCheck;
|
|
13
14
|
exports.checkAgents = checkAgents;
|
|
14
15
|
exports.checkSenses = checkSenses;
|
|
15
16
|
exports.checkHabits = checkHabits;
|
|
@@ -23,6 +24,7 @@ exports.checkLifecycle = checkLifecycle;
|
|
|
23
24
|
exports.runDoctorChecks = runDoctorChecks;
|
|
24
25
|
const runtime_1 = require("../../nerves/runtime");
|
|
25
26
|
const bluebubbles_health_diagnostics_1 = require("./bluebubbles-health-diagnostics");
|
|
27
|
+
const freshness_1 = require("./freshness");
|
|
26
28
|
const ouro_path_installer_1 = require("../versioning/ouro-path-installer");
|
|
27
29
|
const runtime_credentials_1 = require("../runtime-credentials");
|
|
28
30
|
const machine_identity_1 = require("../machine-identity");
|
|
@@ -115,6 +117,102 @@ const SENSITIVE_CONFIG_KEYS = ["apiKey", "token", "secret", "password"];
|
|
|
115
117
|
function credentialKeyLeaks(raw) {
|
|
116
118
|
return SENSITIVE_CONFIG_KEYS.filter((key) => raw.includes(`"${key}"`));
|
|
117
119
|
}
|
|
120
|
+
// ── Pipeline liveness (dead-pipe detection) ──
|
|
121
|
+
//
|
|
122
|
+
// Configuration checks answer "is this wired up?". These answer "is anything
|
|
123
|
+
// actually flowing?". Both are needed: the 77-day mail outage and the
|
|
124
|
+
// multi-day BlueBubbles inbound outage were both invisible to configuration
|
|
125
|
+
// and connection checks that stayed green throughout.
|
|
126
|
+
/**
|
|
127
|
+
* Mail ingest thresholds.
|
|
128
|
+
*
|
|
129
|
+
* A delegated mailbox forwards a human's real inbox, so a full calendar day
|
|
130
|
+
* with zero inbound mail is already anomalous — that is the warn line. Fail
|
|
131
|
+
* waits until 72h so a genuinely quiet weekend on a low-traffic mailbox does
|
|
132
|
+
* not hard-fail doctor; the 2026-05-10 outage would have failed on day four
|
|
133
|
+
* instead of going unnoticed for seventy-seven. Override per agent with
|
|
134
|
+
* `senses.mail.freshness.{warnAfterHours,failAfterHours}` in `agent.json`.
|
|
135
|
+
*/
|
|
136
|
+
exports.DEFAULT_MAIL_INGEST_THRESHOLDS = {
|
|
137
|
+
warnAfterMs: 24 * freshness_1.FRESHNESS_HOUR_MS,
|
|
138
|
+
failAfterMs: 72 * freshness_1.FRESHNESS_HOUR_MS,
|
|
139
|
+
};
|
|
140
|
+
/**
|
|
141
|
+
* Inbound sense-delivery thresholds.
|
|
142
|
+
*
|
|
143
|
+
* Human-driven chat surfaces are bursty — nobody texts an agent every day —
|
|
144
|
+
* so the warn line sits at 72h and failure waits a full week. Seven days
|
|
145
|
+
* without a single inbound message on an attached chat sense is not plausible
|
|
146
|
+
* quiet; it is a dead pipe. Override per agent with
|
|
147
|
+
* `senses.<sense>.freshness.{warnAfterHours,failAfterHours}` in `agent.json`.
|
|
148
|
+
*/
|
|
149
|
+
exports.DEFAULT_SENSE_DELIVERY_THRESHOLDS = {
|
|
150
|
+
warnAfterMs: 72 * freshness_1.FRESHNESS_HOUR_MS,
|
|
151
|
+
failAfterMs: 7 * freshness_1.FRESHNESS_DAY_MS,
|
|
152
|
+
};
|
|
153
|
+
/** Hard bound on how many per-conversation inbound logs a sense probe stats. */
|
|
154
|
+
const SENSE_DELIVERY_MAX_LOG_SCAN = 500;
|
|
155
|
+
function pipelineLivenessCheck(input) {
|
|
156
|
+
const result = (0, freshness_1.evaluateFreshness)({
|
|
157
|
+
activity: input.activity,
|
|
158
|
+
unit: input.unit,
|
|
159
|
+
observation: input.probe.observation,
|
|
160
|
+
provenance: input.probe.provenance,
|
|
161
|
+
nowMs: input.nowMs,
|
|
162
|
+
thresholds: input.thresholds,
|
|
163
|
+
remediation: input.remediation,
|
|
164
|
+
configuredSinceMs: input.configuredSinceMs,
|
|
165
|
+
context: input.context,
|
|
166
|
+
});
|
|
167
|
+
return { id: input.id, label: input.label, status: result.status, detail: result.detail };
|
|
168
|
+
}
|
|
169
|
+
/** Best-effort mtime used as a "this pipe has been wired up since" floor. */
|
|
170
|
+
function pathMtimeMs(deps, filePath) {
|
|
171
|
+
if (!deps.existsSync(filePath))
|
|
172
|
+
return null;
|
|
173
|
+
try {
|
|
174
|
+
return deps.statSync(filePath).mtimeMs;
|
|
175
|
+
}
|
|
176
|
+
catch {
|
|
177
|
+
return null;
|
|
178
|
+
}
|
|
179
|
+
}
|
|
180
|
+
/** Read `senses.<sense>.freshness` from `agent.json`, if present. */
|
|
181
|
+
function senseFreshnessOverride(deps, agentDir, sense) {
|
|
182
|
+
const configPath = `${deps.bundlesRoot}/${agentDir}/agent.json`;
|
|
183
|
+
if (!deps.existsSync(configPath))
|
|
184
|
+
return null;
|
|
185
|
+
try {
|
|
186
|
+
const config = JSON.parse(deps.readFileSync(configPath));
|
|
187
|
+
return asRecord(asRecord(config.senses)?.[sense])?.freshness ?? null;
|
|
188
|
+
}
|
|
189
|
+
catch {
|
|
190
|
+
return null;
|
|
191
|
+
}
|
|
192
|
+
}
|
|
193
|
+
/**
|
|
194
|
+
* Inbound-delivery liveness for a sense that records what it received.
|
|
195
|
+
*
|
|
196
|
+
* This asserts on the *inbound* log, deliberately not on the upstream health
|
|
197
|
+
* probe: BlueBubbles reported `upstreamStatus: ok` for days while inbound
|
|
198
|
+
* delivery was dead because the server was POSTing to a stale webhook port.
|
|
199
|
+
* A sense opts in by calling this with its inbound log directory.
|
|
200
|
+
*/
|
|
201
|
+
function senseInboundDeliveryCheck(input) {
|
|
202
|
+
const suffix = input.logSuffix ?? ".ndjson";
|
|
203
|
+
return pipelineLivenessCheck({
|
|
204
|
+
id: `senses.${input.sense}.inbound_liveness`,
|
|
205
|
+
label: `${input.agentDir} ${input.sense} inbound delivery`,
|
|
206
|
+
activity: `${input.sense} inbound delivery`,
|
|
207
|
+
unit: "inbound message",
|
|
208
|
+
probe: (0, freshness_1.observeAppendPerEventStore)(input.inboundDir, input.deps, { suffix, maxEntries: SENSE_DELIVERY_MAX_LOG_SCAN }),
|
|
209
|
+
thresholds: (0, freshness_1.resolveFreshnessThresholds)(exports.DEFAULT_SENSE_DELIVERY_THRESHOLDS, senseFreshnessOverride(input.deps, input.agentDir, input.sense)),
|
|
210
|
+
remediation: input.remediation,
|
|
211
|
+
configuredSinceMs: input.configuredSincePath ? pathMtimeMs(input.deps, input.configuredSincePath) : null,
|
|
212
|
+
context: input.context,
|
|
213
|
+
nowMs: input.nowMs,
|
|
214
|
+
});
|
|
215
|
+
}
|
|
118
216
|
function checkAgents(deps) {
|
|
119
217
|
const checks = [];
|
|
120
218
|
if (!deps.existsSync(deps.bundlesRoot)) {
|
|
@@ -255,6 +353,7 @@ async function checkSenses(deps) {
|
|
|
255
353
|
status: "pass",
|
|
256
354
|
detail: serverUrl,
|
|
257
355
|
});
|
|
356
|
+
let upstreamContext = "upstream probe not run in this pass";
|
|
258
357
|
if (deps.fetchImpl) {
|
|
259
358
|
const probe = await (0, bluebubbles_health_diagnostics_1.probeBlueBubblesHealth)({
|
|
260
359
|
serverUrl,
|
|
@@ -267,7 +366,21 @@ async function checkSenses(deps) {
|
|
|
267
366
|
status: probe.ok ? "pass" : "fail",
|
|
268
367
|
detail: probe.detail,
|
|
269
368
|
});
|
|
369
|
+
upstreamContext = probe.ok
|
|
370
|
+
? "upstream probe reachable — which does not prove inbound delivery"
|
|
371
|
+
: "upstream probe failing";
|
|
270
372
|
}
|
|
373
|
+
const bluebubblesStateRoot = `${deps.bundlesRoot}/${agentDir}/state/senses/bluebubbles`;
|
|
374
|
+
checks.push(senseInboundDeliveryCheck({
|
|
375
|
+
deps,
|
|
376
|
+
agentDir,
|
|
377
|
+
sense: "bluebubbles",
|
|
378
|
+
inboundDir: `${bluebubblesStateRoot}/inbound`,
|
|
379
|
+
configuredSincePath: bluebubblesStateRoot,
|
|
380
|
+
context: upstreamContext,
|
|
381
|
+
remediation: "no iMessage is reaching this agent — confirm the BlueBubbles server's configured webhook URL/port matches the port this daemon is listening on (a stale webhook port is the known cause), then re-attach with `ouro connect bluebubbles --agent <agent>`",
|
|
382
|
+
nowMs: Date.now(),
|
|
383
|
+
}));
|
|
271
384
|
}
|
|
272
385
|
if (sense === "mail" && senseObj.enabled === true) {
|
|
273
386
|
const runtimeConfig = await (0, runtime_credentials_1.refreshRuntimeCredentialConfig)(agentName, { preserveCachedOnFailure: true });
|
|
@@ -739,26 +852,64 @@ function checkMailroom(deps) {
|
|
|
739
852
|
checks.push({ label: `${agentDir} mailroom`, status: "warn", detail: "registry.json has no mailboxes — provision via `ouro connect mail`" });
|
|
740
853
|
continue;
|
|
741
854
|
}
|
|
742
|
-
let messagesCount = 0;
|
|
743
855
|
const messagesDir = `${mailroomRoot}/messages`;
|
|
744
|
-
|
|
745
|
-
|
|
746
|
-
try {
|
|
747
|
-
messagesCount = deps.readdirSync(messagesDir).filter((name) => name.endsWith(".json")).length;
|
|
748
|
-
}
|
|
749
|
-
catch {
|
|
750
|
-
// ignore — pass detail just won't include the message count
|
|
751
|
-
}
|
|
752
|
-
}
|
|
856
|
+
const listing = listStoredMailMessages(deps, messagesDir);
|
|
857
|
+
/* v8 ignore start -- defensive: pluralization branches depend on filesystem-state fixtures not exhaustively covered @preserve */
|
|
753
858
|
checks.push({
|
|
754
859
|
label: `${agentDir} mailroom`,
|
|
755
860
|
status: "pass",
|
|
756
|
-
detail: `${mailboxes.length} mailbox${mailboxes.length === 1 ? "" : "es"}, ${sourceGrants.length} source grant${sourceGrants.length === 1 ? "" : "s"}, ${
|
|
861
|
+
detail: `${mailboxes.length} mailbox${mailboxes.length === 1 ? "" : "es"}, ${sourceGrants.length} source grant${sourceGrants.length === 1 ? "" : "s"}, ${listing.count} message${listing.count === 1 ? "" : "s"} stored (cumulative — not a liveness signal)`,
|
|
757
862
|
});
|
|
758
863
|
/* v8 ignore stop */
|
|
864
|
+
checks.push(mailIngestLivenessCheck(deps, agentDir, mailroomRoot, messagesDir, listing));
|
|
759
865
|
}
|
|
760
866
|
return { name: "Mailroom", checks };
|
|
761
867
|
}
|
|
868
|
+
function listStoredMailMessages(deps, messagesDir) {
|
|
869
|
+
if (!deps.existsSync(messagesDir))
|
|
870
|
+
return { count: 0, readable: true };
|
|
871
|
+
try {
|
|
872
|
+
return { count: deps.readdirSync(messagesDir).filter((name) => name.endsWith(".json")).length, readable: true };
|
|
873
|
+
}
|
|
874
|
+
catch {
|
|
875
|
+
return { count: 0, readable: false };
|
|
876
|
+
}
|
|
877
|
+
}
|
|
878
|
+
/**
|
|
879
|
+
* Mail-ingest liveness — "is mail still arriving?", as opposed to the check
|
|
880
|
+
* above, which only says "is a mailbox configured?".
|
|
881
|
+
*
|
|
882
|
+
* This is the check that would have caught the 2026-05-10 outage: the mailbox,
|
|
883
|
+
* its config, and a cumulative count of 45,479 messages all reported ✔ for 77
|
|
884
|
+
* days while zero mail was ingested.
|
|
885
|
+
*
|
|
886
|
+
* The signal is the `messages/` directory mtime, which is both the cheapest
|
|
887
|
+
* and the most correct option. Cheapest: one stat regardless of store size,
|
|
888
|
+
* and the production store holds ~45k message files. Most correct: a message's
|
|
889
|
+
* `receivedAt` is when the mail was *sent* (an mbox backfill writes month-old
|
|
890
|
+
* values today), so parsing message bodies would answer a different question
|
|
891
|
+
* than "did this machine ingest anything recently?".
|
|
892
|
+
*/
|
|
893
|
+
function mailIngestLivenessCheck(deps, agentDir, mailroomRoot, messagesDir, listing) {
|
|
894
|
+
const probe = listing.readable
|
|
895
|
+
? (0, freshness_1.observeCreatePerEventStore)(messagesDir, deps, { hasEntries: listing.count > 0 })
|
|
896
|
+
: {
|
|
897
|
+
observation: { kind: "unknown", reason: `${messagesDir} exists but could not be listed` },
|
|
898
|
+
provenance: `attempted directory listing of ${messagesDir}`,
|
|
899
|
+
};
|
|
900
|
+
return pipelineLivenessCheck({
|
|
901
|
+
id: "mail.ingest_liveness",
|
|
902
|
+
label: `${agentDir} mail ingest liveness`,
|
|
903
|
+
activity: "mail ingested",
|
|
904
|
+
unit: "message",
|
|
905
|
+
probe,
|
|
906
|
+
thresholds: (0, freshness_1.resolveFreshnessThresholds)(exports.DEFAULT_MAIL_INGEST_THRESHOLDS, senseFreshnessOverride(deps, agentDir, "mail")),
|
|
907
|
+
remediation: "mail is configured but nothing is arriving — re-check the mailbox grant and keyIds against the vault (a server-side key rotation silently orphans ingestion), run `ouro connect mail --agent <agent>`, and inspect the mailroom ingress logs",
|
|
908
|
+
configuredSinceMs: pathMtimeMs(deps, `${mailroomRoot}/registry.json`),
|
|
909
|
+
context: "mailbox configured",
|
|
910
|
+
nowMs: Date.now(),
|
|
911
|
+
});
|
|
912
|
+
}
|
|
762
913
|
function checkFriends(deps) {
|
|
763
914
|
const checks = [];
|
|
764
915
|
const agents = discoverAgents(deps);
|
|
@@ -0,0 +1,302 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/**
|
|
3
|
+
* Pipeline freshness — "is this pipe still moving?"
|
|
4
|
+
*
|
|
5
|
+
* Every outage this module exists to prevent had the same shape: a health
|
|
6
|
+
* check verified that something was *configured* or *connected*, and reported
|
|
7
|
+
* a green tick next to a pipe that had been dead for weeks.
|
|
8
|
+
*
|
|
9
|
+
* - mail: `mail enabled` / `mail config ok` / `45479 messages` all passed
|
|
10
|
+
* while zero mail had been ingested for 77 days after a vault key rotation
|
|
11
|
+
* changed the mailbox keyIds. The message count is cumulative, so it can
|
|
12
|
+
* only ever go up — it is not a liveness signal.
|
|
13
|
+
* - bluebubbles: the upstream HTTP probe stayed `ok` for days while inbound
|
|
14
|
+
* delivery was dead, because the server was POSTing its webhook to a
|
|
15
|
+
* stale port. A reachable upstream does not prove inbound delivery.
|
|
16
|
+
* - launchd: checks asserted a plist *file* existed rather than that the
|
|
17
|
+
* job was loaded.
|
|
18
|
+
*
|
|
19
|
+
* The fix is to assert on *observed data flow*, with three rules baked into
|
|
20
|
+
* this primitive:
|
|
21
|
+
*
|
|
22
|
+
* 1. Age is always stated in the message, in human units, with the
|
|
23
|
+
* timestamp it was derived from.
|
|
24
|
+
* 2. "Never observed any activity" and "could not observe" are distinct
|
|
25
|
+
* states from "activity, but stale" — and none of them can read as
|
|
26
|
+
* `pass`. An unobservable pipe is unverified, not healthy.
|
|
27
|
+
* 3. Every non-`pass` result carries a concrete remediation hint.
|
|
28
|
+
*
|
|
29
|
+
* Thresholds are always supplied by the caller so freshness policy lives with
|
|
30
|
+
* the pipe it describes, not in this file.
|
|
31
|
+
*/
|
|
32
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
33
|
+
exports.DEFAULT_CLOCK_SKEW_TOLERANCE_MS = exports.FRESHNESS_DAY_MS = exports.FRESHNESS_HOUR_MS = exports.FRESHNESS_MINUTE_MS = void 0;
|
|
34
|
+
exports.formatFreshnessAge = formatFreshnessAge;
|
|
35
|
+
exports.formatFreshnessTimestamp = formatFreshnessTimestamp;
|
|
36
|
+
exports.evaluateFreshness = evaluateFreshness;
|
|
37
|
+
exports.observeCreatePerEventStore = observeCreatePerEventStore;
|
|
38
|
+
exports.observeAppendPerEventStore = observeAppendPerEventStore;
|
|
39
|
+
exports.resolveFreshnessThresholds = resolveFreshnessThresholds;
|
|
40
|
+
const runtime_1 = require("../../nerves/runtime");
|
|
41
|
+
exports.FRESHNESS_MINUTE_MS = 60 * 1000;
|
|
42
|
+
exports.FRESHNESS_HOUR_MS = 60 * exports.FRESHNESS_MINUTE_MS;
|
|
43
|
+
exports.FRESHNESS_DAY_MS = 24 * exports.FRESHNESS_HOUR_MS;
|
|
44
|
+
/** Small tolerance before a future timestamp is treated as clock skew. */
|
|
45
|
+
exports.DEFAULT_CLOCK_SKEW_TOLERANCE_MS = exports.FRESHNESS_MINUTE_MS;
|
|
46
|
+
/**
|
|
47
|
+
* Human-readable duration. Always rounds down so the age is never overstated:
|
|
48
|
+
* "77 days" means at least 77 days have elapsed.
|
|
49
|
+
*/
|
|
50
|
+
function formatFreshnessAge(ms) {
|
|
51
|
+
const clamped = Math.max(0, ms);
|
|
52
|
+
const seconds = Math.floor(clamped / 1000);
|
|
53
|
+
if (seconds < 60)
|
|
54
|
+
return `${seconds} second${seconds === 1 ? "" : "s"}`;
|
|
55
|
+
const minutes = Math.floor(clamped / exports.FRESHNESS_MINUTE_MS);
|
|
56
|
+
if (minutes < 60)
|
|
57
|
+
return `${minutes} minute${minutes === 1 ? "" : "s"}`;
|
|
58
|
+
const hours = Math.floor(clamped / exports.FRESHNESS_HOUR_MS);
|
|
59
|
+
if (hours < 48)
|
|
60
|
+
return `${hours} hour${hours === 1 ? "" : "s"}`;
|
|
61
|
+
// The day bucket only starts at 48h, so it is always plural.
|
|
62
|
+
return `${Math.floor(clamped / exports.FRESHNESS_DAY_MS)} days`;
|
|
63
|
+
}
|
|
64
|
+
/** Minute-precision ISO instant — keeps the incident date visible and unambiguous. */
|
|
65
|
+
function formatFreshnessTimestamp(ms) {
|
|
66
|
+
return new Date(ms).toISOString().replace(/:\d{2}\.\d{3}Z$/, "Z");
|
|
67
|
+
}
|
|
68
|
+
function joinDetail(parts) {
|
|
69
|
+
return parts.filter((part) => !!part && part.trim().length > 0).join("; ");
|
|
70
|
+
}
|
|
71
|
+
function statusForAge(ageMs, thresholds) {
|
|
72
|
+
if (ageMs >= thresholds.failAfterMs)
|
|
73
|
+
return "fail";
|
|
74
|
+
if (ageMs >= thresholds.warnAfterMs)
|
|
75
|
+
return "warn";
|
|
76
|
+
return "pass";
|
|
77
|
+
}
|
|
78
|
+
/**
|
|
79
|
+
* Severity for "readable store, nothing ever recorded".
|
|
80
|
+
*
|
|
81
|
+
* Floors at `warn` on purpose: a configured-but-never-delivered pipe is never
|
|
82
|
+
* healthy, but a pipe wired up minutes ago does not deserve a hard failure.
|
|
83
|
+
*/
|
|
84
|
+
function statusForNever(input) {
|
|
85
|
+
const since = input.configuredSinceMs;
|
|
86
|
+
if (typeof since !== "number" || !Number.isFinite(since)) {
|
|
87
|
+
return { status: "fail", configuredForMs: null };
|
|
88
|
+
}
|
|
89
|
+
const configuredForMs = Math.max(0, input.nowMs - since);
|
|
90
|
+
const escalated = statusForAge(configuredForMs, input.thresholds);
|
|
91
|
+
return { status: escalated === "pass" ? "warn" : escalated, configuredForMs };
|
|
92
|
+
}
|
|
93
|
+
function evaluateActivity(input, atMs) {
|
|
94
|
+
const tolerance = input.clockSkewToleranceMs ?? exports.DEFAULT_CLOCK_SKEW_TOLERANCE_MS;
|
|
95
|
+
const skewMs = atMs - input.nowMs;
|
|
96
|
+
const timestamp = formatFreshnessTimestamp(atMs);
|
|
97
|
+
if (skewMs > tolerance) {
|
|
98
|
+
return {
|
|
99
|
+
status: "warn",
|
|
100
|
+
state: "future",
|
|
101
|
+
ageMs: 0,
|
|
102
|
+
detail: joinDetail([
|
|
103
|
+
input.context,
|
|
104
|
+
`last ${input.unit} is timestamped ${timestamp}, ${formatFreshnessAge(skewMs)} in the future — clock skew means freshness cannot be trusted`,
|
|
105
|
+
input.provenance,
|
|
106
|
+
`fix: ${input.remediation}`,
|
|
107
|
+
]),
|
|
108
|
+
};
|
|
109
|
+
}
|
|
110
|
+
const ageMs = Math.max(0, -skewMs);
|
|
111
|
+
const status = statusForAge(ageMs, input.thresholds);
|
|
112
|
+
if (status === "pass") {
|
|
113
|
+
return {
|
|
114
|
+
status,
|
|
115
|
+
state: "fresh",
|
|
116
|
+
ageMs,
|
|
117
|
+
detail: joinDetail([
|
|
118
|
+
input.context,
|
|
119
|
+
`${input.activity} ${formatFreshnessAge(ageMs)} ago (last ${input.unit} ${timestamp})`,
|
|
120
|
+
input.provenance,
|
|
121
|
+
]),
|
|
122
|
+
};
|
|
123
|
+
}
|
|
124
|
+
return {
|
|
125
|
+
status,
|
|
126
|
+
state: "stale",
|
|
127
|
+
ageMs,
|
|
128
|
+
detail: joinDetail([
|
|
129
|
+
input.context,
|
|
130
|
+
`no ${input.activity} in ${formatFreshnessAge(ageMs)} — last ${input.unit} ${timestamp}`,
|
|
131
|
+
input.provenance,
|
|
132
|
+
`fix: ${input.remediation}`,
|
|
133
|
+
]),
|
|
134
|
+
};
|
|
135
|
+
}
|
|
136
|
+
/**
|
|
137
|
+
* Turn a single observation into a status + actionable detail.
|
|
138
|
+
*
|
|
139
|
+
* The only path that can return `pass` is an observed activity timestamp
|
|
140
|
+
* inside the warn threshold. Everything else — no activity, unreadable store,
|
|
141
|
+
* skewed clock — is surfaced, never swallowed.
|
|
142
|
+
*/
|
|
143
|
+
function evaluateFreshness(input) {
|
|
144
|
+
const observation = input.observation;
|
|
145
|
+
if (observation.kind === "unknown") {
|
|
146
|
+
return {
|
|
147
|
+
status: "fail",
|
|
148
|
+
state: "unknown",
|
|
149
|
+
ageMs: null,
|
|
150
|
+
detail: joinDetail([
|
|
151
|
+
input.context,
|
|
152
|
+
`${input.activity}: last-activity time could not be determined — ${observation.reason}; unverified, not healthy`,
|
|
153
|
+
input.provenance,
|
|
154
|
+
`fix: ${input.remediation}`,
|
|
155
|
+
]),
|
|
156
|
+
};
|
|
157
|
+
}
|
|
158
|
+
if (observation.kind === "none") {
|
|
159
|
+
const { status, configuredForMs } = statusForNever(input);
|
|
160
|
+
return {
|
|
161
|
+
status,
|
|
162
|
+
state: "never",
|
|
163
|
+
ageMs: null,
|
|
164
|
+
detail: joinDetail([
|
|
165
|
+
input.context,
|
|
166
|
+
configuredForMs === null
|
|
167
|
+
? `no ${input.activity} ever — the store is readable and empty`
|
|
168
|
+
: `no ${input.activity} ever — the store is readable and empty, and this pipe has been configured for ${formatFreshnessAge(configuredForMs)}`,
|
|
169
|
+
input.provenance,
|
|
170
|
+
`fix: ${input.remediation}`,
|
|
171
|
+
]),
|
|
172
|
+
};
|
|
173
|
+
}
|
|
174
|
+
if (!Number.isFinite(observation.atMs)) {
|
|
175
|
+
return evaluateFreshness({
|
|
176
|
+
...input,
|
|
177
|
+
observation: { kind: "unknown", reason: `observed timestamp is not a finite epoch-ms value (${String(observation.atMs)})` },
|
|
178
|
+
});
|
|
179
|
+
}
|
|
180
|
+
return evaluateActivity(input, observation.atMs);
|
|
181
|
+
}
|
|
182
|
+
/**
|
|
183
|
+
* Last-write probe for **create-per-event** stores — one new file per event,
|
|
184
|
+
* such as the mailroom's `messages/<id>.json`.
|
|
185
|
+
*
|
|
186
|
+
* A directory's mtime advances whenever an entry is created or removed, so it
|
|
187
|
+
* is an exact answer to "when was a file last added here" for O(1) cost, with
|
|
188
|
+
* no per-entry stat. That matters: the production mailroom holds ~45k message
|
|
189
|
+
* files and must never be stat-walked by a health check.
|
|
190
|
+
*
|
|
191
|
+
* It is also the semantically *correct* signal here. A message's `receivedAt`
|
|
192
|
+
* is when the mail was sent, not when this machine ingested it — an mbox
|
|
193
|
+
* backfill writes month-old `receivedAt` values today — so parsing message
|
|
194
|
+
* bodies would answer a different question than "is the pipe moving?".
|
|
195
|
+
*
|
|
196
|
+
* `hasEntries` is supplied by the caller (which typically already lists the
|
|
197
|
+
* directory for a count) so an empty directory reports `none` rather than
|
|
198
|
+
* reporting its own creation time as activity.
|
|
199
|
+
*/
|
|
200
|
+
function observeCreatePerEventStore(dir, deps, options) {
|
|
201
|
+
const provenance = `derived from ${dir} directory mtime (O(1), no per-entry scan)`;
|
|
202
|
+
if (!deps.existsSync(dir)) {
|
|
203
|
+
return { observation: { kind: "none" }, provenance: `${dir} does not exist` };
|
|
204
|
+
}
|
|
205
|
+
if (!options.hasEntries) {
|
|
206
|
+
return { observation: { kind: "none" }, provenance: `${dir} is empty` };
|
|
207
|
+
}
|
|
208
|
+
try {
|
|
209
|
+
return { observation: { kind: "activity", atMs: deps.statSync(dir).mtimeMs }, provenance };
|
|
210
|
+
}
|
|
211
|
+
catch (error) {
|
|
212
|
+
return {
|
|
213
|
+
observation: { kind: "unknown", reason: `${dir} could not be stat'd: ${error instanceof Error ? error.message : String(error)}` },
|
|
214
|
+
provenance,
|
|
215
|
+
};
|
|
216
|
+
}
|
|
217
|
+
}
|
|
218
|
+
/**
|
|
219
|
+
* Last-write probe for **append-per-event** stores — a small set of long-lived
|
|
220
|
+
* logs appended in place, such as the BlueBubbles inbound ndjson (one file per
|
|
221
|
+
* chat).
|
|
222
|
+
*
|
|
223
|
+
* Directory mtime is the wrong signal here: appending to an existing file does
|
|
224
|
+
* not touch its parent directory, so a busy-but-not-growing log set would look
|
|
225
|
+
* frozen. This stats each matching entry and takes the newest.
|
|
226
|
+
*
|
|
227
|
+
* The scan is bounded by `maxEntries`. When the bound is hit the provenance
|
|
228
|
+
* says so explicitly — a bounded scan must never masquerade as exhaustive.
|
|
229
|
+
*/
|
|
230
|
+
function observeAppendPerEventStore(dir, deps, options) {
|
|
231
|
+
if (!deps.existsSync(dir)) {
|
|
232
|
+
return { observation: { kind: "none" }, provenance: `${dir} does not exist` };
|
|
233
|
+
}
|
|
234
|
+
let entries;
|
|
235
|
+
try {
|
|
236
|
+
entries = deps.readdirSync(dir).filter((name) => name.endsWith(options.suffix));
|
|
237
|
+
}
|
|
238
|
+
catch (error) {
|
|
239
|
+
return {
|
|
240
|
+
observation: { kind: "unknown", reason: `${dir} could not be listed: ${error instanceof Error ? error.message : String(error)}` },
|
|
241
|
+
provenance: `attempted directory listing of ${dir}`,
|
|
242
|
+
};
|
|
243
|
+
}
|
|
244
|
+
if (entries.length === 0) {
|
|
245
|
+
return { observation: { kind: "none" }, provenance: `${dir} holds no ${options.suffix} logs` };
|
|
246
|
+
}
|
|
247
|
+
const scanned = entries.slice(0, options.maxEntries);
|
|
248
|
+
const truncated = entries.length > scanned.length;
|
|
249
|
+
const provenance = truncated
|
|
250
|
+
? `newest mtime across the first ${scanned.length} of ${entries.length} ${options.suffix} logs in ${dir} — scan bounded, newer activity may exist in the unscanned ${entries.length - scanned.length}`
|
|
251
|
+
: `newest mtime across ${scanned.length} ${options.suffix} log${scanned.length === 1 ? "" : "s"} in ${dir}`;
|
|
252
|
+
let newest = Number.NEGATIVE_INFINITY;
|
|
253
|
+
const failures = [];
|
|
254
|
+
for (const name of scanned) {
|
|
255
|
+
try {
|
|
256
|
+
const mtimeMs = deps.statSync(`${dir}/${name}`).mtimeMs;
|
|
257
|
+
if (mtimeMs > newest)
|
|
258
|
+
newest = mtimeMs;
|
|
259
|
+
}
|
|
260
|
+
catch (error) {
|
|
261
|
+
failures.push(`${name}: ${error instanceof Error ? error.message : String(error)}`);
|
|
262
|
+
}
|
|
263
|
+
}
|
|
264
|
+
if (newest === Number.NEGATIVE_INFINITY) {
|
|
265
|
+
return {
|
|
266
|
+
observation: { kind: "unknown", reason: `none of the ${scanned.length} log(s) in ${dir} could be stat'd: ${failures.slice(0, 3).join("; ")}` },
|
|
267
|
+
provenance,
|
|
268
|
+
};
|
|
269
|
+
}
|
|
270
|
+
return {
|
|
271
|
+
observation: { kind: "activity", atMs: newest },
|
|
272
|
+
provenance: failures.length > 0 ? `${provenance} (${failures.length} unreadable and skipped)` : provenance,
|
|
273
|
+
};
|
|
274
|
+
}
|
|
275
|
+
/**
|
|
276
|
+
* Merge an `agent.json` override onto in-repo defaults.
|
|
277
|
+
*
|
|
278
|
+
* Configuration lives in a committed config file rather than an environment
|
|
279
|
+
* variable, per the repo configuration policy. Non-positive and non-finite
|
|
280
|
+
* values are ignored so a malformed override can never disable the check.
|
|
281
|
+
*/
|
|
282
|
+
function resolveFreshnessThresholds(defaults, override) {
|
|
283
|
+
const record = override && typeof override === "object" && !Array.isArray(override)
|
|
284
|
+
? override
|
|
285
|
+
: null;
|
|
286
|
+
const hours = (key) => {
|
|
287
|
+
const value = record?.[key];
|
|
288
|
+
return typeof value === "number" && Number.isFinite(value) && value > 0 ? value * exports.FRESHNESS_HOUR_MS : null;
|
|
289
|
+
};
|
|
290
|
+
return {
|
|
291
|
+
warnAfterMs: hours("warnAfterHours") ?? defaults.warnAfterMs,
|
|
292
|
+
failAfterMs: hours("failAfterHours") ?? defaults.failAfterMs,
|
|
293
|
+
};
|
|
294
|
+
}
|
|
295
|
+
/* v8 ignore start -- module load observability event */
|
|
296
|
+
(0, runtime_1.emitNervesEvent)({
|
|
297
|
+
component: "daemon",
|
|
298
|
+
event: "daemon.pipeline_freshness_loaded",
|
|
299
|
+
message: "pipeline freshness primitive loaded",
|
|
300
|
+
meta: {},
|
|
301
|
+
});
|
|
302
|
+
/* v8 ignore stop */
|