@ouro.bot/cli 0.1.0-alpha.721 → 0.1.0-alpha.723
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/changelog.json +289 -275
- package/dist/heart/daemon/doctor.js +231 -13
- package/dist/heart/daemon/freshness.js +343 -0
- package/dist/heart/orientation-frame.js +82 -11
- package/dist/senses/bluebubbles/client.js +49 -0
- package/dist/senses/bluebubbles/index.js +22 -17
- package/dist/senses/bluebubbles/model.js +74 -8
- package/package.json +1 -1
|
@@ -7,9 +7,10 @@
|
|
|
7
7
|
* "fail" check and the remaining categories still run.
|
|
8
8
|
*/
|
|
9
9
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
10
|
-
exports.KNOWN_DOCTOR_CATEGORIES = void 0;
|
|
10
|
+
exports.KNOWN_DOCTOR_CATEGORIES = exports.DEFAULT_SENSE_DELIVERY_THRESHOLDS = exports.DEFAULT_MAIL_INGEST_THRESHOLDS = void 0;
|
|
11
11
|
exports.checkCliPath = checkCliPath;
|
|
12
12
|
exports.checkDaemon = checkDaemon;
|
|
13
|
+
exports.senseInboundDeliveryCheck = senseInboundDeliveryCheck;
|
|
13
14
|
exports.checkAgents = checkAgents;
|
|
14
15
|
exports.checkSenses = checkSenses;
|
|
15
16
|
exports.checkHabits = checkHabits;
|
|
@@ -23,6 +24,7 @@ exports.checkLifecycle = checkLifecycle;
|
|
|
23
24
|
exports.runDoctorChecks = runDoctorChecks;
|
|
24
25
|
const runtime_1 = require("../../nerves/runtime");
|
|
25
26
|
const bluebubbles_health_diagnostics_1 = require("./bluebubbles-health-diagnostics");
|
|
27
|
+
const freshness_1 = require("./freshness");
|
|
26
28
|
const ouro_path_installer_1 = require("../versioning/ouro-path-installer");
|
|
27
29
|
const runtime_credentials_1 = require("../runtime-credentials");
|
|
28
30
|
const machine_identity_1 = require("../machine-identity");
|
|
@@ -115,6 +117,102 @@ const SENSITIVE_CONFIG_KEYS = ["apiKey", "token", "secret", "password"];
|
|
|
115
117
|
function credentialKeyLeaks(raw) {
|
|
116
118
|
return SENSITIVE_CONFIG_KEYS.filter((key) => raw.includes(`"${key}"`));
|
|
117
119
|
}
|
|
120
|
+
// ── Pipeline liveness (dead-pipe detection) ──
|
|
121
|
+
//
|
|
122
|
+
// Configuration checks answer "is this wired up?". These answer "is anything
|
|
123
|
+
// actually flowing?". Both are needed: the 77-day mail outage and the
|
|
124
|
+
// multi-day BlueBubbles inbound outage were both invisible to configuration
|
|
125
|
+
// and connection checks that stayed green throughout.
|
|
126
|
+
/**
|
|
127
|
+
* Mail ingest thresholds.
|
|
128
|
+
*
|
|
129
|
+
* A delegated mailbox forwards a human's real inbox, so a full calendar day
|
|
130
|
+
* with zero inbound mail is already anomalous — that is the warn line. Fail
|
|
131
|
+
* waits until 72h so a genuinely quiet weekend on a low-traffic mailbox does
|
|
132
|
+
* not hard-fail doctor; the 2026-05-10 outage would have failed on day four
|
|
133
|
+
* instead of going unnoticed for seventy-seven. Override per agent with
|
|
134
|
+
* `senses.mail.freshness.{warnAfterHours,failAfterHours}` in `agent.json`.
|
|
135
|
+
*/
|
|
136
|
+
exports.DEFAULT_MAIL_INGEST_THRESHOLDS = {
|
|
137
|
+
warnAfterMs: 24 * freshness_1.FRESHNESS_HOUR_MS,
|
|
138
|
+
failAfterMs: 72 * freshness_1.FRESHNESS_HOUR_MS,
|
|
139
|
+
};
|
|
140
|
+
/**
|
|
141
|
+
* Inbound sense-delivery thresholds.
|
|
142
|
+
*
|
|
143
|
+
* Human-driven chat surfaces are bursty — nobody texts an agent every day —
|
|
144
|
+
* so the warn line sits at 72h and failure waits a full week. Seven days
|
|
145
|
+
* without a single inbound message on an attached chat sense is not plausible
|
|
146
|
+
* quiet; it is a dead pipe. Override per agent with
|
|
147
|
+
* `senses.<sense>.freshness.{warnAfterHours,failAfterHours}` in `agent.json`.
|
|
148
|
+
*/
|
|
149
|
+
exports.DEFAULT_SENSE_DELIVERY_THRESHOLDS = {
|
|
150
|
+
warnAfterMs: 72 * freshness_1.FRESHNESS_HOUR_MS,
|
|
151
|
+
failAfterMs: 7 * freshness_1.FRESHNESS_DAY_MS,
|
|
152
|
+
};
|
|
153
|
+
/** Hard bound on how many per-conversation inbound logs a sense probe stats. */
|
|
154
|
+
const SENSE_DELIVERY_MAX_LOG_SCAN = 500;
|
|
155
|
+
function pipelineLivenessCheck(input) {
|
|
156
|
+
const result = (0, freshness_1.evaluateFreshness)({
|
|
157
|
+
activity: input.activity,
|
|
158
|
+
unit: input.unit,
|
|
159
|
+
observation: input.probe.observation,
|
|
160
|
+
provenance: input.probe.provenance,
|
|
161
|
+
nowMs: input.nowMs,
|
|
162
|
+
thresholds: input.thresholds,
|
|
163
|
+
remediation: input.remediation,
|
|
164
|
+
configuredSinceMs: input.configuredSinceMs,
|
|
165
|
+
context: input.context,
|
|
166
|
+
});
|
|
167
|
+
return { id: input.id, label: input.label, status: result.status, detail: result.detail };
|
|
168
|
+
}
|
|
169
|
+
/** Best-effort mtime used as a "this pipe has been wired up since" floor. */
|
|
170
|
+
function pathMtimeMs(deps, filePath) {
|
|
171
|
+
if (!deps.existsSync(filePath))
|
|
172
|
+
return null;
|
|
173
|
+
try {
|
|
174
|
+
return deps.statSync(filePath).mtimeMs;
|
|
175
|
+
}
|
|
176
|
+
catch {
|
|
177
|
+
return null;
|
|
178
|
+
}
|
|
179
|
+
}
|
|
180
|
+
/** Read `senses.<sense>.freshness` from `agent.json`, if present. */
|
|
181
|
+
function senseFreshnessOverride(deps, agentDir, sense) {
|
|
182
|
+
const configPath = `${deps.bundlesRoot}/${agentDir}/agent.json`;
|
|
183
|
+
if (!deps.existsSync(configPath))
|
|
184
|
+
return null;
|
|
185
|
+
try {
|
|
186
|
+
const config = JSON.parse(deps.readFileSync(configPath));
|
|
187
|
+
return asRecord(asRecord(config.senses)?.[sense])?.freshness ?? null;
|
|
188
|
+
}
|
|
189
|
+
catch {
|
|
190
|
+
return null;
|
|
191
|
+
}
|
|
192
|
+
}
|
|
193
|
+
/**
|
|
194
|
+
* Inbound-delivery liveness for a sense that records what it received.
|
|
195
|
+
*
|
|
196
|
+
* This asserts on the *inbound* log, deliberately not on the upstream health
|
|
197
|
+
* probe: BlueBubbles reported `upstreamStatus: ok` for days while inbound
|
|
198
|
+
* delivery was dead because the server was POSTing to a stale webhook port.
|
|
199
|
+
* A sense opts in by calling this with its inbound log directory.
|
|
200
|
+
*/
|
|
201
|
+
function senseInboundDeliveryCheck(input) {
|
|
202
|
+
const suffix = input.logSuffix ?? ".ndjson";
|
|
203
|
+
return pipelineLivenessCheck({
|
|
204
|
+
id: `senses.${input.sense}.inbound_liveness`,
|
|
205
|
+
label: `${input.agentDir} ${input.sense} inbound delivery`,
|
|
206
|
+
activity: `${input.sense} inbound delivery`,
|
|
207
|
+
unit: "inbound message",
|
|
208
|
+
probe: (0, freshness_1.observeAppendPerEventStore)(input.inboundDir, input.deps, { suffix, maxEntries: SENSE_DELIVERY_MAX_LOG_SCAN }),
|
|
209
|
+
thresholds: (0, freshness_1.resolveFreshnessThresholds)(exports.DEFAULT_SENSE_DELIVERY_THRESHOLDS, senseFreshnessOverride(input.deps, input.agentDir, input.sense)),
|
|
210
|
+
remediation: input.remediation,
|
|
211
|
+
configuredSinceMs: input.configuredSincePath ? pathMtimeMs(input.deps, input.configuredSincePath) : null,
|
|
212
|
+
context: input.context,
|
|
213
|
+
nowMs: input.nowMs,
|
|
214
|
+
});
|
|
215
|
+
}
|
|
118
216
|
function checkAgents(deps) {
|
|
119
217
|
const checks = [];
|
|
120
218
|
if (!deps.existsSync(deps.bundlesRoot)) {
|
|
@@ -255,6 +353,7 @@ async function checkSenses(deps) {
|
|
|
255
353
|
status: "pass",
|
|
256
354
|
detail: serverUrl,
|
|
257
355
|
});
|
|
356
|
+
let upstreamContext = "upstream probe not run in this pass";
|
|
258
357
|
if (deps.fetchImpl) {
|
|
259
358
|
const probe = await (0, bluebubbles_health_diagnostics_1.probeBlueBubblesHealth)({
|
|
260
359
|
serverUrl,
|
|
@@ -267,7 +366,21 @@ async function checkSenses(deps) {
|
|
|
267
366
|
status: probe.ok ? "pass" : "fail",
|
|
268
367
|
detail: probe.detail,
|
|
269
368
|
});
|
|
369
|
+
upstreamContext = probe.ok
|
|
370
|
+
? "upstream probe reachable — which does not prove inbound delivery"
|
|
371
|
+
: "upstream probe failing";
|
|
270
372
|
}
|
|
373
|
+
const bluebubblesStateRoot = `${deps.bundlesRoot}/${agentDir}/state/senses/bluebubbles`;
|
|
374
|
+
checks.push(senseInboundDeliveryCheck({
|
|
375
|
+
deps,
|
|
376
|
+
agentDir,
|
|
377
|
+
sense: "bluebubbles",
|
|
378
|
+
inboundDir: `${bluebubblesStateRoot}/inbound`,
|
|
379
|
+
configuredSincePath: bluebubblesStateRoot,
|
|
380
|
+
context: upstreamContext,
|
|
381
|
+
remediation: "no iMessage is reaching this agent — confirm the BlueBubbles server's configured webhook URL/port matches the port this daemon is listening on (a stale webhook port is the known cause), then re-attach with `ouro connect bluebubbles --agent <agent>`",
|
|
382
|
+
nowMs: Date.now(),
|
|
383
|
+
}));
|
|
271
384
|
}
|
|
272
385
|
if (sense === "mail" && senseObj.enabled === true) {
|
|
273
386
|
const runtimeConfig = await (0, runtime_credentials_1.refreshRuntimeCredentialConfig)(agentName, { preserveCachedOnFailure: true });
|
|
@@ -695,7 +808,7 @@ function tripStoresDiffer(deps, durableRoot, legacyRoot) {
|
|
|
695
808
|
}
|
|
696
809
|
return false;
|
|
697
810
|
}
|
|
698
|
-
function checkMailroom(deps) {
|
|
811
|
+
async function checkMailroom(deps) {
|
|
699
812
|
const checks = [];
|
|
700
813
|
const agents = discoverAgents(deps);
|
|
701
814
|
if (agents.length === 0) {
|
|
@@ -739,26 +852,131 @@ function checkMailroom(deps) {
|
|
|
739
852
|
checks.push({ label: `${agentDir} mailroom`, status: "warn", detail: "registry.json has no mailboxes — provision via `ouro connect mail`" });
|
|
740
853
|
continue;
|
|
741
854
|
}
|
|
742
|
-
let messagesCount = 0;
|
|
743
855
|
const messagesDir = `${mailroomRoot}/messages`;
|
|
744
|
-
|
|
745
|
-
|
|
746
|
-
try {
|
|
747
|
-
messagesCount = deps.readdirSync(messagesDir).filter((name) => name.endsWith(".json")).length;
|
|
748
|
-
}
|
|
749
|
-
catch {
|
|
750
|
-
// ignore — pass detail just won't include the message count
|
|
751
|
-
}
|
|
752
|
-
}
|
|
856
|
+
const listing = listJsonDocuments(deps, messagesDir);
|
|
857
|
+
/* v8 ignore start -- defensive: pluralization branches depend on filesystem-state fixtures not exhaustively covered @preserve */
|
|
753
858
|
checks.push({
|
|
754
859
|
label: `${agentDir} mailroom`,
|
|
755
860
|
status: "pass",
|
|
756
|
-
detail: `${mailboxes.length} mailbox${mailboxes.length === 1 ? "" : "es"}, ${sourceGrants.length} source grant${sourceGrants.length === 1 ? "" : "s"}, ${
|
|
861
|
+
detail: `${mailboxes.length} mailbox${mailboxes.length === 1 ? "" : "es"}, ${sourceGrants.length} source grant${sourceGrants.length === 1 ? "" : "s"}, ${listing.count} message${listing.count === 1 ? "" : "s"} stored (cumulative — not a liveness signal)`,
|
|
757
862
|
});
|
|
758
863
|
/* v8 ignore stop */
|
|
864
|
+
checks.push(mailIngestLivenessCheck(deps, agentDir, mailroomRoot, messagesDir, listing, await resolveMailStore(agentDir)));
|
|
759
865
|
}
|
|
760
866
|
return { name: "Mailroom", checks };
|
|
761
867
|
}
|
|
868
|
+
function listJsonDocuments(deps, dir) {
|
|
869
|
+
if (!deps.existsSync(dir))
|
|
870
|
+
return { count: 0, readable: true };
|
|
871
|
+
try {
|
|
872
|
+
return { count: deps.readdirSync(dir).filter((name) => name.endsWith(".json")).length, readable: true };
|
|
873
|
+
}
|
|
874
|
+
catch {
|
|
875
|
+
return { count: 0, readable: false };
|
|
876
|
+
}
|
|
877
|
+
}
|
|
878
|
+
async function resolveMailStore(agentDir) {
|
|
879
|
+
const agentName = agentDir.replace(/\.ouro$/, "");
|
|
880
|
+
// An unreadable vault means the mode is unknown, so this falls back to the
|
|
881
|
+
// reader's own default — the local file store. `mail config` in checkSenses
|
|
882
|
+
// is where an unavailable runtime config is reported, and it fails hard.
|
|
883
|
+
const runtimeConfig = await (0, runtime_credentials_1.refreshRuntimeCredentialConfig)(agentName, { preserveCachedOnFailure: true });
|
|
884
|
+
if (!runtimeConfig.ok)
|
|
885
|
+
return { hosted: false };
|
|
886
|
+
const mailroom = asRecord(runtimeConfig.config.mailroom);
|
|
887
|
+
const azureAccountUrl = textField(mailroom, "azureAccountUrl");
|
|
888
|
+
if (!azureAccountUrl)
|
|
889
|
+
return { hosted: false };
|
|
890
|
+
const azureContainer = textField(mailroom, "azureContainer") || "mailroom";
|
|
891
|
+
return { hosted: true, label: `hosted azure-blob ${azureAccountUrl}/${azureContainer}` };
|
|
892
|
+
}
|
|
893
|
+
/**
|
|
894
|
+
* Local-file Mailroom liveness signal.
|
|
895
|
+
*
|
|
896
|
+
* The `messages/` directory mtime is both the cheapest and the most correct
|
|
897
|
+
* option. Cheapest: one stat regardless of store size, and the production store
|
|
898
|
+
* holds ~45k message files. Most correct: a message's `receivedAt` is when the
|
|
899
|
+
* mail was *sent* (an mbox backfill writes month-old values today), so parsing
|
|
900
|
+
* message bodies would answer a different question than "did this machine
|
|
901
|
+
* ingest anything recently?".
|
|
902
|
+
*/
|
|
903
|
+
function localMailStoreProbe(deps, messagesDir, listing) {
|
|
904
|
+
if (!listing.readable) {
|
|
905
|
+
return {
|
|
906
|
+
observation: { kind: "unknown", reason: `${messagesDir} exists but could not be listed` },
|
|
907
|
+
provenance: `attempted directory listing of ${messagesDir}`,
|
|
908
|
+
};
|
|
909
|
+
}
|
|
910
|
+
return (0, freshness_1.observeCreatePerEventStore)(messagesDir, deps, { hasEntries: listing.count > 0 });
|
|
911
|
+
}
|
|
912
|
+
/**
|
|
913
|
+
* Hosted Mailroom liveness signal.
|
|
914
|
+
*
|
|
915
|
+
* Once an agent is cut over to the hosted store, messages live in the Blob
|
|
916
|
+
* container and nothing writes to `state/mailroom/messages` any more, so its
|
|
917
|
+
* mtime freezes at the cutover. Reading it in hosted mode is how this check
|
|
918
|
+
* reported "no mail ingested in 77 days" on 2026-07-27 while the container was
|
|
919
|
+
* taking mail that same morning — a false failure, which trains operators to
|
|
920
|
+
* ignore the check and so undoes the reason it exists.
|
|
921
|
+
*
|
|
922
|
+
* The local artifact that does still move is the hosted reader's search cache:
|
|
923
|
+
* `AzureBlobMailroomStore` writes `state/mail-search/<messageId>.json` for
|
|
924
|
+
* every message it decrypts (see `mailroom/reader.ts`, which hands the hosted
|
|
925
|
+
* store that cache directory). Message ids are content hashes, so a *new* file
|
|
926
|
+
* appears exactly when this machine sees a message it has never seen before;
|
|
927
|
+
* re-reading old mail rewrites existing files and leaves the directory mtime
|
|
928
|
+
* untouched. Same create-per-event shape as the local store, same O(1) stat.
|
|
929
|
+
*
|
|
930
|
+
* It is deliberately one step downstream of ingest: it proves hosted mail
|
|
931
|
+
* reached this machine, which is the strongest claim available without network
|
|
932
|
+
* access or Azure credentials — a health check must need neither. The activity
|
|
933
|
+
* wording, provenance and remediation all say so rather than implying doctor
|
|
934
|
+
* measured the container itself, and an empty mirror reports `unknown` rather
|
|
935
|
+
* than the hosted store being empty.
|
|
936
|
+
*/
|
|
937
|
+
function hostedMailMirrorProbe(deps, agentDir, storeLabel) {
|
|
938
|
+
const mirrorDir = `${deps.bundlesRoot}/${agentDir}/state/mail-search`;
|
|
939
|
+
const listing = listJsonDocuments(deps, mirrorDir);
|
|
940
|
+
if (!listing.readable) {
|
|
941
|
+
return {
|
|
942
|
+
observation: { kind: "unknown", reason: `${mirrorDir} exists but could not be listed` },
|
|
943
|
+
provenance: `attempted directory listing of ${mirrorDir}`,
|
|
944
|
+
};
|
|
945
|
+
}
|
|
946
|
+
return (0, freshness_1.observeMirroredStore)(mirrorDir, deps, { hasEntries: listing.count > 0, remote: storeLabel });
|
|
947
|
+
}
|
|
948
|
+
/**
|
|
949
|
+
* Mail-ingest liveness — "is mail still arriving?", as opposed to the check
|
|
950
|
+
* above, which only says "is a mailbox configured?".
|
|
951
|
+
*
|
|
952
|
+
* This is the check that would have caught the 2026-05-10 outage: the mailbox,
|
|
953
|
+
* its config, and a cumulative count of 45,479 messages all reported ✔ for 77
|
|
954
|
+
* days while zero mail was ingested.
|
|
955
|
+
*
|
|
956
|
+
* Which signal is honest depends on where the store lives, so the probe, the
|
|
957
|
+
* activity wording and the remediation are all chosen per store kind. Reusing
|
|
958
|
+
* the local path in hosted mode measures a directory nothing writes to.
|
|
959
|
+
*/
|
|
960
|
+
function mailIngestLivenessCheck(deps, agentDir, mailroomRoot, messagesDir, listing, store) {
|
|
961
|
+
const hosted = store.hosted ? store : null;
|
|
962
|
+
const probe = hosted
|
|
963
|
+
? hostedMailMirrorProbe(deps, agentDir, hosted.label)
|
|
964
|
+
: localMailStoreProbe(deps, messagesDir, listing);
|
|
965
|
+
return pipelineLivenessCheck({
|
|
966
|
+
id: "mail.ingest_liveness",
|
|
967
|
+
label: `${agentDir} mail ingest liveness`,
|
|
968
|
+
activity: hosted ? "hosted mail observed locally" : "mail ingested",
|
|
969
|
+
unit: "message",
|
|
970
|
+
probe,
|
|
971
|
+
thresholds: (0, freshness_1.resolveFreshnessThresholds)(exports.DEFAULT_MAIL_INGEST_THRESHOLDS, senseFreshnessOverride(deps, agentDir, "mail")),
|
|
972
|
+
remediation: hosted
|
|
973
|
+
? `doctor measures hosted mail only through this machine's local mirror and makes no network calls — confirm the container itself by listing \`messages/\` blobs in ${hosted.label} by Last-Modified, or read the mailbox directly (\`ouro mailbox\`, or the agent's \`mail_recent\` tool); if the mirror is empty or stale while the container is current, the hosted reader on this machine is not running`
|
|
974
|
+
: "mail is configured but nothing is arriving — re-check the mailbox grant and keyIds against the vault (a server-side key rotation silently orphans ingestion), run `ouro connect mail --agent <agent>`, and inspect the mailroom ingress logs",
|
|
975
|
+
configuredSinceMs: pathMtimeMs(deps, `${mailroomRoot}/registry.json`),
|
|
976
|
+
context: hosted ? `mailbox configured; ${hosted.label}` : "mailbox configured",
|
|
977
|
+
nowMs: Date.now(),
|
|
978
|
+
});
|
|
979
|
+
}
|
|
762
980
|
function checkFriends(deps) {
|
|
763
981
|
const checks = [];
|
|
764
982
|
const agents = discoverAgents(deps);
|
|
@@ -0,0 +1,343 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/**
|
|
3
|
+
* Pipeline freshness — "is this pipe still moving?"
|
|
4
|
+
*
|
|
5
|
+
* Every outage this module exists to prevent had the same shape: a health
|
|
6
|
+
* check verified that something was *configured* or *connected*, and reported
|
|
7
|
+
* a green tick next to a pipe that had been dead for weeks.
|
|
8
|
+
*
|
|
9
|
+
* - mail: `mail enabled` / `mail config ok` / `45479 messages` all passed
|
|
10
|
+
* while zero mail had been ingested for 77 days after a vault key rotation
|
|
11
|
+
* changed the mailbox keyIds. The message count is cumulative, so it can
|
|
12
|
+
* only ever go up — it is not a liveness signal.
|
|
13
|
+
* - bluebubbles: the upstream HTTP probe stayed `ok` for days while inbound
|
|
14
|
+
* delivery was dead, because the server was POSTing its webhook to a
|
|
15
|
+
* stale port. A reachable upstream does not prove inbound delivery.
|
|
16
|
+
* - launchd: checks asserted a plist *file* existed rather than that the
|
|
17
|
+
* job was loaded.
|
|
18
|
+
*
|
|
19
|
+
* The fix is to assert on *observed data flow*, with three rules baked into
|
|
20
|
+
* this primitive:
|
|
21
|
+
*
|
|
22
|
+
* 1. Age is always stated in the message, in human units, with the
|
|
23
|
+
* timestamp it was derived from.
|
|
24
|
+
* 2. "Never observed any activity" and "could not observe" are distinct
|
|
25
|
+
* states from "activity, but stale" — and none of them can read as
|
|
26
|
+
* `pass`. An unobservable pipe is unverified, not healthy.
|
|
27
|
+
* 3. Every non-`pass` result carries a concrete remediation hint.
|
|
28
|
+
*
|
|
29
|
+
* Thresholds are always supplied by the caller so freshness policy lives with
|
|
30
|
+
* the pipe it describes, not in this file.
|
|
31
|
+
*/
|
|
32
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
33
|
+
exports.DEFAULT_CLOCK_SKEW_TOLERANCE_MS = exports.FRESHNESS_DAY_MS = exports.FRESHNESS_HOUR_MS = exports.FRESHNESS_MINUTE_MS = void 0;
|
|
34
|
+
exports.formatFreshnessAge = formatFreshnessAge;
|
|
35
|
+
exports.formatFreshnessTimestamp = formatFreshnessTimestamp;
|
|
36
|
+
exports.evaluateFreshness = evaluateFreshness;
|
|
37
|
+
exports.observeCreatePerEventStore = observeCreatePerEventStore;
|
|
38
|
+
exports.observeMirroredStore = observeMirroredStore;
|
|
39
|
+
exports.observeAppendPerEventStore = observeAppendPerEventStore;
|
|
40
|
+
exports.resolveFreshnessThresholds = resolveFreshnessThresholds;
|
|
41
|
+
const runtime_1 = require("../../nerves/runtime");
|
|
42
|
+
exports.FRESHNESS_MINUTE_MS = 60 * 1000;
|
|
43
|
+
exports.FRESHNESS_HOUR_MS = 60 * exports.FRESHNESS_MINUTE_MS;
|
|
44
|
+
exports.FRESHNESS_DAY_MS = 24 * exports.FRESHNESS_HOUR_MS;
|
|
45
|
+
/** Small tolerance before a future timestamp is treated as clock skew. */
|
|
46
|
+
exports.DEFAULT_CLOCK_SKEW_TOLERANCE_MS = exports.FRESHNESS_MINUTE_MS;
|
|
47
|
+
/**
|
|
48
|
+
* Human-readable duration. Always rounds down so the age is never overstated:
|
|
49
|
+
* "77 days" means at least 77 days have elapsed.
|
|
50
|
+
*/
|
|
51
|
+
function formatFreshnessAge(ms) {
|
|
52
|
+
const clamped = Math.max(0, ms);
|
|
53
|
+
const seconds = Math.floor(clamped / 1000);
|
|
54
|
+
if (seconds < 60)
|
|
55
|
+
return `${seconds} second${seconds === 1 ? "" : "s"}`;
|
|
56
|
+
const minutes = Math.floor(clamped / exports.FRESHNESS_MINUTE_MS);
|
|
57
|
+
if (minutes < 60)
|
|
58
|
+
return `${minutes} minute${minutes === 1 ? "" : "s"}`;
|
|
59
|
+
const hours = Math.floor(clamped / exports.FRESHNESS_HOUR_MS);
|
|
60
|
+
if (hours < 48)
|
|
61
|
+
return `${hours} hour${hours === 1 ? "" : "s"}`;
|
|
62
|
+
// The day bucket only starts at 48h, so it is always plural.
|
|
63
|
+
return `${Math.floor(clamped / exports.FRESHNESS_DAY_MS)} days`;
|
|
64
|
+
}
|
|
65
|
+
/** Minute-precision ISO instant — keeps the incident date visible and unambiguous. */
|
|
66
|
+
function formatFreshnessTimestamp(ms) {
|
|
67
|
+
return new Date(ms).toISOString().replace(/:\d{2}\.\d{3}Z$/, "Z");
|
|
68
|
+
}
|
|
69
|
+
function joinDetail(parts) {
|
|
70
|
+
return parts.filter((part) => !!part && part.trim().length > 0).join("; ");
|
|
71
|
+
}
|
|
72
|
+
function statusForAge(ageMs, thresholds) {
|
|
73
|
+
if (ageMs >= thresholds.failAfterMs)
|
|
74
|
+
return "fail";
|
|
75
|
+
if (ageMs >= thresholds.warnAfterMs)
|
|
76
|
+
return "warn";
|
|
77
|
+
return "pass";
|
|
78
|
+
}
|
|
79
|
+
/**
|
|
80
|
+
* Severity for "readable store, nothing ever recorded".
|
|
81
|
+
*
|
|
82
|
+
* Floors at `warn` on purpose: a configured-but-never-delivered pipe is never
|
|
83
|
+
* healthy, but a pipe wired up minutes ago does not deserve a hard failure.
|
|
84
|
+
*/
|
|
85
|
+
function statusForNever(input) {
|
|
86
|
+
const since = input.configuredSinceMs;
|
|
87
|
+
if (typeof since !== "number" || !Number.isFinite(since)) {
|
|
88
|
+
return { status: "fail", configuredForMs: null };
|
|
89
|
+
}
|
|
90
|
+
const configuredForMs = Math.max(0, input.nowMs - since);
|
|
91
|
+
const escalated = statusForAge(configuredForMs, input.thresholds);
|
|
92
|
+
return { status: escalated === "pass" ? "warn" : escalated, configuredForMs };
|
|
93
|
+
}
|
|
94
|
+
function evaluateActivity(input, atMs) {
|
|
95
|
+
const tolerance = input.clockSkewToleranceMs ?? exports.DEFAULT_CLOCK_SKEW_TOLERANCE_MS;
|
|
96
|
+
const skewMs = atMs - input.nowMs;
|
|
97
|
+
const timestamp = formatFreshnessTimestamp(atMs);
|
|
98
|
+
if (skewMs > tolerance) {
|
|
99
|
+
return {
|
|
100
|
+
status: "warn",
|
|
101
|
+
state: "future",
|
|
102
|
+
ageMs: 0,
|
|
103
|
+
detail: joinDetail([
|
|
104
|
+
input.context,
|
|
105
|
+
`last ${input.unit} is timestamped ${timestamp}, ${formatFreshnessAge(skewMs)} in the future — clock skew means freshness cannot be trusted`,
|
|
106
|
+
input.provenance,
|
|
107
|
+
`fix: ${input.remediation}`,
|
|
108
|
+
]),
|
|
109
|
+
};
|
|
110
|
+
}
|
|
111
|
+
const ageMs = Math.max(0, -skewMs);
|
|
112
|
+
const status = statusForAge(ageMs, input.thresholds);
|
|
113
|
+
if (status === "pass") {
|
|
114
|
+
return {
|
|
115
|
+
status,
|
|
116
|
+
state: "fresh",
|
|
117
|
+
ageMs,
|
|
118
|
+
detail: joinDetail([
|
|
119
|
+
input.context,
|
|
120
|
+
`${input.activity} ${formatFreshnessAge(ageMs)} ago (last ${input.unit} ${timestamp})`,
|
|
121
|
+
input.provenance,
|
|
122
|
+
]),
|
|
123
|
+
};
|
|
124
|
+
}
|
|
125
|
+
return {
|
|
126
|
+
status,
|
|
127
|
+
state: "stale",
|
|
128
|
+
ageMs,
|
|
129
|
+
detail: joinDetail([
|
|
130
|
+
input.context,
|
|
131
|
+
`no ${input.activity} in ${formatFreshnessAge(ageMs)} — last ${input.unit} ${timestamp}`,
|
|
132
|
+
input.provenance,
|
|
133
|
+
`fix: ${input.remediation}`,
|
|
134
|
+
]),
|
|
135
|
+
};
|
|
136
|
+
}
|
|
137
|
+
/**
|
|
138
|
+
* Turn a single observation into a status + actionable detail.
|
|
139
|
+
*
|
|
140
|
+
* The only path that can return `pass` is an observed activity timestamp
|
|
141
|
+
* inside the warn threshold. Everything else — no activity, unreadable store,
|
|
142
|
+
* skewed clock — is surfaced, never swallowed.
|
|
143
|
+
*/
|
|
144
|
+
function evaluateFreshness(input) {
|
|
145
|
+
const observation = input.observation;
|
|
146
|
+
if (observation.kind === "unknown") {
|
|
147
|
+
return {
|
|
148
|
+
status: "fail",
|
|
149
|
+
state: "unknown",
|
|
150
|
+
ageMs: null,
|
|
151
|
+
detail: joinDetail([
|
|
152
|
+
input.context,
|
|
153
|
+
`${input.activity}: last-activity time could not be determined — ${observation.reason}; unverified, not healthy`,
|
|
154
|
+
input.provenance,
|
|
155
|
+
`fix: ${input.remediation}`,
|
|
156
|
+
]),
|
|
157
|
+
};
|
|
158
|
+
}
|
|
159
|
+
if (observation.kind === "none") {
|
|
160
|
+
const { status, configuredForMs } = statusForNever(input);
|
|
161
|
+
return {
|
|
162
|
+
status,
|
|
163
|
+
state: "never",
|
|
164
|
+
ageMs: null,
|
|
165
|
+
detail: joinDetail([
|
|
166
|
+
input.context,
|
|
167
|
+
configuredForMs === null
|
|
168
|
+
? `no ${input.activity} ever — the store is readable and empty`
|
|
169
|
+
: `no ${input.activity} ever — the store is readable and empty, and this pipe has been configured for ${formatFreshnessAge(configuredForMs)}`,
|
|
170
|
+
input.provenance,
|
|
171
|
+
`fix: ${input.remediation}`,
|
|
172
|
+
]),
|
|
173
|
+
};
|
|
174
|
+
}
|
|
175
|
+
if (!Number.isFinite(observation.atMs)) {
|
|
176
|
+
return evaluateFreshness({
|
|
177
|
+
...input,
|
|
178
|
+
observation: { kind: "unknown", reason: `observed timestamp is not a finite epoch-ms value (${String(observation.atMs)})` },
|
|
179
|
+
});
|
|
180
|
+
}
|
|
181
|
+
return evaluateActivity(input, observation.atMs);
|
|
182
|
+
}
|
|
183
|
+
/**
|
|
184
|
+
* Last-write probe for **create-per-event** stores — one new file per event,
|
|
185
|
+
* such as the mailroom's `messages/<id>.json`.
|
|
186
|
+
*
|
|
187
|
+
* A directory's mtime advances whenever an entry is created or removed, so it
|
|
188
|
+
* is an exact answer to "when was a file last added here" for O(1) cost, with
|
|
189
|
+
* no per-entry stat. That matters: the production mailroom holds ~45k message
|
|
190
|
+
* files and must never be stat-walked by a health check.
|
|
191
|
+
*
|
|
192
|
+
* It is also the semantically *correct* signal here. A message's `receivedAt`
|
|
193
|
+
* is when the mail was sent, not when this machine ingested it — an mbox
|
|
194
|
+
* backfill writes month-old `receivedAt` values today — so parsing message
|
|
195
|
+
* bodies would answer a different question than "is the pipe moving?".
|
|
196
|
+
*
|
|
197
|
+
* `hasEntries` is supplied by the caller (which typically already lists the
|
|
198
|
+
* directory for a count) so an empty directory reports `none` rather than
|
|
199
|
+
* reporting its own creation time as activity.
|
|
200
|
+
*/
|
|
201
|
+
function observeCreatePerEventStore(dir, deps, options) {
|
|
202
|
+
const provenance = `derived from ${dir} directory mtime (O(1), no per-entry scan)`;
|
|
203
|
+
if (!deps.existsSync(dir)) {
|
|
204
|
+
return { observation: { kind: "none" }, provenance: `${dir} does not exist` };
|
|
205
|
+
}
|
|
206
|
+
if (!options.hasEntries) {
|
|
207
|
+
return { observation: { kind: "none" }, provenance: `${dir} is empty` };
|
|
208
|
+
}
|
|
209
|
+
try {
|
|
210
|
+
return { observation: { kind: "activity", atMs: deps.statSync(dir).mtimeMs }, provenance };
|
|
211
|
+
}
|
|
212
|
+
catch (error) {
|
|
213
|
+
return {
|
|
214
|
+
observation: { kind: "unknown", reason: `${dir} could not be stat'd: ${error instanceof Error ? error.message : String(error)}` },
|
|
215
|
+
provenance,
|
|
216
|
+
};
|
|
217
|
+
}
|
|
218
|
+
}
|
|
219
|
+
/**
|
|
220
|
+
* Last-observation probe for a **local mirror of a remote store** — the cache
|
|
221
|
+
* this machine writes as it reads a store that lives somewhere else, such as
|
|
222
|
+
* the hosted Mailroom's `state/mail-search/<messageId>.json` documents.
|
|
223
|
+
*
|
|
224
|
+
* Mechanically this is `observeCreatePerEventStore`: one file per event, named
|
|
225
|
+
* by a content-addressed id, so the directory mtime answers "when did this
|
|
226
|
+
* machine last record something it had never seen before" for O(1) cost.
|
|
227
|
+
*
|
|
228
|
+
* One rule differs, and it is the entire reason this exists: an absent or empty
|
|
229
|
+
* mirror is `unknown`, never `none`. For a local store the directory *is* the
|
|
230
|
+
* store, so empty means "nothing was ever delivered". For a mirror the
|
|
231
|
+
* authoritative store is remote and deliberately not read here — a health check
|
|
232
|
+
* must not need network access or credentials — so empty means only "this
|
|
233
|
+
* machine holds no record". That is absence of evidence, and rendering it as a
|
|
234
|
+
* confident "never delivered" would be the same crying-wolf failure as reading
|
|
235
|
+
* a local directory that nothing writes to any more.
|
|
236
|
+
*/
|
|
237
|
+
function observeMirroredStore(dir, deps, options) {
|
|
238
|
+
const provenance = `derived from the local mirror at ${dir} (directory mtime, O(1), no per-entry scan); the authoritative store is ${options.remote}, which doctor does not read`;
|
|
239
|
+
const exists = deps.existsSync(dir);
|
|
240
|
+
if (!exists || !options.hasEntries) {
|
|
241
|
+
return {
|
|
242
|
+
observation: {
|
|
243
|
+
kind: "unknown",
|
|
244
|
+
reason: `the local mirror at ${dir} is ${exists ? "empty" : "absent"}, and ${options.remote} is not read by doctor (no network calls, no credentials), so recency cannot be measured on this machine`,
|
|
245
|
+
},
|
|
246
|
+
provenance,
|
|
247
|
+
};
|
|
248
|
+
}
|
|
249
|
+
try {
|
|
250
|
+
return { observation: { kind: "activity", atMs: deps.statSync(dir).mtimeMs }, provenance };
|
|
251
|
+
}
|
|
252
|
+
catch (error) {
|
|
253
|
+
return {
|
|
254
|
+
observation: { kind: "unknown", reason: `${dir} could not be stat'd: ${error instanceof Error ? error.message : String(error)}` },
|
|
255
|
+
provenance,
|
|
256
|
+
};
|
|
257
|
+
}
|
|
258
|
+
}
|
|
259
|
+
/**
|
|
260
|
+
* Last-write probe for **append-per-event** stores — a small set of long-lived
|
|
261
|
+
* logs appended in place, such as the BlueBubbles inbound ndjson (one file per
|
|
262
|
+
* chat).
|
|
263
|
+
*
|
|
264
|
+
* Directory mtime is the wrong signal here: appending to an existing file does
|
|
265
|
+
* not touch its parent directory, so a busy-but-not-growing log set would look
|
|
266
|
+
* frozen. This stats each matching entry and takes the newest.
|
|
267
|
+
*
|
|
268
|
+
* The scan is bounded by `maxEntries`. When the bound is hit the provenance
|
|
269
|
+
* says so explicitly — a bounded scan must never masquerade as exhaustive.
|
|
270
|
+
*/
|
|
271
|
+
function observeAppendPerEventStore(dir, deps, options) {
|
|
272
|
+
if (!deps.existsSync(dir)) {
|
|
273
|
+
return { observation: { kind: "none" }, provenance: `${dir} does not exist` };
|
|
274
|
+
}
|
|
275
|
+
let entries;
|
|
276
|
+
try {
|
|
277
|
+
entries = deps.readdirSync(dir).filter((name) => name.endsWith(options.suffix));
|
|
278
|
+
}
|
|
279
|
+
catch (error) {
|
|
280
|
+
return {
|
|
281
|
+
observation: { kind: "unknown", reason: `${dir} could not be listed: ${error instanceof Error ? error.message : String(error)}` },
|
|
282
|
+
provenance: `attempted directory listing of ${dir}`,
|
|
283
|
+
};
|
|
284
|
+
}
|
|
285
|
+
if (entries.length === 0) {
|
|
286
|
+
return { observation: { kind: "none" }, provenance: `${dir} holds no ${options.suffix} logs` };
|
|
287
|
+
}
|
|
288
|
+
const scanned = entries.slice(0, options.maxEntries);
|
|
289
|
+
const truncated = entries.length > scanned.length;
|
|
290
|
+
const provenance = truncated
|
|
291
|
+
? `newest mtime across the first ${scanned.length} of ${entries.length} ${options.suffix} logs in ${dir} — scan bounded, newer activity may exist in the unscanned ${entries.length - scanned.length}`
|
|
292
|
+
: `newest mtime across ${scanned.length} ${options.suffix} log${scanned.length === 1 ? "" : "s"} in ${dir}`;
|
|
293
|
+
let newest = Number.NEGATIVE_INFINITY;
|
|
294
|
+
const failures = [];
|
|
295
|
+
for (const name of scanned) {
|
|
296
|
+
try {
|
|
297
|
+
const mtimeMs = deps.statSync(`${dir}/${name}`).mtimeMs;
|
|
298
|
+
if (mtimeMs > newest)
|
|
299
|
+
newest = mtimeMs;
|
|
300
|
+
}
|
|
301
|
+
catch (error) {
|
|
302
|
+
failures.push(`${name}: ${error instanceof Error ? error.message : String(error)}`);
|
|
303
|
+
}
|
|
304
|
+
}
|
|
305
|
+
if (newest === Number.NEGATIVE_INFINITY) {
|
|
306
|
+
return {
|
|
307
|
+
observation: { kind: "unknown", reason: `none of the ${scanned.length} log(s) in ${dir} could be stat'd: ${failures.slice(0, 3).join("; ")}` },
|
|
308
|
+
provenance,
|
|
309
|
+
};
|
|
310
|
+
}
|
|
311
|
+
return {
|
|
312
|
+
observation: { kind: "activity", atMs: newest },
|
|
313
|
+
provenance: failures.length > 0 ? `${provenance} (${failures.length} unreadable and skipped)` : provenance,
|
|
314
|
+
};
|
|
315
|
+
}
|
|
316
|
+
/**
|
|
317
|
+
* Merge an `agent.json` override onto in-repo defaults.
|
|
318
|
+
*
|
|
319
|
+
* Configuration lives in a committed config file rather than an environment
|
|
320
|
+
* variable, per the repo configuration policy. Non-positive and non-finite
|
|
321
|
+
* values are ignored so a malformed override can never disable the check.
|
|
322
|
+
*/
|
|
323
|
+
function resolveFreshnessThresholds(defaults, override) {
|
|
324
|
+
const record = override && typeof override === "object" && !Array.isArray(override)
|
|
325
|
+
? override
|
|
326
|
+
: null;
|
|
327
|
+
const hours = (key) => {
|
|
328
|
+
const value = record?.[key];
|
|
329
|
+
return typeof value === "number" && Number.isFinite(value) && value > 0 ? value * exports.FRESHNESS_HOUR_MS : null;
|
|
330
|
+
};
|
|
331
|
+
return {
|
|
332
|
+
warnAfterMs: hours("warnAfterHours") ?? defaults.warnAfterMs,
|
|
333
|
+
failAfterMs: hours("failAfterHours") ?? defaults.failAfterMs,
|
|
334
|
+
};
|
|
335
|
+
}
|
|
336
|
+
/* v8 ignore start -- module load observability event */
|
|
337
|
+
(0, runtime_1.emitNervesEvent)({
|
|
338
|
+
component: "daemon",
|
|
339
|
+
event: "daemon.pipeline_freshness_loaded",
|
|
340
|
+
message: "pipeline freshness primitive loaded",
|
|
341
|
+
meta: {},
|
|
342
|
+
});
|
|
343
|
+
/* v8 ignore stop */
|