@ouro.bot/cli 0.1.0-alpha.720 → 0.1.0-alpha.722
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/changelog.json +12 -0
- package/dist/arc/flight-recorder.js +112 -6
- package/dist/heart/daemon/doctor.js +163 -12
- package/dist/heart/daemon/freshness.js +302 -0
- package/dist/heart/habits/habit-session.js +7 -4
- package/dist/heart/mailbox/mailbox-http-hooks.js +2 -0
- package/dist/heart/mailbox/mailbox-http-routes.js +23 -0
- package/dist/heart/mailbox/mailbox-read.js +3 -1
- package/dist/heart/mailbox/readers/runtime-readers.js +9 -0
- package/dist/rsvp/native-habit-runner.js +173 -5
- package/dist/senses/bluebubbles/context-packet.js +22 -1
- package/dist/senses/bluebubbles/index.js +54 -28
- package/dist/senses/context-packet-ledger.js +126 -0
- package/dist/senses/pipeline.js +37 -1
- package/package.json +1 -1
package/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Arye Mendelow
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
package/changelog.json
CHANGED
|
@@ -1,6 +1,18 @@
|
|
|
1
1
|
{
|
|
2
2
|
"_note": "This changelog is maintained as part of the PR/version-bump workflow. Agent-curated, not auto-generated. Agents read this file directly via read_file to understand what changed between versions.",
|
|
3
3
|
"versions": [
|
|
4
|
+
{
|
|
5
|
+
"version": "0.1.0-alpha.722",
|
|
6
|
+
"changes": [
|
|
7
|
+
"ouro doctor now asserts pipeline liveness (mail ingest age, sense inbound-delivery age) so a configured-but-dead pipe fails loudly instead of reporting healthy."
|
|
8
|
+
]
|
|
9
|
+
},
|
|
10
|
+
{
|
|
11
|
+
"version": "0.1.0-alpha.721",
|
|
12
|
+
"changes": [
|
|
13
|
+
"Make Slugger context hydration and native habit traces inspectable and coverage-gated."
|
|
14
|
+
]
|
|
15
|
+
},
|
|
4
16
|
{
|
|
5
17
|
"version": "0.1.0-alpha.720",
|
|
6
18
|
"changes": [
|
|
@@ -35,6 +35,7 @@ var __importStar = (this && this.__importStar) || (function () {
|
|
|
35
35
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
36
36
|
exports.isHabitRunTrigger = isHabitRunTrigger;
|
|
37
37
|
exports.flightRecorderLatestPath = flightRecorderLatestPath;
|
|
38
|
+
exports.decisionsFromHabitRunTraceSteps = decisionsFromHabitRunTraceSteps;
|
|
38
39
|
exports.isFlightRecorderResume = isFlightRecorderResume;
|
|
39
40
|
exports.readFlightRecorderResume = readFlightRecorderResume;
|
|
40
41
|
exports.writeFlightRecorderResume = writeFlightRecorderResume;
|
|
@@ -86,6 +87,17 @@ function atomicWriteJson(filePath, value) {
|
|
|
86
87
|
function cappedArray(values) {
|
|
87
88
|
return values.map((value) => (0, session_events_1.capStructuredRecordString)(value));
|
|
88
89
|
}
|
|
90
|
+
function decisionsFromHabitRunTraceSteps(traceSteps = []) {
|
|
91
|
+
const decisions = [];
|
|
92
|
+
for (const step of traceSteps) {
|
|
93
|
+
for (const decision of step.decisions ?? []) {
|
|
94
|
+
const trimmed = decision.trim();
|
|
95
|
+
if (trimmed.length > 0)
|
|
96
|
+
decisions.push((0, session_events_1.capStructuredRecordString)(trimmed));
|
|
97
|
+
}
|
|
98
|
+
}
|
|
99
|
+
return uniqueStrings(decisions);
|
|
100
|
+
}
|
|
89
101
|
function normalizeEvent(input) {
|
|
90
102
|
const recordedAt = input.recordedAt ?? new Date().toISOString();
|
|
91
103
|
return {
|
|
@@ -604,6 +616,55 @@ function isHabitSurfaceAttemptArray(value) {
|
|
|
604
616
|
&& (entry.rawStatus === undefined || typeof entry.rawStatus === "string")
|
|
605
617
|
&& (entry.error === undefined || typeof entry.error === "string"));
|
|
606
618
|
}
|
|
619
|
+
function isHabitRunTraceStepKind(value) {
|
|
620
|
+
return value === "trigger"
|
|
621
|
+
|| value === "habit_definition"
|
|
622
|
+
|| value === "fetch"
|
|
623
|
+
|| value === "snapshot"
|
|
624
|
+
|| value === "render"
|
|
625
|
+
|| value === "decision"
|
|
626
|
+
|| value === "produced_ref"
|
|
627
|
+
|| value === "surface_attempt"
|
|
628
|
+
|| value === "send"
|
|
629
|
+
|| value === "ledger"
|
|
630
|
+
|| value === "error"
|
|
631
|
+
|| value === "complete";
|
|
632
|
+
}
|
|
633
|
+
function isHabitRunTraceStepStatus(value) {
|
|
634
|
+
return value === "started"
|
|
635
|
+
|| value === "succeeded"
|
|
636
|
+
|| value === "skipped"
|
|
637
|
+
|| value === "blocked"
|
|
638
|
+
|| value === "failed";
|
|
639
|
+
}
|
|
640
|
+
function isHabitRunTraceRefArray(value) {
|
|
641
|
+
return Array.isArray(value)
|
|
642
|
+
&& value.every((entry) => isPlainRecord(entry)
|
|
643
|
+
&& typeof entry.kind === "string"
|
|
644
|
+
&& entry.kind.trim().length > 0
|
|
645
|
+
&& typeof entry.locator === "string"
|
|
646
|
+
&& entry.locator.trim().length > 0
|
|
647
|
+
&& (entry.label === undefined || typeof entry.label === "string"));
|
|
648
|
+
}
|
|
649
|
+
function isHabitRunTraceStepArray(value) {
|
|
650
|
+
return Array.isArray(value)
|
|
651
|
+
&& value.every((entry) => isPlainRecord(entry)
|
|
652
|
+
&& entry.schemaVersion === 1
|
|
653
|
+
&& typeof entry.stepId === "string"
|
|
654
|
+
&& entry.stepId.trim().length > 0
|
|
655
|
+
&& isHabitRunTraceStepKind(entry.kind)
|
|
656
|
+
&& isHabitRunTraceStepStatus(entry.status)
|
|
657
|
+
&& typeof entry.at === "string"
|
|
658
|
+
&& entry.at.trim().length > 0
|
|
659
|
+
&& typeof entry.summary === "string"
|
|
660
|
+
&& entry.summary.trim().length > 0
|
|
661
|
+
&& (entry.refs === undefined || isHabitRunTraceRefArray(entry.refs))
|
|
662
|
+
&& (entry.producedRefs === undefined || isProducedRefArray(entry.producedRefs))
|
|
663
|
+
&& (entry.decisions === undefined || isStringArray(entry.decisions))
|
|
664
|
+
&& (entry.surfaceAttempt === undefined || isHabitSurfaceAttemptArray([entry.surfaceAttempt]))
|
|
665
|
+
&& (entry.error === undefined || typeof entry.error === "string")
|
|
666
|
+
&& (entry.meta === undefined || isPlainRecord(entry.meta)));
|
|
667
|
+
}
|
|
607
668
|
function isHabitReturnRouteArray(value) {
|
|
608
669
|
return Array.isArray(value)
|
|
609
670
|
&& value.every((entry) => isPlainRecord(entry)
|
|
@@ -633,10 +694,11 @@ function isHabitToolPolicy(value) {
|
|
|
633
694
|
&& typeof value.outwardMessagingAllowed === "boolean";
|
|
634
695
|
}
|
|
635
696
|
function defaultHabitRunSummarySnapshot(receipt) {
|
|
697
|
+
const decisions = decisionsFromHabitRunTraceSteps(receipt.traceSteps);
|
|
636
698
|
if (receipt.errors.length > 0) {
|
|
637
699
|
return {
|
|
638
700
|
summary: `Habit ${receipt.habitName} finished with errors: ${receipt.errors.join("; ")}`,
|
|
639
|
-
decisions
|
|
701
|
+
decisions,
|
|
640
702
|
nextLikelyStep: null,
|
|
641
703
|
};
|
|
642
704
|
}
|
|
@@ -644,7 +706,7 @@ function defaultHabitRunSummarySnapshot(receipt) {
|
|
|
644
706
|
if (surface) {
|
|
645
707
|
return {
|
|
646
708
|
summary: `Habit ${receipt.habitName} surfaced via ${surface.recipient}/${surface.channel}.`,
|
|
647
|
-
decisions
|
|
709
|
+
decisions,
|
|
648
710
|
nextLikelyStep: null,
|
|
649
711
|
};
|
|
650
712
|
}
|
|
@@ -652,13 +714,13 @@ function defaultHabitRunSummarySnapshot(receipt) {
|
|
|
652
714
|
if (produced) {
|
|
653
715
|
return {
|
|
654
716
|
summary: `Habit ${receipt.habitName} produced ${produced.kind}: ${produced.locator}.`,
|
|
655
|
-
decisions
|
|
717
|
+
decisions,
|
|
656
718
|
nextLikelyStep: null,
|
|
657
719
|
};
|
|
658
720
|
}
|
|
659
721
|
return {
|
|
660
722
|
summary: `Habit ${receipt.habitName} finished with ${receipt.outcome}.`,
|
|
661
|
-
decisions
|
|
723
|
+
decisions,
|
|
662
724
|
nextLikelyStep: null,
|
|
663
725
|
};
|
|
664
726
|
}
|
|
@@ -674,7 +736,7 @@ function normalizeHabitRunSummarySnapshot(value, fallback) {
|
|
|
674
736
|
: fallback.nextLikelyStep;
|
|
675
737
|
return {
|
|
676
738
|
summary: (0, session_events_1.capStructuredRecordString)(summary),
|
|
677
|
-
decisions: cappedArray(isStringArray(snapshot.decisions) ? snapshot.decisions : fallback.decisions),
|
|
739
|
+
decisions: cappedArray(isStringArray(snapshot.decisions) && snapshot.decisions.length > 0 ? snapshot.decisions : fallback.decisions),
|
|
678
740
|
nextLikelyStep: nextLikelyStep === null ? null : (0, session_events_1.capStructuredRecordString)(nextLikelyStep),
|
|
679
741
|
};
|
|
680
742
|
}
|
|
@@ -711,6 +773,7 @@ function isHabitRunReceipt(value) {
|
|
|
711
773
|
&& (value.summarySnapshot === undefined || isPlainRecord(value.summarySnapshot))
|
|
712
774
|
&& isProducedRefArray(value.producedRefs)
|
|
713
775
|
&& isHabitSurfaceAttemptArray(value.surfaceAttempts)
|
|
776
|
+
&& (value.traceSteps === undefined || isHabitRunTraceStepArray(value.traceSteps))
|
|
714
777
|
&& isStringArray(value.errors);
|
|
715
778
|
}
|
|
716
779
|
function isLegacyHabitRunReceipt(value) {
|
|
@@ -780,11 +843,53 @@ function normalizeLegacyHabitRunReceipt(receipt) {
|
|
|
780
843
|
summarySnapshot: defaultHabitRunSummarySnapshot(receipt),
|
|
781
844
|
producedRefs: receipt.producedRefs,
|
|
782
845
|
surfaceAttempts: receipt.surfaceAttempts,
|
|
846
|
+
traceSteps: [],
|
|
783
847
|
errors: receipt.errors,
|
|
784
848
|
};
|
|
785
849
|
}
|
|
850
|
+
function capHabitRunTraceRefs(refs) {
|
|
851
|
+
if (!refs || refs.length === 0)
|
|
852
|
+
return undefined;
|
|
853
|
+
return refs.map((ref) => ({
|
|
854
|
+
kind: (0, session_events_1.capStructuredRecordString)(ref.kind),
|
|
855
|
+
locator: (0, session_events_1.capStructuredRecordString)(ref.locator),
|
|
856
|
+
...(ref.label ? { label: (0, session_events_1.capStructuredRecordString)(ref.label) } : {}),
|
|
857
|
+
}));
|
|
858
|
+
}
|
|
859
|
+
function capHabitRunTraceStep(step) {
|
|
860
|
+
const refs = capHabitRunTraceRefs(step.refs);
|
|
861
|
+
const producedRefs = step.producedRefs?.map((ref) => ({ ...ref, locator: (0, session_events_1.capStructuredRecordString)(ref.locator) }));
|
|
862
|
+
const decisions = step.decisions ? cappedArray(step.decisions) : undefined;
|
|
863
|
+
const surfaceAttempt = step.surfaceAttempt
|
|
864
|
+
? {
|
|
865
|
+
...step.surfaceAttempt,
|
|
866
|
+
recipient: (0, session_events_1.capStructuredRecordString)(step.surfaceAttempt.recipient),
|
|
867
|
+
channel: (0, session_events_1.capStructuredRecordString)(step.surfaceAttempt.channel),
|
|
868
|
+
...(step.surfaceAttempt.rawStatus ? { rawStatus: (0, session_events_1.capStructuredRecordString)(step.surfaceAttempt.rawStatus) } : {}),
|
|
869
|
+
...(step.surfaceAttempt.error ? { error: (0, session_events_1.capStructuredRecordString)(step.surfaceAttempt.error) } : {}),
|
|
870
|
+
}
|
|
871
|
+
: undefined;
|
|
872
|
+
return {
|
|
873
|
+
schemaVersion: 1,
|
|
874
|
+
stepId: (0, session_events_1.capStructuredRecordString)(step.stepId),
|
|
875
|
+
kind: step.kind,
|
|
876
|
+
status: step.status,
|
|
877
|
+
at: (0, session_events_1.capStructuredRecordString)(step.at),
|
|
878
|
+
summary: (0, session_events_1.capStructuredRecordString)(step.summary),
|
|
879
|
+
...(refs ? { refs } : {}),
|
|
880
|
+
...(producedRefs && producedRefs.length > 0 ? { producedRefs } : {}),
|
|
881
|
+
...(decisions && decisions.length > 0 ? { decisions } : {}),
|
|
882
|
+
...(surfaceAttempt ? { surfaceAttempt } : {}),
|
|
883
|
+
...(step.error ? { error: (0, session_events_1.capStructuredRecordString)(step.error) } : {}),
|
|
884
|
+
...(step.meta ? { meta: (0, session_events_1.capStructuredRecordStringLeaves)(step.meta) } : {}),
|
|
885
|
+
};
|
|
886
|
+
}
|
|
887
|
+
function normalizeHabitRunTraceSteps(traceSteps) {
|
|
888
|
+
return (traceSteps ?? []).map((step) => capHabitRunTraceStep(step));
|
|
889
|
+
}
|
|
786
890
|
function capHabitRunReceipt(receipt) {
|
|
787
|
-
const
|
|
891
|
+
const traceSteps = normalizeHabitRunTraceSteps(receipt.traceSteps);
|
|
892
|
+
const fallbackSnapshot = defaultHabitRunSummarySnapshot({ ...receipt, traceSteps });
|
|
788
893
|
return {
|
|
789
894
|
...receipt,
|
|
790
895
|
habitName: (0, session_events_1.capStructuredRecordString)(receipt.habitName),
|
|
@@ -822,6 +927,7 @@ function capHabitRunReceipt(receipt) {
|
|
|
822
927
|
...(attempt.rawStatus ? { rawStatus: (0, session_events_1.capStructuredRecordString)(attempt.rawStatus) } : {}),
|
|
823
928
|
...(attempt.error ? { error: (0, session_events_1.capStructuredRecordString)(attempt.error) } : {}),
|
|
824
929
|
})),
|
|
930
|
+
traceSteps,
|
|
825
931
|
errors: receipt.errors.map((error) => (0, session_events_1.capStructuredRecordString)(error)),
|
|
826
932
|
};
|
|
827
933
|
}
|
|
@@ -7,9 +7,10 @@
|
|
|
7
7
|
* "fail" check and the remaining categories still run.
|
|
8
8
|
*/
|
|
9
9
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
10
|
-
exports.KNOWN_DOCTOR_CATEGORIES = void 0;
|
|
10
|
+
exports.KNOWN_DOCTOR_CATEGORIES = exports.DEFAULT_SENSE_DELIVERY_THRESHOLDS = exports.DEFAULT_MAIL_INGEST_THRESHOLDS = void 0;
|
|
11
11
|
exports.checkCliPath = checkCliPath;
|
|
12
12
|
exports.checkDaemon = checkDaemon;
|
|
13
|
+
exports.senseInboundDeliveryCheck = senseInboundDeliveryCheck;
|
|
13
14
|
exports.checkAgents = checkAgents;
|
|
14
15
|
exports.checkSenses = checkSenses;
|
|
15
16
|
exports.checkHabits = checkHabits;
|
|
@@ -23,6 +24,7 @@ exports.checkLifecycle = checkLifecycle;
|
|
|
23
24
|
exports.runDoctorChecks = runDoctorChecks;
|
|
24
25
|
const runtime_1 = require("../../nerves/runtime");
|
|
25
26
|
const bluebubbles_health_diagnostics_1 = require("./bluebubbles-health-diagnostics");
|
|
27
|
+
const freshness_1 = require("./freshness");
|
|
26
28
|
const ouro_path_installer_1 = require("../versioning/ouro-path-installer");
|
|
27
29
|
const runtime_credentials_1 = require("../runtime-credentials");
|
|
28
30
|
const machine_identity_1 = require("../machine-identity");
|
|
@@ -115,6 +117,102 @@ const SENSITIVE_CONFIG_KEYS = ["apiKey", "token", "secret", "password"];
|
|
|
115
117
|
function credentialKeyLeaks(raw) {
|
|
116
118
|
return SENSITIVE_CONFIG_KEYS.filter((key) => raw.includes(`"${key}"`));
|
|
117
119
|
}
|
|
120
|
+
// ── Pipeline liveness (dead-pipe detection) ──
|
|
121
|
+
//
|
|
122
|
+
// Configuration checks answer "is this wired up?". These answer "is anything
|
|
123
|
+
// actually flowing?". Both are needed: the 77-day mail outage and the
|
|
124
|
+
// multi-day BlueBubbles inbound outage were both invisible to configuration
|
|
125
|
+
// and connection checks that stayed green throughout.
|
|
126
|
+
/**
|
|
127
|
+
* Mail ingest thresholds.
|
|
128
|
+
*
|
|
129
|
+
* A delegated mailbox forwards a human's real inbox, so a full calendar day
|
|
130
|
+
* with zero inbound mail is already anomalous — that is the warn line. Fail
|
|
131
|
+
* waits until 72h so a genuinely quiet weekend on a low-traffic mailbox does
|
|
132
|
+
* not hard-fail doctor; the 2026-05-10 outage would have failed on day four
|
|
133
|
+
* instead of going unnoticed for seventy-seven. Override per agent with
|
|
134
|
+
* `senses.mail.freshness.{warnAfterHours,failAfterHours}` in `agent.json`.
|
|
135
|
+
*/
|
|
136
|
+
exports.DEFAULT_MAIL_INGEST_THRESHOLDS = {
|
|
137
|
+
warnAfterMs: 24 * freshness_1.FRESHNESS_HOUR_MS,
|
|
138
|
+
failAfterMs: 72 * freshness_1.FRESHNESS_HOUR_MS,
|
|
139
|
+
};
|
|
140
|
+
/**
|
|
141
|
+
* Inbound sense-delivery thresholds.
|
|
142
|
+
*
|
|
143
|
+
* Human-driven chat surfaces are bursty — nobody texts an agent every day —
|
|
144
|
+
* so the warn line sits at 72h and failure waits a full week. Seven days
|
|
145
|
+
* without a single inbound message on an attached chat sense is not plausible
|
|
146
|
+
* quiet; it is a dead pipe. Override per agent with
|
|
147
|
+
* `senses.<sense>.freshness.{warnAfterHours,failAfterHours}` in `agent.json`.
|
|
148
|
+
*/
|
|
149
|
+
exports.DEFAULT_SENSE_DELIVERY_THRESHOLDS = {
|
|
150
|
+
warnAfterMs: 72 * freshness_1.FRESHNESS_HOUR_MS,
|
|
151
|
+
failAfterMs: 7 * freshness_1.FRESHNESS_DAY_MS,
|
|
152
|
+
};
|
|
153
|
+
/** Hard bound on how many per-conversation inbound logs a sense probe stats. */
|
|
154
|
+
const SENSE_DELIVERY_MAX_LOG_SCAN = 500;
|
|
155
|
+
function pipelineLivenessCheck(input) {
|
|
156
|
+
const result = (0, freshness_1.evaluateFreshness)({
|
|
157
|
+
activity: input.activity,
|
|
158
|
+
unit: input.unit,
|
|
159
|
+
observation: input.probe.observation,
|
|
160
|
+
provenance: input.probe.provenance,
|
|
161
|
+
nowMs: input.nowMs,
|
|
162
|
+
thresholds: input.thresholds,
|
|
163
|
+
remediation: input.remediation,
|
|
164
|
+
configuredSinceMs: input.configuredSinceMs,
|
|
165
|
+
context: input.context,
|
|
166
|
+
});
|
|
167
|
+
return { id: input.id, label: input.label, status: result.status, detail: result.detail };
|
|
168
|
+
}
|
|
169
|
+
/** Best-effort mtime used as a "this pipe has been wired up since" floor. */
|
|
170
|
+
function pathMtimeMs(deps, filePath) {
|
|
171
|
+
if (!deps.existsSync(filePath))
|
|
172
|
+
return null;
|
|
173
|
+
try {
|
|
174
|
+
return deps.statSync(filePath).mtimeMs;
|
|
175
|
+
}
|
|
176
|
+
catch {
|
|
177
|
+
return null;
|
|
178
|
+
}
|
|
179
|
+
}
|
|
180
|
+
/** Read `senses.<sense>.freshness` from `agent.json`, if present. */
|
|
181
|
+
function senseFreshnessOverride(deps, agentDir, sense) {
|
|
182
|
+
const configPath = `${deps.bundlesRoot}/${agentDir}/agent.json`;
|
|
183
|
+
if (!deps.existsSync(configPath))
|
|
184
|
+
return null;
|
|
185
|
+
try {
|
|
186
|
+
const config = JSON.parse(deps.readFileSync(configPath));
|
|
187
|
+
return asRecord(asRecord(config.senses)?.[sense])?.freshness ?? null;
|
|
188
|
+
}
|
|
189
|
+
catch {
|
|
190
|
+
return null;
|
|
191
|
+
}
|
|
192
|
+
}
|
|
193
|
+
/**
|
|
194
|
+
* Inbound-delivery liveness for a sense that records what it received.
|
|
195
|
+
*
|
|
196
|
+
* This asserts on the *inbound* log, deliberately not on the upstream health
|
|
197
|
+
* probe: BlueBubbles reported `upstreamStatus: ok` for days while inbound
|
|
198
|
+
* delivery was dead because the server was POSTing to a stale webhook port.
|
|
199
|
+
* A sense opts in by calling this with its inbound log directory.
|
|
200
|
+
*/
|
|
201
|
+
function senseInboundDeliveryCheck(input) {
|
|
202
|
+
const suffix = input.logSuffix ?? ".ndjson";
|
|
203
|
+
return pipelineLivenessCheck({
|
|
204
|
+
id: `senses.${input.sense}.inbound_liveness`,
|
|
205
|
+
label: `${input.agentDir} ${input.sense} inbound delivery`,
|
|
206
|
+
activity: `${input.sense} inbound delivery`,
|
|
207
|
+
unit: "inbound message",
|
|
208
|
+
probe: (0, freshness_1.observeAppendPerEventStore)(input.inboundDir, input.deps, { suffix, maxEntries: SENSE_DELIVERY_MAX_LOG_SCAN }),
|
|
209
|
+
thresholds: (0, freshness_1.resolveFreshnessThresholds)(exports.DEFAULT_SENSE_DELIVERY_THRESHOLDS, senseFreshnessOverride(input.deps, input.agentDir, input.sense)),
|
|
210
|
+
remediation: input.remediation,
|
|
211
|
+
configuredSinceMs: input.configuredSincePath ? pathMtimeMs(input.deps, input.configuredSincePath) : null,
|
|
212
|
+
context: input.context,
|
|
213
|
+
nowMs: input.nowMs,
|
|
214
|
+
});
|
|
215
|
+
}
|
|
118
216
|
function checkAgents(deps) {
|
|
119
217
|
const checks = [];
|
|
120
218
|
if (!deps.existsSync(deps.bundlesRoot)) {
|
|
@@ -255,6 +353,7 @@ async function checkSenses(deps) {
|
|
|
255
353
|
status: "pass",
|
|
256
354
|
detail: serverUrl,
|
|
257
355
|
});
|
|
356
|
+
let upstreamContext = "upstream probe not run in this pass";
|
|
258
357
|
if (deps.fetchImpl) {
|
|
259
358
|
const probe = await (0, bluebubbles_health_diagnostics_1.probeBlueBubblesHealth)({
|
|
260
359
|
serverUrl,
|
|
@@ -267,7 +366,21 @@ async function checkSenses(deps) {
|
|
|
267
366
|
status: probe.ok ? "pass" : "fail",
|
|
268
367
|
detail: probe.detail,
|
|
269
368
|
});
|
|
369
|
+
upstreamContext = probe.ok
|
|
370
|
+
? "upstream probe reachable — which does not prove inbound delivery"
|
|
371
|
+
: "upstream probe failing";
|
|
270
372
|
}
|
|
373
|
+
const bluebubblesStateRoot = `${deps.bundlesRoot}/${agentDir}/state/senses/bluebubbles`;
|
|
374
|
+
checks.push(senseInboundDeliveryCheck({
|
|
375
|
+
deps,
|
|
376
|
+
agentDir,
|
|
377
|
+
sense: "bluebubbles",
|
|
378
|
+
inboundDir: `${bluebubblesStateRoot}/inbound`,
|
|
379
|
+
configuredSincePath: bluebubblesStateRoot,
|
|
380
|
+
context: upstreamContext,
|
|
381
|
+
remediation: "no iMessage is reaching this agent — confirm the BlueBubbles server's configured webhook URL/port matches the port this daemon is listening on (a stale webhook port is the known cause), then re-attach with `ouro connect bluebubbles --agent <agent>`",
|
|
382
|
+
nowMs: Date.now(),
|
|
383
|
+
}));
|
|
271
384
|
}
|
|
272
385
|
if (sense === "mail" && senseObj.enabled === true) {
|
|
273
386
|
const runtimeConfig = await (0, runtime_credentials_1.refreshRuntimeCredentialConfig)(agentName, { preserveCachedOnFailure: true });
|
|
@@ -739,26 +852,64 @@ function checkMailroom(deps) {
|
|
|
739
852
|
checks.push({ label: `${agentDir} mailroom`, status: "warn", detail: "registry.json has no mailboxes — provision via `ouro connect mail`" });
|
|
740
853
|
continue;
|
|
741
854
|
}
|
|
742
|
-
let messagesCount = 0;
|
|
743
855
|
const messagesDir = `${mailroomRoot}/messages`;
|
|
744
|
-
|
|
745
|
-
|
|
746
|
-
try {
|
|
747
|
-
messagesCount = deps.readdirSync(messagesDir).filter((name) => name.endsWith(".json")).length;
|
|
748
|
-
}
|
|
749
|
-
catch {
|
|
750
|
-
// ignore — pass detail just won't include the message count
|
|
751
|
-
}
|
|
752
|
-
}
|
|
856
|
+
const listing = listStoredMailMessages(deps, messagesDir);
|
|
857
|
+
/* v8 ignore start -- defensive: pluralization branches depend on filesystem-state fixtures not exhaustively covered @preserve */
|
|
753
858
|
checks.push({
|
|
754
859
|
label: `${agentDir} mailroom`,
|
|
755
860
|
status: "pass",
|
|
756
|
-
detail: `${mailboxes.length} mailbox${mailboxes.length === 1 ? "" : "es"}, ${sourceGrants.length} source grant${sourceGrants.length === 1 ? "" : "s"}, ${
|
|
861
|
+
detail: `${mailboxes.length} mailbox${mailboxes.length === 1 ? "" : "es"}, ${sourceGrants.length} source grant${sourceGrants.length === 1 ? "" : "s"}, ${listing.count} message${listing.count === 1 ? "" : "s"} stored (cumulative — not a liveness signal)`,
|
|
757
862
|
});
|
|
758
863
|
/* v8 ignore stop */
|
|
864
|
+
checks.push(mailIngestLivenessCheck(deps, agentDir, mailroomRoot, messagesDir, listing));
|
|
759
865
|
}
|
|
760
866
|
return { name: "Mailroom", checks };
|
|
761
867
|
}
|
|
868
|
+
function listStoredMailMessages(deps, messagesDir) {
|
|
869
|
+
if (!deps.existsSync(messagesDir))
|
|
870
|
+
return { count: 0, readable: true };
|
|
871
|
+
try {
|
|
872
|
+
return { count: deps.readdirSync(messagesDir).filter((name) => name.endsWith(".json")).length, readable: true };
|
|
873
|
+
}
|
|
874
|
+
catch {
|
|
875
|
+
return { count: 0, readable: false };
|
|
876
|
+
}
|
|
877
|
+
}
|
|
878
|
+
/**
|
|
879
|
+
* Mail-ingest liveness — "is mail still arriving?", as opposed to the check
|
|
880
|
+
* above, which only says "is a mailbox configured?".
|
|
881
|
+
*
|
|
882
|
+
* This is the check that would have caught the 2026-05-10 outage: the mailbox,
|
|
883
|
+
* its config, and a cumulative count of 45,479 messages all reported ✔ for 77
|
|
884
|
+
* days while zero mail was ingested.
|
|
885
|
+
*
|
|
886
|
+
* The signal is the `messages/` directory mtime, which is both the cheapest
|
|
887
|
+
* and the most correct option. Cheapest: one stat regardless of store size,
|
|
888
|
+
* and the production store holds ~45k message files. Most correct: a message's
|
|
889
|
+
* `receivedAt` is when the mail was *sent* (an mbox backfill writes month-old
|
|
890
|
+
* values today), so parsing message bodies would answer a different question
|
|
891
|
+
* than "did this machine ingest anything recently?".
|
|
892
|
+
*/
|
|
893
|
+
function mailIngestLivenessCheck(deps, agentDir, mailroomRoot, messagesDir, listing) {
|
|
894
|
+
const probe = listing.readable
|
|
895
|
+
? (0, freshness_1.observeCreatePerEventStore)(messagesDir, deps, { hasEntries: listing.count > 0 })
|
|
896
|
+
: {
|
|
897
|
+
observation: { kind: "unknown", reason: `${messagesDir} exists but could not be listed` },
|
|
898
|
+
provenance: `attempted directory listing of ${messagesDir}`,
|
|
899
|
+
};
|
|
900
|
+
return pipelineLivenessCheck({
|
|
901
|
+
id: "mail.ingest_liveness",
|
|
902
|
+
label: `${agentDir} mail ingest liveness`,
|
|
903
|
+
activity: "mail ingested",
|
|
904
|
+
unit: "message",
|
|
905
|
+
probe,
|
|
906
|
+
thresholds: (0, freshness_1.resolveFreshnessThresholds)(exports.DEFAULT_MAIL_INGEST_THRESHOLDS, senseFreshnessOverride(deps, agentDir, "mail")),
|
|
907
|
+
remediation: "mail is configured but nothing is arriving — re-check the mailbox grant and keyIds against the vault (a server-side key rotation silently orphans ingestion), run `ouro connect mail --agent <agent>`, and inspect the mailroom ingress logs",
|
|
908
|
+
configuredSinceMs: pathMtimeMs(deps, `${mailroomRoot}/registry.json`),
|
|
909
|
+
context: "mailbox configured",
|
|
910
|
+
nowMs: Date.now(),
|
|
911
|
+
});
|
|
912
|
+
}
|
|
762
913
|
function checkFriends(deps) {
|
|
763
914
|
const checks = [];
|
|
764
915
|
const agents = discoverAgents(deps);
|