@ouro.bot/cli 0.1.0-alpha.527 → 0.1.0-alpha.529
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/changelog.json +17 -0
- package/dist/heart/daemon/agent-service.js +165 -3
- package/dist/heart/daemon/cli-exec.js +22 -2
- package/dist/heart/daemon/cli-render.js +77 -2
- package/dist/heart/daemon/daemon-entry.js +7 -0
- package/dist/heart/daemon/daemon.js +36 -7
- package/dist/heart/daemon/health-monitor.js +25 -0
- package/dist/heart/daemon/sense-manager.js +64 -3
- package/dist/heart/mcp/mcp-server.js +1 -1
- package/dist/senses/bluebubbles/index.js +100 -13
- package/dist/senses/bluebubbles/runtime-state.js +9 -0
- package/package.json +1 -1
package/changelog.json
CHANGED
|
@@ -1,6 +1,23 @@
|
|
|
1
1
|
{
|
|
2
2
|
"_note": "This changelog is maintained as part of the PR/version-bump workflow. Agent-curated, not auto-generated. Agents read this file directly via read_file to understand what changed between versions.",
|
|
3
3
|
"versions": [
|
|
4
|
+
{
|
|
5
|
+
"version": "0.1.0-alpha.529",
|
|
6
|
+
"changes": [
|
|
7
|
+
"MCP agent status now asks the daemon for proof-bearing runtime health, including per-agent worker rows and enabled sense rows with proof method, proof age, failure layer, and recovery counters.",
|
|
8
|
+
"MCP status reports daemon/socket unreachability as an explicit status line instead of hiding it behind otherwise successful diary/session facts.",
|
|
9
|
+
"MCP status now includes its own package version and flags MCP-vs-daemon version mismatches so stale long-lived MCP servers are visible during reliability checks."
|
|
10
|
+
]
|
|
11
|
+
},
|
|
12
|
+
{
|
|
13
|
+
"version": "0.1.0-alpha.528",
|
|
14
|
+
"changes": [
|
|
15
|
+
"Daemon overview health now degrades whenever an enabled sense is configured but not externally proven healthy, and `ouro up` refuses a healthy handoff when runtime senses are degraded instead of treating unreachable iMessage as green.",
|
|
16
|
+
"Managed sense health probes now carry restart ownership, so failed BlueBubbles/iMessage probes trigger daemon-managed recovery while preserving proof method, proof age, failure layer, recovery action, and pending recovery age in status surfaces.",
|
|
17
|
+
"BlueBubbles webhooks now durably capture inbound and mutation sidecars before acknowledging the HTTP request, then process the slower agent turn asynchronously so live iMessage delivery is not blocked by model/runtime latency.",
|
|
18
|
+
"MCP agent status now returns concrete runtime facts instead of terse placeholder text, giving external tools a proof-bearing status surface for Slugger and other agents."
|
|
19
|
+
]
|
|
20
|
+
},
|
|
4
21
|
{
|
|
5
22
|
"version": "0.1.0-alpha.527",
|
|
6
23
|
"changes": [
|
|
@@ -58,6 +58,7 @@ const path = __importStar(require("path"));
|
|
|
58
58
|
const identity_1 = require("../identity");
|
|
59
59
|
const diary_1 = require("../../mind/diary");
|
|
60
60
|
const runtime_1 = require("../../nerves/runtime");
|
|
61
|
+
const socket_client_1 = require("./socket-client");
|
|
61
62
|
/** Format diary hits the same way the search_notes tool does. */
|
|
62
63
|
function formatDiaryHits(hits) {
|
|
63
64
|
return hits.map((f) => `[diary] ${f.text} (source=${f.source}, createdAt=${f.createdAt})`);
|
|
@@ -149,6 +150,152 @@ function listTaskFiles(agent) {
|
|
|
149
150
|
function emit(event, message, meta) {
|
|
150
151
|
(0, runtime_1.emitNervesEvent)({ component: "daemon", event, message, meta });
|
|
151
152
|
}
|
|
153
|
+
function objectRecord(value) {
|
|
154
|
+
return value && typeof value === "object" && !Array.isArray(value)
|
|
155
|
+
? value
|
|
156
|
+
: null;
|
|
157
|
+
}
|
|
158
|
+
function stringValue(value) {
|
|
159
|
+
return typeof value === "string" ? value : null;
|
|
160
|
+
}
|
|
161
|
+
function booleanValue(value) {
|
|
162
|
+
return typeof value === "boolean" ? value : null;
|
|
163
|
+
}
|
|
164
|
+
function numberValue(value) {
|
|
165
|
+
return typeof value === "number" && Number.isFinite(value) ? value : null;
|
|
166
|
+
}
|
|
167
|
+
function readMcpRuntimeVersion() {
|
|
168
|
+
const packagePath = path.resolve(__dirname, "..", "..", "..", "package.json");
|
|
169
|
+
try {
|
|
170
|
+
const parsed = JSON.parse(fs.readFileSync(packagePath, "utf-8"));
|
|
171
|
+
return stringValue(parsed.version);
|
|
172
|
+
}
|
|
173
|
+
catch {
|
|
174
|
+
return null;
|
|
175
|
+
}
|
|
176
|
+
}
|
|
177
|
+
function summarizeRuntimeStatus(data, agent) {
|
|
178
|
+
const payload = objectRecord(data);
|
|
179
|
+
if (!payload)
|
|
180
|
+
return null;
|
|
181
|
+
const overview = objectRecord(payload.overview);
|
|
182
|
+
const workers = Array.isArray(payload.workers) ? payload.workers : [];
|
|
183
|
+
const senses = Array.isArray(payload.senses) ? payload.senses : [];
|
|
184
|
+
return {
|
|
185
|
+
daemonReachable: true,
|
|
186
|
+
overview: overview
|
|
187
|
+
? {
|
|
188
|
+
daemon: stringValue(overview.daemon) ?? "unknown",
|
|
189
|
+
health: stringValue(overview.health) ?? "unknown",
|
|
190
|
+
version: stringValue(overview.version),
|
|
191
|
+
mode: stringValue(overview.mode),
|
|
192
|
+
}
|
|
193
|
+
: undefined,
|
|
194
|
+
workers: workers.flatMap((row) => {
|
|
195
|
+
const record = objectRecord(row);
|
|
196
|
+
if (!record || stringValue(record.agent) !== agent)
|
|
197
|
+
return [];
|
|
198
|
+
const worker = stringValue(record.worker);
|
|
199
|
+
const status = stringValue(record.status);
|
|
200
|
+
return worker && status ? [{ worker, status }] : [];
|
|
201
|
+
}),
|
|
202
|
+
senses: senses.flatMap((row) => {
|
|
203
|
+
const record = objectRecord(row);
|
|
204
|
+
if (!record || stringValue(record.agent) !== agent)
|
|
205
|
+
return [];
|
|
206
|
+
const sense = stringValue(record.sense);
|
|
207
|
+
const status = stringValue(record.status);
|
|
208
|
+
const enabled = booleanValue(record.enabled);
|
|
209
|
+
if (!sense || !status || enabled === null)
|
|
210
|
+
return [];
|
|
211
|
+
return [{
|
|
212
|
+
sense,
|
|
213
|
+
status,
|
|
214
|
+
enabled,
|
|
215
|
+
detail: stringValue(record.detail),
|
|
216
|
+
proofMethod: stringValue(record.proofMethod),
|
|
217
|
+
lastProofAt: stringValue(record.lastProofAt),
|
|
218
|
+
proofAgeMs: numberValue(record.proofAgeMs),
|
|
219
|
+
pendingRecoveryCount: numberValue(record.pendingRecoveryCount),
|
|
220
|
+
failedRecoveryCount: numberValue(record.failedRecoveryCount),
|
|
221
|
+
failureLayer: stringValue(record.failureLayer),
|
|
222
|
+
lastFailure: stringValue(record.lastFailure),
|
|
223
|
+
recoveryAction: stringValue(record.recoveryAction),
|
|
224
|
+
}];
|
|
225
|
+
}),
|
|
226
|
+
};
|
|
227
|
+
}
|
|
228
|
+
async function readRuntimeStatus(socketPath, agent) {
|
|
229
|
+
if (!socketPath)
|
|
230
|
+
return null;
|
|
231
|
+
try {
|
|
232
|
+
const response = await (0, socket_client_1.sendDaemonCommand)(socketPath, { kind: "daemon.status" });
|
|
233
|
+
if (!response.ok) {
|
|
234
|
+
return {
|
|
235
|
+
daemonReachable: false,
|
|
236
|
+
workers: [],
|
|
237
|
+
senses: [],
|
|
238
|
+
error: response.error ?? response.message ?? "daemon status did not answer cleanly",
|
|
239
|
+
};
|
|
240
|
+
}
|
|
241
|
+
return summarizeRuntimeStatus(response.data, agent);
|
|
242
|
+
}
|
|
243
|
+
catch (error) {
|
|
244
|
+
return {
|
|
245
|
+
daemonReachable: false,
|
|
246
|
+
workers: [],
|
|
247
|
+
senses: [],
|
|
248
|
+
error: error instanceof Error ? error.message : String(error),
|
|
249
|
+
};
|
|
250
|
+
}
|
|
251
|
+
}
|
|
252
|
+
function formatRuntimeStatusLines(runtime, mcpVersion) {
|
|
253
|
+
if (!runtime)
|
|
254
|
+
return mcpVersion ? [`mcpVersion=${mcpVersion}`] : [];
|
|
255
|
+
if (!runtime.daemonReachable) {
|
|
256
|
+
return [
|
|
257
|
+
`daemon=unreachable${runtime.error ? `\terror=${runtime.error}` : ""}`,
|
|
258
|
+
...(mcpVersion ? [`mcpVersion=${mcpVersion}`] : []),
|
|
259
|
+
];
|
|
260
|
+
}
|
|
261
|
+
const lines = [];
|
|
262
|
+
if (runtime.overview) {
|
|
263
|
+
const versionPart = runtime.overview.version ? `\tdaemonVersion=${runtime.overview.version}` : "";
|
|
264
|
+
const modePart = runtime.overview.mode ? `\tmode=${runtime.overview.mode}` : "";
|
|
265
|
+
const mcpVersionPart = mcpVersion ? `\tmcpVersion=${mcpVersion}` : "";
|
|
266
|
+
const mismatchPart = mcpVersion && runtime.overview.version && mcpVersion !== runtime.overview.version
|
|
267
|
+
? `\tversionMismatch=mcp:${mcpVersion},daemon:${runtime.overview.version}`
|
|
268
|
+
: "";
|
|
269
|
+
lines.push(`daemon=${runtime.overview.daemon}\thealth=${runtime.overview.health}${versionPart}${modePart}${mcpVersionPart}${mismatchPart}`);
|
|
270
|
+
}
|
|
271
|
+
for (const worker of runtime.workers) {
|
|
272
|
+
lines.push(`worker=${worker.worker}:${worker.status}`);
|
|
273
|
+
}
|
|
274
|
+
for (const sense of runtime.senses) {
|
|
275
|
+
const detailPart = sense.detail ? `\tdetail=${sense.detail}` : "";
|
|
276
|
+
const proofPart = sense.proofMethod ? `\tproof=${sense.proofMethod}` : "";
|
|
277
|
+
const lastProofPart = sense.lastProofAt ? `\tlastProofAt=${sense.lastProofAt}` : "";
|
|
278
|
+
const proofAgePart = sense.proofAgeMs !== null ? `\tproofAgeMs=${sense.proofAgeMs}` : "";
|
|
279
|
+
const pendingPart = sense.pendingRecoveryCount !== null ? `\tpendingRecovery=${sense.pendingRecoveryCount}` : "";
|
|
280
|
+
const failedPart = sense.failedRecoveryCount !== null ? `\tfailedRecovery=${sense.failedRecoveryCount}` : "";
|
|
281
|
+
const failureLayerPart = sense.failureLayer ? `\tfailureLayer=${sense.failureLayer}` : "";
|
|
282
|
+
const failurePart = sense.lastFailure ? `\tlastFailure=${sense.lastFailure}` : "";
|
|
283
|
+
const recoveryPart = sense.recoveryAction ? `\trecovery=${sense.recoveryAction}` : "";
|
|
284
|
+
lines.push(`sense=${sense.sense}:${sense.enabled ? sense.status : "disabled"}`
|
|
285
|
+
+ detailPart
|
|
286
|
+
+ proofPart
|
|
287
|
+
+ lastProofPart
|
|
288
|
+
+ proofAgePart
|
|
289
|
+
+ pendingPart
|
|
290
|
+
+ failedPart
|
|
291
|
+
+ failureLayerPart
|
|
292
|
+
+ failurePart
|
|
293
|
+
+ recoveryPart);
|
|
294
|
+
}
|
|
295
|
+
if (lines.length === 0 && mcpVersion)
|
|
296
|
+
lines.push(`mcpVersion=${mcpVersion}`);
|
|
297
|
+
return lines;
|
|
298
|
+
}
|
|
152
299
|
// ─── Handlers ────────────────────────────────────────────────────────────────
|
|
153
300
|
async function handleAgentStatus(params) {
|
|
154
301
|
(0, runtime_1.emitNervesEvent)({ component: "daemon", event: "daemon.agent_service_start", message: "handling agent.status", meta: { agent: params.agent } });
|
|
@@ -156,17 +303,32 @@ async function handleAgentStatus(params) {
|
|
|
156
303
|
const facts = (0, diary_1.readDiaryEntries)(diaryRoot);
|
|
157
304
|
const innerStatus = readInnerDialogStatus(params.agent);
|
|
158
305
|
const sessions = enumerateSessions(params.agent);
|
|
306
|
+
const mcpVersion = readMcpRuntimeVersion();
|
|
307
|
+
const runtime = await readRuntimeStatus(params.socketPath ?? socket_client_1.DEFAULT_DAEMON_SOCKET_PATH, params.agent);
|
|
159
308
|
emit("daemon.agent_service_end", "completed agent.status", { agent: params.agent });
|
|
309
|
+
const innerStatusValue = innerStatus?.status ?? "unknown";
|
|
310
|
+
const lastThoughtAt = innerStatus?.lastCompletedAt ?? null;
|
|
311
|
+
const agentLine = [
|
|
312
|
+
`agent=${params.agent}`,
|
|
313
|
+
`innerStatus=${innerStatusValue}`,
|
|
314
|
+
`lastThoughtAt=${lastThoughtAt ?? "never"}`,
|
|
315
|
+
`sessionCount=${sessions.length}`,
|
|
316
|
+
`diaryEntries=${facts.length}`,
|
|
317
|
+
].join("\t");
|
|
318
|
+
const runtimeLines = formatRuntimeStatusLines(runtime, mcpVersion);
|
|
319
|
+
const message = [agentLine, ...runtimeLines].join("\n");
|
|
160
320
|
return {
|
|
161
321
|
ok: true,
|
|
162
|
-
message
|
|
322
|
+
message,
|
|
163
323
|
data: {
|
|
164
324
|
agent: params.agent,
|
|
165
|
-
innerStatus:
|
|
166
|
-
lastThoughtAt
|
|
325
|
+
innerStatus: innerStatusValue,
|
|
326
|
+
lastThoughtAt,
|
|
167
327
|
sessionCount: sessions.length,
|
|
168
328
|
hasDiaryEntries: facts.length > 0,
|
|
169
329
|
factCount: facts.length,
|
|
330
|
+
runtime,
|
|
331
|
+
mcpVersion,
|
|
170
332
|
},
|
|
171
333
|
};
|
|
172
334
|
}
|
|
@@ -621,7 +621,7 @@ function finalDaemonFailureMessage(deps, reason) {
|
|
|
621
621
|
lines.push("Run `ouro up` again or `ouro doctor` for a deeper diagnosis.");
|
|
622
622
|
return lines.join("\n");
|
|
623
623
|
}
|
|
624
|
-
async function verifyDaemonReadyForHandoff(deps) {
|
|
624
|
+
async function verifyDaemonReadyForHandoff(deps, options = {}) {
|
|
625
625
|
const socketAlive = await deps.checkSocketAlive(deps.socketPath);
|
|
626
626
|
if (!socketAlive) {
|
|
627
627
|
return {
|
|
@@ -654,6 +654,23 @@ async function verifyDaemonReadyForHandoff(deps) {
|
|
|
654
654
|
message: finalDaemonFailureMessage(deps, `the daemon reported state ${payload.overview.daemon}`),
|
|
655
655
|
};
|
|
656
656
|
}
|
|
657
|
+
if (payload.overview.health !== "ok") {
|
|
658
|
+
const degradedSenses = payload.senses
|
|
659
|
+
.filter((sense) => sense.enabled && !["running", "interactive", "disabled", "not_attached"].includes(sense.status))
|
|
660
|
+
.map((sense) => `${sense.agent}/${sense.sense}: ${sense.status} - ${sense.detail}`);
|
|
661
|
+
const detail = degradedSenses.length > 0 ? `; ${degradedSenses.join("; ")}` : "";
|
|
662
|
+
if (options.allowDegradedHealth) {
|
|
663
|
+
return {
|
|
664
|
+
ok: true,
|
|
665
|
+
summary: `runtime health ${payload.overview.health}`,
|
|
666
|
+
};
|
|
667
|
+
}
|
|
668
|
+
return {
|
|
669
|
+
ok: false,
|
|
670
|
+
summary: `daemon health ${payload.overview.health}`,
|
|
671
|
+
message: finalDaemonFailureMessage(deps, `runtime health is ${payload.overview.health}${detail}`),
|
|
672
|
+
};
|
|
673
|
+
}
|
|
657
674
|
const workerCount = payload.workers.length;
|
|
658
675
|
return {
|
|
659
676
|
ok: true,
|
|
@@ -5863,7 +5880,10 @@ async function runOuroCli(args, deps = (0, cli_defaults_1.createDefaultOuroCliDe
|
|
|
5863
5880
|
daemonResult.stability = mergeStartupStability(daemonResult.stability, providerDegraded);
|
|
5864
5881
|
progress.completePhase("provider checks", providerRepairCountSummary(providerDegraded.length));
|
|
5865
5882
|
progress.startPhase("final daemon check");
|
|
5866
|
-
const finalDaemonCheck = await verifyDaemonReadyForHandoff(deps
|
|
5883
|
+
const finalDaemonCheck = await verifyDaemonReadyForHandoff(deps, {
|
|
5884
|
+
allowDegradedHealth: Boolean(command.noRepair
|
|
5885
|
+
&& ((daemonResult.stability?.degraded.length ?? 0) > 0 || providerDegraded.length > 0)),
|
|
5886
|
+
});
|
|
5867
5887
|
if (!finalDaemonCheck.ok) {
|
|
5868
5888
|
;
|
|
5869
5889
|
progress.failPhase?.("final daemon check", finalDaemonCheck.summary);
|
|
@@ -66,6 +66,50 @@ function numberField(value) {
|
|
|
66
66
|
function booleanField(value) {
|
|
67
67
|
return typeof value === "boolean" ? value : null;
|
|
68
68
|
}
|
|
69
|
+
function formatDurationMs(ms) {
|
|
70
|
+
if (ms < 1_000)
|
|
71
|
+
return `${Math.round(ms)}ms`;
|
|
72
|
+
if (ms < 60_000)
|
|
73
|
+
return `${Math.round(ms / 1_000)}s`;
|
|
74
|
+
if (ms < 3_600_000)
|
|
75
|
+
return `${Math.round(ms / 60_000)}m`;
|
|
76
|
+
if (ms < 86_400_000)
|
|
77
|
+
return `${Math.round(ms / 3_600_000)}h`;
|
|
78
|
+
return `${Math.round(ms / 86_400_000)}d`;
|
|
79
|
+
}
|
|
80
|
+
function formatSenseDetail(row) {
|
|
81
|
+
const details = [row.detail];
|
|
82
|
+
const proofParts = [];
|
|
83
|
+
if (row.proofMethod)
|
|
84
|
+
proofParts.push(`proof=${row.proofMethod}`);
|
|
85
|
+
if (row.lastProofAt)
|
|
86
|
+
proofParts.push(`lastProof=${row.lastProofAt}`);
|
|
87
|
+
if (row.proofAgeMs !== undefined)
|
|
88
|
+
proofParts.push(`proofAge=${formatDurationMs(row.proofAgeMs)}`);
|
|
89
|
+
if (proofParts.length > 0)
|
|
90
|
+
details.push(proofParts.join(" "));
|
|
91
|
+
const recoveryParts = [];
|
|
92
|
+
if (row.pendingRecoveryCount !== undefined)
|
|
93
|
+
recoveryParts.push(`pendingRecovery=${row.pendingRecoveryCount}`);
|
|
94
|
+
if (row.failedRecoveryCount !== undefined)
|
|
95
|
+
recoveryParts.push(`failedRecovery=${row.failedRecoveryCount}`);
|
|
96
|
+
if (row.oldestPendingRecoveryAt)
|
|
97
|
+
recoveryParts.push(`oldestPending=${row.oldestPendingRecoveryAt}`);
|
|
98
|
+
if (row.oldestPendingRecoveryAgeMs !== undefined)
|
|
99
|
+
recoveryParts.push(`oldestPendingAge=${formatDurationMs(row.oldestPendingRecoveryAgeMs)}`);
|
|
100
|
+
if (recoveryParts.length > 0)
|
|
101
|
+
details.push(recoveryParts.join(" "));
|
|
102
|
+
const failureParts = [];
|
|
103
|
+
if (row.failureLayer)
|
|
104
|
+
failureParts.push(`failureLayer=${row.failureLayer}`);
|
|
105
|
+
if (row.lastFailure)
|
|
106
|
+
failureParts.push(`lastFailure=${row.lastFailure}`);
|
|
107
|
+
if (row.recoveryAction)
|
|
108
|
+
failureParts.push(`recovery=${row.recoveryAction}`);
|
|
109
|
+
if (failureParts.length > 0)
|
|
110
|
+
details.push(failureParts.join(" "));
|
|
111
|
+
return details.filter(Boolean).join("; ");
|
|
112
|
+
}
|
|
69
113
|
// ── Parsers ──
|
|
70
114
|
function parseStatusPayload(data) {
|
|
71
115
|
if (!data || typeof data !== "object" || Array.isArray(data))
|
|
@@ -116,7 +160,7 @@ function parseStatusPayload(data) {
|
|
|
116
160
|
const enabled = booleanField(row.enabled);
|
|
117
161
|
if (!agent || !sense || !status || detail === null || enabled === null)
|
|
118
162
|
return null;
|
|
119
|
-
|
|
163
|
+
const parsed = {
|
|
120
164
|
agent,
|
|
121
165
|
sense,
|
|
122
166
|
label: stringField(row.label) ?? undefined,
|
|
@@ -124,6 +168,37 @@ function parseStatusPayload(data) {
|
|
|
124
168
|
status,
|
|
125
169
|
detail,
|
|
126
170
|
};
|
|
171
|
+
const proofMethod = stringField(row.proofMethod);
|
|
172
|
+
const lastProofAt = stringField(row.lastProofAt);
|
|
173
|
+
const proofAgeMs = numberField(row.proofAgeMs);
|
|
174
|
+
const lastFailure = stringField(row.lastFailure);
|
|
175
|
+
const failureLayer = stringField(row.failureLayer);
|
|
176
|
+
const recoveryAction = stringField(row.recoveryAction);
|
|
177
|
+
const pendingRecoveryCount = numberField(row.pendingRecoveryCount);
|
|
178
|
+
const failedRecoveryCount = numberField(row.failedRecoveryCount);
|
|
179
|
+
const oldestPendingRecoveryAt = stringField(row.oldestPendingRecoveryAt);
|
|
180
|
+
const oldestPendingRecoveryAgeMs = numberField(row.oldestPendingRecoveryAgeMs);
|
|
181
|
+
if (proofMethod !== null)
|
|
182
|
+
parsed.proofMethod = proofMethod;
|
|
183
|
+
if (lastProofAt !== null)
|
|
184
|
+
parsed.lastProofAt = lastProofAt;
|
|
185
|
+
if (proofAgeMs !== null)
|
|
186
|
+
parsed.proofAgeMs = proofAgeMs;
|
|
187
|
+
if (lastFailure !== null)
|
|
188
|
+
parsed.lastFailure = lastFailure;
|
|
189
|
+
if (failureLayer !== null)
|
|
190
|
+
parsed.failureLayer = failureLayer;
|
|
191
|
+
if (recoveryAction !== null)
|
|
192
|
+
parsed.recoveryAction = recoveryAction;
|
|
193
|
+
if (pendingRecoveryCount !== null)
|
|
194
|
+
parsed.pendingRecoveryCount = pendingRecoveryCount;
|
|
195
|
+
if (failedRecoveryCount !== null)
|
|
196
|
+
parsed.failedRecoveryCount = failedRecoveryCount;
|
|
197
|
+
if (oldestPendingRecoveryAt !== null)
|
|
198
|
+
parsed.oldestPendingRecoveryAt = oldestPendingRecoveryAt;
|
|
199
|
+
if (oldestPendingRecoveryAgeMs !== null)
|
|
200
|
+
parsed.oldestPendingRecoveryAgeMs = oldestPendingRecoveryAgeMs;
|
|
201
|
+
return parsed;
|
|
127
202
|
});
|
|
128
203
|
const parsedWorkers = workers.map((entry) => {
|
|
129
204
|
if (!entry || typeof entry !== "object" || Array.isArray(entry))
|
|
@@ -364,7 +439,7 @@ function formatDaemonStatusOutput(response, fallback) {
|
|
|
364
439
|
const name = humanizeSenseName(row.sense, row.label).padEnd(nameWidth);
|
|
365
440
|
const dot = row.enabled ? statusDot(row.status) : dim("○");
|
|
366
441
|
const statusText = (row.enabled ? row.status : "disabled").padEnd(statusWidth);
|
|
367
|
-
lines.push(` ${name} ${dot} ${statusText} ${dim(row
|
|
442
|
+
lines.push(` ${name} ${dot} ${statusText} ${dim(formatSenseDetail(row))}`);
|
|
368
443
|
}
|
|
369
444
|
}
|
|
370
445
|
lines.push("");
|
|
@@ -148,6 +148,13 @@ const healthMonitor = new health_monitor_1.HealthMonitor({
|
|
|
148
148
|
}
|
|
149
149
|
catch { /* recovery is best-effort */ }
|
|
150
150
|
},
|
|
151
|
+
/* v8 ignore next 3 -- wiring: delegates to senseManager.restartSense which has focused tests @preserve */
|
|
152
|
+
onCriticalSense: (managedName) => {
|
|
153
|
+
try {
|
|
154
|
+
void senseManager.restartSense(managedName);
|
|
155
|
+
}
|
|
156
|
+
catch { /* recovery is best-effort */ }
|
|
157
|
+
},
|
|
151
158
|
});
|
|
152
159
|
const habitSchedulers = [];
|
|
153
160
|
let entryRuntimeStopping = false;
|
|
@@ -345,17 +345,46 @@ function buildWorkerRows(snapshots) {
|
|
|
345
345
|
fixHint: snapshot.fixHint ?? null,
|
|
346
346
|
}));
|
|
347
347
|
}
|
|
348
|
+
function unhealthySenseRows(senses) {
|
|
349
|
+
return senses.filter((row) => {
|
|
350
|
+
if (!row.enabled)
|
|
351
|
+
return false;
|
|
352
|
+
if (row.status === "disabled" || row.status === "not_attached")
|
|
353
|
+
return false;
|
|
354
|
+
if (row.status === "interactive" || row.status === "running")
|
|
355
|
+
return false;
|
|
356
|
+
return true;
|
|
357
|
+
});
|
|
358
|
+
}
|
|
359
|
+
function overviewHealth(workers, senses) {
|
|
360
|
+
if (!workers.every((worker) => worker.status === "running"))
|
|
361
|
+
return "warn";
|
|
362
|
+
if (unhealthySenseRows(senses).length > 0)
|
|
363
|
+
return "warn";
|
|
364
|
+
return "ok";
|
|
365
|
+
}
|
|
348
366
|
function formatStatusSummary(payload) {
|
|
349
367
|
if (payload.overview.workerCount === 0 && payload.overview.senseCount === 0) {
|
|
350
368
|
return "no managed agents";
|
|
351
369
|
}
|
|
352
|
-
const
|
|
353
|
-
...payload.workers
|
|
354
|
-
|
|
355
|
-
.
|
|
356
|
-
|
|
370
|
+
const degraded = [
|
|
371
|
+
...payload.workers
|
|
372
|
+
.filter((row) => row.status !== "running")
|
|
373
|
+
.map((row) => `worker:${row.agent}/${row.worker}:${row.status}`),
|
|
374
|
+
...unhealthySenseRows(payload.senses)
|
|
375
|
+
.map((row) => `sense:${row.agent}/${row.sense}:${row.status}`),
|
|
357
376
|
];
|
|
358
|
-
const detail =
|
|
377
|
+
const detail = degraded.length > 0 ? `\tdegraded=${degraded.join(",")}` : "";
|
|
378
|
+
if (!detail) {
|
|
379
|
+
const rows = [
|
|
380
|
+
...payload.workers.map((row) => `${row.agent}/${row.worker}:${row.status}`),
|
|
381
|
+
...payload.senses
|
|
382
|
+
.filter((row) => row.enabled)
|
|
383
|
+
.map((row) => `${row.agent}/${row.sense}:${row.status}`),
|
|
384
|
+
];
|
|
385
|
+
const items = rows.length > 0 ? `\titems=${rows.join(",")}` : "";
|
|
386
|
+
return `daemon=${payload.overview.daemon}\tworkers=${payload.overview.workerCount}\tsenses=${payload.overview.senseCount}\thealth=${payload.overview.health}${items}`;
|
|
387
|
+
}
|
|
359
388
|
return `daemon=${payload.overview.daemon}\tworkers=${payload.overview.workerCount}\tsenses=${payload.overview.senseCount}\thealth=${payload.overview.health}${detail}`;
|
|
360
389
|
}
|
|
361
390
|
function parseIncomingCommand(raw) {
|
|
@@ -480,7 +509,7 @@ class OuroDaemon {
|
|
|
480
509
|
return {
|
|
481
510
|
overview: {
|
|
482
511
|
daemon: "running",
|
|
483
|
-
health: workers
|
|
512
|
+
health: overviewHealth(workers, senses),
|
|
484
513
|
socketPath: this.socketPath,
|
|
485
514
|
mailboxUrl,
|
|
486
515
|
outlookUrl: mailboxUrl,
|
|
@@ -8,6 +8,7 @@ class HealthMonitor {
|
|
|
8
8
|
alertSink;
|
|
9
9
|
diskUsagePercent;
|
|
10
10
|
onCriticalAgent;
|
|
11
|
+
onCriticalSense;
|
|
11
12
|
senseProbes;
|
|
12
13
|
senseProbeProvider;
|
|
13
14
|
intervalHandle = null;
|
|
@@ -17,9 +18,31 @@ class HealthMonitor {
|
|
|
17
18
|
this.alertSink = options.alertSink ?? (() => undefined);
|
|
18
19
|
this.diskUsagePercent = options.diskUsagePercent ?? (() => 0);
|
|
19
20
|
this.onCriticalAgent = options.onCriticalAgent ?? (() => undefined);
|
|
21
|
+
this.onCriticalSense = options.onCriticalSense ?? (() => undefined);
|
|
20
22
|
this.senseProbes = options.senseProbes ?? [];
|
|
21
23
|
this.senseProbeProvider = options.senseProbeProvider ?? (() => []);
|
|
22
24
|
}
|
|
25
|
+
triggerSenseRecovery(probe, detail) {
|
|
26
|
+
if (!probe.managedName)
|
|
27
|
+
return;
|
|
28
|
+
try {
|
|
29
|
+
(0, runtime_1.emitNervesEvent)({
|
|
30
|
+
level: "warn",
|
|
31
|
+
component: "daemon",
|
|
32
|
+
event: "daemon.health_check_sense_recovery_attempted",
|
|
33
|
+
message: "triggering recovery restart for failed sense probe",
|
|
34
|
+
meta: {
|
|
35
|
+
probeName: probe.name,
|
|
36
|
+
managedName: probe.managedName,
|
|
37
|
+
detail,
|
|
38
|
+
},
|
|
39
|
+
});
|
|
40
|
+
this.onCriticalSense(probe.managedName, probe.name);
|
|
41
|
+
}
|
|
42
|
+
catch {
|
|
43
|
+
// Recovery is best-effort -- callback errors must not crash runChecks
|
|
44
|
+
}
|
|
45
|
+
}
|
|
23
46
|
startPeriodicChecks(intervalMs) {
|
|
24
47
|
if (this.intervalHandle !== null)
|
|
25
48
|
return;
|
|
@@ -137,6 +160,7 @@ class HealthMonitor {
|
|
|
137
160
|
});
|
|
138
161
|
}
|
|
139
162
|
else {
|
|
163
|
+
this.triggerSenseRecovery(probe, outcome.detail ?? "unknown");
|
|
140
164
|
results.push({
|
|
141
165
|
name: `sense-probe:${probe.name}`,
|
|
142
166
|
status: "critical",
|
|
@@ -145,6 +169,7 @@ class HealthMonitor {
|
|
|
145
169
|
}
|
|
146
170
|
}
|
|
147
171
|
catch (error) {
|
|
172
|
+
this.triggerSenseRecovery(probe, error instanceof Error ? error.message : String(error));
|
|
148
173
|
results.push({
|
|
149
174
|
name: `sense-probe:${probe.name}`,
|
|
150
175
|
status: "critical",
|
|
@@ -266,12 +266,19 @@ function readBlueBubblesRuntimeJson(runtimePath) {
|
|
|
266
266
|
? parsed.detail
|
|
267
267
|
: "startup health probe pending",
|
|
268
268
|
lastCheckedAt: typeof parsed.lastCheckedAt === "string" ? parsed.lastCheckedAt : undefined,
|
|
269
|
+
proofMethod: typeof parsed.proofMethod === "string" && parsed.proofMethod.trim()
|
|
270
|
+
? parsed.proofMethod
|
|
271
|
+
: undefined,
|
|
269
272
|
pendingRecoveryCount: typeof parsed.pendingRecoveryCount === "number" && Number.isFinite(parsed.pendingRecoveryCount)
|
|
270
273
|
? parsed.pendingRecoveryCount
|
|
271
274
|
: 0,
|
|
272
275
|
failedRecoveryCount: typeof parsed.failedRecoveryCount === "number" && Number.isFinite(parsed.failedRecoveryCount)
|
|
273
276
|
? parsed.failedRecoveryCount
|
|
274
277
|
: 0,
|
|
278
|
+
oldestPendingRecoveryAt: typeof parsed.oldestPendingRecoveryAt === "string" ? parsed.oldestPendingRecoveryAt : undefined,
|
|
279
|
+
oldestPendingRecoveryAgeMs: typeof parsed.oldestPendingRecoveryAgeMs === "number" && Number.isFinite(parsed.oldestPendingRecoveryAgeMs)
|
|
280
|
+
? parsed.oldestPendingRecoveryAgeMs
|
|
281
|
+
: undefined,
|
|
275
282
|
};
|
|
276
283
|
/* v8 ignore stop */
|
|
277
284
|
/* v8 ignore start -- defensive: catch for missing/corrupt BB runtime state file @preserve */
|
|
@@ -288,34 +295,66 @@ function readBlueBubblesRuntimeFacts(agent, bundlesRoot, snapshot) {
|
|
|
288
295
|
return { runtime: snapshot?.runtime };
|
|
289
296
|
}
|
|
290
297
|
const state = readBlueBubblesRuntimeJson(runtimePath);
|
|
298
|
+
const checkedAtMs = state.lastCheckedAt ? Date.parse(state.lastCheckedAt) : Number.NaN;
|
|
299
|
+
const proofFacts = {
|
|
300
|
+
proofMethod: state.proofMethod ?? "bluebubbles.checkHealth",
|
|
301
|
+
lastProofAt: state.lastCheckedAt,
|
|
302
|
+
proofAgeMs: Number.isFinite(checkedAtMs) ? Math.max(0, Date.now() - checkedAtMs) : undefined,
|
|
303
|
+
pendingRecoveryCount: state.pendingRecoveryCount,
|
|
304
|
+
failedRecoveryCount: state.failedRecoveryCount,
|
|
305
|
+
oldestPendingRecoveryAt: state.oldestPendingRecoveryAt,
|
|
306
|
+
oldestPendingRecoveryAgeMs: state.oldestPendingRecoveryAgeMs,
|
|
307
|
+
};
|
|
291
308
|
if (!blueBubblesRuntimeStateIsFresh(state.lastCheckedAt)) {
|
|
292
|
-
return {
|
|
309
|
+
return {
|
|
310
|
+
runtime: snapshot?.runtime,
|
|
311
|
+
lastFailure: state.lastCheckedAt ? "BlueBubbles proof is stale" : undefined,
|
|
312
|
+
failureLayer: state.lastCheckedAt ? "proof_freshness" : undefined,
|
|
313
|
+
};
|
|
293
314
|
}
|
|
294
315
|
if (snapshot?.runtime !== "running") {
|
|
295
316
|
return {
|
|
296
317
|
runtime: "error",
|
|
297
318
|
detail: "BlueBubbles listener is not running",
|
|
319
|
+
...proofFacts,
|
|
320
|
+
lastFailure: "listener process is not running",
|
|
321
|
+
failureLayer: "listener",
|
|
322
|
+
recoveryAction: "daemon health monitor will restart the BlueBubbles listener when its probe fails",
|
|
298
323
|
};
|
|
299
324
|
}
|
|
300
325
|
if (state.upstreamStatus === "error") {
|
|
301
326
|
return {
|
|
302
327
|
runtime: "error",
|
|
303
328
|
detail: state.detail,
|
|
329
|
+
...proofFacts,
|
|
330
|
+
lastFailure: state.detail,
|
|
331
|
+
failureLayer: "upstream",
|
|
332
|
+
recoveryAction: "verify BlueBubbles server/app auth and local machine attachment, then retry the listener",
|
|
304
333
|
};
|
|
305
334
|
}
|
|
306
335
|
if (state.pendingRecoveryCount > 0) {
|
|
307
336
|
return {
|
|
308
337
|
runtime: "error",
|
|
309
338
|
detail: state.detail,
|
|
339
|
+
...proofFacts,
|
|
340
|
+
lastFailure: state.detail,
|
|
341
|
+
failureLayer: "recovery_queue",
|
|
342
|
+
recoveryAction: "queued recovery will retry; inspect BlueBubbles inbound/recovery sidecar logs if age keeps growing",
|
|
310
343
|
};
|
|
311
344
|
}
|
|
312
345
|
if (state.upstreamStatus === "ok") {
|
|
313
346
|
return {
|
|
314
347
|
runtime: "running",
|
|
348
|
+
...proofFacts,
|
|
315
349
|
...(state.failedRecoveryCount > 0 ? { detail: state.detail } : {}),
|
|
350
|
+
...(state.failedRecoveryCount > 0 ? {
|
|
351
|
+
lastFailure: state.detail,
|
|
352
|
+
failureLayer: "recovery_quarantine",
|
|
353
|
+
recoveryAction: "inspect quarantined BlueBubbles recovery failures; live transport remains reachable",
|
|
354
|
+
} : {}),
|
|
316
355
|
};
|
|
317
356
|
}
|
|
318
|
-
return { runtime: snapshot?.runtime };
|
|
357
|
+
return { runtime: snapshot?.runtime, ...proofFacts };
|
|
319
358
|
}
|
|
320
359
|
class DaemonSenseManager {
|
|
321
360
|
processManager;
|
|
@@ -467,10 +506,20 @@ class DaemonSenseManager {
|
|
|
467
506
|
const machinePayload = machineRuntimeConfig.config;
|
|
468
507
|
const bluebubblesChannel = machinePayload.bluebubblesChannel;
|
|
469
508
|
const port = numberField(bluebubblesChannel, "port", DEFAULT_BLUEBUBBLES_PORT);
|
|
470
|
-
probes.push(
|
|
509
|
+
probes.push({
|
|
510
|
+
...(0, http_health_probe_1.createHttpHealthProbe)(`bluebubbles:${agent}`, port),
|
|
511
|
+
managedName: `${agent}:bluebubbles`,
|
|
512
|
+
});
|
|
471
513
|
}
|
|
472
514
|
return probes;
|
|
473
515
|
}
|
|
516
|
+
async restartSense(managedName) {
|
|
517
|
+
if (this.processManager.restartAgent) {
|
|
518
|
+
await this.processManager.restartAgent(managedName);
|
|
519
|
+
return;
|
|
520
|
+
}
|
|
521
|
+
await this.processManager.startAgent?.(managedName);
|
|
522
|
+
}
|
|
474
523
|
listSenseRows() {
|
|
475
524
|
const runtime = new Map();
|
|
476
525
|
for (const snapshot of this.processManager.listAgentSnapshots()) {
|
|
@@ -513,6 +562,18 @@ class DaemonSenseManager {
|
|
|
513
562
|
?? context.facts[entry.sense].detail
|
|
514
563
|
: context.facts[entry.sense].detail
|
|
515
564
|
: "not enabled in agent.json",
|
|
565
|
+
...(entry.sense === "bluebubbles" ? {
|
|
566
|
+
proofMethod: blueBubblesRuntimeFacts.proofMethod,
|
|
567
|
+
lastProofAt: blueBubblesRuntimeFacts.lastProofAt,
|
|
568
|
+
proofAgeMs: blueBubblesRuntimeFacts.proofAgeMs,
|
|
569
|
+
lastFailure: blueBubblesRuntimeFacts.lastFailure,
|
|
570
|
+
failureLayer: blueBubblesRuntimeFacts.failureLayer,
|
|
571
|
+
recoveryAction: blueBubblesRuntimeFacts.recoveryAction,
|
|
572
|
+
pendingRecoveryCount: blueBubblesRuntimeFacts.pendingRecoveryCount,
|
|
573
|
+
failedRecoveryCount: blueBubblesRuntimeFacts.failedRecoveryCount,
|
|
574
|
+
oldestPendingRecoveryAt: blueBubblesRuntimeFacts.oldestPendingRecoveryAt,
|
|
575
|
+
oldestPendingRecoveryAgeMs: blueBubblesRuntimeFacts.oldestPendingRecoveryAgeMs,
|
|
576
|
+
} : {}),
|
|
516
577
|
}));
|
|
517
578
|
});
|
|
518
579
|
(0, runtime_1.emitNervesEvent)({
|
|
@@ -425,7 +425,7 @@ function createMcpServer(options) {
|
|
|
425
425
|
if (serviceHandler && typeof agentService[serviceHandler] === "function") {
|
|
426
426
|
const handlerFn = agentService[serviceHandler];
|
|
427
427
|
try {
|
|
428
|
-
response = await handlerFn({ agent, friendId, ...toolArgs });
|
|
428
|
+
response = await handlerFn({ agent, friendId, socketPath, ...toolArgs });
|
|
429
429
|
}
|
|
430
430
|
catch (error) {
|
|
431
431
|
const errorMessage = error instanceof Error ? error.message : String(error);
|
|
@@ -1239,6 +1239,9 @@ function countPendingRecoveryCandidates(agentName) {
|
|
|
1239
1239
|
.length;
|
|
1240
1240
|
}
|
|
1241
1241
|
function countPendingCapturedInboundMessages(agentName) {
|
|
1242
|
+
return listPendingCapturedInboundMessages(agentName).length;
|
|
1243
|
+
}
|
|
1244
|
+
function listPendingCapturedInboundMessages(agentName) {
|
|
1242
1245
|
const seenMessageGuids = new Set();
|
|
1243
1246
|
return (0, inbound_log_1.listRecordedBlueBubblesInbound)(agentName)
|
|
1244
1247
|
.filter((entry) => {
|
|
@@ -1247,8 +1250,7 @@ function countPendingCapturedInboundMessages(agentName) {
|
|
|
1247
1250
|
seenMessageGuids.add(entry.messageGuid);
|
|
1248
1251
|
return true;
|
|
1249
1252
|
})
|
|
1250
|
-
.filter((entry) => !(0, processed_log_1.hasProcessedBlueBubblesMessage)(agentName, entry.sessionKey, entry.messageGuid))
|
|
1251
|
-
.length;
|
|
1253
|
+
.filter((entry) => !(0, processed_log_1.hasProcessedBlueBubblesMessage)(agentName, entry.sessionKey, entry.messageGuid));
|
|
1252
1254
|
}
|
|
1253
1255
|
function parseTimestampMs(value) {
|
|
1254
1256
|
if (!value)
|
|
@@ -1273,6 +1275,23 @@ function formatBlueBubblesRuntimeDetail(queued, failed) {
|
|
|
1273
1275
|
return `${failed} message(s) unrecoverable this cycle; upstream ok`;
|
|
1274
1276
|
return "upstream reachable";
|
|
1275
1277
|
}
|
|
1278
|
+
function blueBubblesPendingRecoverySnapshot(agentName, nowMs = Date.now()) {
|
|
1279
|
+
const pendingRecordedAt = [
|
|
1280
|
+
...listPendingCapturedInboundMessages(agentName).map((entry) => entry.recordedAt),
|
|
1281
|
+
...(0, mutation_log_1.listBlueBubblesRecoveryCandidates)(agentName)
|
|
1282
|
+
.filter((entry) => !(0, processed_log_1.hasProcessedBlueBubblesMessage)(agentName, entry.sessionKey, entry.messageGuid))
|
|
1283
|
+
.map((entry) => entry.recordedAt),
|
|
1284
|
+
]
|
|
1285
|
+
.map((value) => ({ value, ms: Date.parse(value) }))
|
|
1286
|
+
.filter((entry) => Number.isFinite(entry.ms))
|
|
1287
|
+
.sort((left, right) => left.ms - right.ms);
|
|
1288
|
+
const oldest = pendingRecordedAt[0];
|
|
1289
|
+
return {
|
|
1290
|
+
pendingRecoveryCount: countPendingCapturedInboundMessages(agentName) + countPendingRecoveryCandidates(agentName),
|
|
1291
|
+
oldestPendingRecoveryAt: oldest?.value,
|
|
1292
|
+
oldestPendingRecoveryAgeMs: oldest ? Math.max(0, nowMs - oldest.ms) : undefined,
|
|
1293
|
+
};
|
|
1294
|
+
}
|
|
1276
1295
|
async function syncBlueBubblesRuntime(deps = {}) {
|
|
1277
1296
|
const resolvedDeps = { ...defaultDeps, ...deps };
|
|
1278
1297
|
const agentName = resolvedDeps.getAgentName();
|
|
@@ -1281,13 +1300,13 @@ async function syncBlueBubblesRuntime(deps = {}) {
|
|
|
1281
1300
|
const previousState = (0, runtime_state_1.readBlueBubblesRuntimeState)(agentName);
|
|
1282
1301
|
try {
|
|
1283
1302
|
await client.checkHealth();
|
|
1284
|
-
const
|
|
1285
|
-
const recoveryPending = countPendingRecoveryCandidates(agentName);
|
|
1303
|
+
const pendingBeforeCatchup = blueBubblesPendingRecoverySnapshot(agentName);
|
|
1286
1304
|
(0, runtime_state_1.writeBlueBubblesRuntimeState)(agentName, {
|
|
1287
1305
|
upstreamStatus: "ok",
|
|
1288
1306
|
detail: "upstream reachable; recovery pass running",
|
|
1289
1307
|
lastCheckedAt: checkedAt,
|
|
1290
|
-
|
|
1308
|
+
proofMethod: "bluebubbles.checkHealth",
|
|
1309
|
+
...pendingBeforeCatchup,
|
|
1291
1310
|
lastRecoveredAt: previousState.lastRecoveredAt,
|
|
1292
1311
|
lastRecoveredMessageGuid: previousState.lastRecoveredMessageGuid,
|
|
1293
1312
|
});
|
|
@@ -1295,7 +1314,8 @@ async function syncBlueBubblesRuntime(deps = {}) {
|
|
|
1295
1314
|
processTurns: false,
|
|
1296
1315
|
});
|
|
1297
1316
|
const failed = catchUp.failed;
|
|
1298
|
-
const
|
|
1317
|
+
const pendingAfterCatchup = blueBubblesPendingRecoverySnapshot(agentName);
|
|
1318
|
+
const queued = pendingAfterCatchup.pendingRecoveryCount;
|
|
1299
1319
|
// upstreamStatus reflects whether BlueBubbles itself and the local bridge
|
|
1300
1320
|
// can answer webhook traffic. The daemon status layer treats
|
|
1301
1321
|
// pendingRecoveryCount as unhealthy for user-facing iMessage reachability,
|
|
@@ -1304,6 +1324,8 @@ async function syncBlueBubblesRuntime(deps = {}) {
|
|
|
1304
1324
|
upstreamStatus: "ok",
|
|
1305
1325
|
detail: formatBlueBubblesRuntimeDetail(queued, failed),
|
|
1306
1326
|
lastCheckedAt: checkedAt,
|
|
1327
|
+
proofMethod: "bluebubbles.checkHealth",
|
|
1328
|
+
...pendingAfterCatchup,
|
|
1307
1329
|
pendingRecoveryCount: queued,
|
|
1308
1330
|
failedRecoveryCount: failed,
|
|
1309
1331
|
lastRecoveredAt: previousState.lastRecoveredAt,
|
|
@@ -1315,7 +1337,8 @@ async function syncBlueBubblesRuntime(deps = {}) {
|
|
|
1315
1337
|
upstreamStatus: "error",
|
|
1316
1338
|
detail: error instanceof Error ? error.message : String(error),
|
|
1317
1339
|
lastCheckedAt: checkedAt,
|
|
1318
|
-
|
|
1340
|
+
proofMethod: "bluebubbles.checkHealth",
|
|
1341
|
+
...blueBubblesPendingRecoverySnapshot(agentName),
|
|
1319
1342
|
failedRecoveryCount: 0,
|
|
1320
1343
|
});
|
|
1321
1344
|
}
|
|
@@ -1324,13 +1347,14 @@ async function recoverQueuedBlueBubblesMessages(deps = {}) {
|
|
|
1324
1347
|
const resolvedDeps = { ...defaultDeps, ...deps };
|
|
1325
1348
|
const agentName = resolvedDeps.getAgentName();
|
|
1326
1349
|
const previousState = (0, runtime_state_1.readBlueBubblesRuntimeState)(agentName);
|
|
1327
|
-
const initialPending =
|
|
1350
|
+
const initialPending = blueBubblesPendingRecoverySnapshot(agentName).pendingRecoveryCount;
|
|
1328
1351
|
if (initialPending === 0) {
|
|
1329
1352
|
return { recovered: 0, skipped: 0, failed: 0, pendingRecoveryCount: 0 };
|
|
1330
1353
|
}
|
|
1331
1354
|
const captured = await recoverCapturedBlueBubblesInboundMessages(resolvedDeps);
|
|
1332
1355
|
const recovery = await recoverMissedBlueBubblesMessages(resolvedDeps);
|
|
1333
|
-
const
|
|
1356
|
+
const pendingSnapshot = blueBubblesPendingRecoverySnapshot(agentName);
|
|
1357
|
+
const pendingRecoveryCount = pendingSnapshot.pendingRecoveryCount;
|
|
1334
1358
|
const failed = captured.failed + recovery.failed;
|
|
1335
1359
|
const recovered = captured.recovered + recovery.recovered;
|
|
1336
1360
|
const skipped = captured.skipped + recovery.skipped;
|
|
@@ -1341,7 +1365,8 @@ async function recoverQueuedBlueBubblesMessages(deps = {}) {
|
|
|
1341
1365
|
upstreamStatus: "ok",
|
|
1342
1366
|
detail: formatBlueBubblesRuntimeDetail(pendingRecoveryCount, failed),
|
|
1343
1367
|
lastCheckedAt: checkedAt,
|
|
1344
|
-
|
|
1368
|
+
proofMethod: "bluebubbles.checkHealth",
|
|
1369
|
+
...pendingSnapshot,
|
|
1345
1370
|
failedRecoveryCount: failed,
|
|
1346
1371
|
lastRecoveredAt: recovered > 0 ? checkedAt : previousState.lastRecoveredAt,
|
|
1347
1372
|
lastRecoveredMessageGuid: previousState.lastRecoveredMessageGuid,
|
|
@@ -1352,7 +1377,8 @@ async function recoverQueuedBlueBubblesMessages(deps = {}) {
|
|
|
1352
1377
|
upstreamStatus: "error",
|
|
1353
1378
|
detail: error instanceof Error ? error.message : String(error),
|
|
1354
1379
|
lastCheckedAt: checkedAt,
|
|
1355
|
-
|
|
1380
|
+
proofMethod: "bluebubbles.checkHealth",
|
|
1381
|
+
...pendingSnapshot,
|
|
1356
1382
|
failedRecoveryCount: failed,
|
|
1357
1383
|
lastRecoveredAt: recovered > 0 ? checkedAt : previousState.lastRecoveredAt,
|
|
1358
1384
|
lastRecoveredMessageGuid: previousState.lastRecoveredMessageGuid,
|
|
@@ -1691,24 +1717,85 @@ function createBlueBubblesWebhookHandler(deps = {}) {
|
|
|
1691
1717
|
writeJson(res, 400, { error: "Invalid JSON body" });
|
|
1692
1718
|
return;
|
|
1693
1719
|
}
|
|
1720
|
+
let normalized;
|
|
1694
1721
|
try {
|
|
1695
|
-
|
|
1696
|
-
writeJson(res, 200, result);
|
|
1722
|
+
normalized = (0, model_1.normalizeBlueBubblesEvent)(payload);
|
|
1697
1723
|
}
|
|
1698
1724
|
catch (error) {
|
|
1725
|
+
if (error instanceof model_1.BlueBubblesIgnoredEventError) {
|
|
1726
|
+
(0, runtime_1.emitNervesEvent)({
|
|
1727
|
+
component: "senses",
|
|
1728
|
+
event: "senses.bluebubbles_event_skipped",
|
|
1729
|
+
message: "skipped ignorable bluebubbles event",
|
|
1730
|
+
meta: {
|
|
1731
|
+
eventType: error.eventType,
|
|
1732
|
+
},
|
|
1733
|
+
});
|
|
1734
|
+
writeJson(res, 200, {
|
|
1735
|
+
handled: true,
|
|
1736
|
+
notifiedAgent: false,
|
|
1737
|
+
reason: "ignored",
|
|
1738
|
+
});
|
|
1739
|
+
return;
|
|
1740
|
+
}
|
|
1699
1741
|
(0, runtime_1.emitNervesEvent)({
|
|
1700
1742
|
level: "error",
|
|
1701
1743
|
component: "senses",
|
|
1702
1744
|
event: "senses.bluebubbles_webhook_error",
|
|
1703
1745
|
message: "bluebubbles webhook handling failed",
|
|
1704
1746
|
meta: {
|
|
1747
|
+
/* v8 ignore next -- normalizeBlueBubblesEvent throws Error subclasses; String fallback is defensive @preserve */
|
|
1705
1748
|
reason: error instanceof Error ? error.message : String(error),
|
|
1706
1749
|
},
|
|
1707
1750
|
});
|
|
1708
1751
|
writeJson(res, 500, {
|
|
1752
|
+
/* v8 ignore next -- normalizeBlueBubblesEvent throws Error subclasses; String fallback is defensive @preserve */
|
|
1709
1753
|
error: error instanceof Error ? error.message : String(error),
|
|
1710
1754
|
});
|
|
1755
|
+
return;
|
|
1756
|
+
}
|
|
1757
|
+
const resolvedDeps = { ...defaultDeps, ...deps };
|
|
1758
|
+
if (normalized.kind === "message" && !normalized.fromMe) {
|
|
1759
|
+
(0, inbound_log_1.recordBlueBubblesInbound)(resolvedDeps.getAgentName(), normalized, "webhook");
|
|
1711
1760
|
}
|
|
1761
|
+
else if (normalized.kind === "mutation") {
|
|
1762
|
+
try {
|
|
1763
|
+
resolvedDeps.recordMutation(resolvedDeps.getAgentName(), normalized);
|
|
1764
|
+
}
|
|
1765
|
+
catch (error) {
|
|
1766
|
+
(0, runtime_1.emitNervesEvent)({
|
|
1767
|
+
level: "error",
|
|
1768
|
+
component: "senses",
|
|
1769
|
+
event: "senses.bluebubbles_mutation_log_error",
|
|
1770
|
+
message: "failed recording bluebubbles mutation sidecar",
|
|
1771
|
+
meta: {
|
|
1772
|
+
messageGuid: normalized.messageGuid,
|
|
1773
|
+
mutationType: normalized.mutationType,
|
|
1774
|
+
reason: error instanceof Error ? error.message : String(error),
|
|
1775
|
+
},
|
|
1776
|
+
});
|
|
1777
|
+
}
|
|
1778
|
+
}
|
|
1779
|
+
writeJson(res, 200, {
|
|
1780
|
+
handled: true,
|
|
1781
|
+
notifiedAgent: false,
|
|
1782
|
+
kind: normalized.kind,
|
|
1783
|
+
queued: true,
|
|
1784
|
+
reason: "queued",
|
|
1785
|
+
});
|
|
1786
|
+
setTimeout(() => {
|
|
1787
|
+
void handleBlueBubblesEvent(payload, deps).catch((error) => {
|
|
1788
|
+
(0, runtime_1.emitNervesEvent)({
|
|
1789
|
+
level: "error",
|
|
1790
|
+
component: "senses",
|
|
1791
|
+
event: "senses.bluebubbles_webhook_async_error",
|
|
1792
|
+
message: "bluebubbles webhook async handling failed after durable capture",
|
|
1793
|
+
meta: {
|
|
1794
|
+
reason: error instanceof Error ? error.message : String(error),
|
|
1795
|
+
},
|
|
1796
|
+
});
|
|
1797
|
+
});
|
|
1798
|
+
}, 0);
|
|
1712
1799
|
};
|
|
1713
1800
|
}
|
|
1714
1801
|
function findImessageHandle(friend) {
|
|
@@ -64,10 +64,19 @@ function readBlueBubblesRuntimeState(agentName, agentRoot) {
|
|
|
64
64
|
? parsed.detail
|
|
65
65
|
: DEFAULT_RUNTIME_STATE.detail,
|
|
66
66
|
lastCheckedAt: typeof parsed.lastCheckedAt === "string" ? parsed.lastCheckedAt : undefined,
|
|
67
|
+
proofMethod: typeof parsed.proofMethod === "string" && parsed.proofMethod.trim()
|
|
68
|
+
? parsed.proofMethod
|
|
69
|
+
: undefined,
|
|
67
70
|
pendingRecoveryCount: typeof parsed.pendingRecoveryCount === "number" && Number.isFinite(parsed.pendingRecoveryCount)
|
|
68
71
|
? parsed.pendingRecoveryCount
|
|
69
72
|
: 0,
|
|
70
73
|
...(typeof failedRecoveryCount === "number" ? { failedRecoveryCount } : {}),
|
|
74
|
+
oldestPendingRecoveryAt: typeof parsed.oldestPendingRecoveryAt === "string"
|
|
75
|
+
? parsed.oldestPendingRecoveryAt
|
|
76
|
+
: undefined,
|
|
77
|
+
oldestPendingRecoveryAgeMs: typeof parsed.oldestPendingRecoveryAgeMs === "number" && Number.isFinite(parsed.oldestPendingRecoveryAgeMs)
|
|
78
|
+
? parsed.oldestPendingRecoveryAgeMs
|
|
79
|
+
: undefined,
|
|
71
80
|
lastRecoveredAt: typeof parsed.lastRecoveredAt === "string" ? parsed.lastRecoveredAt : undefined,
|
|
72
81
|
lastRecoveredMessageGuid: typeof parsed.lastRecoveredMessageGuid === "string"
|
|
73
82
|
? parsed.lastRecoveredMessageGuid
|