@ouro.bot/cli 0.1.0-alpha.543 → 0.1.0-alpha.545
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/changelog.json +16 -0
- package/dist/heart/daemon/cli-defaults.js +1 -1
- package/dist/heart/daemon/cli-render.js +20 -0
- package/dist/heart/daemon/os-cron-deps.js +1 -0
- package/dist/heart/daemon/os-cron.js +10 -11
- package/dist/heart/daemon/sense-manager.js +28 -0
- package/dist/senses/bluebubbles/active-turns.js +216 -0
- package/dist/senses/bluebubbles/index.js +124 -63
- package/dist/senses/bluebubbles/replay.js +8 -0
- package/dist/senses/bluebubbles/runtime-state.js +14 -0
- package/package.json +1 -1
package/changelog.json
CHANGED
|
@@ -1,6 +1,22 @@
|
|
|
1
1
|
{
|
|
2
2
|
"_note": "This changelog is maintained as part of the PR/version-bump workflow. Agent-curated, not auto-generated. Agents read this file directly via read_file to understand what changed between versions.",
|
|
3
3
|
"versions": [
|
|
4
|
+
{
|
|
5
|
+
"version": "0.1.0-alpha.545",
|
|
6
|
+
"changes": [
|
|
7
|
+
"Launchd-managed habit jobs now inherit the runtime PATH in their plist environment, preventing login-started jobs from failing to find the Node runtime.",
|
|
8
|
+
"The default daemon starter now launches with the current process executable instead of a bare `node` lookup, so background restarts survive sparse launch contexts.",
|
|
9
|
+
"Adds regression coverage for restart launch path hardening and mail-import discovery timestamp fallbacks."
|
|
10
|
+
]
|
|
11
|
+
},
|
|
12
|
+
{
|
|
13
|
+
"version": "0.1.0-alpha.544",
|
|
14
|
+
"changes": [
|
|
15
|
+
"BlueBubbles live webhook turns now have a bounded timeout, visible timeout notice, and recoverable captured-message retry path, so a long iMessage turn cannot silently monopolize the live chat lane.",
|
|
16
|
+
"BlueBubbles runtime health now records active and stalled live turns, exposes them in status output, and marks the sense unhealthy when the listener is reachable but a live iMessage turn is stuck.",
|
|
17
|
+
"The `ouro bluebubbles replay` debug command now refreshes this machine's BlueBubbles runtime credentials before creating the client, matching the daemon-launched listener's vault-backed configuration."
|
|
18
|
+
]
|
|
19
|
+
},
|
|
4
20
|
{
|
|
5
21
|
"version": "0.1.0-alpha.543",
|
|
6
22
|
"changes": [
|
|
@@ -81,7 +81,7 @@ function defaultStartDaemonProcess(socketPath) {
|
|
|
81
81
|
// when the daemon's logging system writes to stderr after the parent exits.
|
|
82
82
|
const outFd = fs.openSync(os.devNull, "w");
|
|
83
83
|
const errFd = fs.openSync(os.devNull, "w");
|
|
84
|
-
const child = (0, child_process_1.spawn)(
|
|
84
|
+
const child = (0, child_process_1.spawn)(process.execPath, [entry, "--socket", socketPath], {
|
|
85
85
|
detached: true,
|
|
86
86
|
stdio: ["ignore", outFd, errFd],
|
|
87
87
|
});
|
|
@@ -97,6 +97,14 @@ function formatSenseDetail(row) {
|
|
|
97
97
|
recoveryParts.push(`oldestPending=${row.oldestPendingRecoveryAt}`);
|
|
98
98
|
if (row.oldestPendingRecoveryAgeMs !== undefined)
|
|
99
99
|
recoveryParts.push(`oldestPendingAge=${formatDurationMs(row.oldestPendingRecoveryAgeMs)}`);
|
|
100
|
+
if (row.activeTurnCount !== undefined)
|
|
101
|
+
recoveryParts.push(`activeTurns=${row.activeTurnCount}`);
|
|
102
|
+
if (row.stalledTurnCount !== undefined)
|
|
103
|
+
recoveryParts.push(`stalledTurns=${row.stalledTurnCount}`);
|
|
104
|
+
if (row.oldestActiveTurnStartedAt)
|
|
105
|
+
recoveryParts.push(`oldestActive=${row.oldestActiveTurnStartedAt}`);
|
|
106
|
+
if (row.oldestActiveTurnAgeMs !== undefined)
|
|
107
|
+
recoveryParts.push(`oldestActiveAge=${formatDurationMs(row.oldestActiveTurnAgeMs)}`);
|
|
100
108
|
if (recoveryParts.length > 0)
|
|
101
109
|
details.push(recoveryParts.join(" "));
|
|
102
110
|
const failureParts = [];
|
|
@@ -181,6 +189,10 @@ function parseStatusPayload(data) {
|
|
|
181
189
|
const failedRecoveryCount = numberField(row.failedRecoveryCount);
|
|
182
190
|
const oldestPendingRecoveryAt = stringField(row.oldestPendingRecoveryAt);
|
|
183
191
|
const oldestPendingRecoveryAgeMs = numberField(row.oldestPendingRecoveryAgeMs);
|
|
192
|
+
const activeTurnCount = numberField(row.activeTurnCount);
|
|
193
|
+
const stalledTurnCount = numberField(row.stalledTurnCount);
|
|
194
|
+
const oldestActiveTurnStartedAt = stringField(row.oldestActiveTurnStartedAt);
|
|
195
|
+
const oldestActiveTurnAgeMs = numberField(row.oldestActiveTurnAgeMs);
|
|
184
196
|
if (proofMethod !== null)
|
|
185
197
|
parsed.proofMethod = proofMethod;
|
|
186
198
|
if (lastProofAt !== null)
|
|
@@ -201,6 +213,14 @@ function parseStatusPayload(data) {
|
|
|
201
213
|
parsed.oldestPendingRecoveryAt = oldestPendingRecoveryAt;
|
|
202
214
|
if (oldestPendingRecoveryAgeMs !== null)
|
|
203
215
|
parsed.oldestPendingRecoveryAgeMs = oldestPendingRecoveryAgeMs;
|
|
216
|
+
if (activeTurnCount !== null)
|
|
217
|
+
parsed.activeTurnCount = activeTurnCount;
|
|
218
|
+
if (stalledTurnCount !== null)
|
|
219
|
+
parsed.stalledTurnCount = stalledTurnCount;
|
|
220
|
+
if (oldestActiveTurnStartedAt !== null)
|
|
221
|
+
parsed.oldestActiveTurnStartedAt = oldestActiveTurnStartedAt;
|
|
222
|
+
if (oldestActiveTurnAgeMs !== null)
|
|
223
|
+
parsed.oldestActiveTurnAgeMs = oldestActiveTurnAgeMs;
|
|
204
224
|
return parsed;
|
|
205
225
|
});
|
|
206
226
|
const parsedWorkers = workers.map((entry) => {
|
|
@@ -80,7 +80,7 @@ function scheduleToCalendarInterval(schedule) {
|
|
|
80
80
|
result.Month = parseInt(month, 10);
|
|
81
81
|
return Object.keys(result).length > 0 ? result : null;
|
|
82
82
|
}
|
|
83
|
-
function generatePlistXml(job) {
|
|
83
|
+
function generatePlistXml(job, envPath) {
|
|
84
84
|
const label = plistLabel(job);
|
|
85
85
|
const seconds = cadenceToSeconds(job.schedule);
|
|
86
86
|
const calendar = seconds === null ? scheduleToCalendarInterval(job.schedule) : null;
|
|
@@ -97,7 +97,7 @@ function generatePlistXml(job) {
|
|
|
97
97
|
else {
|
|
98
98
|
triggerXml = ` <key>StartInterval</key>\n <integer>1800</integer>`;
|
|
99
99
|
}
|
|
100
|
-
|
|
100
|
+
const lines = [
|
|
101
101
|
`<?xml version="1.0" encoding="UTF-8"?>`,
|
|
102
102
|
`<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">`,
|
|
103
103
|
`<plist version="1.0">`,
|
|
@@ -110,14 +110,12 @@ function generatePlistXml(job) {
|
|
|
110
110
|
...job.command.split(" ").slice(1).map((arg) => ` <string>${arg}</string>`),
|
|
111
111
|
` </array>`,
|
|
112
112
|
triggerXml,
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
` <key>
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
``,
|
|
120
|
-
].join("\n");
|
|
113
|
+
];
|
|
114
|
+
if (envPath) {
|
|
115
|
+
lines.push(` <key>EnvironmentVariables</key>`, ` <dict>`, ` <key>PATH</key>`, ` <string>${envPath}</string>`, ` </dict>`);
|
|
116
|
+
}
|
|
117
|
+
lines.push(` <key>StandardOutPath</key>`, ` <string>/tmp/${label}.stdout.log</string>`, ` <key>StandardErrorPath</key>`, ` <string>/tmp/${label}.stderr.log</string>`, `</dict>`, `</plist>`, ``);
|
|
118
|
+
return lines.join("\n");
|
|
121
119
|
}
|
|
122
120
|
class LaunchdCronManager {
|
|
123
121
|
deps;
|
|
@@ -148,7 +146,7 @@ class LaunchdCronManager {
|
|
|
148
146
|
const label = plistLabel(job);
|
|
149
147
|
const filename = `${label}.plist`;
|
|
150
148
|
const fullPath = `${this.launchAgentsDir}/${filename}`;
|
|
151
|
-
const xml = generatePlistXml(job);
|
|
149
|
+
const xml = generatePlistXml(job, this.deps.envPath);
|
|
152
150
|
try {
|
|
153
151
|
this.deps.exec(`launchctl unload "${fullPath}"`);
|
|
154
152
|
}
|
|
@@ -248,6 +246,7 @@ function createOsCronManager(options = {}) {
|
|
|
248
246
|
listDir: () => [],
|
|
249
247
|
mkdirp: () => { },
|
|
250
248
|
homeDir: os.homedir(),
|
|
249
|
+
envPath: process.env.PATH ?? "",
|
|
251
250
|
};
|
|
252
251
|
/* v8 ignore stop */
|
|
253
252
|
return new LaunchdCronManager(deps);
|
|
@@ -279,6 +279,16 @@ function readBlueBubblesRuntimeJson(runtimePath) {
|
|
|
279
279
|
oldestPendingRecoveryAgeMs: typeof parsed.oldestPendingRecoveryAgeMs === "number" && Number.isFinite(parsed.oldestPendingRecoveryAgeMs)
|
|
280
280
|
? parsed.oldestPendingRecoveryAgeMs
|
|
281
281
|
: undefined,
|
|
282
|
+
activeTurnCount: typeof parsed.activeTurnCount === "number" && Number.isFinite(parsed.activeTurnCount)
|
|
283
|
+
? parsed.activeTurnCount
|
|
284
|
+
: undefined,
|
|
285
|
+
stalledTurnCount: typeof parsed.stalledTurnCount === "number" && Number.isFinite(parsed.stalledTurnCount)
|
|
286
|
+
? parsed.stalledTurnCount
|
|
287
|
+
: undefined,
|
|
288
|
+
oldestActiveTurnStartedAt: typeof parsed.oldestActiveTurnStartedAt === "string" ? parsed.oldestActiveTurnStartedAt : undefined,
|
|
289
|
+
oldestActiveTurnAgeMs: typeof parsed.oldestActiveTurnAgeMs === "number" && Number.isFinite(parsed.oldestActiveTurnAgeMs)
|
|
290
|
+
? parsed.oldestActiveTurnAgeMs
|
|
291
|
+
: undefined,
|
|
282
292
|
};
|
|
283
293
|
/* v8 ignore stop */
|
|
284
294
|
/* v8 ignore start -- defensive: catch for missing/corrupt BB runtime state file @preserve */
|
|
@@ -304,6 +314,10 @@ function readBlueBubblesRuntimeFacts(agent, bundlesRoot, snapshot) {
|
|
|
304
314
|
failedRecoveryCount: state.failedRecoveryCount,
|
|
305
315
|
oldestPendingRecoveryAt: state.oldestPendingRecoveryAt,
|
|
306
316
|
oldestPendingRecoveryAgeMs: state.oldestPendingRecoveryAgeMs,
|
|
317
|
+
activeTurnCount: state.activeTurnCount,
|
|
318
|
+
stalledTurnCount: state.stalledTurnCount,
|
|
319
|
+
oldestActiveTurnStartedAt: state.oldestActiveTurnStartedAt,
|
|
320
|
+
oldestActiveTurnAgeMs: state.oldestActiveTurnAgeMs,
|
|
307
321
|
};
|
|
308
322
|
if (!blueBubblesRuntimeStateIsFresh(state.lastCheckedAt)) {
|
|
309
323
|
return {
|
|
@@ -342,6 +356,16 @@ function readBlueBubblesRuntimeFacts(agent, bundlesRoot, snapshot) {
|
|
|
342
356
|
recoveryAction: "queued recovery will retry; inspect BlueBubbles inbound/recovery sidecar logs if age keeps growing",
|
|
343
357
|
};
|
|
344
358
|
}
|
|
359
|
+
if ((state.stalledTurnCount ?? 0) > 0) {
|
|
360
|
+
return {
|
|
361
|
+
runtime: "error",
|
|
362
|
+
detail: state.detail,
|
|
363
|
+
...proofFacts,
|
|
364
|
+
lastFailure: state.detail,
|
|
365
|
+
failureLayer: "live_turn_stall",
|
|
366
|
+
recoveryAction: "live iMessage turn timeout/watchdog will release the lane and recovery will retry captured messages",
|
|
367
|
+
};
|
|
368
|
+
}
|
|
345
369
|
if (state.upstreamStatus === "ok") {
|
|
346
370
|
return {
|
|
347
371
|
runtime: "running",
|
|
@@ -573,6 +597,10 @@ class DaemonSenseManager {
|
|
|
573
597
|
failedRecoveryCount: blueBubblesRuntimeFacts.failedRecoveryCount,
|
|
574
598
|
oldestPendingRecoveryAt: blueBubblesRuntimeFacts.oldestPendingRecoveryAt,
|
|
575
599
|
oldestPendingRecoveryAgeMs: blueBubblesRuntimeFacts.oldestPendingRecoveryAgeMs,
|
|
600
|
+
activeTurnCount: blueBubblesRuntimeFacts.activeTurnCount,
|
|
601
|
+
stalledTurnCount: blueBubblesRuntimeFacts.stalledTurnCount,
|
|
602
|
+
oldestActiveTurnStartedAt: blueBubblesRuntimeFacts.oldestActiveTurnStartedAt,
|
|
603
|
+
oldestActiveTurnAgeMs: blueBubblesRuntimeFacts.oldestActiveTurnAgeMs,
|
|
576
604
|
} : {}),
|
|
577
605
|
}));
|
|
578
606
|
});
|
|
@@ -0,0 +1,216 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
3
|
+
if (k2 === undefined) k2 = k;
|
|
4
|
+
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
5
|
+
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
6
|
+
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
7
|
+
}
|
|
8
|
+
Object.defineProperty(o, k2, desc);
|
|
9
|
+
}) : (function(o, m, k, k2) {
|
|
10
|
+
if (k2 === undefined) k2 = k;
|
|
11
|
+
o[k2] = m[k];
|
|
12
|
+
}));
|
|
13
|
+
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
|
14
|
+
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
|
15
|
+
}) : function(o, v) {
|
|
16
|
+
o["default"] = v;
|
|
17
|
+
});
|
|
18
|
+
var __importStar = (this && this.__importStar) || (function () {
|
|
19
|
+
var ownKeys = function(o) {
|
|
20
|
+
ownKeys = Object.getOwnPropertyNames || function (o) {
|
|
21
|
+
var ar = [];
|
|
22
|
+
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
|
|
23
|
+
return ar;
|
|
24
|
+
};
|
|
25
|
+
return ownKeys(o);
|
|
26
|
+
};
|
|
27
|
+
return function (mod) {
|
|
28
|
+
if (mod && mod.__esModule) return mod;
|
|
29
|
+
var result = {};
|
|
30
|
+
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
|
|
31
|
+
__setModuleDefault(result, mod);
|
|
32
|
+
return result;
|
|
33
|
+
};
|
|
34
|
+
})();
|
|
35
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
36
|
+
exports.beginBlueBubblesActiveTurn = beginBlueBubblesActiveTurn;
|
|
37
|
+
exports.noteBlueBubblesActiveTurnVisibleActivity = noteBlueBubblesActiveTurnVisibleActivity;
|
|
38
|
+
exports.finishBlueBubblesActiveTurn = finishBlueBubblesActiveTurn;
|
|
39
|
+
exports.listBlueBubblesActiveTurns = listBlueBubblesActiveTurns;
|
|
40
|
+
exports.snapshotBlueBubblesActiveTurns = snapshotBlueBubblesActiveTurns;
|
|
41
|
+
const fs = __importStar(require("node:fs"));
|
|
42
|
+
const path = __importStar(require("node:path"));
|
|
43
|
+
const config_1 = require("../../heart/config");
|
|
44
|
+
const identity_1 = require("../../heart/identity");
|
|
45
|
+
const runtime_1 = require("../../nerves/runtime");
|
|
46
|
+
function activeTurnsDir(agentName) {
|
|
47
|
+
return path.join((0, identity_1.getAgentRoot)(agentName), "state", "senses", "bluebubbles", "active-turns");
|
|
48
|
+
}
|
|
49
|
+
function activeTurnPath(agentName, turnId) {
|
|
50
|
+
return path.join(activeTurnsDir(agentName), `${(0, config_1.sanitizeKey)(turnId)}.json`);
|
|
51
|
+
}
|
|
52
|
+
function isPidAlive(pid) {
|
|
53
|
+
try {
|
|
54
|
+
process.kill(pid, 0);
|
|
55
|
+
return true;
|
|
56
|
+
}
|
|
57
|
+
catch {
|
|
58
|
+
return false;
|
|
59
|
+
}
|
|
60
|
+
}
|
|
61
|
+
function parseActiveTurn(raw) {
|
|
62
|
+
try {
|
|
63
|
+
const parsed = JSON.parse(raw);
|
|
64
|
+
if (!parsed.turnId || !parsed.messageGuid || !parsed.sessionKey || !parsed.startedAt)
|
|
65
|
+
return null;
|
|
66
|
+
if (typeof parsed.pid !== "number")
|
|
67
|
+
return null;
|
|
68
|
+
return {
|
|
69
|
+
turnId: parsed.turnId,
|
|
70
|
+
pid: parsed.pid,
|
|
71
|
+
startedAt: parsed.startedAt,
|
|
72
|
+
lastVisibleActivityAt: typeof parsed.lastVisibleActivityAt === "string"
|
|
73
|
+
? parsed.lastVisibleActivityAt
|
|
74
|
+
: undefined,
|
|
75
|
+
messageGuid: parsed.messageGuid,
|
|
76
|
+
sessionKey: parsed.sessionKey,
|
|
77
|
+
chatGuid: parsed.chatGuid ?? null,
|
|
78
|
+
chatIdentifier: parsed.chatIdentifier ?? null,
|
|
79
|
+
};
|
|
80
|
+
}
|
|
81
|
+
catch {
|
|
82
|
+
return null;
|
|
83
|
+
}
|
|
84
|
+
}
|
|
85
|
+
function beginBlueBubblesActiveTurn(agentName, event) {
|
|
86
|
+
const turnId = `${event.chat.sessionKey}:${event.messageGuid}:${process.pid}`;
|
|
87
|
+
const entry = {
|
|
88
|
+
turnId,
|
|
89
|
+
pid: process.pid,
|
|
90
|
+
startedAt: new Date().toISOString(),
|
|
91
|
+
messageGuid: event.messageGuid,
|
|
92
|
+
sessionKey: event.chat.sessionKey,
|
|
93
|
+
chatGuid: event.chat.chatGuid ?? null,
|
|
94
|
+
chatIdentifier: event.chat.chatIdentifier ?? null,
|
|
95
|
+
};
|
|
96
|
+
const filePath = activeTurnPath(agentName, turnId);
|
|
97
|
+
try {
|
|
98
|
+
fs.mkdirSync(path.dirname(filePath), { recursive: true });
|
|
99
|
+
fs.writeFileSync(filePath, JSON.stringify(entry, null, 2) + "\n", "utf-8");
|
|
100
|
+
}
|
|
101
|
+
catch (error) {
|
|
102
|
+
(0, runtime_1.emitNervesEvent)({
|
|
103
|
+
level: "warn",
|
|
104
|
+
component: "senses",
|
|
105
|
+
event: "senses.bluebubbles_active_turn_write_error",
|
|
106
|
+
message: "failed to record active bluebubbles turn",
|
|
107
|
+
meta: {
|
|
108
|
+
agentName,
|
|
109
|
+
messageGuid: event.messageGuid,
|
|
110
|
+
sessionKey: event.chat.sessionKey,
|
|
111
|
+
/* v8 ignore next -- filesystem writes throw Error instances; stringify guard is defensive @preserve */
|
|
112
|
+
reason: error instanceof Error ? error.message : String(error),
|
|
113
|
+
},
|
|
114
|
+
});
|
|
115
|
+
}
|
|
116
|
+
return turnId;
|
|
117
|
+
}
|
|
118
|
+
function noteBlueBubblesActiveTurnVisibleActivity(agentName, turnId) {
|
|
119
|
+
const filePath = activeTurnPath(agentName, turnId);
|
|
120
|
+
let entry = null;
|
|
121
|
+
try {
|
|
122
|
+
entry = parseActiveTurn(fs.readFileSync(filePath, "utf-8"));
|
|
123
|
+
}
|
|
124
|
+
catch {
|
|
125
|
+
return;
|
|
126
|
+
}
|
|
127
|
+
if (!entry)
|
|
128
|
+
return;
|
|
129
|
+
try {
|
|
130
|
+
fs.writeFileSync(filePath, JSON.stringify({ ...entry, lastVisibleActivityAt: new Date().toISOString() }, null, 2) + "\n", "utf-8");
|
|
131
|
+
}
|
|
132
|
+
catch (error) {
|
|
133
|
+
(0, runtime_1.emitNervesEvent)({
|
|
134
|
+
level: "warn",
|
|
135
|
+
component: "senses",
|
|
136
|
+
event: "senses.bluebubbles_active_turn_write_error",
|
|
137
|
+
message: "failed to update active bluebubbles turn",
|
|
138
|
+
meta: {
|
|
139
|
+
agentName,
|
|
140
|
+
turnId,
|
|
141
|
+
/* v8 ignore next -- filesystem writes throw Error instances; stringify guard is defensive @preserve */
|
|
142
|
+
reason: error instanceof Error ? error.message : String(error),
|
|
143
|
+
},
|
|
144
|
+
});
|
|
145
|
+
}
|
|
146
|
+
}
|
|
147
|
+
function finishBlueBubblesActiveTurn(agentName, turnId) {
|
|
148
|
+
try {
|
|
149
|
+
fs.unlinkSync(activeTurnPath(agentName, turnId));
|
|
150
|
+
}
|
|
151
|
+
catch {
|
|
152
|
+
// Missing active-turn files are harmless: this is best-effort telemetry.
|
|
153
|
+
}
|
|
154
|
+
}
|
|
155
|
+
function listBlueBubblesActiveTurns(agentName) {
|
|
156
|
+
let files;
|
|
157
|
+
try {
|
|
158
|
+
files = fs.readdirSync(activeTurnsDir(agentName)).filter((name) => name.endsWith(".json")).sort();
|
|
159
|
+
}
|
|
160
|
+
catch {
|
|
161
|
+
return [];
|
|
162
|
+
}
|
|
163
|
+
const entries = [];
|
|
164
|
+
for (const file of files) {
|
|
165
|
+
const filePath = path.join(activeTurnsDir(agentName), file);
|
|
166
|
+
let entry = null;
|
|
167
|
+
try {
|
|
168
|
+
entry = parseActiveTurn(fs.readFileSync(filePath, "utf-8"));
|
|
169
|
+
}
|
|
170
|
+
catch {
|
|
171
|
+
entry = null;
|
|
172
|
+
}
|
|
173
|
+
if (!entry) {
|
|
174
|
+
try {
|
|
175
|
+
fs.unlinkSync(filePath);
|
|
176
|
+
}
|
|
177
|
+
catch { /* ignore corrupt cleanup races */ }
|
|
178
|
+
continue;
|
|
179
|
+
}
|
|
180
|
+
if (!isPidAlive(entry.pid)) {
|
|
181
|
+
try {
|
|
182
|
+
fs.unlinkSync(filePath);
|
|
183
|
+
}
|
|
184
|
+
catch { /* ignore stale cleanup races */ }
|
|
185
|
+
(0, runtime_1.emitNervesEvent)({
|
|
186
|
+
level: "warn",
|
|
187
|
+
component: "senses",
|
|
188
|
+
event: "senses.bluebubbles_active_turn_pruned",
|
|
189
|
+
message: "pruned stale bluebubbles active-turn marker",
|
|
190
|
+
meta: {
|
|
191
|
+
agentName,
|
|
192
|
+
messageGuid: entry.messageGuid,
|
|
193
|
+
sessionKey: entry.sessionKey,
|
|
194
|
+
pid: entry.pid,
|
|
195
|
+
},
|
|
196
|
+
});
|
|
197
|
+
continue;
|
|
198
|
+
}
|
|
199
|
+
entries.push(entry);
|
|
200
|
+
}
|
|
201
|
+
return entries;
|
|
202
|
+
}
|
|
203
|
+
function snapshotBlueBubblesActiveTurns(agentName, stalledAfterMs, nowMs = Date.now()) {
|
|
204
|
+
const entries = listBlueBubblesActiveTurns(agentName);
|
|
205
|
+
const started = entries
|
|
206
|
+
.map((entry) => ({ value: entry.startedAt, ms: Date.parse(entry.startedAt) }))
|
|
207
|
+
.filter((entry) => Number.isFinite(entry.ms))
|
|
208
|
+
.sort((left, right) => left.ms - right.ms);
|
|
209
|
+
const oldest = started[0];
|
|
210
|
+
return {
|
|
211
|
+
activeTurnCount: entries.length,
|
|
212
|
+
stalledTurnCount: started.filter((entry) => nowMs - entry.ms >= stalledAfterMs).length,
|
|
213
|
+
oldestActiveTurnStartedAt: oldest?.value,
|
|
214
|
+
oldestActiveTurnAgeMs: oldest ? Math.max(0, nowMs - oldest.ms) : undefined,
|
|
215
|
+
};
|
|
216
|
+
}
|
|
@@ -76,6 +76,7 @@ const mutation_log_1 = require("./mutation-log");
|
|
|
76
76
|
const processed_log_1 = require("./processed-log");
|
|
77
77
|
const runtime_state_1 = require("./runtime-state");
|
|
78
78
|
const session_cleanup_1 = require("./session-cleanup");
|
|
79
|
+
const active_turns_1 = require("./active-turns");
|
|
79
80
|
const tool_activity_callbacks_1 = require("../../heart/tool-activity-callbacks");
|
|
80
81
|
const commands_1 = require("../commands");
|
|
81
82
|
const trust_gate_1 = require("../trust-gate");
|
|
@@ -170,7 +171,10 @@ const defaultDeps = {
|
|
|
170
171
|
const BLUEBUBBLES_RUNTIME_SYNC_INTERVAL_MS = 30_000;
|
|
171
172
|
const BLUEBUBBLES_RECOVERY_PASS_DELAY_MS = 1_000;
|
|
172
173
|
const BLUEBUBBLES_RECOVERY_PASS_INTERVAL_MS = 30_000;
|
|
174
|
+
const BLUEBUBBLES_LIVE_TURN_TIMEOUT_MS = 2 * 60_000;
|
|
173
175
|
const BLUEBUBBLES_RECOVERY_TURN_TIMEOUT_MS = 10 * 60_000;
|
|
176
|
+
const BLUEBUBBLES_LIVE_TURN_STALLED_MS = 90_000;
|
|
177
|
+
const BLUEBUBBLES_SILENCE_WATCHDOG_MS = 75_000;
|
|
174
178
|
const BLUEBUBBLES_CATCHUP_PAGE_SIZE = 50;
|
|
175
179
|
const BLUEBUBBLES_CATCHUP_MAX_PAGES = 20;
|
|
176
180
|
const BLUEBUBBLES_HEALTHY_CATCHUP_OVERLAP_MS = 90_000;
|
|
@@ -447,10 +451,12 @@ function emitBlueBubblesMarkReadWarning(chat, error) {
|
|
|
447
451
|
},
|
|
448
452
|
});
|
|
449
453
|
}
|
|
450
|
-
function createBlueBubblesCallbacks(client, chat, replyTarget, isGroupChat) {
|
|
454
|
+
function createBlueBubblesCallbacks(client, chat, replyTarget, isGroupChat, onVisibleActivity) {
|
|
451
455
|
let textBuffer = "";
|
|
452
456
|
let typingActive = false;
|
|
453
457
|
let queue = Promise.resolve();
|
|
458
|
+
let lastVisibleActivityMs = Date.now();
|
|
459
|
+
let silenceWatchdog = null;
|
|
454
460
|
function enqueue(operation, task) {
|
|
455
461
|
queue = queue.then(task).catch((error) => {
|
|
456
462
|
(0, runtime_1.emitNervesEvent)({
|
|
@@ -480,6 +486,29 @@ function createBlueBubblesCallbacks(client, chat, replyTarget, isGroupChat) {
|
|
|
480
486
|
}
|
|
481
487
|
});
|
|
482
488
|
}
|
|
489
|
+
function recordVisibleActivity() {
|
|
490
|
+
lastVisibleActivityMs = Date.now();
|
|
491
|
+
onVisibleActivity?.();
|
|
492
|
+
}
|
|
493
|
+
function stopSilenceWatchdog() {
|
|
494
|
+
if (silenceWatchdog === null)
|
|
495
|
+
return;
|
|
496
|
+
clearInterval(silenceWatchdog);
|
|
497
|
+
silenceWatchdog = null;
|
|
498
|
+
}
|
|
499
|
+
function startSilenceWatchdog() {
|
|
500
|
+
if (silenceWatchdog !== null)
|
|
501
|
+
return;
|
|
502
|
+
silenceWatchdog = setInterval(() => {
|
|
503
|
+
if (Date.now() - lastVisibleActivityMs < BLUEBUBBLES_SILENCE_WATCHDOG_MS)
|
|
504
|
+
return;
|
|
505
|
+
sendStatus("still working on this...");
|
|
506
|
+
}, BLUEBUBBLES_SILENCE_WATCHDOG_MS);
|
|
507
|
+
/* v8 ignore next -- timer handles expose unref only in some runtimes @preserve */
|
|
508
|
+
if (typeof silenceWatchdog.unref === "function") {
|
|
509
|
+
silenceWatchdog.unref();
|
|
510
|
+
}
|
|
511
|
+
}
|
|
483
512
|
function sendStatus(text) {
|
|
484
513
|
enqueue("send_status", async () => {
|
|
485
514
|
await client.sendText({
|
|
@@ -487,6 +516,7 @@ function createBlueBubblesCallbacks(client, chat, replyTarget, isGroupChat) {
|
|
|
487
516
|
text,
|
|
488
517
|
replyToMessageGuid: replyTarget.getReplyToMessageGuid(),
|
|
489
518
|
});
|
|
519
|
+
recordVisibleActivity();
|
|
490
520
|
// Re-enable typing indicator — sending a message clears the typing bubble
|
|
491
521
|
await client.setTyping(chat, true);
|
|
492
522
|
});
|
|
@@ -502,6 +532,7 @@ function createBlueBubblesCallbacks(client, chat, replyTarget, isGroupChat) {
|
|
|
502
532
|
});
|
|
503
533
|
return {
|
|
504
534
|
onModelStart() {
|
|
535
|
+
startSilenceWatchdog();
|
|
505
536
|
if (!isGroupChat)
|
|
506
537
|
startTypingNow();
|
|
507
538
|
(0, runtime_1.emitNervesEvent)({
|
|
@@ -590,6 +621,7 @@ function createBlueBubblesCallbacks(client, chat, replyTarget, isGroupChat) {
|
|
|
590
621
|
text: trimmed,
|
|
591
622
|
replyToMessageGuid: replyTarget.getReplyToMessageGuid(),
|
|
592
623
|
});
|
|
624
|
+
recordVisibleActivity();
|
|
593
625
|
// Note: do NOT call client.setTyping(chat, false) here — the agent is
|
|
594
626
|
// still mid-turn, so the typing indicator stays ACTIVE.
|
|
595
627
|
(0, runtime_1.emitNervesEvent)({
|
|
@@ -623,8 +655,10 @@ function createBlueBubblesCallbacks(client, chat, replyTarget, isGroupChat) {
|
|
|
623
655
|
text: trimmed,
|
|
624
656
|
replyToMessageGuid: replyTarget.getReplyToMessageGuid(),
|
|
625
657
|
});
|
|
658
|
+
recordVisibleActivity();
|
|
626
659
|
},
|
|
627
660
|
async finish() {
|
|
661
|
+
stopSilenceWatchdog();
|
|
628
662
|
statusBatcher.flush();
|
|
629
663
|
if (!typingActive) {
|
|
630
664
|
await queue;
|
|
@@ -824,6 +858,7 @@ async function handleBlueBubblesNormalizedEvent(event, resolvedDeps, source, opt
|
|
|
824
858
|
}
|
|
825
859
|
let ownsInFlightMessage = false;
|
|
826
860
|
let releaseInFlightAfterTurnSettles = false;
|
|
861
|
+
let activeTurnId = null;
|
|
827
862
|
if (event.kind === "message") {
|
|
828
863
|
if ((0, processed_log_1.hasProcessedBlueBubblesMessage)(agentName, event.chat.sessionKey, event.messageGuid)
|
|
829
864
|
|| (0, processed_log_1.hasProcessedBlueBubblesMessageGuid)(agentName, event.messageGuid)) {
|
|
@@ -855,6 +890,7 @@ async function handleBlueBubblesNormalizedEvent(event, resolvedDeps, source, opt
|
|
|
855
890
|
return { handled: true, notifiedAgent: false, kind: event.kind, reason: "already_processed" };
|
|
856
891
|
}
|
|
857
892
|
ownsInFlightMessage = true;
|
|
893
|
+
activeTurnId = (0, active_turns_1.beginBlueBubblesActiveTurn)(agentName, event);
|
|
858
894
|
}
|
|
859
895
|
try {
|
|
860
896
|
// ── Adapter setup: friend, session, content, callbacks ──────────
|
|
@@ -945,10 +981,12 @@ async function handleBlueBubblesNormalizedEvent(event, resolvedDeps, source, opt
|
|
|
945
981
|
role: "user",
|
|
946
982
|
content: buildInboundContent(event, existing?.messages ?? sessionMessages, repliedToText),
|
|
947
983
|
};
|
|
948
|
-
const callbacks = createBlueBubblesCallbacks(client, event.chat, replyTarget, event.chat.isGroup
|
|
984
|
+
const callbacks = createBlueBubblesCallbacks(client, event.chat, replyTarget, event.chat.isGroup, activeTurnId
|
|
985
|
+
? () => (0, active_turns_1.noteBlueBubblesActiveTurnVisibleActivity)(agentName, activeTurnId)
|
|
986
|
+
: undefined);
|
|
949
987
|
const controller = new AbortController();
|
|
950
|
-
let timeoutTimer
|
|
951
|
-
let timeoutPromise
|
|
988
|
+
let timeoutTimer;
|
|
989
|
+
let timeoutPromise;
|
|
952
990
|
let timeoutReject;
|
|
953
991
|
let recoveryTimedOut = false;
|
|
954
992
|
// BB-specific tool context wrappers
|
|
@@ -978,34 +1016,39 @@ async function handleBlueBubblesNormalizedEvent(event, resolvedDeps, source, opt
|
|
|
978
1016
|
};
|
|
979
1017
|
/* v8 ignore stop */
|
|
980
1018
|
try {
|
|
981
|
-
const
|
|
982
|
-
|
|
983
|
-
|
|
984
|
-
|
|
985
|
-
|
|
986
|
-
|
|
987
|
-
|
|
988
|
-
|
|
1019
|
+
const liveWebhookTimeout = source === "webhook" && options.timeoutMs === undefined;
|
|
1020
|
+
const timeoutMs = options.timeoutMs ?? BLUEBUBBLES_LIVE_TURN_TIMEOUT_MS;
|
|
1021
|
+
timeoutPromise = new Promise((_, reject) => {
|
|
1022
|
+
timeoutReject = reject;
|
|
1023
|
+
});
|
|
1024
|
+
timeoutTimer = setTimeout(() => {
|
|
1025
|
+
const reason = new BlueBubblesRecoveryTurnTimeoutError(timeoutMs);
|
|
1026
|
+
recoveryTimedOut = true;
|
|
1027
|
+
if (liveWebhookTimeout && ownsInFlightMessage && event.kind === "message") {
|
|
1028
|
+
endBlueBubblesMessageInFlight(event.chat.sessionKey, event.messageGuid);
|
|
1029
|
+
ownsInFlightMessage = false;
|
|
1030
|
+
}
|
|
1031
|
+
else {
|
|
989
1032
|
releaseInFlightAfterTurnSettles = true;
|
|
990
|
-
controller.abort(reason);
|
|
991
|
-
timeoutReject?.(reason);
|
|
992
|
-
(0, runtime_1.emitNervesEvent)({
|
|
993
|
-
level: "warn",
|
|
994
|
-
component: "senses",
|
|
995
|
-
event: "senses.bluebubbles_turn_timeout",
|
|
996
|
-
message: "bluebubbles recovery turn timed out",
|
|
997
|
-
meta: {
|
|
998
|
-
messageGuid: event.messageGuid,
|
|
999
|
-
sessionKey: event.chat.sessionKey,
|
|
1000
|
-
source,
|
|
1001
|
-
timeoutMs,
|
|
1002
|
-
},
|
|
1003
|
-
});
|
|
1004
|
-
}, timeoutMs);
|
|
1005
|
-
/* v8 ignore next -- timer handles expose unref only in some runtimes @preserve */
|
|
1006
|
-
if (typeof timeoutTimer.unref === "function") {
|
|
1007
|
-
timeoutTimer.unref();
|
|
1008
1033
|
}
|
|
1034
|
+
controller.abort(reason);
|
|
1035
|
+
timeoutReject?.(reason);
|
|
1036
|
+
(0, runtime_1.emitNervesEvent)({
|
|
1037
|
+
level: "warn",
|
|
1038
|
+
component: "senses",
|
|
1039
|
+
event: "senses.bluebubbles_turn_timeout",
|
|
1040
|
+
message: "bluebubbles turn timed out",
|
|
1041
|
+
meta: {
|
|
1042
|
+
messageGuid: event.messageGuid,
|
|
1043
|
+
sessionKey: event.chat.sessionKey,
|
|
1044
|
+
source,
|
|
1045
|
+
timeoutMs,
|
|
1046
|
+
},
|
|
1047
|
+
});
|
|
1048
|
+
}, timeoutMs);
|
|
1049
|
+
/* v8 ignore next -- timer handles expose unref only in some runtimes @preserve */
|
|
1050
|
+
if (typeof timeoutTimer.unref === "function") {
|
|
1051
|
+
timeoutTimer.unref();
|
|
1009
1052
|
}
|
|
1010
1053
|
const turnPromise = (0, pipeline_1.handleInboundTurn)({
|
|
1011
1054
|
channel: "bluebubbles",
|
|
@@ -1069,34 +1112,40 @@ async function handleBlueBubblesNormalizedEvent(event, resolvedDeps, source, opt
|
|
|
1069
1112
|
})(),
|
|
1070
1113
|
});
|
|
1071
1114
|
/* v8 ignore start -- detached late-rejection telemetry is asserted in timeout tests, but V8 does not reliably attribute Promise.catch callbacks @preserve */
|
|
1072
|
-
|
|
1073
|
-
|
|
1074
|
-
|
|
1075
|
-
|
|
1076
|
-
|
|
1077
|
-
|
|
1078
|
-
|
|
1079
|
-
|
|
1080
|
-
|
|
1081
|
-
|
|
1082
|
-
|
|
1083
|
-
|
|
1084
|
-
|
|
1085
|
-
|
|
1086
|
-
|
|
1087
|
-
},
|
|
1088
|
-
});
|
|
1089
|
-
})
|
|
1090
|
-
.finally(() => {
|
|
1091
|
-
if (releaseInFlightAfterTurnSettles && ownsInFlightMessage && event.kind === "message") {
|
|
1092
|
-
endBlueBubblesMessageInFlight(event.chat.sessionKey, event.messageGuid);
|
|
1093
|
-
}
|
|
1115
|
+
void turnPromise
|
|
1116
|
+
.catch((error) => {
|
|
1117
|
+
if (!recoveryTimedOut)
|
|
1118
|
+
return;
|
|
1119
|
+
(0, runtime_1.emitNervesEvent)({
|
|
1120
|
+
level: "warn",
|
|
1121
|
+
component: "senses",
|
|
1122
|
+
event: "senses.bluebubbles_recovery_error",
|
|
1123
|
+
message: "bluebubbles recovery turn rejected after timeout",
|
|
1124
|
+
meta: {
|
|
1125
|
+
messageGuid: event.messageGuid,
|
|
1126
|
+
sessionKey: event.chat.sessionKey,
|
|
1127
|
+
source,
|
|
1128
|
+
reason: error instanceof Error ? error.message : String(error),
|
|
1129
|
+
},
|
|
1094
1130
|
});
|
|
1095
|
-
}
|
|
1131
|
+
})
|
|
1132
|
+
.finally(() => {
|
|
1133
|
+
if (releaseInFlightAfterTurnSettles && ownsInFlightMessage && event.kind === "message") {
|
|
1134
|
+
endBlueBubblesMessageInFlight(event.chat.sessionKey, event.messageGuid);
|
|
1135
|
+
}
|
|
1136
|
+
});
|
|
1096
1137
|
/* v8 ignore stop */
|
|
1097
|
-
const result =
|
|
1098
|
-
|
|
1099
|
-
|
|
1138
|
+
const result = await (async () => {
|
|
1139
|
+
try {
|
|
1140
|
+
return await Promise.race([turnPromise, timeoutPromise]);
|
|
1141
|
+
}
|
|
1142
|
+
catch (error) {
|
|
1143
|
+
if (error instanceof BlueBubblesRecoveryTurnTimeoutError) {
|
|
1144
|
+
callbacks.onError(new Error("live iMessage turn timed out; I captured it for recovery instead of silently hanging"), "terminal");
|
|
1145
|
+
}
|
|
1146
|
+
throw error;
|
|
1147
|
+
}
|
|
1148
|
+
})();
|
|
1100
1149
|
/* v8 ignore start -- failover display + error replay @preserve */
|
|
1101
1150
|
if (result.failoverMessage) {
|
|
1102
1151
|
// Failover handled it — show the failover message, skip the buffered error
|
|
@@ -1155,15 +1204,14 @@ async function handleBlueBubblesNormalizedEvent(event, resolvedDeps, source, opt
|
|
|
1155
1204
|
bufferedTerminalError = null;
|
|
1156
1205
|
}
|
|
1157
1206
|
/* v8 ignore stop */
|
|
1158
|
-
|
|
1159
|
-
clearTimeout(timeoutTimer);
|
|
1160
|
-
timeoutTimer = null;
|
|
1161
|
-
}
|
|
1207
|
+
clearTimeout(timeoutTimer);
|
|
1162
1208
|
await callbacks.finish();
|
|
1163
1209
|
}
|
|
1164
1210
|
});
|
|
1165
1211
|
}
|
|
1166
1212
|
finally {
|
|
1213
|
+
if (activeTurnId)
|
|
1214
|
+
(0, active_turns_1.finishBlueBubblesActiveTurn)(agentName, activeTurnId);
|
|
1167
1215
|
if (ownsInFlightMessage && event.kind === "message" && !releaseInFlightAfterTurnSettles) {
|
|
1168
1216
|
endBlueBubblesMessageInFlight(event.chat.sessionKey, event.messageGuid);
|
|
1169
1217
|
}
|
|
@@ -1292,7 +1340,12 @@ function resolveBlueBubblesCatchUpSince(previousState, nowMs = Date.now()) {
|
|
|
1292
1340
|
}
|
|
1293
1341
|
return nowMs - BLUEBUBBLES_FIRST_CATCHUP_LOOKBACK_MS;
|
|
1294
1342
|
}
|
|
1295
|
-
function formatBlueBubblesRuntimeDetail(queued, failed) {
|
|
1343
|
+
function formatBlueBubblesRuntimeDetail(queued, failed, active) {
|
|
1344
|
+
if (active.stalledTurnCount > 0) {
|
|
1345
|
+
return `iMessage live turn appears stalled; ${active.stalledTurnCount} active turn(s) older than ${BLUEBUBBLES_LIVE_TURN_STALLED_MS}ms`;
|
|
1346
|
+
}
|
|
1347
|
+
if (active.activeTurnCount > 0)
|
|
1348
|
+
return `upstream reachable; ${active.activeTurnCount} live turn(s) active`;
|
|
1296
1349
|
if (queued > 0)
|
|
1297
1350
|
return `upstream reachable but iMessage is not caught up; ${queued} recovery item(s) queued`;
|
|
1298
1351
|
if (failed > 0)
|
|
@@ -1322,12 +1375,14 @@ async function syncBlueBubblesRuntime(deps = {}) {
|
|
|
1322
1375
|
try {
|
|
1323
1376
|
await client.checkHealth();
|
|
1324
1377
|
const pendingBeforeCatchup = blueBubblesPendingRecoverySnapshot(agentName);
|
|
1378
|
+
const activeBeforeCatchup = (0, active_turns_1.snapshotBlueBubblesActiveTurns)(agentName, BLUEBUBBLES_LIVE_TURN_STALLED_MS);
|
|
1325
1379
|
(0, runtime_state_1.writeBlueBubblesRuntimeState)(agentName, {
|
|
1326
1380
|
upstreamStatus: "ok",
|
|
1327
1381
|
detail: "upstream reachable; recovery pass running",
|
|
1328
1382
|
lastCheckedAt: checkedAt,
|
|
1329
1383
|
proofMethod: "bluebubbles.checkHealth",
|
|
1330
1384
|
...pendingBeforeCatchup,
|
|
1385
|
+
...activeBeforeCatchup,
|
|
1331
1386
|
lastRecoveredAt: previousState.lastRecoveredAt,
|
|
1332
1387
|
lastRecoveredMessageGuid: previousState.lastRecoveredMessageGuid,
|
|
1333
1388
|
});
|
|
@@ -1336,6 +1391,7 @@ async function syncBlueBubblesRuntime(deps = {}) {
|
|
|
1336
1391
|
});
|
|
1337
1392
|
const failed = catchUp.failed;
|
|
1338
1393
|
const pendingAfterCatchup = blueBubblesPendingRecoverySnapshot(agentName);
|
|
1394
|
+
const activeAfterCatchup = (0, active_turns_1.snapshotBlueBubblesActiveTurns)(agentName, BLUEBUBBLES_LIVE_TURN_STALLED_MS);
|
|
1339
1395
|
const queued = pendingAfterCatchup.pendingRecoveryCount;
|
|
1340
1396
|
// upstreamStatus reflects whether BlueBubbles itself and the local bridge
|
|
1341
1397
|
// can answer webhook traffic. The daemon status layer treats
|
|
@@ -1343,10 +1399,11 @@ async function syncBlueBubblesRuntime(deps = {}) {
|
|
|
1343
1399
|
// while this field stays scoped to upstream transport reachability.
|
|
1344
1400
|
(0, runtime_state_1.writeBlueBubblesRuntimeState)(agentName, {
|
|
1345
1401
|
upstreamStatus: "ok",
|
|
1346
|
-
detail: formatBlueBubblesRuntimeDetail(queued, failed),
|
|
1402
|
+
detail: formatBlueBubblesRuntimeDetail(queued, failed, activeAfterCatchup),
|
|
1347
1403
|
lastCheckedAt: checkedAt,
|
|
1348
1404
|
proofMethod: "bluebubbles.checkHealth",
|
|
1349
1405
|
...pendingAfterCatchup,
|
|
1406
|
+
...activeAfterCatchup,
|
|
1350
1407
|
pendingRecoveryCount: queued,
|
|
1351
1408
|
failedRecoveryCount: failed,
|
|
1352
1409
|
lastRecoveredAt: previousState.lastRecoveredAt,
|
|
@@ -1360,6 +1417,7 @@ async function syncBlueBubblesRuntime(deps = {}) {
|
|
|
1360
1417
|
lastCheckedAt: checkedAt,
|
|
1361
1418
|
proofMethod: "bluebubbles.checkHealth",
|
|
1362
1419
|
...blueBubblesPendingRecoverySnapshot(agentName),
|
|
1420
|
+
...(0, active_turns_1.snapshotBlueBubblesActiveTurns)(agentName, BLUEBUBBLES_LIVE_TURN_STALLED_MS),
|
|
1363
1421
|
failedRecoveryCount: 0,
|
|
1364
1422
|
});
|
|
1365
1423
|
}
|
|
@@ -1382,12 +1440,14 @@ async function recoverQueuedBlueBubblesMessages(deps = {}) {
|
|
|
1382
1440
|
const checkedAt = new Date().toISOString();
|
|
1383
1441
|
try {
|
|
1384
1442
|
await resolvedDeps.createClient().checkHealth();
|
|
1443
|
+
const activeSnapshot = (0, active_turns_1.snapshotBlueBubblesActiveTurns)(agentName, BLUEBUBBLES_LIVE_TURN_STALLED_MS);
|
|
1385
1444
|
(0, runtime_state_1.writeBlueBubblesRuntimeState)(agentName, {
|
|
1386
1445
|
upstreamStatus: "ok",
|
|
1387
|
-
detail: formatBlueBubblesRuntimeDetail(pendingRecoveryCount, failed),
|
|
1446
|
+
detail: formatBlueBubblesRuntimeDetail(pendingRecoveryCount, failed, activeSnapshot),
|
|
1388
1447
|
lastCheckedAt: checkedAt,
|
|
1389
1448
|
proofMethod: "bluebubbles.checkHealth",
|
|
1390
1449
|
...pendingSnapshot,
|
|
1450
|
+
...activeSnapshot,
|
|
1391
1451
|
failedRecoveryCount: failed,
|
|
1392
1452
|
lastRecoveredAt: recovered > 0 ? checkedAt : previousState.lastRecoveredAt,
|
|
1393
1453
|
lastRecoveredMessageGuid: previousState.lastRecoveredMessageGuid,
|
|
@@ -1400,6 +1460,7 @@ async function recoverQueuedBlueBubblesMessages(deps = {}) {
|
|
|
1400
1460
|
lastCheckedAt: checkedAt,
|
|
1401
1461
|
proofMethod: "bluebubbles.checkHealth",
|
|
1402
1462
|
...pendingSnapshot,
|
|
1463
|
+
...(0, active_turns_1.snapshotBlueBubblesActiveTurns)(agentName, BLUEBUBBLES_LIVE_TURN_STALLED_MS),
|
|
1403
1464
|
failedRecoveryCount: failed,
|
|
1404
1465
|
lastRecoveredAt: recovered > 0 ? checkedAt : previousState.lastRecoveredAt,
|
|
1405
1466
|
lastRecoveredMessageGuid: previousState.lastRecoveredMessageGuid,
|
|
@@ -4,7 +4,9 @@ exports.replayBlueBubblesMessage = replayBlueBubblesMessage;
|
|
|
4
4
|
exports.formatBlueBubblesReplayText = formatBlueBubblesReplayText;
|
|
5
5
|
const render_1 = require("../../heart/attachments/render");
|
|
6
6
|
const bluebubbles_1 = require("../../heart/attachments/sources/bluebubbles");
|
|
7
|
+
const machine_identity_1 = require("../../heart/machine-identity");
|
|
7
8
|
const identity_1 = require("../../heart/identity");
|
|
9
|
+
const runtime_credentials_1 = require("../../heart/runtime-credentials");
|
|
8
10
|
const runtime_1 = require("../../nerves/runtime");
|
|
9
11
|
const client_1 = require("./client");
|
|
10
12
|
const model_1 = require("./model");
|
|
@@ -29,6 +31,8 @@ async function replayBlueBubblesMessage(params, deps = {}) {
|
|
|
29
31
|
const setReplayAgentName = deps.setAgentName ?? identity_1.setAgentName;
|
|
30
32
|
const resetReplayIdentity = deps.resetIdentity ?? identity_1.resetIdentity;
|
|
31
33
|
const normalizeEvent = deps.normalizeEvent ?? model_1.normalizeBlueBubblesEvent;
|
|
34
|
+
const loadMachineId = deps.loadMachineId ?? (() => (0, machine_identity_1.loadOrCreateMachineIdentity)().machineId);
|
|
35
|
+
const refreshMachineRuntimeConfig = deps.refreshMachineRuntimeConfig ?? runtime_credentials_1.refreshMachineRuntimeCredentialConfig;
|
|
32
36
|
(0, runtime_1.emitNervesEvent)({
|
|
33
37
|
component: "senses",
|
|
34
38
|
event: "senses.bluebubbles_replay_start",
|
|
@@ -41,6 +45,10 @@ async function replayBlueBubblesMessage(params, deps = {}) {
|
|
|
41
45
|
});
|
|
42
46
|
setReplayAgentName(agentName);
|
|
43
47
|
try {
|
|
48
|
+
if (!deps.createClient) {
|
|
49
|
+
const machineId = loadMachineId();
|
|
50
|
+
await refreshMachineRuntimeConfig(agentName, machineId, { preserveCachedOnFailure: true });
|
|
51
|
+
}
|
|
44
52
|
const client = deps.createClient ? deps.createClient() : (0, client_1.createBlueBubblesClient)();
|
|
45
53
|
const probe = normalizeEvent({
|
|
46
54
|
type: eventType,
|
|
@@ -77,6 +77,18 @@ function readBlueBubblesRuntimeState(agentName, agentRoot) {
|
|
|
77
77
|
oldestPendingRecoveryAgeMs: typeof parsed.oldestPendingRecoveryAgeMs === "number" && Number.isFinite(parsed.oldestPendingRecoveryAgeMs)
|
|
78
78
|
? parsed.oldestPendingRecoveryAgeMs
|
|
79
79
|
: undefined,
|
|
80
|
+
activeTurnCount: typeof parsed.activeTurnCount === "number" && Number.isFinite(parsed.activeTurnCount)
|
|
81
|
+
? parsed.activeTurnCount
|
|
82
|
+
: undefined,
|
|
83
|
+
stalledTurnCount: typeof parsed.stalledTurnCount === "number" && Number.isFinite(parsed.stalledTurnCount)
|
|
84
|
+
? parsed.stalledTurnCount
|
|
85
|
+
: undefined,
|
|
86
|
+
oldestActiveTurnStartedAt: typeof parsed.oldestActiveTurnStartedAt === "string"
|
|
87
|
+
? parsed.oldestActiveTurnStartedAt
|
|
88
|
+
: undefined,
|
|
89
|
+
oldestActiveTurnAgeMs: typeof parsed.oldestActiveTurnAgeMs === "number" && Number.isFinite(parsed.oldestActiveTurnAgeMs)
|
|
90
|
+
? parsed.oldestActiveTurnAgeMs
|
|
91
|
+
: undefined,
|
|
80
92
|
lastRecoveredAt: typeof parsed.lastRecoveredAt === "string" ? parsed.lastRecoveredAt : undefined,
|
|
81
93
|
lastRecoveredMessageGuid: typeof parsed.lastRecoveredMessageGuid === "string"
|
|
82
94
|
? parsed.lastRecoveredMessageGuid
|
|
@@ -116,6 +128,8 @@ function writeBlueBubblesRuntimeState(agentName, state, agentRoot) {
|
|
|
116
128
|
upstreamStatus: state.upstreamStatus,
|
|
117
129
|
pendingRecoveryCount: state.pendingRecoveryCount,
|
|
118
130
|
failedRecoveryCount: state.failedRecoveryCount ?? 0,
|
|
131
|
+
activeTurnCount: state.activeTurnCount ?? 0,
|
|
132
|
+
stalledTurnCount: state.stalledTurnCount ?? 0,
|
|
119
133
|
path: filePath,
|
|
120
134
|
},
|
|
121
135
|
});
|