@ouro.bot/cli 0.1.0-alpha.525 → 0.1.0-alpha.527
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/changelog.json
CHANGED
|
@@ -1,6 +1,24 @@
|
|
|
1
1
|
{
|
|
2
2
|
"_note": "This changelog is maintained as part of the PR/version-bump workflow. Agent-curated, not auto-generated. Agents read this file directly via read_file to understand what changed between versions.",
|
|
3
3
|
"versions": [
|
|
4
|
+
{
|
|
5
|
+
"version": "0.1.0-alpha.527",
|
|
6
|
+
"changes": [
|
|
7
|
+
"Suppresses `onResult`/`onFailure` in the shared tool-activity callbacks factory for any tool that started hidden, so a hidden tool's END never re-emits its raw args into chat surfaces — fixing rejected `settle` calls leaking `answer=`/`intent=` into BlueBubbles and Teams threads.",
|
|
8
|
+
"Tracks hidden-at-start tools by per-name counter to stay sound across concurrent same-name hidden starts, with no behavior change for visible tools.",
|
|
9
|
+
"Adds heart-level regression tests for hidden-tool END suppression (success and failure paths, concurrent same-name) and senses-level regression tests against `createBlueBubblesCallbacks` and `createTeamsCallbacks` asserting that a rejected settle following a visible read_file produces no chat output containing the settle answer text or `intent=`/`answer=` substrings."
|
|
10
|
+
]
|
|
11
|
+
},
|
|
12
|
+
{
|
|
13
|
+
"version": "0.1.0-alpha.526",
|
|
14
|
+
"changes": [
|
|
15
|
+
"Marks BlueBubbles/iMessage as unhealthy in daemon status whenever fresh runtime state has queued recovery work, even if the BlueBubbles process and upstream health probe are answering.",
|
|
16
|
+
"Keeps the runtime `upstreamStatus` scoped to transport reachability while rendering pending iMessage recovery as a user-facing service failure through `pendingRecoveryCount`.",
|
|
17
|
+
"Requires a running BlueBubbles listener process before fresh healthy runtime state can make daemon status green, and hard-times out queued recovery turns so one stuck message cannot wedge backlog draining.",
|
|
18
|
+
"Quarantines timed-out recovery messages with a `recovery-timeout` processed outcome, allowing later queued iMessage recovery to continue while preserving the audit trail.",
|
|
19
|
+
"Clarifies BlueBubbles runtime detail text so queued recovery reads as iMessage not caught up instead of implying the sense is healthy."
|
|
20
|
+
]
|
|
21
|
+
},
|
|
4
22
|
{
|
|
5
23
|
"version": "0.1.0-alpha.525",
|
|
6
24
|
"changes": [
|
|
@@ -266,12 +266,18 @@ function readBlueBubblesRuntimeJson(runtimePath) {
|
|
|
266
266
|
? parsed.detail
|
|
267
267
|
: "startup health probe pending",
|
|
268
268
|
lastCheckedAt: typeof parsed.lastCheckedAt === "string" ? parsed.lastCheckedAt : undefined,
|
|
269
|
+
pendingRecoveryCount: typeof parsed.pendingRecoveryCount === "number" && Number.isFinite(parsed.pendingRecoveryCount)
|
|
270
|
+
? parsed.pendingRecoveryCount
|
|
271
|
+
: 0,
|
|
272
|
+
failedRecoveryCount: typeof parsed.failedRecoveryCount === "number" && Number.isFinite(parsed.failedRecoveryCount)
|
|
273
|
+
? parsed.failedRecoveryCount
|
|
274
|
+
: 0,
|
|
269
275
|
};
|
|
270
276
|
/* v8 ignore stop */
|
|
271
277
|
/* v8 ignore start -- defensive: catch for missing/corrupt BB runtime state file @preserve */
|
|
272
278
|
}
|
|
273
279
|
catch {
|
|
274
|
-
return { upstreamStatus: "unknown", detail: "startup health probe pending" };
|
|
280
|
+
return { upstreamStatus: "unknown", detail: "startup health probe pending", pendingRecoveryCount: 0, failedRecoveryCount: 0 };
|
|
275
281
|
}
|
|
276
282
|
/* v8 ignore stop */
|
|
277
283
|
}
|
|
@@ -285,14 +291,29 @@ function readBlueBubblesRuntimeFacts(agent, bundlesRoot, snapshot) {
|
|
|
285
291
|
if (!blueBubblesRuntimeStateIsFresh(state.lastCheckedAt)) {
|
|
286
292
|
return { runtime: snapshot?.runtime };
|
|
287
293
|
}
|
|
294
|
+
if (snapshot?.runtime !== "running") {
|
|
295
|
+
return {
|
|
296
|
+
runtime: "error",
|
|
297
|
+
detail: "BlueBubbles listener is not running",
|
|
298
|
+
};
|
|
299
|
+
}
|
|
288
300
|
if (state.upstreamStatus === "error") {
|
|
289
301
|
return {
|
|
290
302
|
runtime: "error",
|
|
291
303
|
detail: state.detail,
|
|
292
304
|
};
|
|
293
305
|
}
|
|
306
|
+
if (state.pendingRecoveryCount > 0) {
|
|
307
|
+
return {
|
|
308
|
+
runtime: "error",
|
|
309
|
+
detail: state.detail,
|
|
310
|
+
};
|
|
311
|
+
}
|
|
294
312
|
if (state.upstreamStatus === "ok") {
|
|
295
|
-
return {
|
|
313
|
+
return {
|
|
314
|
+
runtime: "running",
|
|
315
|
+
...(state.failedRecoveryCount > 0 ? { detail: state.detail } : {}),
|
|
316
|
+
};
|
|
296
317
|
}
|
|
297
318
|
return { runtime: snapshot?.runtime };
|
|
298
319
|
}
|
|
@@ -12,24 +12,47 @@ function createToolActivityCallbacks(options) {
|
|
|
12
12
|
});
|
|
13
13
|
// Track the last description so we can reference it in END messages
|
|
14
14
|
let lastDescription = null;
|
|
15
|
+
// Track in-flight hidden tools so onToolEnd can SYMMETRICALLY suppress
|
|
16
|
+
// emission for the same tools that onToolStart already suppresses.
|
|
17
|
+
// Without this, a rejected hidden tool (e.g. settle blocked by the
|
|
18
|
+
// mustResolveBeforeHandoff gate or the inner-dialog attention-queue gate)
|
|
19
|
+
// would emit "✗ <previous visible tool's description> — <hidden tool's args summary>"
|
|
20
|
+
// because lastDescription persists across calls and the hidden tool's summary
|
|
21
|
+
// (built via summarizeArgs) leaks args like settle's `answer`/`intent` into
|
|
22
|
+
// the visible chat. Counter map (not bool) so concurrent hidden starts don't
|
|
23
|
+
// underflow if ends arrive in any order.
|
|
24
|
+
const hiddenInFlight = new Map();
|
|
15
25
|
return {
|
|
16
26
|
onToolStart(name, args) {
|
|
17
27
|
const description = (0, tool_description_1.humanReadableToolDescription)(name, args);
|
|
18
|
-
if (description === null)
|
|
19
|
-
|
|
28
|
+
if (description === null) {
|
|
29
|
+
// hidden tool (settle, rest, descend, observe, speak) — track so the
|
|
30
|
+
// matching onToolEnd is also suppressed symmetrically.
|
|
31
|
+
hiddenInFlight.set(name, (hiddenInFlight.get(name) ?? 0) + 1);
|
|
32
|
+
return;
|
|
33
|
+
}
|
|
20
34
|
lastDescription = description;
|
|
21
35
|
options.onDescription(description);
|
|
22
36
|
},
|
|
23
37
|
onToolEnd(name, summary, success) {
|
|
38
|
+
const hiddenCount = hiddenInFlight.get(name) ?? 0;
|
|
39
|
+
if (hiddenCount > 0) {
|
|
40
|
+
// Hidden tool's start was suppressed; suppress its end too.
|
|
41
|
+
if (hiddenCount === 1)
|
|
42
|
+
hiddenInFlight.delete(name);
|
|
43
|
+
else
|
|
44
|
+
hiddenInFlight.set(name, hiddenCount - 1);
|
|
45
|
+
return;
|
|
46
|
+
}
|
|
24
47
|
const desc = lastDescription ?? name;
|
|
25
48
|
// Strip trailing "..." from description for the result line
|
|
26
49
|
const cleanDesc = desc.endsWith("...") ? desc.slice(0, -3) : desc;
|
|
27
50
|
if (!success) {
|
|
28
|
-
options.onFailure(
|
|
51
|
+
options.onFailure(`✗ ${cleanDesc} — ${summary}`);
|
|
29
52
|
return;
|
|
30
53
|
}
|
|
31
54
|
if (options.isDebug()) {
|
|
32
|
-
options.onResult(
|
|
55
|
+
options.onResult(`✓ ${cleanDesc}`);
|
|
33
56
|
}
|
|
34
57
|
},
|
|
35
58
|
};
|
|
@@ -41,6 +41,7 @@ exports.getDiscoveredOwnHandles = getDiscoveredOwnHandles;
|
|
|
41
41
|
exports.clearDiscoveredOwnHandles = clearDiscoveredOwnHandles;
|
|
42
42
|
exports.recordDiscoveredOwnHandle = recordDiscoveredOwnHandle;
|
|
43
43
|
exports.handleBlueBubblesEvent = handleBlueBubblesEvent;
|
|
44
|
+
exports.recoverQueuedBlueBubblesMessages = recoverQueuedBlueBubblesMessages;
|
|
44
45
|
exports.catchUpMissedBlueBubblesMessages = catchUpMissedBlueBubblesMessages;
|
|
45
46
|
exports.recoverCapturedBlueBubblesInboundMessages = recoverCapturedBlueBubblesInboundMessages;
|
|
46
47
|
exports.recoverMissedBlueBubblesMessages = recoverMissedBlueBubblesMessages;
|
|
@@ -166,11 +167,24 @@ const defaultDeps = {
|
|
|
166
167
|
getOwnHandles: () => [...(0, config_1.getBlueBubblesConfig)().ownHandles, ...discoveredOwnHandles],
|
|
167
168
|
};
|
|
168
169
|
const BLUEBUBBLES_RUNTIME_SYNC_INTERVAL_MS = 30_000;
|
|
170
|
+
const BLUEBUBBLES_RECOVERY_PASS_DELAY_MS = 1_000;
|
|
171
|
+
const BLUEBUBBLES_RECOVERY_PASS_INTERVAL_MS = 30_000;
|
|
172
|
+
const BLUEBUBBLES_RECOVERY_TURN_TIMEOUT_MS = 60_000;
|
|
169
173
|
const BLUEBUBBLES_CATCHUP_PAGE_SIZE = 50;
|
|
170
174
|
const BLUEBUBBLES_CATCHUP_MAX_PAGES = 20;
|
|
171
175
|
const BLUEBUBBLES_HEALTHY_CATCHUP_OVERLAP_MS = 90_000;
|
|
172
176
|
const BLUEBUBBLES_RECOVERY_CATCHUP_LOOKBACK_MS = 24 * 60 * 60 * 1000;
|
|
173
177
|
const BLUEBUBBLES_FIRST_CATCHUP_LOOKBACK_MS = 10 * 60 * 1000;
|
|
178
|
+
class BlueBubblesRecoveryTurnTimeoutError extends Error {
|
|
179
|
+
constructor(timeoutMs) {
|
|
180
|
+
super(`bluebubbles recovery turn timed out after ${timeoutMs}ms`);
|
|
181
|
+
this.name = "BlueBubblesRecoveryTurnTimeoutError";
|
|
182
|
+
}
|
|
183
|
+
}
|
|
184
|
+
function isBlueBubblesRecoveryTurnTimeoutError(error) {
|
|
185
|
+
return error instanceof BlueBubblesRecoveryTurnTimeoutError
|
|
186
|
+
|| (error instanceof Error && error.name === "BlueBubblesRecoveryTurnTimeoutError");
|
|
187
|
+
}
|
|
174
188
|
function resolveFriendParams(event) {
|
|
175
189
|
if (event.chat.isGroup) {
|
|
176
190
|
const groupKey = event.chat.chatGuid ?? event.chat.chatIdentifier ?? event.sender.externalId;
|
|
@@ -742,7 +756,7 @@ async function shouldFilterAgentSelfHandle(event, resolvedDeps) {
|
|
|
742
756
|
}
|
|
743
757
|
return true;
|
|
744
758
|
}
|
|
745
|
-
async function handleBlueBubblesNormalizedEvent(event, resolvedDeps, source) {
|
|
759
|
+
async function handleBlueBubblesNormalizedEvent(event, resolvedDeps, source, options = {}) {
|
|
746
760
|
const client = resolvedDeps.createClient();
|
|
747
761
|
const agentName = resolvedDeps.getAgentName();
|
|
748
762
|
if (event.fromMe) {
|
|
@@ -933,6 +947,10 @@ async function handleBlueBubblesNormalizedEvent(event, resolvedDeps, source) {
|
|
|
933
947
|
};
|
|
934
948
|
const callbacks = createBlueBubblesCallbacks(client, event.chat, replyTarget, event.chat.isGroup);
|
|
935
949
|
const controller = new AbortController();
|
|
950
|
+
let timeoutTimer = null;
|
|
951
|
+
let timeoutPromise = null;
|
|
952
|
+
let timeoutReject;
|
|
953
|
+
let recoveryTimedOut = false;
|
|
936
954
|
// BB-specific tool context wrappers
|
|
937
955
|
const summarize = (0, core_1.createSummarize)("human");
|
|
938
956
|
const bbCapabilities = (0, channel_1.getChannelCapabilities)("bluebubbles");
|
|
@@ -960,7 +978,35 @@ async function handleBlueBubblesNormalizedEvent(event, resolvedDeps, source) {
|
|
|
960
978
|
};
|
|
961
979
|
/* v8 ignore stop */
|
|
962
980
|
try {
|
|
963
|
-
const
|
|
981
|
+
const timeoutMs = options.timeoutMs;
|
|
982
|
+
if (typeof timeoutMs === "number" && Number.isFinite(timeoutMs) && timeoutMs > 0) {
|
|
983
|
+
timeoutPromise = new Promise((_, reject) => {
|
|
984
|
+
timeoutReject = reject;
|
|
985
|
+
});
|
|
986
|
+
timeoutTimer = setTimeout(() => {
|
|
987
|
+
const reason = new BlueBubblesRecoveryTurnTimeoutError(timeoutMs);
|
|
988
|
+
recoveryTimedOut = true;
|
|
989
|
+
controller.abort(reason);
|
|
990
|
+
timeoutReject?.(reason);
|
|
991
|
+
(0, runtime_1.emitNervesEvent)({
|
|
992
|
+
level: "warn",
|
|
993
|
+
component: "senses",
|
|
994
|
+
event: "senses.bluebubbles_turn_timeout",
|
|
995
|
+
message: "bluebubbles recovery turn timed out",
|
|
996
|
+
meta: {
|
|
997
|
+
messageGuid: event.messageGuid,
|
|
998
|
+
sessionKey: event.chat.sessionKey,
|
|
999
|
+
source,
|
|
1000
|
+
timeoutMs,
|
|
1001
|
+
},
|
|
1002
|
+
});
|
|
1003
|
+
}, timeoutMs);
|
|
1004
|
+
/* v8 ignore next -- timer handles expose unref only in some runtimes @preserve */
|
|
1005
|
+
if (typeof timeoutTimer.unref === "function") {
|
|
1006
|
+
timeoutTimer.unref();
|
|
1007
|
+
}
|
|
1008
|
+
}
|
|
1009
|
+
const turnPromise = (0, pipeline_1.handleInboundTurn)({
|
|
964
1010
|
channel: "bluebubbles",
|
|
965
1011
|
sessionKey: event.chat.sessionKey,
|
|
966
1012
|
capabilities: bbCapabilities,
|
|
@@ -1021,6 +1067,29 @@ async function handleBlueBubblesNormalizedEvent(event, resolvedDeps, source) {
|
|
|
1021
1067
|
return bbFailoverStates.get(event.chat.sessionKey);
|
|
1022
1068
|
})(),
|
|
1023
1069
|
});
|
|
1070
|
+
/* v8 ignore start -- detached late-rejection telemetry is asserted in timeout tests, but V8 does not reliably attribute Promise.catch callbacks @preserve */
|
|
1071
|
+
if (timeoutPromise) {
|
|
1072
|
+
void turnPromise.catch((error) => {
|
|
1073
|
+
if (!recoveryTimedOut)
|
|
1074
|
+
return;
|
|
1075
|
+
(0, runtime_1.emitNervesEvent)({
|
|
1076
|
+
level: "warn",
|
|
1077
|
+
component: "senses",
|
|
1078
|
+
event: "senses.bluebubbles_recovery_error",
|
|
1079
|
+
message: "bluebubbles recovery turn rejected after timeout",
|
|
1080
|
+
meta: {
|
|
1081
|
+
messageGuid: event.messageGuid,
|
|
1082
|
+
sessionKey: event.chat.sessionKey,
|
|
1083
|
+
source,
|
|
1084
|
+
reason: error instanceof Error ? error.message : String(error),
|
|
1085
|
+
},
|
|
1086
|
+
});
|
|
1087
|
+
});
|
|
1088
|
+
}
|
|
1089
|
+
/* v8 ignore stop */
|
|
1090
|
+
const result = timeoutPromise
|
|
1091
|
+
? await Promise.race([turnPromise, timeoutPromise])
|
|
1092
|
+
: await turnPromise;
|
|
1024
1093
|
/* v8 ignore start -- failover display + error replay @preserve */
|
|
1025
1094
|
if (result.failoverMessage) {
|
|
1026
1095
|
// Failover handled it — show the failover message, skip the buffered error
|
|
@@ -1079,6 +1148,10 @@ async function handleBlueBubblesNormalizedEvent(event, resolvedDeps, source) {
|
|
|
1079
1148
|
bufferedTerminalError = null;
|
|
1080
1149
|
}
|
|
1081
1150
|
/* v8 ignore stop */
|
|
1151
|
+
if (timeoutTimer !== null) {
|
|
1152
|
+
clearTimeout(timeoutTimer);
|
|
1153
|
+
timeoutTimer = null;
|
|
1154
|
+
}
|
|
1082
1155
|
await callbacks.finish();
|
|
1083
1156
|
}
|
|
1084
1157
|
});
|
|
@@ -1193,6 +1266,13 @@ function resolveBlueBubblesCatchUpSince(previousState, nowMs = Date.now()) {
|
|
|
1193
1266
|
}
|
|
1194
1267
|
return nowMs - BLUEBUBBLES_FIRST_CATCHUP_LOOKBACK_MS;
|
|
1195
1268
|
}
|
|
1269
|
+
function formatBlueBubblesRuntimeDetail(queued, failed) {
|
|
1270
|
+
if (queued > 0)
|
|
1271
|
+
return `upstream reachable but iMessage is not caught up; ${queued} recovery item(s) queued`;
|
|
1272
|
+
if (failed > 0)
|
|
1273
|
+
return `${failed} message(s) unrecoverable this cycle; upstream ok`;
|
|
1274
|
+
return "upstream reachable";
|
|
1275
|
+
}
|
|
1196
1276
|
async function syncBlueBubblesRuntime(deps = {}) {
|
|
1197
1277
|
const resolvedDeps = { ...defaultDeps, ...deps };
|
|
1198
1278
|
const agentName = resolvedDeps.getAgentName();
|
|
@@ -1216,21 +1296,16 @@ async function syncBlueBubblesRuntime(deps = {}) {
|
|
|
1216
1296
|
});
|
|
1217
1297
|
const failed = catchUp.failed;
|
|
1218
1298
|
const queued = capturedPending + recoveryPending + (catchUp.queued ?? 0);
|
|
1219
|
-
// upstreamStatus reflects whether BlueBubbles itself
|
|
1220
|
-
//
|
|
1221
|
-
//
|
|
1222
|
-
//
|
|
1223
|
-
// would otherwise stick the sense in "error" forever, contradicting `ouro
|
|
1224
|
-
// doctor` which only checks upstream reachability.
|
|
1299
|
+
// upstreamStatus reflects whether BlueBubbles itself and the local bridge
|
|
1300
|
+
// can answer webhook traffic. The daemon status layer treats
|
|
1301
|
+
// pendingRecoveryCount as unhealthy for user-facing iMessage reachability,
|
|
1302
|
+
// while this field stays scoped to upstream transport reachability.
|
|
1225
1303
|
(0, runtime_state_1.writeBlueBubblesRuntimeState)(agentName, {
|
|
1226
1304
|
upstreamStatus: "ok",
|
|
1227
|
-
detail: queued
|
|
1228
|
-
? `upstream reachable; ${queued} recovery item(s) queued`
|
|
1229
|
-
: failed > 0
|
|
1230
|
-
? `${failed} message(s) unrecoverable this cycle; upstream ok`
|
|
1231
|
-
: "upstream reachable",
|
|
1305
|
+
detail: formatBlueBubblesRuntimeDetail(queued, failed),
|
|
1232
1306
|
lastCheckedAt: checkedAt,
|
|
1233
1307
|
pendingRecoveryCount: queued,
|
|
1308
|
+
failedRecoveryCount: failed,
|
|
1234
1309
|
lastRecoveredAt: previousState.lastRecoveredAt,
|
|
1235
1310
|
lastRecoveredMessageGuid: previousState.lastRecoveredMessageGuid,
|
|
1236
1311
|
});
|
|
@@ -1241,8 +1316,49 @@ async function syncBlueBubblesRuntime(deps = {}) {
|
|
|
1241
1316
|
detail: error instanceof Error ? error.message : String(error),
|
|
1242
1317
|
lastCheckedAt: checkedAt,
|
|
1243
1318
|
pendingRecoveryCount: countPendingCapturedInboundMessages(agentName) + countPendingRecoveryCandidates(agentName),
|
|
1319
|
+
failedRecoveryCount: 0,
|
|
1320
|
+
});
|
|
1321
|
+
}
|
|
1322
|
+
}
|
|
1323
|
+
async function recoverQueuedBlueBubblesMessages(deps = {}) {
|
|
1324
|
+
const resolvedDeps = { ...defaultDeps, ...deps };
|
|
1325
|
+
const agentName = resolvedDeps.getAgentName();
|
|
1326
|
+
const previousState = (0, runtime_state_1.readBlueBubblesRuntimeState)(agentName);
|
|
1327
|
+
const initialPending = countPendingCapturedInboundMessages(agentName) + countPendingRecoveryCandidates(agentName);
|
|
1328
|
+
if (initialPending === 0) {
|
|
1329
|
+
return { recovered: 0, skipped: 0, failed: 0, pendingRecoveryCount: 0 };
|
|
1330
|
+
}
|
|
1331
|
+
const captured = await recoverCapturedBlueBubblesInboundMessages(resolvedDeps);
|
|
1332
|
+
const recovery = await recoverMissedBlueBubblesMessages(resolvedDeps);
|
|
1333
|
+
const pendingRecoveryCount = countPendingCapturedInboundMessages(agentName) + countPendingRecoveryCandidates(agentName);
|
|
1334
|
+
const failed = captured.failed + recovery.failed;
|
|
1335
|
+
const recovered = captured.recovered + recovery.recovered;
|
|
1336
|
+
const skipped = captured.skipped + recovery.skipped;
|
|
1337
|
+
const checkedAt = new Date().toISOString();
|
|
1338
|
+
try {
|
|
1339
|
+
await resolvedDeps.createClient().checkHealth();
|
|
1340
|
+
(0, runtime_state_1.writeBlueBubblesRuntimeState)(agentName, {
|
|
1341
|
+
upstreamStatus: "ok",
|
|
1342
|
+
detail: formatBlueBubblesRuntimeDetail(pendingRecoveryCount, failed),
|
|
1343
|
+
lastCheckedAt: checkedAt,
|
|
1344
|
+
pendingRecoveryCount,
|
|
1345
|
+
failedRecoveryCount: failed,
|
|
1346
|
+
lastRecoveredAt: recovered > 0 ? checkedAt : previousState.lastRecoveredAt,
|
|
1347
|
+
lastRecoveredMessageGuid: previousState.lastRecoveredMessageGuid,
|
|
1348
|
+
});
|
|
1349
|
+
}
|
|
1350
|
+
catch (error) {
|
|
1351
|
+
(0, runtime_state_1.writeBlueBubblesRuntimeState)(agentName, {
|
|
1352
|
+
upstreamStatus: "error",
|
|
1353
|
+
detail: error instanceof Error ? error.message : String(error),
|
|
1354
|
+
lastCheckedAt: checkedAt,
|
|
1355
|
+
pendingRecoveryCount,
|
|
1356
|
+
failedRecoveryCount: failed,
|
|
1357
|
+
lastRecoveredAt: recovered > 0 ? checkedAt : previousState.lastRecoveredAt,
|
|
1358
|
+
lastRecoveredMessageGuid: previousState.lastRecoveredMessageGuid,
|
|
1244
1359
|
});
|
|
1245
1360
|
}
|
|
1361
|
+
return { recovered, skipped, failed, pendingRecoveryCount };
|
|
1246
1362
|
}
|
|
1247
1363
|
async function catchUpMissedBlueBubblesMessages(deps = {}, previousState, options = {}) {
|
|
1248
1364
|
const resolvedDeps = { ...defaultDeps, ...deps };
|
|
@@ -1338,13 +1454,17 @@ async function catchUpMissedBlueBubblesMessages(deps = {}, previousState, option
|
|
|
1338
1454
|
result.queued = (result.queued ?? 0) + 1;
|
|
1339
1455
|
continue;
|
|
1340
1456
|
}
|
|
1457
|
+
let repairedMessage = null;
|
|
1341
1458
|
try {
|
|
1342
1459
|
const repaired = await client.repairEvent(event);
|
|
1343
1460
|
if (repaired.kind !== "message") {
|
|
1344
1461
|
result.skipped++;
|
|
1345
1462
|
continue;
|
|
1346
1463
|
}
|
|
1347
|
-
|
|
1464
|
+
repairedMessage = repaired;
|
|
1465
|
+
const handled = await handleBlueBubblesNormalizedEvent(repaired, resolvedDeps, "upstream-catchup", {
|
|
1466
|
+
timeoutMs: BLUEBUBBLES_RECOVERY_TURN_TIMEOUT_MS,
|
|
1467
|
+
});
|
|
1348
1468
|
if (handled.reason === "already_processed") {
|
|
1349
1469
|
result.skipped++;
|
|
1350
1470
|
}
|
|
@@ -1355,6 +1475,9 @@ async function catchUpMissedBlueBubblesMessages(deps = {}, previousState, option
|
|
|
1355
1475
|
}
|
|
1356
1476
|
catch (error) {
|
|
1357
1477
|
result.failed++;
|
|
1478
|
+
if (repairedMessage && isBlueBubblesRecoveryTurnTimeoutError(error)) {
|
|
1479
|
+
(0, processed_log_1.recordProcessedBlueBubblesMessage)(agentName, repairedMessage, "upstream-catchup", "recovery-timeout");
|
|
1480
|
+
}
|
|
1358
1481
|
(0, runtime_1.emitNervesEvent)({
|
|
1359
1482
|
level: "warn",
|
|
1360
1483
|
component: "senses",
|
|
@@ -1431,13 +1554,17 @@ async function recoverCapturedBlueBubblesInboundMessages(deps = {}) {
|
|
|
1431
1554
|
result.skipped++;
|
|
1432
1555
|
continue;
|
|
1433
1556
|
}
|
|
1557
|
+
let repairedMessage = null;
|
|
1434
1558
|
try {
|
|
1435
1559
|
const repaired = await client.repairEvent(inboundEntryToRecoveryEvent(entry));
|
|
1436
1560
|
if (repaired.kind !== "message") {
|
|
1437
1561
|
result.skipped++;
|
|
1438
1562
|
continue;
|
|
1439
1563
|
}
|
|
1440
|
-
|
|
1564
|
+
repairedMessage = repaired;
|
|
1565
|
+
const handled = await handleBlueBubblesNormalizedEvent(repaired, resolvedDeps, entry.source, {
|
|
1566
|
+
timeoutMs: BLUEBUBBLES_RECOVERY_TURN_TIMEOUT_MS,
|
|
1567
|
+
});
|
|
1441
1568
|
if (handled.reason === "already_processed") {
|
|
1442
1569
|
result.skipped++;
|
|
1443
1570
|
}
|
|
@@ -1447,6 +1574,9 @@ async function recoverCapturedBlueBubblesInboundMessages(deps = {}) {
|
|
|
1447
1574
|
}
|
|
1448
1575
|
catch (error) {
|
|
1449
1576
|
result.failed++;
|
|
1577
|
+
if (repairedMessage && isBlueBubblesRecoveryTurnTimeoutError(error)) {
|
|
1578
|
+
(0, processed_log_1.recordProcessedBlueBubblesMessage)(agentName, repairedMessage, entry.source, "recovery-timeout");
|
|
1579
|
+
}
|
|
1450
1580
|
(0, runtime_1.emitNervesEvent)({
|
|
1451
1581
|
level: "warn",
|
|
1452
1582
|
component: "senses",
|
|
@@ -1473,13 +1603,17 @@ async function recoverMissedBlueBubblesMessages(deps = {}) {
|
|
|
1473
1603
|
result.skipped++;
|
|
1474
1604
|
continue;
|
|
1475
1605
|
}
|
|
1606
|
+
let repairedMessage = null;
|
|
1476
1607
|
try {
|
|
1477
1608
|
const repaired = await client.repairEvent(mutationEntryToEvent(candidate));
|
|
1478
1609
|
if (repaired.kind !== "message") {
|
|
1479
1610
|
result.pending++;
|
|
1480
1611
|
continue;
|
|
1481
1612
|
}
|
|
1482
|
-
|
|
1613
|
+
repairedMessage = repaired;
|
|
1614
|
+
const handled = await handleBlueBubblesNormalizedEvent(repaired, resolvedDeps, "mutation-recovery", {
|
|
1615
|
+
timeoutMs: BLUEBUBBLES_RECOVERY_TURN_TIMEOUT_MS,
|
|
1616
|
+
});
|
|
1483
1617
|
if (handled.reason === "already_processed") {
|
|
1484
1618
|
result.skipped++;
|
|
1485
1619
|
}
|
|
@@ -1489,6 +1623,9 @@ async function recoverMissedBlueBubblesMessages(deps = {}) {
|
|
|
1489
1623
|
}
|
|
1490
1624
|
catch (error) {
|
|
1491
1625
|
result.failed++;
|
|
1626
|
+
if (repairedMessage && isBlueBubblesRecoveryTurnTimeoutError(error)) {
|
|
1627
|
+
(0, processed_log_1.recordProcessedBlueBubblesMessage)(agentName, repairedMessage, "mutation-recovery", "recovery-timeout");
|
|
1628
|
+
}
|
|
1492
1629
|
(0, runtime_1.emitNervesEvent)({
|
|
1493
1630
|
level: "warn",
|
|
1494
1631
|
component: "senses",
|
|
@@ -1955,11 +2092,49 @@ function startBlueBubblesApp(deps = {}) {
|
|
|
1955
2092
|
resolvedDeps.createClient();
|
|
1956
2093
|
const channelConfig = (0, config_1.getBlueBubblesChannelConfig)();
|
|
1957
2094
|
const server = resolvedDeps.createServer(createBlueBubblesWebhookHandler(deps));
|
|
2095
|
+
let recoveryPassRunning = false;
|
|
2096
|
+
let recoveryDelayTimer = null;
|
|
2097
|
+
function triggerRecoveryPass() {
|
|
2098
|
+
/* v8 ignore next -- re-entrant timer guard; difficult to force deterministically without timing the turn lock @preserve */
|
|
2099
|
+
if (recoveryPassRunning)
|
|
2100
|
+
return;
|
|
2101
|
+
recoveryPassRunning = true;
|
|
2102
|
+
void recoverQueuedBlueBubblesMessages(resolvedDeps)
|
|
2103
|
+
/* v8 ignore start -- defensive wrapper; expected per-message failures are handled inside recovery helpers @preserve */
|
|
2104
|
+
.catch((error) => {
|
|
2105
|
+
(0, runtime_1.emitNervesEvent)({
|
|
2106
|
+
level: "warn",
|
|
2107
|
+
component: "senses",
|
|
2108
|
+
event: "senses.bluebubbles_recovery_error",
|
|
2109
|
+
message: "bluebubbles queued recovery pass failed",
|
|
2110
|
+
meta: { reason: error instanceof Error ? error.message : String(error) },
|
|
2111
|
+
});
|
|
2112
|
+
})
|
|
2113
|
+
/* v8 ignore stop */
|
|
2114
|
+
.finally(() => {
|
|
2115
|
+
recoveryPassRunning = false;
|
|
2116
|
+
});
|
|
2117
|
+
}
|
|
2118
|
+
function scheduleRecoveryPass() {
|
|
2119
|
+
/* v8 ignore next -- duplicate scheduling guard for overlapping health sync completions @preserve */
|
|
2120
|
+
if (recoveryDelayTimer !== null)
|
|
2121
|
+
return;
|
|
2122
|
+
recoveryDelayTimer = setTimeout(() => {
|
|
2123
|
+
recoveryDelayTimer = null;
|
|
2124
|
+
triggerRecoveryPass();
|
|
2125
|
+
}, BLUEBUBBLES_RECOVERY_PASS_DELAY_MS);
|
|
2126
|
+
}
|
|
1958
2127
|
const runtimeTimer = setInterval(() => {
|
|
1959
|
-
void syncBlueBubblesRuntime(resolvedDeps);
|
|
2128
|
+
void syncBlueBubblesRuntime(resolvedDeps).then(scheduleRecoveryPass);
|
|
1960
2129
|
}, BLUEBUBBLES_RUNTIME_SYNC_INTERVAL_MS);
|
|
2130
|
+
const recoveryTimer = setInterval(triggerRecoveryPass, BLUEBUBBLES_RECOVERY_PASS_INTERVAL_MS);
|
|
1961
2131
|
server.on?.("close", () => {
|
|
1962
2132
|
clearInterval(runtimeTimer);
|
|
2133
|
+
clearInterval(recoveryTimer);
|
|
2134
|
+
if (recoveryDelayTimer !== null) {
|
|
2135
|
+
clearTimeout(recoveryDelayTimer);
|
|
2136
|
+
recoveryDelayTimer = null;
|
|
2137
|
+
}
|
|
1963
2138
|
});
|
|
1964
2139
|
server.listen(channelConfig.port, () => {
|
|
1965
2140
|
(0, runtime_1.emitNervesEvent)({
|
|
@@ -1969,6 +2144,6 @@ function startBlueBubblesApp(deps = {}) {
|
|
|
1969
2144
|
meta: { port: channelConfig.port, webhookPath: channelConfig.webhookPath },
|
|
1970
2145
|
});
|
|
1971
2146
|
});
|
|
1972
|
-
void syncBlueBubblesRuntime(resolvedDeps);
|
|
2147
|
+
void syncBlueBubblesRuntime(resolvedDeps).then(scheduleRecoveryPass);
|
|
1973
2148
|
return server;
|
|
1974
2149
|
}
|
|
@@ -53,6 +53,9 @@ function readBlueBubblesRuntimeState(agentName, agentRoot) {
|
|
|
53
53
|
try {
|
|
54
54
|
const raw = fs.readFileSync(filePath, "utf-8");
|
|
55
55
|
const parsed = JSON.parse(raw);
|
|
56
|
+
const failedRecoveryCount = typeof parsed.failedRecoveryCount === "number" && Number.isFinite(parsed.failedRecoveryCount)
|
|
57
|
+
? parsed.failedRecoveryCount
|
|
58
|
+
: undefined;
|
|
56
59
|
return {
|
|
57
60
|
upstreamStatus: parsed.upstreamStatus === "ok" || parsed.upstreamStatus === "error"
|
|
58
61
|
? parsed.upstreamStatus
|
|
@@ -64,6 +67,7 @@ function readBlueBubblesRuntimeState(agentName, agentRoot) {
|
|
|
64
67
|
pendingRecoveryCount: typeof parsed.pendingRecoveryCount === "number" && Number.isFinite(parsed.pendingRecoveryCount)
|
|
65
68
|
? parsed.pendingRecoveryCount
|
|
66
69
|
: 0,
|
|
70
|
+
...(typeof failedRecoveryCount === "number" ? { failedRecoveryCount } : {}),
|
|
67
71
|
lastRecoveredAt: typeof parsed.lastRecoveredAt === "string" ? parsed.lastRecoveredAt : undefined,
|
|
68
72
|
lastRecoveredMessageGuid: typeof parsed.lastRecoveredMessageGuid === "string"
|
|
69
73
|
? parsed.lastRecoveredMessageGuid
|
|
@@ -102,6 +106,7 @@ function writeBlueBubblesRuntimeState(agentName, state, agentRoot) {
|
|
|
102
106
|
agentName,
|
|
103
107
|
upstreamStatus: state.upstreamStatus,
|
|
104
108
|
pendingRecoveryCount: state.pendingRecoveryCount,
|
|
109
|
+
failedRecoveryCount: state.failedRecoveryCount ?? 0,
|
|
105
110
|
path: filePath,
|
|
106
111
|
},
|
|
107
112
|
});
|