@ouro.bot/cli 0.1.0-alpha.525 → 0.1.0-alpha.526

package/changelog.json

@@ -1,6 +1,16 @@
 {
   "_note": "This changelog is maintained as part of the PR/version-bump workflow. Agent-curated, not auto-generated. Agents read this file directly via read_file to understand what changed between versions.",
   "versions": [
+    {
+      "version": "0.1.0-alpha.526",
+      "changes": [
+        "Marks BlueBubbles/iMessage as unhealthy in daemon status whenever fresh runtime state has queued recovery work, even if the BlueBubbles process and upstream health probe are answering.",
+        "Keeps the runtime `upstreamStatus` scoped to transport reachability while rendering pending iMessage recovery as a user-facing service failure through `pendingRecoveryCount`.",
+        "Requires a running BlueBubbles listener process before fresh healthy runtime state can make daemon status green, and hard-times out queued recovery turns so one stuck message cannot wedge backlog draining.",
+        "Quarantines timed-out recovery messages with a `recovery-timeout` processed outcome, allowing later queued iMessage recovery to continue while preserving the audit trail.",
+        "Clarifies BlueBubbles runtime detail text so queued recovery reads as iMessage not caught up instead of implying the sense is healthy."
+      ]
+    },
     {
       "version": "0.1.0-alpha.525",
       "changes": [

package/dist/heart/daemon/sense-manager.js

@@ -266,12 +266,18 @@ function readBlueBubblesRuntimeJson(runtimePath) {
                 ? parsed.detail
                 : "startup health probe pending",
             lastCheckedAt: typeof parsed.lastCheckedAt === "string" ? parsed.lastCheckedAt : undefined,
+            pendingRecoveryCount: typeof parsed.pendingRecoveryCount === "number" && Number.isFinite(parsed.pendingRecoveryCount)
+                ? parsed.pendingRecoveryCount
+                : 0,
+            failedRecoveryCount: typeof parsed.failedRecoveryCount === "number" && Number.isFinite(parsed.failedRecoveryCount)
+                ? parsed.failedRecoveryCount
+                : 0,
         };
         /* v8 ignore stop */
         /* v8 ignore start -- defensive: catch for missing/corrupt BB runtime state file @preserve */
     }
     catch {
-        return { upstreamStatus: "unknown", detail: "startup health probe pending" };
+        return { upstreamStatus: "unknown", detail: "startup health probe pending", pendingRecoveryCount: 0, failedRecoveryCount: 0 };
     }
     /* v8 ignore stop */
 }
@@ -285,14 +291,29 @@ function readBlueBubblesRuntimeFacts(agent, bundlesRoot, snapshot) {
     if (!blueBubblesRuntimeStateIsFresh(state.lastCheckedAt)) {
         return { runtime: snapshot?.runtime };
     }
+    if (snapshot?.runtime !== "running") {
+        return {
+            runtime: "error",
+            detail: "BlueBubbles listener is not running",
+        };
+    }
     if (state.upstreamStatus === "error") {
         return {
             runtime: "error",
             detail: state.detail,
         };
     }
+    if (state.pendingRecoveryCount > 0) {
+        return {
+            runtime: "error",
+            detail: state.detail,
+        };
+    }
     if (state.upstreamStatus === "ok") {
-        return { runtime: "running" };
+        return {
+            runtime: "running",
+            ...(state.failedRecoveryCount > 0 ? { detail: state.detail } : {}),
+        };
     }
     return { runtime: snapshot?.runtime };
 }

package/dist/senses/bluebubbles/index.js

@@ -41,6 +41,7 @@ exports.getDiscoveredOwnHandles = getDiscoveredOwnHandles;
 exports.clearDiscoveredOwnHandles = clearDiscoveredOwnHandles;
 exports.recordDiscoveredOwnHandle = recordDiscoveredOwnHandle;
 exports.handleBlueBubblesEvent = handleBlueBubblesEvent;
+exports.recoverQueuedBlueBubblesMessages = recoverQueuedBlueBubblesMessages;
 exports.catchUpMissedBlueBubblesMessages = catchUpMissedBlueBubblesMessages;
 exports.recoverCapturedBlueBubblesInboundMessages = recoverCapturedBlueBubblesInboundMessages;
 exports.recoverMissedBlueBubblesMessages = recoverMissedBlueBubblesMessages;
@@ -166,11 +167,24 @@ const defaultDeps = {
     getOwnHandles: () => [...(0, config_1.getBlueBubblesConfig)().ownHandles, ...discoveredOwnHandles],
 };
 const BLUEBUBBLES_RUNTIME_SYNC_INTERVAL_MS = 30_000;
+const BLUEBUBBLES_RECOVERY_PASS_DELAY_MS = 1_000;
+const BLUEBUBBLES_RECOVERY_PASS_INTERVAL_MS = 30_000;
+const BLUEBUBBLES_RECOVERY_TURN_TIMEOUT_MS = 60_000;
 const BLUEBUBBLES_CATCHUP_PAGE_SIZE = 50;
 const BLUEBUBBLES_CATCHUP_MAX_PAGES = 20;
 const BLUEBUBBLES_HEALTHY_CATCHUP_OVERLAP_MS = 90_000;
 const BLUEBUBBLES_RECOVERY_CATCHUP_LOOKBACK_MS = 24 * 60 * 60 * 1000;
 const BLUEBUBBLES_FIRST_CATCHUP_LOOKBACK_MS = 10 * 60 * 1000;
+class BlueBubblesRecoveryTurnTimeoutError extends Error {
+    constructor(timeoutMs) {
+        super(`bluebubbles recovery turn timed out after ${timeoutMs}ms`);
+        this.name = "BlueBubblesRecoveryTurnTimeoutError";
+    }
+}
+function isBlueBubblesRecoveryTurnTimeoutError(error) {
+    return error instanceof BlueBubblesRecoveryTurnTimeoutError
+        || (error instanceof Error && error.name === "BlueBubblesRecoveryTurnTimeoutError");
+}
 function resolveFriendParams(event) {
     if (event.chat.isGroup) {
         const groupKey = event.chat.chatGuid ?? event.chat.chatIdentifier ?? event.sender.externalId;
@@ -742,7 +756,7 @@ async function shouldFilterAgentSelfHandle(event, resolvedDeps) {
     }
     return true;
 }
-async function handleBlueBubblesNormalizedEvent(event, resolvedDeps, source) {
+async function handleBlueBubblesNormalizedEvent(event, resolvedDeps, source, options = {}) {
     const client = resolvedDeps.createClient();
     const agentName = resolvedDeps.getAgentName();
     if (event.fromMe) {
@@ -933,6 +947,10 @@ async function handleBlueBubblesNormalizedEvent(event, resolvedDeps, source) {
             };
             const callbacks = createBlueBubblesCallbacks(client, event.chat, replyTarget, event.chat.isGroup);
             const controller = new AbortController();
+            let timeoutTimer = null;
+            let timeoutPromise = null;
+            let timeoutReject;
+            let recoveryTimedOut = false;
             // BB-specific tool context wrappers
             const summarize = (0, core_1.createSummarize)("human");
             const bbCapabilities = (0, channel_1.getChannelCapabilities)("bluebubbles");
@@ -960,7 +978,35 @@ async function handleBlueBubblesNormalizedEvent(event, resolvedDeps, source) {
             };
             /* v8 ignore stop */
             try {
-                const result = await (0, pipeline_1.handleInboundTurn)({
+                const timeoutMs = options.timeoutMs;
+                if (typeof timeoutMs === "number" && Number.isFinite(timeoutMs) && timeoutMs > 0) {
+                    timeoutPromise = new Promise((_, reject) => {
+                        timeoutReject = reject;
+                    });
+                    timeoutTimer = setTimeout(() => {
+                        const reason = new BlueBubblesRecoveryTurnTimeoutError(timeoutMs);
+                        recoveryTimedOut = true;
+                        controller.abort(reason);
+                        timeoutReject?.(reason);
+                        (0, runtime_1.emitNervesEvent)({
+                            level: "warn",
+                            component: "senses",
+                            event: "senses.bluebubbles_turn_timeout",
+                            message: "bluebubbles recovery turn timed out",
+                            meta: {
+                                messageGuid: event.messageGuid,
+                                sessionKey: event.chat.sessionKey,
+                                source,
+                                timeoutMs,
+                            },
+                        });
+                    }, timeoutMs);
+                    /* v8 ignore next -- timer handles expose unref only in some runtimes @preserve */
+                    if (typeof timeoutTimer.unref === "function") {
+                        timeoutTimer.unref();
+                    }
+                }
+                const turnPromise = (0, pipeline_1.handleInboundTurn)({
                     channel: "bluebubbles",
                     sessionKey: event.chat.sessionKey,
                     capabilities: bbCapabilities,
@@ -1021,6 +1067,29 @@ async function handleBlueBubblesNormalizedEvent(event, resolvedDeps, source) {
                         return bbFailoverStates.get(event.chat.sessionKey);
                     })(),
                 });
+                /* v8 ignore start -- detached late-rejection telemetry is asserted in timeout tests, but V8 does not reliably attribute Promise.catch callbacks @preserve */
+                if (timeoutPromise) {
+                    void turnPromise.catch((error) => {
+                        if (!recoveryTimedOut)
+                            return;
+                        (0, runtime_1.emitNervesEvent)({
+                            level: "warn",
+                            component: "senses",
+                            event: "senses.bluebubbles_recovery_error",
+                            message: "bluebubbles recovery turn rejected after timeout",
+                            meta: {
+                                messageGuid: event.messageGuid,
+                                sessionKey: event.chat.sessionKey,
+                                source,
+                                reason: error instanceof Error ? error.message : String(error),
+                            },
+                        });
+                    });
+                }
+                /* v8 ignore stop */
+                const result = timeoutPromise
+                    ? await Promise.race([turnPromise, timeoutPromise])
+                    : await turnPromise;
                 /* v8 ignore start -- failover display + error replay @preserve */
                 if (result.failoverMessage) {
                     // Failover handled it — show the failover message, skip the buffered error
@@ -1079,6 +1148,10 @@ async function handleBlueBubblesNormalizedEvent(event, resolvedDeps, source) {
                     bufferedTerminalError = null;
                 }
                 /* v8 ignore stop */
+                if (timeoutTimer !== null) {
+                    clearTimeout(timeoutTimer);
+                    timeoutTimer = null;
+                }
                 await callbacks.finish();
             }
         });
@@ -1193,6 +1266,13 @@ function resolveBlueBubblesCatchUpSince(previousState, nowMs = Date.now()) {
     }
     return nowMs - BLUEBUBBLES_FIRST_CATCHUP_LOOKBACK_MS;
 }
+function formatBlueBubblesRuntimeDetail(queued, failed) {
+    if (queued > 0)
+        return `upstream reachable but iMessage is not caught up; ${queued} recovery item(s) queued`;
+    if (failed > 0)
+        return `${failed} message(s) unrecoverable this cycle; upstream ok`;
+    return "upstream reachable";
+}
 async function syncBlueBubblesRuntime(deps = {}) {
     const resolvedDeps = { ...defaultDeps, ...deps };
     const agentName = resolvedDeps.getAgentName();
@@ -1216,21 +1296,16 @@ async function syncBlueBubblesRuntime(deps = {}) {
         });
         const failed = catchUp.failed;
         const queued = capturedPending + recoveryPending + (catchUp.queued ?? 0);
-        // upstreamStatus reflects whether BlueBubbles itself is healthy and
-        // whether the local bridge can answer webhook traffic. Queued recovery work
-        // and per-cycle failures are noted in `detail` for transparency but do NOT
-        // flip the status to error: a single permanently-unrecoverable message
-        // would otherwise stick the sense in "error" forever, contradicting `ouro
-        // doctor` which only checks upstream reachability.
+        // upstreamStatus reflects whether BlueBubbles itself and the local bridge
+        // can answer webhook traffic. The daemon status layer treats
+        // pendingRecoveryCount as unhealthy for user-facing iMessage reachability,
+        // while this field stays scoped to upstream transport reachability.
         (0, runtime_state_1.writeBlueBubblesRuntimeState)(agentName, {
             upstreamStatus: "ok",
-            detail: queued > 0
-                ? `upstream reachable; ${queued} recovery item(s) queued`
-                : failed > 0
-                    ? `${failed} message(s) unrecoverable this cycle; upstream ok`
-                    : "upstream reachable",
+            detail: formatBlueBubblesRuntimeDetail(queued, failed),
             lastCheckedAt: checkedAt,
             pendingRecoveryCount: queued,
+            failedRecoveryCount: failed,
             lastRecoveredAt: previousState.lastRecoveredAt,
             lastRecoveredMessageGuid: previousState.lastRecoveredMessageGuid,
         });
@@ -1241,8 +1316,49 @@ async function syncBlueBubblesRuntime(deps = {}) {
             detail: error instanceof Error ? error.message : String(error),
             lastCheckedAt: checkedAt,
             pendingRecoveryCount: countPendingCapturedInboundMessages(agentName) + countPendingRecoveryCandidates(agentName),
+            failedRecoveryCount: 0,
+        });
+    }
+}
+async function recoverQueuedBlueBubblesMessages(deps = {}) {
+    const resolvedDeps = { ...defaultDeps, ...deps };
+    const agentName = resolvedDeps.getAgentName();
+    const previousState = (0, runtime_state_1.readBlueBubblesRuntimeState)(agentName);
+    const initialPending = countPendingCapturedInboundMessages(agentName) + countPendingRecoveryCandidates(agentName);
+    if (initialPending === 0) {
+        return { recovered: 0, skipped: 0, failed: 0, pendingRecoveryCount: 0 };
+    }
+    const captured = await recoverCapturedBlueBubblesInboundMessages(resolvedDeps);
+    const recovery = await recoverMissedBlueBubblesMessages(resolvedDeps);
+    const pendingRecoveryCount = countPendingCapturedInboundMessages(agentName) + countPendingRecoveryCandidates(agentName);
+    const failed = captured.failed + recovery.failed;
+    const recovered = captured.recovered + recovery.recovered;
+    const skipped = captured.skipped + recovery.skipped;
+    const checkedAt = new Date().toISOString();
+    try {
+        await resolvedDeps.createClient().checkHealth();
+        (0, runtime_state_1.writeBlueBubblesRuntimeState)(agentName, {
+            upstreamStatus: "ok",
+            detail: formatBlueBubblesRuntimeDetail(pendingRecoveryCount, failed),
+            lastCheckedAt: checkedAt,
+            pendingRecoveryCount,
+            failedRecoveryCount: failed,
+            lastRecoveredAt: recovered > 0 ? checkedAt : previousState.lastRecoveredAt,
+            lastRecoveredMessageGuid: previousState.lastRecoveredMessageGuid,
+        });
+    }
+    catch (error) {
+        (0, runtime_state_1.writeBlueBubblesRuntimeState)(agentName, {
+            upstreamStatus: "error",
+            detail: error instanceof Error ? error.message : String(error),
+            lastCheckedAt: checkedAt,
+            pendingRecoveryCount,
+            failedRecoveryCount: failed,
+            lastRecoveredAt: recovered > 0 ? checkedAt : previousState.lastRecoveredAt,
+            lastRecoveredMessageGuid: previousState.lastRecoveredMessageGuid,
         });
     }
+    return { recovered, skipped, failed, pendingRecoveryCount };
 }
 async function catchUpMissedBlueBubblesMessages(deps = {}, previousState, options = {}) {
     const resolvedDeps = { ...defaultDeps, ...deps };
@@ -1338,13 +1454,17 @@ async function catchUpMissedBlueBubblesMessages(deps = {}, previousState, option
             result.queued = (result.queued ?? 0) + 1;
             continue;
         }
+        let repairedMessage = null;
         try {
             const repaired = await client.repairEvent(event);
             if (repaired.kind !== "message") {
                 result.skipped++;
                 continue;
             }
-            const handled = await handleBlueBubblesNormalizedEvent(repaired, resolvedDeps, "upstream-catchup");
+            repairedMessage = repaired;
+            const handled = await handleBlueBubblesNormalizedEvent(repaired, resolvedDeps, "upstream-catchup", {
+                timeoutMs: BLUEBUBBLES_RECOVERY_TURN_TIMEOUT_MS,
+            });
             if (handled.reason === "already_processed") {
                 result.skipped++;
             }
@@ -1355,6 +1475,9 @@ async function catchUpMissedBlueBubblesMessages(deps = {}, previousState, option
         }
         catch (error) {
             result.failed++;
+            if (repairedMessage && isBlueBubblesRecoveryTurnTimeoutError(error)) {
+                (0, processed_log_1.recordProcessedBlueBubblesMessage)(agentName, repairedMessage, "upstream-catchup", "recovery-timeout");
+            }
             (0, runtime_1.emitNervesEvent)({
                 level: "warn",
                 component: "senses",
@@ -1431,13 +1554,17 @@ async function recoverCapturedBlueBubblesInboundMessages(deps = {}) {
             result.skipped++;
             continue;
         }
+        let repairedMessage = null;
         try {
             const repaired = await client.repairEvent(inboundEntryToRecoveryEvent(entry));
             if (repaired.kind !== "message") {
                 result.skipped++;
                 continue;
             }
-            const handled = await handleBlueBubblesNormalizedEvent(repaired, resolvedDeps, entry.source);
+            repairedMessage = repaired;
+            const handled = await handleBlueBubblesNormalizedEvent(repaired, resolvedDeps, entry.source, {
+                timeoutMs: BLUEBUBBLES_RECOVERY_TURN_TIMEOUT_MS,
+            });
             if (handled.reason === "already_processed") {
                 result.skipped++;
             }
@@ -1447,6 +1574,9 @@ async function recoverCapturedBlueBubblesInboundMessages(deps = {}) {
         }
         catch (error) {
             result.failed++;
+            if (repairedMessage && isBlueBubblesRecoveryTurnTimeoutError(error)) {
+                (0, processed_log_1.recordProcessedBlueBubblesMessage)(agentName, repairedMessage, entry.source, "recovery-timeout");
+            }
             (0, runtime_1.emitNervesEvent)({
                 level: "warn",
                 component: "senses",
@@ -1473,13 +1603,17 @@ async function recoverMissedBlueBubblesMessages(deps = {}) {
             result.skipped++;
             continue;
         }
+        let repairedMessage = null;
         try {
             const repaired = await client.repairEvent(mutationEntryToEvent(candidate));
             if (repaired.kind !== "message") {
                 result.pending++;
                 continue;
             }
-            const handled = await handleBlueBubblesNormalizedEvent(repaired, resolvedDeps, "mutation-recovery");
+            repairedMessage = repaired;
+            const handled = await handleBlueBubblesNormalizedEvent(repaired, resolvedDeps, "mutation-recovery", {
+                timeoutMs: BLUEBUBBLES_RECOVERY_TURN_TIMEOUT_MS,
+            });
             if (handled.reason === "already_processed") {
                 result.skipped++;
             }
@@ -1489,6 +1623,9 @@ async function recoverMissedBlueBubblesMessages(deps = {}) {
         }
         catch (error) {
             result.failed++;
+            if (repairedMessage && isBlueBubblesRecoveryTurnTimeoutError(error)) {
+                (0, processed_log_1.recordProcessedBlueBubblesMessage)(agentName, repairedMessage, "mutation-recovery", "recovery-timeout");
+            }
             (0, runtime_1.emitNervesEvent)({
                 level: "warn",
                 component: "senses",
@@ -1955,11 +2092,49 @@ function startBlueBubblesApp(deps = {}) {
     resolvedDeps.createClient();
     const channelConfig = (0, config_1.getBlueBubblesChannelConfig)();
     const server = resolvedDeps.createServer(createBlueBubblesWebhookHandler(deps));
+    let recoveryPassRunning = false;
+    let recoveryDelayTimer = null;
+    function triggerRecoveryPass() {
+        /* v8 ignore next -- re-entrant timer guard; difficult to force deterministically without timing the turn lock @preserve */
+        if (recoveryPassRunning)
+            return;
+        recoveryPassRunning = true;
+        void recoverQueuedBlueBubblesMessages(resolvedDeps)
+            /* v8 ignore start -- defensive wrapper; expected per-message failures are handled inside recovery helpers @preserve */
+            .catch((error) => {
+            (0, runtime_1.emitNervesEvent)({
+                level: "warn",
+                component: "senses",
+                event: "senses.bluebubbles_recovery_error",
+                message: "bluebubbles queued recovery pass failed",
+                meta: { reason: error instanceof Error ? error.message : String(error) },
+            });
+        })
+            /* v8 ignore stop */
+            .finally(() => {
+            recoveryPassRunning = false;
+        });
+    }
+    function scheduleRecoveryPass() {
+        /* v8 ignore next -- duplicate scheduling guard for overlapping health sync completions @preserve */
+        if (recoveryDelayTimer !== null)
+            return;
+        recoveryDelayTimer = setTimeout(() => {
+            recoveryDelayTimer = null;
+            triggerRecoveryPass();
+        }, BLUEBUBBLES_RECOVERY_PASS_DELAY_MS);
+    }
     const runtimeTimer = setInterval(() => {
-        void syncBlueBubblesRuntime(resolvedDeps);
+        void syncBlueBubblesRuntime(resolvedDeps).then(scheduleRecoveryPass);
     }, BLUEBUBBLES_RUNTIME_SYNC_INTERVAL_MS);
+    const recoveryTimer = setInterval(triggerRecoveryPass, BLUEBUBBLES_RECOVERY_PASS_INTERVAL_MS);
     server.on?.("close", () => {
         clearInterval(runtimeTimer);
+        clearInterval(recoveryTimer);
+        if (recoveryDelayTimer !== null) {
+            clearTimeout(recoveryDelayTimer);
+            recoveryDelayTimer = null;
+        }
     });
     server.listen(channelConfig.port, () => {
         (0, runtime_1.emitNervesEvent)({
@@ -1969,6 +2144,6 @@ function startBlueBubblesApp(deps = {}) {
             meta: { port: channelConfig.port, webhookPath: channelConfig.webhookPath },
         });
     });
-    void syncBlueBubblesRuntime(resolvedDeps);
+    void syncBlueBubblesRuntime(resolvedDeps).then(scheduleRecoveryPass);
     return server;
 }

package/dist/senses/bluebubbles/runtime-state.js

@@ -53,6 +53,9 @@ function readBlueBubblesRuntimeState(agentName, agentRoot) {
     try {
         const raw = fs.readFileSync(filePath, "utf-8");
         const parsed = JSON.parse(raw);
+        const failedRecoveryCount = typeof parsed.failedRecoveryCount === "number" && Number.isFinite(parsed.failedRecoveryCount)
+            ? parsed.failedRecoveryCount
+            : undefined;
         return {
             upstreamStatus: parsed.upstreamStatus === "ok" || parsed.upstreamStatus === "error"
                 ? parsed.upstreamStatus
@@ -64,6 +67,7 @@ function readBlueBubblesRuntimeState(agentName, agentRoot) {
             pendingRecoveryCount: typeof parsed.pendingRecoveryCount === "number" && Number.isFinite(parsed.pendingRecoveryCount)
                 ? parsed.pendingRecoveryCount
                 : 0,
+            ...(typeof failedRecoveryCount === "number" ? { failedRecoveryCount } : {}),
             lastRecoveredAt: typeof parsed.lastRecoveredAt === "string" ? parsed.lastRecoveredAt : undefined,
             lastRecoveredMessageGuid: typeof parsed.lastRecoveredMessageGuid === "string"
                 ? parsed.lastRecoveredMessageGuid
@@ -102,6 +106,7 @@ function writeBlueBubblesRuntimeState(agentName, state, agentRoot) {
             agentName,
             upstreamStatus: state.upstreamStatus,
             pendingRecoveryCount: state.pendingRecoveryCount,
+            failedRecoveryCount: state.failedRecoveryCount ?? 0,
             path: filePath,
         },
     });

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@ouro.bot/cli",
-  "version": "0.1.0-alpha.525",
+  "version": "0.1.0-alpha.526",
   "main": "dist/heart/daemon/ouro-entry.js",
   "bin": {
     "cli": "dist/heart/daemon/ouro-bot-entry.js",