@ouro.bot/cli 0.1.0-alpha.543 → 0.1.0-alpha.545

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/changelog.json CHANGED
@@ -1,6 +1,22 @@
1
1
  {
2
2
  "_note": "This changelog is maintained as part of the PR/version-bump workflow. Agent-curated, not auto-generated. Agents read this file directly via read_file to understand what changed between versions.",
3
3
  "versions": [
4
+ {
5
+ "version": "0.1.0-alpha.545",
6
+ "changes": [
7
+ "Launchd-managed habit jobs now inherit the runtime PATH in their plist environment, preventing login-started jobs from failing to find the Node runtime.",
8
+ "The default daemon starter now launches with the current process executable instead of a bare `node` lookup, so background restarts survive sparse launch contexts.",
9
+ "Adds regression coverage for restart launch path hardening and mail-import discovery timestamp fallbacks."
10
+ ]
11
+ },
12
+ {
13
+ "version": "0.1.0-alpha.544",
14
+ "changes": [
15
+ "BlueBubbles live webhook turns now have a bounded timeout, visible timeout notice, and recoverable captured-message retry path, so a long iMessage turn cannot silently monopolize the live chat lane.",
16
+ "BlueBubbles runtime health now records active and stalled live turns, exposes them in status output, and marks the sense unhealthy when the listener is reachable but a live iMessage turn is stuck.",
17
+ "The `ouro bluebubbles replay` debug command now refreshes this machine's BlueBubbles runtime credentials before creating the client, matching the daemon-launched listener's vault-backed configuration."
18
+ ]
19
+ },
4
20
  {
5
21
  "version": "0.1.0-alpha.543",
6
22
  "changes": [
@@ -81,7 +81,7 @@ function defaultStartDaemonProcess(socketPath) {
81
81
  // when the daemon's logging system writes to stderr after the parent exits.
82
82
  const outFd = fs.openSync(os.devNull, "w");
83
83
  const errFd = fs.openSync(os.devNull, "w");
84
- const child = (0, child_process_1.spawn)("node", [entry, "--socket", socketPath], {
84
+ const child = (0, child_process_1.spawn)(process.execPath, [entry, "--socket", socketPath], {
85
85
  detached: true,
86
86
  stdio: ["ignore", outFd, errFd],
87
87
  });
@@ -97,6 +97,14 @@ function formatSenseDetail(row) {
97
97
  recoveryParts.push(`oldestPending=${row.oldestPendingRecoveryAt}`);
98
98
  if (row.oldestPendingRecoveryAgeMs !== undefined)
99
99
  recoveryParts.push(`oldestPendingAge=${formatDurationMs(row.oldestPendingRecoveryAgeMs)}`);
100
+ if (row.activeTurnCount !== undefined)
101
+ recoveryParts.push(`activeTurns=${row.activeTurnCount}`);
102
+ if (row.stalledTurnCount !== undefined)
103
+ recoveryParts.push(`stalledTurns=${row.stalledTurnCount}`);
104
+ if (row.oldestActiveTurnStartedAt)
105
+ recoveryParts.push(`oldestActive=${row.oldestActiveTurnStartedAt}`);
106
+ if (row.oldestActiveTurnAgeMs !== undefined)
107
+ recoveryParts.push(`oldestActiveAge=${formatDurationMs(row.oldestActiveTurnAgeMs)}`);
100
108
  if (recoveryParts.length > 0)
101
109
  details.push(recoveryParts.join(" "));
102
110
  const failureParts = [];
@@ -181,6 +189,10 @@ function parseStatusPayload(data) {
181
189
  const failedRecoveryCount = numberField(row.failedRecoveryCount);
182
190
  const oldestPendingRecoveryAt = stringField(row.oldestPendingRecoveryAt);
183
191
  const oldestPendingRecoveryAgeMs = numberField(row.oldestPendingRecoveryAgeMs);
192
+ const activeTurnCount = numberField(row.activeTurnCount);
193
+ const stalledTurnCount = numberField(row.stalledTurnCount);
194
+ const oldestActiveTurnStartedAt = stringField(row.oldestActiveTurnStartedAt);
195
+ const oldestActiveTurnAgeMs = numberField(row.oldestActiveTurnAgeMs);
184
196
  if (proofMethod !== null)
185
197
  parsed.proofMethod = proofMethod;
186
198
  if (lastProofAt !== null)
@@ -201,6 +213,14 @@ function parseStatusPayload(data) {
201
213
  parsed.oldestPendingRecoveryAt = oldestPendingRecoveryAt;
202
214
  if (oldestPendingRecoveryAgeMs !== null)
203
215
  parsed.oldestPendingRecoveryAgeMs = oldestPendingRecoveryAgeMs;
216
+ if (activeTurnCount !== null)
217
+ parsed.activeTurnCount = activeTurnCount;
218
+ if (stalledTurnCount !== null)
219
+ parsed.stalledTurnCount = stalledTurnCount;
220
+ if (oldestActiveTurnStartedAt !== null)
221
+ parsed.oldestActiveTurnStartedAt = oldestActiveTurnStartedAt;
222
+ if (oldestActiveTurnAgeMs !== null)
223
+ parsed.oldestActiveTurnAgeMs = oldestActiveTurnAgeMs;
204
224
  return parsed;
205
225
  });
206
226
  const parsedWorkers = workers.map((entry) => {
@@ -77,6 +77,7 @@ function createRealOsCronDeps() {
77
77
  },
78
78
  mkdirp: (dir) => fs.mkdirSync(dir, { recursive: true }),
79
79
  homeDir: os.homedir(),
80
+ envPath: process.env.PATH ?? "",
80
81
  };
81
82
  }
82
83
  function createRealCrontabDeps() {
@@ -80,7 +80,7 @@ function scheduleToCalendarInterval(schedule) {
80
80
  result.Month = parseInt(month, 10);
81
81
  return Object.keys(result).length > 0 ? result : null;
82
82
  }
83
- function generatePlistXml(job) {
83
+ function generatePlistXml(job, envPath) {
84
84
  const label = plistLabel(job);
85
85
  const seconds = cadenceToSeconds(job.schedule);
86
86
  const calendar = seconds === null ? scheduleToCalendarInterval(job.schedule) : null;
@@ -97,7 +97,7 @@ function generatePlistXml(job) {
97
97
  else {
98
98
  triggerXml = ` <key>StartInterval</key>\n <integer>1800</integer>`;
99
99
  }
100
- return [
100
+ const lines = [
101
101
  `<?xml version="1.0" encoding="UTF-8"?>`,
102
102
  `<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">`,
103
103
  `<plist version="1.0">`,
@@ -110,14 +110,12 @@ function generatePlistXml(job) {
110
110
  ...job.command.split(" ").slice(1).map((arg) => ` <string>${arg}</string>`),
111
111
  ` </array>`,
112
112
  triggerXml,
113
- ` <key>StandardOutPath</key>`,
114
- ` <string>/tmp/${label}.stdout.log</string>`,
115
- ` <key>StandardErrorPath</key>`,
116
- ` <string>/tmp/${label}.stderr.log</string>`,
117
- `</dict>`,
118
- `</plist>`,
119
- ``,
120
- ].join("\n");
113
+ ];
114
+ if (envPath) {
115
+ lines.push(` <key>EnvironmentVariables</key>`, ` <dict>`, ` <key>PATH</key>`, ` <string>${envPath}</string>`, ` </dict>`);
116
+ }
117
+ lines.push(` <key>StandardOutPath</key>`, ` <string>/tmp/${label}.stdout.log</string>`, ` <key>StandardErrorPath</key>`, ` <string>/tmp/${label}.stderr.log</string>`, `</dict>`, `</plist>`, ``);
118
+ return lines.join("\n");
121
119
  }
122
120
  class LaunchdCronManager {
123
121
  deps;
@@ -148,7 +146,7 @@ class LaunchdCronManager {
148
146
  const label = plistLabel(job);
149
147
  const filename = `${label}.plist`;
150
148
  const fullPath = `${this.launchAgentsDir}/${filename}`;
151
- const xml = generatePlistXml(job);
149
+ const xml = generatePlistXml(job, this.deps.envPath);
152
150
  try {
153
151
  this.deps.exec(`launchctl unload "${fullPath}"`);
154
152
  }
@@ -248,6 +246,7 @@ function createOsCronManager(options = {}) {
248
246
  listDir: () => [],
249
247
  mkdirp: () => { },
250
248
  homeDir: os.homedir(),
249
+ envPath: process.env.PATH ?? "",
251
250
  };
252
251
  /* v8 ignore stop */
253
252
  return new LaunchdCronManager(deps);
@@ -279,6 +279,16 @@ function readBlueBubblesRuntimeJson(runtimePath) {
279
279
  oldestPendingRecoveryAgeMs: typeof parsed.oldestPendingRecoveryAgeMs === "number" && Number.isFinite(parsed.oldestPendingRecoveryAgeMs)
280
280
  ? parsed.oldestPendingRecoveryAgeMs
281
281
  : undefined,
282
+ activeTurnCount: typeof parsed.activeTurnCount === "number" && Number.isFinite(parsed.activeTurnCount)
283
+ ? parsed.activeTurnCount
284
+ : undefined,
285
+ stalledTurnCount: typeof parsed.stalledTurnCount === "number" && Number.isFinite(parsed.stalledTurnCount)
286
+ ? parsed.stalledTurnCount
287
+ : undefined,
288
+ oldestActiveTurnStartedAt: typeof parsed.oldestActiveTurnStartedAt === "string" ? parsed.oldestActiveTurnStartedAt : undefined,
289
+ oldestActiveTurnAgeMs: typeof parsed.oldestActiveTurnAgeMs === "number" && Number.isFinite(parsed.oldestActiveTurnAgeMs)
290
+ ? parsed.oldestActiveTurnAgeMs
291
+ : undefined,
282
292
  };
283
293
  /* v8 ignore stop */
284
294
  /* v8 ignore start -- defensive: catch for missing/corrupt BB runtime state file @preserve */
@@ -304,6 +314,10 @@ function readBlueBubblesRuntimeFacts(agent, bundlesRoot, snapshot) {
304
314
  failedRecoveryCount: state.failedRecoveryCount,
305
315
  oldestPendingRecoveryAt: state.oldestPendingRecoveryAt,
306
316
  oldestPendingRecoveryAgeMs: state.oldestPendingRecoveryAgeMs,
317
+ activeTurnCount: state.activeTurnCount,
318
+ stalledTurnCount: state.stalledTurnCount,
319
+ oldestActiveTurnStartedAt: state.oldestActiveTurnStartedAt,
320
+ oldestActiveTurnAgeMs: state.oldestActiveTurnAgeMs,
307
321
  };
308
322
  if (!blueBubblesRuntimeStateIsFresh(state.lastCheckedAt)) {
309
323
  return {
@@ -342,6 +356,16 @@ function readBlueBubblesRuntimeFacts(agent, bundlesRoot, snapshot) {
342
356
  recoveryAction: "queued recovery will retry; inspect BlueBubbles inbound/recovery sidecar logs if age keeps growing",
343
357
  };
344
358
  }
359
+ if ((state.stalledTurnCount ?? 0) > 0) {
360
+ return {
361
+ runtime: "error",
362
+ detail: state.detail,
363
+ ...proofFacts,
364
+ lastFailure: state.detail,
365
+ failureLayer: "live_turn_stall",
366
+ recoveryAction: "live iMessage turn timeout/watchdog will release the lane and recovery will retry captured messages",
367
+ };
368
+ }
345
369
  if (state.upstreamStatus === "ok") {
346
370
  return {
347
371
  runtime: "running",
@@ -573,6 +597,10 @@ class DaemonSenseManager {
573
597
  failedRecoveryCount: blueBubblesRuntimeFacts.failedRecoveryCount,
574
598
  oldestPendingRecoveryAt: blueBubblesRuntimeFacts.oldestPendingRecoveryAt,
575
599
  oldestPendingRecoveryAgeMs: blueBubblesRuntimeFacts.oldestPendingRecoveryAgeMs,
600
+ activeTurnCount: blueBubblesRuntimeFacts.activeTurnCount,
601
+ stalledTurnCount: blueBubblesRuntimeFacts.stalledTurnCount,
602
+ oldestActiveTurnStartedAt: blueBubblesRuntimeFacts.oldestActiveTurnStartedAt,
603
+ oldestActiveTurnAgeMs: blueBubblesRuntimeFacts.oldestActiveTurnAgeMs,
576
604
  } : {}),
577
605
  }));
578
606
  });
@@ -0,0 +1,216 @@
1
+ "use strict";
2
+ var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
3
+ if (k2 === undefined) k2 = k;
4
+ var desc = Object.getOwnPropertyDescriptor(m, k);
5
+ if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
6
+ desc = { enumerable: true, get: function() { return m[k]; } };
7
+ }
8
+ Object.defineProperty(o, k2, desc);
9
+ }) : (function(o, m, k, k2) {
10
+ if (k2 === undefined) k2 = k;
11
+ o[k2] = m[k];
12
+ }));
13
+ var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
14
+ Object.defineProperty(o, "default", { enumerable: true, value: v });
15
+ }) : function(o, v) {
16
+ o["default"] = v;
17
+ });
18
+ var __importStar = (this && this.__importStar) || (function () {
19
+ var ownKeys = function(o) {
20
+ ownKeys = Object.getOwnPropertyNames || function (o) {
21
+ var ar = [];
22
+ for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
23
+ return ar;
24
+ };
25
+ return ownKeys(o);
26
+ };
27
+ return function (mod) {
28
+ if (mod && mod.__esModule) return mod;
29
+ var result = {};
30
+ if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
31
+ __setModuleDefault(result, mod);
32
+ return result;
33
+ };
34
+ })();
35
+ Object.defineProperty(exports, "__esModule", { value: true });
36
+ exports.beginBlueBubblesActiveTurn = beginBlueBubblesActiveTurn;
37
+ exports.noteBlueBubblesActiveTurnVisibleActivity = noteBlueBubblesActiveTurnVisibleActivity;
38
+ exports.finishBlueBubblesActiveTurn = finishBlueBubblesActiveTurn;
39
+ exports.listBlueBubblesActiveTurns = listBlueBubblesActiveTurns;
40
+ exports.snapshotBlueBubblesActiveTurns = snapshotBlueBubblesActiveTurns;
41
+ const fs = __importStar(require("node:fs"));
42
+ const path = __importStar(require("node:path"));
43
+ const config_1 = require("../../heart/config");
44
+ const identity_1 = require("../../heart/identity");
45
+ const runtime_1 = require("../../nerves/runtime");
46
+ function activeTurnsDir(agentName) {
47
+ return path.join((0, identity_1.getAgentRoot)(agentName), "state", "senses", "bluebubbles", "active-turns");
48
+ }
49
+ function activeTurnPath(agentName, turnId) {
50
+ return path.join(activeTurnsDir(agentName), `${(0, config_1.sanitizeKey)(turnId)}.json`);
51
+ }
52
+ function isPidAlive(pid) {
53
+ try {
54
+ process.kill(pid, 0);
55
+ return true;
56
+ }
57
+ catch {
58
+ return false;
59
+ }
60
+ }
61
+ function parseActiveTurn(raw) {
62
+ try {
63
+ const parsed = JSON.parse(raw);
64
+ if (!parsed.turnId || !parsed.messageGuid || !parsed.sessionKey || !parsed.startedAt)
65
+ return null;
66
+ if (typeof parsed.pid !== "number")
67
+ return null;
68
+ return {
69
+ turnId: parsed.turnId,
70
+ pid: parsed.pid,
71
+ startedAt: parsed.startedAt,
72
+ lastVisibleActivityAt: typeof parsed.lastVisibleActivityAt === "string"
73
+ ? parsed.lastVisibleActivityAt
74
+ : undefined,
75
+ messageGuid: parsed.messageGuid,
76
+ sessionKey: parsed.sessionKey,
77
+ chatGuid: parsed.chatGuid ?? null,
78
+ chatIdentifier: parsed.chatIdentifier ?? null,
79
+ };
80
+ }
81
+ catch {
82
+ return null;
83
+ }
84
+ }
85
+ function beginBlueBubblesActiveTurn(agentName, event) {
86
+ const turnId = `${event.chat.sessionKey}:${event.messageGuid}:${process.pid}`;
87
+ const entry = {
88
+ turnId,
89
+ pid: process.pid,
90
+ startedAt: new Date().toISOString(),
91
+ messageGuid: event.messageGuid,
92
+ sessionKey: event.chat.sessionKey,
93
+ chatGuid: event.chat.chatGuid ?? null,
94
+ chatIdentifier: event.chat.chatIdentifier ?? null,
95
+ };
96
+ const filePath = activeTurnPath(agentName, turnId);
97
+ try {
98
+ fs.mkdirSync(path.dirname(filePath), { recursive: true });
99
+ fs.writeFileSync(filePath, JSON.stringify(entry, null, 2) + "\n", "utf-8");
100
+ }
101
+ catch (error) {
102
+ (0, runtime_1.emitNervesEvent)({
103
+ level: "warn",
104
+ component: "senses",
105
+ event: "senses.bluebubbles_active_turn_write_error",
106
+ message: "failed to record active bluebubbles turn",
107
+ meta: {
108
+ agentName,
109
+ messageGuid: event.messageGuid,
110
+ sessionKey: event.chat.sessionKey,
111
+ /* v8 ignore next -- filesystem writes throw Error instances; stringify guard is defensive @preserve */
112
+ reason: error instanceof Error ? error.message : String(error),
113
+ },
114
+ });
115
+ }
116
+ return turnId;
117
+ }
118
+ function noteBlueBubblesActiveTurnVisibleActivity(agentName, turnId) {
119
+ const filePath = activeTurnPath(agentName, turnId);
120
+ let entry = null;
121
+ try {
122
+ entry = parseActiveTurn(fs.readFileSync(filePath, "utf-8"));
123
+ }
124
+ catch {
125
+ return;
126
+ }
127
+ if (!entry)
128
+ return;
129
+ try {
130
+ fs.writeFileSync(filePath, JSON.stringify({ ...entry, lastVisibleActivityAt: new Date().toISOString() }, null, 2) + "\n", "utf-8");
131
+ }
132
+ catch (error) {
133
+ (0, runtime_1.emitNervesEvent)({
134
+ level: "warn",
135
+ component: "senses",
136
+ event: "senses.bluebubbles_active_turn_write_error",
137
+ message: "failed to update active bluebubbles turn",
138
+ meta: {
139
+ agentName,
140
+ turnId,
141
+ /* v8 ignore next -- filesystem writes throw Error instances; stringify guard is defensive @preserve */
142
+ reason: error instanceof Error ? error.message : String(error),
143
+ },
144
+ });
145
+ }
146
+ }
147
+ function finishBlueBubblesActiveTurn(agentName, turnId) {
148
+ try {
149
+ fs.unlinkSync(activeTurnPath(agentName, turnId));
150
+ }
151
+ catch {
152
+ // Missing active-turn files are harmless: this is best-effort telemetry.
153
+ }
154
+ }
155
+ function listBlueBubblesActiveTurns(agentName) {
156
+ let files;
157
+ try {
158
+ files = fs.readdirSync(activeTurnsDir(agentName)).filter((name) => name.endsWith(".json")).sort();
159
+ }
160
+ catch {
161
+ return [];
162
+ }
163
+ const entries = [];
164
+ for (const file of files) {
165
+ const filePath = path.join(activeTurnsDir(agentName), file);
166
+ let entry = null;
167
+ try {
168
+ entry = parseActiveTurn(fs.readFileSync(filePath, "utf-8"));
169
+ }
170
+ catch {
171
+ entry = null;
172
+ }
173
+ if (!entry) {
174
+ try {
175
+ fs.unlinkSync(filePath);
176
+ }
177
+ catch { /* ignore corrupt cleanup races */ }
178
+ continue;
179
+ }
180
+ if (!isPidAlive(entry.pid)) {
181
+ try {
182
+ fs.unlinkSync(filePath);
183
+ }
184
+ catch { /* ignore stale cleanup races */ }
185
+ (0, runtime_1.emitNervesEvent)({
186
+ level: "warn",
187
+ component: "senses",
188
+ event: "senses.bluebubbles_active_turn_pruned",
189
+ message: "pruned stale bluebubbles active-turn marker",
190
+ meta: {
191
+ agentName,
192
+ messageGuid: entry.messageGuid,
193
+ sessionKey: entry.sessionKey,
194
+ pid: entry.pid,
195
+ },
196
+ });
197
+ continue;
198
+ }
199
+ entries.push(entry);
200
+ }
201
+ return entries;
202
+ }
203
+ function snapshotBlueBubblesActiveTurns(agentName, stalledAfterMs, nowMs = Date.now()) {
204
+ const entries = listBlueBubblesActiveTurns(agentName);
205
+ const started = entries
206
+ .map((entry) => ({ value: entry.startedAt, ms: Date.parse(entry.startedAt) }))
207
+ .filter((entry) => Number.isFinite(entry.ms))
208
+ .sort((left, right) => left.ms - right.ms);
209
+ const oldest = started[0];
210
+ return {
211
+ activeTurnCount: entries.length,
212
+ stalledTurnCount: started.filter((entry) => nowMs - entry.ms >= stalledAfterMs).length,
213
+ oldestActiveTurnStartedAt: oldest?.value,
214
+ oldestActiveTurnAgeMs: oldest ? Math.max(0, nowMs - oldest.ms) : undefined,
215
+ };
216
+ }
@@ -76,6 +76,7 @@ const mutation_log_1 = require("./mutation-log");
76
76
  const processed_log_1 = require("./processed-log");
77
77
  const runtime_state_1 = require("./runtime-state");
78
78
  const session_cleanup_1 = require("./session-cleanup");
79
+ const active_turns_1 = require("./active-turns");
79
80
  const tool_activity_callbacks_1 = require("../../heart/tool-activity-callbacks");
80
81
  const commands_1 = require("../commands");
81
82
  const trust_gate_1 = require("../trust-gate");
@@ -170,7 +171,10 @@ const defaultDeps = {
170
171
  const BLUEBUBBLES_RUNTIME_SYNC_INTERVAL_MS = 30_000;
171
172
  const BLUEBUBBLES_RECOVERY_PASS_DELAY_MS = 1_000;
172
173
  const BLUEBUBBLES_RECOVERY_PASS_INTERVAL_MS = 30_000;
174
+ const BLUEBUBBLES_LIVE_TURN_TIMEOUT_MS = 2 * 60_000;
173
175
  const BLUEBUBBLES_RECOVERY_TURN_TIMEOUT_MS = 10 * 60_000;
176
+ const BLUEBUBBLES_LIVE_TURN_STALLED_MS = 90_000;
177
+ const BLUEBUBBLES_SILENCE_WATCHDOG_MS = 75_000;
174
178
  const BLUEBUBBLES_CATCHUP_PAGE_SIZE = 50;
175
179
  const BLUEBUBBLES_CATCHUP_MAX_PAGES = 20;
176
180
  const BLUEBUBBLES_HEALTHY_CATCHUP_OVERLAP_MS = 90_000;
@@ -447,10 +451,12 @@ function emitBlueBubblesMarkReadWarning(chat, error) {
447
451
  },
448
452
  });
449
453
  }
450
- function createBlueBubblesCallbacks(client, chat, replyTarget, isGroupChat) {
454
+ function createBlueBubblesCallbacks(client, chat, replyTarget, isGroupChat, onVisibleActivity) {
451
455
  let textBuffer = "";
452
456
  let typingActive = false;
453
457
  let queue = Promise.resolve();
458
+ let lastVisibleActivityMs = Date.now();
459
+ let silenceWatchdog = null;
454
460
  function enqueue(operation, task) {
455
461
  queue = queue.then(task).catch((error) => {
456
462
  (0, runtime_1.emitNervesEvent)({
@@ -480,6 +486,29 @@ function createBlueBubblesCallbacks(client, chat, replyTarget, isGroupChat) {
480
486
  }
481
487
  });
482
488
  }
489
+ function recordVisibleActivity() {
490
+ lastVisibleActivityMs = Date.now();
491
+ onVisibleActivity?.();
492
+ }
493
+ function stopSilenceWatchdog() {
494
+ if (silenceWatchdog === null)
495
+ return;
496
+ clearInterval(silenceWatchdog);
497
+ silenceWatchdog = null;
498
+ }
499
+ function startSilenceWatchdog() {
500
+ if (silenceWatchdog !== null)
501
+ return;
502
+ silenceWatchdog = setInterval(() => {
503
+ if (Date.now() - lastVisibleActivityMs < BLUEBUBBLES_SILENCE_WATCHDOG_MS)
504
+ return;
505
+ sendStatus("still working on this...");
506
+ }, BLUEBUBBLES_SILENCE_WATCHDOG_MS);
507
+ /* v8 ignore next -- timer handles expose unref only in some runtimes @preserve */
508
+ if (typeof silenceWatchdog.unref === "function") {
509
+ silenceWatchdog.unref();
510
+ }
511
+ }
483
512
  function sendStatus(text) {
484
513
  enqueue("send_status", async () => {
485
514
  await client.sendText({
@@ -487,6 +516,7 @@ function createBlueBubblesCallbacks(client, chat, replyTarget, isGroupChat) {
487
516
  text,
488
517
  replyToMessageGuid: replyTarget.getReplyToMessageGuid(),
489
518
  });
519
+ recordVisibleActivity();
490
520
  // Re-enable typing indicator — sending a message clears the typing bubble
491
521
  await client.setTyping(chat, true);
492
522
  });
@@ -502,6 +532,7 @@ function createBlueBubblesCallbacks(client, chat, replyTarget, isGroupChat) {
502
532
  });
503
533
  return {
504
534
  onModelStart() {
535
+ startSilenceWatchdog();
505
536
  if (!isGroupChat)
506
537
  startTypingNow();
507
538
  (0, runtime_1.emitNervesEvent)({
@@ -590,6 +621,7 @@ function createBlueBubblesCallbacks(client, chat, replyTarget, isGroupChat) {
590
621
  text: trimmed,
591
622
  replyToMessageGuid: replyTarget.getReplyToMessageGuid(),
592
623
  });
624
+ recordVisibleActivity();
593
625
  // Note: do NOT call client.setTyping(chat, false) here — the agent is
594
626
  // still mid-turn, so the typing indicator stays ACTIVE.
595
627
  (0, runtime_1.emitNervesEvent)({
@@ -623,8 +655,10 @@ function createBlueBubblesCallbacks(client, chat, replyTarget, isGroupChat) {
623
655
  text: trimmed,
624
656
  replyToMessageGuid: replyTarget.getReplyToMessageGuid(),
625
657
  });
658
+ recordVisibleActivity();
626
659
  },
627
660
  async finish() {
661
+ stopSilenceWatchdog();
628
662
  statusBatcher.flush();
629
663
  if (!typingActive) {
630
664
  await queue;
@@ -824,6 +858,7 @@ async function handleBlueBubblesNormalizedEvent(event, resolvedDeps, source, opt
824
858
  }
825
859
  let ownsInFlightMessage = false;
826
860
  let releaseInFlightAfterTurnSettles = false;
861
+ let activeTurnId = null;
827
862
  if (event.kind === "message") {
828
863
  if ((0, processed_log_1.hasProcessedBlueBubblesMessage)(agentName, event.chat.sessionKey, event.messageGuid)
829
864
  || (0, processed_log_1.hasProcessedBlueBubblesMessageGuid)(agentName, event.messageGuid)) {
@@ -855,6 +890,7 @@ async function handleBlueBubblesNormalizedEvent(event, resolvedDeps, source, opt
855
890
  return { handled: true, notifiedAgent: false, kind: event.kind, reason: "already_processed" };
856
891
  }
857
892
  ownsInFlightMessage = true;
893
+ activeTurnId = (0, active_turns_1.beginBlueBubblesActiveTurn)(agentName, event);
858
894
  }
859
895
  try {
860
896
  // ── Adapter setup: friend, session, content, callbacks ──────────
@@ -945,10 +981,12 @@ async function handleBlueBubblesNormalizedEvent(event, resolvedDeps, source, opt
945
981
  role: "user",
946
982
  content: buildInboundContent(event, existing?.messages ?? sessionMessages, repliedToText),
947
983
  };
948
- const callbacks = createBlueBubblesCallbacks(client, event.chat, replyTarget, event.chat.isGroup);
984
+ const callbacks = createBlueBubblesCallbacks(client, event.chat, replyTarget, event.chat.isGroup, activeTurnId
985
+ ? () => (0, active_turns_1.noteBlueBubblesActiveTurnVisibleActivity)(agentName, activeTurnId)
986
+ : undefined);
949
987
  const controller = new AbortController();
950
- let timeoutTimer = null;
951
- let timeoutPromise = null;
988
+ let timeoutTimer;
989
+ let timeoutPromise;
952
990
  let timeoutReject;
953
991
  let recoveryTimedOut = false;
954
992
  // BB-specific tool context wrappers
@@ -978,34 +1016,39 @@ async function handleBlueBubblesNormalizedEvent(event, resolvedDeps, source, opt
978
1016
  };
979
1017
  /* v8 ignore stop */
980
1018
  try {
981
- const timeoutMs = options.timeoutMs;
982
- if (typeof timeoutMs === "number" && Number.isFinite(timeoutMs) && timeoutMs > 0) {
983
- timeoutPromise = new Promise((_, reject) => {
984
- timeoutReject = reject;
985
- });
986
- timeoutTimer = setTimeout(() => {
987
- const reason = new BlueBubblesRecoveryTurnTimeoutError(timeoutMs);
988
- recoveryTimedOut = true;
1019
+ const liveWebhookTimeout = source === "webhook" && options.timeoutMs === undefined;
1020
+ const timeoutMs = options.timeoutMs ?? BLUEBUBBLES_LIVE_TURN_TIMEOUT_MS;
1021
+ timeoutPromise = new Promise((_, reject) => {
1022
+ timeoutReject = reject;
1023
+ });
1024
+ timeoutTimer = setTimeout(() => {
1025
+ const reason = new BlueBubblesRecoveryTurnTimeoutError(timeoutMs);
1026
+ recoveryTimedOut = true;
1027
+ if (liveWebhookTimeout && ownsInFlightMessage && event.kind === "message") {
1028
+ endBlueBubblesMessageInFlight(event.chat.sessionKey, event.messageGuid);
1029
+ ownsInFlightMessage = false;
1030
+ }
1031
+ else {
989
1032
  releaseInFlightAfterTurnSettles = true;
990
- controller.abort(reason);
991
- timeoutReject?.(reason);
992
- (0, runtime_1.emitNervesEvent)({
993
- level: "warn",
994
- component: "senses",
995
- event: "senses.bluebubbles_turn_timeout",
996
- message: "bluebubbles recovery turn timed out",
997
- meta: {
998
- messageGuid: event.messageGuid,
999
- sessionKey: event.chat.sessionKey,
1000
- source,
1001
- timeoutMs,
1002
- },
1003
- });
1004
- }, timeoutMs);
1005
- /* v8 ignore next -- timer handles expose unref only in some runtimes @preserve */
1006
- if (typeof timeoutTimer.unref === "function") {
1007
- timeoutTimer.unref();
1008
1033
  }
1034
+ controller.abort(reason);
1035
+ timeoutReject?.(reason);
1036
+ (0, runtime_1.emitNervesEvent)({
1037
+ level: "warn",
1038
+ component: "senses",
1039
+ event: "senses.bluebubbles_turn_timeout",
1040
+ message: "bluebubbles turn timed out",
1041
+ meta: {
1042
+ messageGuid: event.messageGuid,
1043
+ sessionKey: event.chat.sessionKey,
1044
+ source,
1045
+ timeoutMs,
1046
+ },
1047
+ });
1048
+ }, timeoutMs);
1049
+ /* v8 ignore next -- timer handles expose unref only in some runtimes @preserve */
1050
+ if (typeof timeoutTimer.unref === "function") {
1051
+ timeoutTimer.unref();
1009
1052
  }
1010
1053
  const turnPromise = (0, pipeline_1.handleInboundTurn)({
1011
1054
  channel: "bluebubbles",
@@ -1069,34 +1112,40 @@ async function handleBlueBubblesNormalizedEvent(event, resolvedDeps, source, opt
1069
1112
  })(),
1070
1113
  });
1071
1114
  /* v8 ignore start -- detached late-rejection telemetry is asserted in timeout tests, but V8 does not reliably attribute Promise.catch callbacks @preserve */
1072
- if (timeoutPromise) {
1073
- void turnPromise
1074
- .catch((error) => {
1075
- if (!recoveryTimedOut)
1076
- return;
1077
- (0, runtime_1.emitNervesEvent)({
1078
- level: "warn",
1079
- component: "senses",
1080
- event: "senses.bluebubbles_recovery_error",
1081
- message: "bluebubbles recovery turn rejected after timeout",
1082
- meta: {
1083
- messageGuid: event.messageGuid,
1084
- sessionKey: event.chat.sessionKey,
1085
- source,
1086
- reason: error instanceof Error ? error.message : String(error),
1087
- },
1088
- });
1089
- })
1090
- .finally(() => {
1091
- if (releaseInFlightAfterTurnSettles && ownsInFlightMessage && event.kind === "message") {
1092
- endBlueBubblesMessageInFlight(event.chat.sessionKey, event.messageGuid);
1093
- }
1115
+ void turnPromise
1116
+ .catch((error) => {
1117
+ if (!recoveryTimedOut)
1118
+ return;
1119
+ (0, runtime_1.emitNervesEvent)({
1120
+ level: "warn",
1121
+ component: "senses",
1122
+ event: "senses.bluebubbles_recovery_error",
1123
+ message: "bluebubbles recovery turn rejected after timeout",
1124
+ meta: {
1125
+ messageGuid: event.messageGuid,
1126
+ sessionKey: event.chat.sessionKey,
1127
+ source,
1128
+ reason: error instanceof Error ? error.message : String(error),
1129
+ },
1094
1130
  });
1095
- }
1131
+ })
1132
+ .finally(() => {
1133
+ if (releaseInFlightAfterTurnSettles && ownsInFlightMessage && event.kind === "message") {
1134
+ endBlueBubblesMessageInFlight(event.chat.sessionKey, event.messageGuid);
1135
+ }
1136
+ });
1096
1137
  /* v8 ignore stop */
1097
- const result = timeoutPromise
1098
- ? await Promise.race([turnPromise, timeoutPromise])
1099
- : await turnPromise;
1138
+ const result = await (async () => {
1139
+ try {
1140
+ return await Promise.race([turnPromise, timeoutPromise]);
1141
+ }
1142
+ catch (error) {
1143
+ if (error instanceof BlueBubblesRecoveryTurnTimeoutError) {
1144
+ callbacks.onError(new Error("live iMessage turn timed out; I captured it for recovery instead of silently hanging"), "terminal");
1145
+ }
1146
+ throw error;
1147
+ }
1148
+ })();
1100
1149
  /* v8 ignore start -- failover display + error replay @preserve */
1101
1150
  if (result.failoverMessage) {
1102
1151
  // Failover handled it — show the failover message, skip the buffered error
@@ -1155,15 +1204,14 @@ async function handleBlueBubblesNormalizedEvent(event, resolvedDeps, source, opt
1155
1204
  bufferedTerminalError = null;
1156
1205
  }
1157
1206
  /* v8 ignore stop */
1158
- if (timeoutTimer !== null) {
1159
- clearTimeout(timeoutTimer);
1160
- timeoutTimer = null;
1161
- }
1207
+ clearTimeout(timeoutTimer);
1162
1208
  await callbacks.finish();
1163
1209
  }
1164
1210
  });
1165
1211
  }
1166
1212
  finally {
1213
+ if (activeTurnId)
1214
+ (0, active_turns_1.finishBlueBubblesActiveTurn)(agentName, activeTurnId);
1167
1215
  if (ownsInFlightMessage && event.kind === "message" && !releaseInFlightAfterTurnSettles) {
1168
1216
  endBlueBubblesMessageInFlight(event.chat.sessionKey, event.messageGuid);
1169
1217
  }
@@ -1292,7 +1340,12 @@ function resolveBlueBubblesCatchUpSince(previousState, nowMs = Date.now()) {
1292
1340
  }
1293
1341
  return nowMs - BLUEBUBBLES_FIRST_CATCHUP_LOOKBACK_MS;
1294
1342
  }
1295
- function formatBlueBubblesRuntimeDetail(queued, failed) {
1343
+ function formatBlueBubblesRuntimeDetail(queued, failed, active) {
1344
+ if (active.stalledTurnCount > 0) {
1345
+ return `iMessage live turn appears stalled; ${active.stalledTurnCount} active turn(s) older than ${BLUEBUBBLES_LIVE_TURN_STALLED_MS}ms`;
1346
+ }
1347
+ if (active.activeTurnCount > 0)
1348
+ return `upstream reachable; ${active.activeTurnCount} live turn(s) active`;
1296
1349
  if (queued > 0)
1297
1350
  return `upstream reachable but iMessage is not caught up; ${queued} recovery item(s) queued`;
1298
1351
  if (failed > 0)
@@ -1322,12 +1375,14 @@ async function syncBlueBubblesRuntime(deps = {}) {
1322
1375
  try {
1323
1376
  await client.checkHealth();
1324
1377
  const pendingBeforeCatchup = blueBubblesPendingRecoverySnapshot(agentName);
1378
+ const activeBeforeCatchup = (0, active_turns_1.snapshotBlueBubblesActiveTurns)(agentName, BLUEBUBBLES_LIVE_TURN_STALLED_MS);
1325
1379
  (0, runtime_state_1.writeBlueBubblesRuntimeState)(agentName, {
1326
1380
  upstreamStatus: "ok",
1327
1381
  detail: "upstream reachable; recovery pass running",
1328
1382
  lastCheckedAt: checkedAt,
1329
1383
  proofMethod: "bluebubbles.checkHealth",
1330
1384
  ...pendingBeforeCatchup,
1385
+ ...activeBeforeCatchup,
1331
1386
  lastRecoveredAt: previousState.lastRecoveredAt,
1332
1387
  lastRecoveredMessageGuid: previousState.lastRecoveredMessageGuid,
1333
1388
  });
@@ -1336,6 +1391,7 @@ async function syncBlueBubblesRuntime(deps = {}) {
1336
1391
  });
1337
1392
  const failed = catchUp.failed;
1338
1393
  const pendingAfterCatchup = blueBubblesPendingRecoverySnapshot(agentName);
1394
+ const activeAfterCatchup = (0, active_turns_1.snapshotBlueBubblesActiveTurns)(agentName, BLUEBUBBLES_LIVE_TURN_STALLED_MS);
1339
1395
  const queued = pendingAfterCatchup.pendingRecoveryCount;
1340
1396
  // upstreamStatus reflects whether BlueBubbles itself and the local bridge
1341
1397
  // can answer webhook traffic. The daemon status layer treats
@@ -1343,10 +1399,11 @@ async function syncBlueBubblesRuntime(deps = {}) {
1343
1399
  // while this field stays scoped to upstream transport reachability.
1344
1400
  (0, runtime_state_1.writeBlueBubblesRuntimeState)(agentName, {
1345
1401
  upstreamStatus: "ok",
1346
- detail: formatBlueBubblesRuntimeDetail(queued, failed),
1402
+ detail: formatBlueBubblesRuntimeDetail(queued, failed, activeAfterCatchup),
1347
1403
  lastCheckedAt: checkedAt,
1348
1404
  proofMethod: "bluebubbles.checkHealth",
1349
1405
  ...pendingAfterCatchup,
1406
+ ...activeAfterCatchup,
1350
1407
  pendingRecoveryCount: queued,
1351
1408
  failedRecoveryCount: failed,
1352
1409
  lastRecoveredAt: previousState.lastRecoveredAt,
@@ -1360,6 +1417,7 @@ async function syncBlueBubblesRuntime(deps = {}) {
1360
1417
  lastCheckedAt: checkedAt,
1361
1418
  proofMethod: "bluebubbles.checkHealth",
1362
1419
  ...blueBubblesPendingRecoverySnapshot(agentName),
1420
+ ...(0, active_turns_1.snapshotBlueBubblesActiveTurns)(agentName, BLUEBUBBLES_LIVE_TURN_STALLED_MS),
1363
1421
  failedRecoveryCount: 0,
1364
1422
  });
1365
1423
  }
@@ -1382,12 +1440,14 @@ async function recoverQueuedBlueBubblesMessages(deps = {}) {
1382
1440
  const checkedAt = new Date().toISOString();
1383
1441
  try {
1384
1442
  await resolvedDeps.createClient().checkHealth();
1443
+ const activeSnapshot = (0, active_turns_1.snapshotBlueBubblesActiveTurns)(agentName, BLUEBUBBLES_LIVE_TURN_STALLED_MS);
1385
1444
  (0, runtime_state_1.writeBlueBubblesRuntimeState)(agentName, {
1386
1445
  upstreamStatus: "ok",
1387
- detail: formatBlueBubblesRuntimeDetail(pendingRecoveryCount, failed),
1446
+ detail: formatBlueBubblesRuntimeDetail(pendingRecoveryCount, failed, activeSnapshot),
1388
1447
  lastCheckedAt: checkedAt,
1389
1448
  proofMethod: "bluebubbles.checkHealth",
1390
1449
  ...pendingSnapshot,
1450
+ ...activeSnapshot,
1391
1451
  failedRecoveryCount: failed,
1392
1452
  lastRecoveredAt: recovered > 0 ? checkedAt : previousState.lastRecoveredAt,
1393
1453
  lastRecoveredMessageGuid: previousState.lastRecoveredMessageGuid,
@@ -1400,6 +1460,7 @@ async function recoverQueuedBlueBubblesMessages(deps = {}) {
1400
1460
  lastCheckedAt: checkedAt,
1401
1461
  proofMethod: "bluebubbles.checkHealth",
1402
1462
  ...pendingSnapshot,
1463
+ ...(0, active_turns_1.snapshotBlueBubblesActiveTurns)(agentName, BLUEBUBBLES_LIVE_TURN_STALLED_MS),
1403
1464
  failedRecoveryCount: failed,
1404
1465
  lastRecoveredAt: recovered > 0 ? checkedAt : previousState.lastRecoveredAt,
1405
1466
  lastRecoveredMessageGuid: previousState.lastRecoveredMessageGuid,
@@ -4,7 +4,9 @@ exports.replayBlueBubblesMessage = replayBlueBubblesMessage;
4
4
  exports.formatBlueBubblesReplayText = formatBlueBubblesReplayText;
5
5
  const render_1 = require("../../heart/attachments/render");
6
6
  const bluebubbles_1 = require("../../heart/attachments/sources/bluebubbles");
7
+ const machine_identity_1 = require("../../heart/machine-identity");
7
8
  const identity_1 = require("../../heart/identity");
9
+ const runtime_credentials_1 = require("../../heart/runtime-credentials");
8
10
  const runtime_1 = require("../../nerves/runtime");
9
11
  const client_1 = require("./client");
10
12
  const model_1 = require("./model");
@@ -29,6 +31,8 @@ async function replayBlueBubblesMessage(params, deps = {}) {
29
31
  const setReplayAgentName = deps.setAgentName ?? identity_1.setAgentName;
30
32
  const resetReplayIdentity = deps.resetIdentity ?? identity_1.resetIdentity;
31
33
  const normalizeEvent = deps.normalizeEvent ?? model_1.normalizeBlueBubblesEvent;
34
+ const loadMachineId = deps.loadMachineId ?? (() => (0, machine_identity_1.loadOrCreateMachineIdentity)().machineId);
35
+ const refreshMachineRuntimeConfig = deps.refreshMachineRuntimeConfig ?? runtime_credentials_1.refreshMachineRuntimeCredentialConfig;
32
36
  (0, runtime_1.emitNervesEvent)({
33
37
  component: "senses",
34
38
  event: "senses.bluebubbles_replay_start",
@@ -41,6 +45,10 @@ async function replayBlueBubblesMessage(params, deps = {}) {
41
45
  });
42
46
  setReplayAgentName(agentName);
43
47
  try {
48
+ if (!deps.createClient) {
49
+ const machineId = loadMachineId();
50
+ await refreshMachineRuntimeConfig(agentName, machineId, { preserveCachedOnFailure: true });
51
+ }
44
52
  const client = deps.createClient ? deps.createClient() : (0, client_1.createBlueBubblesClient)();
45
53
  const probe = normalizeEvent({
46
54
  type: eventType,
@@ -77,6 +77,18 @@ function readBlueBubblesRuntimeState(agentName, agentRoot) {
77
77
  oldestPendingRecoveryAgeMs: typeof parsed.oldestPendingRecoveryAgeMs === "number" && Number.isFinite(parsed.oldestPendingRecoveryAgeMs)
78
78
  ? parsed.oldestPendingRecoveryAgeMs
79
79
  : undefined,
80
+ activeTurnCount: typeof parsed.activeTurnCount === "number" && Number.isFinite(parsed.activeTurnCount)
81
+ ? parsed.activeTurnCount
82
+ : undefined,
83
+ stalledTurnCount: typeof parsed.stalledTurnCount === "number" && Number.isFinite(parsed.stalledTurnCount)
84
+ ? parsed.stalledTurnCount
85
+ : undefined,
86
+ oldestActiveTurnStartedAt: typeof parsed.oldestActiveTurnStartedAt === "string"
87
+ ? parsed.oldestActiveTurnStartedAt
88
+ : undefined,
89
+ oldestActiveTurnAgeMs: typeof parsed.oldestActiveTurnAgeMs === "number" && Number.isFinite(parsed.oldestActiveTurnAgeMs)
90
+ ? parsed.oldestActiveTurnAgeMs
91
+ : undefined,
80
92
  lastRecoveredAt: typeof parsed.lastRecoveredAt === "string" ? parsed.lastRecoveredAt : undefined,
81
93
  lastRecoveredMessageGuid: typeof parsed.lastRecoveredMessageGuid === "string"
82
94
  ? parsed.lastRecoveredMessageGuid
@@ -116,6 +128,8 @@ function writeBlueBubblesRuntimeState(agentName, state, agentRoot) {
116
128
  upstreamStatus: state.upstreamStatus,
117
129
  pendingRecoveryCount: state.pendingRecoveryCount,
118
130
  failedRecoveryCount: state.failedRecoveryCount ?? 0,
131
+ activeTurnCount: state.activeTurnCount ?? 0,
132
+ stalledTurnCount: state.stalledTurnCount ?? 0,
119
133
  path: filePath,
120
134
  },
121
135
  });
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@ouro.bot/cli",
3
- "version": "0.1.0-alpha.543",
3
+ "version": "0.1.0-alpha.545",
4
4
  "main": "dist/heart/daemon/ouro-entry.js",
5
5
  "bin": {
6
6
  "cli": "dist/heart/daemon/ouro-bot-entry.js",