@ouro.bot/cli 0.1.0-alpha.321 → 0.1.0-alpha.323
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/changelog.json +14 -0
- package/dist/heart/daemon/bluebubbles-health-diagnostics.js +122 -0
- package/dist/heart/daemon/cli-exec.js +1 -0
- package/dist/heart/daemon/doctor.js +80 -1
- package/dist/senses/bluebubbles/client.js +91 -9
- package/dist/senses/bluebubbles/index.js +159 -5
- package/package.json +1 -1
package/changelog.json
CHANGED
|
@@ -1,6 +1,20 @@
|
|
|
1
1
|
{
|
|
2
2
|
"_note": "This changelog is maintained as part of the PR/version-bump workflow. Agent-curated, not auto-generated. Agents read this file directly via read_file to understand what changed between versions.",
|
|
3
3
|
"versions": [
|
|
4
|
+
{
|
|
5
|
+
"version": "0.1.0-alpha.323",
|
|
6
|
+
"changes": [
|
|
7
|
+
"fix(bluebubbles): drain missed upstream BlueBubbles messages after an outage by querying recent messages when the upstream health probe recovers, deduping against the inbound sidecar, repairing candidates, and replaying them through the normal BlueBubbles turn path oldest-first.",
|
|
8
|
+
"fix(bluebubbles): make upstream catch-up bounded, paginated, and observable with nerves events for query start/end/skip/error, catch-up start/complete/error, runtime status updates, and explicit failure state if the bounded page limit is reached before the outage window is drained."
|
|
9
|
+
]
|
|
10
|
+
},
|
|
11
|
+
{
|
|
12
|
+
"version": "0.1.0-alpha.322",
|
|
13
|
+
"changes": [
|
|
14
|
+
"fix(bluebubbles): share BlueBubbles health diagnostics between runtime status and doctor so fetch, malformed URL, auth, and upstream server failures produce actionable repair guidance instead of a bare `fetch failed`.",
|
|
15
|
+
"fix(doctor): actively probe enabled BlueBubbles upstreams, validate `bluebubbles.serverUrl` and `bluebubbles.password`, and report exactly whether config, reachability, auth, or upstream health needs repair."
|
|
16
|
+
]
|
|
17
|
+
},
|
|
4
18
|
{
|
|
5
19
|
"version": "0.1.0-alpha.321",
|
|
6
20
|
"changes": [
|
|
@@ -0,0 +1,122 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.stringifyBlueBubblesHealthError = stringifyBlueBubblesHealthError;
|
|
4
|
+
exports.redactBlueBubblesHealthDetailForNerves = redactBlueBubblesHealthDetailForNerves;
|
|
5
|
+
exports.formatBlueBubblesHealthcheckFailure = formatBlueBubblesHealthcheckFailure;
|
|
6
|
+
exports.probeBlueBubblesHealth = probeBlueBubblesHealth;
|
|
7
|
+
const runtime_1 = require("../../nerves/runtime");
|
|
8
|
+
const error_classification_1 = require("../providers/error-classification");
|
|
9
|
+
function buildBlueBubblesApiUrl(baseUrl, endpoint, password) {
|
|
10
|
+
const root = baseUrl.endsWith("/") ? baseUrl : `${baseUrl}/`;
|
|
11
|
+
const url = new URL(endpoint.replace(/^\//, ""), root);
|
|
12
|
+
url.searchParams.set("password", password);
|
|
13
|
+
return url.toString();
|
|
14
|
+
}
|
|
15
|
+
function stringifyBlueBubblesHealthError(error) {
|
|
16
|
+
if (error instanceof Error) {
|
|
17
|
+
const message = error.message.trim();
|
|
18
|
+
if (message)
|
|
19
|
+
return message;
|
|
20
|
+
return error.name || "unknown";
|
|
21
|
+
}
|
|
22
|
+
const value = String(error).trim();
|
|
23
|
+
return value || "unknown";
|
|
24
|
+
}
|
|
25
|
+
function redactBlueBubblesHealthDetailForNerves(detail) {
|
|
26
|
+
return detail
|
|
27
|
+
.replace(/\bbluebubbles\.password\b/gi, "bluebubbles credential")
|
|
28
|
+
.replace(/\bpassword\b/gi, "credential");
|
|
29
|
+
}
|
|
30
|
+
function blueBubblesHealthStatus(error) {
|
|
31
|
+
return error instanceof Error && typeof error.status === "number"
|
|
32
|
+
? error.status
|
|
33
|
+
: null;
|
|
34
|
+
}
|
|
35
|
+
function blueBubblesHealthClassification(error) {
|
|
36
|
+
return error instanceof Error ? (0, error_classification_1.classifyHttpError)(error) : "unknown";
|
|
37
|
+
}
|
|
38
|
+
function formatBlueBubblesHealthcheckFailure(serverUrlInput, error) {
|
|
39
|
+
const serverUrl = serverUrlInput.trim() || "configured BlueBubbles server";
|
|
40
|
+
const rawReason = stringifyBlueBubblesHealthError(error);
|
|
41
|
+
const status = blueBubblesHealthStatus(error);
|
|
42
|
+
if (!(error instanceof Error)) {
|
|
43
|
+
return `BlueBubbles health check failed at ${serverUrl}. Check \`bluebubbles.serverUrl\`, confirm the BlueBubbles app/API is running, and inspect daemon logs. Raw error: ${rawReason}`;
|
|
44
|
+
}
|
|
45
|
+
switch (blueBubblesHealthClassification(error)) {
|
|
46
|
+
case "network-error":
|
|
47
|
+
return `Cannot reach BlueBubbles at ${serverUrl}. Check \`bluebubbles.serverUrl\`, confirm the BlueBubbles app/API is running, and verify this machine can reach it. Raw error: ${rawReason}`;
|
|
48
|
+
case "auth-failure":
|
|
49
|
+
return `BlueBubbles auth failed at ${serverUrl} (HTTP ${status}). Check \`bluebubbles.password\` in secrets.json and confirm the server accepts it. Raw error: ${rawReason}`;
|
|
50
|
+
case "server-error":
|
|
51
|
+
return `BlueBubbles upstream returned HTTP ${status} at ${serverUrl}. Check the BlueBubbles app/server logs and confirm the upstream API is healthy. Raw error: ${rawReason}`;
|
|
52
|
+
default:
|
|
53
|
+
return `BlueBubbles health check failed at ${serverUrl}${status === null ? "" : ` (HTTP ${status})`}. Check \`bluebubbles.serverUrl\`, the BlueBubbles server configuration, and daemon logs. Raw error: ${rawReason}`;
|
|
54
|
+
}
|
|
55
|
+
}
|
|
56
|
+
async function probeBlueBubblesHealth(input) {
|
|
57
|
+
try {
|
|
58
|
+
const url = buildBlueBubblesApiUrl(input.serverUrl, "/api/v1/message/count", input.password);
|
|
59
|
+
const response = await input.fetchImpl(url, {
|
|
60
|
+
method: "GET",
|
|
61
|
+
signal: AbortSignal.timeout(input.requestTimeoutMs),
|
|
62
|
+
});
|
|
63
|
+
if (!response.ok) {
|
|
64
|
+
const errorText = await response.text().catch(() => "");
|
|
65
|
+
const error = new Error(errorText || "unknown");
|
|
66
|
+
error.status = response.status;
|
|
67
|
+
throw error;
|
|
68
|
+
}
|
|
69
|
+
(0, runtime_1.emitNervesEvent)({
|
|
70
|
+
component: "daemon",
|
|
71
|
+
event: "daemon.bluebubbles_health_probe_checked",
|
|
72
|
+
message: "checked bluebubbles upstream health",
|
|
73
|
+
meta: {
|
|
74
|
+
serverUrl: input.serverUrl,
|
|
75
|
+
ok: true,
|
|
76
|
+
status: response.status,
|
|
77
|
+
},
|
|
78
|
+
});
|
|
79
|
+
return {
|
|
80
|
+
ok: true,
|
|
81
|
+
detail: "upstream reachable",
|
|
82
|
+
reason: null,
|
|
83
|
+
status: response.status,
|
|
84
|
+
classification: null,
|
|
85
|
+
};
|
|
86
|
+
}
|
|
87
|
+
catch (error) {
|
|
88
|
+
const detail = formatBlueBubblesHealthcheckFailure(input.serverUrl, error);
|
|
89
|
+
const reason = stringifyBlueBubblesHealthError(error);
|
|
90
|
+
const status = blueBubblesHealthStatus(error);
|
|
91
|
+
const classification = blueBubblesHealthClassification(error);
|
|
92
|
+
(0, runtime_1.emitNervesEvent)({
|
|
93
|
+
level: "warn",
|
|
94
|
+
component: "daemon",
|
|
95
|
+
event: "daemon.bluebubbles_health_probe_checked",
|
|
96
|
+
message: "checked bluebubbles upstream health",
|
|
97
|
+
meta: {
|
|
98
|
+
serverUrl: input.serverUrl,
|
|
99
|
+
ok: false,
|
|
100
|
+
status,
|
|
101
|
+
reason,
|
|
102
|
+
classification,
|
|
103
|
+
detail: redactBlueBubblesHealthDetailForNerves(detail),
|
|
104
|
+
},
|
|
105
|
+
});
|
|
106
|
+
return {
|
|
107
|
+
ok: false,
|
|
108
|
+
detail,
|
|
109
|
+
reason,
|
|
110
|
+
status,
|
|
111
|
+
classification,
|
|
112
|
+
};
|
|
113
|
+
}
|
|
114
|
+
}
|
|
115
|
+
/* v8 ignore start -- module load observability event */
|
|
116
|
+
(0, runtime_1.emitNervesEvent)({
|
|
117
|
+
component: "daemon",
|
|
118
|
+
event: "daemon.bluebubbles_health_diagnostics_loaded",
|
|
119
|
+
message: "bluebubbles health diagnostics loaded",
|
|
120
|
+
meta: {},
|
|
121
|
+
});
|
|
122
|
+
/* v8 ignore stop */
|
|
@@ -2189,6 +2189,7 @@ async function runOuroCli(args, deps = (0, cli_defaults_1.createDefaultOuroCliDe
|
|
|
2189
2189
|
statSync: (p) => fs.statSync(p),
|
|
2190
2190
|
/* v8 ignore stop */
|
|
2191
2191
|
checkSocketAlive: deps.checkSocketAlive,
|
|
2192
|
+
fetchImpl: deps.fetchImpl ?? fetch,
|
|
2192
2193
|
socketPath: deps.socketPath,
|
|
2193
2194
|
bundlesRoot: deps.bundlesRoot ?? (0, identity_1.getAgentBundlesRoot)(),
|
|
2194
2195
|
secretsRoot: deps.secretsRoot ?? path.join(os.homedir(), ".agentsecrets"),
|
|
@@ -15,6 +15,8 @@ exports.checkSecurity = checkSecurity;
|
|
|
15
15
|
exports.checkDisk = checkDisk;
|
|
16
16
|
exports.runDoctorChecks = runDoctorChecks;
|
|
17
17
|
const runtime_1 = require("../../nerves/runtime");
|
|
18
|
+
const bluebubbles_health_diagnostics_1 = require("./bluebubbles-health-diagnostics");
|
|
19
|
+
const DEFAULT_BLUEBUBBLES_REQUEST_TIMEOUT_MS = 30_000;
|
|
18
20
|
// ── Category checkers ──
|
|
19
21
|
async function checkDaemon(deps) {
|
|
20
22
|
const checks = [];
|
|
@@ -47,6 +49,27 @@ function discoverAgents(deps) {
|
|
|
47
49
|
return [];
|
|
48
50
|
return deps.readdirSync(deps.bundlesRoot).filter((name) => name.endsWith(".ouro"));
|
|
49
51
|
}
|
|
52
|
+
function asRecord(value) {
|
|
53
|
+
return value && typeof value === "object" && !Array.isArray(value)
|
|
54
|
+
? value
|
|
55
|
+
: null;
|
|
56
|
+
}
|
|
57
|
+
function textField(record, key) {
|
|
58
|
+
const value = record?.[key];
|
|
59
|
+
return typeof value === "string" ? value.trim() : "";
|
|
60
|
+
}
|
|
61
|
+
function numberField(record, key, fallback) {
|
|
62
|
+
const value = record?.[key];
|
|
63
|
+
return typeof value === "number" && Number.isFinite(value) ? value : fallback;
|
|
64
|
+
}
|
|
65
|
+
function readJsonObject(deps, filePath) {
|
|
66
|
+
try {
|
|
67
|
+
return asRecord(JSON.parse(deps.readFileSync(filePath)));
|
|
68
|
+
}
|
|
69
|
+
catch {
|
|
70
|
+
return null;
|
|
71
|
+
}
|
|
72
|
+
}
|
|
50
73
|
function checkAgents(deps) {
|
|
51
74
|
const checks = [];
|
|
52
75
|
if (!deps.existsSync(deps.bundlesRoot)) {
|
|
@@ -105,10 +128,11 @@ function checkAgents(deps) {
|
|
|
105
128
|
}
|
|
106
129
|
return { name: "Agents", checks };
|
|
107
130
|
}
|
|
108
|
-
function checkSenses(deps) {
|
|
131
|
+
async function checkSenses(deps) {
|
|
109
132
|
const checks = [];
|
|
110
133
|
const agents = discoverAgents(deps);
|
|
111
134
|
for (const agentDir of agents) {
|
|
135
|
+
const agentName = agentDir.replace(/\.ouro$/, "");
|
|
112
136
|
const configPath = `${deps.bundlesRoot}/${agentDir}/agent.json`;
|
|
113
137
|
if (!deps.existsSync(configPath))
|
|
114
138
|
continue;
|
|
@@ -145,6 +169,61 @@ function checkSenses(deps) {
|
|
|
145
169
|
detail: senseObj.enabled ? "enabled" : "disabled",
|
|
146
170
|
});
|
|
147
171
|
}
|
|
172
|
+
if (sense === "bluebubbles" && senseObj.enabled === true) {
|
|
173
|
+
const secretsPath = `${deps.secretsRoot}/${agentName}/secrets.json`;
|
|
174
|
+
if (!deps.existsSync(secretsPath)) {
|
|
175
|
+
checks.push({
|
|
176
|
+
label: `${agentDir} bluebubbles config`,
|
|
177
|
+
status: "fail",
|
|
178
|
+
detail: "missing secrets.json",
|
|
179
|
+
});
|
|
180
|
+
continue;
|
|
181
|
+
}
|
|
182
|
+
const secrets = readJsonObject(deps, secretsPath);
|
|
183
|
+
if (!secrets) {
|
|
184
|
+
checks.push({
|
|
185
|
+
label: `${agentDir} bluebubbles config`,
|
|
186
|
+
status: "fail",
|
|
187
|
+
detail: "secrets.json unparseable",
|
|
188
|
+
});
|
|
189
|
+
continue;
|
|
190
|
+
}
|
|
191
|
+
const bluebubbles = asRecord(secrets.bluebubbles);
|
|
192
|
+
const bluebubblesChannel = asRecord(secrets.bluebubblesChannel);
|
|
193
|
+
const serverUrl = textField(bluebubbles, "serverUrl");
|
|
194
|
+
const password = textField(bluebubbles, "password");
|
|
195
|
+
const missing = [];
|
|
196
|
+
if (!serverUrl)
|
|
197
|
+
missing.push("bluebubbles.serverUrl");
|
|
198
|
+
if (!password)
|
|
199
|
+
missing.push("bluebubbles.password");
|
|
200
|
+
if (missing.length > 0) {
|
|
201
|
+
checks.push({
|
|
202
|
+
label: `${agentDir} bluebubbles config`,
|
|
203
|
+
status: "fail",
|
|
204
|
+
detail: `missing ${missing.join("/")}`,
|
|
205
|
+
});
|
|
206
|
+
continue;
|
|
207
|
+
}
|
|
208
|
+
checks.push({
|
|
209
|
+
label: `${agentDir} bluebubbles config`,
|
|
210
|
+
status: "pass",
|
|
211
|
+
detail: serverUrl,
|
|
212
|
+
});
|
|
213
|
+
if (deps.fetchImpl) {
|
|
214
|
+
const probe = await (0, bluebubbles_health_diagnostics_1.probeBlueBubblesHealth)({
|
|
215
|
+
serverUrl,
|
|
216
|
+
password,
|
|
217
|
+
requestTimeoutMs: numberField(bluebubblesChannel, "requestTimeoutMs", DEFAULT_BLUEBUBBLES_REQUEST_TIMEOUT_MS),
|
|
218
|
+
fetchImpl: deps.fetchImpl,
|
|
219
|
+
});
|
|
220
|
+
checks.push({
|
|
221
|
+
label: `${agentDir} bluebubbles upstream`,
|
|
222
|
+
status: probe.ok ? "pass" : "fail",
|
|
223
|
+
detail: probe.detail,
|
|
224
|
+
});
|
|
225
|
+
}
|
|
226
|
+
}
|
|
148
227
|
}
|
|
149
228
|
}
|
|
150
229
|
if (checks.length === 0) {
|
|
@@ -4,6 +4,7 @@ exports.createBlueBubblesClient = createBlueBubblesClient;
|
|
|
4
4
|
const node_crypto_1 = require("node:crypto");
|
|
5
5
|
const config_1 = require("../../heart/config");
|
|
6
6
|
const identity_1 = require("../../heart/identity");
|
|
7
|
+
const bluebubbles_health_diagnostics_1 = require("../../heart/daemon/bluebubbles-health-diagnostics");
|
|
7
8
|
const runtime_1 = require("../../nerves/runtime");
|
|
8
9
|
const minimax_1 = require("../../heart/providers/minimax");
|
|
9
10
|
const minimax_vlm_1 = require("../../heart/providers/minimax-vlm");
|
|
@@ -57,6 +58,9 @@ async function parseJsonBody(response) {
|
|
|
57
58
|
return null;
|
|
58
59
|
}
|
|
59
60
|
}
|
|
61
|
+
function describeCaughtValue(error) {
|
|
62
|
+
return error instanceof Error ? error.message : String(error);
|
|
63
|
+
}
|
|
60
64
|
function buildRepairUrl(baseUrl, messageGuid, password) {
|
|
61
65
|
const url = buildBlueBubblesApiUrl(baseUrl, `/api/v1/message/${encodeURIComponent(messageGuid)}`, password);
|
|
62
66
|
const parsed = new URL(url);
|
|
@@ -101,6 +105,17 @@ function extractChatQueryRows(payload) {
|
|
|
101
105
|
}
|
|
102
106
|
return data.map((entry) => asRecord(entry)).filter((entry) => entry !== null);
|
|
103
107
|
}
|
|
108
|
+
function extractMessageQueryRows(payload) {
|
|
109
|
+
const record = asRecord(payload);
|
|
110
|
+
const data = asRecord(record?.data);
|
|
111
|
+
const rows = Array.isArray(record?.data) ? record.data
|
|
112
|
+
: Array.isArray(data?.messages) ? data.messages
|
|
113
|
+
: Array.isArray(data?.results) ? data.results
|
|
114
|
+
: Array.isArray(record?.messages) ? record.messages
|
|
115
|
+
: Array.isArray(payload) ? payload
|
|
116
|
+
: [];
|
|
117
|
+
return rows.map((entry) => asRecord(entry)).filter((entry) => entry !== null);
|
|
118
|
+
}
|
|
104
119
|
async function resolveChatGuidForIdentifier(config, channelConfig, chatIdentifier) {
|
|
105
120
|
const trimmedIdentifier = chatIdentifier.trim();
|
|
106
121
|
if (!trimmedIdentifier)
|
|
@@ -332,19 +347,19 @@ function createBlueBubblesClient(config = (0, config_1.getBlueBubblesConfig)(),
|
|
|
332
347
|
}
|
|
333
348
|
},
|
|
334
349
|
async checkHealth() {
|
|
335
|
-
const url = buildBlueBubblesApiUrl(config.serverUrl, "/api/v1/message/count", config.password);
|
|
336
350
|
(0, runtime_1.emitNervesEvent)({
|
|
337
351
|
component: "senses",
|
|
338
352
|
event: "senses.bluebubbles_healthcheck_start",
|
|
339
353
|
message: "probing bluebubbles upstream health",
|
|
340
354
|
meta: { serverUrl: config.serverUrl },
|
|
341
355
|
});
|
|
342
|
-
const
|
|
343
|
-
|
|
344
|
-
|
|
356
|
+
const result = await (0, bluebubbles_health_diagnostics_1.probeBlueBubblesHealth)({
|
|
357
|
+
serverUrl: config.serverUrl,
|
|
358
|
+
password: config.password,
|
|
359
|
+
requestTimeoutMs: channelConfig.requestTimeoutMs,
|
|
360
|
+
fetchImpl: fetch,
|
|
345
361
|
});
|
|
346
|
-
if (!
|
|
347
|
-
const errorText = await response.text().catch(() => "");
|
|
362
|
+
if (!result.ok) {
|
|
348
363
|
(0, runtime_1.emitNervesEvent)({
|
|
349
364
|
level: "warn",
|
|
350
365
|
component: "senses",
|
|
@@ -352,11 +367,13 @@ function createBlueBubblesClient(config = (0, config_1.getBlueBubblesConfig)(),
|
|
|
352
367
|
message: "bluebubbles upstream health probe failed",
|
|
353
368
|
meta: {
|
|
354
369
|
serverUrl: config.serverUrl,
|
|
355
|
-
status:
|
|
356
|
-
reason:
|
|
370
|
+
status: result.status,
|
|
371
|
+
reason: result.reason,
|
|
372
|
+
classification: result.classification,
|
|
373
|
+
detail: (0, bluebubbles_health_diagnostics_1.redactBlueBubblesHealthDetailForNerves)(result.detail),
|
|
357
374
|
},
|
|
358
375
|
});
|
|
359
|
-
throw new Error(
|
|
376
|
+
throw new Error(result.detail);
|
|
360
377
|
}
|
|
361
378
|
(0, runtime_1.emitNervesEvent)({
|
|
362
379
|
component: "senses",
|
|
@@ -365,6 +382,71 @@ function createBlueBubblesClient(config = (0, config_1.getBlueBubblesConfig)(),
|
|
|
365
382
|
meta: { serverUrl: config.serverUrl },
|
|
366
383
|
});
|
|
367
384
|
},
|
|
385
|
+
async listRecentMessages(params = {}) {
|
|
386
|
+
const limit = Math.max(1, Math.min(100, Math.floor(params.limit ?? 50)));
|
|
387
|
+
const offset = Math.max(0, Math.floor(params.offset ?? 0));
|
|
388
|
+
const url = buildBlueBubblesApiUrl(config.serverUrl, "/api/v1/message/query", config.password);
|
|
389
|
+
(0, runtime_1.emitNervesEvent)({
|
|
390
|
+
component: "senses",
|
|
391
|
+
event: "senses.bluebubbles_query_recent_start",
|
|
392
|
+
message: "querying recent bluebubbles messages",
|
|
393
|
+
meta: { limit, offset },
|
|
394
|
+
});
|
|
395
|
+
const response = await fetch(url, {
|
|
396
|
+
method: "POST",
|
|
397
|
+
headers: { "Content-Type": "application/json" },
|
|
398
|
+
body: JSON.stringify({
|
|
399
|
+
limit,
|
|
400
|
+
offset,
|
|
401
|
+
sort: "DESC",
|
|
402
|
+
with: ["chats", "attachments", "payloadData", "messageSummaryInfo"],
|
|
403
|
+
}),
|
|
404
|
+
signal: AbortSignal.timeout(channelConfig.requestTimeoutMs),
|
|
405
|
+
});
|
|
406
|
+
if (!response.ok) {
|
|
407
|
+
const errorText = await response.text().catch(() => "");
|
|
408
|
+
(0, runtime_1.emitNervesEvent)({
|
|
409
|
+
level: "warn",
|
|
410
|
+
component: "senses",
|
|
411
|
+
event: "senses.bluebubbles_query_recent_error",
|
|
412
|
+
message: "bluebubbles recent message query failed",
|
|
413
|
+
meta: {
|
|
414
|
+
status: response.status,
|
|
415
|
+
reason: errorText || "unknown",
|
|
416
|
+
},
|
|
417
|
+
});
|
|
418
|
+
throw new Error(`BlueBubbles recent message query failed (${response.status}): ${errorText || "unknown"}`);
|
|
419
|
+
}
|
|
420
|
+
const payload = await parseJsonBody(response);
|
|
421
|
+
const rows = extractMessageQueryRows(payload);
|
|
422
|
+
const messages = [];
|
|
423
|
+
for (const row of rows) {
|
|
424
|
+
try {
|
|
425
|
+
messages.push((0, model_1.normalizeBlueBubblesEvent)({ type: "new-message", data: row }));
|
|
426
|
+
}
|
|
427
|
+
catch (error) {
|
|
428
|
+
(0, runtime_1.emitNervesEvent)({
|
|
429
|
+
level: "warn",
|
|
430
|
+
component: "senses",
|
|
431
|
+
event: "senses.bluebubbles_query_recent_skip",
|
|
432
|
+
message: "skipped unusable bluebubbles recent message row",
|
|
433
|
+
meta: {
|
|
434
|
+
reason: describeCaughtValue(error),
|
|
435
|
+
},
|
|
436
|
+
});
|
|
437
|
+
}
|
|
438
|
+
}
|
|
439
|
+
(0, runtime_1.emitNervesEvent)({
|
|
440
|
+
component: "senses",
|
|
441
|
+
event: "senses.bluebubbles_query_recent_end",
|
|
442
|
+
message: "queried recent bluebubbles messages",
|
|
443
|
+
meta: {
|
|
444
|
+
rows: rows.length,
|
|
445
|
+
normalized: messages.length,
|
|
446
|
+
},
|
|
447
|
+
});
|
|
448
|
+
return messages;
|
|
449
|
+
},
|
|
368
450
|
async repairEvent(event) {
|
|
369
451
|
if (!event.requiresRepair) {
|
|
370
452
|
(0, runtime_1.emitNervesEvent)({
|
|
@@ -36,6 +36,7 @@ Object.defineProperty(exports, "__esModule", { value: true });
|
|
|
36
36
|
exports.enrichReactionText = enrichReactionText;
|
|
37
37
|
exports.createStatusBatcher = createStatusBatcher;
|
|
38
38
|
exports.handleBlueBubblesEvent = handleBlueBubblesEvent;
|
|
39
|
+
exports.catchUpMissedBlueBubblesMessages = catchUpMissedBlueBubblesMessages;
|
|
39
40
|
exports.recoverMissedBlueBubblesMessages = recoverMissedBlueBubblesMessages;
|
|
40
41
|
exports.createBlueBubblesWebhookHandler = createBlueBubblesWebhookHandler;
|
|
41
42
|
exports.sendProactiveBlueBubblesMessageToSession = sendProactiveBlueBubblesMessageToSession;
|
|
@@ -135,6 +136,11 @@ const defaultDeps = {
|
|
|
135
136
|
createServer: http.createServer,
|
|
136
137
|
};
|
|
137
138
|
const BLUEBUBBLES_RUNTIME_SYNC_INTERVAL_MS = 30_000;
|
|
139
|
+
const BLUEBUBBLES_CATCHUP_PAGE_SIZE = 50;
|
|
140
|
+
const BLUEBUBBLES_CATCHUP_MAX_PAGES = 20;
|
|
141
|
+
const BLUEBUBBLES_HEALTHY_CATCHUP_OVERLAP_MS = 90_000;
|
|
142
|
+
const BLUEBUBBLES_RECOVERY_CATCHUP_LOOKBACK_MS = 24 * 60 * 60 * 1000;
|
|
143
|
+
const BLUEBUBBLES_FIRST_CATCHUP_LOOKBACK_MS = 10 * 60 * 1000;
|
|
138
144
|
function resolveFriendParams(event) {
|
|
139
145
|
if (event.chat.isGroup) {
|
|
140
146
|
const groupKey = event.chat.chatGuid ?? event.chat.chatIdentifier ?? event.sender.externalId;
|
|
@@ -921,24 +927,50 @@ function countPendingRecoveryCandidates(agentName) {
|
|
|
921
927
|
.filter((entry) => !(0, inbound_log_1.hasRecordedBlueBubblesInbound)(agentName, entry.sessionKey, entry.messageGuid))
|
|
922
928
|
.length;
|
|
923
929
|
}
|
|
930
|
+
function parseTimestampMs(value) {
|
|
931
|
+
if (!value)
|
|
932
|
+
return null;
|
|
933
|
+
const parsed = Date.parse(value);
|
|
934
|
+
return Number.isFinite(parsed) ? parsed : null;
|
|
935
|
+
}
|
|
936
|
+
function resolveBlueBubblesCatchUpSince(previousState, nowMs = Date.now()) {
|
|
937
|
+
if (previousState.upstreamStatus === "error") {
|
|
938
|
+
return nowMs - BLUEBUBBLES_RECOVERY_CATCHUP_LOOKBACK_MS;
|
|
939
|
+
}
|
|
940
|
+
const lastCheckedAt = parseTimestampMs(previousState.lastCheckedAt);
|
|
941
|
+
if (lastCheckedAt !== null) {
|
|
942
|
+
return Math.max(0, lastCheckedAt - BLUEBUBBLES_HEALTHY_CATCHUP_OVERLAP_MS);
|
|
943
|
+
}
|
|
944
|
+
return nowMs - BLUEBUBBLES_FIRST_CATCHUP_LOOKBACK_MS;
|
|
945
|
+
}
|
|
946
|
+
function formatRecoveredCount(count) {
|
|
947
|
+
return `caught up ${count} missed message(s)`;
|
|
948
|
+
}
|
|
924
949
|
async function syncBlueBubblesRuntime(deps = {}) {
|
|
925
950
|
const resolvedDeps = { ...defaultDeps, ...deps };
|
|
926
951
|
const agentName = resolvedDeps.getAgentName();
|
|
927
952
|
const client = resolvedDeps.createClient();
|
|
928
953
|
const checkedAt = new Date().toISOString();
|
|
954
|
+
const previousState = (0, runtime_state_1.readBlueBubblesRuntimeState)(agentName);
|
|
929
955
|
try {
|
|
930
956
|
await client.checkHealth();
|
|
931
957
|
const recovery = await recoverMissedBlueBubblesMessages(resolvedDeps);
|
|
958
|
+
const catchUp = await catchUpMissedBlueBubblesMessages(resolvedDeps, previousState);
|
|
959
|
+
const failed = recovery.failed + catchUp.failed;
|
|
960
|
+
const recovered = recovery.recovered + catchUp.recovered;
|
|
932
961
|
(0, runtime_state_1.writeBlueBubblesRuntimeState)(agentName, {
|
|
933
|
-
upstreamStatus: recovery.pending > 0 ||
|
|
934
|
-
detail:
|
|
935
|
-
? `recovery failures: ${
|
|
962
|
+
upstreamStatus: recovery.pending > 0 || failed > 0 ? "error" : "ok",
|
|
963
|
+
detail: failed > 0
|
|
964
|
+
? `recovery failures: ${failed}`
|
|
936
965
|
: recovery.pending > 0
|
|
937
966
|
? `pending recovery: ${recovery.pending}`
|
|
938
|
-
:
|
|
967
|
+
: catchUp.recovered > 0
|
|
968
|
+
? formatRecoveredCount(catchUp.recovered)
|
|
969
|
+
: "upstream reachable",
|
|
939
970
|
lastCheckedAt: checkedAt,
|
|
940
971
|
pendingRecoveryCount: recovery.pending,
|
|
941
|
-
lastRecoveredAt:
|
|
972
|
+
lastRecoveredAt: recovered > 0 ? checkedAt : previousState.lastRecoveredAt,
|
|
973
|
+
lastRecoveredMessageGuid: catchUp.lastRecoveredMessageGuid ?? previousState.lastRecoveredMessageGuid,
|
|
942
974
|
});
|
|
943
975
|
}
|
|
944
976
|
catch (error) {
|
|
@@ -950,6 +982,128 @@ async function syncBlueBubblesRuntime(deps = {}) {
|
|
|
950
982
|
});
|
|
951
983
|
}
|
|
952
984
|
}
|
|
985
|
+
async function catchUpMissedBlueBubblesMessages(deps = {}, previousState) {
|
|
986
|
+
const resolvedDeps = { ...defaultDeps, ...deps };
|
|
987
|
+
const agentName = resolvedDeps.getAgentName();
|
|
988
|
+
const client = resolvedDeps.createClient();
|
|
989
|
+
const result = { inspected: 0, recovered: 0, skipped: 0, failed: 0 };
|
|
990
|
+
const state = previousState ?? (0, runtime_state_1.readBlueBubblesRuntimeState)(agentName);
|
|
991
|
+
const catchUpSince = resolveBlueBubblesCatchUpSince(state);
|
|
992
|
+
/* v8 ignore next -- older injected test doubles may omit the catch-up query method */
|
|
993
|
+
if (!client.listRecentMessages)
|
|
994
|
+
return result;
|
|
995
|
+
(0, runtime_1.emitNervesEvent)({
|
|
996
|
+
component: "senses",
|
|
997
|
+
event: "senses.bluebubbles_catchup_start",
|
|
998
|
+
message: "bluebubbles upstream catch-up pass started",
|
|
999
|
+
meta: {
|
|
1000
|
+
since: new Date(catchUpSince).toISOString(),
|
|
1001
|
+
pageSize: BLUEBUBBLES_CATCHUP_PAGE_SIZE,
|
|
1002
|
+
maxPages: BLUEBUBBLES_CATCHUP_MAX_PAGES,
|
|
1003
|
+
},
|
|
1004
|
+
});
|
|
1005
|
+
const recentEvents = [];
|
|
1006
|
+
for (let page = 0; page < BLUEBUBBLES_CATCHUP_MAX_PAGES; page++) {
|
|
1007
|
+
let pageEvents;
|
|
1008
|
+
try {
|
|
1009
|
+
pageEvents = await client.listRecentMessages({
|
|
1010
|
+
limit: BLUEBUBBLES_CATCHUP_PAGE_SIZE,
|
|
1011
|
+
offset: page * BLUEBUBBLES_CATCHUP_PAGE_SIZE,
|
|
1012
|
+
});
|
|
1013
|
+
}
|
|
1014
|
+
catch (error) {
|
|
1015
|
+
result.failed++;
|
|
1016
|
+
(0, runtime_1.emitNervesEvent)({
|
|
1017
|
+
level: "warn",
|
|
1018
|
+
component: "senses",
|
|
1019
|
+
event: "senses.bluebubbles_catchup_error",
|
|
1020
|
+
message: "bluebubbles upstream catch-up query failed",
|
|
1021
|
+
meta: {
|
|
1022
|
+
offset: page * BLUEBUBBLES_CATCHUP_PAGE_SIZE,
|
|
1023
|
+
reason: error instanceof Error ? error.message : String(error),
|
|
1024
|
+
},
|
|
1025
|
+
});
|
|
1026
|
+
break;
|
|
1027
|
+
}
|
|
1028
|
+
recentEvents.push(...pageEvents);
|
|
1029
|
+
if (pageEvents.length < BLUEBUBBLES_CATCHUP_PAGE_SIZE)
|
|
1030
|
+
break;
|
|
1031
|
+
const oldestMessageTimestamp = pageEvents
|
|
1032
|
+
.filter((event) => event.kind === "message")
|
|
1033
|
+
.reduce((oldest, event) => Math.min(oldest, event.timestamp), Number.POSITIVE_INFINITY);
|
|
1034
|
+
if (oldestMessageTimestamp <= catchUpSince)
|
|
1035
|
+
break;
|
|
1036
|
+
if (page === BLUEBUBBLES_CATCHUP_MAX_PAGES - 1) {
|
|
1037
|
+
result.failed++;
|
|
1038
|
+
(0, runtime_1.emitNervesEvent)({
|
|
1039
|
+
level: "warn",
|
|
1040
|
+
component: "senses",
|
|
1041
|
+
event: "senses.bluebubbles_catchup_error",
|
|
1042
|
+
message: "bluebubbles upstream catch-up reached the bounded page limit",
|
|
1043
|
+
meta: {
|
|
1044
|
+
inspectedPages: BLUEBUBBLES_CATCHUP_MAX_PAGES,
|
|
1045
|
+
reason: "catch-up page limit reached before the outage window cutoff",
|
|
1046
|
+
},
|
|
1047
|
+
});
|
|
1048
|
+
}
|
|
1049
|
+
}
|
|
1050
|
+
const seenMessageGuids = new Set();
|
|
1051
|
+
const candidates = recentEvents
|
|
1052
|
+
.filter((event) => event.kind === "message")
|
|
1053
|
+
.filter((event) => {
|
|
1054
|
+
if (seenMessageGuids.has(event.messageGuid))
|
|
1055
|
+
return false;
|
|
1056
|
+
seenMessageGuids.add(event.messageGuid);
|
|
1057
|
+
return true;
|
|
1058
|
+
})
|
|
1059
|
+
.sort((left, right) => left.timestamp - right.timestamp);
|
|
1060
|
+
for (const event of candidates) {
|
|
1061
|
+
result.inspected++;
|
|
1062
|
+
if (event.fromMe
|
|
1063
|
+
|| event.timestamp < catchUpSince
|
|
1064
|
+
|| (0, inbound_log_1.hasRecordedBlueBubblesInbound)(agentName, event.chat.sessionKey, event.messageGuid)) {
|
|
1065
|
+
result.skipped++;
|
|
1066
|
+
continue;
|
|
1067
|
+
}
|
|
1068
|
+
try {
|
|
1069
|
+
const repaired = await client.repairEvent(event);
|
|
1070
|
+
if (repaired.kind !== "message") {
|
|
1071
|
+
result.skipped++;
|
|
1072
|
+
continue;
|
|
1073
|
+
}
|
|
1074
|
+
const handled = await handleBlueBubblesNormalizedEvent(repaired, resolvedDeps, "upstream-catchup");
|
|
1075
|
+
if (handled.reason === "already_processed") {
|
|
1076
|
+
result.skipped++;
|
|
1077
|
+
}
|
|
1078
|
+
else {
|
|
1079
|
+
result.recovered++;
|
|
1080
|
+
result.lastRecoveredMessageGuid = repaired.messageGuid;
|
|
1081
|
+
}
|
|
1082
|
+
}
|
|
1083
|
+
catch (error) {
|
|
1084
|
+
result.failed++;
|
|
1085
|
+
(0, runtime_1.emitNervesEvent)({
|
|
1086
|
+
level: "warn",
|
|
1087
|
+
component: "senses",
|
|
1088
|
+
event: "senses.bluebubbles_catchup_error",
|
|
1089
|
+
message: "bluebubbles upstream catch-up message failed",
|
|
1090
|
+
meta: {
|
|
1091
|
+
messageGuid: event.messageGuid,
|
|
1092
|
+
reason: error instanceof Error ? error.message : String(error),
|
|
1093
|
+
},
|
|
1094
|
+
});
|
|
1095
|
+
}
|
|
1096
|
+
}
|
|
1097
|
+
if (result.inspected > 0 || result.recovered > 0 || result.skipped > 0 || result.failed > 0) {
|
|
1098
|
+
(0, runtime_1.emitNervesEvent)({
|
|
1099
|
+
component: "senses",
|
|
1100
|
+
event: "senses.bluebubbles_catchup_complete",
|
|
1101
|
+
message: "bluebubbles upstream catch-up pass completed",
|
|
1102
|
+
meta: { ...result },
|
|
1103
|
+
});
|
|
1104
|
+
}
|
|
1105
|
+
return result;
|
|
1106
|
+
}
|
|
953
1107
|
async function recoverMissedBlueBubblesMessages(deps = {}) {
|
|
954
1108
|
const resolvedDeps = { ...defaultDeps, ...deps };
|
|
955
1109
|
const agentName = resolvedDeps.getAgentName();
|