@ouro.bot/cli 0.1.0-alpha.730 → 0.1.0-alpha.731
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +8 -0
- package/changelog.json +12 -0
- package/dist/heart/daemon/cli-exec.js +38 -4
- package/dist/heart/daemon/cli-help.js +9 -4
- package/dist/heart/daemon/cli-parse.js +30 -6
- package/dist/heart/daemon/doctor.js +277 -28
- package/dist/heart/daemon/freshness.js +4 -2
- package/dist/heart/daemon/logs-prune.js +26 -4
- package/dist/heart/daemon/prunable-bundle.js +144 -0
- package/dist/mailroom/blob-store.js +102 -6
- package/dist/mailroom/cache-sync-cli.js +54 -0
- package/dist/mailroom/hosted-cache-sync.js +249 -0
- package/dist/mailroom/reader.js +59 -0
- package/dist/mailroom/search-cache.js +76 -6
- package/dist/repertoire/guardrails.js +674 -0
- package/dist/repertoire/tools-mail.js +5 -152
- package/package.json +1 -1
|
@@ -35,6 +35,9 @@ const nerves_1 = require("../../nerves");
|
|
|
35
35
|
const config_1 = require("../../rsvp/config");
|
|
36
36
|
const cutover_1 = require("../../rsvp/cutover");
|
|
37
37
|
const diagnostics_1 = require("../../rsvp/diagnostics");
|
|
38
|
+
const hosted_cache_sync_1 = require("../../mailroom/hosted-cache-sync");
|
|
39
|
+
const search_cache_1 = require("../../mailroom/search-cache");
|
|
40
|
+
const prunable_bundle_1 = require("./prunable-bundle");
|
|
38
41
|
const DEFAULT_BLUEBUBBLES_REQUEST_TIMEOUT_MS = 30_000;
|
|
39
42
|
const DEFAULT_BLUEBUBBLES_PORT = 18_790;
|
|
40
43
|
const DEFAULT_BLUEBUBBLES_WEBHOOK_PATH = "/bluebubbles-webhook";
|
|
@@ -87,11 +90,11 @@ async function checkDaemon(deps) {
|
|
|
87
90
|
}
|
|
88
91
|
return { name: "Daemon", checks };
|
|
89
92
|
}
|
|
90
|
-
/** Discover
|
|
93
|
+
/** Discover doctor candidates: *.ouro directories with a present agent.json. */
|
|
91
94
|
function discoverAgents(deps) {
|
|
92
95
|
if (!deps.existsSync(deps.bundlesRoot))
|
|
93
96
|
return [];
|
|
94
|
-
return deps.readdirSync(deps.bundlesRoot).filter((name) => name.endsWith(".ouro"));
|
|
97
|
+
return deps.readdirSync(deps.bundlesRoot).filter((name) => name.endsWith(".ouro") && deps.existsSync(`${deps.bundlesRoot}/${name}/agent.json`));
|
|
95
98
|
}
|
|
96
99
|
function asRecord(value) {
|
|
97
100
|
return value && typeof value === "object" && !Array.isArray(value)
|
|
@@ -206,13 +209,17 @@ function senseFreshnessOverride(deps, agentDir, sense) {
|
|
|
206
209
|
* A sense opts in by calling this with its inbound log directory.
|
|
207
210
|
*/
|
|
208
211
|
function senseInboundDeliveryCheck(input) {
|
|
212
|
+
const result = senseInboundDeliveryResult(input);
|
|
213
|
+
return { id: `senses.${input.sense}.inbound_liveness`, label: `${input.agentDir} ${input.sense} inbound delivery`, status: result.status, detail: result.detail };
|
|
214
|
+
}
|
|
215
|
+
function senseInboundDeliveryResult(input) {
|
|
209
216
|
const suffix = input.logSuffix ?? ".ndjson";
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
label: `${input.agentDir} ${input.sense} inbound delivery`,
|
|
217
|
+
const probe = (0, freshness_1.observeAppendPerEventStore)(input.inboundDir, input.deps, { suffix, maxEntries: SENSE_DELIVERY_MAX_LOG_SCAN });
|
|
218
|
+
return (0, freshness_1.evaluateFreshness)({
|
|
213
219
|
activity: `${input.sense} inbound delivery`,
|
|
214
220
|
unit: "inbound message",
|
|
215
|
-
|
|
221
|
+
observation: probe.observation,
|
|
222
|
+
provenance: probe.provenance,
|
|
216
223
|
thresholds: (0, freshness_1.resolveFreshnessThresholds)(exports.DEFAULT_SENSE_DELIVERY_THRESHOLDS, senseFreshnessOverride(input.deps, input.agentDir, input.sense)),
|
|
217
224
|
remediation: input.remediation,
|
|
218
225
|
configuredSinceMs: input.configuredSincePath ? pathMtimeMs(input.deps, input.configuredSincePath) : null,
|
|
@@ -220,6 +227,57 @@ function senseInboundDeliveryCheck(input) {
|
|
|
220
227
|
nowMs: input.nowMs,
|
|
221
228
|
});
|
|
222
229
|
}
|
|
230
|
+
const DEFINITIVE_BLUEBUBBLES_FAILURES = new Set([
|
|
231
|
+
"auth-failure",
|
|
232
|
+
"rate-limit",
|
|
233
|
+
"server-error",
|
|
234
|
+
"usage-limit",
|
|
235
|
+
]);
|
|
236
|
+
const DEFINITIVE_BLUEBUBBLES_WEBHOOK_STATES = new Set([
|
|
237
|
+
"missing",
|
|
238
|
+
"drifted",
|
|
239
|
+
"auth-failed",
|
|
240
|
+
"malformed",
|
|
241
|
+
"listener-not-ready",
|
|
242
|
+
]);
|
|
243
|
+
function blueBubblesUpstreamEvidence(probe) {
|
|
244
|
+
if (probe.ok)
|
|
245
|
+
return "healthy";
|
|
246
|
+
if (DEFINITIVE_BLUEBUBBLES_FAILURES.has(probe.classification)) {
|
|
247
|
+
return "definitive-fault";
|
|
248
|
+
}
|
|
249
|
+
return "unavailable";
|
|
250
|
+
}
|
|
251
|
+
function blueBubblesInboundDeliveryCheck(input, evidence) {
|
|
252
|
+
const result = senseInboundDeliveryResult(input);
|
|
253
|
+
const base = {
|
|
254
|
+
id: "senses.bluebubbles.inbound_liveness",
|
|
255
|
+
label: `${input.agentDir} bluebubbles inbound delivery`,
|
|
256
|
+
};
|
|
257
|
+
if (result.state !== "stale" && result.state !== "never") {
|
|
258
|
+
return { ...base, status: result.status, detail: result.detail };
|
|
259
|
+
}
|
|
260
|
+
const webhookDefinitive = DEFINITIVE_BLUEBUBBLES_WEBHOOK_STATES.has(evidence.webhook);
|
|
261
|
+
if (evidence.upstream === "definitive-fault" || webhookDefinitive) {
|
|
262
|
+
const corroboration = evidence.webhook === "missing"
|
|
263
|
+
? "missing owned webhook corroborates the silence"
|
|
264
|
+
: webhookDefinitive
|
|
265
|
+
? `${evidence.webhook} owned-webhook evidence corroborates the silence`
|
|
266
|
+
: "a definitive upstream fault corroborates the silence";
|
|
267
|
+
return { ...base, status: "fail", detail: `${corroboration}; ${result.detail}` };
|
|
268
|
+
}
|
|
269
|
+
const orientation = evidence.upstream === "healthy" && evidence.webhook === "exact"
|
|
270
|
+
? "no recent inbound was observed; upstream and exact owned webhook are healthy"
|
|
271
|
+
: evidence.upstream === "not-run"
|
|
272
|
+
? "no recent inbound was observed; infrastructure probes were not run, so delivery is unverified"
|
|
273
|
+
: "no recent inbound was observed; infrastructure probes were unavailable, so delivery is unverified";
|
|
274
|
+
const observed = result.detail.replace(/; fix: .*$/u, "");
|
|
275
|
+
return {
|
|
276
|
+
...base,
|
|
277
|
+
status: "warn",
|
|
278
|
+
detail: `${orientation}; quiet and delivery failure are indistinguishable without a new message; doctor did not send a test message; ${observed}`,
|
|
279
|
+
};
|
|
280
|
+
}
|
|
223
281
|
function checkAgents(deps) {
|
|
224
282
|
const checks = [];
|
|
225
283
|
if (!deps.existsSync(deps.bundlesRoot)) {
|
|
@@ -228,16 +286,12 @@ function checkAgents(deps) {
|
|
|
228
286
|
}
|
|
229
287
|
const agents = discoverAgents(deps);
|
|
230
288
|
if (agents.length === 0) {
|
|
231
|
-
checks.push({ label: "agent bundles", status: "warn", detail: "no
|
|
289
|
+
checks.push({ label: "agent bundles", status: "warn", detail: "no agent bundles found" });
|
|
232
290
|
return { name: "Agents", checks };
|
|
233
291
|
}
|
|
234
292
|
for (const agentDir of agents) {
|
|
235
293
|
const agentPath = `${deps.bundlesRoot}/${agentDir}`;
|
|
236
294
|
const configPath = `${agentPath}/agent.json`;
|
|
237
|
-
if (!deps.existsSync(configPath)) {
|
|
238
|
-
checks.push({ label: `${agentDir}/agent.json`, status: "fail", detail: "missing" });
|
|
239
|
-
continue;
|
|
240
|
-
}
|
|
241
295
|
let config;
|
|
242
296
|
try {
|
|
243
297
|
config = JSON.parse(deps.readFileSync(configPath));
|
|
@@ -414,6 +468,7 @@ async function checkSenses(deps) {
|
|
|
414
468
|
detail: serverUrl,
|
|
415
469
|
});
|
|
416
470
|
let upstreamContext = "upstream probe not run in this pass";
|
|
471
|
+
let infrastructure = { upstream: "not-run", webhook: null };
|
|
417
472
|
if (deps.fetchImpl) {
|
|
418
473
|
const requestTimeoutMs = numberField(bluebubblesChannel, "requestTimeoutMs", DEFAULT_BLUEBUBBLES_REQUEST_TIMEOUT_MS);
|
|
419
474
|
const probe = await (0, bluebubbles_health_diagnostics_1.probeBlueBubblesHealth)({
|
|
@@ -430,6 +485,7 @@ async function checkSenses(deps) {
|
|
|
430
485
|
upstreamContext = probe.ok
|
|
431
486
|
? "upstream probe reachable — which does not prove inbound delivery"
|
|
432
487
|
: "upstream probe failing";
|
|
488
|
+
infrastructure = { ...infrastructure, upstream: blueBubblesUpstreamEvidence(probe) };
|
|
433
489
|
const webhook = await (0, webhook_registration_1.inspectBlueBubblesWebhookRegistration)({
|
|
434
490
|
serverUrl,
|
|
435
491
|
password,
|
|
@@ -446,9 +502,10 @@ async function checkSenses(deps) {
|
|
|
446
502
|
status: webhook.ok ? "pass" : "fail",
|
|
447
503
|
detail: webhook.detail,
|
|
448
504
|
});
|
|
505
|
+
infrastructure = { ...infrastructure, webhook: webhook.state };
|
|
449
506
|
}
|
|
450
507
|
const bluebubblesStateRoot = `${deps.bundlesRoot}/${agentDir}/state/senses/bluebubbles`;
|
|
451
|
-
checks.push(
|
|
508
|
+
checks.push(blueBubblesInboundDeliveryCheck({
|
|
452
509
|
deps,
|
|
453
510
|
agentDir,
|
|
454
511
|
sense: "bluebubbles",
|
|
@@ -457,7 +514,7 @@ async function checkSenses(deps) {
|
|
|
457
514
|
context: upstreamContext,
|
|
458
515
|
remediation: "no iMessage is reaching this agent — confirm the BlueBubbles server's configured webhook URL/port matches the port this daemon is listening on (a stale webhook port is the known cause), then re-attach with `ouro connect bluebubbles --agent <agent>`",
|
|
459
516
|
nowMs: Date.now(),
|
|
460
|
-
}));
|
|
517
|
+
}, infrastructure));
|
|
461
518
|
}
|
|
462
519
|
if (sense === "mail" && senseObj.enabled === true) {
|
|
463
520
|
const runtimeConfig = await (0, runtime_credentials_1.refreshRuntimeCredentialConfig)(agentName, { preserveCachedOnFailure: true });
|
|
@@ -957,7 +1014,29 @@ async function checkMailroom(deps) {
|
|
|
957
1014
|
detail: `${mailboxes.length} mailbox${mailboxes.length === 1 ? "" : "es"}, ${sourceGrants.length} source grant${sourceGrants.length === 1 ? "" : "s"}, ${listing.count} message${listing.count === 1 ? "" : "s"} stored (cumulative — not a liveness signal)`,
|
|
958
1015
|
});
|
|
959
1016
|
/* v8 ignore stop */
|
|
960
|
-
|
|
1017
|
+
const store = await resolveMailStore(agentDir);
|
|
1018
|
+
if (store.hosted === null) {
|
|
1019
|
+
checks.push({
|
|
1020
|
+
id: "mail.ingest_liveness",
|
|
1021
|
+
label: `${agentDir} mail ingest liveness`,
|
|
1022
|
+
status: store.status,
|
|
1023
|
+
detail: `mail store mode is unverified because fresh runtime credentials are unavailable: ${store.detail}; local-file and hosted liveness were not inferred from stale cached configuration`,
|
|
1024
|
+
});
|
|
1025
|
+
checks.push({
|
|
1026
|
+
id: "mail.cache_authority",
|
|
1027
|
+
label: `${agentDir} hosted mail cache authority`,
|
|
1028
|
+
status: store.status,
|
|
1029
|
+
detail: `fresh runtime credentials are unavailable: ${store.detail}; hosted authority was not probed from stale cached coordinates`,
|
|
1030
|
+
});
|
|
1031
|
+
continue;
|
|
1032
|
+
}
|
|
1033
|
+
const authorityCheck = store.hosted && deps.observeHostedMailAuthority
|
|
1034
|
+
? await hostedMailCacheAuthorityCheck(deps, agentDir, store)
|
|
1035
|
+
: undefined;
|
|
1036
|
+
checks.push(mailIngestLivenessCheck(deps, agentDir, mailroomRoot, messagesDir, listing, store, authorityCheck));
|
|
1037
|
+
if (authorityCheck) {
|
|
1038
|
+
checks.push(authorityCheck);
|
|
1039
|
+
}
|
|
961
1040
|
}
|
|
962
1041
|
return { name: "Mailroom", checks };
|
|
963
1042
|
}
|
|
@@ -973,18 +1052,135 @@ function listJsonDocuments(deps, dir) {
|
|
|
973
1052
|
}
|
|
974
1053
|
async function resolveMailStore(agentDir) {
|
|
975
1054
|
const agentName = agentDir.replace(/\.ouro$/, "");
|
|
976
|
-
|
|
977
|
-
|
|
978
|
-
|
|
979
|
-
|
|
980
|
-
|
|
981
|
-
|
|
1055
|
+
const runtimeConfig = await (0, runtime_credentials_1.refreshRuntimeCredentialConfig)(agentName, { preserveCachedOnFailure: false });
|
|
1056
|
+
if (!runtimeConfig.ok) {
|
|
1057
|
+
return {
|
|
1058
|
+
hosted: null,
|
|
1059
|
+
status: runtimeConfig.reason === "unavailable" ? "warn" : "fail",
|
|
1060
|
+
detail: `${runtimeConfig.error} (${runtimeConfig.itemPath})`,
|
|
1061
|
+
};
|
|
1062
|
+
}
|
|
982
1063
|
const mailroom = asRecord(runtimeConfig.config.mailroom);
|
|
983
1064
|
const azureAccountUrl = textField(mailroom, "azureAccountUrl");
|
|
984
1065
|
if (!azureAccountUrl)
|
|
985
1066
|
return { hosted: false };
|
|
986
1067
|
const azureContainer = textField(mailroom, "azureContainer") || "mailroom";
|
|
987
|
-
|
|
1068
|
+
const privateKeys = asRecord(mailroom?.privateKeys);
|
|
1069
|
+
return {
|
|
1070
|
+
hosted: true,
|
|
1071
|
+
label: `hosted azure-blob ${azureAccountUrl}/${azureContainer}`,
|
|
1072
|
+
privateKeyIds: new Set(Object.entries(privateKeys ?? {})
|
|
1073
|
+
.filter(([, value]) => typeof value === "string" && value.trim().length > 0)
|
|
1074
|
+
.map(([keyId]) => keyId)),
|
|
1075
|
+
};
|
|
1076
|
+
}
|
|
1077
|
+
function parseCacheDocument(deps, filePath) {
|
|
1078
|
+
try {
|
|
1079
|
+
const parsed = JSON.parse(deps.readFileSync(filePath));
|
|
1080
|
+
return asRecord(parsed);
|
|
1081
|
+
}
|
|
1082
|
+
catch {
|
|
1083
|
+
return null;
|
|
1084
|
+
}
|
|
1085
|
+
}
|
|
1086
|
+
async function hostedMailCacheAuthorityCheck(deps, agentDir, store) {
|
|
1087
|
+
const agentName = agentDir.replace(/\.ouro$/, "");
|
|
1088
|
+
const base = { id: "mail.cache_authority", label: `${agentDir} hosted mail cache authority` };
|
|
1089
|
+
let probe;
|
|
1090
|
+
try {
|
|
1091
|
+
probe = await deps.observeHostedMailAuthority(agentName);
|
|
1092
|
+
}
|
|
1093
|
+
catch (error) {
|
|
1094
|
+
return {
|
|
1095
|
+
...base,
|
|
1096
|
+
status: "warn",
|
|
1097
|
+
detail: `hosted authority is temporarily unverified: ${error instanceof Error ? error.message : String(error)}; no local cache mutation was attempted`,
|
|
1098
|
+
};
|
|
1099
|
+
}
|
|
1100
|
+
if (!probe.ok) {
|
|
1101
|
+
return {
|
|
1102
|
+
...base,
|
|
1103
|
+
status: probe.definitive ? "fail" : "warn",
|
|
1104
|
+
detail: `hosted authority unavailable: ${probe.detail}; no local cache mutation was attempted`,
|
|
1105
|
+
};
|
|
1106
|
+
}
|
|
1107
|
+
const { observation } = probe;
|
|
1108
|
+
if (observation.parseFailureCount > 0 || observation.duplicateIds.length > 0) {
|
|
1109
|
+
return {
|
|
1110
|
+
...base,
|
|
1111
|
+
status: "warn",
|
|
1112
|
+
detail: `hosted authority is ambiguous (${observation.parseFailureCount} malformed index name(s), ${observation.duplicateIds.length} duplicate id(s)); no local cache mutation was attempted`,
|
|
1113
|
+
};
|
|
1114
|
+
}
|
|
1115
|
+
if (observation.records.length === 0) {
|
|
1116
|
+
return {
|
|
1117
|
+
...base,
|
|
1118
|
+
status: "warn",
|
|
1119
|
+
detail: "hosted authority is empty and remains unverified because an empty index namespace cannot prove the mailbox is empty; no local cache mutation was attempted",
|
|
1120
|
+
};
|
|
1121
|
+
}
|
|
1122
|
+
const cacheDir = `${deps.bundlesRoot}/${agentDir}/state/mail-search`;
|
|
1123
|
+
let fileNames = [];
|
|
1124
|
+
if (deps.existsSync(cacheDir)) {
|
|
1125
|
+
try {
|
|
1126
|
+
fileNames = deps.readdirSync(cacheDir).filter((name) => name.endsWith(".json"));
|
|
1127
|
+
}
|
|
1128
|
+
catch (error) {
|
|
1129
|
+
return {
|
|
1130
|
+
...base,
|
|
1131
|
+
status: "warn",
|
|
1132
|
+
detail: `hosted authority is sound but local cache is unreadable: ${error instanceof Error ? error.message : String(error)}; run \`ouro mail sync-cache --agent ${agentName}\``,
|
|
1133
|
+
};
|
|
1134
|
+
}
|
|
1135
|
+
}
|
|
1136
|
+
const recordsById = new Map(observation.records.map((record) => [record.id, record]));
|
|
1137
|
+
const documentsById = new Map();
|
|
1138
|
+
let malformedOrNoncanonical = 0;
|
|
1139
|
+
for (const fileName of fileNames) {
|
|
1140
|
+
const document = parseCacheDocument(deps, `${cacheDir}/${fileName}`);
|
|
1141
|
+
if (!document || document.agentId !== agentName || fileName !== `${document.messageId}.json` || !recordsById.has(document.messageId)) {
|
|
1142
|
+
malformedOrNoncanonical += 1;
|
|
1143
|
+
continue;
|
|
1144
|
+
}
|
|
1145
|
+
documentsById.set(document.messageId, document);
|
|
1146
|
+
}
|
|
1147
|
+
let mismatched = 0;
|
|
1148
|
+
let invalidKeyProvenance = 0;
|
|
1149
|
+
for (const record of observation.records) {
|
|
1150
|
+
const document = documentsById.get(record.id);
|
|
1151
|
+
if (!document || !(0, hosted_cache_sync_1.mailSearchCacheDocumentMatchesRecord)(document, record, true)) {
|
|
1152
|
+
mismatched += 1;
|
|
1153
|
+
continue;
|
|
1154
|
+
}
|
|
1155
|
+
if (!document.decryptionKeyId || !store.privateKeyIds.has(document.decryptionKeyId)) {
|
|
1156
|
+
invalidKeyProvenance += 1;
|
|
1157
|
+
}
|
|
1158
|
+
}
|
|
1159
|
+
const coverage = (0, search_cache_1.readMailSearchCoverageRecord)({
|
|
1160
|
+
agentId: agentName,
|
|
1161
|
+
storeKind: "azure-blob",
|
|
1162
|
+
}, { cacheDirForAgent: () => cacheDir });
|
|
1163
|
+
const expectedCachedCount = documentsById.size;
|
|
1164
|
+
const coverageMatches = coverage !== null
|
|
1165
|
+
&& coverage.textProjectionVersion === search_cache_1.MAIL_SEARCH_TEXT_PROJECTION_VERSION
|
|
1166
|
+
&& coverage.messageIndexFingerprint === observation.snapshot.messageIndexFingerprint
|
|
1167
|
+
&& coverage.visibleMessageCount === observation.snapshot.visibleMessageCount
|
|
1168
|
+
&& coverage.cachedMessageCount === expectedCachedCount
|
|
1169
|
+
&& coverage.decryptableMessageCount === expectedCachedCount
|
|
1170
|
+
&& coverage.skippedMessageCount === observation.snapshot.visibleMessageCount - expectedCachedCount;
|
|
1171
|
+
if (malformedOrNoncanonical > 0 || mismatched > 0 || invalidKeyProvenance > 0 || !coverageMatches) {
|
|
1172
|
+
return {
|
|
1173
|
+
...base,
|
|
1174
|
+
status: "warn",
|
|
1175
|
+
detail: `hosted authority is sound but cache diverges: ${mismatched} missing/stale canonical document(s), ${malformedOrNoncanonical} malformed/orphan/noncanonical/duplicate file(s), ${invalidKeyProvenance} invalid key provenance document(s), all-scope coverage ${coverageMatches ? "matches" : "missing/malformed/stale"}; run \`ouro mail sync-cache --agent ${agentName}\``,
|
|
1176
|
+
};
|
|
1177
|
+
}
|
|
1178
|
+
const count = observation.records.length;
|
|
1179
|
+
return {
|
|
1180
|
+
...base,
|
|
1181
|
+
status: "pass",
|
|
1182
|
+
detail: `${count} authoritative hosted message${count === 1 ? "" : "s"}; local search cache converged with current metadata, projection, and key provenance`,
|
|
1183
|
+
};
|
|
988
1184
|
}
|
|
989
1185
|
/**
|
|
990
1186
|
* Local-file Mailroom liveness signal.
|
|
@@ -1039,7 +1235,11 @@ function hostedMailMirrorProbe(deps, agentDir, storeLabel) {
|
|
|
1039
1235
|
provenance: `attempted directory listing of ${mirrorDir}`,
|
|
1040
1236
|
};
|
|
1041
1237
|
}
|
|
1042
|
-
return (0, freshness_1.observeMirroredStore)(mirrorDir, deps, {
|
|
1238
|
+
return (0, freshness_1.observeMirroredStore)(mirrorDir, deps, {
|
|
1239
|
+
hasEntries: listing.count > 0,
|
|
1240
|
+
remote: storeLabel,
|
|
1241
|
+
authorityReadNote: "which the separate hosted cache authority check reads without mutation; this liveness check does not use remote data for recency",
|
|
1242
|
+
});
|
|
1043
1243
|
}
|
|
1044
1244
|
/**
|
|
1045
1245
|
* Mail-ingest liveness — "is mail still arriving?", as opposed to the check
|
|
@@ -1053,8 +1253,24 @@ function hostedMailMirrorProbe(deps, agentDir, storeLabel) {
|
|
|
1053
1253
|
* activity wording and the remediation are all chosen per store kind. Reusing
|
|
1054
1254
|
* the local path in hosted mode measures a directory nothing writes to.
|
|
1055
1255
|
*/
|
|
1056
|
-
function mailIngestLivenessCheck(deps, agentDir, mailroomRoot, messagesDir, listing, store) {
|
|
1256
|
+
function mailIngestLivenessCheck(deps, agentDir, mailroomRoot, messagesDir, listing, store, authorityCheck) {
|
|
1057
1257
|
const hosted = store.hosted ? store : null;
|
|
1258
|
+
if (hosted && authorityCheck) {
|
|
1259
|
+
if (authorityCheck.status === "pass") {
|
|
1260
|
+
return {
|
|
1261
|
+
id: "mail.ingest_liveness",
|
|
1262
|
+
label: `${agentDir} mail ingest liveness`,
|
|
1263
|
+
status: "pass",
|
|
1264
|
+
detail: `hosted authority and local search cache are converged; this is healthy quiet even when the local mirror mtime is old or unchanged (${hosted.label}); mirror age is not treated as hosted-ingest authority`,
|
|
1265
|
+
};
|
|
1266
|
+
}
|
|
1267
|
+
return {
|
|
1268
|
+
id: "mail.ingest_liveness",
|
|
1269
|
+
label: `${agentDir} mail ingest liveness`,
|
|
1270
|
+
status: authorityCheck.status,
|
|
1271
|
+
detail: `hosted ingest liveness follows the available authority result rather than local mirror age: ${authorityCheck.detail}`,
|
|
1272
|
+
};
|
|
1273
|
+
}
|
|
1058
1274
|
const probe = hosted
|
|
1059
1275
|
? hostedMailMirrorProbe(deps, agentDir, hosted.label)
|
|
1060
1276
|
: localMailStoreProbe(deps, messagesDir, listing);
|
|
@@ -1066,7 +1282,7 @@ function mailIngestLivenessCheck(deps, agentDir, mailroomRoot, messagesDir, list
|
|
|
1066
1282
|
probe,
|
|
1067
1283
|
thresholds: (0, freshness_1.resolveFreshnessThresholds)(exports.DEFAULT_MAIL_INGEST_THRESHOLDS, senseFreshnessOverride(deps, agentDir, "mail")),
|
|
1068
1284
|
remediation: hosted
|
|
1069
|
-
? `
|
|
1285
|
+
? `this liveness check measures recency only through this machine's local mirror; the separate hosted cache authority check reads index names without mutation — if the mirror is empty or stale while authority is current, run \`ouro mail sync-cache --agent ${agentDir.replace(/\.ouro$/, "")}\` and verify the hosted reader on this machine`
|
|
1070
1286
|
: "mail is configured but nothing is arriving — re-check the mailbox grant and keyIds against the vault (a server-side key rotation silently orphans ingestion), run `ouro connect mail --agent <agent>`, and inspect the mailroom ingress logs",
|
|
1071
1287
|
configuredSinceMs: pathMtimeMs(deps, `${mailroomRoot}/registry.json`),
|
|
1072
1288
|
context: hosted ? `mailbox configured; ${hosted.label}` : "mailbox configured",
|
|
@@ -1160,7 +1376,7 @@ function checkDisk(deps) {
|
|
|
1160
1376
|
return !/\.\d+\.log$/.test(name);
|
|
1161
1377
|
return false;
|
|
1162
1378
|
};
|
|
1163
|
-
const addLogSizeCheck = (labelPrefix, logsDir) => {
|
|
1379
|
+
const addLogSizeCheck = (labelPrefix, logsDir, pruneCommand) => {
|
|
1164
1380
|
let totalSize = 0;
|
|
1165
1381
|
let activeSize = 0;
|
|
1166
1382
|
const oversizedActive = [];
|
|
@@ -1187,14 +1403,15 @@ function checkDisk(deps) {
|
|
|
1187
1403
|
}
|
|
1188
1404
|
const sizeMB = totalSize / (1024 * 1024);
|
|
1189
1405
|
const activeSizeMB = activeSize / (1024 * 1024);
|
|
1406
|
+
const repair = pruneCommand ? ` — run \`${pruneCommand}\`` : "";
|
|
1190
1407
|
if (activeSizeMB > 500) {
|
|
1191
|
-
checks.push({ label: `${labelPrefix} daemon log size`, status: "fail", detail: `${activeSizeMB.toFixed(1)}MB active / ${sizeMB.toFixed(1)}MB total — active logs exceed 500MB limit` });
|
|
1408
|
+
checks.push({ label: `${labelPrefix} daemon log size`, status: "fail", detail: `${activeSizeMB.toFixed(1)}MB active / ${sizeMB.toFixed(1)}MB total — active logs exceed 500MB limit${repair}` });
|
|
1192
1409
|
}
|
|
1193
1410
|
else if (oversizedActive.length > 0) {
|
|
1194
1411
|
checks.push({
|
|
1195
1412
|
label: `${labelPrefix} daemon log size`,
|
|
1196
1413
|
status: "warn",
|
|
1197
|
-
detail: `${activeSizeMB.toFixed(1)}MB active / ${sizeMB.toFixed(1)}MB total; active stream(s) over ${Math.round(nerves_1.DEFAULT_MAX_LOG_SIZE_BYTES / (1024 * 1024))}MB: ${oversizedActive.join(", ")}
|
|
1414
|
+
detail: `${activeSizeMB.toFixed(1)}MB active / ${sizeMB.toFixed(1)}MB total; active stream(s) over ${Math.round(nerves_1.DEFAULT_MAX_LOG_SIZE_BYTES / (1024 * 1024))}MB: ${oversizedActive.join(", ")}${repair}`,
|
|
1198
1415
|
});
|
|
1199
1416
|
}
|
|
1200
1417
|
else {
|
|
@@ -1207,11 +1424,43 @@ function checkDisk(deps) {
|
|
|
1207
1424
|
}
|
|
1208
1425
|
for (const agentDir of agents) {
|
|
1209
1426
|
const logsDir = `${deps.bundlesRoot}/${agentDir}/state/daemon/logs`;
|
|
1427
|
+
let pruneCommand;
|
|
1428
|
+
if (deps.lstatSync && deps.realpathSync) {
|
|
1429
|
+
const io = {
|
|
1430
|
+
existsSync: deps.existsSync,
|
|
1431
|
+
lstatSync: deps.lstatSync,
|
|
1432
|
+
readdirSync: deps.readdirSync,
|
|
1433
|
+
realpathSync: deps.realpathSync,
|
|
1434
|
+
};
|
|
1435
|
+
try {
|
|
1436
|
+
const agentName = agentDir.slice(0, -".ouro".length);
|
|
1437
|
+
const target = (0, prunable_bundle_1.resolvePrunableAgentBundle)({ bundlesRoot: deps.bundlesRoot, agentName, fs: io });
|
|
1438
|
+
if (!deps.existsSync(target.logsDir))
|
|
1439
|
+
throw new Error("logs directory does not exist");
|
|
1440
|
+
const logsIo = {
|
|
1441
|
+
lstatSync: (filePath) => {
|
|
1442
|
+
const stat = deps.lstatSync(filePath);
|
|
1443
|
+
return {
|
|
1444
|
+
isDirectory: stat.isDirectory,
|
|
1445
|
+
isFile: stat.isFile ?? (() => false),
|
|
1446
|
+
isSymbolicLink: stat.isSymbolicLink,
|
|
1447
|
+
};
|
|
1448
|
+
},
|
|
1449
|
+
readdirSync: deps.readdirSync,
|
|
1450
|
+
realpathSync: deps.realpathSync,
|
|
1451
|
+
};
|
|
1452
|
+
(0, prunable_bundle_1.validatePrunableLogsTarget)(target, logsIo);
|
|
1453
|
+
pruneCommand = `ouro logs prune --agent ${agentName}`;
|
|
1454
|
+
}
|
|
1455
|
+
catch {
|
|
1456
|
+
// Doctor reports size but only offers repair for an exactly validated target.
|
|
1457
|
+
}
|
|
1458
|
+
}
|
|
1210
1459
|
if (!deps.existsSync(logsDir)) {
|
|
1211
1460
|
checks.push({ label: `${agentDir} daemon logs dir`, status: "warn", detail: `${logsDir} not found` });
|
|
1212
1461
|
}
|
|
1213
1462
|
else {
|
|
1214
|
-
addLogSizeCheck(agentDir, logsDir);
|
|
1463
|
+
addLogSizeCheck(agentDir, logsDir, pruneCommand);
|
|
1215
1464
|
}
|
|
1216
1465
|
}
|
|
1217
1466
|
// Check AgentBundles root
|
|
@@ -235,13 +235,15 @@ function observeCreatePerEventStore(dir, deps, options) {
|
|
|
235
235
|
* a local directory that nothing writes to any more.
|
|
236
236
|
*/
|
|
237
237
|
function observeMirroredStore(dir, deps, options) {
|
|
238
|
-
const
|
|
238
|
+
const provenanceReadNote = options.authorityReadNote ?? "which doctor does not read";
|
|
239
|
+
const unknownReadNote = options.authorityReadNote ?? "is not read by doctor (no network calls, no credentials)";
|
|
240
|
+
const provenance = `derived from the local mirror at ${dir} (directory mtime, O(1), no per-entry scan); the authoritative store is ${options.remote}, ${provenanceReadNote}`;
|
|
239
241
|
const exists = deps.existsSync(dir);
|
|
240
242
|
if (!exists || !options.hasEntries) {
|
|
241
243
|
return {
|
|
242
244
|
observation: {
|
|
243
245
|
kind: "unknown",
|
|
244
|
-
reason: `the local mirror at ${dir} is ${exists ? "empty" : "absent"}, and ${options.remote}
|
|
246
|
+
reason: `the local mirror at ${dir} is ${exists ? "empty" : "absent"}, and ${options.remote} ${unknownReadNote}, so recency cannot be measured on this machine`,
|
|
245
247
|
},
|
|
246
248
|
provenance,
|
|
247
249
|
};
|
|
@@ -7,6 +7,17 @@ const crypto_1 = require("crypto");
|
|
|
7
7
|
const nerves_1 = require("../../nerves");
|
|
8
8
|
const runtime_1 = require("../../nerves/runtime");
|
|
9
9
|
const identity_1 = require("../identity");
|
|
10
|
+
const prunable_bundle_1 = require("./prunable-bundle");
|
|
11
|
+
function resolveLogsTarget(options) {
|
|
12
|
+
if (options.logsDir)
|
|
13
|
+
return { logsDir: options.logsDir };
|
|
14
|
+
if (!options.agentName) {
|
|
15
|
+
throw new Error("daemon logs prune requires an explicit agent or an internal logs directory");
|
|
16
|
+
}
|
|
17
|
+
const bundlesRoot = options.bundlesRoot ?? (0, identity_1.getAgentBundlesRoot)();
|
|
18
|
+
const target = (0, prunable_bundle_1.resolvePrunableAgentBundle)({ bundlesRoot, agentName: options.agentName });
|
|
19
|
+
return { logsDir: target.logsDir, target };
|
|
20
|
+
}
|
|
10
21
|
function isActiveLogStream(name) {
|
|
11
22
|
if (name.endsWith(".ndjson")) {
|
|
12
23
|
return !/\.\d+\.ndjson$/.test(name);
|
|
@@ -17,20 +28,26 @@ function isActiveLogStream(name) {
|
|
|
17
28
|
return false;
|
|
18
29
|
}
|
|
19
30
|
function pruneDaemonLogs(options = {}) {
|
|
20
|
-
/* v8 ignore next -- defensive: tests always pass logsDir to avoid prod paths @preserve */
|
|
21
|
-
const logsDir = options.logsDir ?? (0, identity_1.getAgentDaemonLogsDir)(options.agentName);
|
|
22
31
|
const maxSizeBytes = options.maxSizeBytes ?? nerves_1.DEFAULT_MAX_LOG_SIZE_BYTES;
|
|
23
32
|
const maxGenerations = options.maxGenerations ?? nerves_1.DEFAULT_MAX_GENERATIONS;
|
|
24
33
|
const traceId = (0, crypto_1.randomUUID)();
|
|
34
|
+
let logsDir = options.logsDir ?? "unresolved";
|
|
25
35
|
(0, runtime_1.emitNervesEvent)({
|
|
26
36
|
component: "nerves",
|
|
27
37
|
event: "nerves.logs_prune_start",
|
|
28
38
|
trace_id: traceId,
|
|
29
39
|
message: "pruning daemon logs",
|
|
30
|
-
meta: {
|
|
40
|
+
meta: {
|
|
41
|
+
logsDir: options.logsDir ?? null,
|
|
42
|
+
agentName: options.agentName ?? null,
|
|
43
|
+
maxSizeBytes,
|
|
44
|
+
maxGenerations,
|
|
45
|
+
},
|
|
31
46
|
});
|
|
32
47
|
let completed = false;
|
|
33
48
|
try {
|
|
49
|
+
const resolved = resolveLogsTarget(options);
|
|
50
|
+
logsDir = resolved.logsDir;
|
|
34
51
|
if (!(0, fs_1.existsSync)(logsDir)) {
|
|
35
52
|
completed = true;
|
|
36
53
|
(0, runtime_1.emitNervesEvent)({
|
|
@@ -42,6 +59,10 @@ function pruneDaemonLogs(options = {}) {
|
|
|
42
59
|
});
|
|
43
60
|
return { filesCompacted: 0, bytesFreed: 0 };
|
|
44
61
|
}
|
|
62
|
+
const inspection = resolved.target
|
|
63
|
+
? (0, prunable_bundle_1.validatePrunableLogsTarget)(resolved.target)
|
|
64
|
+
: (0, prunable_bundle_1.validatePrunableLogsDirectory)(logsDir);
|
|
65
|
+
const { entries, logsReal } = inspection;
|
|
45
66
|
let filesCompacted = 0;
|
|
46
67
|
let bytesFreed = 0;
|
|
47
68
|
// Enumerate and rotate each active structured stream plus legacy launchd
|
|
@@ -49,10 +70,11 @@ function pruneDaemonLogs(options = {}) {
|
|
|
49
70
|
// stream can be a rotation candidate. Legacy generation files are handled
|
|
50
71
|
// inside rotateIfNeeded's generation-shift step, so we skip them here to
|
|
51
72
|
// avoid double-rotating.
|
|
52
|
-
for (const name of
|
|
73
|
+
for (const name of entries) {
|
|
53
74
|
if (!isActiveLogStream(name))
|
|
54
75
|
continue;
|
|
55
76
|
const filePath = (0, path_1.join)(logsDir, name);
|
|
77
|
+
(0, prunable_bundle_1.validatePrunableLogEntry)(filePath, logsReal);
|
|
56
78
|
let sizeBefore;
|
|
57
79
|
try {
|
|
58
80
|
sizeBefore = (0, fs_1.statSync)(filePath).size;
|