@ouro.bot/cli 0.1.0-alpha.730 → 0.1.0-alpha.732
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +8 -0
- package/changelog.json +18 -0
- package/dist/heart/daemon/cli-exec.js +38 -4
- package/dist/heart/daemon/cli-help.js +9 -4
- package/dist/heart/daemon/cli-parse.js +30 -6
- package/dist/heart/daemon/doctor.js +294 -28
- package/dist/heart/daemon/freshness.js +4 -2
- package/dist/heart/daemon/logs-prune.js +26 -4
- package/dist/heart/daemon/prunable-bundle.js +144 -0
- package/dist/mailroom/blob-store.js +102 -6
- package/dist/mailroom/cache-sync-cli.js +54 -0
- package/dist/mailroom/hosted-cache-sync.js +249 -0
- package/dist/mailroom/reader.js +59 -0
- package/dist/mailroom/search-cache.js +76 -6
- package/dist/repertoire/guardrails.js +674 -0
- package/dist/repertoire/tools-mail.js +5 -152
- package/package.json +1 -1
|
@@ -0,0 +1,144 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.isSafePrunableAgentName = isSafePrunableAgentName;
|
|
4
|
+
exports.resolvePrunableAgentBundle = resolvePrunableAgentBundle;
|
|
5
|
+
exports.listPrunableAgentBundles = listPrunableAgentBundles;
|
|
6
|
+
exports.isManagedDaemonLogEntry = isManagedDaemonLogEntry;
|
|
7
|
+
exports.validatePrunableLogEntry = validatePrunableLogEntry;
|
|
8
|
+
exports.validatePrunableLogsDirectory = validatePrunableLogsDirectory;
|
|
9
|
+
exports.validatePrunableLogsTarget = validatePrunableLogsTarget;
|
|
10
|
+
const path_1 = require("path");
|
|
11
|
+
const runtime_1 = require("../../nerves/runtime");
|
|
12
|
+
const defaultFs = {
|
|
13
|
+
existsSync: (filePath) => require("fs").existsSync(filePath),
|
|
14
|
+
readdirSync: (dirPath) => require("fs").readdirSync(dirPath),
|
|
15
|
+
lstatSync: (filePath) => require("fs").lstatSync(filePath),
|
|
16
|
+
realpathSync: (filePath) => require("fs").realpathSync(filePath),
|
|
17
|
+
};
|
|
18
|
+
const defaultLogsFs = {
|
|
19
|
+
readdirSync: (dirPath) => require("fs").readdirSync(dirPath),
|
|
20
|
+
lstatSync: (filePath) => require("fs").lstatSync(filePath),
|
|
21
|
+
realpathSync: (filePath) => require("fs").realpathSync(filePath),
|
|
22
|
+
};
|
|
23
|
+
const SAFE_AGENT_NAME = /^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$/;
|
|
24
|
+
function isSafePrunableAgentName(agentName) {
|
|
25
|
+
return SAFE_AGENT_NAME.test(agentName) && agentName !== "." && agentName !== "..";
|
|
26
|
+
}
|
|
27
|
+
function invalidTarget(agentName, reason) {
|
|
28
|
+
return new Error(`agent ${JSON.stringify(agentName)} is not a prunable agent bundle: ${reason}`);
|
|
29
|
+
}
|
|
30
|
+
function resolvePrunableAgentBundle(options) {
|
|
31
|
+
const io = options.fs ?? defaultFs;
|
|
32
|
+
const agentName = options.agentName;
|
|
33
|
+
if (!isSafePrunableAgentName(agentName)) {
|
|
34
|
+
throw invalidTarget(agentName, "name must match [A-Za-z0-9][A-Za-z0-9._-]* and be at most 128 characters");
|
|
35
|
+
}
|
|
36
|
+
let rootReal;
|
|
37
|
+
try {
|
|
38
|
+
rootReal = io.realpathSync(options.bundlesRoot);
|
|
39
|
+
}
|
|
40
|
+
catch {
|
|
41
|
+
throw invalidTarget(agentName, "bundles root does not exist");
|
|
42
|
+
}
|
|
43
|
+
const bundleDir = (0, path_1.join)(options.bundlesRoot, `${agentName}.ouro`);
|
|
44
|
+
let bundleStat;
|
|
45
|
+
try {
|
|
46
|
+
bundleStat = io.lstatSync(bundleDir);
|
|
47
|
+
}
|
|
48
|
+
catch {
|
|
49
|
+
throw invalidTarget(agentName, "bundle does not exist");
|
|
50
|
+
}
|
|
51
|
+
if (bundleStat.isSymbolicLink() || !bundleStat.isDirectory()) {
|
|
52
|
+
throw invalidTarget(agentName, "bundle must be a real directory, not a symlink or other entry");
|
|
53
|
+
}
|
|
54
|
+
let bundleReal;
|
|
55
|
+
try {
|
|
56
|
+
bundleReal = io.realpathSync(bundleDir);
|
|
57
|
+
}
|
|
58
|
+
catch {
|
|
59
|
+
throw invalidTarget(agentName, "bundle cannot be resolved");
|
|
60
|
+
}
|
|
61
|
+
if ((0, path_1.dirname)(bundleReal) !== rootReal || (0, path_1.basename)(bundleReal) !== `${agentName}.ouro`) {
|
|
62
|
+
throw invalidTarget(agentName, "bundle must be a direct child of the bundles root");
|
|
63
|
+
}
|
|
64
|
+
if (!io.existsSync((0, path_1.join)(bundleReal, "agent.json"))) {
|
|
65
|
+
throw invalidTarget(agentName, "agent.json is not present");
|
|
66
|
+
}
|
|
67
|
+
const target = { agentName, bundleDir: bundleReal, logsDir: (0, path_1.join)(bundleReal, "state", "daemon", "logs") };
|
|
68
|
+
(0, runtime_1.emitNervesEvent)({
|
|
69
|
+
component: "nerves",
|
|
70
|
+
event: "nerves.prunable_bundle_resolved",
|
|
71
|
+
message: "resolved prunable agent bundle",
|
|
72
|
+
meta: { agentName, bundleDir: bundleReal },
|
|
73
|
+
});
|
|
74
|
+
return target;
|
|
75
|
+
}
|
|
76
|
+
function listPrunableAgentBundles(options) {
|
|
77
|
+
const io = options.fs ?? defaultFs;
|
|
78
|
+
let entries;
|
|
79
|
+
try {
|
|
80
|
+
entries = io.readdirSync(options.bundlesRoot);
|
|
81
|
+
}
|
|
82
|
+
catch {
|
|
83
|
+
entries = [];
|
|
84
|
+
}
|
|
85
|
+
const agents = [];
|
|
86
|
+
for (const entry of entries) {
|
|
87
|
+
if (!entry.endsWith(".ouro"))
|
|
88
|
+
continue;
|
|
89
|
+
const agentName = entry.slice(0, -".ouro".length);
|
|
90
|
+
if (!isSafePrunableAgentName(agentName))
|
|
91
|
+
continue;
|
|
92
|
+
try {
|
|
93
|
+
resolvePrunableAgentBundle({ ...options, agentName });
|
|
94
|
+
agents.push(agentName);
|
|
95
|
+
}
|
|
96
|
+
catch {
|
|
97
|
+
// Invalid entries are not prune candidates.
|
|
98
|
+
}
|
|
99
|
+
}
|
|
100
|
+
agents.sort((left, right) => left.localeCompare(right));
|
|
101
|
+
(0, runtime_1.emitNervesEvent)({
|
|
102
|
+
component: "nerves",
|
|
103
|
+
event: "nerves.prunable_bundles_listed",
|
|
104
|
+
message: "listed prunable agent bundles",
|
|
105
|
+
meta: { bundlesRoot: options.bundlesRoot, count: agents.length },
|
|
106
|
+
});
|
|
107
|
+
return agents;
|
|
108
|
+
}
|
|
109
|
+
function isManagedDaemonLogEntry(name) {
|
|
110
|
+
const activeNdjson = name.endsWith(".ndjson") && !/\.\d+\.ndjson$/.test(name);
|
|
111
|
+
const activeLog = name.endsWith(".log") && !/\.\d+\.log$/.test(name);
|
|
112
|
+
return activeNdjson || activeLog ||
|
|
113
|
+
/\.\d+\.ndjson(?:\.gz)?$/.test(name) ||
|
|
114
|
+
/\.log\.\d+(?:\.gz)?$/.test(name) ||
|
|
115
|
+
/\.\d+\.log$/.test(name);
|
|
116
|
+
}
|
|
117
|
+
function validatePrunableLogEntry(filePath, logsReal, io = defaultLogsFs) {
|
|
118
|
+
const stat = io.lstatSync(filePath);
|
|
119
|
+
if (stat.isSymbolicLink() || !stat.isFile()) {
|
|
120
|
+
throw new Error(`daemon log entry must be a regular non-symlink file: ${filePath}`);
|
|
121
|
+
}
|
|
122
|
+
if ((0, path_1.dirname)(io.realpathSync(filePath)) !== logsReal) {
|
|
123
|
+
throw new Error(`daemon log entry escapes the canonical logs directory: ${filePath}`);
|
|
124
|
+
}
|
|
125
|
+
}
|
|
126
|
+
function validatePrunableLogsDirectory(logsDir, expectedCanonicalLogsDir, io = defaultLogsFs) {
|
|
127
|
+
const stat = io.lstatSync(logsDir);
|
|
128
|
+
if (stat.isSymbolicLink() || !stat.isDirectory()) {
|
|
129
|
+
throw new Error(`daemon logs directory must be a real directory: ${logsDir}`);
|
|
130
|
+
}
|
|
131
|
+
const logsReal = io.realpathSync(logsDir);
|
|
132
|
+
if (expectedCanonicalLogsDir && logsReal !== expectedCanonicalLogsDir) {
|
|
133
|
+
throw new Error(`daemon logs directory must remain inside its canonical agent bundle: ${logsDir}`);
|
|
134
|
+
}
|
|
135
|
+
const entries = io.readdirSync(logsDir);
|
|
136
|
+
for (const name of entries) {
|
|
137
|
+
if (isManagedDaemonLogEntry(name))
|
|
138
|
+
validatePrunableLogEntry((0, path_1.join)(logsDir, name), logsReal, io);
|
|
139
|
+
}
|
|
140
|
+
return { entries, logsReal };
|
|
141
|
+
}
|
|
142
|
+
function validatePrunableLogsTarget(target, io = defaultLogsFs) {
|
|
143
|
+
return validatePrunableLogsDirectory(target.logsDir, target.logsDir, io);
|
|
144
|
+
}
|
|
@@ -1,7 +1,9 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
3
|
exports.AzureBlobMailroomStore = void 0;
|
|
4
|
+
exports.canonicalMailIndexRecordSnapshot = canonicalMailIndexRecordSnapshot;
|
|
4
5
|
exports.decryptBlobMessages = decryptBlobMessages;
|
|
6
|
+
const node_crypto_1 = require("node:crypto");
|
|
5
7
|
const runtime_1 = require("../nerves/runtime");
|
|
6
8
|
const core_1 = require("./core");
|
|
7
9
|
const search_cache_1 = require("./search-cache");
|
|
@@ -198,21 +200,53 @@ function parseMessageIndexBlobName(name) {
|
|
|
198
200
|
const agentId = parts[1];
|
|
199
201
|
const stem = parts[2].slice(0, -5);
|
|
200
202
|
const [sortKey, compartmentKind, placement, sourceToken, ...idParts] = stem.split("__");
|
|
201
|
-
|
|
203
|
+
const id = idParts.join("__");
|
|
204
|
+
if (!agentId || !sortKey || !compartmentKind || !placement || !sourceToken || !id)
|
|
205
|
+
return null;
|
|
206
|
+
if (!/^\d{13}$/.test(sortKey))
|
|
202
207
|
return null;
|
|
203
208
|
if (compartmentKind !== "native" && compartmentKind !== "delegated")
|
|
204
209
|
return null;
|
|
205
|
-
|
|
210
|
+
if (!["imbox", "screener", "discarded", "quarantine", "draft", "sent"].includes(placement))
|
|
211
|
+
return null;
|
|
212
|
+
const decodedSource = decodeSourceToken(sourceToken);
|
|
213
|
+
const receivedAtMs = MESSAGE_INDEX_SORT_MAX_MS - Number(sortKey);
|
|
206
214
|
return {
|
|
207
215
|
schemaVersion: 1,
|
|
208
|
-
id
|
|
216
|
+
id,
|
|
209
217
|
agentId,
|
|
210
218
|
compartmentKind,
|
|
211
219
|
placement: placement,
|
|
212
|
-
...(
|
|
213
|
-
receivedAt:
|
|
220
|
+
...(decodedSource ? { source: decodedSource } : {}),
|
|
221
|
+
receivedAt: new Date(receivedAtMs).toISOString(),
|
|
222
|
+
};
|
|
223
|
+
}
|
|
224
|
+
function canonicalIndexRecord(record) {
|
|
225
|
+
return {
|
|
226
|
+
id: record.id,
|
|
227
|
+
receivedAt: record.receivedAt,
|
|
228
|
+
placement: record.placement,
|
|
229
|
+
compartmentKind: record.compartmentKind,
|
|
230
|
+
source: record.source?.toLowerCase() ?? null,
|
|
231
|
+
};
|
|
232
|
+
}
|
|
233
|
+
function canonicalMailIndexRecordSnapshot(records) {
|
|
234
|
+
const normalizedRecords = records
|
|
235
|
+
.map(canonicalIndexRecord)
|
|
236
|
+
.sort((left, right) => JSON.stringify(left).localeCompare(JSON.stringify(right)));
|
|
237
|
+
const receivedTimes = records
|
|
238
|
+
.map((record) => record.receivedAt)
|
|
239
|
+
.sort((left, right) => Date.parse(left) - Date.parse(right));
|
|
240
|
+
return {
|
|
241
|
+
visibleMessageCount: records.length,
|
|
242
|
+
messageIndexFingerprint: (0, node_crypto_1.createHash)("sha256").update(JSON.stringify(normalizedRecords)).digest("hex"),
|
|
243
|
+
...(receivedTimes[0] ? { oldestReceivedAt: receivedTimes[0] } : {}),
|
|
244
|
+
...(receivedTimes.at(-1) ? { newestReceivedAt: receivedTimes.at(-1) } : {}),
|
|
214
245
|
};
|
|
215
246
|
}
|
|
247
|
+
function canonicalMailIndexRecordOrder(left, right) {
|
|
248
|
+
return JSON.stringify(canonicalIndexRecord(left)).localeCompare(JSON.stringify(canonicalIndexRecord(right)));
|
|
249
|
+
}
|
|
216
250
|
function sourceMatchesFilter(source, filter) {
|
|
217
251
|
if (!filter)
|
|
218
252
|
return true;
|
|
@@ -344,6 +378,69 @@ class AzureBlobMailroomStore {
|
|
|
344
378
|
});
|
|
345
379
|
return applyOptionalLimit(sorted, filters.limit);
|
|
346
380
|
}
|
|
381
|
+
async observeMessageIndexAuthority(agentId) {
|
|
382
|
+
const prefix = messageIndexPrefix(agentId);
|
|
383
|
+
const operationTimeout = timeoutSignal(this.blobOperationTimeoutMs);
|
|
384
|
+
const records = [];
|
|
385
|
+
let totalNameCount = 0;
|
|
386
|
+
let parseFailureCount = 0;
|
|
387
|
+
try {
|
|
388
|
+
for await (const item of this.container.listBlobsFlat({ prefix, abortSignal: operationTimeout.signal })) {
|
|
389
|
+
totalNameCount += 1;
|
|
390
|
+
let parsed = null;
|
|
391
|
+
try {
|
|
392
|
+
parsed = parseMessageIndexBlobName(item.name);
|
|
393
|
+
}
|
|
394
|
+
catch {
|
|
395
|
+
parsed = null;
|
|
396
|
+
}
|
|
397
|
+
if (!parsed) {
|
|
398
|
+
parseFailureCount += 1;
|
|
399
|
+
continue;
|
|
400
|
+
}
|
|
401
|
+
records.push(parsed);
|
|
402
|
+
}
|
|
403
|
+
}
|
|
404
|
+
catch (error) {
|
|
405
|
+
const message = (error instanceof Error ? error.message : String(error)).toLowerCase();
|
|
406
|
+
if ((error instanceof Error && error.name === "AbortError") || message.includes("aborted")) {
|
|
407
|
+
throw normalizeBlobOperationError("list", { name: prefix }, this.blobOperationTimeoutMs, error);
|
|
408
|
+
}
|
|
409
|
+
throw error;
|
|
410
|
+
}
|
|
411
|
+
finally {
|
|
412
|
+
operationTimeout.dispose();
|
|
413
|
+
}
|
|
414
|
+
records.sort(canonicalMailIndexRecordOrder);
|
|
415
|
+
const counts = new Map();
|
|
416
|
+
for (const record of records)
|
|
417
|
+
counts.set(record.id, (counts.get(record.id) ?? 0) + 1);
|
|
418
|
+
const duplicateIds = [...counts.entries()]
|
|
419
|
+
.filter(([, count]) => count > 1)
|
|
420
|
+
.map(([id]) => id)
|
|
421
|
+
.sort((left, right) => left.localeCompare(right));
|
|
422
|
+
const observation = {
|
|
423
|
+
totalNameCount,
|
|
424
|
+
parsedRecordCount: records.length,
|
|
425
|
+
parseFailureCount,
|
|
426
|
+
duplicateIds,
|
|
427
|
+
records,
|
|
428
|
+
snapshot: canonicalMailIndexRecordSnapshot(records),
|
|
429
|
+
};
|
|
430
|
+
(0, runtime_1.emitNervesEvent)({
|
|
431
|
+
component: "senses",
|
|
432
|
+
event: "senses.mail_blob_store_authority_observed",
|
|
433
|
+
message: "azure blob mailroom index authority observed",
|
|
434
|
+
meta: {
|
|
435
|
+
agentId,
|
|
436
|
+
totalNameCount,
|
|
437
|
+
parsedRecordCount: records.length,
|
|
438
|
+
parseFailureCount,
|
|
439
|
+
duplicateIdCount: duplicateIds.length,
|
|
440
|
+
},
|
|
441
|
+
});
|
|
442
|
+
return observation;
|
|
443
|
+
}
|
|
347
444
|
async listMessagesFromIndexes(filters) {
|
|
348
445
|
const records = await this.listMessageIndexRecords(filters);
|
|
349
446
|
if (records === null)
|
|
@@ -465,7 +562,6 @@ class AzureBlobMailroomStore {
|
|
|
465
562
|
return message;
|
|
466
563
|
}
|
|
467
564
|
async getIndexedMessageById(id) {
|
|
468
|
-
await this.ensureContainer();
|
|
469
565
|
const message = await downloadIndexedJson(this.messageBlob(id), this.blobOperationTimeoutMs);
|
|
470
566
|
(0, runtime_1.emitNervesEvent)({
|
|
471
567
|
component: "senses",
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.runHostedMailCacheSync = runHostedMailCacheSync;
|
|
4
|
+
const runtime_credentials_1 = require("../heart/runtime-credentials");
|
|
5
|
+
const runtime_1 = require("../nerves/runtime");
|
|
6
|
+
const hosted_cache_sync_1 = require("./hosted-cache-sync");
|
|
7
|
+
const reader_1 = require("./reader");
|
|
8
|
+
async function runHostedMailCacheSync(agentName) {
|
|
9
|
+
const freshRuntime = await (0, runtime_credentials_1.refreshRuntimeCredentialConfig)(agentName, { preserveCachedOnFailure: false });
|
|
10
|
+
if (!freshRuntime.ok) {
|
|
11
|
+
throw new Error(`mail cache sync for ${agentName} requires fresh ${freshRuntime.itemPath}: ${freshRuntime.error}`);
|
|
12
|
+
}
|
|
13
|
+
const reader = (0, reader_1.resolveMailroomReader)(agentName);
|
|
14
|
+
if (!reader.ok)
|
|
15
|
+
throw new Error(reader.error);
|
|
16
|
+
if (reader.storeKind !== "azure-blob") {
|
|
17
|
+
throw new Error(`mail cache sync for ${agentName} requires hosted Azure Mailroom; resolved ${reader.storeKind}`);
|
|
18
|
+
}
|
|
19
|
+
const authority = (0, reader_1.resolveHostedMailAuthority)(agentName);
|
|
20
|
+
if (!authority.ok)
|
|
21
|
+
throw new Error(authority.error);
|
|
22
|
+
const result = await (0, hosted_cache_sync_1.syncHostedMailSearchCache)({
|
|
23
|
+
agentId: agentName,
|
|
24
|
+
mode: "full-convergence",
|
|
25
|
+
authority: authority.authority,
|
|
26
|
+
store: reader.store,
|
|
27
|
+
privateKeys: reader.config.privateKeys,
|
|
28
|
+
storeKind: reader.storeKind,
|
|
29
|
+
cacheOptions: reader.store.mailSearchCacheOptions?.(),
|
|
30
|
+
});
|
|
31
|
+
const text = [
|
|
32
|
+
`hosted mail cache converged for ${agentName}.`,
|
|
33
|
+
`visible authoritative messages: ${result.coverage.visibleMessageCount}`,
|
|
34
|
+
`decryptable cached messages: ${result.coverage.decryptableMessageCount}`,
|
|
35
|
+
`fetched this run: ${result.fetched}`,
|
|
36
|
+
`already current: ${result.alreadyCached}`,
|
|
37
|
+
`removed stale local files: ${result.removed}`,
|
|
38
|
+
`skipped unavailable messages: ${result.skipped}`,
|
|
39
|
+
`indexed at: ${result.coverage.indexedAt}`,
|
|
40
|
+
].join("\n");
|
|
41
|
+
(0, runtime_1.emitNervesEvent)({
|
|
42
|
+
component: "senses",
|
|
43
|
+
event: "senses.mail_cache_sync_cli_completed",
|
|
44
|
+
message: "foreground hosted mail cache sync completed",
|
|
45
|
+
meta: {
|
|
46
|
+
agentName,
|
|
47
|
+
visible: result.coverage.visibleMessageCount,
|
|
48
|
+
cached: result.coverage.decryptableMessageCount,
|
|
49
|
+
removed: result.removed,
|
|
50
|
+
skipped: result.skipped,
|
|
51
|
+
},
|
|
52
|
+
});
|
|
53
|
+
return text;
|
|
54
|
+
}
|
|
@@ -0,0 +1,249 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.mailSearchCacheDocumentMatchesRecord = mailSearchCacheDocumentMatchesRecord;
|
|
4
|
+
exports.syncHostedMailSearchCache = syncHostedMailSearchCache;
|
|
5
|
+
const node_crypto_1 = require("node:crypto");
|
|
6
|
+
const runtime_1 = require("../nerves/runtime");
|
|
7
|
+
const blob_store_1 = require("./blob-store");
|
|
8
|
+
const core_1 = require("./core");
|
|
9
|
+
const search_cache_1 = require("./search-cache");
|
|
10
|
+
function filtersFor(input) {
|
|
11
|
+
return {
|
|
12
|
+
agentId: input.agentId,
|
|
13
|
+
...(input.placement ? { placement: input.placement } : {}),
|
|
14
|
+
...(input.scope ? { compartmentKind: input.scope } : {}),
|
|
15
|
+
...(input.source ? { source: input.source } : {}),
|
|
16
|
+
};
|
|
17
|
+
}
|
|
18
|
+
function recordFor(message) {
|
|
19
|
+
return {
|
|
20
|
+
schemaVersion: 1,
|
|
21
|
+
id: message.id,
|
|
22
|
+
agentId: message.agentId,
|
|
23
|
+
compartmentKind: message.compartmentKind,
|
|
24
|
+
placement: message.placement,
|
|
25
|
+
...(message.source ? { source: message.source } : {}),
|
|
26
|
+
receivedAt: message.receivedAt,
|
|
27
|
+
};
|
|
28
|
+
}
|
|
29
|
+
function sameOptionalText(left, right) {
|
|
30
|
+
return (left ?? "").toLowerCase() === (right ?? "").toLowerCase();
|
|
31
|
+
}
|
|
32
|
+
function mailSearchCacheDocumentMatchesRecord(document, record, requireKeyProvenance) {
|
|
33
|
+
return document.schemaVersion === 1
|
|
34
|
+
&& document.agentId === record.agentId
|
|
35
|
+
&& document.messageId === record.id
|
|
36
|
+
&& document.receivedAt === record.receivedAt
|
|
37
|
+
&& document.placement === record.placement
|
|
38
|
+
&& document.compartmentKind === record.compartmentKind
|
|
39
|
+
&& sameOptionalText(document.source, record.source)
|
|
40
|
+
&& document.textProjectionVersion === search_cache_1.MAIL_SEARCH_TEXT_PROJECTION_VERSION
|
|
41
|
+
&& (!requireKeyProvenance || typeof document.decryptionKeyId === "string");
|
|
42
|
+
}
|
|
43
|
+
function messageMatchesRecord(message, record) {
|
|
44
|
+
return message.id === record.id
|
|
45
|
+
&& message.agentId === record.agentId
|
|
46
|
+
&& message.receivedAt === record.receivedAt
|
|
47
|
+
&& message.placement === record.placement
|
|
48
|
+
&& message.compartmentKind === record.compartmentKind
|
|
49
|
+
&& sameOptionalText(message.source, record.source);
|
|
50
|
+
}
|
|
51
|
+
async function scopedRecords(input) {
|
|
52
|
+
const filters = filtersFor(input);
|
|
53
|
+
const indexed = await input.store.listMessageIndexRecords?.(filters);
|
|
54
|
+
if (indexed)
|
|
55
|
+
return indexed;
|
|
56
|
+
return (await input.store.listMessages(filters)).map(recordFor);
|
|
57
|
+
}
|
|
58
|
+
async function fetchIndexedMessage(store, id) {
|
|
59
|
+
return store.getIndexedMessageById
|
|
60
|
+
? store.getIndexedMessageById(id)
|
|
61
|
+
: store.getMessage(id);
|
|
62
|
+
}
|
|
63
|
+
function removeCanonicalIfPresent(agentId, messageId, options) {
|
|
64
|
+
return (0, search_cache_1.removeMailSearchCacheDocument)(agentId, messageId, options) ? 1 : 0;
|
|
65
|
+
}
|
|
66
|
+
function missingPrivateKeyId(error) {
|
|
67
|
+
if (!(error instanceof Error))
|
|
68
|
+
return null;
|
|
69
|
+
const keyId = error.keyId;
|
|
70
|
+
return typeof keyId === "string" && keyId.length > 0 ? keyId : null;
|
|
71
|
+
}
|
|
72
|
+
/**
|
|
73
|
+
* Converge a local decrypted cache against hosted index authority, or perform
|
|
74
|
+
* the legacy bounded upsert used by the in-agent refresh tool. This core never
|
|
75
|
+
* writes remote mail state and never records access; adapters own audit policy.
|
|
76
|
+
*/
|
|
77
|
+
async function performHostedMailSearchCacheSync(input) {
|
|
78
|
+
let authority = null;
|
|
79
|
+
let records;
|
|
80
|
+
if (input.mode === "full-convergence") {
|
|
81
|
+
const observer = input.authority ?? input.store;
|
|
82
|
+
if (!observer.observeMessageIndexAuthority) {
|
|
83
|
+
throw new Error("hosted mail authority observer is unavailable");
|
|
84
|
+
}
|
|
85
|
+
authority = await observer.observeMessageIndexAuthority(input.agentId);
|
|
86
|
+
if (authority.parseFailureCount > 0 || authority.duplicateIds.length > 0) {
|
|
87
|
+
throw new Error(`hosted mail authority is ambiguous: ${authority.parseFailureCount} malformed index name(s), ${authority.duplicateIds.length} duplicate message id(s)`);
|
|
88
|
+
}
|
|
89
|
+
if (authority.records.length === 0) {
|
|
90
|
+
throw new Error("hosted mail authority is empty and cannot safely drive full convergence");
|
|
91
|
+
}
|
|
92
|
+
records = authority.records;
|
|
93
|
+
}
|
|
94
|
+
else {
|
|
95
|
+
records = await scopedRecords(input);
|
|
96
|
+
}
|
|
97
|
+
const recordsById = new Map(records.map((record) => [record.id, record]));
|
|
98
|
+
const canonicalDocuments = new Map();
|
|
99
|
+
let removed = 0;
|
|
100
|
+
if (input.mode === "full-convergence") {
|
|
101
|
+
for (const inspected of (0, search_cache_1.inspectMailSearchCacheFiles)(input.agentId, input.cacheOptions)) {
|
|
102
|
+
const document = inspected.document;
|
|
103
|
+
const canonicalName = document ? `${document.messageId}.json` : null;
|
|
104
|
+
const record = document ? recordsById.get(document.messageId) : undefined;
|
|
105
|
+
if (!document || inspected.fileName !== canonicalName || document.agentId !== input.agentId || !record) {
|
|
106
|
+
removed += Number((0, search_cache_1.removeMailSearchCacheFile)(input.agentId, inspected.fileName, input.cacheOptions));
|
|
107
|
+
continue;
|
|
108
|
+
}
|
|
109
|
+
canonicalDocuments.set(document.messageId, document);
|
|
110
|
+
}
|
|
111
|
+
}
|
|
112
|
+
else {
|
|
113
|
+
for (const document of (0, search_cache_1.searchMailSearchCache)({
|
|
114
|
+
agentId: input.agentId,
|
|
115
|
+
placement: input.placement,
|
|
116
|
+
compartmentKind: input.scope,
|
|
117
|
+
source: input.source,
|
|
118
|
+
}, input.cacheOptions)) {
|
|
119
|
+
canonicalDocuments.set(document.messageId, document);
|
|
120
|
+
}
|
|
121
|
+
}
|
|
122
|
+
let fetched = 0;
|
|
123
|
+
let alreadyCached = 0;
|
|
124
|
+
let skipped = 0;
|
|
125
|
+
const failures = [];
|
|
126
|
+
for (const record of records) {
|
|
127
|
+
const cached = canonicalDocuments.get(record.id);
|
|
128
|
+
if (cached && input.mode === "full-convergence" && cached.decryptionKeyId && !input.privateKeys[cached.decryptionKeyId]) {
|
|
129
|
+
removed += removeCanonicalIfPresent(input.agentId, record.id, input.cacheOptions);
|
|
130
|
+
skipped += 1;
|
|
131
|
+
continue;
|
|
132
|
+
}
|
|
133
|
+
if (cached && mailSearchCacheDocumentMatchesRecord(cached, record, input.mode === "full-convergence")) {
|
|
134
|
+
alreadyCached += 1;
|
|
135
|
+
continue;
|
|
136
|
+
}
|
|
137
|
+
try {
|
|
138
|
+
const message = await fetchIndexedMessage(input.store, record.id);
|
|
139
|
+
if (!message)
|
|
140
|
+
throw new Error("indexed message was not retrievable");
|
|
141
|
+
fetched += 1;
|
|
142
|
+
if (!messageMatchesRecord(message, record)) {
|
|
143
|
+
throw new Error(`indexed message metadata did not match authority for ${record.id}`);
|
|
144
|
+
}
|
|
145
|
+
const decrypted = (0, core_1.readDecryptedMailMessage)(message, input.privateKeys);
|
|
146
|
+
(0, search_cache_1.upsertMailSearchCacheDocument)(message, decrypted.private, input.cacheOptions, {
|
|
147
|
+
...(message.bodyForm === "encrypted" ? { decryptionKeyId: message.privateEnvelope.keyId } : {}),
|
|
148
|
+
});
|
|
149
|
+
}
|
|
150
|
+
catch (error) {
|
|
151
|
+
if (input.mode === "full-convergence" || missingPrivateKeyId(error)) {
|
|
152
|
+
removed += removeCanonicalIfPresent(input.agentId, record.id, input.cacheOptions);
|
|
153
|
+
skipped += 1;
|
|
154
|
+
}
|
|
155
|
+
else {
|
|
156
|
+
failures.push({ id: record.id, error: error instanceof Error ? error.message : String(error) });
|
|
157
|
+
}
|
|
158
|
+
}
|
|
159
|
+
}
|
|
160
|
+
if (failures.length > 0) {
|
|
161
|
+
const sample = failures.slice(0, 3).map((entry) => `${entry.id}: ${entry.error}`).join("; ");
|
|
162
|
+
throw new Error(`mail search index refresh incomplete; ${failures.length} fetch failed. first failure(s): ${sample}`);
|
|
163
|
+
}
|
|
164
|
+
const cachedIds = new Set((0, search_cache_1.searchMailSearchCache)({
|
|
165
|
+
agentId: input.agentId,
|
|
166
|
+
placement: input.placement,
|
|
167
|
+
compartmentKind: input.scope,
|
|
168
|
+
source: input.source,
|
|
169
|
+
}, input.cacheOptions).filter((document) => recordsById.has(document.messageId)).map((document) => document.messageId));
|
|
170
|
+
const snapshot = authority?.snapshot ?? (0, blob_store_1.canonicalMailIndexRecordSnapshot)(records);
|
|
171
|
+
const coverage = (0, search_cache_1.writeMailSearchCoverageRecord)({
|
|
172
|
+
schemaVersion: 1,
|
|
173
|
+
agentId: input.agentId,
|
|
174
|
+
storeKind: input.storeKind,
|
|
175
|
+
...(input.placement ? { placement: input.placement } : {}),
|
|
176
|
+
...(input.scope ? { compartmentKind: input.scope } : {}),
|
|
177
|
+
...(input.source ? { source: input.source } : {}),
|
|
178
|
+
indexedAt: new Date().toISOString(),
|
|
179
|
+
visibleMessageCount: records.length,
|
|
180
|
+
cachedMessageCount: cachedIds.size,
|
|
181
|
+
decryptableMessageCount: cachedIds.size,
|
|
182
|
+
skippedMessageCount: skipped,
|
|
183
|
+
messageIndexFingerprint: snapshot.messageIndexFingerprint,
|
|
184
|
+
textProjectionVersion: search_cache_1.MAIL_SEARCH_TEXT_PROJECTION_VERSION,
|
|
185
|
+
...(snapshot.oldestReceivedAt ? { oldestReceivedAt: snapshot.oldestReceivedAt } : {}),
|
|
186
|
+
...(snapshot.newestReceivedAt ? { newestReceivedAt: snapshot.newestReceivedAt } : {}),
|
|
187
|
+
}, input.cacheOptions);
|
|
188
|
+
(0, runtime_1.emitNervesEvent)({
|
|
189
|
+
component: "senses",
|
|
190
|
+
event: "senses.mail_search_cache_synced",
|
|
191
|
+
message: "hosted mail search cache synchronized",
|
|
192
|
+
meta: {
|
|
193
|
+
agentId: input.agentId,
|
|
194
|
+
mode: input.mode,
|
|
195
|
+
visible: records.length,
|
|
196
|
+
fetched,
|
|
197
|
+
alreadyCached,
|
|
198
|
+
removed,
|
|
199
|
+
skipped,
|
|
200
|
+
},
|
|
201
|
+
});
|
|
202
|
+
return { coverage, fetched, alreadyCached, removed, skipped };
|
|
203
|
+
}
|
|
204
|
+
async function syncHostedMailSearchCache(input) {
|
|
205
|
+
const traceId = (0, node_crypto_1.randomUUID)();
|
|
206
|
+
(0, runtime_1.emitNervesEvent)({
|
|
207
|
+
component: "senses",
|
|
208
|
+
event: "senses.mail_search_cache_sync_start",
|
|
209
|
+
trace_id: traceId,
|
|
210
|
+
message: "synchronizing hosted mail search cache",
|
|
211
|
+
meta: { agentId: input.agentId, mode: input.mode, storeKind: input.storeKind },
|
|
212
|
+
});
|
|
213
|
+
try {
|
|
214
|
+
const result = await performHostedMailSearchCacheSync(input);
|
|
215
|
+
(0, runtime_1.emitNervesEvent)({
|
|
216
|
+
component: "senses",
|
|
217
|
+
event: "senses.mail_search_cache_sync_end",
|
|
218
|
+
trace_id: traceId,
|
|
219
|
+
message: "hosted mail search cache synchronization completed",
|
|
220
|
+
meta: {
|
|
221
|
+
agentId: input.agentId,
|
|
222
|
+
mode: input.mode,
|
|
223
|
+
storeKind: input.storeKind,
|
|
224
|
+
visible: result.coverage.visibleMessageCount,
|
|
225
|
+
fetched: result.fetched,
|
|
226
|
+
alreadyCached: result.alreadyCached,
|
|
227
|
+
removed: result.removed,
|
|
228
|
+
skipped: result.skipped,
|
|
229
|
+
},
|
|
230
|
+
});
|
|
231
|
+
return result;
|
|
232
|
+
}
|
|
233
|
+
catch (error) {
|
|
234
|
+
(0, runtime_1.emitNervesEvent)({
|
|
235
|
+
component: "senses",
|
|
236
|
+
event: "senses.mail_search_cache_sync_error",
|
|
237
|
+
trace_id: traceId,
|
|
238
|
+
level: "error",
|
|
239
|
+
message: "hosted mail search cache synchronization failed",
|
|
240
|
+
meta: {
|
|
241
|
+
agentId: input.agentId,
|
|
242
|
+
mode: input.mode,
|
|
243
|
+
storeKind: input.storeKind,
|
|
244
|
+
error: String(error),
|
|
245
|
+
},
|
|
246
|
+
});
|
|
247
|
+
throw error;
|
|
248
|
+
}
|
|
249
|
+
}
|
package/dist/mailroom/reader.js
CHANGED
|
@@ -37,6 +37,7 @@ exports.parseMailroomConfig = parseMailroomConfig;
|
|
|
37
37
|
exports.readMailroomRegistry = readMailroomRegistry;
|
|
38
38
|
exports.writeMailroomRegistry = writeMailroomRegistry;
|
|
39
39
|
exports.resolveMailroomReader = resolveMailroomReader;
|
|
40
|
+
exports.resolveHostedMailAuthority = resolveHostedMailAuthority;
|
|
40
41
|
exports.resolveMailroomReaderWithRefresh = resolveMailroomReaderWithRefresh;
|
|
41
42
|
const fs = __importStar(require("node:fs"));
|
|
42
43
|
const path = __importStar(require("node:path"));
|
|
@@ -221,6 +222,64 @@ function resolveMailroomReader(agentName = (0, identity_2.getAgentName)()) {
|
|
|
221
222
|
});
|
|
222
223
|
return result;
|
|
223
224
|
}
|
|
225
|
+
function resolveHostedMailAuthority(agentName = (0, identity_2.getAgentName)()) {
|
|
226
|
+
const runtime = (0, runtime_credentials_1.readRuntimeCredentialConfig)(agentName);
|
|
227
|
+
if (!runtime.ok) {
|
|
228
|
+
const result = {
|
|
229
|
+
ok: false,
|
|
230
|
+
agentName,
|
|
231
|
+
reason: "auth-required",
|
|
232
|
+
error: `AUTH_REQUIRED:mailroom -- Hosted mail authority is unavailable because ${runtime.itemPath} is ${runtime.reason}: ${runtime.error}`,
|
|
233
|
+
};
|
|
234
|
+
(0, runtime_1.emitNervesEvent)({
|
|
235
|
+
component: "senses",
|
|
236
|
+
event: "senses.mail_hosted_authority_resolved",
|
|
237
|
+
message: "hosted mail authority resolved",
|
|
238
|
+
meta: { agentName, status: result.reason },
|
|
239
|
+
});
|
|
240
|
+
return result;
|
|
241
|
+
}
|
|
242
|
+
const mailroom = isRecord(runtime.config.mailroom) ? runtime.config.mailroom : null;
|
|
243
|
+
const azureAccountUrl = mailroom ? textField(mailroom, "azureAccountUrl") : undefined;
|
|
244
|
+
if (!mailroom || !azureAccountUrl) {
|
|
245
|
+
const result = {
|
|
246
|
+
ok: false,
|
|
247
|
+
agentName,
|
|
248
|
+
reason: "misconfigured",
|
|
249
|
+
error: `Hosted mail authority for ${agentName} is missing mailroom.azureAccountUrl in vault runtime/config.`,
|
|
250
|
+
};
|
|
251
|
+
(0, runtime_1.emitNervesEvent)({
|
|
252
|
+
component: "senses",
|
|
253
|
+
event: "senses.mail_hosted_authority_resolved",
|
|
254
|
+
message: "hosted mail authority resolved",
|
|
255
|
+
meta: { agentName, status: result.reason },
|
|
256
|
+
});
|
|
257
|
+
return result;
|
|
258
|
+
}
|
|
259
|
+
const azureContainer = textField(mailroom, "azureContainer") ?? "mailroom";
|
|
260
|
+
const azureManagedIdentityClientId = textField(mailroom, "azureManagedIdentityClientId");
|
|
261
|
+
const store = new blob_store_1.AzureBlobMailroomStore({
|
|
262
|
+
serviceClient: new storage_blob_1.BlobServiceClient(azureAccountUrl, azureManagedIdentityClientId
|
|
263
|
+
? new identity_1.DefaultAzureCredential({ managedIdentityClientId: azureManagedIdentityClientId })
|
|
264
|
+
: new identity_1.DefaultAzureCredential()),
|
|
265
|
+
containerName: azureContainer,
|
|
266
|
+
});
|
|
267
|
+
const result = {
|
|
268
|
+
ok: true,
|
|
269
|
+
agentName,
|
|
270
|
+
authority: {
|
|
271
|
+
observeMessageIndexAuthority: (authorityAgentId) => store.observeMessageIndexAuthority(authorityAgentId),
|
|
272
|
+
},
|
|
273
|
+
storeLabel: `${azureAccountUrl}/${azureContainer}`,
|
|
274
|
+
};
|
|
275
|
+
(0, runtime_1.emitNervesEvent)({
|
|
276
|
+
component: "senses",
|
|
277
|
+
event: "senses.mail_hosted_authority_resolved",
|
|
278
|
+
message: "hosted mail authority resolved",
|
|
279
|
+
meta: { agentName, status: "ready" },
|
|
280
|
+
});
|
|
281
|
+
return result;
|
|
282
|
+
}
|
|
224
283
|
async function resolveMailroomReaderWithRefresh(agentName = (0, identity_2.getAgentName)()) {
|
|
225
284
|
const resolved = resolveMailroomReader(agentName);
|
|
226
285
|
if (resolved.ok
|