@holon-run/agentinbox 1.7.2 → 1.8.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +22 -0
- package/dist/src/cli.js +150 -8
- package/dist/src/client.js +52 -0
- package/dist/src/email_attachment.js +199 -0
- package/dist/src/email_attachment_content.js +759 -0
- package/dist/src/email_body.js +392 -0
- package/dist/src/http.js +147 -3
- package/dist/src/service.js +61 -3
- package/dist/src/source_schema.js +4 -1
- package/dist/src/sources/email.js +72 -3
- package/dist/src/sources/remote_modules.js +2 -1
- package/dist/src/store.js +451 -1
- package/drizzle/migrations/0006_email_body_cache.sql +19 -0
- package/drizzle/migrations/0007_email_attachment_materializations.sql +20 -0
- package/drizzle/migrations/0008_email_attachment_audit.sql +17 -0
- package/drizzle/schema.ts +62 -0
- package/package.json +2 -2
- package/skills/agentinbox/SKILL.md +20 -0
|
@@ -0,0 +1,759 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
3
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
|
+
};
|
|
5
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
+
exports.EmailAttachmentContentManager = exports.EMAIL_ATTACHMENT_DEFAULT_MAX_TOTAL_BYTES = void 0;
|
|
7
|
+
const node_crypto_1 = __importDefault(require("node:crypto"));
|
|
8
|
+
const node_fs_1 = __importDefault(require("node:fs"));
|
|
9
|
+
const node_path_1 = __importDefault(require("node:path"));
|
|
10
|
+
const uxc_daemon_client_1 = require("@holon-run/uxc-daemon-client");
|
|
11
|
+
const email_attachment_1 = require("./email_attachment");
|
|
12
|
+
const email_1 = require("./sources/email");
|
|
13
|
+
const util_1 = require("./util");
|
|
14
|
+
exports.EMAIL_ATTACHMENT_DEFAULT_MAX_TOTAL_BYTES = 512 * 1024 * 1024;
|
|
15
|
+
const EMAIL_ATTACHMENT_DEFAULT_GC_BATCH_SIZE = 256;
|
|
16
|
+
const EMAIL_ATTACHMENT_STAGING_MAX_AGE_MS = 60 * 60 * 1000;
|
|
17
|
+
class EmailAttachmentError extends Error {
|
|
18
|
+
code;
|
|
19
|
+
rejected;
|
|
20
|
+
statusCode;
|
|
21
|
+
constructor(code, rejected, message) {
|
|
22
|
+
super(message);
|
|
23
|
+
this.code = code;
|
|
24
|
+
this.rejected = rejected;
|
|
25
|
+
this.statusCode = code === "attachment_content_unavailable"
|
|
26
|
+
? 404
|
|
27
|
+
: rejected
|
|
28
|
+
? 400
|
|
29
|
+
: 503;
|
|
30
|
+
}
|
|
31
|
+
}
|
|
32
|
+
class EmailAttachmentContentManager {
|
|
33
|
+
store;
|
|
34
|
+
uxc;
|
|
35
|
+
inFlight = new Map();
|
|
36
|
+
itemInFlight = new Map();
|
|
37
|
+
publishingObjects = new Map();
|
|
38
|
+
stagingDir;
|
|
39
|
+
objectsDir;
|
|
40
|
+
scanner;
|
|
41
|
+
maxTotalBytes;
|
|
42
|
+
gcBatchSize;
|
|
43
|
+
constructor(store, rootDir, uxc = new uxc_daemon_client_1.UxcDaemonClient({ env: process.env }), options = {}) {
|
|
44
|
+
this.store = store;
|
|
45
|
+
this.uxc = uxc;
|
|
46
|
+
this.stagingDir = node_path_1.default.join(rootDir, "staging");
|
|
47
|
+
this.objectsDir = node_path_1.default.join(rootDir, "objects");
|
|
48
|
+
this.scanner = options.scanner ?? null;
|
|
49
|
+
this.maxTotalBytes = positiveIntegerOrDefault(options.maxTotalBytes ?? process.env.AGENTINBOX_EMAIL_ATTACHMENT_MAX_TOTAL_BYTES, exports.EMAIL_ATTACHMENT_DEFAULT_MAX_TOTAL_BYTES);
|
|
50
|
+
this.gcBatchSize = positiveIntegerOrDefault(options.gcBatchSize, EMAIL_ATTACHMENT_DEFAULT_GC_BATCH_SIZE);
|
|
51
|
+
}
|
|
52
|
+
inspect(agentId, attachmentRef) {
|
|
53
|
+
try {
|
|
54
|
+
const owned = this.resolveOwned(agentId, attachmentRef);
|
|
55
|
+
if (owned.attachment.status === "deleted") {
|
|
56
|
+
throw unknownAttachment(attachmentRef);
|
|
57
|
+
}
|
|
58
|
+
this.appendAudit(agentId, attachmentRef, "metadata_read", "success", owned);
|
|
59
|
+
return owned.attachment;
|
|
60
|
+
}
|
|
61
|
+
catch (error) {
|
|
62
|
+
this.appendAudit(agentId, attachmentRef, "metadata_read", "rejected", null, auditErrorCode(error));
|
|
63
|
+
throw error;
|
|
64
|
+
}
|
|
65
|
+
}
|
|
66
|
+
async materialize(agentId, attachmentRef) {
|
|
67
|
+
let owned;
|
|
68
|
+
try {
|
|
69
|
+
owned = this.resolveOwned(agentId, attachmentRef);
|
|
70
|
+
if (owned.attachment.status === "deleted") {
|
|
71
|
+
throw unknownAttachment(attachmentRef);
|
|
72
|
+
}
|
|
73
|
+
this.appendAudit(agentId, attachmentRef, "materialize", "started", owned);
|
|
74
|
+
}
|
|
75
|
+
catch (error) {
|
|
76
|
+
this.appendAudit(agentId, attachmentRef, "materialize", "rejected", null, auditErrorCode(error));
|
|
77
|
+
throw error;
|
|
78
|
+
}
|
|
79
|
+
const current = this.store.getEmailAttachmentMaterialization(owned.item.itemId, owned.attachmentSelector);
|
|
80
|
+
if (current?.status === "available" && current.objectKey && this.objectExists(current.objectKey)) {
|
|
81
|
+
const attachment = this.resolveOwned(agentId, attachmentRef).attachment;
|
|
82
|
+
this.appendAudit(agentId, attachmentRef, "materialize", "available", owned, null, current);
|
|
83
|
+
return attachment;
|
|
84
|
+
}
|
|
85
|
+
const key = `${owned.inboxId}:${owned.item.itemId}:${owned.attachmentSelector}`;
|
|
86
|
+
const existing = this.inFlight.get(key);
|
|
87
|
+
if (existing) {
|
|
88
|
+
return this.auditMaterializationResult(agentId, attachmentRef, owned, existing);
|
|
89
|
+
}
|
|
90
|
+
const itemKey = `${owned.inboxId}:${owned.item.itemId}`;
|
|
91
|
+
const previousItem = this.itemInFlight.get(itemKey) ?? Promise.resolve();
|
|
92
|
+
const pending = previousItem
|
|
93
|
+
.catch(() => undefined)
|
|
94
|
+
.then(() => this.materializeOnce(agentId, attachmentRef, owned))
|
|
95
|
+
.finally(() => this.inFlight.delete(key));
|
|
96
|
+
const itemCompletion = pending.then(() => undefined, () => undefined).finally(() => {
|
|
97
|
+
if (this.itemInFlight.get(itemKey) === itemCompletion) {
|
|
98
|
+
this.itemInFlight.delete(itemKey);
|
|
99
|
+
}
|
|
100
|
+
});
|
|
101
|
+
this.itemInFlight.set(itemKey, itemCompletion);
|
|
102
|
+
this.inFlight.set(key, pending);
|
|
103
|
+
return this.auditMaterializationResult(agentId, attachmentRef, owned, pending);
|
|
104
|
+
}
|
|
105
|
+
openContent(agentId, attachmentRef) {
|
|
106
|
+
try {
|
|
107
|
+
const content = this.openContentOnce(agentId, attachmentRef);
|
|
108
|
+
const owned = this.resolveOwned(agentId, attachmentRef);
|
|
109
|
+
const materialization = this.store.getEmailAttachmentMaterialization(owned.item.itemId, owned.attachmentSelector);
|
|
110
|
+
this.appendAudit(agentId, attachmentRef, "content_read", "success", owned, null, materialization);
|
|
111
|
+
return content;
|
|
112
|
+
}
|
|
113
|
+
catch (error) {
|
|
114
|
+
this.appendAudit(agentId, attachmentRef, "content_read", "rejected", null, auditErrorCode(error));
|
|
115
|
+
throw error;
|
|
116
|
+
}
|
|
117
|
+
}
|
|
118
|
+
openContentOnce(agentId, attachmentRef) {
|
|
119
|
+
const owned = this.resolveOwned(agentId, attachmentRef);
|
|
120
|
+
const materialization = this.store.getEmailAttachmentMaterialization(owned.item.itemId, owned.attachmentSelector);
|
|
121
|
+
if (materialization?.status !== "available"
|
|
122
|
+
|| !materialization.objectKey
|
|
123
|
+
|| !materialization.sha256
|
|
124
|
+
|| materialization.storedSize == null) {
|
|
125
|
+
throw new EmailAttachmentError("attachment_content_unavailable", false, "Email attachment content is not available.");
|
|
126
|
+
}
|
|
127
|
+
const objectPath = this.resolveObjectPath(materialization.objectKey);
|
|
128
|
+
let fd = null;
|
|
129
|
+
try {
|
|
130
|
+
fd = node_fs_1.default.openSync(objectPath, "r");
|
|
131
|
+
const stat = node_fs_1.default.fstatSync(fd);
|
|
132
|
+
if (!stat.isFile() || stat.size !== materialization.storedSize) {
|
|
133
|
+
throw new Error("managed attachment object mismatch");
|
|
134
|
+
}
|
|
135
|
+
}
|
|
136
|
+
catch {
|
|
137
|
+
if (fd != null) {
|
|
138
|
+
node_fs_1.default.closeSync(fd);
|
|
139
|
+
}
|
|
140
|
+
throw new EmailAttachmentError("attachment_content_unavailable", false, "Email attachment content is not available.");
|
|
141
|
+
}
|
|
142
|
+
if (fd == null) {
|
|
143
|
+
throw new EmailAttachmentError("attachment_content_unavailable", false, "Email attachment content is not available.");
|
|
144
|
+
}
|
|
145
|
+
let verified;
|
|
146
|
+
try {
|
|
147
|
+
// Re-check ownership after opening the managed object and before returning bytes.
|
|
148
|
+
verified = this.resolveOwned(agentId, attachmentRef);
|
|
149
|
+
}
|
|
150
|
+
catch (error) {
|
|
151
|
+
node_fs_1.default.closeSync(fd);
|
|
152
|
+
throw error;
|
|
153
|
+
}
|
|
154
|
+
return {
|
|
155
|
+
attachment: verified.attachment,
|
|
156
|
+
stream: node_fs_1.default.createReadStream(objectPath, { fd, autoClose: true }),
|
|
157
|
+
size: materialization.storedSize,
|
|
158
|
+
sha256: materialization.sha256,
|
|
159
|
+
contentType: materialization.detectedContentType ?? "application/octet-stream",
|
|
160
|
+
};
|
|
161
|
+
}
|
|
162
|
+
async delete(agentId, attachmentRef) {
|
|
163
|
+
let owned;
|
|
164
|
+
try {
|
|
165
|
+
owned = this.resolveOwned(agentId, attachmentRef);
|
|
166
|
+
}
|
|
167
|
+
catch (error) {
|
|
168
|
+
this.appendAudit(agentId, attachmentRef, "delete", "rejected", null, auditErrorCode(error));
|
|
169
|
+
throw error;
|
|
170
|
+
}
|
|
171
|
+
const itemKey = `${owned.inboxId}:${owned.item.itemId}`;
|
|
172
|
+
const previousItem = this.itemInFlight.get(itemKey) ?? Promise.resolve();
|
|
173
|
+
const deletion = previousItem
|
|
174
|
+
.catch(() => undefined)
|
|
175
|
+
.then(() => this.deleteOnce(agentId, attachmentRef, owned));
|
|
176
|
+
const itemCompletion = deletion.then(() => undefined, () => undefined).finally(() => {
|
|
177
|
+
if (this.itemInFlight.get(itemKey) === itemCompletion) {
|
|
178
|
+
this.itemInFlight.delete(itemKey);
|
|
179
|
+
}
|
|
180
|
+
});
|
|
181
|
+
this.itemInFlight.set(itemKey, itemCompletion);
|
|
182
|
+
return deletion;
|
|
183
|
+
}
|
|
184
|
+
gc(now = new Date()) {
|
|
185
|
+
const objectKeys = new Set();
|
|
186
|
+
let expired = 0;
|
|
187
|
+
let capacityEvicted = 0;
|
|
188
|
+
let objectsDeleted = 0;
|
|
189
|
+
const cutoffIso = now.toISOString();
|
|
190
|
+
for (const candidate of this.store.listExpiredEmailAttachmentMaterializations(cutoffIso, this.gcBatchSize)) {
|
|
191
|
+
const previous = this.tombstoneForGc(candidate, "attachment_retention_expired", cutoffIso);
|
|
192
|
+
if (previous) {
|
|
193
|
+
expired += 1;
|
|
194
|
+
if (previous.objectKey) {
|
|
195
|
+
objectKeys.add(previous.objectKey);
|
|
196
|
+
}
|
|
197
|
+
}
|
|
198
|
+
}
|
|
199
|
+
let managedBytes = this.store.sumManagedEmailAttachmentBytes();
|
|
200
|
+
if (managedBytes > this.maxTotalBytes) {
|
|
201
|
+
for (const object of this.store.listOldestEmailAttachmentObjects(this.gcBatchSize)) {
|
|
202
|
+
if (managedBytes <= this.maxTotalBytes) {
|
|
203
|
+
break;
|
|
204
|
+
}
|
|
205
|
+
const references = this.store.listEmailAttachmentMaterializationsForObject(object.objectKey);
|
|
206
|
+
let evictedObject = false;
|
|
207
|
+
for (const candidate of references) {
|
|
208
|
+
const previous = this.tombstoneForGc(candidate, "attachment_capacity_evicted", cutoffIso);
|
|
209
|
+
if (previous) {
|
|
210
|
+
capacityEvicted += 1;
|
|
211
|
+
evictedObject = true;
|
|
212
|
+
}
|
|
213
|
+
}
|
|
214
|
+
if (evictedObject) {
|
|
215
|
+
objectKeys.add(object.objectKey);
|
|
216
|
+
managedBytes = Math.max(0, managedBytes - object.storedSize);
|
|
217
|
+
}
|
|
218
|
+
}
|
|
219
|
+
}
|
|
220
|
+
for (const objectKey of objectKeys) {
|
|
221
|
+
if (this.deleteObjectIfUnreferenced(objectKey)) {
|
|
222
|
+
objectsDeleted += 1;
|
|
223
|
+
}
|
|
224
|
+
}
|
|
225
|
+
const orphanObjectsDeleted = this.deleteOrphanObjects(this.gcBatchSize);
|
|
226
|
+
const stagingFilesDeleted = this.deleteStaleStagingFiles(now.getTime() - EMAIL_ATTACHMENT_STAGING_MAX_AGE_MS, this.gcBatchSize);
|
|
227
|
+
return {
|
|
228
|
+
expired,
|
|
229
|
+
capacityEvicted,
|
|
230
|
+
objectsDeleted,
|
|
231
|
+
orphanObjectsDeleted,
|
|
232
|
+
stagingFilesDeleted,
|
|
233
|
+
managedBytes: this.store.sumManagedEmailAttachmentBytes(),
|
|
234
|
+
};
|
|
235
|
+
}
|
|
236
|
+
async materializeOnce(agentId, attachmentRef, owned) {
|
|
237
|
+
const policy = (0, email_1.parseEmailAttachmentPolicy)(owned.source.config?.attachmentPolicy ?? owned.source.config?.attachment_policy);
|
|
238
|
+
if (policy.mode !== "store_reference") {
|
|
239
|
+
throw new EmailAttachmentError("attachment_policy_metadata_only", true, "This email source does not allow attachment content storage.");
|
|
240
|
+
}
|
|
241
|
+
if (!owned.handle) {
|
|
242
|
+
throw new EmailAttachmentError("attachment_not_retrievable", true, "This attachment does not have a retrievable provider reference.");
|
|
243
|
+
}
|
|
244
|
+
const collection = (0, email_attachment_1.publicEmailAttachmentCollection)(owned.item.itemId, owned.item.metadata);
|
|
245
|
+
const knownCount = collection.attachmentCount ?? collection.attachments.length;
|
|
246
|
+
if (knownCount > policy.maxAttachmentsPerMessage) {
|
|
247
|
+
throw new EmailAttachmentError("attachment_count_limit_exceeded", true, "The message exceeds the configured attachment count limit.");
|
|
248
|
+
}
|
|
249
|
+
if (owned.attachment.size != null
|
|
250
|
+
&& owned.attachment.size > policy.maxBytesPerAttachment) {
|
|
251
|
+
throw new EmailAttachmentError("attachment_size_limit_exceeded", true, "The attachment exceeds the configured size limit.");
|
|
252
|
+
}
|
|
253
|
+
assertContentTypeAllowed(owned.attachment.contentType, policy, "declared");
|
|
254
|
+
const storedBeforeFetch = this.store.sumAvailableEmailAttachmentBytes(owned.item.itemId, owned.attachmentSelector);
|
|
255
|
+
if (owned.attachment.size != null
|
|
256
|
+
&& storedBeforeFetch + owned.attachment.size > policy.maxBytesPerMessage) {
|
|
257
|
+
throw new EmailAttachmentError("attachment_message_size_limit_exceeded", true, "The message exceeds the configured stored attachment byte limit.");
|
|
258
|
+
}
|
|
259
|
+
const now = (0, util_1.nowIso)();
|
|
260
|
+
const prior = this.store.getEmailAttachmentMaterialization(owned.item.itemId, owned.attachmentSelector);
|
|
261
|
+
const pending = {
|
|
262
|
+
itemId: owned.item.itemId,
|
|
263
|
+
attachmentSelector: owned.attachmentSelector,
|
|
264
|
+
status: "pending",
|
|
265
|
+
lastErrorCode: null,
|
|
266
|
+
objectKey: null,
|
|
267
|
+
sha256: null,
|
|
268
|
+
declaredContentType: owned.attachment.contentType,
|
|
269
|
+
detectedContentType: null,
|
|
270
|
+
declaredSize: owned.attachment.size,
|
|
271
|
+
storedSize: null,
|
|
272
|
+
createdAt: prior?.createdAt ?? now,
|
|
273
|
+
updatedAt: now,
|
|
274
|
+
expiresAt: null,
|
|
275
|
+
deletedAt: null,
|
|
276
|
+
};
|
|
277
|
+
if (!this.store.beginEmailAttachmentMaterialization(owned.inboxId, pending)) {
|
|
278
|
+
throw unknownAttachment(attachmentRef);
|
|
279
|
+
}
|
|
280
|
+
node_fs_1.default.mkdirSync(this.stagingDir, { recursive: true, mode: 0o700 });
|
|
281
|
+
node_fs_1.default.chmodSync(this.stagingDir, 0o700);
|
|
282
|
+
const stagingPath = node_path_1.default.join(this.stagingDir, node_crypto_1.default.randomBytes(18).toString("hex"));
|
|
283
|
+
try {
|
|
284
|
+
const result = await this.retrieve(owned.handle, stagingPath, policy.maxBytesPerAttachment);
|
|
285
|
+
const stat = node_fs_1.default.statSync(stagingPath);
|
|
286
|
+
if (!stat.isFile() || stat.size > policy.maxBytesPerAttachment || stat.size !== result.size_bytes) {
|
|
287
|
+
throw new EmailAttachmentError("attachment_size_mismatch", true, "The retrieved attachment size did not match the provider result.");
|
|
288
|
+
}
|
|
289
|
+
node_fs_1.default.chmodSync(stagingPath, 0o600);
|
|
290
|
+
const sha256 = await hashFile(stagingPath);
|
|
291
|
+
if (sha256 !== result.sha256.toLowerCase()) {
|
|
292
|
+
throw new EmailAttachmentError("attachment_hash_mismatch", true, "The retrieved attachment hash did not match the provider result.");
|
|
293
|
+
}
|
|
294
|
+
const detectedContentType = detectContentType(stagingPath);
|
|
295
|
+
assertContentTypeAllowed(detectedContentType, policy, "detected");
|
|
296
|
+
const scanResult = this.scanner
|
|
297
|
+
? await this.scanner.scan({
|
|
298
|
+
path: stagingPath,
|
|
299
|
+
sha256,
|
|
300
|
+
size: stat.size,
|
|
301
|
+
contentType: detectedContentType,
|
|
302
|
+
})
|
|
303
|
+
: { status: "clean", code: null };
|
|
304
|
+
if (scanResult.status === "rejected") {
|
|
305
|
+
throw new EmailAttachmentError(stableAttachmentErrorCode(scanResult.code, "attachment_scanner_rejected"), true, "The attachment was rejected by the configured scanner.");
|
|
306
|
+
}
|
|
307
|
+
const alreadyStored = this.store.sumAvailableEmailAttachmentBytes(owned.item.itemId, owned.attachmentSelector);
|
|
308
|
+
if (alreadyStored + stat.size > policy.maxBytesPerMessage) {
|
|
309
|
+
throw new EmailAttachmentError("attachment_message_size_limit_exceeded", true, "The message exceeds the configured stored attachment byte limit.");
|
|
310
|
+
}
|
|
311
|
+
const verified = this.resolveOwned(agentId, attachmentRef);
|
|
312
|
+
syncFile(stagingPath);
|
|
313
|
+
const objectKey = objectKeyForSha256(sha256);
|
|
314
|
+
this.beginObjectPublication(objectKey);
|
|
315
|
+
try {
|
|
316
|
+
await this.putObject(stagingPath, sha256, objectKey);
|
|
317
|
+
const completedAt = (0, util_1.nowIso)();
|
|
318
|
+
const record = {
|
|
319
|
+
...pending,
|
|
320
|
+
status: scanResult.status === "quarantined" ? "quarantined" : "available",
|
|
321
|
+
lastErrorCode: scanResult.status === "quarantined"
|
|
322
|
+
? stableAttachmentErrorCode(scanResult.code, "attachment_scanner_quarantined")
|
|
323
|
+
: null,
|
|
324
|
+
objectKey,
|
|
325
|
+
sha256,
|
|
326
|
+
detectedContentType,
|
|
327
|
+
storedSize: stat.size,
|
|
328
|
+
updatedAt: completedAt,
|
|
329
|
+
expiresAt: new Date(Date.parse(completedAt) + policy.retentionSecs * 1000).toISOString(),
|
|
330
|
+
};
|
|
331
|
+
if (verified.item.itemId !== owned.item.itemId
|
|
332
|
+
|| verified.attachmentSelector !== owned.attachmentSelector
|
|
333
|
+
|| !this.store.finishEmailAttachmentMaterialization(owned.inboxId, record)) {
|
|
334
|
+
throw unknownAttachment(attachmentRef);
|
|
335
|
+
}
|
|
336
|
+
}
|
|
337
|
+
finally {
|
|
338
|
+
this.endObjectPublication(objectKey);
|
|
339
|
+
}
|
|
340
|
+
return this.resolveOwned(agentId, attachmentRef).attachment;
|
|
341
|
+
}
|
|
342
|
+
catch (error) {
|
|
343
|
+
node_fs_1.default.rmSync(stagingPath, { force: true });
|
|
344
|
+
if (error instanceof Error && error.message.startsWith("unknown inbox attachment:")) {
|
|
345
|
+
throw error;
|
|
346
|
+
}
|
|
347
|
+
const failure = attachmentFailure(error);
|
|
348
|
+
const failedAt = (0, util_1.nowIso)();
|
|
349
|
+
this.store.finishEmailAttachmentMaterialization(owned.inboxId, {
|
|
350
|
+
...pending,
|
|
351
|
+
status: failure.rejected ? "rejected" : "failed",
|
|
352
|
+
lastErrorCode: failure.code,
|
|
353
|
+
updatedAt: failedAt,
|
|
354
|
+
});
|
|
355
|
+
throw failure;
|
|
356
|
+
}
|
|
357
|
+
}
|
|
358
|
+
async auditMaterializationResult(agentId, attachmentRef, owned, operation) {
|
|
359
|
+
try {
|
|
360
|
+
const attachment = await operation;
|
|
361
|
+
const materialization = this.store.getEmailAttachmentMaterialization(owned.item.itemId, owned.attachmentSelector);
|
|
362
|
+
this.appendAudit(agentId, attachmentRef, "materialize", materialization?.status ?? attachment.status, owned, materialization?.lastErrorCode ?? null, materialization);
|
|
363
|
+
return attachment;
|
|
364
|
+
}
|
|
365
|
+
catch (error) {
|
|
366
|
+
const materialization = this.store.getEmailAttachmentMaterialization(owned.item.itemId, owned.attachmentSelector);
|
|
367
|
+
this.appendAudit(agentId, attachmentRef, "materialize", materialization?.status ?? "failed", owned, materialization?.lastErrorCode ?? auditErrorCode(error), materialization);
|
|
368
|
+
throw error;
|
|
369
|
+
}
|
|
370
|
+
}
|
|
371
|
+
deleteOnce(agentId, attachmentRef, initial) {
|
|
372
|
+
const owned = this.resolveOwned(agentId, attachmentRef);
|
|
373
|
+
if (owned.item.itemId !== initial.item.itemId
|
|
374
|
+
|| owned.attachmentSelector !== initial.attachmentSelector) {
|
|
375
|
+
throw unknownAttachment(attachmentRef);
|
|
376
|
+
}
|
|
377
|
+
const current = this.store.getEmailAttachmentMaterialization(owned.item.itemId, owned.attachmentSelector);
|
|
378
|
+
if (current?.status === "deleted") {
|
|
379
|
+
this.appendAudit(agentId, attachmentRef, "delete", "already_deleted", owned);
|
|
380
|
+
return { attachmentRef, deleted: false };
|
|
381
|
+
}
|
|
382
|
+
const deletedAt = (0, util_1.nowIso)();
|
|
383
|
+
const tombstoneFallback = {
|
|
384
|
+
itemId: owned.item.itemId,
|
|
385
|
+
attachmentSelector: owned.attachmentSelector,
|
|
386
|
+
status: "deleted",
|
|
387
|
+
lastErrorCode: "attachment_explicitly_deleted",
|
|
388
|
+
objectKey: null,
|
|
389
|
+
sha256: null,
|
|
390
|
+
declaredContentType: owned.attachment.contentType,
|
|
391
|
+
detectedContentType: owned.attachment.detectedContentType,
|
|
392
|
+
declaredSize: owned.attachment.size,
|
|
393
|
+
storedSize: null,
|
|
394
|
+
createdAt: deletedAt,
|
|
395
|
+
updatedAt: deletedAt,
|
|
396
|
+
expiresAt: null,
|
|
397
|
+
deletedAt,
|
|
398
|
+
};
|
|
399
|
+
const previous = this.store.tombstoneEmailAttachmentMaterialization(owned.inboxId, owned.item.itemId, owned.attachmentSelector, deletedAt, this.auditEvent(agentId, attachmentRef, "delete", "deleted", owned, "attachment_explicitly_deleted", current), tombstoneFallback);
|
|
400
|
+
if (!previous) {
|
|
401
|
+
throw unknownAttachment(attachmentRef);
|
|
402
|
+
}
|
|
403
|
+
if (previous.objectKey) {
|
|
404
|
+
this.deleteObjectIfUnreferenced(previous.objectKey);
|
|
405
|
+
}
|
|
406
|
+
return { attachmentRef, deleted: true };
|
|
407
|
+
}
|
|
408
|
+
tombstoneForGc(candidate, errorCode, deletedAt) {
|
|
409
|
+
const materialization = candidate.materialization;
|
|
410
|
+
const attachmentRef = `att_v1.${materialization.itemId}.${materialization.attachmentSelector}`;
|
|
411
|
+
return this.store.tombstoneEmailAttachmentMaterialization(candidate.inboxId, materialization.itemId, materialization.attachmentSelector, deletedAt, {
|
|
412
|
+
auditId: (0, util_1.generateCanonicalId)("aat"),
|
|
413
|
+
attachmentRef,
|
|
414
|
+
claimedAgentId: candidate.agentId,
|
|
415
|
+
inboxId: candidate.inboxId,
|
|
416
|
+
itemId: materialization.itemId,
|
|
417
|
+
sourceId: candidate.sourceId,
|
|
418
|
+
action: "gc",
|
|
419
|
+
result: "deleted",
|
|
420
|
+
errorCode,
|
|
421
|
+
bytes: materialization.storedSize,
|
|
422
|
+
sha256: materialization.sha256,
|
|
423
|
+
createdAt: deletedAt,
|
|
424
|
+
});
|
|
425
|
+
}
|
|
426
|
+
appendAudit(agentId, attachmentRef, action, result, owned, errorCode = null, materialization = null) {
|
|
427
|
+
this.store.appendEmailAttachmentAudit(this.auditEvent(agentId, attachmentRef, action, result, owned, errorCode, materialization));
|
|
428
|
+
}
|
|
429
|
+
auditEvent(agentId, attachmentRef, action, result, owned, errorCode = null, materialization = null) {
|
|
430
|
+
return {
|
|
431
|
+
auditId: (0, util_1.generateCanonicalId)("aat"),
|
|
432
|
+
attachmentRef: safeAuditAttachmentRef(attachmentRef),
|
|
433
|
+
claimedAgentId: agentId,
|
|
434
|
+
inboxId: owned?.inboxId ?? null,
|
|
435
|
+
itemId: owned?.item.itemId ?? null,
|
|
436
|
+
sourceId: owned?.source.sourceId ?? null,
|
|
437
|
+
action,
|
|
438
|
+
result,
|
|
439
|
+
errorCode: errorCode
|
|
440
|
+
? stableAttachmentErrorCode(errorCode, "attachment_operation_failed")
|
|
441
|
+
: null,
|
|
442
|
+
bytes: materialization?.storedSize ?? null,
|
|
443
|
+
sha256: materialization?.sha256 ?? null,
|
|
444
|
+
createdAt: (0, util_1.nowIso)(),
|
|
445
|
+
};
|
|
446
|
+
}
|
|
447
|
+
deleteObjectIfUnreferenced(objectKey) {
|
|
448
|
+
if (this.publishingObjects.has(objectKey)) {
|
|
449
|
+
return false;
|
|
450
|
+
}
|
|
451
|
+
if (this.store.countEmailAttachmentObjectReferences(objectKey) > 0) {
|
|
452
|
+
return false;
|
|
453
|
+
}
|
|
454
|
+
try {
|
|
455
|
+
const objectPath = this.resolveObjectPath(objectKey);
|
|
456
|
+
node_fs_1.default.rmSync(objectPath, { force: true });
|
|
457
|
+
try {
|
|
458
|
+
node_fs_1.default.rmdirSync(node_path_1.default.dirname(objectPath));
|
|
459
|
+
}
|
|
460
|
+
catch {
|
|
461
|
+
// A shared hash prefix may still contain other managed objects.
|
|
462
|
+
}
|
|
463
|
+
return true;
|
|
464
|
+
}
|
|
465
|
+
catch {
|
|
466
|
+
return false;
|
|
467
|
+
}
|
|
468
|
+
}
|
|
469
|
+
deleteOrphanObjects(limit) {
|
|
470
|
+
const referenced = new Set(this.store.listReferencedEmailAttachmentObjectKeys());
|
|
471
|
+
let deleted = 0;
|
|
472
|
+
let prefixes;
|
|
473
|
+
try {
|
|
474
|
+
prefixes = node_fs_1.default.readdirSync(this.objectsDir, { withFileTypes: true });
|
|
475
|
+
}
|
|
476
|
+
catch {
|
|
477
|
+
return 0;
|
|
478
|
+
}
|
|
479
|
+
for (const prefix of prefixes) {
|
|
480
|
+
if (deleted >= limit || !prefix.isDirectory() || !/^[a-f0-9]{2}$/.test(prefix.name)) {
|
|
481
|
+
continue;
|
|
482
|
+
}
|
|
483
|
+
const prefixPath = node_path_1.default.join(this.objectsDir, prefix.name);
|
|
484
|
+
let objects;
|
|
485
|
+
try {
|
|
486
|
+
objects = node_fs_1.default.readdirSync(prefixPath, { withFileTypes: true });
|
|
487
|
+
}
|
|
488
|
+
catch {
|
|
489
|
+
continue;
|
|
490
|
+
}
|
|
491
|
+
for (const object of objects) {
|
|
492
|
+
if (deleted >= limit) {
|
|
493
|
+
break;
|
|
494
|
+
}
|
|
495
|
+
const objectKey = node_path_1.default.posix.join("objects", prefix.name, object.name);
|
|
496
|
+
if (object.isFile()
|
|
497
|
+
&& /^[a-f0-9]{64}$/.test(object.name)
|
|
498
|
+
&& !referenced.has(objectKey)
|
|
499
|
+
&& this.deleteObjectIfUnreferenced(objectKey)) {
|
|
500
|
+
deleted += 1;
|
|
501
|
+
}
|
|
502
|
+
}
|
|
503
|
+
}
|
|
504
|
+
return deleted;
|
|
505
|
+
}
|
|
506
|
+
deleteStaleStagingFiles(olderThanMs, limit) {
|
|
507
|
+
let entries;
|
|
508
|
+
try {
|
|
509
|
+
entries = node_fs_1.default.readdirSync(this.stagingDir, { withFileTypes: true });
|
|
510
|
+
}
|
|
511
|
+
catch {
|
|
512
|
+
return 0;
|
|
513
|
+
}
|
|
514
|
+
let deleted = 0;
|
|
515
|
+
for (const entry of entries) {
|
|
516
|
+
if (deleted >= limit || !entry.isFile() || !/^[a-f0-9]{36}$/.test(entry.name)) {
|
|
517
|
+
continue;
|
|
518
|
+
}
|
|
519
|
+
const stagingPath = node_path_1.default.join(this.stagingDir, entry.name);
|
|
520
|
+
try {
|
|
521
|
+
if (node_fs_1.default.statSync(stagingPath).mtimeMs <= olderThanMs) {
|
|
522
|
+
node_fs_1.default.rmSync(stagingPath, { force: true });
|
|
523
|
+
deleted += 1;
|
|
524
|
+
}
|
|
525
|
+
}
|
|
526
|
+
catch {
|
|
527
|
+
// A concurrent materialization may already have published or removed it.
|
|
528
|
+
}
|
|
529
|
+
}
|
|
530
|
+
return deleted;
|
|
531
|
+
}
|
|
532
|
+
async retrieve(handle, stagingPath, maxBytes) {
|
|
533
|
+
const result = await this.uxc.request("email.attachment.get", {
|
|
534
|
+
handle: JSON.stringify(handle),
|
|
535
|
+
output: stagingPath,
|
|
536
|
+
max_bytes: maxBytes,
|
|
537
|
+
});
|
|
538
|
+
if (!isRecord(result)) {
|
|
539
|
+
throw new EmailAttachmentError("attachment_provider_invalid_result", false, "UXC returned an invalid attachment result.");
|
|
540
|
+
}
|
|
541
|
+
const size = optionalInteger(result.size_bytes);
|
|
542
|
+
const sha256 = optionalString(result.sha256);
|
|
543
|
+
if (size == null || !sha256 || !/^[a-f0-9]{64}$/i.test(sha256)) {
|
|
544
|
+
throw new EmailAttachmentError("attachment_provider_invalid_result", false, "UXC returned an invalid attachment result.");
|
|
545
|
+
}
|
|
546
|
+
return {
|
|
547
|
+
size_bytes: size,
|
|
548
|
+
sha256: sha256.toLowerCase(),
|
|
549
|
+
content_type: optionalString(result.content_type),
|
|
550
|
+
};
|
|
551
|
+
}
|
|
552
|
+
resolveOwned(agentId, attachmentRef) {
|
|
553
|
+
const inbox = this.store.getInboxByAgentId(agentId);
|
|
554
|
+
const parsed = inbox ? attachmentRefParts(attachmentRef) : null;
|
|
555
|
+
const item = parsed && inbox
|
|
556
|
+
? this.store.getInboxItemForInbox(inbox.inboxId, parsed.itemId)
|
|
557
|
+
: null;
|
|
558
|
+
const source = item ? this.store.getSource(item.sourceId) : null;
|
|
559
|
+
if (!inbox
|
|
560
|
+
|| !item
|
|
561
|
+
|| item.eventVariant !== "email.message.received"
|
|
562
|
+
|| source?.sourceType !== "email_mailbox") {
|
|
563
|
+
throw unknownAttachment(attachmentRef);
|
|
564
|
+
}
|
|
565
|
+
const resolved = (0, email_attachment_1.resolveEmailAttachment)(item, attachmentRef, (itemId, selector) => this.store.getEmailAttachmentMaterialization(itemId, selector));
|
|
566
|
+
if (!resolved) {
|
|
567
|
+
throw unknownAttachment(attachmentRef);
|
|
568
|
+
}
|
|
569
|
+
return {
|
|
570
|
+
inboxId: inbox.inboxId,
|
|
571
|
+
item,
|
|
572
|
+
source,
|
|
573
|
+
...resolved,
|
|
574
|
+
};
|
|
575
|
+
}
|
|
576
|
+
async putObject(stagingPath, sha256, objectKey) {
|
|
577
|
+
const objectPath = this.resolveObjectPath(objectKey);
|
|
578
|
+
node_fs_1.default.mkdirSync(node_path_1.default.dirname(objectPath), { recursive: true, mode: 0o700 });
|
|
579
|
+
node_fs_1.default.chmodSync(node_path_1.default.dirname(objectPath), 0o700);
|
|
580
|
+
if (node_fs_1.default.existsSync(objectPath)) {
|
|
581
|
+
const stagedSize = node_fs_1.default.statSync(stagingPath).size;
|
|
582
|
+
const existing = node_fs_1.default.statSync(objectPath);
|
|
583
|
+
if (!existing.isFile()
|
|
584
|
+
|| existing.size !== stagedSize
|
|
585
|
+
|| await hashFile(objectPath) !== sha256) {
|
|
586
|
+
throw new EmailAttachmentError("attachment_object_integrity_mismatch", false, "The managed attachment object failed integrity verification.");
|
|
587
|
+
}
|
|
588
|
+
node_fs_1.default.rmSync(stagingPath, { force: true });
|
|
589
|
+
}
|
|
590
|
+
else {
|
|
591
|
+
node_fs_1.default.renameSync(stagingPath, objectPath);
|
|
592
|
+
node_fs_1.default.chmodSync(objectPath, 0o600);
|
|
593
|
+
syncDirectory(node_path_1.default.dirname(objectPath));
|
|
594
|
+
}
|
|
595
|
+
}
|
|
596
|
+
beginObjectPublication(objectKey) {
|
|
597
|
+
this.publishingObjects.set(objectKey, (this.publishingObjects.get(objectKey) ?? 0) + 1);
|
|
598
|
+
}
|
|
599
|
+
endObjectPublication(objectKey) {
|
|
600
|
+
const count = this.publishingObjects.get(objectKey) ?? 0;
|
|
601
|
+
if (count <= 1) {
|
|
602
|
+
this.publishingObjects.delete(objectKey);
|
|
603
|
+
return;
|
|
604
|
+
}
|
|
605
|
+
this.publishingObjects.set(objectKey, count - 1);
|
|
606
|
+
}
|
|
607
|
+
objectExists(objectKey) {
|
|
608
|
+
try {
|
|
609
|
+
return node_fs_1.default.statSync(this.resolveObjectPath(objectKey)).isFile();
|
|
610
|
+
}
|
|
611
|
+
catch {
|
|
612
|
+
return false;
|
|
613
|
+
}
|
|
614
|
+
}
|
|
615
|
+
resolveObjectPath(objectKey) {
|
|
616
|
+
if (!/^objects\/[a-f0-9]{2}\/[a-f0-9]{64}$/.test(objectKey)) {
|
|
617
|
+
throw new EmailAttachmentError("attachment_content_unavailable", false, "Email attachment content is not available.");
|
|
618
|
+
}
|
|
619
|
+
const resolved = node_path_1.default.resolve(node_path_1.default.dirname(this.objectsDir), objectKey);
|
|
620
|
+
const root = `${node_path_1.default.resolve(node_path_1.default.dirname(this.objectsDir))}${node_path_1.default.sep}`;
|
|
621
|
+
if (!resolved.startsWith(root)) {
|
|
622
|
+
throw new EmailAttachmentError("attachment_content_unavailable", false, "Email attachment content is not available.");
|
|
623
|
+
}
|
|
624
|
+
return resolved;
|
|
625
|
+
}
|
|
626
|
+
}
|
|
627
|
+
exports.EmailAttachmentContentManager = EmailAttachmentContentManager;
|
|
628
|
+
function attachmentRefParts(attachmentRef) {
|
|
629
|
+
const match = /^att_v1\.([A-Za-z0-9_-]+)\.[A-Za-z0-9_-]{22}$/.exec(attachmentRef);
|
|
630
|
+
return match ? { itemId: match[1] } : null;
|
|
631
|
+
}
|
|
632
|
+
function objectKeyForSha256(sha256) {
|
|
633
|
+
return node_path_1.default.posix.join("objects", sha256.slice(0, 2), sha256);
|
|
634
|
+
}
|
|
635
|
+
function unknownAttachment(attachmentRef) {
|
|
636
|
+
return new Error(`unknown inbox attachment: ${attachmentRef}`);
|
|
637
|
+
}
|
|
638
|
+
function attachmentFailure(error) {
|
|
639
|
+
if (error instanceof EmailAttachmentError) {
|
|
640
|
+
return error;
|
|
641
|
+
}
|
|
642
|
+
const data = isRecord(error) && isRecord(error.data) ? error.data : {};
|
|
643
|
+
const code = stableAttachmentErrorCode(data.code, "");
|
|
644
|
+
if (code) {
|
|
645
|
+
return new EmailAttachmentError(code, false, "Email attachment retrieval failed.");
|
|
646
|
+
}
|
|
647
|
+
return new EmailAttachmentError("attachment_provider_unavailable", false, "Email attachment retrieval is temporarily unavailable.");
|
|
648
|
+
}
|
|
649
|
+
function assertContentTypeAllowed(contentType, policy, phase) {
|
|
650
|
+
if (!contentType) {
|
|
651
|
+
return;
|
|
652
|
+
}
|
|
653
|
+
const normalized = contentType.split(";", 1)[0].trim().toLowerCase();
|
|
654
|
+
if (policy.denyContentTypes.some((pattern) => contentTypeMatches(normalized, pattern))) {
|
|
655
|
+
throw new EmailAttachmentError(`attachment_${phase}_content_type_denied`, true, "The attachment content type is denied by source policy.");
|
|
656
|
+
}
|
|
657
|
+
if (policy.allowContentTypes.length > 0
|
|
658
|
+
&& !policy.allowContentTypes.some((pattern) => contentTypeMatches(normalized, pattern))) {
|
|
659
|
+
throw new EmailAttachmentError(`attachment_${phase}_content_type_not_allowed`, true, "The attachment content type is not allowed by source policy.");
|
|
660
|
+
}
|
|
661
|
+
}
|
|
662
|
+
function contentTypeMatches(contentType, pattern) {
|
|
663
|
+
return pattern.endsWith("/*")
|
|
664
|
+
? contentType.startsWith(pattern.slice(0, -1))
|
|
665
|
+
: contentType === pattern;
|
|
666
|
+
}
|
|
667
|
+
async function hashFile(filePath) {
|
|
668
|
+
const hash = node_crypto_1.default.createHash("sha256");
|
|
669
|
+
const stream = node_fs_1.default.createReadStream(filePath);
|
|
670
|
+
for await (const chunk of stream) {
|
|
671
|
+
hash.update(chunk);
|
|
672
|
+
}
|
|
673
|
+
return hash.digest("hex");
|
|
674
|
+
}
|
|
675
|
+
function syncFile(filePath) {
|
|
676
|
+
const fd = node_fs_1.default.openSync(filePath, "r");
|
|
677
|
+
try {
|
|
678
|
+
node_fs_1.default.fsyncSync(fd);
|
|
679
|
+
}
|
|
680
|
+
finally {
|
|
681
|
+
node_fs_1.default.closeSync(fd);
|
|
682
|
+
}
|
|
683
|
+
}
|
|
684
|
+
function syncDirectory(directoryPath) {
|
|
685
|
+
const fd = node_fs_1.default.openSync(directoryPath, "r");
|
|
686
|
+
try {
|
|
687
|
+
node_fs_1.default.fsyncSync(fd);
|
|
688
|
+
}
|
|
689
|
+
finally {
|
|
690
|
+
node_fs_1.default.closeSync(fd);
|
|
691
|
+
}
|
|
692
|
+
}
|
|
693
|
+
function detectContentType(filePath) {
|
|
694
|
+
const fd = node_fs_1.default.openSync(filePath, "r");
|
|
695
|
+
try {
|
|
696
|
+
const buffer = Buffer.alloc(512);
|
|
697
|
+
const bytes = node_fs_1.default.readSync(fd, buffer, 0, buffer.length, 0);
|
|
698
|
+
const header = buffer.subarray(0, bytes);
|
|
699
|
+
if (header.subarray(0, 5).toString("ascii") === "%PDF-") {
|
|
700
|
+
return "application/pdf";
|
|
701
|
+
}
|
|
702
|
+
if (header.subarray(0, 8).equals(Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]))) {
|
|
703
|
+
return "image/png";
|
|
704
|
+
}
|
|
705
|
+
if (header[0] === 0xff && header[1] === 0xd8 && header[2] === 0xff) {
|
|
706
|
+
return "image/jpeg";
|
|
707
|
+
}
|
|
708
|
+
const gif = header.subarray(0, 6).toString("ascii");
|
|
709
|
+
if (gif === "GIF87a" || gif === "GIF89a") {
|
|
710
|
+
return "image/gif";
|
|
711
|
+
}
|
|
712
|
+
if (header[0] === 0x50 && header[1] === 0x4b && header[2] === 0x03 && header[3] === 0x04) {
|
|
713
|
+
return "application/zip";
|
|
714
|
+
}
|
|
715
|
+
if (header.length > 0 && !header.includes(0) && Buffer.from(header.toString("utf8"), "utf8").equals(header)) {
|
|
716
|
+
return "text/plain";
|
|
717
|
+
}
|
|
718
|
+
return "application/octet-stream";
|
|
719
|
+
}
|
|
720
|
+
finally {
|
|
721
|
+
node_fs_1.default.closeSync(fd);
|
|
722
|
+
}
|
|
723
|
+
}
|
|
724
|
+
function isRecord(value) {
|
|
725
|
+
return Boolean(value) && typeof value === "object" && !Array.isArray(value);
|
|
726
|
+
}
|
|
727
|
+
function optionalString(value) {
|
|
728
|
+
return typeof value === "string" ? value : null;
|
|
729
|
+
}
|
|
730
|
+
function optionalInteger(value) {
|
|
731
|
+
return Number.isSafeInteger(value) && Number(value) >= 0 ? Number(value) : null;
|
|
732
|
+
}
|
|
733
|
+
function safeAuditAttachmentRef(attachmentRef) {
|
|
734
|
+
return /^att_v1\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]{22}$/.test(attachmentRef)
|
|
735
|
+
? attachmentRef
|
|
736
|
+
: "invalid_attachment_ref";
|
|
737
|
+
}
|
|
738
|
+
function stableAttachmentErrorCode(value, fallback) {
|
|
739
|
+
return typeof value === "string" && /^[a-z][a-z0-9_]{0,63}$/.test(value)
|
|
740
|
+
? value
|
|
741
|
+
: fallback;
|
|
742
|
+
}
|
|
743
|
+
function auditErrorCode(error) {
|
|
744
|
+
if (error instanceof EmailAttachmentError) {
|
|
745
|
+
return error.code;
|
|
746
|
+
}
|
|
747
|
+
if (error instanceof Error && error.message.startsWith("unknown inbox attachment:")) {
|
|
748
|
+
return "attachment_not_found";
|
|
749
|
+
}
|
|
750
|
+
return "attachment_operation_failed";
|
|
751
|
+
}
|
|
752
|
+
function positiveIntegerOrDefault(value, fallback) {
|
|
753
|
+
const parsed = typeof value === "string" && value.trim().length > 0
|
|
754
|
+
? Number(value)
|
|
755
|
+
: value;
|
|
756
|
+
return Number.isSafeInteger(parsed) && Number(parsed) > 0
|
|
757
|
+
? Number(parsed)
|
|
758
|
+
: fallback;
|
|
759
|
+
}
|