@remit/mailbox-service 0.0.14 → 0.0.16
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/src/body-parse.test.ts +55 -0
- package/src/body-parse.ts +61 -0
- package/src/body-sync-quarantine.test.ts +219 -0
- package/src/body-sync.ts +167 -3
- package/src/imapflow-connection.test.ts +49 -0
- package/src/imapflow-connection.ts +19 -23
- package/src/index.ts +15 -0
- package/src/message-sync-quarantine.test.ts +407 -0
- package/src/message-sync.ts +135 -18
- package/src/quarantine.test.ts +191 -0
- package/src/quarantine.ts +228 -0
- package/src/sync-watermarks.test.ts +0 -9
- package/src/sync-watermarks.ts +1 -4
package/src/message-sync.ts
CHANGED
|
@@ -36,6 +36,7 @@ import {
|
|
|
36
36
|
} from "./mailbox-cursor-rebuild.js";
|
|
37
37
|
import { ROOT_PART_PATH, walkMimeStructure } from "./mime-walker.js";
|
|
38
38
|
import { PassThroughUnitOfWork } from "./pass-through-unit-of-work.js";
|
|
39
|
+
import type { QuarantineService } from "./quarantine.js";
|
|
39
40
|
import { reconcileStaleMessage } from "./stale-message-reconcile.js";
|
|
40
41
|
import {
|
|
41
42
|
advanceChangeCursor,
|
|
@@ -146,8 +147,11 @@ interface SaveMessageResult extends SyncedMessage {
|
|
|
146
147
|
/**
|
|
147
148
|
* Wrapper outcome for a single message in the batch. A `failed` outcome means
|
|
148
149
|
* the save threw (and was caught) — its UID must NOT advance the watermark, so
|
|
149
|
-
* the message is re-fetched and retried on the next cycle. `
|
|
150
|
-
*
|
|
150
|
+
* the message is re-fetched and retried on the next cycle. A `saved` outcome
|
|
151
|
+
* with a `null` result is a UID this round finished with but created no row
|
|
152
|
+
* for: a cross-mailbox collision, a change applied to an existing row, or a
|
|
153
|
+
* message quarantined instead of applied (issue #72). Its watermark advances
|
|
154
|
+
* either way, because there is nothing left to retry.
|
|
151
155
|
*/
|
|
152
156
|
type BatchOutcome =
|
|
153
157
|
| { kind: "saved"; uid: number; result: SaveMessageResult | null }
|
|
@@ -224,6 +228,13 @@ export class MessageSyncService {
|
|
|
224
228
|
* flip is never dismissed as redundant.
|
|
225
229
|
*/
|
|
226
230
|
private messageFlagService?: IMessageFlagRepository,
|
|
231
|
+
/**
|
|
232
|
+
* The set of messages already set aside (issue #72). Message sync never
|
|
233
|
+
* writes a record — only the body path can attribute a failure to the
|
|
234
|
+
* message — but it reads the set so a uid the body path quarantined is
|
|
235
|
+
* not re-fetched on every round.
|
|
236
|
+
*/
|
|
237
|
+
private quarantineService?: QuarantineService,
|
|
227
238
|
) {
|
|
228
239
|
this.log = logger ?? noopLogger;
|
|
229
240
|
this.unitOfWork =
|
|
@@ -309,6 +320,10 @@ export class MessageSyncService {
|
|
|
309
320
|
});
|
|
310
321
|
}
|
|
311
322
|
|
|
323
|
+
// One read per round, not per message (issue #72). The list is small by
|
|
324
|
+
// design — a growing one is a bug being reported, not a page to paginate.
|
|
325
|
+
const quarantined = await this.quarantineService?.load(accountConfigId);
|
|
326
|
+
|
|
312
327
|
const allUids = await connection.search(["ALL"]);
|
|
313
328
|
const uids = selectUidsToSync(allUids, lastSyncUid, highWaterMarkUid);
|
|
314
329
|
const unseenCount = status.unseen;
|
|
@@ -350,7 +365,42 @@ export class MessageSyncService {
|
|
|
350
365
|
|
|
351
366
|
// Process only the first batch
|
|
352
367
|
const batchUids = uids.slice(0, batchSize);
|
|
353
|
-
|
|
368
|
+
|
|
369
|
+
// A quarantined UID is not fetched again, but it stays in `batchUids` so
|
|
370
|
+
// the watermark still advances over it. Filtering it out of the selection
|
|
371
|
+
// instead would hold the watermark below a message that is already
|
|
372
|
+
// durably resolved, which is the stall by another route.
|
|
373
|
+
const fetchUids = batchUids.filter(
|
|
374
|
+
(uid) => !quarantined?.has(mailboxId, box.uidvalidity, uid),
|
|
375
|
+
);
|
|
376
|
+
const messages =
|
|
377
|
+
fetchUids.length > 0 ? await this.fetchMessageBatch(fetchUids) : [];
|
|
378
|
+
|
|
379
|
+
// A UID the round could not act on. Two shapes reach here and neither is
|
|
380
|
+
// the message's fault: the FETCH returned no row at all (the connection
|
|
381
|
+
// layer drops rows imapflow yields without a usable UID or INTERNALDATE,
|
|
382
|
+
// #408, and a message can be expunged between the SEARCH and the FETCH),
|
|
383
|
+
// or it returned a row carrying no ENVELOPE, which names no message and
|
|
384
|
+
// is the same client-side glitch one field further in.
|
|
385
|
+
//
|
|
386
|
+
// They join `failedUids` rather than merely being left out of the batch.
|
|
387
|
+
// Absence alone does not hold a watermark: `advanceUidWatermarks` takes
|
|
388
|
+
// the MAX of what was applied, so a gap in the middle of a batch is
|
|
389
|
+
// stepped straight over — [23, 22, 21] with 22 missing still advances to
|
|
390
|
+
// 23 and loses 22 for good. A failure is the one thing a watermark is
|
|
391
|
+
// built to stop below.
|
|
392
|
+
const applicable = messages.filter((msg) => msg.envelope !== undefined);
|
|
393
|
+
const unusableUids = batchUids.filter(
|
|
394
|
+
(uid) =>
|
|
395
|
+
!applicable.some((msg) => msg.uid === uid) &&
|
|
396
|
+
!quarantined?.has(mailboxId, box.uidvalidity, uid),
|
|
397
|
+
);
|
|
398
|
+
if (unusableUids.length > 0) {
|
|
399
|
+
this.log.warn(
|
|
400
|
+
{ mailboxId, mailboxPath, unusableUids },
|
|
401
|
+
"FETCH returned no usable row for some requested UIDs; holding the watermark below them",
|
|
402
|
+
);
|
|
403
|
+
}
|
|
354
404
|
|
|
355
405
|
// Process messages in parallel with concurrency limit. `stopOnError` stays
|
|
356
406
|
// at its default — but each message is saved through `trySaveMessage`,
|
|
@@ -358,7 +408,7 @@ export class MessageSyncService {
|
|
|
358
408
|
// rejecting. So one bad message can no longer abort the whole batch (the
|
|
359
409
|
// poison pill that previously froze the mailbox, #817).
|
|
360
410
|
const outcomes = await pMap(
|
|
361
|
-
|
|
411
|
+
applicable,
|
|
362
412
|
(msg) => this.trySaveMessage(mailboxId, accountId, accountConfigId, msg),
|
|
363
413
|
{ concurrency: MESSAGE_SAVE_CONCURRENCY },
|
|
364
414
|
);
|
|
@@ -379,15 +429,16 @@ export class MessageSyncService {
|
|
|
379
429
|
|
|
380
430
|
// UIDs whose save threw. They must stay inside the next cycle's fetch
|
|
381
431
|
// window, so the watermark may not advance past them (no silent loss).
|
|
382
|
-
const
|
|
383
|
-
|
|
432
|
+
const saveFailedUids = outcomes.flatMap((o) =>
|
|
433
|
+
o.kind === "failed" ? [o.uid] : [],
|
|
384
434
|
);
|
|
385
|
-
if (
|
|
435
|
+
if (saveFailedUids.length > 0) {
|
|
386
436
|
this.log.warn(
|
|
387
|
-
{ mailboxId, mailboxPath, failedUids:
|
|
437
|
+
{ mailboxId, mailboxPath, failedUids: saveFailedUids },
|
|
388
438
|
"Some messages failed to save; holding watermark below them for retry",
|
|
389
439
|
);
|
|
390
440
|
}
|
|
441
|
+
const failedUids = new Set([...saveFailedUids, ...unusableUids]);
|
|
391
442
|
|
|
392
443
|
// Watermarks advance over every SUCCESSFULLY-consumed UID in the batch,
|
|
393
444
|
// independent of ownership. `selectUidsToSync` reselects work purely by UID
|
|
@@ -413,7 +464,8 @@ export class MessageSyncService {
|
|
|
413
464
|
// enumeration work behind and lost no message to a failed save. Seeding
|
|
414
465
|
// it earlier would switch the mailbox to CHANGEDSINCE while UIDs it has
|
|
415
466
|
// never fetched still sit below the watermark, and those messages would
|
|
416
|
-
// never be discovered.
|
|
467
|
+
// never be discovered. A UID the FETCH returned nothing usable for is in
|
|
468
|
+
// `failedUids` too: the round did not finish with it either.
|
|
417
469
|
const enumerationComplete = !hasMore && failedUids.size === 0;
|
|
418
470
|
|
|
419
471
|
await this.mailboxService.update(accountId, mailboxId, {
|
|
@@ -587,8 +639,9 @@ export class MessageSyncService {
|
|
|
587
639
|
|
|
588
640
|
const newMessages =
|
|
589
641
|
newUids.length > 0 ? await this.fetchMessageBatch(newUids) : [];
|
|
642
|
+
const applicable = newMessages.filter((msg) => msg.envelope !== undefined);
|
|
590
643
|
const outcomes = await pMap(
|
|
591
|
-
|
|
644
|
+
applicable,
|
|
592
645
|
(msg) => this.trySaveMessage(mailboxId, accountId, accountConfigId, msg),
|
|
593
646
|
{ concurrency: MESSAGE_SAVE_CONCURRENCY },
|
|
594
647
|
);
|
|
@@ -600,6 +653,31 @@ export class MessageSyncService {
|
|
|
600
653
|
|
|
601
654
|
const serverUids = serverSnapshots.map((s) => s.uid);
|
|
602
655
|
|
|
656
|
+
// A UID this pass could not account for, in any of the three ways it can
|
|
657
|
+
// go missing: the snapshot FETCH never returned a row for it, the
|
|
658
|
+
// message FETCH never returned one, or the row it returned carried no
|
|
659
|
+
// ENVELOPE. None of the three is the message's fault and none can be
|
|
660
|
+
// quarantined, so each has to keep the UID selectable.
|
|
661
|
+
//
|
|
662
|
+
// This matters more here than on the enumeration path, not less. The
|
|
663
|
+
// covered region is computed from `serverUids` rather than from what was
|
|
664
|
+
// applied, so a UID missing anywhere inside its span is silently inside
|
|
665
|
+
// it; and this round seeds the mod-sequence and returns the mailbox to
|
|
666
|
+
// `normal`, so the next round takes CHANGEDSINCE, which never
|
|
667
|
+
// enumerates. A UID lost here is lost for good.
|
|
668
|
+
const savedUids = new Set(applicable.map((msg) => msg.uid));
|
|
669
|
+
const snapshotUids = new Set(serverUids);
|
|
670
|
+
const unusableUids = [
|
|
671
|
+
...allUids.filter((uid) => !snapshotUids.has(uid)),
|
|
672
|
+
...newUids.filter((uid) => !savedUids.has(uid)),
|
|
673
|
+
];
|
|
674
|
+
if (unusableUids.length > 0) {
|
|
675
|
+
this.log.warn(
|
|
676
|
+
{ mailboxId, mailboxPath, unusableUids },
|
|
677
|
+
"Cursor rebuild could not account for some UIDs; holding the watermark below them",
|
|
678
|
+
);
|
|
679
|
+
}
|
|
680
|
+
|
|
603
681
|
// A new message whose save threw must stay selectable, so the forward
|
|
604
682
|
// watermark stops below it and every UID above it is re-enumerated next
|
|
605
683
|
// round. The mod-sequence seed is withheld entirely in that case: the
|
|
@@ -607,9 +685,10 @@ export class MessageSyncService {
|
|
|
607
685
|
// it on a UIDVALIDITY change) and the new one would sit above the
|
|
608
686
|
// message that failed, so the mailbox goes back to enumeration until a
|
|
609
687
|
// clean round seeds it.
|
|
610
|
-
const failedUids = new Set(
|
|
611
|
-
outcomes.flatMap((o) => (o.kind === "failed" ? [o.uid] : [])),
|
|
612
|
-
|
|
688
|
+
const failedUids = new Set([
|
|
689
|
+
...outcomes.flatMap((o) => (o.kind === "failed" ? [o.uid] : [])),
|
|
690
|
+
...unusableUids,
|
|
691
|
+
]);
|
|
613
692
|
const lowestFailure = failedUids.size
|
|
614
693
|
? Math.min(...failedUids)
|
|
615
694
|
: Number.POSITIVE_INFINITY;
|
|
@@ -756,21 +835,53 @@ export class MessageSyncService {
|
|
|
756
835
|
const ordered = dropAppliedPrefix(orderByModseq(changed), cursor);
|
|
757
836
|
const batch = ordered.slice(0, batchSize);
|
|
758
837
|
|
|
838
|
+
// A quarantined UID stays in `batch`, so the cursor still advances over
|
|
839
|
+
// it; only the work of re-applying it is skipped.
|
|
840
|
+
const quarantined = await this.quarantineService?.load(accountConfigId);
|
|
841
|
+
const applicable = batch.filter(
|
|
842
|
+
(msg) =>
|
|
843
|
+
!quarantined?.has(mailboxId, box.uidvalidity, msg.uid) &&
|
|
844
|
+
msg.envelope !== undefined,
|
|
845
|
+
);
|
|
846
|
+
|
|
847
|
+
// A change row carrying no ENVELOPE holds the cursor and is retried; it
|
|
848
|
+
// is never set aside. On this path the message is usually one already
|
|
849
|
+
// stored — it demonstrably had a sender, a date and a Message-ID when it
|
|
850
|
+
// was first saved — so an envelope-less row is the FETCH glitching
|
|
851
|
+
// (#408), not the message being defective. Quarantining it would filter
|
|
852
|
+
// that UID out of every later round, stopping its flag sync until a
|
|
853
|
+
// purge, and tell the user a message they can open and read "arrived
|
|
854
|
+
// without a sender". A transient glitch heals on the retry; a persistent
|
|
855
|
+
// one trips the stalled-cursor alert, which is what that alert is for.
|
|
856
|
+
const unusableUids = batch.flatMap((msg) =>
|
|
857
|
+
msg.envelope === undefined &&
|
|
858
|
+
!quarantined?.has(mailboxId, box.uidvalidity, msg.uid)
|
|
859
|
+
? [msg.uid]
|
|
860
|
+
: [],
|
|
861
|
+
);
|
|
862
|
+
if (unusableUids.length > 0) {
|
|
863
|
+
this.log.warn(
|
|
864
|
+
{ mailboxId, mailboxPath, unusableUids },
|
|
865
|
+
"Change rows carried no ENVELOPE; holding the sync cursor below them",
|
|
866
|
+
);
|
|
867
|
+
}
|
|
868
|
+
|
|
759
869
|
const outcomes = await pMap(
|
|
760
|
-
|
|
870
|
+
applicable,
|
|
761
871
|
(msg) => this.tryApplyChange(mailboxId, accountId, accountConfigId, msg),
|
|
762
872
|
{ concurrency: MESSAGE_SAVE_CONCURRENCY },
|
|
763
873
|
);
|
|
764
874
|
|
|
765
|
-
const
|
|
766
|
-
|
|
875
|
+
const saveFailedUids = outcomes.flatMap((o) =>
|
|
876
|
+
o.kind === "failed" ? [o.uid] : [],
|
|
767
877
|
);
|
|
768
|
-
if (
|
|
878
|
+
if (saveFailedUids.length > 0) {
|
|
769
879
|
this.log.warn(
|
|
770
|
-
{ mailboxId, mailboxPath, failedUids:
|
|
880
|
+
{ mailboxId, mailboxPath, failedUids: saveFailedUids },
|
|
771
881
|
"Some changes failed to apply; holding the sync cursor below them for retry",
|
|
772
882
|
);
|
|
773
883
|
}
|
|
884
|
+
const failedUids = new Set([...saveFailedUids, ...unusableUids]);
|
|
774
885
|
|
|
775
886
|
// Body sync only concerns messages this round created — a metadata
|
|
776
887
|
// change has no new body to fetch.
|
|
@@ -1231,6 +1342,12 @@ export class MessageSyncService {
|
|
|
1231
1342
|
order: number;
|
|
1232
1343
|
}> = [];
|
|
1233
1344
|
|
|
1345
|
+
// An unusable address is dropped and the message is written anyway, and
|
|
1346
|
+
// that stays deliberate under the quarantine rules (issue #72). What is
|
|
1347
|
+
// lost is one envelope address, not the message: it is stored, listed and
|
|
1348
|
+
// readable, and its body is untouched. Setting the whole message aside
|
|
1349
|
+
// over a malformed From would take readable mail out of the mailbox to
|
|
1350
|
+
// protect a display name.
|
|
1234
1351
|
for (let i = 0; i < addresses.length; i++) {
|
|
1235
1352
|
const addr = addresses[i];
|
|
1236
1353
|
if (!isParseableEmailAddress(addr)) continue;
|
|
@@ -0,0 +1,191 @@
|
|
|
1
|
+
import assert from "node:assert/strict";
|
|
2
|
+
import { describe, it } from "node:test";
|
|
3
|
+
import type {
|
|
4
|
+
IMailboxSpecialUseRepository,
|
|
5
|
+
IQuarantineRepository,
|
|
6
|
+
MessageData,
|
|
7
|
+
MessageItem,
|
|
8
|
+
QuarantineItem,
|
|
9
|
+
QuarantineUpsertInput,
|
|
10
|
+
} from "@remit/data-ports";
|
|
11
|
+
import {
|
|
12
|
+
QuarantinedUids,
|
|
13
|
+
QuarantineService,
|
|
14
|
+
resolveMailboxRole,
|
|
15
|
+
shapeFromMessageData,
|
|
16
|
+
} from "./quarantine.js";
|
|
17
|
+
|
|
18
|
+
const noopLog = { info: () => {}, warn: () => {} };
|
|
19
|
+
|
|
20
|
+
const buildService = (
|
|
21
|
+
specialUse: string[] = [],
|
|
22
|
+
): {
|
|
23
|
+
service: QuarantineService;
|
|
24
|
+
writes: QuarantineUpsertInput[];
|
|
25
|
+
listed: string[];
|
|
26
|
+
} => {
|
|
27
|
+
const writes: QuarantineUpsertInput[] = [];
|
|
28
|
+
const listed: string[] = [];
|
|
29
|
+
const repository = {
|
|
30
|
+
listByAccountConfigId: async (accountConfigId: string) => {
|
|
31
|
+
listed.push(accountConfigId);
|
|
32
|
+
return [] as QuarantineItem[];
|
|
33
|
+
},
|
|
34
|
+
upsert: async (input: QuarantineUpsertInput) => {
|
|
35
|
+
writes.push(input);
|
|
36
|
+
},
|
|
37
|
+
} satisfies IQuarantineRepository;
|
|
38
|
+
|
|
39
|
+
const mailboxSpecialUseService = {
|
|
40
|
+
listByMailboxId: async () => specialUse.map((use) => ({ specialUse: use })),
|
|
41
|
+
} as unknown as IMailboxSpecialUseRepository;
|
|
42
|
+
|
|
43
|
+
return {
|
|
44
|
+
service: new QuarantineService(
|
|
45
|
+
repository,
|
|
46
|
+
mailboxSpecialUseService,
|
|
47
|
+
"sha-abc",
|
|
48
|
+
noopLog,
|
|
49
|
+
),
|
|
50
|
+
writes,
|
|
51
|
+
listed,
|
|
52
|
+
};
|
|
53
|
+
};
|
|
54
|
+
|
|
55
|
+
const context = {
|
|
56
|
+
accountId: "acct-1",
|
|
57
|
+
accountConfigId: "cfg-1",
|
|
58
|
+
mailboxId: "mbx-1",
|
|
59
|
+
mailboxPath: "INBOX",
|
|
60
|
+
uidValidity: 1_712_000_000,
|
|
61
|
+
attempts: 2,
|
|
62
|
+
};
|
|
63
|
+
|
|
64
|
+
const failure = {
|
|
65
|
+
stage: "BodyParse" as const,
|
|
66
|
+
code: "UnreadableBody" as const,
|
|
67
|
+
message: "the parser said no",
|
|
68
|
+
};
|
|
69
|
+
|
|
70
|
+
describe("QuarantineService.record", () => {
|
|
71
|
+
it("stamps the worker build, not the client's", async () => {
|
|
72
|
+
const { service, writes } = buildService();
|
|
73
|
+
await service.record(context, 40217, failure);
|
|
74
|
+
assert.equal(writes[0]?.workerVersion, "sha-abc");
|
|
75
|
+
});
|
|
76
|
+
|
|
77
|
+
it("names the message by mailbox, uidValidity and uid", async () => {
|
|
78
|
+
const { service, writes } = buildService();
|
|
79
|
+
await service.record(context, 40217, failure);
|
|
80
|
+
assert.deepEqual(
|
|
81
|
+
{
|
|
82
|
+
mailboxId: writes[0]?.mailboxId,
|
|
83
|
+
uidValidity: writes[0]?.uidValidity,
|
|
84
|
+
uid: writes[0]?.uid,
|
|
85
|
+
},
|
|
86
|
+
{ mailboxId: "mbx-1", uidValidity: 1_712_000_000, uid: 40217 },
|
|
87
|
+
);
|
|
88
|
+
});
|
|
89
|
+
|
|
90
|
+
it("supplies no quarantineId, so the random-id default is unreachable", async () => {
|
|
91
|
+
const { service, writes } = buildService();
|
|
92
|
+
await service.record(context, 40217, failure);
|
|
93
|
+
assert.ok(!("quarantineId" in (writes[0] as object)));
|
|
94
|
+
});
|
|
95
|
+
|
|
96
|
+
it("omits an absent diagnostic rather than writing a null through", async () => {
|
|
97
|
+
const { service, writes } = buildService();
|
|
98
|
+
await service.record(
|
|
99
|
+
{ ...context, mailboxPath: "Clients/Acme" },
|
|
100
|
+
1,
|
|
101
|
+
failure,
|
|
102
|
+
);
|
|
103
|
+
assert.ok(!("failurePartPath" in (writes[0] as object)));
|
|
104
|
+
assert.ok(!("mailboxRole" in (writes[0] as object)));
|
|
105
|
+
});
|
|
106
|
+
|
|
107
|
+
it("reads the folder's role off the server's SPECIAL-USE", async () => {
|
|
108
|
+
const { service, writes } = buildService(["Junk"]);
|
|
109
|
+
await service.record({ ...context, mailboxPath: "Spam" }, 1, failure);
|
|
110
|
+
assert.equal(writes[0]?.mailboxRole, "Junk");
|
|
111
|
+
});
|
|
112
|
+
});
|
|
113
|
+
|
|
114
|
+
describe("resolveMailboxRole", () => {
|
|
115
|
+
it("gives INBOX its role without a SPECIAL-USE flag, which it never has", () => {
|
|
116
|
+
assert.equal(resolveMailboxRole("INBOX", []), "Inbox");
|
|
117
|
+
});
|
|
118
|
+
|
|
119
|
+
it("leaves a plain folder roleless instead of inventing one", () => {
|
|
120
|
+
assert.equal(resolveMailboxRole("Clients/Acme", []), undefined);
|
|
121
|
+
});
|
|
122
|
+
|
|
123
|
+
it("drops a SPECIAL-USE that is not a canonical role", () => {
|
|
124
|
+
assert.equal(resolveMailboxRole("Priority", ["Important"]), undefined);
|
|
125
|
+
});
|
|
126
|
+
});
|
|
127
|
+
|
|
128
|
+
describe("QuarantinedUids", () => {
|
|
129
|
+
const entries = [
|
|
130
|
+
{ mailboxId: "mbx-1", uidValidity: 10, uid: 5 },
|
|
131
|
+
] as QuarantineItem[];
|
|
132
|
+
|
|
133
|
+
it("matches a uid on the same mailbox and UIDVALIDITY", () => {
|
|
134
|
+
assert.equal(new QuarantinedUids(entries).has("mbx-1", 10, 5), true);
|
|
135
|
+
});
|
|
136
|
+
|
|
137
|
+
it("does not match the same uid on a new UIDVALIDITY, which is a different message", () => {
|
|
138
|
+
assert.equal(new QuarantinedUids(entries).has("mbx-1", 11, 5), false);
|
|
139
|
+
});
|
|
140
|
+
|
|
141
|
+
it("does not match the same uid in another mailbox", () => {
|
|
142
|
+
assert.equal(new QuarantinedUids(entries).has("mbx-2", 10, 5), false);
|
|
143
|
+
});
|
|
144
|
+
});
|
|
145
|
+
|
|
146
|
+
describe("shapeFromMessageData", () => {
|
|
147
|
+
const message = {
|
|
148
|
+
rfc822Size: 2048,
|
|
149
|
+
messageIdHeader: "<xyz@example.com>",
|
|
150
|
+
} as MessageItem;
|
|
151
|
+
|
|
152
|
+
const data = {
|
|
153
|
+
bodyPart: [
|
|
154
|
+
{
|
|
155
|
+
bodyPartId: "bp-0",
|
|
156
|
+
partPath: "0",
|
|
157
|
+
mediaType: "multipart",
|
|
158
|
+
mediaSubtype: "alternative",
|
|
159
|
+
transferEncoding: "7bit",
|
|
160
|
+
},
|
|
161
|
+
{
|
|
162
|
+
bodyPartId: "bp-1",
|
|
163
|
+
partPath: "1",
|
|
164
|
+
mediaType: "text",
|
|
165
|
+
mediaSubtype: "plain",
|
|
166
|
+
transferEncoding: "quoted-printable",
|
|
167
|
+
},
|
|
168
|
+
],
|
|
169
|
+
bodyPartParameter: [
|
|
170
|
+
{
|
|
171
|
+
bodyPartId: "bp-0",
|
|
172
|
+
parameterName: "charset",
|
|
173
|
+
parameterValue: "iso-8859-1",
|
|
174
|
+
},
|
|
175
|
+
],
|
|
176
|
+
} as unknown as MessageData;
|
|
177
|
+
|
|
178
|
+
it("rebuilds the tree from the rows metadata sync already wrote", () => {
|
|
179
|
+
assert.deepEqual(shapeFromMessageData(message, data).structure, [
|
|
180
|
+
{ depth: 0, contentType: "multipart/alternative" },
|
|
181
|
+
{ depth: 1, contentType: "text/plain" },
|
|
182
|
+
]);
|
|
183
|
+
});
|
|
184
|
+
|
|
185
|
+
it("takes the root part's declared charset and encoding", () => {
|
|
186
|
+
const shape = shapeFromMessageData(message, data);
|
|
187
|
+
assert.equal(shape.charset, "iso-8859-1");
|
|
188
|
+
assert.equal(shape.transferEncoding, "7bit");
|
|
189
|
+
assert.equal(shape.contentType, "multipart/alternative");
|
|
190
|
+
});
|
|
191
|
+
});
|
|
@@ -0,0 +1,228 @@
|
|
|
1
|
+
import type {
|
|
2
|
+
IMailboxSpecialUseRepository,
|
|
3
|
+
IQuarantineRepository,
|
|
4
|
+
MessageData,
|
|
5
|
+
MessageItem,
|
|
6
|
+
QuarantineItem,
|
|
7
|
+
QuarantineMimeNodeItem,
|
|
8
|
+
} from "@remit/data-ports";
|
|
9
|
+
import { quarantineMessageIdHash, ROOT_PART_PATH } from "@remit/data-ports/id";
|
|
10
|
+
import { CanonicalMailboxRole } from "@remit/domain-enums";
|
|
11
|
+
|
|
12
|
+
type CanonicalRole = NonNullable<QuarantineItem["mailboxRole"]>;
|
|
13
|
+
type FailureStage = QuarantineItem["failureStage"];
|
|
14
|
+
type FailureCode = QuarantineItem["failureCode"];
|
|
15
|
+
|
|
16
|
+
const CANONICAL_ROLES = new Set<string>(Object.values(CanonicalMailboxRole));
|
|
17
|
+
|
|
18
|
+
/**
|
|
19
|
+
* The canonical role of the folder a quarantined message arrived in, read from
|
|
20
|
+
* the server's own RFC 6154 SPECIAL-USE declaration — the same source the
|
|
21
|
+
* appointment flow seeds from. A folder the server declares nothing about has
|
|
22
|
+
* no role, which is the normal state for a plain folder and is why the field
|
|
23
|
+
* is optional. `Important` is a SPECIAL-USE with no canonical role and drops
|
|
24
|
+
* out here rather than being invented into one.
|
|
25
|
+
*/
|
|
26
|
+
export const resolveMailboxRole = (
|
|
27
|
+
mailboxPath: string,
|
|
28
|
+
specialUse: readonly string[],
|
|
29
|
+
): CanonicalRole | undefined => {
|
|
30
|
+
if (mailboxPath.toUpperCase() === "INBOX") return CanonicalMailboxRole.Inbox;
|
|
31
|
+
const matched = specialUse.find((use) => CANONICAL_ROLES.has(use));
|
|
32
|
+
return matched as CanonicalRole | undefined;
|
|
33
|
+
};
|
|
34
|
+
|
|
35
|
+
/** Everything about the quarantine that is the same for every uid in a round. */
|
|
36
|
+
export interface QuarantineContext {
|
|
37
|
+
accountId: string;
|
|
38
|
+
accountConfigId: string;
|
|
39
|
+
mailboxId: string;
|
|
40
|
+
mailboxPath: string;
|
|
41
|
+
uidValidity: number;
|
|
42
|
+
/** Rounds tried before the message was set aside. */
|
|
43
|
+
attempts: number;
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
/**
|
|
47
|
+
* The repro fingerprint: what the message declared itself to be. Every field
|
|
48
|
+
* is optional-by-absence because a message can fail before its shape was read,
|
|
49
|
+
* and a required column would force the writer to invent a value or drop the
|
|
50
|
+
* record.
|
|
51
|
+
*/
|
|
52
|
+
export interface QuarantineMessageShape {
|
|
53
|
+
contentType?: string;
|
|
54
|
+
transferEncoding?: string;
|
|
55
|
+
charset?: string;
|
|
56
|
+
sizeBytes?: number;
|
|
57
|
+
structure: QuarantineMimeNodeItem[];
|
|
58
|
+
messageIdHash?: string;
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
export interface QuarantineFailure {
|
|
62
|
+
stage: FailureStage;
|
|
63
|
+
code: FailureCode;
|
|
64
|
+
/** Parser error text. Stored, shown on screen, never published. */
|
|
65
|
+
message: string;
|
|
66
|
+
partPath?: string;
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
const EMPTY_SHAPE: QuarantineMessageShape = { structure: [] };
|
|
70
|
+
|
|
71
|
+
/**
|
|
72
|
+
* Message shape off the rows metadata sync already wrote. The body path has no
|
|
73
|
+
* FETCH result to read — it streams raw bytes — but the MIME tree it needs was
|
|
74
|
+
* walked into BodyPart rows when the message's metadata was synced, so the
|
|
75
|
+
* fingerprint comes from there rather than from re-reading headers the parser
|
|
76
|
+
* has just refused.
|
|
77
|
+
*/
|
|
78
|
+
export const shapeFromMessageData = (
|
|
79
|
+
message: MessageItem,
|
|
80
|
+
data: MessageData,
|
|
81
|
+
): QuarantineMessageShape => {
|
|
82
|
+
const parts = [...data.bodyPart].sort((a, b) =>
|
|
83
|
+
a.partPath.localeCompare(b.partPath, "en", { numeric: true }),
|
|
84
|
+
);
|
|
85
|
+
const root = parts.find((part) => part.partPath === ROOT_PART_PATH);
|
|
86
|
+
const rootCharset = root
|
|
87
|
+
? data.bodyPartParameter.find(
|
|
88
|
+
(param) =>
|
|
89
|
+
param.bodyPartId === root.bodyPartId &&
|
|
90
|
+
param.parameterName.toLowerCase() === "charset",
|
|
91
|
+
)?.parameterValue
|
|
92
|
+
: undefined;
|
|
93
|
+
|
|
94
|
+
return {
|
|
95
|
+
...(root ? { contentType: `${root.mediaType}/${root.mediaSubtype}` } : {}),
|
|
96
|
+
...(root ? { transferEncoding: root.transferEncoding } : {}),
|
|
97
|
+
...(rootCharset ? { charset: rootCharset } : {}),
|
|
98
|
+
...(message.rfc822Size ? { sizeBytes: message.rfc822Size } : {}),
|
|
99
|
+
structure: parts.map((part) => ({
|
|
100
|
+
depth: partPathDepth(part.partPath),
|
|
101
|
+
contentType: `${part.mediaType}/${part.mediaSubtype}`,
|
|
102
|
+
})),
|
|
103
|
+
...hashOf(message.messageIdHeader),
|
|
104
|
+
};
|
|
105
|
+
};
|
|
106
|
+
|
|
107
|
+
const partPathDepth = (partPath: string): number =>
|
|
108
|
+
partPath === ROOT_PART_PATH ? 0 : partPath.split(".").length;
|
|
109
|
+
|
|
110
|
+
const hashOf = (
|
|
111
|
+
messageIdHeader: string | undefined,
|
|
112
|
+
): { messageIdHash?: string } => {
|
|
113
|
+
const messageIdHash = quarantineMessageIdHash(messageIdHeader);
|
|
114
|
+
return messageIdHash ? { messageIdHash } : {};
|
|
115
|
+
};
|
|
116
|
+
|
|
117
|
+
/**
|
|
118
|
+
* The set of messages already set aside, held for the duration of one sync
|
|
119
|
+
* round. A round loads it once and filters against it in memory: the list is
|
|
120
|
+
* small by design — a growing one is a bug being reported, not a page to
|
|
121
|
+
* paginate — and a lookup per message would put a query on the hot path for a
|
|
122
|
+
* state that is almost always empty.
|
|
123
|
+
*/
|
|
124
|
+
export class QuarantinedUids {
|
|
125
|
+
private readonly keys: ReadonlySet<string>;
|
|
126
|
+
|
|
127
|
+
constructor(entries: readonly QuarantineItem[]) {
|
|
128
|
+
this.keys = new Set(
|
|
129
|
+
entries.map((entry) =>
|
|
130
|
+
uidKey(entry.mailboxId, entry.uidValidity, entry.uid),
|
|
131
|
+
),
|
|
132
|
+
);
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
get size(): number {
|
|
136
|
+
return this.keys.size;
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
has(mailboxId: string, uidValidity: number, uid: number): boolean {
|
|
140
|
+
return this.keys.has(uidKey(mailboxId, uidValidity, uid));
|
|
141
|
+
}
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
const uidKey = (mailboxId: string, uidValidity: number, uid: number): string =>
|
|
145
|
+
`${mailboxId}:${uidValidity}:${uid}`;
|
|
146
|
+
|
|
147
|
+
export interface QuarantineLogger {
|
|
148
|
+
info(obj: Record<string, unknown>, msg: string): void;
|
|
149
|
+
warn(obj: Record<string, unknown>, msg: string): void;
|
|
150
|
+
}
|
|
151
|
+
|
|
152
|
+
/**
|
|
153
|
+
* Writes the record a message becomes when the sync path cannot apply it
|
|
154
|
+
* (issue #72), and loads the set a round filters against.
|
|
155
|
+
*
|
|
156
|
+
* Only a message defect reaches this class. Deciding that is the caller's job
|
|
157
|
+
* and it is made at exactly one kind of catch site — one narrow enough that
|
|
158
|
+
* the error can only have come from the message itself. An S3, queue or
|
|
159
|
+
* database failure propagates; writing one here would advance a cursor past
|
|
160
|
+
* mail that is fine and tell the user it was unreadable.
|
|
161
|
+
*/
|
|
162
|
+
export class QuarantineService {
|
|
163
|
+
constructor(
|
|
164
|
+
private readonly repository: IQuarantineRepository,
|
|
165
|
+
private readonly mailboxSpecialUseService: IMailboxSpecialUseRepository,
|
|
166
|
+
private readonly workerVersion: string,
|
|
167
|
+
private readonly log: QuarantineLogger,
|
|
168
|
+
) {}
|
|
169
|
+
|
|
170
|
+
async load(accountConfigId: string): Promise<QuarantinedUids> {
|
|
171
|
+
return new QuarantinedUids(
|
|
172
|
+
await this.repository.listByAccountConfigId(accountConfigId),
|
|
173
|
+
);
|
|
174
|
+
}
|
|
175
|
+
|
|
176
|
+
/**
|
|
177
|
+
* Set a message aside. Resolves only once the row is durable, because the
|
|
178
|
+
* caller's next act is to let a cursor move past this uid — and a cursor
|
|
179
|
+
* that moves past work no record survives is the silent loss this whole
|
|
180
|
+
* feature exists to end.
|
|
181
|
+
*/
|
|
182
|
+
async record(
|
|
183
|
+
context: QuarantineContext,
|
|
184
|
+
uid: number,
|
|
185
|
+
failure: QuarantineFailure,
|
|
186
|
+
shape: QuarantineMessageShape = EMPTY_SHAPE,
|
|
187
|
+
): Promise<void> {
|
|
188
|
+
// Resolved here rather than per round: a round almost never writes a row,
|
|
189
|
+
// so the lookup belongs on the write and not on the hot path.
|
|
190
|
+
const specialUse = await this.mailboxSpecialUseService.listByMailboxId(
|
|
191
|
+
context.mailboxId,
|
|
192
|
+
);
|
|
193
|
+
const mailboxRole = resolveMailboxRole(
|
|
194
|
+
context.mailboxPath,
|
|
195
|
+
specialUse.map((entry) => entry.specialUse),
|
|
196
|
+
);
|
|
197
|
+
|
|
198
|
+
await this.repository.upsert({
|
|
199
|
+
accountConfigId: context.accountConfigId,
|
|
200
|
+
accountId: context.accountId,
|
|
201
|
+
mailboxId: context.mailboxId,
|
|
202
|
+
uidValidity: context.uidValidity,
|
|
203
|
+
uid,
|
|
204
|
+
...(mailboxRole ? { mailboxRole } : {}),
|
|
205
|
+
mailboxPath: context.mailboxPath,
|
|
206
|
+
quarantinedAt: Date.now(),
|
|
207
|
+
attempts: context.attempts,
|
|
208
|
+
failureStage: failure.stage,
|
|
209
|
+
failureCode: failure.code,
|
|
210
|
+
failureMessage: failure.message,
|
|
211
|
+
...(failure.partPath ? { failurePartPath: failure.partPath } : {}),
|
|
212
|
+
workerVersion: this.workerVersion,
|
|
213
|
+
...shape,
|
|
214
|
+
});
|
|
215
|
+
|
|
216
|
+
this.log.warn(
|
|
217
|
+
{
|
|
218
|
+
mailboxId: context.mailboxId,
|
|
219
|
+
mailboxPath: context.mailboxPath,
|
|
220
|
+
uid,
|
|
221
|
+
uidValidity: context.uidValidity,
|
|
222
|
+
failureStage: failure.stage,
|
|
223
|
+
failureCode: failure.code,
|
|
224
|
+
},
|
|
225
|
+
"Message quarantined; cursor may advance past it",
|
|
226
|
+
);
|
|
227
|
+
}
|
|
228
|
+
}
|
|
@@ -135,15 +135,6 @@ describe("parseChangeCursor / formatChangeCursor", () => {
|
|
|
135
135
|
assert.equal(parseChangeCursor("abc").modseq, 0n);
|
|
136
136
|
assert.equal(parseChangeCursor("500:abc").modseq, 500n);
|
|
137
137
|
});
|
|
138
|
-
|
|
139
|
-
it("survives a numeric column value", () => {
|
|
140
|
-
// SQLite hands back a number whatever the declared column type.
|
|
141
|
-
assert.deepEqual(parseChangeCursor(900 as unknown as string), {
|
|
142
|
-
modseq: 900n,
|
|
143
|
-
group: 0n,
|
|
144
|
-
uid: 0,
|
|
145
|
-
});
|
|
146
|
-
});
|
|
147
138
|
});
|
|
148
139
|
|
|
149
140
|
describe("dropAppliedPrefix", () => {
|