@remit/mailbox-service 0.0.15 → 0.0.17

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,407 @@
1
+ /**
2
+ * The ways an enumeration round used to let go of a message without a record of
3
+ * it (issue #72).
4
+ *
5
+ * A row carrying no ENVELOPE was counted as saved and the watermark moved past
6
+ * it; so was a UID the FETCH never returned a row for. Neither is attributable
7
+ * to the message — an absent envelope is indistinguishable from the FETCH row
8
+ * glitching — so neither is quarantined. Both are now failures the watermark
9
+ * must stop below, which is the only treatment that survives a gap in the
10
+ * middle of a batch.
11
+ */
12
+
13
+ import assert from "node:assert/strict";
14
+ import { describe, it } from "node:test";
15
+ import type {
16
+ IAddressRepository,
17
+ IEnvelopeRepository,
18
+ IMailboxRepository,
19
+ IMailboxSpecialUseRepository,
20
+ IMessageRepository,
21
+ IQuarantineRepository,
22
+ IThreadMessageRepository,
23
+ IUnitOfWork,
24
+ MailboxItem,
25
+ QuarantineItem,
26
+ QuarantineUpsertInput,
27
+ UpdateMailboxInput,
28
+ } from "@remit/data-ports";
29
+ import { MailboxCursorState } from "@remit/domain-enums";
30
+ import type { ManagedConnectionFactory } from "./connection-factory.js";
31
+ import { MessageSyncService, selectUidsToSync } from "./message-sync.js";
32
+ import { QuarantineService } from "./quarantine.js";
33
+ import type { IImapConnection, ImapMessage } from "./types.js";
34
+
35
+ const ACCOUNT_ID = "acc-1";
36
+ const ACCOUNT_CONFIG_ID = "cfg-1";
37
+ const MAILBOX_ID = "mbx-1";
38
+ const UID_VALIDITY = 100;
39
+
40
+ const buildMailbox = (over: Partial<MailboxItem> = {}): MailboxItem =>
41
+ ({
42
+ mailboxId: MAILBOX_ID,
43
+ accountId: ACCOUNT_ID,
44
+ fullPath: "INBOX",
45
+ uidValidity: UID_VALIDITY,
46
+ lastSyncUid: 0,
47
+ highWaterMarkUid: 20,
48
+ highestModseq: "0",
49
+ cursorState: MailboxCursorState.normal,
50
+ ...over,
51
+ }) as MailboxItem;
52
+
53
+ const withEnvelope = (uid: number): ImapMessage => ({
54
+ uid,
55
+ seq: uid,
56
+ flags: [],
57
+ internalDate: new Date("2026-01-01T00:00:00Z"),
58
+ size: 42,
59
+ envelope: {
60
+ date: "Thu, 01 Jan 2026 00:00:00 +0000",
61
+ subject: "Hello",
62
+ from: [{ mailbox: "sender", host: "example.com" }],
63
+ sender: [],
64
+ replyTo: [],
65
+ to: [{ mailbox: "me", host: "example.com" }],
66
+ cc: [],
67
+ bcc: [],
68
+ inReplyTo: "",
69
+ messageId: `<${uid}@example.com>`,
70
+ },
71
+ });
72
+
73
+ const withoutEnvelope = (uid: number): ImapMessage => ({
74
+ uid,
75
+ seq: uid,
76
+ flags: [],
77
+ internalDate: new Date("2026-01-01T00:00:00Z"),
78
+ size: 42,
79
+ bodyStructure: { type: "text/plain", encoding: "7bit" },
80
+ });
81
+
82
+ const buildHarness = (options: {
83
+ allUids: number[];
84
+ enumerated: ImapMessage[];
85
+ existing?: QuarantineItem[];
86
+ lastSyncUid?: number;
87
+ highWaterMarkUid?: number;
88
+ /** UIDs the cheap snapshot FETCH returns; defaults to `allUids`. */
89
+ snapshotUids?: number[];
90
+ /** Set to `cursor_invalid` to take the rebuild path. */
91
+ cursorState?: MailboxItem["cursorState"];
92
+ }) => {
93
+ const mailbox = buildMailbox({
94
+ ...(options.lastSyncUid !== undefined
95
+ ? { lastSyncUid: options.lastSyncUid }
96
+ : {}),
97
+ ...(options.highWaterMarkUid !== undefined
98
+ ? { highWaterMarkUid: options.highWaterMarkUid }
99
+ : {}),
100
+ ...(options.cursorState ? { cursorState: options.cursorState } : {}),
101
+ });
102
+ const mailboxUpdates: UpdateMailboxInput[] = [];
103
+ const writes: QuarantineUpsertInput[] = [];
104
+ const fetched: number[][] = [];
105
+
106
+ const connection = {
107
+ openBox: async () => ({ uidvalidity: UID_VALIDITY, uidnext: 99 }),
108
+ getMailboxStatus: async () => ({
109
+ messages: 10,
110
+ recent: 0,
111
+ unseen: 1,
112
+ uidNext: 99,
113
+ uidValidity: UID_VALIDITY,
114
+ highestModseq: "600",
115
+ deletedCount: 0,
116
+ }),
117
+ supportsCondstore: () => false,
118
+ search: async () => options.allUids,
119
+ fetchEnvelopeSnapshots: async () =>
120
+ (options.snapshotUids ?? options.allUids).map((uid) => ({
121
+ uid,
122
+ messageId: `<${uid}@example.com>`,
123
+ internalDate: new Date("2026-01-01T00:00:00Z"),
124
+ })),
125
+ fetchMessages: async (uids: number[]) => {
126
+ fetched.push(uids);
127
+ return options.enumerated.filter((msg) => uids.includes(msg.uid));
128
+ },
129
+ } as unknown as IImapConnection;
130
+
131
+ const mailboxService = {
132
+ get: async () => mailbox,
133
+ update: async (_a: string, _m: string, input: UpdateMailboxInput) => {
134
+ mailboxUpdates.push(input);
135
+ return mailbox;
136
+ },
137
+ } as unknown as IMailboxRepository;
138
+
139
+ const threadMessageService = {
140
+ findByMessageId: async () => null,
141
+ findAllByMessageId: async () => [],
142
+ listByMailbox: async () => ({ items: [], continuationToken: undefined }),
143
+ create: async () => ({ threadMessageId: "tm-1" }),
144
+ update: async () => ({ threadMessageId: "tm-1" }),
145
+ } as unknown as IThreadMessageRepository;
146
+
147
+ const unitOfWork: IUnitOfWork = {
148
+ transaction: (fn) =>
149
+ fn({
150
+ message: {
151
+ upsertWithStatus: async (input: { mailboxId: string }) => ({
152
+ item: { mailboxId: input.mailboxId },
153
+ created: true,
154
+ }),
155
+ updateUid: async () => undefined,
156
+ } as unknown as IMessageRepository,
157
+ envelope: {
158
+ upsertEnvelope: async () => undefined,
159
+ upsertBodyParts: async () => undefined,
160
+ } as unknown as IEnvelopeRepository,
161
+ address: {
162
+ upsertAddress: async () => undefined,
163
+ upsertEnvelopeAddress: async () => undefined,
164
+ } as unknown as IAddressRepository,
165
+ threadMessage: threadMessageService,
166
+ }),
167
+ };
168
+
169
+ const repository = {
170
+ listByAccountConfigId: async () => options.existing ?? [],
171
+ upsert: async (input: QuarantineUpsertInput) => {
172
+ writes.push(input);
173
+ },
174
+ } satisfies IQuarantineRepository;
175
+
176
+ const service = new MessageSyncService(
177
+ {
178
+ getConnection: () => connection,
179
+ close: async () => {},
180
+ } as ManagedConnectionFactory,
181
+ mailboxService,
182
+ {} as IMessageRepository,
183
+ {} as IEnvelopeRepository,
184
+ {} as IAddressRepository,
185
+ threadMessageService,
186
+ { info: () => {}, warn: () => {}, error: () => {} },
187
+ unitOfWork,
188
+ undefined,
189
+ undefined,
190
+ new QuarantineService(
191
+ repository,
192
+ {
193
+ listByMailboxId: async () => [],
194
+ } as unknown as IMailboxSpecialUseRepository,
195
+ "sha-abc",
196
+ { info: () => {}, warn: () => {} },
197
+ ),
198
+ );
199
+
200
+ return { service, mailboxUpdates, writes, fetched };
201
+ };
202
+
203
+ const sync = (service: MessageSyncService) =>
204
+ service.syncMessages(MAILBOX_ID, ACCOUNT_ID, ACCOUNT_CONFIG_ID, 50);
205
+
206
+ describe("a row that carried no ENVELOPE", () => {
207
+ it("is held for retry rather than set aside", async () => {
208
+ const harness = buildHarness({
209
+ allUids: [21],
210
+ enumerated: [withoutEnvelope(21)],
211
+ });
212
+
213
+ await sync(harness.service);
214
+
215
+ // Nothing can tell an envelope-less row apart from the FETCH glitching,
216
+ // and the client is far more often the cause than the message. Recording
217
+ // it would set aside mail that is fine.
218
+ assert.deepEqual(harness.writes, []);
219
+ assert.equal(harness.mailboxUpdates[0]?.highWaterMarkUid, 20);
220
+ });
221
+
222
+ it("keeps the mailbox on enumeration rather than seeding a mod-sequence over it", async () => {
223
+ const harness = buildHarness({
224
+ allUids: [21],
225
+ enumerated: [withoutEnvelope(21)],
226
+ });
227
+
228
+ await sync(harness.service);
229
+
230
+ assert.equal(harness.mailboxUpdates[0]?.highestModseq, undefined);
231
+ });
232
+ });
233
+
234
+ describe("a uid already quarantined", () => {
235
+ it("is not fetched again, but the watermark still passes it", async () => {
236
+ const harness = buildHarness({
237
+ allUids: [21, 22],
238
+ enumerated: [withEnvelope(22)],
239
+ existing: [
240
+ {
241
+ mailboxId: MAILBOX_ID,
242
+ uidValidity: UID_VALIDITY,
243
+ uid: 21,
244
+ } as QuarantineItem,
245
+ ],
246
+ });
247
+
248
+ await sync(harness.service);
249
+
250
+ assert.deepEqual(harness.fetched, [[22]]);
251
+ assert.equal(harness.mailboxUpdates[0]?.highWaterMarkUid, 22);
252
+ });
253
+ });
254
+
255
+ describe("a uid the FETCH returned no row for", () => {
256
+ it("stops the watermark below it when it is the highest of the batch", async () => {
257
+ const harness = buildHarness({
258
+ allUids: [22, 21],
259
+ enumerated: [withEnvelope(21)],
260
+ });
261
+
262
+ await sync(harness.service);
263
+
264
+ assert.equal(harness.mailboxUpdates[0]?.highWaterMarkUid, 21);
265
+ });
266
+
267
+ it("stops the watermark below it when it sits in the middle of the batch", async () => {
268
+ const harness = buildHarness({
269
+ allUids: [23, 22, 21],
270
+ // 22's row is dropped; 23 above it saves fine.
271
+ enumerated: [withEnvelope(23), withEnvelope(21)],
272
+ });
273
+
274
+ await sync(harness.service);
275
+
276
+ // The watermark advances by MAX of what was applied, so absence alone
277
+ // would let 23 carry it straight over the gap and lose 22 for good.
278
+ assert.equal(harness.mailboxUpdates[0]?.highWaterMarkUid, 21);
279
+ });
280
+
281
+ it("stops the backfill floor above a gap, so it stays selectable", async () => {
282
+ const harness = buildHarness({
283
+ allUids: [12, 11, 10],
284
+ enumerated: [withEnvelope(12), withEnvelope(10)],
285
+ lastSyncUid: 20,
286
+ highWaterMarkUid: 20,
287
+ });
288
+
289
+ await sync(harness.service);
290
+
291
+ assert.equal(harness.mailboxUpdates[0]?.lastSyncUid, 12);
292
+ });
293
+
294
+ it("keeps the mailbox on enumeration rather than seeding a mod-sequence over it", async () => {
295
+ const harness = buildHarness({
296
+ allUids: [22, 21],
297
+ enumerated: [withEnvelope(21)],
298
+ });
299
+
300
+ await sync(harness.service);
301
+
302
+ assert.equal(harness.mailboxUpdates[0]?.highestModseq, undefined);
303
+ });
304
+ });
305
+
306
+ /**
307
+ * The cursor rebuild is the third save path, and the one where a lost UID is
308
+ * lost for good: its covered region is computed from the server snapshot
309
+ * rather than from what it applied, and it seeds the mod-sequence and returns
310
+ * the mailbox to `normal` — so the next round takes CHANGEDSINCE, which never
311
+ * enumerates.
312
+ */
313
+ describe("cursor rebuild", () => {
314
+ const rebuild = (over: {
315
+ allUids: number[];
316
+ enumerated: ImapMessage[];
317
+ snapshotUids?: number[];
318
+ }) =>
319
+ buildHarness({
320
+ ...over,
321
+ cursorState: MailboxCursorState.cursor_invalid,
322
+ lastSyncUid: 0,
323
+ highWaterMarkUid: 0,
324
+ });
325
+
326
+ it("holds the watermark below a UID whose row carried no ENVELOPE", async () => {
327
+ const harness = rebuild({
328
+ allUids: [23, 22, 21],
329
+ enumerated: [withEnvelope(23), withoutEnvelope(22), withEnvelope(21)],
330
+ });
331
+
332
+ await sync(harness.service);
333
+
334
+ const final = harness.mailboxUpdates.at(-1);
335
+ assert.equal(final?.highWaterMarkUid, 21);
336
+ });
337
+
338
+ it("holds the watermark below a UID the message FETCH did not return", async () => {
339
+ const harness = rebuild({
340
+ allUids: [23, 22, 21],
341
+ enumerated: [withEnvelope(23), withEnvelope(21)],
342
+ });
343
+
344
+ await sync(harness.service);
345
+
346
+ assert.equal(harness.mailboxUpdates.at(-1)?.highWaterMarkUid, 21);
347
+ });
348
+
349
+ it("holds the watermark below a UID the snapshot FETCH did not return", async () => {
350
+ const harness = rebuild({
351
+ allUids: [23, 22, 21],
352
+ snapshotUids: [23, 21],
353
+ enumerated: [withEnvelope(23), withEnvelope(21)],
354
+ });
355
+
356
+ await sync(harness.service);
357
+
358
+ // This UID never reaches the snapshot, so it is not even a candidate for
359
+ // saving — but the covered region spans it, which is the whole hazard.
360
+ assert.equal(harness.mailboxUpdates.at(-1)?.highWaterMarkUid, 21);
361
+ });
362
+
363
+ it("withholds the mod-sequence seed, so the mailbox stays on enumeration", async () => {
364
+ const harness = rebuild({
365
+ allUids: [23, 22, 21],
366
+ enumerated: [withEnvelope(23), withEnvelope(21)],
367
+ });
368
+
369
+ await sync(harness.service);
370
+
371
+ // Seeding it would flip the mailbox to CHANGEDSINCE, which never
372
+ // enumerates — the UID would never be looked for again.
373
+ assert.equal(harness.mailboxUpdates.at(-1)?.highestModseq, "0");
374
+ });
375
+
376
+ it("leaves the missing UID selectable by the next round", async () => {
377
+ const harness = rebuild({
378
+ allUids: [23, 22, 21],
379
+ enumerated: [withEnvelope(23), withEnvelope(21)],
380
+ });
381
+
382
+ await sync(harness.service);
383
+
384
+ const final = harness.mailboxUpdates.at(-1);
385
+ assert.deepEqual(
386
+ selectUidsToSync(
387
+ [23, 22, 21],
388
+ final?.lastSyncUid ?? 0,
389
+ final?.highWaterMarkUid ?? 0,
390
+ ),
391
+ [23, 22],
392
+ );
393
+ });
394
+
395
+ it("seeds the mod-sequence when every UID was accounted for", async () => {
396
+ const harness = rebuild({
397
+ allUids: [23, 22, 21],
398
+ enumerated: [withEnvelope(23), withEnvelope(22), withEnvelope(21)],
399
+ });
400
+
401
+ await sync(harness.service);
402
+
403
+ const final = harness.mailboxUpdates.at(-1);
404
+ assert.equal(final?.highestModseq, "600");
405
+ assert.equal(final?.highWaterMarkUid, 23);
406
+ });
407
+ });
@@ -36,6 +36,7 @@ import {
36
36
  } from "./mailbox-cursor-rebuild.js";
37
37
  import { ROOT_PART_PATH, walkMimeStructure } from "./mime-walker.js";
38
38
  import { PassThroughUnitOfWork } from "./pass-through-unit-of-work.js";
39
+ import type { QuarantineService } from "./quarantine.js";
39
40
  import { reconcileStaleMessage } from "./stale-message-reconcile.js";
40
41
  import {
41
42
  advanceChangeCursor,
@@ -146,8 +147,11 @@ interface SaveMessageResult extends SyncedMessage {
146
147
  /**
147
148
  * Wrapper outcome for a single message in the batch. A `failed` outcome means
148
149
  * the save threw (and was caught) — its UID must NOT advance the watermark, so
149
- * the message is re-fetched and retried on the next cycle. `null` means the
150
- * message carried no envelope and was intentionally skipped (nothing to retry).
150
+ * the message is re-fetched and retried on the next cycle. A `saved` outcome
151
+ * with a `null` result is a UID this round finished with but created no row
152
+ * for: a cross-mailbox collision, a change applied to an existing row, or a
153
+ * message quarantined instead of applied (issue #72). Its watermark advances
154
+ * either way, because there is nothing left to retry.
151
155
  */
152
156
  type BatchOutcome =
153
157
  | { kind: "saved"; uid: number; result: SaveMessageResult | null }
@@ -224,6 +228,13 @@ export class MessageSyncService {
224
228
  * flip is never dismissed as redundant.
225
229
  */
226
230
  private messageFlagService?: IMessageFlagRepository,
231
+ /**
232
+ * The set of messages already set aside (issue #72). Message sync never
233
+ * writes a record — only the body path can attribute a failure to the
234
+ * message — but it reads the set so a uid the body path quarantined is
235
+ * not re-fetched on every round.
236
+ */
237
+ private quarantineService?: QuarantineService,
227
238
  ) {
228
239
  this.log = logger ?? noopLogger;
229
240
  this.unitOfWork =
@@ -309,6 +320,10 @@ export class MessageSyncService {
309
320
  });
310
321
  }
311
322
 
323
+ // One read per round, not per message (issue #72). The list is small by
324
+ // design — a growing one is a bug being reported, not a page to paginate.
325
+ const quarantined = await this.quarantineService?.load(accountConfigId);
326
+
312
327
  const allUids = await connection.search(["ALL"]);
313
328
  const uids = selectUidsToSync(allUids, lastSyncUid, highWaterMarkUid);
314
329
  const unseenCount = status.unseen;
@@ -350,7 +365,42 @@ export class MessageSyncService {
350
365
 
351
366
  // Process only the first batch
352
367
  const batchUids = uids.slice(0, batchSize);
353
- const messages = await this.fetchMessageBatch(batchUids);
368
+
369
+ // A quarantined UID is not fetched again, but it stays in `batchUids` so
370
+ // the watermark still advances over it. Filtering it out of the selection
371
+ // instead would hold the watermark below a message that is already
372
+ // durably resolved, which is the stall by another route.
373
+ const fetchUids = batchUids.filter(
374
+ (uid) => !quarantined?.has(mailboxId, box.uidvalidity, uid),
375
+ );
376
+ const messages =
377
+ fetchUids.length > 0 ? await this.fetchMessageBatch(fetchUids) : [];
378
+
379
+ // A UID the round could not act on. Two shapes reach here and neither is
380
+ // the message's fault: the FETCH returned no row at all (the connection
381
+ // layer drops rows imapflow yields without a usable UID or INTERNALDATE,
382
+ // #408, and a message can be expunged between the SEARCH and the FETCH),
383
+ // or it returned a row carrying no ENVELOPE, which names no message and
384
+ // is the same client-side glitch one field further in.
385
+ //
386
+ // They join `failedUids` rather than merely being left out of the batch.
387
+ // Absence alone does not hold a watermark: `advanceUidWatermarks` takes
388
+ // the MAX of what was applied, so a gap in the middle of a batch is
389
+ // stepped straight over — [23, 22, 21] with 22 missing still advances to
390
+ // 23 and loses 22 for good. A failure is the one thing a watermark is
391
+ // built to stop below.
392
+ const applicable = messages.filter((msg) => msg.envelope !== undefined);
393
+ const unusableUids = batchUids.filter(
394
+ (uid) =>
395
+ !applicable.some((msg) => msg.uid === uid) &&
396
+ !quarantined?.has(mailboxId, box.uidvalidity, uid),
397
+ );
398
+ if (unusableUids.length > 0) {
399
+ this.log.warn(
400
+ { mailboxId, mailboxPath, unusableUids },
401
+ "FETCH returned no usable row for some requested UIDs; holding the watermark below them",
402
+ );
403
+ }
354
404
 
355
405
  // Process messages in parallel with concurrency limit. `stopOnError` stays
356
406
  // at its default — but each message is saved through `trySaveMessage`,
@@ -358,7 +408,7 @@ export class MessageSyncService {
358
408
  // rejecting. So one bad message can no longer abort the whole batch (the
359
409
  // poison pill that previously froze the mailbox, #817).
360
410
  const outcomes = await pMap(
361
- messages,
411
+ applicable,
362
412
  (msg) => this.trySaveMessage(mailboxId, accountId, accountConfigId, msg),
363
413
  { concurrency: MESSAGE_SAVE_CONCURRENCY },
364
414
  );
@@ -379,15 +429,16 @@ export class MessageSyncService {
379
429
 
380
430
  // UIDs whose save threw. They must stay inside the next cycle's fetch
381
431
  // window, so the watermark may not advance past them (no silent loss).
382
- const failedUids = new Set(
383
- outcomes.flatMap((o) => (o.kind === "failed" ? [o.uid] : [])),
432
+ const saveFailedUids = outcomes.flatMap((o) =>
433
+ o.kind === "failed" ? [o.uid] : [],
384
434
  );
385
- if (failedUids.size > 0) {
435
+ if (saveFailedUids.length > 0) {
386
436
  this.log.warn(
387
- { mailboxId, mailboxPath, failedUids: [...failedUids] },
437
+ { mailboxId, mailboxPath, failedUids: saveFailedUids },
388
438
  "Some messages failed to save; holding watermark below them for retry",
389
439
  );
390
440
  }
441
+ const failedUids = new Set([...saveFailedUids, ...unusableUids]);
391
442
 
392
443
  // Watermarks advance over every SUCCESSFULLY-consumed UID in the batch,
393
444
  // independent of ownership. `selectUidsToSync` reselects work purely by UID
@@ -413,7 +464,8 @@ export class MessageSyncService {
413
464
  // enumeration work behind and lost no message to a failed save. Seeding
414
465
  // it earlier would switch the mailbox to CHANGEDSINCE while UIDs it has
415
466
  // never fetched still sit below the watermark, and those messages would
416
- // never be discovered.
467
+ // never be discovered. A UID the FETCH returned nothing usable for is in
468
+ // `failedUids` too: the round did not finish with it either.
417
469
  const enumerationComplete = !hasMore && failedUids.size === 0;
418
470
 
419
471
  await this.mailboxService.update(accountId, mailboxId, {
@@ -587,8 +639,9 @@ export class MessageSyncService {
587
639
 
588
640
  const newMessages =
589
641
  newUids.length > 0 ? await this.fetchMessageBatch(newUids) : [];
642
+ const applicable = newMessages.filter((msg) => msg.envelope !== undefined);
590
643
  const outcomes = await pMap(
591
- newMessages,
644
+ applicable,
592
645
  (msg) => this.trySaveMessage(mailboxId, accountId, accountConfigId, msg),
593
646
  { concurrency: MESSAGE_SAVE_CONCURRENCY },
594
647
  );
@@ -600,6 +653,31 @@ export class MessageSyncService {
600
653
 
601
654
  const serverUids = serverSnapshots.map((s) => s.uid);
602
655
 
656
+ // A UID this pass could not account for, in any of the three ways it can
657
+ // go missing: the snapshot FETCH never returned a row for it, the
658
+ // message FETCH never returned one, or the row it returned carried no
659
+ // ENVELOPE. None of the three is the message's fault and none can be
660
+ // quarantined, so each has to keep the UID selectable.
661
+ //
662
+ // This matters more here than on the enumeration path, not less. The
663
+ // covered region is computed from `serverUids` rather than from what was
664
+ // applied, so a UID missing anywhere inside its span is silently inside
665
+ // it; and this round seeds the mod-sequence and returns the mailbox to
666
+ // `normal`, so the next round takes CHANGEDSINCE, which never
667
+ // enumerates. A UID lost here is lost for good.
668
+ const savedUids = new Set(applicable.map((msg) => msg.uid));
669
+ const snapshotUids = new Set(serverUids);
670
+ const unusableUids = [
671
+ ...allUids.filter((uid) => !snapshotUids.has(uid)),
672
+ ...newUids.filter((uid) => !savedUids.has(uid)),
673
+ ];
674
+ if (unusableUids.length > 0) {
675
+ this.log.warn(
676
+ { mailboxId, mailboxPath, unusableUids },
677
+ "Cursor rebuild could not account for some UIDs; holding the watermark below them",
678
+ );
679
+ }
680
+
603
681
  // A new message whose save threw must stay selectable, so the forward
604
682
  // watermark stops below it and every UID above it is re-enumerated next
605
683
  // round. The mod-sequence seed is withheld entirely in that case: the
@@ -607,9 +685,10 @@ export class MessageSyncService {
607
685
  // it on a UIDVALIDITY change) and the new one would sit above the
608
686
  // message that failed, so the mailbox goes back to enumeration until a
609
687
  // clean round seeds it.
610
- const failedUids = new Set(
611
- outcomes.flatMap((o) => (o.kind === "failed" ? [o.uid] : [])),
612
- );
688
+ const failedUids = new Set([
689
+ ...outcomes.flatMap((o) => (o.kind === "failed" ? [o.uid] : [])),
690
+ ...unusableUids,
691
+ ]);
613
692
  const lowestFailure = failedUids.size
614
693
  ? Math.min(...failedUids)
615
694
  : Number.POSITIVE_INFINITY;
@@ -756,21 +835,53 @@ export class MessageSyncService {
756
835
  const ordered = dropAppliedPrefix(orderByModseq(changed), cursor);
757
836
  const batch = ordered.slice(0, batchSize);
758
837
 
838
+ // A quarantined UID stays in `batch`, so the cursor still advances over
839
+ // it; only the work of re-applying it is skipped.
840
+ const quarantined = await this.quarantineService?.load(accountConfigId);
841
+ const applicable = batch.filter(
842
+ (msg) =>
843
+ !quarantined?.has(mailboxId, box.uidvalidity, msg.uid) &&
844
+ msg.envelope !== undefined,
845
+ );
846
+
847
+ // A change row carrying no ENVELOPE holds the cursor and is retried; it
848
+ // is never set aside. On this path the message is usually one already
849
+ // stored — it demonstrably had a sender, a date and a Message-ID when it
850
+ // was first saved — so an envelope-less row is the FETCH glitching
851
+ // (#408), not the message being defective. Quarantining it would filter
852
+ // that UID out of every later round, stopping its flag sync until a
853
+ // purge, and tell the user a message they can open and read "arrived
854
+ // without a sender". A transient glitch heals on the retry; a persistent
855
+ // one trips the stalled-cursor alert, which is what that alert is for.
856
+ const unusableUids = batch.flatMap((msg) =>
857
+ msg.envelope === undefined &&
858
+ !quarantined?.has(mailboxId, box.uidvalidity, msg.uid)
859
+ ? [msg.uid]
860
+ : [],
861
+ );
862
+ if (unusableUids.length > 0) {
863
+ this.log.warn(
864
+ { mailboxId, mailboxPath, unusableUids },
865
+ "Change rows carried no ENVELOPE; holding the sync cursor below them",
866
+ );
867
+ }
868
+
759
869
  const outcomes = await pMap(
760
- batch,
870
+ applicable,
761
871
  (msg) => this.tryApplyChange(mailboxId, accountId, accountConfigId, msg),
762
872
  { concurrency: MESSAGE_SAVE_CONCURRENCY },
763
873
  );
764
874
 
765
- const failedUids = new Set(
766
- outcomes.flatMap((o) => (o.kind === "failed" ? [o.uid] : [])),
875
+ const saveFailedUids = outcomes.flatMap((o) =>
876
+ o.kind === "failed" ? [o.uid] : [],
767
877
  );
768
- if (failedUids.size > 0) {
878
+ if (saveFailedUids.length > 0) {
769
879
  this.log.warn(
770
- { mailboxId, mailboxPath, failedUids: [...failedUids] },
880
+ { mailboxId, mailboxPath, failedUids: saveFailedUids },
771
881
  "Some changes failed to apply; holding the sync cursor below them for retry",
772
882
  );
773
883
  }
884
+ const failedUids = new Set([...saveFailedUids, ...unusableUids]);
774
885
 
775
886
  // Body sync only concerns messages this round created — a metadata
776
887
  // change has no new body to fetch.
@@ -1231,6 +1342,12 @@ export class MessageSyncService {
1231
1342
  order: number;
1232
1343
  }> = [];
1233
1344
 
1345
+ // An unusable address is dropped and the message is written anyway, and
1346
+ // that stays deliberate under the quarantine rules (issue #72). What is
1347
+ // lost is one envelope address, not the message: it is stored, listed and
1348
+ // readable, and its body is untouched. Setting the whole message aside
1349
+ // over a malformed From would take readable mail out of the mailbox to
1350
+ // protect a display name.
1234
1351
  for (let i = 0; i < addresses.length; i++) {
1235
1352
  const addr = addresses[i];
1236
1353
  if (!isParseableEmailAddress(addr)) continue;