@remit/mailbox-service 0.0.111 → 0.0.112

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (123) hide show
  1. package/package.json +21 -6
  2. package/src/body-part-mapper.ts +51 -1
  3. package/src/body-sync-queue.test.ts +39 -99
  4. package/src/body-sync-terminal.test.ts +115 -277
  5. package/src/body-sync-terminal.ts +10 -12
  6. package/src/body-sync.test.ts +925 -0
  7. package/src/body-sync.ts +156 -235
  8. package/src/classification-backfill.test.ts +197 -527
  9. package/src/classify-message.test.ts +11 -74
  10. package/src/external-move.test.ts +102 -209
  11. package/src/external-move.ts +13 -0
  12. package/src/filter-operations.test.ts +319 -0
  13. package/src/filter-operations.ts +377 -0
  14. package/src/filter-update.test.ts +313 -0
  15. package/src/filters/filter-config.test.ts +55 -197
  16. package/src/filters/filter-config.ts +4 -39
  17. package/src/filters/match.test.ts +1 -38
  18. package/src/filters/match.ts +3 -52
  19. package/src/filters/pipeline.test.ts +158 -289
  20. package/src/filters/pipeline.ts +56 -170
  21. package/src/flag-push-terminal.ts +9 -11
  22. package/src/flag-push.test.ts +76 -372
  23. package/src/flag-queue.test.ts +197 -0
  24. package/src/folder-operations.test.ts +273 -0
  25. package/src/folder-operations.ts +217 -0
  26. package/src/forwarded-message.test.ts +122 -0
  27. package/src/imapflow-connect-retry.test.ts +76 -113
  28. package/src/imapflow-connection.test.ts +34 -0
  29. package/src/imapflow-connection.ts +5 -4
  30. package/src/index.ts +59 -12
  31. package/src/list-id-backfill.test.ts +133 -499
  32. package/src/mailbox-cursor.test.ts +81 -189
  33. package/src/mailbox-delete-intent.test.ts +78 -205
  34. package/src/mailbox-intent.ts +5 -3
  35. package/src/mailbox-management.test.ts +107 -220
  36. package/src/mailbox-management.ts +1 -2
  37. package/src/mailbox-presence.test.ts +36 -34
  38. package/src/mailbox-queue.ts +2 -20
  39. package/src/mailbox-rename-intent.test.ts +156 -325
  40. package/src/mailbox-rename-subtree.test.ts +205 -582
  41. package/src/mailbox-sync.test.ts +182 -744
  42. package/src/message-move.test.ts +726 -0
  43. package/src/message-move.ts +227 -57
  44. package/src/message-read.ts +87 -0
  45. package/src/message-save-retry.test.ts +90 -0
  46. package/src/message-sync-flags.test.ts +10 -1
  47. package/src/message-sync.test.ts +762 -0
  48. package/src/message-sync.ts +409 -239
  49. package/src/mime-walker.ts +16 -15
  50. package/src/organize-job.ts +93 -0
  51. package/src/outbox-attachment.test.ts +97 -209
  52. package/src/outbox-failed-recovery.test.ts +198 -269
  53. package/src/outbox-queue.test.ts +24 -0
  54. package/src/outbox-queue.ts +25 -8
  55. package/src/ownership.ts +140 -0
  56. package/src/placement-move-terminal.ts +9 -12
  57. package/src/placement-move.ts +33 -6
  58. package/src/placement-settled.ts +28 -4
  59. package/src/placement.test.ts +404 -0
  60. package/src/quarantine.test.ts +90 -148
  61. package/src/quarantine.ts +3 -2
  62. package/src/refusal.ts +29 -0
  63. package/src/restore-source-placement.ts +0 -4
  64. package/src/spam-report.test.ts +238 -710
  65. package/src/sync-warning.test.ts +213 -0
  66. package/src/sync-warning.ts +56 -0
  67. package/src/test-helpers/fake-imap.ts +690 -0
  68. package/src/test-helpers/test-queue.ts +56 -0
  69. package/src/test-helpers/test-store.ts +299 -0
  70. package/src/thread-root.test.ts +29 -0
  71. package/src/thread-root.ts +51 -0
  72. package/src/types/mailparser-augment.d.ts +8 -0
  73. package/src/adversarial-mime.e2e.test.ts +0 -307
  74. package/src/body-sync-auto-read-writeonce.test.ts +0 -219
  75. package/src/body-sync-calendar-suggestion.test.ts +0 -433
  76. package/src/body-sync-category-denormalize.test.ts +0 -352
  77. package/src/body-sync-category-override.test.ts +0 -501
  78. package/src/body-sync-category-writeonce.test.ts +0 -234
  79. package/src/body-sync-filter.test.ts +0 -237
  80. package/src/body-sync-message-gone.test.ts +0 -139
  81. package/src/body-sync-placement-flags.test.ts +0 -347
  82. package/src/body-sync-placement-writeonce.test.ts +0 -337
  83. package/src/body-sync-quarantine.test.ts +0 -276
  84. package/src/body-sync-resync.test.ts +0 -605
  85. package/src/body-sync-unclassifiable-once.test.ts +0 -260
  86. package/src/body-sync-unsubscribed-auto-read.test.ts +0 -176
  87. package/src/filters/anchor-drift.ts +0 -55
  88. package/src/filters/message-embedder.test.ts +0 -39
  89. package/src/filters/message-embedder.ts +0 -33
  90. package/src/flag-push-terminal.test.ts +0 -209
  91. package/src/flag-queue-stars.test.ts +0 -171
  92. package/src/imapflow-connection.e2e.test.ts +0 -335
  93. package/src/imapflow-connection.integ.test.ts +0 -316
  94. package/src/message-delete-junk-reconcile.test.ts +0 -117
  95. package/src/message-delete-no-trash.test.ts +0 -245
  96. package/src/message-delete-unsettled-placement.test.ts +0 -396
  97. package/src/message-move-copy.test.ts +0 -562
  98. package/src/message-move-empty-trash.test.ts +0 -326
  99. package/src/message-move-same-mailbox.test.ts +0 -139
  100. package/src/message-move-unsettled-placement.test.ts +0 -378
  101. package/src/message-sync-address-spoof.test.ts +0 -293
  102. package/src/message-sync-category.test.ts +0 -142
  103. package/src/message-sync-changedsince.test.ts +0 -840
  104. package/src/message-sync-junk-harvest.test.ts +0 -244
  105. package/src/message-sync-quarantine.test.ts +0 -416
  106. package/src/message-sync-repoint.test.ts +0 -419
  107. package/src/outbox-queue-settled-row.test.ts +0 -140
  108. package/src/pass-through-unit-of-work.ts +0 -17
  109. package/src/placement-move-terminal.test.ts +0 -300
  110. package/src/placement-move-unsettled.test.ts +0 -275
  111. package/src/placement-transition-race.test.ts +0 -482
  112. package/src/stale-message-reconcile.test.ts +0 -82
  113. package/src/stale-message-reconcile.ts +0 -55
  114. package/src/test-helpers/isolated-mailbox.ts +0 -44
  115. package/src/test-helpers/mailfuzz-connection.ts +0 -27
  116. package/test/fixtures/mime-adversarial-e2e/01-octet-stream-pdf.eml +0 -21
  117. package/test/fixtures/mime-adversarial-e2e/02-nested-related-cid-image.eml +0 -31
  118. package/test/fixtures/mime-adversarial-e2e/03-calendar-invite.eml +0 -33
  119. package/test/fixtures/mime-adversarial-e2e/04-forwarded-rfc822-with-attachments.eml +0 -42
  120. package/test/fixtures/mime-adversarial-e2e/05-tnef-winmail.eml +0 -25
  121. package/test/fixtures/mime-adversarial-e2e/06-8bit-non-ascii.eml +0 -15
  122. package/test/fixtures/mime-adversarial-e2e/07-quoted-printable-soft-breaks.eml +0 -17
  123. package/test/fixtures/mime-adversarial-e2e/08-alternative-empty-text.eml +0 -18
package/src/body-sync.ts CHANGED
@@ -5,15 +5,15 @@ import {
5
5
  recordCalendarSuggestion,
6
6
  } from "@remit/calendar-service";
7
7
  import {
8
+ type BodyPartItem,
8
9
  type IAddressRepository,
10
+ type ICalendarStore,
9
11
  type ICalendarSuggestionRepository,
10
- type ICalendarUnitOfWork,
11
12
  type IEnvelopeRepository,
12
13
  type IFilterRepository,
13
14
  type IMailboxSpecialUseRepository,
14
15
  type IMessageRepository,
15
16
  type IThreadMessageRepository,
16
- type IUnitOfWork,
17
17
  isSenderMuted,
18
18
  type MessageItem,
19
19
  type ThreadMessageItem,
@@ -25,6 +25,7 @@ import {
25
25
  NotFoundError,
26
26
  } from "@remit/data-ports/errors";
27
27
  import { deriveAddressId, deriveBodyPartId } from "@remit/data-ports/id";
28
+ import { isEncapsulatedMessage } from "@remit/data-ports/message-parts";
28
29
  import { isBulkSender } from "@remit/data-ports/wellknown";
29
30
  import {
30
31
  FilterState,
@@ -33,6 +34,7 @@ import {
33
34
  PlacementConfidence,
34
35
  QuarantineFailureStage,
35
36
  SenderTrust,
37
+ SyncWarningStage,
36
38
  } from "@remit/domain-enums";
37
39
  import {
38
40
  isStorageNotFoundError,
@@ -42,12 +44,16 @@ import {
42
44
  import { type ParsedMail, simpleParser } from "mailparser";
43
45
  import pMap from "p-map";
44
46
  import { BodyParseError, parseMessageBody } from "./body-parse.js";
45
- import { mapBodyPartsToContent } from "./body-part-mapper.js";
47
+ import {
48
+ mapMessagePartsToContent,
49
+ parsePartSource,
50
+ } from "./body-part-mapper.js";
46
51
  import { calendarParts } from "./calendar-parts.js";
47
52
  import {
48
53
  classifyParsedMessage,
49
54
  extractPrimaryFromEmail,
50
55
  } from "./classify-message.js";
56
+
51
57
  import { extractListId } from "./filters/list-id.js";
52
58
  import type { FilterMessage } from "./filters/match.js";
53
59
  import {
@@ -68,11 +74,11 @@ import {
68
74
  hasDecidedCategory,
69
75
  } from "./message-category.js";
70
76
  import { isMessageGoneFromOpenMailbox } from "./message-presence.js";
71
- import { PassThroughUnitOfWork } from "./pass-through-unit-of-work.js";
72
77
  import type { PlacementMoveService } from "./placement-move.js";
73
- import { isPlacementUnsettled } from "./placement-settled.js";
78
+ import { isPlacementUnsettled, uidMailboxIdOf } from "./placement-settled.js";
74
79
  import { type QuarantineService, shapeFromMessageData } from "./quarantine.js";
75
80
  import { extractSnippetFromEmail } from "./snippet.js";
81
+ import type { SyncWarningService } from "./sync-warning.js";
76
82
  import { type IImapConnection, MailConnectionError } from "./types.js";
77
83
 
78
84
  const BODY_PART_STORE_CONCURRENCY = 4;
@@ -120,12 +126,20 @@ export const isBodyPartDeferralEnabled = (): boolean => {
120
126
  * single HEAD and early-returns, skipping the body.eml GET, the MIME re-parse,
121
127
  * and the per-leaf HEADs on every warm re-open.
122
128
  *
123
- * Deliberately an S3-only marker (no Message-row flag): a DDB write would
124
- * re-fire the redundant vector upsert that #607 removed. A leading dot keeps it
129
+ * Deliberately an S3-only marker (no Message-row flag). A leading dot keeps it
125
130
  * out of any real IMAP section path (those are dot-separated digits).
126
131
  */
127
132
  const MATERIALIZED_SENTINEL_PART_PATH = ".materialized";
128
133
 
134
+ /**
135
+ * The sentinel for a message carrying a forwarded message. Those parts were
136
+ * once stored under a layout in which the forwarded message's root overwrote
137
+ * its own row, so a `.materialized` written then vouches for objects at the
138
+ * wrong paths. Its own name makes every such message materialize once more,
139
+ * overwriting what is there; no other message is touched.
140
+ */
141
+ const MATERIALIZED_SENTINEL_V2_PART_PATH = ".materialized.2";
142
+
129
143
  /**
130
144
  * A mid-stream socket drop during the ranged body fetch surfaces as a typed
131
145
  * `MailConnectionError` (the connection layer classifies imapflow's
@@ -142,7 +156,7 @@ const isConnectionDrop = (error: unknown): boolean => {
142
156
 
143
157
  /**
144
158
  * Project a parsed message onto the fields a filter matches against (RFC 034) —
145
- * the literal-clause targets plus the text a semantic anchor embeds. Kept
159
+ * the literal-clause targets. Kept
146
160
  * separate from `ParsedMail` so {@link FilterPipeline} stays parser-agnostic.
147
161
  */
148
162
  const toFilterMessage = (parsed: ParsedMail): FilterMessage => ({
@@ -249,10 +263,15 @@ export interface StoreBodyPartContentsResult {
249
263
  export type ConnectionGetter = () => Promise<IImapConnection>;
250
264
 
251
265
  export interface BodySyncFolder {
266
+ mailboxId: string;
252
267
  fullPath: string;
253
268
  placement: FolderPlacement;
254
269
  }
255
270
 
271
+ export interface BodySyncRoundFolder extends BodySyncFolder {
272
+ uidValidity: number;
273
+ }
274
+
256
275
  export interface PlacementConfig {
257
276
  mailboxSpecialUseService: IMailboxSpecialUseRepository;
258
277
  /**
@@ -297,7 +316,7 @@ export interface UnsubscribeConfig {
297
316
  /**
298
317
  * What body sync needs to offer a mail's invitation as a card (issue #1033).
299
318
  *
300
- * The unit of work is here to provision the account's default collection on
319
+ * The calendar store is here to provision the account's default collection on
301
320
  * first use, through the one function every other caller provisions with. That
302
321
  * is not bookkeeping: an invitation whose DTSTART names no zone is RFC 5545
303
322
  * floating time, and reading it anywhere but where the user's calendar lives
@@ -309,14 +328,13 @@ export interface UnsubscribeConfig {
309
328
  */
310
329
  export interface CalendarSuggestionConfig {
311
330
  calendarSuggestionService: ICalendarSuggestionRepository;
312
- calendarUnitOfWork: ICalendarUnitOfWork;
331
+ calendarStore: ICalendarStore;
313
332
  filterService: Pick<IFilterRepository, "listByAccountAndState">;
314
333
  }
315
334
 
316
335
  export class BodySyncService {
317
336
  private log: BodySyncLogger;
318
337
  private readonly filterPipeline?: FilterPipeline;
319
- private readonly unitOfWork: IUnitOfWork;
320
338
 
321
339
  constructor(
322
340
  private messageService: IMessageRepository,
@@ -324,56 +342,30 @@ export class BodySyncService {
324
342
  private threadMessageService: IThreadMessageRepository,
325
343
  private addressService: IAddressRepository,
326
344
  private envelopeService: IEnvelopeRepository,
345
+ private readonly syncWarningService: SyncWarningService,
327
346
  logger?: BodySyncLogger,
328
347
  private readonly placementConfig?: PlacementConfig,
329
348
  private readonly filterConfig?: FilterConfig,
330
349
  private readonly quarantineConfig?: QuarantineConfig,
331
350
  private readonly unsubscribeConfig?: UnsubscribeConfig,
332
351
  private readonly calendarConfig?: CalendarSuggestionConfig,
333
- unitOfWork?: IUnitOfWork,
334
352
  ) {
335
353
  this.log = logger ?? noopLogger;
336
- this.unitOfWork =
337
- unitOfWork ??
338
- new PassThroughUnitOfWork({
339
- message: messageService,
340
- envelope: envelopeService,
341
- address: addressService,
342
- threadMessage: threadMessageService,
343
- });
344
354
  this.filterPipeline = filterConfig
345
355
  ? new FilterPipeline(filterConfig, this.log)
346
356
  : undefined;
347
357
  }
348
358
 
349
- /**
350
- * Sync message bodies for a batch of messages.
351
- *
352
- * Fail-fast on connection errors: if the IMAP connection is lost,
353
- * immediately stop processing and return all remaining messages as failed
354
- * so they can be requeued for later retry.
355
- *
356
- * @param messageIds - The message IDs to sync bodies for
357
- * @param accountId - The account ID (for storage path)
358
- * @param accountConfigId - The account config ID (for thread updates)
359
- * @param folder - The IMAP mailbox path and its folder placement
360
- * @param getConnection - Function to get a (lazy) IMAP connection
361
- * @param force - Bypass the "already stored" skip guard and re-fetch every
362
- * message in this batch even though `bodyStorageKey` is already set. Set
363
- * only by the read-miss re-arm cue (a `/content` read found the storage
364
- * object missing despite the DB row saying otherwise); bulk metadata-sync
365
- * never sets this, so it keeps skipping on `bodyStorageKey` with no
366
- * existence check.
367
- */
368
359
  async syncBodies(
369
360
  messageIds: string[],
370
361
  accountId: string,
371
362
  accountConfigId: string,
372
- folder: BodySyncFolder,
363
+ folder: BodySyncRoundFolder,
373
364
  getConnection: ConnectionGetter,
374
365
  force = false,
375
366
  ): Promise<SyncBodiesResult> {
376
367
  const syncedMessageIds: string[] = [];
368
+ const syncedUids: number[] = [];
377
369
  let skippedCount = 0;
378
370
  const location = {
379
371
  accountId,
@@ -403,8 +395,22 @@ export class BodySyncService {
403
395
  const quarantined =
404
396
  await this.quarantineConfig?.quarantineService.load(accountConfigId);
405
397
 
398
+ const rows = new Map(
399
+ (await this.messageService.get(messageIds)).map((row) => [
400
+ row.messageId,
401
+ row,
402
+ ]),
403
+ );
404
+
406
405
  for (const messageId of messageIds) {
407
- const message = await this.messageService.get(messageId);
406
+ const message = rows.get(messageId);
407
+
408
+ // A row deleted since the batch was cut, or moved out of this folder
409
+ // for good, has no uid here; its new uid would fetch another message.
410
+ if (!message || uidMailboxIdOf(message) !== folder.mailboxId) {
411
+ skippedCount++;
412
+ continue;
413
+ }
408
414
 
409
415
  // A quarantined message has no stored body, so nothing else here would
410
416
  // stop it being fetched and re-parsed on every redelivery, forever.
@@ -472,43 +478,31 @@ export class BodySyncService {
472
478
  reStoredMessageIds.has(messageId),
473
479
  );
474
480
  } catch (error) {
475
- // A per-message store failure (e.g. the parsed-body S3 write) is a
476
- // real fault, but it must not silently mark the message synced —
477
- // that would leave parsedBody null and the message unindexed. Leave
478
- // the UID in `pending` so it lands in failedMessageIds and the SQS
479
- // batch requeues just this message. A dropped connection is handled
480
- // by the outer catch (fail-fast on the whole remaining batch).
481
- if (isConnectionDrop(error)) throw error;
482
-
483
481
  // The one error in this frame that is the message's own fault. It
484
- // reaches here from the parse call and nowhere else, so the S3, DDB
485
- // and SQS work this same frame wraps cannot be mistaken for it and
486
- // keeps requeueing below. The row is durable before the UID leaves
487
- // `pending`, so the message is never let go of unrecorded.
488
- if (
489
- error instanceof BodyParseError &&
490
- (await this.quarantineBodyParse(messageId, location, error))
491
- ) {
482
+ // reaches here from the parse call and nowhere else, so the S3, DB
483
+ // and queue work this same frame wraps cannot be mistaken for it.
484
+ // The row is durable before the UID leaves `pending`, so the
485
+ // message is never let go of unrecorded.
486
+ if (error instanceof BodyParseError && this.quarantineConfig) {
487
+ await this.recordQuarantine(
488
+ this.quarantineConfig,
489
+ messageId,
490
+ location,
491
+ error,
492
+ );
492
493
  pending.delete(uid);
493
494
  skippedCount++;
494
495
  source.resume();
495
496
  continue;
496
497
  }
497
498
 
498
- this.log.error?.(
499
- {
500
- messageId,
501
- errorName: (error as { name?: string }).name,
502
- error: (error as Error).message,
503
- },
504
- "Body store failed for message; leaving for requeue",
505
- );
506
- // Drain the stream so the connection stays usable for the next UID.
507
- source.resume();
508
- continue;
499
+ // Anything else is our own storage, database or queue failing. It
500
+ // aborts the batch, which redelivers whole; what stored skips.
501
+ throw error;
509
502
  }
510
503
  pending.delete(uid);
511
504
  syncedMessageIds.push(messageId);
505
+ syncedUids.push(uid);
512
506
  }
513
507
  } catch (error) {
514
508
  this.log.error?.(
@@ -532,6 +526,21 @@ export class BodySyncService {
532
526
  // server silently omitted) — re-enqueue it.
533
527
  const failedMessageIds = [...pending.values()];
534
528
 
529
+ await this.syncWarningService.settle(
530
+ {
531
+ accountConfigId,
532
+ accountId,
533
+ mailboxId: folder.mailboxId,
534
+ uidValidity: folder.uidValidity,
535
+ },
536
+ SyncWarningStage.Body,
537
+ {
538
+ failedUids: [...pending.keys()],
539
+ syncedUids,
540
+ serverUids: { kind: "Unlisted" },
541
+ },
542
+ );
543
+
535
544
  this.log.info(
536
545
  {
537
546
  synced: syncedMessageIds.length,
@@ -547,53 +556,10 @@ export class BodySyncService {
547
556
  }
548
557
 
549
558
  /**
550
- * Set a message aside because its body will not parse (issue #72).
551
- *
552
- * Returns false when no record was written — either because this service was
553
- * built without a quarantine writer, or because writing it failed. The
554
- * caller then falls back to requeueing rather than dropping a message no
555
- * record survives. The failure the row carries is the parser's, and only the
556
- * parser's — see {@link BodyParseError}.
557
- *
558
- * Reading and writing the record is storage and database work, so its own
559
- * failure is contained per message the same way the save it replaces is: it
560
- * must not take down a batch that has other messages to fetch, and in the
561
- * stream loop a throw would also skip `source.resume()` and leave the
562
- * literal undrained on an otherwise usable connection.
563
- *
564
- * The fingerprint comes from the BodyPart rows metadata sync already wrote:
565
- * the body path streams raw bytes and has no FETCH result to read, and
566
- * re-reading headers the parser has just refused would be reading the very
567
- * thing that is broken.
559
+ * Set a message aside because its body will not parse (issue #72). The
560
+ * fingerprint comes from the BodyPart rows metadata sync already wrote: the
561
+ * body path streams raw bytes and has no FETCH result to read.
568
562
  */
569
- private async quarantineBodyParse(
570
- messageId: string,
571
- location: {
572
- accountId: string;
573
- accountConfigId: string;
574
- mailboxPath: string;
575
- },
576
- error: BodyParseError,
577
- ): Promise<boolean> {
578
- const config = this.quarantineConfig;
579
- if (!config) return false;
580
-
581
- return this.recordQuarantine(config, messageId, location, error).then(
582
- () => true,
583
- (writeError: unknown) => {
584
- this.log.error?.(
585
- {
586
- messageId,
587
- errorName: (writeError as { name?: string }).name,
588
- error: inspect(writeError),
589
- },
590
- "Could not record quarantine; leaving the message for requeue",
591
- );
592
- return false;
593
- },
594
- );
595
- }
596
-
597
563
  private async recordQuarantine(
598
564
  config: QuarantineConfig,
599
565
  messageId: string,
@@ -606,6 +572,10 @@ export class BodySyncService {
606
572
  ): Promise<void> {
607
573
  const message = await this.messageService.get(messageId);
608
574
  const messageData = await this.envelopeService.getMessageData(messageId);
575
+ const threadMessage = await this.threadMessageService.findByMessageId(
576
+ location.accountConfigId,
577
+ messageId,
578
+ );
609
579
 
610
580
  await config.quarantineService.record(
611
581
  {
@@ -622,7 +592,11 @@ export class BodySyncService {
622
592
  code: error.failureCode,
623
593
  message: error.message,
624
594
  },
625
- shapeFromMessageData(message, messageData),
595
+ shapeFromMessageData(
596
+ message,
597
+ messageData,
598
+ threadMessage?.messageIdHeader,
599
+ ),
626
600
  );
627
601
  }
628
602
 
@@ -779,7 +753,8 @@ export class BodySyncService {
779
753
  accountConfigId,
780
754
  accountId,
781
755
  messageId,
782
- parsed,
756
+ await this.envelopeService.listBodyParts(messageId),
757
+ body,
783
758
  );
784
759
  }
785
760
  await this.messageService.update(messageId, {
@@ -824,6 +799,7 @@ export class BodySyncService {
824
799
  // The DECISION is computed here; the applies below are unswallowed.
825
800
  const filterDecision = await this.evaluateFilters(
826
801
  accountConfigId,
802
+ accountId,
827
803
  messageId,
828
804
  parsed,
829
805
  );
@@ -831,10 +807,10 @@ export class BodySyncService {
831
807
  // Store the parsed-body cache BEFORE persisting bodyStorageKey. The skip
832
808
  // guard in syncBodies treats a stored bodyStorageKey as "fully synced",
833
809
  // so if we wrote it first and the parsed-cache write then failed, the
834
- // requeued retry would skip the message — leaving parsedBody null and the
835
- // search-index upsert bodyless forever. A parsed-cache failure here throws
836
- // and propagates before bodyStorageKey is written, so the requeued message
837
- // genuinely re-attempts the parsed write and gets indexed.
810
+ // requeued retry would skip the message — leaving parsedBody null forever.
811
+ // A parsed-cache failure here throws and propagates before bodyStorageKey
812
+ // is written, so the requeued message genuinely re-attempts the parsed
813
+ // write.
838
814
  await this.storeParsedBodyCache(
839
815
  accountConfigId,
840
816
  accountId,
@@ -851,7 +827,8 @@ export class BodySyncService {
851
827
  accountConfigId,
852
828
  accountId,
853
829
  messageId,
854
- parsed,
830
+ await this.envelopeService.listBodyParts(messageId),
831
+ body,
855
832
  );
856
833
  }
857
834
 
@@ -872,10 +849,13 @@ export class BodySyncService {
872
849
  // offer an undo back to it (issue #223). Snapshotted into the marker
873
850
  // rather than read from `originalMailboxId` at derive time, which a
874
851
  // later user move would overwrite — the filter's source must survive an
875
- // undo to stay derivable.
876
- const sourceMailboxId = (await this.messageService.get(messageId))
877
- .mailboxId;
878
- await this.filterConfig?.placementMoveService.moveMessage(
852
+ // undo to stay derivable. A redelivery after a failed enqueue finds
853
+ // the move already applied locally, so the source is the folder the
854
+ // uid still names, not the one the row now points at.
855
+ const sourceMailboxId = uidMailboxIdOf(
856
+ await this.messageService.get(messageId),
857
+ );
858
+ const filed = await this.filterConfig?.placementMoveService.moveMessage(
879
859
  accountConfigId,
880
860
  messageId,
881
861
  destinationMailboxId,
@@ -886,7 +866,7 @@ export class BodySyncService {
886
866
  // there matches but was not moved by Remit. `moveMessage` no-ops in that
887
867
  // case; record neither the marker nor `movedByRemit`, so the badge never
888
868
  // claims a move that did not happen and its Undo is never a no-op.
889
- if (sourceMailboxId !== destinationMailboxId) {
869
+ if (sourceMailboxId !== destinationMailboxId && filed !== "refused") {
890
870
  filterMoved = true;
891
871
  filterMove = {
892
872
  filterId: filterDecision.move.filterId,
@@ -946,9 +926,8 @@ export class BodySyncService {
946
926
  const moved = Boolean(resolved.move || filterMoved);
947
927
 
948
928
  // ONE Message UpdateItem per synced message: bodyStorageKey + every
949
- // classification/derived field + the move flag + the audit verdict. Each
950
- // extra Message mutation emits a DDB stream record that fans out to a
951
- // redundant S3-Vectors upsert, so we collapse them into a single write.
929
+ // classification/derived field + the move flag + the audit verdict,
930
+ // collapsed into a single write.
952
931
  // The flag is folded in only when a move WAS applied (by the classifier or
953
932
  // a filter); the verdict is folded in whenever Remit decided to act.
954
933
  // Written LAST so bodyStorageKey — the skip-guard signal — is only durable
@@ -1140,12 +1119,7 @@ export class BodySyncService {
1140
1119
  );
1141
1120
  }
1142
1121
 
1143
- return this.reconcileGoneMessage(
1144
- message,
1145
- accountId,
1146
- accountConfigId,
1147
- mailboxPath,
1148
- );
1122
+ return this.reconcileGoneMessage(message, accountId, mailboxPath);
1149
1123
  }
1150
1124
 
1151
1125
  /**
@@ -1158,7 +1132,6 @@ export class BodySyncService {
1158
1132
  private async reconcileGoneMessage(
1159
1133
  message: MessageItem,
1160
1134
  accountId: string,
1161
- accountConfigId: string,
1162
1135
  mailboxPath: string,
1163
1136
  ): Promise<never> {
1164
1137
  const { messageId, uid } = message;
@@ -1171,26 +1144,10 @@ export class BodySyncService {
1171
1144
  );
1172
1145
  }
1173
1146
 
1174
- const reconciled = await this.unitOfWork.transaction(async (repos) => {
1175
- const deleted = await repos.message.deleteIfPlacement(messageId, {
1176
- status: message.status,
1177
- mailboxId: message.mailboxId,
1178
- uid,
1179
- });
1180
- if (!deleted) return false;
1181
- const threadMessages = await repos.threadMessage.findAllByMessageId(
1182
- accountConfigId,
1183
- messageId,
1184
- );
1185
- if (threadMessages.length > 0) {
1186
- await repos.threadMessage.deleteMany(
1187
- threadMessages.map((row) => ({
1188
- accountConfigId: row.accountConfigId,
1189
- threadMessageId: row.threadMessageId,
1190
- })),
1191
- );
1192
- }
1193
- return true;
1147
+ const reconciled = await this.messageService.deleteIfPlacement(messageId, {
1148
+ status: message.status,
1149
+ mailboxId: message.mailboxId,
1150
+ uid,
1194
1151
  });
1195
1152
 
1196
1153
  if (!reconciled) {
@@ -1403,7 +1360,7 @@ export class BodySyncService {
1403
1360
  const parts = calendarParts(parsed);
1404
1361
  if (parts.length === 0) return;
1405
1362
 
1406
- const { calendarUnitOfWork, calendarSuggestionService, filterService } =
1363
+ const { calendarStore, calendarSuggestionService, filterService } =
1407
1364
  this.calendarConfig;
1408
1365
 
1409
1366
  await Promise.resolve()
@@ -1433,7 +1390,7 @@ export class BodySyncService {
1433
1390
  // the organizer meant it. Idempotent: the id is derived, so a second
1434
1391
  // first use returns the collection the first one made.
1435
1392
  const collection = await provisionDefaultCalendar(
1436
- calendarUnitOfWork,
1393
+ calendarStore,
1437
1394
  accountConfigId,
1438
1395
  );
1439
1396
  for (const part of parts) {
@@ -1503,12 +1460,14 @@ export class BodySyncService {
1503
1460
  */
1504
1461
  private async evaluateFilters(
1505
1462
  accountConfigId: string,
1463
+ accountId: string,
1506
1464
  messageId: string,
1507
1465
  parsed: ParsedMail,
1508
1466
  ): Promise<FilterDecision> {
1509
1467
  if (!this.filterPipeline) return { labels: [] };
1510
1468
  return this.filterPipeline.evaluate(
1511
1469
  accountConfigId,
1470
+ accountId,
1512
1471
  messageId,
1513
1472
  toFilterMessage(parsed),
1514
1473
  );
@@ -1517,8 +1476,7 @@ export class BodySyncService {
1517
1476
  /**
1518
1477
  * Resolve the placement decision for a message, BEFORE the single Message
1519
1478
  * update — so both the `movedByRemit` flag and the audit verdict join that
1520
- * one UpdateItem instead of a second mutation that would fan out another
1521
- * redundant S3-Vectors upsert. Drives both directions (junk → inbox rescue,
1479
+ * one UpdateItem instead of a second mutation. Drives both directions (junk → inbox rescue,
1522
1480
  * inbox → junk demote) off the pure {@link classifyPlacement} verdict.
1523
1481
  *
1524
1482
  * Returns a {@link PlacementOutcome}: a `verdict` to persist whenever Remit
@@ -1772,10 +1730,10 @@ export class BodySyncService {
1772
1730
  accountConfigId: string,
1773
1731
  accountId: string,
1774
1732
  messageId: string,
1775
- parsed: ParsedMail,
1733
+ bodyParts: readonly BodyPartItem[],
1734
+ body: Buffer,
1776
1735
  options?: { skipExisting?: boolean },
1777
1736
  ): Promise<StoreBodyPartContentsResult> {
1778
- const bodyParts = await this.envelopeService.listBodyParts(messageId);
1779
1737
  if (bodyParts.length === 0) {
1780
1738
  this.log.debug?.(
1781
1739
  { messageId },
@@ -1785,31 +1743,32 @@ export class BodySyncService {
1785
1743
  }
1786
1744
 
1787
1745
  const log = this.log;
1788
- const pairs = mapBodyPartsToContent(bodyParts, parsed, {
1789
- messageId,
1790
- logger:
1791
- log.warn || log.debug
1792
- ? {
1793
- warn: (obj, msg) => {
1794
- log.warn?.(obj, msg);
1795
- },
1796
- debug: (obj, msg) => {
1797
- log.debug?.(obj, msg);
1798
- },
1799
- }
1800
- : undefined,
1801
- });
1746
+ // A parse of its own: classification and the snippet, which search reads,
1747
+ // need an inline forwarded message's text, which `ignoreEmbedded` keeps out.
1748
+ const pairs = await mapMessagePartsToContent(
1749
+ bodyParts,
1750
+ await parsePartSource(body),
1751
+ {
1752
+ messageId,
1753
+ logger:
1754
+ log.warn || log.debug
1755
+ ? {
1756
+ warn: (obj, msg) => {
1757
+ log.warn?.(obj, msg);
1758
+ },
1759
+ debug: (obj, msg) => {
1760
+ log.debug?.(obj, msg);
1761
+ },
1762
+ }
1763
+ : undefined,
1764
+ },
1765
+ );
1802
1766
 
1803
1767
  if (pairs.length === 0) {
1804
1768
  return { stored: 0 };
1805
1769
  }
1806
1770
 
1807
- const bodyPartIdByPath = new Map(
1808
- bodyParts.map((bp) => [bp.partPath, bp.bodyPartId]),
1809
- );
1810
-
1811
1771
  let stored = 0;
1812
- const textContents: { bodyPartId: string; content: string }[] = [];
1813
1772
  await pMap(
1814
1773
  pairs,
1815
1774
  async (entry) => {
@@ -1833,30 +1792,10 @@ export class BodySyncService {
1833
1792
  contentType: entry.contentType,
1834
1793
  });
1835
1794
  stored++;
1836
-
1837
- if (
1838
- entry.contentType.toLowerCase().startsWith("text/") &&
1839
- entry.content.length > 0
1840
- ) {
1841
- const bodyPartId = bodyPartIdByPath.get(entry.partPath);
1842
- if (bodyPartId) {
1843
- textContents.push({
1844
- bodyPartId,
1845
- content: entry.content.toString("utf8"),
1846
- });
1847
- }
1848
- }
1849
1795
  },
1850
1796
  { concurrency: BODY_PART_STORE_CONCURRENCY },
1851
1797
  );
1852
1798
 
1853
- if (textContents.length > 0) {
1854
- await this.envelopeService.upsertBodyPartContents(
1855
- messageId,
1856
- textContents,
1857
- );
1858
- }
1859
-
1860
1799
  this.log.info(
1861
1800
  { messageId, partCount: pairs.length, stored },
1862
1801
  "Body parts stored",
@@ -1892,12 +1831,17 @@ export class BodySyncService {
1892
1831
  messageId: string,
1893
1832
  bodyStorageKey: string,
1894
1833
  ): Promise<StoreBodyPartContentsResult> {
1834
+ const bodyParts = await this.envelopeService.listBodyParts(messageId);
1835
+ const carriesMessage = bodyParts.some(isEncapsulatedMessage);
1836
+ const sentinel = carriesMessage
1837
+ ? MATERIALIZED_SENTINEL_V2_PART_PATH
1838
+ : MATERIALIZED_SENTINEL_PART_PATH;
1895
1839
  if (
1896
1840
  await this.storageService.bodyPartExists(
1897
1841
  accountConfigId,
1898
1842
  accountId,
1899
1843
  messageId,
1900
- MATERIALIZED_SENTINEL_PART_PATH,
1844
+ sentinel,
1901
1845
  )
1902
1846
  ) {
1903
1847
  this.log.debug?.(
@@ -1908,13 +1852,13 @@ export class BodySyncService {
1908
1852
  }
1909
1853
 
1910
1854
  const body = await this.storageService.retrieve(bodyStorageKey);
1911
- const parsed = await simpleParser(body);
1912
1855
  const result = await this.storeBodyPartContents(
1913
1856
  accountConfigId,
1914
1857
  accountId,
1915
1858
  messageId,
1916
- parsed,
1917
- { skipExisting: true },
1859
+ bodyParts,
1860
+ body,
1861
+ { skipExisting: !carriesMessage },
1918
1862
  );
1919
1863
 
1920
1864
  // Reaching here means storeBodyPartContents resolved — `pMap` rejects on
@@ -1924,47 +1868,24 @@ export class BodySyncService {
1924
1868
  accountConfigId,
1925
1869
  accountId,
1926
1870
  messageId,
1927
- partPath: MATERIALIZED_SENTINEL_PART_PATH,
1871
+ partPath: sentinel,
1928
1872
  content: Buffer.alloc(0),
1929
1873
  });
1930
1874
 
1931
1875
  return result;
1932
1876
  }
1933
1877
 
1934
- /**
1935
- * Persist the pre-parsed body cache. A failure here MUST fail the
1936
- * surrounding body-sync: the parsed-body object is what the search-index
1937
- * pipeline reads, so swallowing a write failure leaves the message marked
1938
- * synced while `parsedBody` is null and it never gets indexed — a silent
1939
- * search gap. Propagate so the message lands in failedMessageIds and the
1940
- * SQS batch requeues it.
1941
- */
1942
1878
  private async storeParsedBodyCache(
1943
1879
  accountConfigId: string,
1944
1880
  accountId: string,
1945
1881
  messageId: string,
1946
1882
  parsed: ParsedMail,
1947
1883
  ): Promise<void> {
1948
- const parsedBody = toParsedBody(parsed);
1949
- try {
1950
- await this.storageService.storeParsedBody({
1951
- accountConfigId,
1952
- accountId,
1953
- messageId,
1954
- parsed: parsedBody,
1955
- });
1956
- this.log.debug?.({ messageId }, "Parsed body cache stored");
1957
- } catch (err: unknown) {
1958
- this.log.error?.(
1959
- {
1960
- messageId,
1961
- errorName: (err as { name?: string }).name,
1962
- errorCode: (err as { Code?: string }).Code,
1963
- error: inspect(err),
1964
- },
1965
- "Failed to store parsed body cache; failing sync to requeue",
1966
- );
1967
- throw err;
1968
- }
1884
+ await this.storageService.storeParsedBody({
1885
+ accountConfigId,
1886
+ accountId,
1887
+ messageId,
1888
+ parsed: toParsedBody(parsed),
1889
+ });
1969
1890
  }
1970
1891
  }