@remit/mailbox-service 0.0.111 → 0.0.113

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (298) hide show
  1. package/dist/account-credentials.d.ts +83 -0
  2. package/dist/account-credentials.js +104 -0
  3. package/dist/attribute-mapper.d.ts +36 -0
  4. package/dist/attribute-mapper.js +119 -0
  5. package/dist/body-parse.d.ts +28 -0
  6. package/{src/body-parse.ts → dist/body-parse.js} +18 -26
  7. package/dist/body-part-mapper.d.ts +92 -0
  8. package/dist/body-part-mapper.js +301 -0
  9. package/dist/body-sync-queue.d.ts +59 -0
  10. package/dist/body-sync-queue.js +56 -0
  11. package/dist/body-sync-terminal.d.ts +58 -0
  12. package/dist/body-sync-terminal.js +95 -0
  13. package/dist/body-sync.d.ts +351 -0
  14. package/dist/body-sync.js +1204 -0
  15. package/dist/calendar-parts.d.ts +23 -0
  16. package/dist/calendar-parts.js +18 -0
  17. package/dist/classification-backfill.d.ts +91 -0
  18. package/dist/classification-backfill.js +180 -0
  19. package/dist/classify-message.d.ts +31 -0
  20. package/dist/classify-message.js +89 -0
  21. package/dist/connection-factory.d.ts +54 -0
  22. package/dist/connection-factory.js +54 -0
  23. package/dist/connection-test.d.ts +41 -0
  24. package/dist/connection-test.js +152 -0
  25. package/dist/external-move.d.ts +94 -0
  26. package/dist/external-move.js +173 -0
  27. package/dist/filter-operations.d.ts +42 -0
  28. package/dist/filter-operations.js +211 -0
  29. package/dist/filters/filter-config.d.ts +14 -0
  30. package/dist/filters/filter-config.js +10 -0
  31. package/{src/filters/list-id.ts → dist/filters/list-id.d.ts} +2 -16
  32. package/dist/filters/list-id.js +26 -0
  33. package/dist/filters/match.d.ts +69 -0
  34. package/dist/filters/match.js +165 -0
  35. package/dist/filters/pipeline.d.ts +64 -0
  36. package/dist/filters/pipeline.js +100 -0
  37. package/{src/flag-push-terminal.ts → dist/flag-push-terminal.d.ts} +19 -77
  38. package/dist/flag-push-terminal.js +63 -0
  39. package/dist/flag-push.d.ts +69 -0
  40. package/dist/flag-push.js +111 -0
  41. package/dist/flag-queue.d.ts +175 -0
  42. package/dist/flag-queue.js +291 -0
  43. package/dist/folder-operations.d.ts +39 -0
  44. package/dist/folder-operations.js +112 -0
  45. package/dist/folder-placement.d.ts +13 -0
  46. package/dist/folder-placement.js +23 -0
  47. package/dist/heuristics/authenticationResults.d.ts +10 -0
  48. package/dist/heuristics/authenticationResults.js +187 -0
  49. package/dist/heuristics/classifyByHeaders.d.ts +82 -0
  50. package/dist/heuristics/classifyByHeaders.js +365 -0
  51. package/dist/heuristics/classifyPlacement.d.ts +67 -0
  52. package/dist/heuristics/classifyPlacement.js +153 -0
  53. package/dist/heuristics/machineSenders.d.ts +21 -0
  54. package/{src/heuristics/machineSenders.ts → dist/heuristics/machineSenders.js} +34 -40
  55. package/dist/heuristics/senderMismatch.d.ts +81 -0
  56. package/dist/heuristics/senderMismatch.js +192 -0
  57. package/dist/heuristics/socialDomains.d.ts +14 -0
  58. package/{src/heuristics/socialDomains.ts → dist/heuristics/socialDomains.js} +9 -9
  59. package/dist/heuristics/transactionalDomains.d.ts +18 -0
  60. package/{src/heuristics/transactionalDomains.ts → dist/heuristics/transactionalDomains.js} +8 -8
  61. package/dist/imapflow-connection.d.ts +377 -0
  62. package/dist/imapflow-connection.js +1147 -0
  63. package/dist/index.d.ts +63 -0
  64. package/dist/index.js +65 -0
  65. package/dist/list-id-backfill.d.ts +83 -0
  66. package/dist/list-id-backfill.js +132 -0
  67. package/dist/mailbox-cursor-rebuild.d.ts +74 -0
  68. package/dist/mailbox-cursor-rebuild.js +71 -0
  69. package/{src/mailbox-cursor.ts → dist/mailbox-cursor.d.ts} +20 -98
  70. package/dist/mailbox-cursor.js +102 -0
  71. package/dist/mailbox-intent.d.ts +41 -0
  72. package/{src/mailbox-intent.ts → dist/mailbox-intent.js} +39 -63
  73. package/dist/mailbox-management.d.ts +240 -0
  74. package/dist/mailbox-management.js +576 -0
  75. package/dist/mailbox-presence.d.ts +52 -0
  76. package/{src/mailbox-presence.ts → dist/mailbox-presence.js} +19 -30
  77. package/dist/mailbox-queue.d.ts +126 -0
  78. package/dist/mailbox-queue.js +214 -0
  79. package/dist/mailbox-sync.d.ts +93 -0
  80. package/dist/mailbox-sync.js +490 -0
  81. package/dist/mailbox-upstream.d.ts +38 -0
  82. package/{src/mailbox-upstream.ts → dist/mailbox-upstream.js} +37 -47
  83. package/dist/message-category.d.ts +51 -0
  84. package/{src/message-category.ts → dist/message-category.js} +28 -71
  85. package/dist/message-move.d.ts +285 -0
  86. package/dist/message-move.js +858 -0
  87. package/dist/message-parser.d.ts +14 -0
  88. package/dist/message-parser.js +17 -0
  89. package/{src/message-presence.ts → dist/message-presence.d.ts} +1 -13
  90. package/dist/message-presence.js +29 -0
  91. package/dist/message-read.d.ts +51 -0
  92. package/dist/message-read.js +31 -0
  93. package/dist/message-sync.d.ts +345 -0
  94. package/dist/message-sync.js +1358 -0
  95. package/dist/mime-walker.d.ts +102 -0
  96. package/dist/mime-walker.js +198 -0
  97. package/dist/organize-job.d.ts +20 -0
  98. package/dist/organize-job.js +37 -0
  99. package/dist/outbox-attachment-content.d.ts +37 -0
  100. package/dist/outbox-attachment-content.js +50 -0
  101. package/dist/outbox-attachment-filename.d.ts +19 -0
  102. package/dist/outbox-attachment-filename.js +64 -0
  103. package/dist/outbox-attachment.d.ts +148 -0
  104. package/dist/outbox-attachment.js +273 -0
  105. package/dist/outbox-queue.d.ts +73 -0
  106. package/dist/outbox-queue.js +284 -0
  107. package/{src/outbox-status.ts → dist/outbox-status.d.ts} +1 -6
  108. package/dist/outbox-status.js +18 -0
  109. package/dist/ownership.d.ts +24 -0
  110. package/dist/ownership.js +83 -0
  111. package/dist/placement-move-terminal.d.ts +67 -0
  112. package/dist/placement-move-terminal.js +88 -0
  113. package/dist/placement-move.d.ts +83 -0
  114. package/dist/placement-move.js +216 -0
  115. package/{src/placement-settled.ts → dist/placement-settled.d.ts} +19 -50
  116. package/dist/placement-settled.js +75 -0
  117. package/dist/quarantine.d.ts +95 -0
  118. package/dist/quarantine.js +131 -0
  119. package/dist/refusal.d.ts +31 -0
  120. package/dist/refusal.js +3 -0
  121. package/{src/restore-source-placement.ts → dist/restore-source-placement.d.ts} +29 -101
  122. package/dist/restore-source-placement.js +79 -0
  123. package/dist/search-text-backfill.d.ts +15 -0
  124. package/dist/search-text-backfill.js +33 -0
  125. package/dist/snippet.d.ts +61 -0
  126. package/dist/snippet.js +129 -0
  127. package/dist/spam-report.d.ts +86 -0
  128. package/dist/spam-report.js +241 -0
  129. package/dist/sync-warning.d.ts +21 -0
  130. package/dist/sync-warning.js +26 -0
  131. package/dist/sync-watermarks.d.ts +138 -0
  132. package/dist/sync-watermarks.js +205 -0
  133. package/dist/test-helpers/fake-imap.d.ts +151 -0
  134. package/dist/test-helpers/fake-imap.js +508 -0
  135. package/dist/test-helpers/folder-roles.d.ts +34 -0
  136. package/dist/test-helpers/folder-roles.js +47 -0
  137. package/dist/test-helpers/mime-tree-generator.d.ts +82 -0
  138. package/dist/test-helpers/mime-tree-generator.js +329 -0
  139. package/dist/test-helpers/test-store.d.ts +69 -0
  140. package/dist/test-helpers/test-store.js +193 -0
  141. package/dist/text/normalizer.d.ts +20 -0
  142. package/dist/text/normalizer.js +102 -0
  143. package/dist/thread-message-rows.d.ts +69 -0
  144. package/dist/thread-message-rows.js +40 -0
  145. package/dist/thread-root.d.ts +7 -0
  146. package/dist/thread-root.js +25 -0
  147. package/dist/types.d.ts +343 -0
  148. package/dist/types.js +19 -0
  149. package/package.json +86 -73
  150. package/src/account-credentials.ts +0 -210
  151. package/src/adversarial-mime.e2e.test.ts +0 -307
  152. package/src/attribute-mapper.test.ts +0 -104
  153. package/src/attribute-mapper.ts +0 -148
  154. package/src/body-parse.test.ts +0 -55
  155. package/src/body-part-mapper.property.test.ts +0 -354
  156. package/src/body-part-mapper.ts +0 -360
  157. package/src/body-sync-auto-read-writeonce.test.ts +0 -219
  158. package/src/body-sync-calendar-suggestion.test.ts +0 -433
  159. package/src/body-sync-category-denormalize.test.ts +0 -352
  160. package/src/body-sync-category-override.test.ts +0 -501
  161. package/src/body-sync-category-writeonce.test.ts +0 -234
  162. package/src/body-sync-filter.test.ts +0 -237
  163. package/src/body-sync-message-gone.test.ts +0 -139
  164. package/src/body-sync-placement-flags.test.ts +0 -347
  165. package/src/body-sync-placement-writeonce.test.ts +0 -337
  166. package/src/body-sync-quarantine.test.ts +0 -276
  167. package/src/body-sync-queue.test.ts +0 -124
  168. package/src/body-sync-queue.ts +0 -131
  169. package/src/body-sync-resync.test.ts +0 -605
  170. package/src/body-sync-terminal.test.ts +0 -310
  171. package/src/body-sync-terminal.ts +0 -176
  172. package/src/body-sync-unclassifiable-once.test.ts +0 -260
  173. package/src/body-sync-unsubscribed-auto-read.test.ts +0 -176
  174. package/src/body-sync.ts +0 -1970
  175. package/src/calendar-parts.ts +0 -41
  176. package/src/classification-backfill.test.ts +0 -615
  177. package/src/classification-backfill.ts +0 -363
  178. package/src/classify-message.test.ts +0 -205
  179. package/src/classify-message.ts +0 -116
  180. package/src/connection-factory.ts +0 -102
  181. package/src/connection-test.ts +0 -208
  182. package/src/credentials.test.ts +0 -181
  183. package/src/external-move.test.ts +0 -369
  184. package/src/external-move.ts +0 -272
  185. package/src/filters/anchor-drift.ts +0 -55
  186. package/src/filters/filter-config.test.ts +0 -229
  187. package/src/filters/filter-config.ts +0 -69
  188. package/src/filters/list-id.test.ts +0 -50
  189. package/src/filters/match.test.ts +0 -429
  190. package/src/filters/match.ts +0 -274
  191. package/src/filters/message-embedder.test.ts +0 -39
  192. package/src/filters/message-embedder.ts +0 -33
  193. package/src/filters/pipeline.test.ts +0 -330
  194. package/src/filters/pipeline.ts +0 -284
  195. package/src/flag-push-terminal.test.ts +0 -209
  196. package/src/flag-push.test.ts +0 -408
  197. package/src/flag-push.ts +0 -198
  198. package/src/flag-queue-stars.test.ts +0 -171
  199. package/src/flag-queue.ts +0 -509
  200. package/src/folder-placement.ts +0 -56
  201. package/src/heuristics/authenticationResults.test.ts +0 -189
  202. package/src/heuristics/authenticationResults.ts +0 -212
  203. package/src/heuristics/classifyByHeaders.test.ts +0 -889
  204. package/src/heuristics/classifyByHeaders.ts +0 -467
  205. package/src/heuristics/classifyPlacement.test.ts +0 -366
  206. package/src/heuristics/classifyPlacement.ts +0 -211
  207. package/src/heuristics/classifyRealMail.test.ts +0 -242
  208. package/src/heuristics/machineSenders.test.ts +0 -101
  209. package/src/heuristics/senderMismatch.test.ts +0 -346
  210. package/src/heuristics/senderMismatch.ts +0 -259
  211. package/src/imapflow-connect-retry.test.ts +0 -134
  212. package/src/imapflow-connection.e2e.test.ts +0 -335
  213. package/src/imapflow-connection.integ.test.ts +0 -316
  214. package/src/imapflow-connection.test.ts +0 -469
  215. package/src/imapflow-connection.ts +0 -1475
  216. package/src/index.ts +0 -350
  217. package/src/list-id-backfill.test.ts +0 -573
  218. package/src/list-id-backfill.ts +0 -274
  219. package/src/mailbox-cursor-rebuild.test.ts +0 -171
  220. package/src/mailbox-cursor-rebuild.ts +0 -146
  221. package/src/mailbox-cursor.test.ts +0 -249
  222. package/src/mailbox-delete-intent.test.ts +0 -246
  223. package/src/mailbox-management.test.ts +0 -356
  224. package/src/mailbox-management.ts +0 -839
  225. package/src/mailbox-presence.test.ts +0 -72
  226. package/src/mailbox-queue.ts +0 -415
  227. package/src/mailbox-rename-intent.test.ts +0 -389
  228. package/src/mailbox-rename-subtree.test.ts +0 -664
  229. package/src/mailbox-sync.test.ts +0 -848
  230. package/src/mailbox-sync.ts +0 -724
  231. package/src/message-delete-junk-reconcile.test.ts +0 -117
  232. package/src/message-delete-no-trash.test.ts +0 -245
  233. package/src/message-delete-unsettled-placement.test.ts +0 -396
  234. package/src/message-move-copy.test.ts +0 -562
  235. package/src/message-move-empty-trash.test.ts +0 -326
  236. package/src/message-move-same-mailbox.test.ts +0 -139
  237. package/src/message-move-unsettled-placement.test.ts +0 -378
  238. package/src/message-move.ts +0 -1348
  239. package/src/message-parser.ts +0 -26
  240. package/src/message-presence.test.ts +0 -50
  241. package/src/message-sync-address-spoof.test.ts +0 -293
  242. package/src/message-sync-category.test.ts +0 -142
  243. package/src/message-sync-changedsince.test.ts +0 -840
  244. package/src/message-sync-flags.test.ts +0 -158
  245. package/src/message-sync-junk-harvest.test.ts +0 -244
  246. package/src/message-sync-quarantine.test.ts +0 -416
  247. package/src/message-sync-repoint.test.ts +0 -419
  248. package/src/message-sync.ts +0 -1963
  249. package/src/mime-walker.test.ts +0 -311
  250. package/src/mime-walker.ts +0 -305
  251. package/src/outbox-attachment-content.test.ts +0 -153
  252. package/src/outbox-attachment-content.ts +0 -104
  253. package/src/outbox-attachment-filename.test.ts +0 -132
  254. package/src/outbox-attachment-filename.ts +0 -76
  255. package/src/outbox-attachment.test.ts +0 -738
  256. package/src/outbox-attachment.ts +0 -536
  257. package/src/outbox-failed-recovery.test.ts +0 -370
  258. package/src/outbox-queue-settled-row.test.ts +0 -140
  259. package/src/outbox-queue.test.ts +0 -291
  260. package/src/outbox-queue.ts +0 -508
  261. package/src/pass-through-unit-of-work.ts +0 -17
  262. package/src/placement-move-terminal.test.ts +0 -300
  263. package/src/placement-move-terminal.ts +0 -148
  264. package/src/placement-move-unsettled.test.ts +0 -275
  265. package/src/placement-move.ts +0 -336
  266. package/src/placement-settled.test.ts +0 -147
  267. package/src/placement-transition-race.test.ts +0 -482
  268. package/src/quarantine.test.ts +0 -191
  269. package/src/quarantine.ts +0 -228
  270. package/src/snippet.test.ts +0 -213
  271. package/src/snippet.ts +0 -152
  272. package/src/spam-report.test.ts +0 -801
  273. package/src/spam-report.ts +0 -357
  274. package/src/stale-message-reconcile.test.ts +0 -82
  275. package/src/stale-message-reconcile.ts +0 -55
  276. package/src/sync-watermarks.test.ts +0 -409
  277. package/src/sync-watermarks.ts +0 -319
  278. package/src/system-flag-wire-format.test.ts +0 -77
  279. package/src/test-helpers/folder-roles.ts +0 -66
  280. package/src/test-helpers/isolated-mailbox.ts +0 -44
  281. package/src/test-helpers/mailfuzz-connection.ts +0 -27
  282. package/src/test-helpers/mime-tree-generator.ts +0 -461
  283. package/src/text/normalizer.test.ts +0 -223
  284. package/src/text/normalizer.ts +0 -140
  285. package/src/thread-message-rows.ts +0 -77
  286. package/src/types/mailparser-augment.d.ts +0 -15
  287. package/src/types/natural-porter-stemmer.d.ts +0 -50
  288. package/src/types/stopword.d.ts +0 -69
  289. package/src/types.ts +0 -371
  290. package/test/fixtures/mime-adversarial-e2e/01-octet-stream-pdf.eml +0 -21
  291. package/test/fixtures/mime-adversarial-e2e/02-nested-related-cid-image.eml +0 -31
  292. package/test/fixtures/mime-adversarial-e2e/03-calendar-invite.eml +0 -33
  293. package/test/fixtures/mime-adversarial-e2e/04-forwarded-rfc822-with-attachments.eml +0 -42
  294. package/test/fixtures/mime-adversarial-e2e/05-tnef-winmail.eml +0 -25
  295. package/test/fixtures/mime-adversarial-e2e/06-8bit-non-ascii.eml +0 -15
  296. package/test/fixtures/mime-adversarial-e2e/07-quoted-printable-soft-breaks.eml +0 -17
  297. package/test/fixtures/mime-adversarial-e2e/08-alternative-empty-text.eml +0 -18
  298. package/tsconfig.json +0 -8
package/src/body-sync.ts DELETED
@@ -1,1970 +0,0 @@
1
- import { PassThrough, type Readable } from "node:stream";
2
- import { inspect } from "node:util";
3
- import {
4
- provisionDefaultCalendar,
5
- recordCalendarSuggestion,
6
- } from "@remit/calendar-service";
7
- import {
8
- type IAddressRepository,
9
- type ICalendarSuggestionRepository,
10
- type ICalendarUnitOfWork,
11
- type IEnvelopeRepository,
12
- type IFilterRepository,
13
- type IMailboxSpecialUseRepository,
14
- type IMessageRepository,
15
- type IThreadMessageRepository,
16
- type IUnitOfWork,
17
- isSenderMuted,
18
- type MessageItem,
19
- type ThreadMessageItem,
20
- type UpdateMessageInput,
21
- } from "@remit/data-ports";
22
- import {
23
- MessageGoneFromServerError,
24
- MessagePlacementUnsettledError,
25
- NotFoundError,
26
- } from "@remit/data-ports/errors";
27
- import { deriveAddressId, deriveBodyPartId } from "@remit/data-ports/id";
28
- import { isBulkSender } from "@remit/data-ports/wellknown";
29
- import {
30
- FilterState,
31
- MessageClassificationState,
32
- PlacementAction,
33
- PlacementConfidence,
34
- QuarantineFailureStage,
35
- SenderTrust,
36
- } from "@remit/domain-enums";
37
- import {
38
- isStorageNotFoundError,
39
- type ParsedBody,
40
- type StorageService,
41
- } from "@remit/storage-service";
42
- import { type ParsedMail, simpleParser } from "mailparser";
43
- import pMap from "p-map";
44
- import { BodyParseError, parseMessageBody } from "./body-parse.js";
45
- import { mapBodyPartsToContent } from "./body-part-mapper.js";
46
- import { calendarParts } from "./calendar-parts.js";
47
- import {
48
- classifyParsedMessage,
49
- extractPrimaryFromEmail,
50
- } from "./classify-message.js";
51
- import { extractListId } from "./filters/list-id.js";
52
- import type { FilterMessage } from "./filters/match.js";
53
- import {
54
- type FilterConfig,
55
- type FilterDecision,
56
- FilterPipeline,
57
- } from "./filters/pipeline.js";
58
- import type { FlagQueueService } from "./flag-queue.js";
59
- import { placementOf, resolveFolderRoleMailboxes } from "./folder-placement.js";
60
- import {
61
- classifyPlacement,
62
- type FolderPlacement,
63
- type PlacementVerdict,
64
- resolveSenderPlacement,
65
- } from "./heuristics/classifyPlacement.js";
66
- import {
67
- denormalizeMessageCategory,
68
- hasDecidedCategory,
69
- } from "./message-category.js";
70
- import { isMessageGoneFromOpenMailbox } from "./message-presence.js";
71
- import { PassThroughUnitOfWork } from "./pass-through-unit-of-work.js";
72
- import type { PlacementMoveService } from "./placement-move.js";
73
- import { isPlacementUnsettled } from "./placement-settled.js";
74
- import { type QuarantineService, shapeFromMessageData } from "./quarantine.js";
75
- import { extractSnippetFromEmail } from "./snippet.js";
76
- import { type IImapConnection, MailConnectionError } from "./types.js";
77
-
78
- const BODY_PART_STORE_CONCURRENCY = 4;
79
-
80
- type MessagePlacementVerdict = NonNullable<
81
- UpdateMessageInput["placementVerdict"]
82
- >;
83
-
84
- type MessageFilterMove = NonNullable<UpdateMessageInput["filterMove"]>;
85
-
86
- type ThreadMessageCategory = ThreadMessageItem["category"];
87
-
88
- /**
89
- * Outcome of {@link BodySyncService.resolvePlacement}: the audit `verdict` to
90
- * persist on the Message (present whenever Remit decided to act, i.e. action
91
- * != leave), and the IMAP `move` to enqueue (present only for a confident,
92
- * actionable verdict). An unsure verdict carries a `verdict` but no `move`.
93
- */
94
- interface PlacementOutcome {
95
- verdict?: MessagePlacementVerdict;
96
- move?: { destinationMailboxId: string; destinationPath: string };
97
- placementDecidedAt?: number;
98
- }
99
-
100
- /**
101
- * Whether the body-sync hot path defers per-part S3 objects. Each MIME leaf is
102
- * its own PutObject, so during bulk sync these per-part writes dominate S3 write
103
- * count and cost. With deferral ON a synced message writes only `body.eml` +
104
- * `parsed.json.gz` (2 writes); the per-part objects are materialized lazily on
105
- * the first `contentUrl` read via {@link BodySyncService.ensureBodyPartsStored}.
106
- *
107
- * Default ON (cost savings). Set `DEFER_BODY_PARTS=false`/`0`/`off` to restore
108
- * the original eager per-part writes — a safe, reversible escape hatch.
109
- */
110
- export const isBodyPartDeferralEnabled = (): boolean => {
111
- const raw = process.env.DEFER_BODY_PARTS;
112
- if (raw === undefined) return true;
113
- const normalized = raw.trim().toLowerCase();
114
- return normalized !== "false" && normalized !== "0" && normalized !== "off";
115
- };
116
-
117
- /**
118
- * Sentinel object written under a message's parts prefix once lazy
119
- * materialization has stored EVERY leaf. The read path checks it first with a
120
- * single HEAD and early-returns, skipping the body.eml GET, the MIME re-parse,
121
- * and the per-leaf HEADs on every warm re-open.
122
- *
123
- * Deliberately an S3-only marker (no Message-row flag): a DDB write would
124
- * re-fire the redundant vector upsert that #607 removed. A leading dot keeps it
125
- * out of any real IMAP section path (those are dot-separated digits).
126
- */
127
- const MATERIALIZED_SENTINEL_PART_PATH = ".materialized";
128
-
129
- /**
130
- * A mid-stream socket drop during the ranged body fetch surfaces as a typed
131
- * `MailConnectionError` (the connection layer classifies imapflow's
132
- * `EConnectionClosed`/`NoConnection`). Detect by type/code, never by message
133
- * text — the underlying library strings vary.
134
- */
135
- const isConnectionDrop = (error: unknown): boolean => {
136
- if (error instanceof MailConnectionError) {
137
- return error.kind === "network";
138
- }
139
- const code = (error as { code?: string }).code;
140
- return code === "EConnectionClosed" || code === "NoConnection";
141
- };
142
-
143
- /**
144
- * Project a parsed message onto the fields a filter matches against (RFC 034) —
145
- * the literal-clause targets plus the text a semantic anchor embeds. Kept
146
- * separate from `ParsedMail` so {@link FilterPipeline} stays parser-agnostic.
147
- */
148
- const toFilterMessage = (parsed: ParsedMail): FilterMessage => ({
149
- from: extractPrimaryFromEmail(parsed) ?? "",
150
- fromName: parsed.from?.value?.[0]?.name ?? "",
151
- subject: parsed.subject ?? "",
152
- text: parsed.text ?? "",
153
- listId: extractListId(parsed),
154
- });
155
-
156
- const SNIPPET_LENGTH = 256;
157
-
158
- /**
159
- * The snippet the list row shows, from whichever body part carries text.
160
- * Exported for the classification backfill (issue #1197), which denormalizes
161
- * the same fields the body-store path does, from the same bytes.
162
- */
163
- export const extractSnippet = (parsed: ParsedMail): string =>
164
- extractSnippetFromEmail(
165
- parsed.text,
166
- typeof parsed.html === "string" ? parsed.html : undefined,
167
- SNIPPET_LENGTH,
168
- );
169
-
170
- /**
171
- * Issue #499: whether a completed body-sync pass has already stored this
172
- * message's body. `bodyStorageKey` is written last, once every derivation of
173
- * that pass has run, so its presence means that pass finished.
174
- *
175
- * Key presence, and nothing more. What the classifier decided is
176
- * `Message.category`; whether it ran at all is `Message.classificationState`.
177
- * Neither is inferred from here — the field this reads is the storage fact.
178
- *
179
- * The re-fetch paths that re-enter with the key already set (`syncBodies(...,
180
- * force: true)` and `fetchAndGetBody`'s `NoSuchKey` fallback) are re-stores
181
- * that skip the derivations entirely (issue #1011), so this guard is what
182
- * keeps a first store that lands on an already-stored row from re-deciding
183
- * anything.
184
- */
185
- const hasStoredBody = (bodyStorageKey: string | undefined): boolean =>
186
- Boolean(bodyStorageKey);
187
-
188
- /**
189
- * Issue #398: `flags.autoArchive` is a filing preference relative to the Inbox.
190
- * A `leave` verdict reaches it for two unrelated reasons and only one of them
191
- * means "nothing confident to say":
192
- *
193
- * - `already-moved-by-remit` is {@link classifyPlacement}'s guard against
194
- * re-deciding a settled placement, and auto-archive is a re-decision. It
195
- * fires on messages Remit itself placed, or that a user moved back
196
- * afterwards.
197
- * - A message already in Junk gets an unsure `leave` whatever the sender's
198
- * `blocked` flag says, since the demote branch only fires outside Junk.
199
- * Moving mail out of Junk is a rescue, which this file reserves for a
200
- * confident signal plus sender trust.
201
- */
202
- const autoArchiveApplies = (
203
- verdict: PlacementVerdict,
204
- placement: FolderPlacement,
205
- ): boolean =>
206
- placement !== "junk" && !verdict.reasons.includes("already-moved-by-remit");
207
-
208
- export const toParsedBody = (parsed: ParsedMail): ParsedBody => ({
209
- text: parsed.text ?? null,
210
- html: typeof parsed.html === "string" ? parsed.html : null,
211
- attachments: (parsed.attachments ?? []).map((a) => ({
212
- filename: a.filename ?? null,
213
- contentType: a.contentType,
214
- contentDisposition: a.contentDisposition ?? null,
215
- contentId: a.contentId ?? null,
216
- size: a.size,
217
- })),
218
- });
219
-
220
- export interface BodySyncLogger {
221
- info(obj: Record<string, unknown>, msg: string): void;
222
- debug?(obj: Record<string, unknown>, msg: string): void;
223
- warn?(obj: Record<string, unknown>, msg: string): void;
224
- error?(obj: Record<string, unknown>, msg: string): void;
225
- }
226
-
227
- const noopLogger: BodySyncLogger = {
228
- info: () => {},
229
- };
230
-
231
- export interface SyncBodiesResult {
232
- syncedCount: number;
233
- syncedMessageIds: string[];
234
- skippedCount: number;
235
- failedCount: number;
236
- failedMessageIds: string[];
237
- }
238
-
239
- export interface FetchBodyResult {
240
- text: string | null;
241
- html: string | null;
242
- storedAt: string;
243
- }
244
-
245
- export interface StoreBodyPartContentsResult {
246
- stored: number;
247
- }
248
-
249
- export type ConnectionGetter = () => Promise<IImapConnection>;
250
-
251
- export interface BodySyncFolder {
252
- fullPath: string;
253
- placement: FolderPlacement;
254
- }
255
-
256
- export interface PlacementConfig {
257
- mailboxSpecialUseService: IMailboxSpecialUseRepository;
258
- /**
259
- * Local-first mover for a confident placement verdict (issue #1271). Same
260
- * `moveMessage(accountConfigId, messageId, destinationMailboxId, accountId)`
261
- * signature as the general-purpose `MessageMoveService` it replaced here,
262
- * but backed by a pending-move marker + its own reconciler queue instead of
263
- * the general MESSAGE_MOVE event (see `placement-move.ts`).
264
- */
265
- placementMoveService: PlacementMoveService;
266
- }
267
-
268
- /**
269
- * What body sync needs to set a message aside (issue #72). The mailbox facts
270
- * are supplied by the caller because they are properties of the round, not of
271
- * any one message.
272
- */
273
- export interface QuarantineConfig {
274
- quarantineService: QuarantineService;
275
- mailboxId: string;
276
- /**
277
- * The mailbox's stored UIDVALIDITY. Taken from the mailbox row rather than
278
- * from the open box because a round whose two disagree is paused by the
279
- * cursor guard before it reaches any UID.
280
- */
281
- uidValidity: number;
282
- /** Redeliveries of this batch so far, so the record says how hard we tried. */
283
- attempts: number;
284
- }
285
-
286
- /**
287
- * What body sync needs to auto-mark-read a message from an unsubscribed
288
- * sender (issue #302, RFC 039 Decision 3). Reuses the same
289
- * `FlagQueueService.markAsRead` a manual mark-as-read already goes through —
290
- * local `\Seen` + `ThreadMessage.isRead` + a durable pending IMAP flag-push
291
- * marker — so this fires the same round-trip, not a second primitive.
292
- */
293
- export interface UnsubscribeConfig {
294
- flagQueueService: FlagQueueService;
295
- }
296
-
297
- /**
298
- * What body sync needs to offer a mail's invitation as a card (issue #1033).
299
- *
300
- * The unit of work is here to provision the account's default collection on
301
- * first use, through the one function every other caller provisions with. That
302
- * is not bookkeeping: an invitation whose DTSTART names no zone is RFC 5545
303
- * floating time, and reading it anywhere but where the user's calendar lives
304
- * shows the meeting at the wrong hour. Without a collection there is no zone
305
- * to read it in, and every such event silently lands in UTC.
306
- *
307
- * `filterService` is read to honour `dismiss{muteSender:true}`: a sender the
308
- * user muted gets no card at all.
309
- */
310
- export interface CalendarSuggestionConfig {
311
- calendarSuggestionService: ICalendarSuggestionRepository;
312
- calendarUnitOfWork: ICalendarUnitOfWork;
313
- filterService: Pick<IFilterRepository, "listByAccountAndState">;
314
- }
315
-
316
- export class BodySyncService {
317
- private log: BodySyncLogger;
318
- private readonly filterPipeline?: FilterPipeline;
319
- private readonly unitOfWork: IUnitOfWork;
320
-
321
- constructor(
322
- private messageService: IMessageRepository,
323
- private storageService: StorageService,
324
- private threadMessageService: IThreadMessageRepository,
325
- private addressService: IAddressRepository,
326
- private envelopeService: IEnvelopeRepository,
327
- logger?: BodySyncLogger,
328
- private readonly placementConfig?: PlacementConfig,
329
- private readonly filterConfig?: FilterConfig,
330
- private readonly quarantineConfig?: QuarantineConfig,
331
- private readonly unsubscribeConfig?: UnsubscribeConfig,
332
- private readonly calendarConfig?: CalendarSuggestionConfig,
333
- unitOfWork?: IUnitOfWork,
334
- ) {
335
- this.log = logger ?? noopLogger;
336
- this.unitOfWork =
337
- unitOfWork ??
338
- new PassThroughUnitOfWork({
339
- message: messageService,
340
- envelope: envelopeService,
341
- address: addressService,
342
- threadMessage: threadMessageService,
343
- });
344
- this.filterPipeline = filterConfig
345
- ? new FilterPipeline(filterConfig, this.log)
346
- : undefined;
347
- }
348
-
349
- /**
350
- * Sync message bodies for a batch of messages.
351
- *
352
- * Fail-fast on connection errors: if the IMAP connection is lost,
353
- * immediately stop processing and return all remaining messages as failed
354
- * so they can be requeued for later retry.
355
- *
356
- * @param messageIds - The message IDs to sync bodies for
357
- * @param accountId - The account ID (for storage path)
358
- * @param accountConfigId - The account config ID (for thread updates)
359
- * @param folder - The IMAP mailbox path and its folder placement
360
- * @param getConnection - Function to get a (lazy) IMAP connection
361
- * @param force - Bypass the "already stored" skip guard and re-fetch every
362
- * message in this batch even though `bodyStorageKey` is already set. Set
363
- * only by the read-miss re-arm cue (a `/content` read found the storage
364
- * object missing despite the DB row saying otherwise); bulk metadata-sync
365
- * never sets this, so it keeps skipping on `bodyStorageKey` with no
366
- * existence check.
367
- */
368
- async syncBodies(
369
- messageIds: string[],
370
- accountId: string,
371
- accountConfigId: string,
372
- folder: BodySyncFolder,
373
- getConnection: ConnectionGetter,
374
- force = false,
375
- ): Promise<SyncBodiesResult> {
376
- const syncedMessageIds: string[] = [];
377
- let skippedCount = 0;
378
- const location = {
379
- accountId,
380
- accountConfigId,
381
- mailboxPath: folder.fullPath,
382
- };
383
-
384
- // Resolve every message up front so we can issue ONE ranged FETCH for the
385
- // whole batch (the desktop-client pattern) instead of a SELECT + download
386
- // per message. Messages whose body is already stored are skipped here and
387
- // never hit the wire — UNLESS `force` is set, in which case every message
388
- // is re-fetched regardless of `bodyStorageKey` (the read-miss cue already
389
- // confirmed the stored object is gone). `pending` maps each UID to its
390
- // messageId so we can match FETCH rows back and re-enqueue any UID the
391
- // server never returns.
392
- const pending = new Map<number, string>();
393
- // Message IDs in this batch whose row already carried `bodyStorageKey`
394
- // when `force` re-added them to `pending`: a forced re-fetch of an
395
- // already-stored body is a re-store (issue #1011) — bytes only, no
396
- // decision re-run. A message with no stored body inside a forced batch
397
- // is still a first store and takes the full decision pass.
398
- const reStoredMessageIds = new Set<string>();
399
-
400
- // One read per round, not per message (issue #72). The list is small by
401
- // design and almost always empty, so a lookup per message would put a
402
- // query on the hot path for a state that is nearly never set.
403
- const quarantined =
404
- await this.quarantineConfig?.quarantineService.load(accountConfigId);
405
-
406
- for (const messageId of messageIds) {
407
- const message = await this.messageService.get(messageId);
408
-
409
- // A quarantined message has no stored body, so nothing else here would
410
- // stop it being fetched and re-parsed on every redelivery, forever.
411
- if (
412
- this.quarantineConfig &&
413
- quarantined?.has(
414
- this.quarantineConfig.mailboxId,
415
- this.quarantineConfig.uidValidity,
416
- message.uid,
417
- )
418
- ) {
419
- this.log.debug?.(
420
- { messageId, uid: message.uid },
421
- "Message is quarantined, skipping",
422
- );
423
- skippedCount++;
424
- continue;
425
- }
426
-
427
- // The body is stored, so there is nothing to fetch (issue #331). This
428
- // used to re-read those bytes from storage to re-derive a category,
429
- // keyed on `category === uncategorized` — which is also the classifier's
430
- // own "nothing to say", so a message it declined cost one storage read
431
- // and two writes on every pass, forever, for the same answer. What the
432
- // classifier did is now recorded as `classificationState`, so a sync
433
- // pass never has to guess it from the row, and the cohort it has not
434
- // reached is selectable by a backfill instead of re-derived here.
435
- if (hasStoredBody(message.bodyStorageKey) && !force) {
436
- this.log.debug?.({ messageId }, "Body already stored, skipping");
437
- skippedCount++;
438
- continue;
439
- }
440
- if (message.bodyStorageKey) reStoredMessageIds.add(messageId);
441
- pending.set(message.uid, messageId);
442
- }
443
-
444
- if (pending.size === 0) {
445
- return this.buildResult(syncedMessageIds, skippedCount, []);
446
- }
447
-
448
- const connection = await getConnection();
449
- // Single SELECT for the whole batch. openBox is idempotent, so a warm
450
- // connection already on this mailbox skips the SELECT entirely.
451
- await connection.openBox(folder.fullPath);
452
-
453
- let connectionLost = false;
454
- try {
455
- for await (const { uid, source } of connection.fetchMessageBodies([
456
- ...pending.keys(),
457
- ])) {
458
- const messageId = pending.get(uid);
459
- if (!messageId) {
460
- // A UID we didn't ask for — drain the stream so the connection
461
- // stays usable, then ignore it.
462
- source.resume();
463
- continue;
464
- }
465
- try {
466
- await this.storeStreamedBody(
467
- messageId,
468
- accountId,
469
- accountConfigId,
470
- source,
471
- folder.placement,
472
- reStoredMessageIds.has(messageId),
473
- );
474
- } catch (error) {
475
- // A per-message store failure (e.g. the parsed-body S3 write) is a
476
- // real fault, but it must not silently mark the message synced —
477
- // that would leave parsedBody null and the message unindexed. Leave
478
- // the UID in `pending` so it lands in failedMessageIds and the SQS
479
- // batch requeues just this message. A dropped connection is handled
480
- // by the outer catch (fail-fast on the whole remaining batch).
481
- if (isConnectionDrop(error)) throw error;
482
-
483
- // The one error in this frame that is the message's own fault. It
484
- // reaches here from the parse call and nowhere else, so the S3, DDB
485
- // and SQS work this same frame wraps cannot be mistaken for it and
486
- // keeps requeueing below. The row is durable before the UID leaves
487
- // `pending`, so the message is never let go of unrecorded.
488
- if (
489
- error instanceof BodyParseError &&
490
- (await this.quarantineBodyParse(messageId, location, error))
491
- ) {
492
- pending.delete(uid);
493
- skippedCount++;
494
- source.resume();
495
- continue;
496
- }
497
-
498
- this.log.error?.(
499
- {
500
- messageId,
501
- errorName: (error as { name?: string }).name,
502
- error: (error as Error).message,
503
- },
504
- "Body store failed for message; leaving for requeue",
505
- );
506
- // Drain the stream so the connection stays usable for the next UID.
507
- source.resume();
508
- continue;
509
- }
510
- pending.delete(uid);
511
- syncedMessageIds.push(messageId);
512
- }
513
- } catch (error) {
514
- this.log.error?.(
515
- { error: (error as Error).message },
516
- "Body fetch stream failed",
517
- );
518
- // Fail-fast: a dropped connection mid-stream leaves every not-yet-yielded
519
- // UID in `pending`; they fall through to failedMessageIds and re-enqueue.
520
- // Any other error is a real fault — let it crash.
521
- if (!isConnectionDrop(error)) {
522
- throw error;
523
- }
524
- connectionLost = true;
525
- this.log.info?.(
526
- { remainingCount: pending.size },
527
- "Connection lost, aborting batch",
528
- );
529
- }
530
-
531
- // Anything still pending was never yielded (mid-stream drop or a UID the
532
- // server silently omitted) — re-enqueue it.
533
- const failedMessageIds = [...pending.values()];
534
-
535
- this.log.info(
536
- {
537
- synced: syncedMessageIds.length,
538
- skipped: skippedCount,
539
- failed: failedMessageIds.length,
540
- total: messageIds.length,
541
- aborted: connectionLost,
542
- },
543
- "Body sync complete",
544
- );
545
-
546
- return this.buildResult(syncedMessageIds, skippedCount, failedMessageIds);
547
- }
548
-
549
- /**
550
- * Set a message aside because its body will not parse (issue #72).
551
- *
552
- * Returns false when no record was written — either because this service was
553
- * built without a quarantine writer, or because writing it failed. The
554
- * caller then falls back to requeueing rather than dropping a message no
555
- * record survives. The failure the row carries is the parser's, and only the
556
- * parser's — see {@link BodyParseError}.
557
- *
558
- * Reading and writing the record is storage and database work, so its own
559
- * failure is contained per message the same way the save it replaces is: it
560
- * must not take down a batch that has other messages to fetch, and in the
561
- * stream loop a throw would also skip `source.resume()` and leave the
562
- * literal undrained on an otherwise usable connection.
563
- *
564
- * The fingerprint comes from the BodyPart rows metadata sync already wrote:
565
- * the body path streams raw bytes and has no FETCH result to read, and
566
- * re-reading headers the parser has just refused would be reading the very
567
- * thing that is broken.
568
- */
569
- private async quarantineBodyParse(
570
- messageId: string,
571
- location: {
572
- accountId: string;
573
- accountConfigId: string;
574
- mailboxPath: string;
575
- },
576
- error: BodyParseError,
577
- ): Promise<boolean> {
578
- const config = this.quarantineConfig;
579
- if (!config) return false;
580
-
581
- return this.recordQuarantine(config, messageId, location, error).then(
582
- () => true,
583
- (writeError: unknown) => {
584
- this.log.error?.(
585
- {
586
- messageId,
587
- errorName: (writeError as { name?: string }).name,
588
- error: inspect(writeError),
589
- },
590
- "Could not record quarantine; leaving the message for requeue",
591
- );
592
- return false;
593
- },
594
- );
595
- }
596
-
597
- private async recordQuarantine(
598
- config: QuarantineConfig,
599
- messageId: string,
600
- location: {
601
- accountId: string;
602
- accountConfigId: string;
603
- mailboxPath: string;
604
- },
605
- error: BodyParseError,
606
- ): Promise<void> {
607
- const message = await this.messageService.get(messageId);
608
- const messageData = await this.envelopeService.getMessageData(messageId);
609
-
610
- await config.quarantineService.record(
611
- {
612
- accountId: location.accountId,
613
- accountConfigId: location.accountConfigId,
614
- mailboxId: config.mailboxId,
615
- mailboxPath: location.mailboxPath,
616
- uidValidity: config.uidValidity,
617
- attempts: config.attempts,
618
- },
619
- message.uid,
620
- {
621
- stage: QuarantineFailureStage.BodyParse,
622
- code: error.failureCode,
623
- message: error.message,
624
- },
625
- shapeFromMessageData(message, messageData),
626
- );
627
- }
628
-
629
- private buildResult(
630
- syncedMessageIds: string[],
631
- skippedCount: number,
632
- failedMessageIds: string[],
633
- ): SyncBodiesResult {
634
- return {
635
- syncedCount: syncedMessageIds.length,
636
- syncedMessageIds,
637
- skippedCount,
638
- failedCount: failedMessageIds.length,
639
- failedMessageIds,
640
- };
641
- }
642
-
643
- /**
644
- * Stream one message body straight to storage while teeing the bytes into a
645
- * buffer for the parse-dependent steps (snippet, classification, parsed-body
646
- * cache, per-part objects). The S3 upload never sees a whole-body concat —
647
- * the storage service streams it — but mailparser still needs the full bytes,
648
- * so we collect them in parallel. Later issues move parsing off the hot path.
649
- *
650
- * `isReStore` marks a re-fetch of a message whose row already carried
651
- * `bodyStorageKey` (the `force` re-arm in {@link syncBodies}): the
652
- * post-store steps then re-materialize the body artifacts without
653
- * re-running any decision (issue #1011).
654
- */
655
- private async storeStreamedBody(
656
- messageId: string,
657
- accountId: string,
658
- accountConfigId: string,
659
- source: Readable,
660
- placement: FolderPlacement,
661
- isReStore: boolean,
662
- ): Promise<void> {
663
- const toStorage = new PassThrough();
664
- const chunks: Buffer[] = [];
665
-
666
- // Tee the source: bytes flow to storage as a stream (no whole-body concat
667
- // on the upload path) while we also collect them for the parse-dependent
668
- // steps below, which still need the full body for mailparser. The store
669
- // and the tee are awaited together so neither rejection is orphaned.
670
- const tee = new Promise<void>((resolve, reject) => {
671
- source.on("data", (chunk: Buffer) => {
672
- chunks.push(chunk);
673
- toStorage.write(chunk);
674
- });
675
- source.on("end", () => {
676
- toStorage.end();
677
- resolve();
678
- });
679
- source.on("error", (err) => {
680
- toStorage.destroy(err);
681
- reject(err);
682
- });
683
- });
684
-
685
- const [ref] = await Promise.all([
686
- this.storageService.storeMessageBodyStream({
687
- accountConfigId,
688
- accountId,
689
- messageId,
690
- content: toStorage,
691
- }),
692
- tee,
693
- ]);
694
-
695
- const body = Buffer.concat(chunks);
696
-
697
- await this.applyPostStoreSteps(
698
- messageId,
699
- accountId,
700
- accountConfigId,
701
- body,
702
- {
703
- uri: ref.uri,
704
- },
705
- placement,
706
- isReStore,
707
- );
708
- }
709
-
710
- /**
711
- * The steps every synced body must go through regardless of which path
712
- * fetched it first — the sync path ({@link storeStreamedBody}) and the
713
- * read-path backfill ({@link fetchAndGetBody}) both call this instead of
714
- * duplicating (and drifting on) their own subset. Before issue #1271,
715
- * `fetchAndGetBody` skipped classification/placement entirely — a message
716
- * materialized by a read got different treatment than one synced in bulk.
717
- *
718
- * Order matters for two independent reasons, both load-bearing:
719
- * 1. The parsed-body cache must be durable before `bodyStorageKey` (the
720
- * skip-guard signal) — see the comment on the write below.
721
- * 2. A placement move — local mailboxId change, pending marker, IMAP-push
722
- * enqueue (`PlacementMoveService.moveMessage`, issue #1271) — must
723
- * complete, or fully fail, BEFORE `bodyStorageKey` is written. That
724
- * failure is never swallowed: if it throws, `bodyStorageKey` is not yet
725
- * durable, so a retry reprocesses the message from scratch (marker
726
- * included) instead of the skip-guard stranding a `movedByRemit` flag
727
- * with no marker behind it — the defect this issue fixes.
728
- *
729
- * Returns the parsed mail so callers that already need it (both do) don't
730
- * pay for mailparser twice.
731
- */
732
- private async applyPostStoreSteps(
733
- messageId: string,
734
- accountId: string,
735
- accountConfigId: string,
736
- body: Buffer,
737
- bodyRef: { uri: string },
738
- placement: FolderPlacement,
739
- /**
740
- * Whether this call is a re-store of an already-classified message
741
- * rather than a first store. The two re-entrant paths (`force: true`
742
- * in `syncBodies`, and `fetchAndGetBody`'s `NoSuchKey` IMAP fallback) are
743
- * both body-only re-fetches: a body re-sync fetches and stores bytes and
744
- * runs no decision logic (issue #1011). When `isReStore` is true the
745
- * placement/filter/category machinery is skipped entirely and the
746
- * message's existing category is carried forward unchanged.
747
- */
748
- isReStore: boolean,
749
- ): Promise<ParsedMail> {
750
- // The one operation on this path that can fail because of how the message
751
- // is built. Its own try block lives in `parseMessageBody`, so this frame —
752
- // which also wraps the S3 body write, the parsed-body cache, the placement
753
- // move, the label writes and the counter update — can tell the two apart
754
- // and quarantine only this one (issue #72). The ThreadMessage write these
755
- // feed happens later, alongside the Message write (see below), once the
756
- // write-once category is decided.
757
- const parsed = await parseMessageBody(body);
758
-
759
- // Body-only re-fetch (issue #1011): the decision pass already ran when
760
- // this message's body first landed, so a re-store re-materializes only
761
- // the body-derived artifacts — parsed cache, body parts, the storage
762
- // key — and re-decides nothing: no classification, placement, filters,
763
- // labels, read state, and no second engagement-counter increment.
764
- // Skipping the decisions here is what lets the write-once fields stay
765
- // safe without per-derivation re-entrancy guards.
766
- if (isReStore) {
767
- // Same ordering rule as the first-store path below: the parsed-body
768
- // cache must be durable before `bodyStorageKey` — the skip-guard
769
- // signal — is written, so a failure here requeues the message and
770
- // the retry genuinely re-attempts the artifact writes.
771
- await this.storeParsedBodyCache(
772
- accountConfigId,
773
- accountId,
774
- messageId,
775
- parsed,
776
- );
777
- if (!isBodyPartDeferralEnabled()) {
778
- await this.storeBodyPartContents(
779
- accountConfigId,
780
- accountId,
781
- messageId,
782
- parsed,
783
- );
784
- }
785
- await this.messageService.update(messageId, {
786
- bodyStorageKey: bodyRef.uri,
787
- });
788
- this.log.info(
789
- { messageId, storageKey: bodyRef.uri },
790
- "Body re-stored without re-running decisions",
791
- );
792
- return parsed;
793
- }
794
-
795
- const snippet = extractSnippet(parsed);
796
- const listId = extractListId(parsed);
797
-
798
- // Compute the header classification once, folding in the sender's
799
- // `flags.category` override when one applies (issue #299). The derived
800
- // fields are folded into the single Message update below — they are NOT
801
- // written here.
802
- const classification = await this.classifyMessage(
803
- accountConfigId,
804
- parsed,
805
- placement,
806
- );
807
-
808
- // Decide the placement move from the in-memory classification before the
809
- // write, so its `movedByRemit` flag and audit verdict join the same
810
- // UpdateItem. The verdict is recorded whenever Remit decided to act
811
- // (action != leave); the move is enqueued only for a confident,
812
- // actionable verdict.
813
- const resolved = await this.resolvePlacement(
814
- messageId,
815
- accountId,
816
- accountConfigId,
817
- parsed,
818
- classification,
819
- );
820
-
821
- // Index-time filter pass (RFC 034). Isolated the same way placement is —
822
- // FilterPipeline.evaluate swallows any read/decision failure and returns
823
- // no actions, so a schema-drifted filter query never fails the body store.
824
- // The DECISION is computed here; the applies below are unswallowed.
825
- const filterDecision = await this.evaluateFilters(
826
- accountConfigId,
827
- messageId,
828
- parsed,
829
- );
830
-
831
- // Store the parsed-body cache BEFORE persisting bodyStorageKey. The skip
832
- // guard in syncBodies treats a stored bodyStorageKey as "fully synced",
833
- // so if we wrote it first and the parsed-cache write then failed, the
834
- // requeued retry would skip the message — leaving parsedBody null and the
835
- // search-index upsert bodyless forever. A parsed-cache failure here throws
836
- // and propagates before bodyStorageKey is written, so the requeued message
837
- // genuinely re-attempts the parsed write and gets indexed.
838
- await this.storeParsedBodyCache(
839
- accountConfigId,
840
- accountId,
841
- messageId,
842
- parsed,
843
- );
844
-
845
- // Per-part S3 objects dominate S3 writes during bulk sync. When deferral
846
- // is enabled we skip them here; they are materialized lazily on the first
847
- // `contentUrl` read (see ensureBodyPartsStored), keeping bulk sync at 2
848
- // writes/message (body.eml + parsed.json.gz).
849
- if (!isBodyPartDeferralEnabled()) {
850
- await this.storeBodyPartContents(
851
- accountConfigId,
852
- accountId,
853
- messageId,
854
- parsed,
855
- );
856
- }
857
-
858
- // Local-first move (issue #1271): local mailboxId change + pending marker +
859
- // IMAP-push enqueue via PlacementMoveService, UNSWALLOWED — a throw here
860
- // propagates before bodyStorageKey is written (see the method doc above),
861
- // so a genuine SQS/DDB write failure requeues the message rather than being
862
- // absorbed. A matched filter's move is exclusive and outranks the
863
- // classifier's placement move (RFC 034 Decision 3.1) — an explicit user
864
- // rule wins the single mailbox a message occupies — so at most one move is
865
- // enqueued. A non-confident/leave verdict with no filter move is a no-op.
866
- let filterMove: MessageFilterMove | undefined;
867
- let filterMoved = false;
868
- if (filterDecision.move) {
869
- const destinationMailboxId = filterDecision.move.destinationMailboxId;
870
- // Capture the source mailbox BEFORE the move rewrites mailboxId, so the
871
- // auto-moved badge can name where the filter took the message from and
872
- // offer an undo back to it (issue #223). Snapshotted into the marker
873
- // rather than read from `originalMailboxId` at derive time, which a
874
- // later user move would overwrite — the filter's source must survive an
875
- // undo to stay derivable.
876
- const sourceMailboxId = (await this.messageService.get(messageId))
877
- .mailboxId;
878
- await this.filterConfig?.placementMoveService.moveMessage(
879
- accountConfigId,
880
- messageId,
881
- destinationMailboxId,
882
- accountId,
883
- );
884
- // A filter matches by content, not location: a message delivered
885
- // straight into the destination (a server-side rule) or already filed
886
- // there matches but was not moved by Remit. `moveMessage` no-ops in that
887
- // case; record neither the marker nor `movedByRemit`, so the badge never
888
- // claims a move that did not happen and its Undo is never a no-op.
889
- if (sourceMailboxId !== destinationMailboxId) {
890
- filterMoved = true;
891
- filterMove = {
892
- filterId: filterDecision.move.filterId,
893
- sourceMailboxId,
894
- destinationMailboxId,
895
- decidedAt: Date.now(),
896
- };
897
- }
898
- } else if (resolved.move) {
899
- await this.placementConfig?.placementMoveService.moveMessage(
900
- accountConfigId,
901
- messageId,
902
- resolved.move.destinationMailboxId,
903
- accountId,
904
- );
905
- }
906
-
907
- // Additive label actions (RFC 034 Decision 3.1): every matching filter's
908
- // label applies. The deterministic MessageLabel upsert is idempotent, so a
909
- // requeue after a later failure re-applies safely; UNSWALLOWED, so a DDB
910
- // write failure propagates before bodyStorageKey is written.
911
- for (const label of filterDecision.labels) {
912
- await this.filterConfig?.messageLabelService.apply({
913
- messageId,
914
- labelId: label.labelId,
915
- accountConfigId,
916
- appliedByFilterId: label.filterId,
917
- });
918
- }
919
-
920
- // Read once for the two write-once guards below — the auto-read decision
921
- // and the category carry-forward. Taken here rather than earlier because
922
- // nothing between this point and the Message update writes either field;
923
- // the moves above touch `mailboxId` only.
924
- const existingMessage = await this.messageService.get(messageId);
925
-
926
- // `flags.unsubscribed` (issue #302, RFC 039 Decision 3): auto-mark-read,
927
- // reusing the same FlagQueueService.markAsRead round-trip a manual
928
- // mark-as-read goes through — idempotent on a retry (flipFlag no-ops when
929
- // the message is already \Seen), so a failure here safely re-fires on the
930
- // next attempt rather than being lost behind the bodyStorageKey skip guard.
931
- await this.applyUnsubscribedAutoRead(
932
- messageId,
933
- accountId,
934
- accountConfigId,
935
- parsed,
936
- existingMessage.bodyStorageKey,
937
- );
938
-
939
- await this.deriveCalendarSuggestions(
940
- messageId,
941
- accountConfigId,
942
- parsed,
943
- existingMessage.bodyStorageKey,
944
- );
945
-
946
- const moved = Boolean(resolved.move || filterMoved);
947
-
948
- // ONE Message UpdateItem per synced message: bodyStorageKey + every
949
- // classification/derived field + the move flag + the audit verdict. Each
950
- // extra Message mutation emits a DDB stream record that fans out to a
951
- // redundant S3-Vectors upsert, so we collapse them into a single write.
952
- // The flag is folded in only when a move WAS applied (by the classifier or
953
- // a filter); the verdict is folded in whenever Remit decided to act.
954
- // Written LAST so bodyStorageKey — the skip-guard signal — is only durable
955
- // once the parsed cache AND the move (when any) are.
956
- //
957
- // `category` is RFC 034 D3.1's write-once field (RFC 030's message-list
958
- // GSI sort key depends on it never churning). The re-entrant re-store
959
- // paths (issue #1011) return before this write; the carry-forward keeps
960
- // the field write-once even when a first store lands on a row that
961
- // already carries a real category — a previously-decided category wins
962
- // over the just-recomputed one. It likewise protects a `flags.category`
963
- // override (issue #299): recomputing would let a *later* override silently
964
- // rewrite a category already decided, which is exactly the churn RFC
965
- // 030's GSI-safety argument forbids.
966
- //
967
- // `resolved.placementDecidedAt` (issue #383) is only ever present on
968
- // the pass that first decided it: the re-store paths skip placement
969
- // entirely (issue #1011), so `computePlacement` is reached only on a
970
- // first store.
971
- const finalCategory = hasDecidedCategory(existingMessage.category)
972
- ? existingMessage.category
973
- : classification.category;
974
-
975
- // Thread-list denormalization, using the same write-once category as the
976
- // Message update below — the two rows must never disagree on category.
977
- await denormalizeMessageCategory(
978
- { threadMessageService: this.threadMessageService },
979
- accountConfigId,
980
- messageId,
981
- { category: finalCategory, snippet, listId },
982
- );
983
-
984
- // `classificationState` records that the classifier ran, in the same
985
- // UpdateItem as the answer it gave (issue #331). Without it `uncategorized`
986
- // carries two meanings at once — not reached yet, and reached with nothing
987
- // to say — and only one of them should ever be looked at again.
988
- const update: UpdateMessageInput = {
989
- bodyStorageKey: bodyRef.uri,
990
- ...classification,
991
- category: finalCategory,
992
- classificationState: MessageClassificationState.Examined,
993
- ...(moved ? { movedByRemit: true } : {}),
994
- ...(resolved.verdict ? { placementVerdict: resolved.verdict } : {}),
995
- ...(filterMove ? { filterMove } : {}),
996
- ...(resolved.placementDecidedAt
997
- ? { placementDecidedAt: resolved.placementDecidedAt }
998
- : {}),
999
- };
1000
- await this.messageService.update(messageId, update);
1001
- this.log.info({ messageId, storageKey: bodyRef.uri }, "Body stored");
1002
-
1003
- // From-Address engagement counter (Address entity, not Message).
1004
- await this.incrementInboundCount(
1005
- messageId,
1006
- accountConfigId,
1007
- parsed,
1008
- classification,
1009
- );
1010
-
1011
- return parsed;
1012
- }
1013
-
1014
- /**
1015
- * Fetch a single message body, store it, and return the parsed content.
1016
- *
1017
- * If the body is already stored, retrieves it from storage.
1018
- * Otherwise fetches from IMAP, stores it, and returns the parsed content.
1019
- *
1020
- * @param messageId - The message ID to fetch
1021
- * @param accountId - The account ID (for storage path)
1022
- * @param accountConfigId - The account config ID (for thread updates)
1023
- * @param folder - The IMAP mailbox path and its folder placement
1024
- * @param getConnection - Function to get a (lazy) IMAP connection
1025
- * @returns Parsed text and HTML content
1026
- */
1027
- async fetchAndGetBody(
1028
- messageId: string,
1029
- accountId: string,
1030
- accountConfigId: string,
1031
- folder: BodySyncFolder,
1032
- getConnection: ConnectionGetter,
1033
- ): Promise<FetchBodyResult> {
1034
- const message = await this.messageService.get(messageId);
1035
-
1036
- let body: Buffer;
1037
- let needsStore = false;
1038
-
1039
- if (message.bodyStorageKey) {
1040
- // Body already stored, try to retrieve from storage
1041
- this.log.debug?.({ messageId }, "Retrieving body from storage");
1042
- try {
1043
- body = await this.storageService.retrieve(message.bodyStorageKey);
1044
- } catch (err) {
1045
- // Only a genuinely-missing object (NoSuchKey) is a safe IMAP
1046
- // fallback — that's the cross-environment / never-stored case.
1047
- // A permission/infra error (AccessDenied, throttle) must NOT be
1048
- // masked as a missing object: let it crash so it's observable.
1049
- if (!isStorageNotFoundError(err)) {
1050
- this.log.error?.(
1051
- {
1052
- messageId,
1053
- storageKey: message.bodyStorageKey,
1054
- errorName: (err as { name?: string }).name,
1055
- errorCode: (err as { Code?: string }).Code,
1056
- error: inspect(err),
1057
- },
1058
- "Body storage retrieval failed (non-NoSuchKey); not falling back to IMAP",
1059
- );
1060
- throw err;
1061
- }
1062
- this.log.debug?.(
1063
- { messageId, error: (err as Error).message },
1064
- "Body object missing (NoSuchKey), falling back to IMAP",
1065
- );
1066
- needsStore = true;
1067
- body = await this.fetchFromImap(
1068
- message,
1069
- accountId,
1070
- accountConfigId,
1071
- folder.fullPath,
1072
- getConnection,
1073
- );
1074
- }
1075
- } else {
1076
- // Fetch from IMAP and store
1077
- needsStore = true;
1078
- body = await this.fetchFromImap(
1079
- message,
1080
- accountId,
1081
- accountConfigId,
1082
- folder.fullPath,
1083
- getConnection,
1084
- );
1085
- }
1086
-
1087
- let parsed: ParsedMail;
1088
-
1089
- if (needsStore) {
1090
- const ref = await this.storageService.storeMessageBody({
1091
- accountConfigId,
1092
- accountId,
1093
- messageId,
1094
- content: body,
1095
- });
1096
-
1097
- // Same shared step the sync path (storeStreamedBody) runs — issue
1098
- // #1271 unified the two body paths so classification/placement no
1099
- // longer depends on which one fetched the body first. A store from
1100
- // here is a re-store exactly when the row already claimed a stored
1101
- // body — the NoSuchKey fallback above; a row with no bodyStorageKey is
1102
- // a genuine first store and takes the full decision pass.
1103
- parsed = await this.applyPostStoreSteps(
1104
- messageId,
1105
- accountId,
1106
- accountConfigId,
1107
- body,
1108
- { uri: ref.uri },
1109
- folder.placement,
1110
- message.bodyStorageKey != null,
1111
- );
1112
- } else {
1113
- parsed = await simpleParser(body);
1114
- }
1115
-
1116
- return {
1117
- text: parsed.text ?? null,
1118
- html: typeof parsed.html === "string" ? parsed.html : null,
1119
- storedAt: message.bodyStorageKey ?? "newly-stored",
1120
- };
1121
- }
1122
-
1123
- private async fetchFromImap(
1124
- message: MessageItem,
1125
- accountId: string,
1126
- accountConfigId: string,
1127
- mailboxPath: string,
1128
- getConnection: ConnectionGetter,
1129
- ): Promise<Buffer> {
1130
- const { uid } = message;
1131
- this.log.debug?.({ uid }, "Fetching body from IMAP");
1132
- const connection = await getConnection();
1133
- await connection.openBox(mailboxPath);
1134
- const body = await connection.fetchMessageBody(uid);
1135
- if (body) return body;
1136
-
1137
- if (!(await isMessageGoneFromOpenMailbox(connection, uid))) {
1138
- throw new Error(
1139
- `IMAP FETCH returned no source for UID ${uid} in ${mailboxPath}, which the server still lists`,
1140
- );
1141
- }
1142
-
1143
- return this.reconcileGoneMessage(
1144
- message,
1145
- accountId,
1146
- accountConfigId,
1147
- mailboxPath,
1148
- );
1149
- }
1150
-
1151
- /**
1152
- * The server no longer holds the message at the row's UID. A row with our
1153
- * own move or delete in flight belongs to that mutation (409). A settled row
1154
- * is removed with its thread entries, but only while its placement still
1155
- * matches what was read (imap-mutations R3); losing that predicate is a 409
1156
- * like any other lost placement transition.
1157
- */
1158
- private async reconcileGoneMessage(
1159
- message: MessageItem,
1160
- accountId: string,
1161
- accountConfigId: string,
1162
- mailboxPath: string,
1163
- ): Promise<never> {
1164
- const { messageId, uid } = message;
1165
- if (isPlacementUnsettled(message)) {
1166
- throw new MessagePlacementUnsettledError(
1167
- `Message ${messageId} is not at its UID while a move or delete is in flight`,
1168
- accountId,
1169
- messageId,
1170
- "in_flight",
1171
- );
1172
- }
1173
-
1174
- const reconciled = await this.unitOfWork.transaction(async (repos) => {
1175
- const deleted = await repos.message.deleteIfPlacement(messageId, {
1176
- status: message.status,
1177
- mailboxId: message.mailboxId,
1178
- uid,
1179
- });
1180
- if (!deleted) return false;
1181
- const threadMessages = await repos.threadMessage.findAllByMessageId(
1182
- accountConfigId,
1183
- messageId,
1184
- );
1185
- if (threadMessages.length > 0) {
1186
- await repos.threadMessage.deleteMany(
1187
- threadMessages.map((row) => ({
1188
- accountConfigId: row.accountConfigId,
1189
- threadMessageId: row.threadMessageId,
1190
- })),
1191
- );
1192
- }
1193
- return true;
1194
- });
1195
-
1196
- if (!reconciled) {
1197
- throw new MessagePlacementUnsettledError(
1198
- `Message ${messageId} changed placement while its absence from ${mailboxPath} was being reconciled`,
1199
- accountId,
1200
- messageId,
1201
- "in_flight",
1202
- );
1203
- }
1204
-
1205
- this.log.info(
1206
- {
1207
- metric: "message_gone_stale_row_reconciled",
1208
- messageId,
1209
- uid,
1210
- mailboxPath,
1211
- },
1212
- "Message no longer on the mail server (expunged or moved by another client); stale row reconciled",
1213
- );
1214
- throw new MessageGoneFromServerError(messageId);
1215
- }
1216
-
1217
- /**
1218
- * Header classification, with the sender's `Address.flags.category`
1219
- * override (issue #299, RFC 039 Decision 3) substituted for the
1220
- * header-derived category when one is set. Shared with the classification
1221
- * backfill (issue #1197) as {@link classifyParsedMessage}, so the one-time
1222
- * pass and the body-store path can never classify the same bytes
1223
- * differently.
1224
- */
1225
- private async classifyMessage(
1226
- accountConfigId: string,
1227
- parsed: ParsedMail,
1228
- placement: FolderPlacement,
1229
- ): Promise<UpdateMessageInput & { category: ThreadMessageCategory }> {
1230
- return classifyParsedMessage(
1231
- this.addressService,
1232
- accountConfigId,
1233
- parsed,
1234
- placement,
1235
- );
1236
- }
1237
-
1238
- private async incrementInboundCount(
1239
- messageId: string,
1240
- accountConfigId: string,
1241
- parsed: ParsedMail,
1242
- classification: UpdateMessageInput,
1243
- ): Promise<void> {
1244
- const fromEmail = extractPrimaryFromEmail(parsed);
1245
- if (!fromEmail) {
1246
- this.log.debug?.(
1247
- { messageId },
1248
- "No From address; skipping inbound counter",
1249
- );
1250
- return;
1251
- }
1252
-
1253
- const addressId = deriveAddressId(accountConfigId, fromEmail);
1254
- const bulk = isBulkSender(
1255
- classification.category,
1256
- classification.hasListUnsubscribe ?? false,
1257
- );
1258
- await this.addressService.incrementInboundCount(
1259
- accountConfigId,
1260
- addressId,
1261
- Date.now(),
1262
- bulk,
1263
- );
1264
- }
1265
-
1266
- /**
1267
- * The per-sender signals {@link computePlacement} needs, from ONE `Address`
1268
- * fetch (RFC 039 Decision 3/3a, issue #300): the trust reads exactly as
1269
- * `deriveSenderTrust` always did — `vip → wellknown → unknown`, untouched by
1270
- * `blocked` — plus `blocked`/`autoArchive` off the same row. `trustSetAt` is
1271
- * the `setAt` of whichever flag produced the trust value, needed by
1272
- * {@link resolveSenderPlacement}'s tie-break; it stays local to placement and
1273
- * never reaches `deriveSenderTrust`'s own contract (the trust badge).
1274
- */
1275
- private async deriveSenderPlacementSignals(
1276
- accountConfigId: string,
1277
- fromEmail: string,
1278
- ): Promise<{
1279
- trust: (typeof SenderTrust)[keyof typeof SenderTrust];
1280
- trustSetAt?: number;
1281
- blocked: boolean;
1282
- blockedSetAt?: number;
1283
- neverSpam: boolean;
1284
- autoArchive: boolean;
1285
- }> {
1286
- const unknown = {
1287
- trust: SenderTrust.Unknown,
1288
- blocked: false,
1289
- neverSpam: false,
1290
- autoArchive: false,
1291
- } as const;
1292
- try {
1293
- const addressId = deriveAddressId(accountConfigId, fromEmail);
1294
- const address = await this.addressService.getAddress(
1295
- accountConfigId,
1296
- addressId,
1297
- );
1298
- const flags = address.flags;
1299
- if (flags?.vip?.value === true) {
1300
- return {
1301
- trust: SenderTrust.Vip,
1302
- trustSetAt: flags.vip.setAt,
1303
- blocked: flags.blocked?.value === true,
1304
- blockedSetAt: flags.blocked?.setAt,
1305
- neverSpam: flags.neverSpam?.value === true,
1306
- autoArchive: flags.autoArchive?.value === true,
1307
- };
1308
- }
1309
- if (flags?.wellknown?.value === true) {
1310
- return {
1311
- trust: SenderTrust.Wellknown,
1312
- trustSetAt: flags.wellknown.setAt,
1313
- blocked: flags.blocked?.value === true,
1314
- blockedSetAt: flags.blocked?.setAt,
1315
- neverSpam: flags.neverSpam?.value === true,
1316
- autoArchive: flags.autoArchive?.value === true,
1317
- };
1318
- }
1319
- return {
1320
- ...unknown,
1321
- blocked: flags?.blocked?.value === true,
1322
- blockedSetAt: flags?.blocked?.setAt,
1323
- neverSpam: flags?.neverSpam?.value === true,
1324
- autoArchive: flags?.autoArchive?.value === true,
1325
- };
1326
- } catch (err) {
1327
- // A genuinely-absent address means "no signals". Any other failure
1328
- // (AccessDenied, throttle, infra) must NOT be silently downgraded — let
1329
- // it crash so the placement decision isn't made on bad data.
1330
- if (!(err instanceof NotFoundError)) throw err;
1331
- }
1332
- return unknown;
1333
- }
1334
-
1335
- /**
1336
- * `flags.unsubscribed` (issue #302, RFC 039 Decision 3): "auto-mark-read
1337
- * until sender stops" — fires on every new message from that sender for as
1338
- * long as the flag stays set, per the flag's own doc comment; there is no
1339
- * separate expiry mechanism, and no caching of the decision beyond this
1340
- * per-message `Address` read. A no-op when body sync was built without an
1341
- * {@link UnsubscribeConfig} or the message carries no `From` address.
1342
- *
1343
- * Write-once per message (issue #499), like the `category` and placement
1344
- * derivations alongside it: read state is applied on the pass that first
1345
- * classifies a message and never re-decided. A user who marks such a message
1346
- * unread afterwards owns it — a re-entrant pass over an already-classified
1347
- * body would otherwise silently undo that.
1348
- */
1349
- private async applyUnsubscribedAutoRead(
1350
- messageId: string,
1351
- accountId: string,
1352
- accountConfigId: string,
1353
- parsed: ParsedMail,
1354
- storedBodyKey: string | undefined,
1355
- ): Promise<void> {
1356
- if (!this.unsubscribeConfig) return;
1357
- if (hasStoredBody(storedBodyKey)) return;
1358
-
1359
- const fromEmail = extractPrimaryFromEmail(parsed);
1360
- if (!fromEmail) return;
1361
-
1362
- const unsubscribed = await this.deriveSenderUnsubscribed(
1363
- accountConfigId,
1364
- fromEmail,
1365
- );
1366
- if (!unsubscribed) return;
1367
-
1368
- await this.unsubscribeConfig.flagQueueService.markAsRead(
1369
- accountConfigId,
1370
- messageId,
1371
- accountId,
1372
- );
1373
- }
1374
-
1375
- /**
1376
- * Offer what a message's `text/calendar` parts propose, as cards beside the
1377
- * message (issue #1033). Nothing here reaches a calendar: a suggestion is
1378
- * written `Pending` and waits for a person, and a `METHOD:CANCEL` is a card
1379
- * of its own rather than a withdrawal of an event the user added.
1380
- *
1381
- * Write-once per message, like the `category`, placement and read-state
1382
- * derivations alongside it (issues #499, #1011): a card is offered on the
1383
- * pass that first classifies a message — the same moment the account's
1384
- * filters run — and never again. The two re-entrant paths a forced body
1385
- * re-sync goes through (`fetchAndGetBody`'s `NoSuchKey` fallback,
1386
- * `syncBodies(..., force: true)`) reach this with `bodyStorageKey` already
1387
- * set, and re-offering there would resurrect a card the user dismissed.
1388
- *
1389
- * Isolated the same way placement and filters are: a card is auxiliary to
1390
- * storing the mail, which is already durable by the time this runs. A
1391
- * repository failure here is logged loudly with the messageId rather than
1392
- * failing a body store that otherwise succeeded.
1393
- */
1394
- private async deriveCalendarSuggestions(
1395
- messageId: string,
1396
- accountConfigId: string,
1397
- parsed: ParsedMail,
1398
- storedBodyKey: string | undefined,
1399
- ): Promise<void> {
1400
- if (!this.calendarConfig) return;
1401
- if (hasStoredBody(storedBodyKey)) return;
1402
-
1403
- const parts = calendarParts(parsed);
1404
- if (parts.length === 0) return;
1405
-
1406
- const { calendarUnitOfWork, calendarSuggestionService, filterService } =
1407
- this.calendarConfig;
1408
-
1409
- await Promise.resolve()
1410
- .then(async () => {
1411
- // `dismiss{muteSender:true}` wrote a standing rule naming this
1412
- // sender. Honouring it here is what makes that button do anything:
1413
- // the index-time filter pipeline skips a rule with no label and no
1414
- // move, so this is the only reader such a rule has.
1415
- const sender = extractPrimaryFromEmail(parsed);
1416
- if (sender) {
1417
- const active = await filterService.listByAccountAndState(
1418
- accountConfigId,
1419
- FilterState.Active,
1420
- );
1421
- if (isSenderMuted(active, sender)) {
1422
- this.log.debug?.(
1423
- { messageId, sender },
1424
- "Sender is muted; offering no calendar suggestion",
1425
- );
1426
- return;
1427
- }
1428
- }
1429
-
1430
- // Provisioned rather than looked up, through the one function every
1431
- // caller provisions with. A missing collection has no timezone, and
1432
- // a floating DTSTART read in UTC puts the meeting hours from where
1433
- // the organizer meant it. Idempotent: the id is derived, so a second
1434
- // first use returns the collection the first one made.
1435
- const collection = await provisionDefaultCalendar(
1436
- calendarUnitOfWork,
1437
- accountConfigId,
1438
- );
1439
- for (const part of parts) {
1440
- const recorded = await recordCalendarSuggestion(
1441
- calendarSuggestionService,
1442
- {
1443
- accountConfigId,
1444
- messageId,
1445
- bodyPartId: deriveBodyPartId(messageId, part.partPath),
1446
- source: part.source,
1447
- icalData: part.icalData,
1448
- timezone: collection.timezone,
1449
- },
1450
- );
1451
- if (!recorded.ok) {
1452
- this.log.info(
1453
- {
1454
- messageId,
1455
- partPath: part.partPath,
1456
- code: recorded.error.code,
1457
- reason: recorded.error.message,
1458
- },
1459
- "Message carries iCalendar this cannot read; no suggestion offered",
1460
- );
1461
- }
1462
- }
1463
- })
1464
- .catch((error: unknown) => {
1465
- this.log.error?.(
1466
- { messageId, accountConfigId, error: inspect(error) },
1467
- "Calendar suggestion derivation failed; body is stored without a card",
1468
- );
1469
- });
1470
- }
1471
-
1472
- private async deriveSenderUnsubscribed(
1473
- accountConfigId: string,
1474
- fromEmail: string,
1475
- ): Promise<boolean> {
1476
- try {
1477
- const addressId = deriveAddressId(accountConfigId, fromEmail);
1478
- const address = await this.addressService.getAddress(
1479
- accountConfigId,
1480
- addressId,
1481
- );
1482
- return address.flags?.unsubscribed?.value === true;
1483
- } catch (err) {
1484
- // A genuinely-absent address means "not unsubscribed". Any other
1485
- // failure (AccessDenied, throttle, infra) must NOT be silently
1486
- // downgraded — let it crash so the read-state decision isn't made on
1487
- // bad data.
1488
- if (!(err instanceof NotFoundError)) throw err;
1489
- return false;
1490
- }
1491
- }
1492
-
1493
- /**
1494
- * Evaluate the account's active filters against a synced message (RFC 034),
1495
- * BEFORE the single Message update — so a filter's `movedByRemit` flag joins
1496
- * that same UpdateItem. Returns the actions to apply (labels + at most one
1497
- * move); the caller applies them. A no-op returning no actions when no
1498
- * {@link FilterConfig} is wired.
1499
- *
1500
- * Isolation lives in {@link FilterPipeline.evaluate}: this read/decision phase
1501
- * never fails the body store (the #1246 placement precedent). The applies of
1502
- * the returned decision are the caller's and are deliberately unswallowed.
1503
- */
1504
- private async evaluateFilters(
1505
- accountConfigId: string,
1506
- messageId: string,
1507
- parsed: ParsedMail,
1508
- ): Promise<FilterDecision> {
1509
- if (!this.filterPipeline) return { labels: [] };
1510
- return this.filterPipeline.evaluate(
1511
- accountConfigId,
1512
- messageId,
1513
- toFilterMessage(parsed),
1514
- );
1515
- }
1516
-
1517
- /**
1518
- * Resolve the placement decision for a message, BEFORE the single Message
1519
- * update — so both the `movedByRemit` flag and the audit verdict join that
1520
- * one UpdateItem instead of a second mutation that would fan out another
1521
- * redundant S3-Vectors upsert. Drives both directions (junk → inbox rescue,
1522
- * inbox → junk demote) off the pure {@link classifyPlacement} verdict.
1523
- *
1524
- * Returns a {@link PlacementOutcome}: a `verdict` to persist whenever Remit
1525
- * decided to act (action != leave), confident and unsure alike, so the
1526
- * distribution is queryable on the message; a `move` to enqueue only for a
1527
- * confident verdict; and `placementDecidedAt` whenever a verdict — including
1528
- * "leave" — was genuinely computed (issue #383), so a re-entrant call short-
1529
- * circuits instead of re-deciding a placement the user may have since
1530
- * overridden by hand.
1531
- *
1532
- * Always logs a structured verdict line for confident, actionable verdicts so
1533
- * the real distribution is observable on a live mailbox.
1534
- *
1535
- * The verdict reads the just-computed classification rather than a persisted
1536
- * row, since those fields are not written until the single update. Placement
1537
- * is auxiliary to the body store — the primary artifact (body.eml) is already
1538
- * durable by the time this runs — so a failure here (e.g. a placement-repo
1539
- * query erroring on a schema-drifted DB) is caught and logged loudly with an
1540
- * alertable field instead of failing the surrounding message store. The empty
1541
- * {@link PlacementOutcome} means "no action", whether Remit genuinely decided
1542
- * to leave the message alone or placement itself failed; the alert log is
1543
- * what distinguishes the latter, and neither case marks the message decided.
1544
- */
1545
- private async resolvePlacement(
1546
- messageId: string,
1547
- accountId: string,
1548
- accountConfigId: string,
1549
- parsed: ParsedMail,
1550
- classification: UpdateMessageInput,
1551
- ): Promise<PlacementOutcome> {
1552
- if (!this.placementConfig) return {};
1553
- return this.computePlacement(
1554
- this.placementConfig,
1555
- messageId,
1556
- accountId,
1557
- accountConfigId,
1558
- parsed,
1559
- classification,
1560
- ).catch((error: unknown) => {
1561
- this.log.error?.(
1562
- {
1563
- alert: "body_sync_placement_failed",
1564
- messageId,
1565
- accountId,
1566
- accountConfigId,
1567
- errorName: (error as { name?: string })?.name,
1568
- error: inspect(error),
1569
- },
1570
- "Placement resolution failed; body already stored, continuing without placement (best-effort, non-fatal)",
1571
- );
1572
- return {};
1573
- });
1574
- }
1575
-
1576
- private async computePlacement(
1577
- placementConfig: PlacementConfig,
1578
- messageId: string,
1579
- accountId: string,
1580
- accountConfigId: string,
1581
- parsed: ParsedMail,
1582
- classification: UpdateMessageInput,
1583
- ): Promise<PlacementOutcome> {
1584
- const { mailboxSpecialUseService } = placementConfig;
1585
-
1586
- const message = await this.messageService.get(messageId);
1587
-
1588
- // Issue #383 / #1011: placement still runs once per message (RFC 039
1589
- // Non-goals), but the once-per-message guarantee now comes from the
1590
- // caller rather than a guard here — the re-store paths
1591
- // (`fetchAndGetBody`'s `NoSuchKey` fallback, `syncBodies(..., force:
1592
- // true)`) skip the decision steps entirely, so only the pass that first
1593
- // stores a body ever reaches this method.
1594
-
1595
- const roleMailboxes = await resolveFolderRoleMailboxes(
1596
- mailboxSpecialUseService,
1597
- accountId,
1598
- );
1599
- const { junkMailbox, inboxMailbox } = roleMailboxes;
1600
- const placement = placementOf(message.mailboxId, roleMailboxes);
1601
-
1602
- const fromEmail = extractPrimaryFromEmail(parsed);
1603
- const signals = fromEmail
1604
- ? await this.deriveSenderPlacementSignals(accountConfigId, fromEmail)
1605
- : {
1606
- trust: SenderTrust.Unknown,
1607
- blocked: false,
1608
- neverSpam: false,
1609
- autoArchive: false,
1610
- };
1611
-
1612
- const { senderTrust, senderOverride } = resolveSenderPlacement(
1613
- { trust: signals.trust, setAt: signals.trustSetAt },
1614
- { blocked: signals.blocked, setAt: signals.blockedSetAt },
1615
- signals.neverSpam,
1616
- );
1617
-
1618
- // The verdict needs the classification signals (providerSpam,
1619
- // authResult, authenticity) that this body-sync pass just derived; the
1620
- // stored row does not carry them yet, so overlay them onto the message.
1621
- const candidate = { ...message, ...classification };
1622
- const verdict = classifyPlacement(
1623
- candidate,
1624
- placement,
1625
- senderTrust,
1626
- senderOverride,
1627
- );
1628
-
1629
- // A `leave` verdict — including "nothing confident to say" — carries no
1630
- // audit record of its own. `flags.autoArchive` (issue #300) is a distinct,
1631
- // lower-priority filing preference: it only files a message away when
1632
- // `blocked`/DKIM/DMARC had nothing to say, never overriding a confident
1633
- // junk/inbox verdict computed above. Still marked decided (issue #383):
1634
- // "leave" is itself a verdict, not "not yet evaluated".
1635
- if (verdict.action === "leave") {
1636
- const outcome = autoArchiveApplies(verdict, placement)
1637
- ? await this.resolveAutoArchive(
1638
- mailboxSpecialUseService,
1639
- message,
1640
- accountId,
1641
- signals.autoArchive,
1642
- )
1643
- : {};
1644
- return { ...outcome, placementDecidedAt: Date.now() };
1645
- }
1646
-
1647
- // Audit verdict — recorded for every actionable verdict (both
1648
- // confidences), so the distribution is queryable on the message.
1649
- // Independent of whether a move is enqueued.
1650
- const audit: MessagePlacementVerdict = {
1651
- action:
1652
- verdict.action === "move-to-inbox"
1653
- ? PlacementAction.MoveToInbox
1654
- : PlacementAction.MoveToJunk,
1655
- confidence:
1656
- verdict.confidence === "confident"
1657
- ? PlacementConfidence.Confident
1658
- : PlacementConfidence.Unsure,
1659
- fromPlacement: placement,
1660
- reasons: verdict.reasons,
1661
- dryRun: false,
1662
- decidedAt: Date.now(),
1663
- };
1664
-
1665
- // Only a confident verdict moves mail. An unsure verdict is recorded
1666
- // but never enqueues a move.
1667
- if (verdict.confidence !== "confident") {
1668
- return { verdict: audit, placementDecidedAt: audit.decidedAt };
1669
- }
1670
-
1671
- const target =
1672
- verdict.action === "move-to-inbox" ? inboxMailbox : junkMailbox;
1673
- if (!target) return { verdict: audit, placementDecidedAt: audit.decidedAt };
1674
-
1675
- // Structured verdict line — emitted for confident, actionable verdicts
1676
- // so the real verdict distribution is observable on a live mailbox.
1677
- this.log.info(
1678
- {
1679
- messageId,
1680
- accountId,
1681
- placement,
1682
- action: verdict.action,
1683
- confidence: verdict.confidence,
1684
- reasons: verdict.reasons,
1685
- destinationMailboxId: target.mailboxId,
1686
- },
1687
- "Placement verdict",
1688
- );
1689
-
1690
- return {
1691
- verdict: audit,
1692
- move: {
1693
- destinationMailboxId: target.mailboxId,
1694
- destinationPath: target.fullPath,
1695
- },
1696
- placementDecidedAt: audit.decidedAt,
1697
- };
1698
- }
1699
-
1700
- /**
1701
- * `flags.autoArchive` (issue #300, RFC 039 Decision 3): file a message
1702
- * straight to Archive, skipping Inbox. Only reached from
1703
- * {@link computePlacement} when {@link classifyPlacement} had no confident
1704
- * junk/inbox verdict of its own — `blocked`/DKIM/DMARC always take priority
1705
- * over this filing preference — and only for the `leave` verdicts
1706
- * {@link autoArchiveApplies} admits.
1707
- *
1708
- * No {@link MessagePlacementVerdict} is recorded: `PlacementAction` (the
1709
- * audit enum) has only `MoveToInbox`/`MoveToJunk` — issue #300 is scoped to
1710
- * no TypeSpec change, so an archive move carries no audit verdict, same as
1711
- * a matched filter's move. The move itself reuses the same
1712
- * `placementMoveService.moveMessage` path as every other confident move.
1713
- *
1714
- * Idempotent the same way {@link classifyPlacement}'s own branches are: a
1715
- * message already sitting in Archive is left alone, not moved again.
1716
- */
1717
- private async resolveAutoArchive(
1718
- mailboxSpecialUseService: IMailboxSpecialUseRepository,
1719
- message: MessageItem,
1720
- accountId: string,
1721
- autoArchive: boolean,
1722
- ): Promise<PlacementOutcome> {
1723
- if (!autoArchive) return {};
1724
-
1725
- const archiveMailbox =
1726
- await mailboxSpecialUseService.findArchiveMailbox(accountId);
1727
- if (!archiveMailbox || message.mailboxId === archiveMailbox.mailboxId) {
1728
- return {};
1729
- }
1730
-
1731
- this.log.info(
1732
- {
1733
- messageId: message.messageId,
1734
- accountId,
1735
- destinationMailboxId: archiveMailbox.mailboxId,
1736
- },
1737
- "Auto-archive verdict",
1738
- );
1739
-
1740
- return {
1741
- move: {
1742
- destinationMailboxId: archiveMailbox.mailboxId,
1743
- destinationPath: archiveMailbox.fullPath,
1744
- },
1745
- };
1746
- }
1747
-
1748
- // The old `enqueuePlacementMove` (best-effort, catch-and-log) lived here.
1749
- // Issue #1271: it ran AFTER `bodyStorageKey` was already durable, so a
1750
- // failure was swallowed to avoid stranding the message behind the
1751
- // body-sync skip guard — but that meant `movedByRemit` could be true with
1752
- // no record the move ever reached IMAP. Replaced by the unswallowed call
1753
- // inside `applyPostStoreSteps`, sequenced BEFORE `bodyStorageKey` is
1754
- // written, backed by `PlacementMoveService`'s pending-move marker.
1755
-
1756
- /**
1757
- * Persist one S3 object per non-multipart leaf so the SPA can resolve
1758
- * `BodyPartResponse.contentUrl` (#298). Keys follow the layout
1759
- * `accounts/{accountConfigId}/{accountId}/messages/{messageId}/parts/{partPath}`
1760
- * so they line up with the URL shape `derive/contentUrl.ts` emits.
1761
- *
1762
- * The mapper is total (#395 PR B): every leaf gets a `BodyPartContentPair`,
1763
- * possibly with a zero-byte content for leaves that have no source bytes
1764
- * (genuinely empty parts, or pathological inputs the positional fallback
1765
- * couldn't pair). No try/catch is needed; the only failure surface here is
1766
- * an S3 write itself, which `pMap` surfaces directly.
1767
- *
1768
- * If `listBodyParts` returns an empty list (e.g. a legacy message synced
1769
- * before #133 populated BodyPart rows), this is a no-op.
1770
- */
1771
- private async storeBodyPartContents(
1772
- accountConfigId: string,
1773
- accountId: string,
1774
- messageId: string,
1775
- parsed: ParsedMail,
1776
- options?: { skipExisting?: boolean },
1777
- ): Promise<StoreBodyPartContentsResult> {
1778
- const bodyParts = await this.envelopeService.listBodyParts(messageId);
1779
- if (bodyParts.length === 0) {
1780
- this.log.debug?.(
1781
- { messageId },
1782
- "No BodyPart rows; skipping per-part storage",
1783
- );
1784
- return { stored: 0 };
1785
- }
1786
-
1787
- const log = this.log;
1788
- const pairs = mapBodyPartsToContent(bodyParts, parsed, {
1789
- messageId,
1790
- logger:
1791
- log.warn || log.debug
1792
- ? {
1793
- warn: (obj, msg) => {
1794
- log.warn?.(obj, msg);
1795
- },
1796
- debug: (obj, msg) => {
1797
- log.debug?.(obj, msg);
1798
- },
1799
- }
1800
- : undefined,
1801
- });
1802
-
1803
- if (pairs.length === 0) {
1804
- return { stored: 0 };
1805
- }
1806
-
1807
- const bodyPartIdByPath = new Map(
1808
- bodyParts.map((bp) => [bp.partPath, bp.bodyPartId]),
1809
- );
1810
-
1811
- let stored = 0;
1812
- const textContents: { bodyPartId: string; content: string }[] = [];
1813
- await pMap(
1814
- pairs,
1815
- async (entry) => {
1816
- if (
1817
- options?.skipExisting &&
1818
- (await this.storageService.bodyPartExists(
1819
- accountConfigId,
1820
- accountId,
1821
- messageId,
1822
- entry.partPath,
1823
- ))
1824
- ) {
1825
- return;
1826
- }
1827
- await this.storageService.storeBodyPart({
1828
- accountConfigId,
1829
- accountId,
1830
- messageId,
1831
- partPath: entry.partPath,
1832
- content: entry.content,
1833
- contentType: entry.contentType,
1834
- });
1835
- stored++;
1836
-
1837
- if (
1838
- entry.contentType.toLowerCase().startsWith("text/") &&
1839
- entry.content.length > 0
1840
- ) {
1841
- const bodyPartId = bodyPartIdByPath.get(entry.partPath);
1842
- if (bodyPartId) {
1843
- textContents.push({
1844
- bodyPartId,
1845
- content: entry.content.toString("utf8"),
1846
- });
1847
- }
1848
- }
1849
- },
1850
- { concurrency: BODY_PART_STORE_CONCURRENCY },
1851
- );
1852
-
1853
- if (textContents.length > 0) {
1854
- await this.envelopeService.upsertBodyPartContents(
1855
- messageId,
1856
- textContents,
1857
- );
1858
- }
1859
-
1860
- this.log.info(
1861
- { messageId, partCount: pairs.length, stored },
1862
- "Body parts stored",
1863
- );
1864
-
1865
- return { stored };
1866
- }
1867
-
1868
- /**
1869
- * Lazily materialize the per-part S3 objects for an already-synced message
1870
- * whose parts were deferred during bulk sync (DEFER_BODY_PARTS).
1871
- *
1872
- * Called from the read path (the API `describeMessage` handler) before the
1873
- * SPA fetches any `contentUrl`: the part bytes are served directly from S3 by
1874
- * CloudFront with no Lambda in the request path, so a missing object would
1875
- * surface to the SPA as a hard "body-missing" failure. Generating the parts
1876
- * here — re-parsing the stored `body.eml` with the same `mapBodyPartsToContent`
1877
- * logic body-sync uses — guarantees every `contentUrl` resolves, while bulk
1878
- * sync stays at 2 writes/message.
1879
- *
1880
- * Idempotent: skips any leaf already on S3, so repeat reads cost only HEAD
1881
- * checks, and a re-read after a partial failure fills the gaps. Requires the
1882
- * message body to be stored (`bodyStorageKey`); callers gate on that.
1883
- *
1884
- * Warm-open fast path: a single HEAD on the `.materialized` sentinel
1885
- * short-circuits the whole pass — no body.eml GET, no MIME re-parse, no
1886
- * per-leaf HEADs. The sentinel is written only after every leaf is confirmed
1887
- * stored, so its presence guarantees all `contentUrl`s resolve.
1888
- */
1889
- async ensureBodyPartsStored(
1890
- accountConfigId: string,
1891
- accountId: string,
1892
- messageId: string,
1893
- bodyStorageKey: string,
1894
- ): Promise<StoreBodyPartContentsResult> {
1895
- if (
1896
- await this.storageService.bodyPartExists(
1897
- accountConfigId,
1898
- accountId,
1899
- messageId,
1900
- MATERIALIZED_SENTINEL_PART_PATH,
1901
- )
1902
- ) {
1903
- this.log.debug?.(
1904
- { messageId },
1905
- "Body parts already materialized (sentinel hit); skipping",
1906
- );
1907
- return { stored: 0 };
1908
- }
1909
-
1910
- const body = await this.storageService.retrieve(bodyStorageKey);
1911
- const parsed = await simpleParser(body);
1912
- const result = await this.storeBodyPartContents(
1913
- accountConfigId,
1914
- accountId,
1915
- messageId,
1916
- parsed,
1917
- { skipExisting: true },
1918
- );
1919
-
1920
- // Reaching here means storeBodyPartContents resolved — `pMap` rejects on
1921
- // any single failed write, so every leaf is now durably on S3. Drop the
1922
- // sentinel so subsequent opens take the HEAD-only fast path above.
1923
- await this.storageService.storeBodyPart({
1924
- accountConfigId,
1925
- accountId,
1926
- messageId,
1927
- partPath: MATERIALIZED_SENTINEL_PART_PATH,
1928
- content: Buffer.alloc(0),
1929
- });
1930
-
1931
- return result;
1932
- }
1933
-
1934
- /**
1935
- * Persist the pre-parsed body cache. A failure here MUST fail the
1936
- * surrounding body-sync: the parsed-body object is what the search-index
1937
- * pipeline reads, so swallowing a write failure leaves the message marked
1938
- * synced while `parsedBody` is null and it never gets indexed — a silent
1939
- * search gap. Propagate so the message lands in failedMessageIds and the
1940
- * SQS batch requeues it.
1941
- */
1942
- private async storeParsedBodyCache(
1943
- accountConfigId: string,
1944
- accountId: string,
1945
- messageId: string,
1946
- parsed: ParsedMail,
1947
- ): Promise<void> {
1948
- const parsedBody = toParsedBody(parsed);
1949
- try {
1950
- await this.storageService.storeParsedBody({
1951
- accountConfigId,
1952
- accountId,
1953
- messageId,
1954
- parsed: parsedBody,
1955
- });
1956
- this.log.debug?.({ messageId }, "Parsed body cache stored");
1957
- } catch (err: unknown) {
1958
- this.log.error?.(
1959
- {
1960
- messageId,
1961
- errorName: (err as { name?: string }).name,
1962
- errorCode: (err as { Code?: string }).Code,
1963
- error: inspect(err),
1964
- },
1965
- "Failed to store parsed body cache; failing sync to requeue",
1966
- );
1967
- throw err;
1968
- }
1969
- }
1970
- }