@remit/mailbox-service 0.0.112 → 0.0.113

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (270) hide show
  1. package/dist/account-credentials.d.ts +83 -0
  2. package/dist/account-credentials.js +104 -0
  3. package/dist/attribute-mapper.d.ts +36 -0
  4. package/dist/attribute-mapper.js +119 -0
  5. package/dist/body-parse.d.ts +28 -0
  6. package/{src/body-parse.ts → dist/body-parse.js} +18 -26
  7. package/dist/body-part-mapper.d.ts +92 -0
  8. package/dist/body-part-mapper.js +301 -0
  9. package/dist/body-sync-queue.d.ts +59 -0
  10. package/dist/body-sync-queue.js +56 -0
  11. package/dist/body-sync-terminal.d.ts +58 -0
  12. package/dist/body-sync-terminal.js +95 -0
  13. package/dist/body-sync.d.ts +351 -0
  14. package/dist/body-sync.js +1204 -0
  15. package/dist/calendar-parts.d.ts +23 -0
  16. package/dist/calendar-parts.js +18 -0
  17. package/dist/classification-backfill.d.ts +91 -0
  18. package/dist/classification-backfill.js +180 -0
  19. package/dist/classify-message.d.ts +31 -0
  20. package/dist/classify-message.js +89 -0
  21. package/dist/connection-factory.d.ts +54 -0
  22. package/dist/connection-factory.js +54 -0
  23. package/dist/connection-test.d.ts +41 -0
  24. package/dist/connection-test.js +152 -0
  25. package/dist/external-move.d.ts +94 -0
  26. package/dist/external-move.js +173 -0
  27. package/dist/filter-operations.d.ts +42 -0
  28. package/dist/filter-operations.js +211 -0
  29. package/dist/filters/filter-config.d.ts +14 -0
  30. package/dist/filters/filter-config.js +10 -0
  31. package/{src/filters/list-id.ts → dist/filters/list-id.d.ts} +2 -16
  32. package/dist/filters/list-id.js +26 -0
  33. package/dist/filters/match.d.ts +69 -0
  34. package/dist/filters/match.js +165 -0
  35. package/dist/filters/pipeline.d.ts +64 -0
  36. package/dist/filters/pipeline.js +100 -0
  37. package/{src/flag-push-terminal.ts → dist/flag-push-terminal.d.ts} +16 -72
  38. package/dist/flag-push-terminal.js +63 -0
  39. package/dist/flag-push.d.ts +69 -0
  40. package/dist/flag-push.js +111 -0
  41. package/dist/flag-queue.d.ts +175 -0
  42. package/dist/flag-queue.js +291 -0
  43. package/dist/folder-operations.d.ts +39 -0
  44. package/dist/folder-operations.js +112 -0
  45. package/dist/folder-placement.d.ts +13 -0
  46. package/dist/folder-placement.js +23 -0
  47. package/dist/heuristics/authenticationResults.d.ts +10 -0
  48. package/dist/heuristics/authenticationResults.js +187 -0
  49. package/dist/heuristics/classifyByHeaders.d.ts +82 -0
  50. package/dist/heuristics/classifyByHeaders.js +365 -0
  51. package/dist/heuristics/classifyPlacement.d.ts +67 -0
  52. package/dist/heuristics/classifyPlacement.js +153 -0
  53. package/dist/heuristics/machineSenders.d.ts +21 -0
  54. package/{src/heuristics/machineSenders.ts → dist/heuristics/machineSenders.js} +34 -40
  55. package/dist/heuristics/senderMismatch.d.ts +81 -0
  56. package/dist/heuristics/senderMismatch.js +192 -0
  57. package/dist/heuristics/socialDomains.d.ts +14 -0
  58. package/{src/heuristics/socialDomains.ts → dist/heuristics/socialDomains.js} +9 -9
  59. package/dist/heuristics/transactionalDomains.d.ts +18 -0
  60. package/{src/heuristics/transactionalDomains.ts → dist/heuristics/transactionalDomains.js} +8 -8
  61. package/dist/imapflow-connection.d.ts +377 -0
  62. package/dist/imapflow-connection.js +1147 -0
  63. package/dist/index.d.ts +63 -0
  64. package/dist/index.js +65 -0
  65. package/dist/list-id-backfill.d.ts +83 -0
  66. package/dist/list-id-backfill.js +132 -0
  67. package/dist/mailbox-cursor-rebuild.d.ts +74 -0
  68. package/dist/mailbox-cursor-rebuild.js +71 -0
  69. package/{src/mailbox-cursor.ts → dist/mailbox-cursor.d.ts} +20 -98
  70. package/dist/mailbox-cursor.js +102 -0
  71. package/dist/mailbox-intent.d.ts +41 -0
  72. package/{src/mailbox-intent.ts → dist/mailbox-intent.js} +38 -64
  73. package/dist/mailbox-management.d.ts +240 -0
  74. package/dist/mailbox-management.js +576 -0
  75. package/dist/mailbox-presence.d.ts +52 -0
  76. package/{src/mailbox-presence.ts → dist/mailbox-presence.js} +19 -30
  77. package/dist/mailbox-queue.d.ts +126 -0
  78. package/dist/mailbox-queue.js +214 -0
  79. package/dist/mailbox-sync.d.ts +93 -0
  80. package/dist/mailbox-sync.js +490 -0
  81. package/dist/mailbox-upstream.d.ts +38 -0
  82. package/{src/mailbox-upstream.ts → dist/mailbox-upstream.js} +37 -47
  83. package/dist/message-category.d.ts +51 -0
  84. package/{src/message-category.ts → dist/message-category.js} +28 -71
  85. package/dist/message-move.d.ts +285 -0
  86. package/dist/message-move.js +858 -0
  87. package/dist/message-parser.d.ts +14 -0
  88. package/dist/message-parser.js +17 -0
  89. package/{src/message-presence.ts → dist/message-presence.d.ts} +1 -13
  90. package/dist/message-presence.js +29 -0
  91. package/dist/message-read.d.ts +51 -0
  92. package/dist/message-read.js +31 -0
  93. package/dist/message-sync.d.ts +345 -0
  94. package/dist/message-sync.js +1358 -0
  95. package/dist/mime-walker.d.ts +102 -0
  96. package/dist/mime-walker.js +198 -0
  97. package/dist/organize-job.d.ts +20 -0
  98. package/dist/organize-job.js +37 -0
  99. package/dist/outbox-attachment-content.d.ts +37 -0
  100. package/dist/outbox-attachment-content.js +50 -0
  101. package/dist/outbox-attachment-filename.d.ts +19 -0
  102. package/dist/outbox-attachment-filename.js +64 -0
  103. package/dist/outbox-attachment.d.ts +148 -0
  104. package/dist/outbox-attachment.js +273 -0
  105. package/dist/outbox-queue.d.ts +73 -0
  106. package/dist/outbox-queue.js +284 -0
  107. package/{src/outbox-status.ts → dist/outbox-status.d.ts} +1 -6
  108. package/dist/outbox-status.js +18 -0
  109. package/dist/ownership.d.ts +24 -0
  110. package/dist/ownership.js +83 -0
  111. package/dist/placement-move-terminal.d.ts +67 -0
  112. package/dist/placement-move-terminal.js +88 -0
  113. package/dist/placement-move.d.ts +83 -0
  114. package/dist/placement-move.js +216 -0
  115. package/{src/placement-settled.ts → dist/placement-settled.d.ts} +13 -68
  116. package/dist/placement-settled.js +75 -0
  117. package/dist/quarantine.d.ts +95 -0
  118. package/dist/quarantine.js +131 -0
  119. package/dist/refusal.d.ts +31 -0
  120. package/dist/refusal.js +3 -0
  121. package/{src/restore-source-placement.ts → dist/restore-source-placement.d.ts} +29 -97
  122. package/dist/restore-source-placement.js +79 -0
  123. package/dist/search-text-backfill.d.ts +15 -0
  124. package/dist/search-text-backfill.js +33 -0
  125. package/dist/snippet.d.ts +61 -0
  126. package/dist/snippet.js +129 -0
  127. package/dist/spam-report.d.ts +86 -0
  128. package/dist/spam-report.js +241 -0
  129. package/dist/sync-warning.d.ts +21 -0
  130. package/dist/sync-warning.js +26 -0
  131. package/dist/sync-watermarks.d.ts +138 -0
  132. package/dist/sync-watermarks.js +205 -0
  133. package/dist/test-helpers/fake-imap.d.ts +151 -0
  134. package/dist/test-helpers/fake-imap.js +508 -0
  135. package/dist/test-helpers/folder-roles.d.ts +34 -0
  136. package/dist/test-helpers/folder-roles.js +47 -0
  137. package/dist/test-helpers/mime-tree-generator.d.ts +82 -0
  138. package/dist/test-helpers/mime-tree-generator.js +329 -0
  139. package/dist/test-helpers/test-store.d.ts +69 -0
  140. package/dist/test-helpers/test-store.js +193 -0
  141. package/dist/text/normalizer.d.ts +20 -0
  142. package/dist/text/normalizer.js +102 -0
  143. package/dist/thread-message-rows.d.ts +69 -0
  144. package/dist/thread-message-rows.js +40 -0
  145. package/dist/thread-root.d.ts +7 -0
  146. package/dist/thread-root.js +25 -0
  147. package/dist/types.d.ts +343 -0
  148. package/dist/types.js +19 -0
  149. package/package.json +86 -88
  150. package/src/account-credentials.ts +0 -210
  151. package/src/attribute-mapper.test.ts +0 -104
  152. package/src/attribute-mapper.ts +0 -148
  153. package/src/body-parse.test.ts +0 -55
  154. package/src/body-part-mapper.property.test.ts +0 -354
  155. package/src/body-part-mapper.ts +0 -410
  156. package/src/body-sync-queue.test.ts +0 -64
  157. package/src/body-sync-queue.ts +0 -131
  158. package/src/body-sync-terminal.test.ts +0 -148
  159. package/src/body-sync-terminal.ts +0 -174
  160. package/src/body-sync.test.ts +0 -925
  161. package/src/body-sync.ts +0 -1891
  162. package/src/calendar-parts.ts +0 -41
  163. package/src/classification-backfill.test.ts +0 -285
  164. package/src/classification-backfill.ts +0 -363
  165. package/src/classify-message.test.ts +0 -142
  166. package/src/classify-message.ts +0 -116
  167. package/src/connection-factory.ts +0 -102
  168. package/src/connection-test.ts +0 -208
  169. package/src/credentials.test.ts +0 -181
  170. package/src/external-move.test.ts +0 -262
  171. package/src/external-move.ts +0 -285
  172. package/src/filter-operations.test.ts +0 -319
  173. package/src/filter-operations.ts +0 -377
  174. package/src/filter-update.test.ts +0 -313
  175. package/src/filters/filter-config.test.ts +0 -87
  176. package/src/filters/filter-config.ts +0 -34
  177. package/src/filters/list-id.test.ts +0 -50
  178. package/src/filters/match.test.ts +0 -392
  179. package/src/filters/match.ts +0 -225
  180. package/src/filters/pipeline.test.ts +0 -199
  181. package/src/filters/pipeline.ts +0 -170
  182. package/src/flag-push.test.ts +0 -112
  183. package/src/flag-push.ts +0 -198
  184. package/src/flag-queue.test.ts +0 -197
  185. package/src/flag-queue.ts +0 -509
  186. package/src/folder-operations.test.ts +0 -273
  187. package/src/folder-operations.ts +0 -217
  188. package/src/folder-placement.ts +0 -56
  189. package/src/forwarded-message.test.ts +0 -122
  190. package/src/heuristics/authenticationResults.test.ts +0 -189
  191. package/src/heuristics/authenticationResults.ts +0 -212
  192. package/src/heuristics/classifyByHeaders.test.ts +0 -889
  193. package/src/heuristics/classifyByHeaders.ts +0 -467
  194. package/src/heuristics/classifyPlacement.test.ts +0 -366
  195. package/src/heuristics/classifyPlacement.ts +0 -211
  196. package/src/heuristics/classifyRealMail.test.ts +0 -242
  197. package/src/heuristics/machineSenders.test.ts +0 -101
  198. package/src/heuristics/senderMismatch.test.ts +0 -346
  199. package/src/heuristics/senderMismatch.ts +0 -259
  200. package/src/imapflow-connect-retry.test.ts +0 -97
  201. package/src/imapflow-connection.test.ts +0 -503
  202. package/src/imapflow-connection.ts +0 -1476
  203. package/src/index.ts +0 -397
  204. package/src/list-id-backfill.test.ts +0 -207
  205. package/src/list-id-backfill.ts +0 -274
  206. package/src/mailbox-cursor-rebuild.test.ts +0 -171
  207. package/src/mailbox-cursor-rebuild.ts +0 -146
  208. package/src/mailbox-cursor.test.ts +0 -141
  209. package/src/mailbox-delete-intent.test.ts +0 -119
  210. package/src/mailbox-management.test.ts +0 -243
  211. package/src/mailbox-management.ts +0 -838
  212. package/src/mailbox-presence.test.ts +0 -74
  213. package/src/mailbox-queue.ts +0 -397
  214. package/src/mailbox-rename-intent.test.ts +0 -220
  215. package/src/mailbox-rename-subtree.test.ts +0 -287
  216. package/src/mailbox-sync.test.ts +0 -286
  217. package/src/mailbox-sync.ts +0 -724
  218. package/src/message-move.test.ts +0 -726
  219. package/src/message-move.ts +0 -1518
  220. package/src/message-parser.ts +0 -26
  221. package/src/message-presence.test.ts +0 -50
  222. package/src/message-read.ts +0 -87
  223. package/src/message-save-retry.test.ts +0 -90
  224. package/src/message-sync-flags.test.ts +0 -167
  225. package/src/message-sync.test.ts +0 -762
  226. package/src/message-sync.ts +0 -2133
  227. package/src/mime-walker.test.ts +0 -311
  228. package/src/mime-walker.ts +0 -306
  229. package/src/organize-job.ts +0 -93
  230. package/src/outbox-attachment-content.test.ts +0 -153
  231. package/src/outbox-attachment-content.ts +0 -104
  232. package/src/outbox-attachment-filename.test.ts +0 -132
  233. package/src/outbox-attachment-filename.ts +0 -76
  234. package/src/outbox-attachment.test.ts +0 -626
  235. package/src/outbox-attachment.ts +0 -536
  236. package/src/outbox-failed-recovery.test.ts +0 -299
  237. package/src/outbox-queue.test.ts +0 -315
  238. package/src/outbox-queue.ts +0 -525
  239. package/src/ownership.ts +0 -140
  240. package/src/placement-move-terminal.ts +0 -145
  241. package/src/placement-move.ts +0 -363
  242. package/src/placement-settled.test.ts +0 -147
  243. package/src/placement.test.ts +0 -404
  244. package/src/quarantine.test.ts +0 -133
  245. package/src/quarantine.ts +0 -229
  246. package/src/refusal.ts +0 -29
  247. package/src/snippet.test.ts +0 -213
  248. package/src/snippet.ts +0 -152
  249. package/src/spam-report.test.ts +0 -329
  250. package/src/spam-report.ts +0 -357
  251. package/src/sync-warning.test.ts +0 -213
  252. package/src/sync-warning.ts +0 -56
  253. package/src/sync-watermarks.test.ts +0 -409
  254. package/src/sync-watermarks.ts +0 -319
  255. package/src/system-flag-wire-format.test.ts +0 -77
  256. package/src/test-helpers/fake-imap.ts +0 -690
  257. package/src/test-helpers/folder-roles.ts +0 -66
  258. package/src/test-helpers/mime-tree-generator.ts +0 -461
  259. package/src/test-helpers/test-queue.ts +0 -56
  260. package/src/test-helpers/test-store.ts +0 -299
  261. package/src/text/normalizer.test.ts +0 -223
  262. package/src/text/normalizer.ts +0 -140
  263. package/src/thread-message-rows.ts +0 -77
  264. package/src/thread-root.test.ts +0 -29
  265. package/src/thread-root.ts +0 -51
  266. package/src/types/mailparser-augment.d.ts +0 -23
  267. package/src/types/natural-porter-stemmer.d.ts +0 -50
  268. package/src/types/stopword.d.ts +0 -69
  269. package/src/types.ts +0 -371
  270. package/tsconfig.json +0 -8
package/src/body-sync.ts DELETED
@@ -1,1891 +0,0 @@
1
- import { PassThrough, type Readable } from "node:stream";
2
- import { inspect } from "node:util";
3
- import {
4
- provisionDefaultCalendar,
5
- recordCalendarSuggestion,
6
- } from "@remit/calendar-service";
7
- import {
8
- type BodyPartItem,
9
- type IAddressRepository,
10
- type ICalendarStore,
11
- type ICalendarSuggestionRepository,
12
- type IEnvelopeRepository,
13
- type IFilterRepository,
14
- type IMailboxSpecialUseRepository,
15
- type IMessageRepository,
16
- type IThreadMessageRepository,
17
- isSenderMuted,
18
- type MessageItem,
19
- type ThreadMessageItem,
20
- type UpdateMessageInput,
21
- } from "@remit/data-ports";
22
- import {
23
- MessageGoneFromServerError,
24
- MessagePlacementUnsettledError,
25
- NotFoundError,
26
- } from "@remit/data-ports/errors";
27
- import { deriveAddressId, deriveBodyPartId } from "@remit/data-ports/id";
28
- import { isEncapsulatedMessage } from "@remit/data-ports/message-parts";
29
- import { isBulkSender } from "@remit/data-ports/wellknown";
30
- import {
31
- FilterState,
32
- MessageClassificationState,
33
- PlacementAction,
34
- PlacementConfidence,
35
- QuarantineFailureStage,
36
- SenderTrust,
37
- SyncWarningStage,
38
- } from "@remit/domain-enums";
39
- import {
40
- isStorageNotFoundError,
41
- type ParsedBody,
42
- type StorageService,
43
- } from "@remit/storage-service";
44
- import { type ParsedMail, simpleParser } from "mailparser";
45
- import pMap from "p-map";
46
- import { BodyParseError, parseMessageBody } from "./body-parse.js";
47
- import {
48
- mapMessagePartsToContent,
49
- parsePartSource,
50
- } from "./body-part-mapper.js";
51
- import { calendarParts } from "./calendar-parts.js";
52
- import {
53
- classifyParsedMessage,
54
- extractPrimaryFromEmail,
55
- } from "./classify-message.js";
56
-
57
- import { extractListId } from "./filters/list-id.js";
58
- import type { FilterMessage } from "./filters/match.js";
59
- import {
60
- type FilterConfig,
61
- type FilterDecision,
62
- FilterPipeline,
63
- } from "./filters/pipeline.js";
64
- import type { FlagQueueService } from "./flag-queue.js";
65
- import { placementOf, resolveFolderRoleMailboxes } from "./folder-placement.js";
66
- import {
67
- classifyPlacement,
68
- type FolderPlacement,
69
- type PlacementVerdict,
70
- resolveSenderPlacement,
71
- } from "./heuristics/classifyPlacement.js";
72
- import {
73
- denormalizeMessageCategory,
74
- hasDecidedCategory,
75
- } from "./message-category.js";
76
- import { isMessageGoneFromOpenMailbox } from "./message-presence.js";
77
- import type { PlacementMoveService } from "./placement-move.js";
78
- import { isPlacementUnsettled, uidMailboxIdOf } from "./placement-settled.js";
79
- import { type QuarantineService, shapeFromMessageData } from "./quarantine.js";
80
- import { extractSnippetFromEmail } from "./snippet.js";
81
- import type { SyncWarningService } from "./sync-warning.js";
82
- import { type IImapConnection, MailConnectionError } from "./types.js";
83
-
84
- const BODY_PART_STORE_CONCURRENCY = 4;
85
-
86
- type MessagePlacementVerdict = NonNullable<
87
- UpdateMessageInput["placementVerdict"]
88
- >;
89
-
90
- type MessageFilterMove = NonNullable<UpdateMessageInput["filterMove"]>;
91
-
92
- type ThreadMessageCategory = ThreadMessageItem["category"];
93
-
94
- /**
95
- * Outcome of {@link BodySyncService.resolvePlacement}: the audit `verdict` to
96
- * persist on the Message (present whenever Remit decided to act, i.e. action
97
- * != leave), and the IMAP `move` to enqueue (present only for a confident,
98
- * actionable verdict). An unsure verdict carries a `verdict` but no `move`.
99
- */
100
- interface PlacementOutcome {
101
- verdict?: MessagePlacementVerdict;
102
- move?: { destinationMailboxId: string; destinationPath: string };
103
- placementDecidedAt?: number;
104
- }
105
-
106
- /**
107
- * Whether the body-sync hot path defers per-part S3 objects. Each MIME leaf is
108
- * its own PutObject, so during bulk sync these per-part writes dominate S3 write
109
- * count and cost. With deferral ON a synced message writes only `body.eml` +
110
- * `parsed.json.gz` (2 writes); the per-part objects are materialized lazily on
111
- * the first `contentUrl` read via {@link BodySyncService.ensureBodyPartsStored}.
112
- *
113
- * Default ON (cost savings). Set `DEFER_BODY_PARTS=false`/`0`/`off` to restore
114
- * the original eager per-part writes — a safe, reversible escape hatch.
115
- */
116
- export const isBodyPartDeferralEnabled = (): boolean => {
117
- const raw = process.env.DEFER_BODY_PARTS;
118
- if (raw === undefined) return true;
119
- const normalized = raw.trim().toLowerCase();
120
- return normalized !== "false" && normalized !== "0" && normalized !== "off";
121
- };
122
-
123
- /**
124
- * Sentinel object written under a message's parts prefix once lazy
125
- * materialization has stored EVERY leaf. The read path checks it first with a
126
- * single HEAD and early-returns, skipping the body.eml GET, the MIME re-parse,
127
- * and the per-leaf HEADs on every warm re-open.
128
- *
129
- * Deliberately an S3-only marker (no Message-row flag). A leading dot keeps it
130
- * out of any real IMAP section path (those are dot-separated digits).
131
- */
132
- const MATERIALIZED_SENTINEL_PART_PATH = ".materialized";
133
-
134
- /**
135
- * The sentinel for a message carrying a forwarded message. Those parts were
136
- * once stored under a layout in which the forwarded message's root overwrote
137
- * its own row, so a `.materialized` written then vouches for objects at the
138
- * wrong paths. Its own name makes every such message materialize once more,
139
- * overwriting what is there; no other message is touched.
140
- */
141
- const MATERIALIZED_SENTINEL_V2_PART_PATH = ".materialized.2";
142
-
143
- /**
144
- * A mid-stream socket drop during the ranged body fetch surfaces as a typed
145
- * `MailConnectionError` (the connection layer classifies imapflow's
146
- * `EConnectionClosed`/`NoConnection`). Detect by type/code, never by message
147
- * text — the underlying library strings vary.
148
- */
149
- const isConnectionDrop = (error: unknown): boolean => {
150
- if (error instanceof MailConnectionError) {
151
- return error.kind === "network";
152
- }
153
- const code = (error as { code?: string }).code;
154
- return code === "EConnectionClosed" || code === "NoConnection";
155
- };
156
-
157
- /**
158
- * Project a parsed message onto the fields a filter matches against (RFC 034) —
159
- * the literal-clause targets. Kept
160
- * separate from `ParsedMail` so {@link FilterPipeline} stays parser-agnostic.
161
- */
162
- const toFilterMessage = (parsed: ParsedMail): FilterMessage => ({
163
- from: extractPrimaryFromEmail(parsed) ?? "",
164
- fromName: parsed.from?.value?.[0]?.name ?? "",
165
- subject: parsed.subject ?? "",
166
- text: parsed.text ?? "",
167
- listId: extractListId(parsed),
168
- });
169
-
170
- const SNIPPET_LENGTH = 256;
171
-
172
- /**
173
- * The snippet the list row shows, from whichever body part carries text.
174
- * Exported for the classification backfill (issue #1197), which denormalizes
175
- * the same fields the body-store path does, from the same bytes.
176
- */
177
- export const extractSnippet = (parsed: ParsedMail): string =>
178
- extractSnippetFromEmail(
179
- parsed.text,
180
- typeof parsed.html === "string" ? parsed.html : undefined,
181
- SNIPPET_LENGTH,
182
- );
183
-
184
- /**
185
- * Issue #499: whether a completed body-sync pass has already stored this
186
- * message's body. `bodyStorageKey` is written last, once every derivation of
187
- * that pass has run, so its presence means that pass finished.
188
- *
189
- * Key presence, and nothing more. What the classifier decided is
190
- * `Message.category`; whether it ran at all is `Message.classificationState`.
191
- * Neither is inferred from here — the field this reads is the storage fact.
192
- *
193
- * The re-fetch paths that re-enter with the key already set (`syncBodies(...,
194
- * force: true)` and `fetchAndGetBody`'s `NoSuchKey` fallback) are re-stores
195
- * that skip the derivations entirely (issue #1011), so this guard is what
196
- * keeps a first store that lands on an already-stored row from re-deciding
197
- * anything.
198
- */
199
- const hasStoredBody = (bodyStorageKey: string | undefined): boolean =>
200
- Boolean(bodyStorageKey);
201
-
202
- /**
203
- * Issue #398: `flags.autoArchive` is a filing preference relative to the Inbox.
204
- * A `leave` verdict reaches it for two unrelated reasons and only one of them
205
- * means "nothing confident to say":
206
- *
207
- * - `already-moved-by-remit` is {@link classifyPlacement}'s guard against
208
- * re-deciding a settled placement, and auto-archive is a re-decision. It
209
- * fires on messages Remit itself placed, or that a user moved back
210
- * afterwards.
211
- * - A message already in Junk gets an unsure `leave` whatever the sender's
212
- * `blocked` flag says, since the demote branch only fires outside Junk.
213
- * Moving mail out of Junk is a rescue, which this file reserves for a
214
- * confident signal plus sender trust.
215
- */
216
- const autoArchiveApplies = (
217
- verdict: PlacementVerdict,
218
- placement: FolderPlacement,
219
- ): boolean =>
220
- placement !== "junk" && !verdict.reasons.includes("already-moved-by-remit");
221
-
222
- export const toParsedBody = (parsed: ParsedMail): ParsedBody => ({
223
- text: parsed.text ?? null,
224
- html: typeof parsed.html === "string" ? parsed.html : null,
225
- attachments: (parsed.attachments ?? []).map((a) => ({
226
- filename: a.filename ?? null,
227
- contentType: a.contentType,
228
- contentDisposition: a.contentDisposition ?? null,
229
- contentId: a.contentId ?? null,
230
- size: a.size,
231
- })),
232
- });
233
-
234
- export interface BodySyncLogger {
235
- info(obj: Record<string, unknown>, msg: string): void;
236
- debug?(obj: Record<string, unknown>, msg: string): void;
237
- warn?(obj: Record<string, unknown>, msg: string): void;
238
- error?(obj: Record<string, unknown>, msg: string): void;
239
- }
240
-
241
- const noopLogger: BodySyncLogger = {
242
- info: () => {},
243
- };
244
-
245
- export interface SyncBodiesResult {
246
- syncedCount: number;
247
- syncedMessageIds: string[];
248
- skippedCount: number;
249
- failedCount: number;
250
- failedMessageIds: string[];
251
- }
252
-
253
- export interface FetchBodyResult {
254
- text: string | null;
255
- html: string | null;
256
- storedAt: string;
257
- }
258
-
259
- export interface StoreBodyPartContentsResult {
260
- stored: number;
261
- }
262
-
263
- export type ConnectionGetter = () => Promise<IImapConnection>;
264
-
265
- export interface BodySyncFolder {
266
- mailboxId: string;
267
- fullPath: string;
268
- placement: FolderPlacement;
269
- }
270
-
271
- export interface BodySyncRoundFolder extends BodySyncFolder {
272
- uidValidity: number;
273
- }
274
-
275
- export interface PlacementConfig {
276
- mailboxSpecialUseService: IMailboxSpecialUseRepository;
277
- /**
278
- * Local-first mover for a confident placement verdict (issue #1271). Same
279
- * `moveMessage(accountConfigId, messageId, destinationMailboxId, accountId)`
280
- * signature as the general-purpose `MessageMoveService` it replaced here,
281
- * but backed by a pending-move marker + its own reconciler queue instead of
282
- * the general MESSAGE_MOVE event (see `placement-move.ts`).
283
- */
284
- placementMoveService: PlacementMoveService;
285
- }
286
-
287
- /**
288
- * What body sync needs to set a message aside (issue #72). The mailbox facts
289
- * are supplied by the caller because they are properties of the round, not of
290
- * any one message.
291
- */
292
- export interface QuarantineConfig {
293
- quarantineService: QuarantineService;
294
- mailboxId: string;
295
- /**
296
- * The mailbox's stored UIDVALIDITY. Taken from the mailbox row rather than
297
- * from the open box because a round whose two disagree is paused by the
298
- * cursor guard before it reaches any UID.
299
- */
300
- uidValidity: number;
301
- /** Redeliveries of this batch so far, so the record says how hard we tried. */
302
- attempts: number;
303
- }
304
-
305
- /**
306
- * What body sync needs to auto-mark-read a message from an unsubscribed
307
- * sender (issue #302, RFC 039 Decision 3). Reuses the same
308
- * `FlagQueueService.markAsRead` a manual mark-as-read already goes through —
309
- * local `\Seen` + `ThreadMessage.isRead` + a durable pending IMAP flag-push
310
- * marker — so this fires the same round-trip, not a second primitive.
311
- */
312
- export interface UnsubscribeConfig {
313
- flagQueueService: FlagQueueService;
314
- }
315
-
316
- /**
317
- * What body sync needs to offer a mail's invitation as a card (issue #1033).
318
- *
319
- * The calendar store is here to provision the account's default collection on
320
- * first use, through the one function every other caller provisions with. That
321
- * is not bookkeeping: an invitation whose DTSTART names no zone is RFC 5545
322
- * floating time, and reading it anywhere but where the user's calendar lives
323
- * shows the meeting at the wrong hour. Without a collection there is no zone
324
- * to read it in, and every such event silently lands in UTC.
325
- *
326
- * `filterService` is read to honour `dismiss{muteSender:true}`: a sender the
327
- * user muted gets no card at all.
328
- */
329
- export interface CalendarSuggestionConfig {
330
- calendarSuggestionService: ICalendarSuggestionRepository;
331
- calendarStore: ICalendarStore;
332
- filterService: Pick<IFilterRepository, "listByAccountAndState">;
333
- }
334
-
335
- export class BodySyncService {
336
- private log: BodySyncLogger;
337
- private readonly filterPipeline?: FilterPipeline;
338
-
339
- constructor(
340
- private messageService: IMessageRepository,
341
- private storageService: StorageService,
342
- private threadMessageService: IThreadMessageRepository,
343
- private addressService: IAddressRepository,
344
- private envelopeService: IEnvelopeRepository,
345
- private readonly syncWarningService: SyncWarningService,
346
- logger?: BodySyncLogger,
347
- private readonly placementConfig?: PlacementConfig,
348
- private readonly filterConfig?: FilterConfig,
349
- private readonly quarantineConfig?: QuarantineConfig,
350
- private readonly unsubscribeConfig?: UnsubscribeConfig,
351
- private readonly calendarConfig?: CalendarSuggestionConfig,
352
- ) {
353
- this.log = logger ?? noopLogger;
354
- this.filterPipeline = filterConfig
355
- ? new FilterPipeline(filterConfig, this.log)
356
- : undefined;
357
- }
358
-
359
- async syncBodies(
360
- messageIds: string[],
361
- accountId: string,
362
- accountConfigId: string,
363
- folder: BodySyncRoundFolder,
364
- getConnection: ConnectionGetter,
365
- force = false,
366
- ): Promise<SyncBodiesResult> {
367
- const syncedMessageIds: string[] = [];
368
- const syncedUids: number[] = [];
369
- let skippedCount = 0;
370
- const location = {
371
- accountId,
372
- accountConfigId,
373
- mailboxPath: folder.fullPath,
374
- };
375
-
376
- // Resolve every message up front so we can issue ONE ranged FETCH for the
377
- // whole batch (the desktop-client pattern) instead of a SELECT + download
378
- // per message. Messages whose body is already stored are skipped here and
379
- // never hit the wire — UNLESS `force` is set, in which case every message
380
- // is re-fetched regardless of `bodyStorageKey` (the read-miss cue already
381
- // confirmed the stored object is gone). `pending` maps each UID to its
382
- // messageId so we can match FETCH rows back and re-enqueue any UID the
383
- // server never returns.
384
- const pending = new Map<number, string>();
385
- // Message IDs in this batch whose row already carried `bodyStorageKey`
386
- // when `force` re-added them to `pending`: a forced re-fetch of an
387
- // already-stored body is a re-store (issue #1011) — bytes only, no
388
- // decision re-run. A message with no stored body inside a forced batch
389
- // is still a first store and takes the full decision pass.
390
- const reStoredMessageIds = new Set<string>();
391
-
392
- // One read per round, not per message (issue #72). The list is small by
393
- // design and almost always empty, so a lookup per message would put a
394
- // query on the hot path for a state that is nearly never set.
395
- const quarantined =
396
- await this.quarantineConfig?.quarantineService.load(accountConfigId);
397
-
398
- const rows = new Map(
399
- (await this.messageService.get(messageIds)).map((row) => [
400
- row.messageId,
401
- row,
402
- ]),
403
- );
404
-
405
- for (const messageId of messageIds) {
406
- const message = rows.get(messageId);
407
-
408
- // A row deleted since the batch was cut, or moved out of this folder
409
- // for good, has no uid here; its new uid would fetch another message.
410
- if (!message || uidMailboxIdOf(message) !== folder.mailboxId) {
411
- skippedCount++;
412
- continue;
413
- }
414
-
415
- // A quarantined message has no stored body, so nothing else here would
416
- // stop it being fetched and re-parsed on every redelivery, forever.
417
- if (
418
- this.quarantineConfig &&
419
- quarantined?.has(
420
- this.quarantineConfig.mailboxId,
421
- this.quarantineConfig.uidValidity,
422
- message.uid,
423
- )
424
- ) {
425
- this.log.debug?.(
426
- { messageId, uid: message.uid },
427
- "Message is quarantined, skipping",
428
- );
429
- skippedCount++;
430
- continue;
431
- }
432
-
433
- // The body is stored, so there is nothing to fetch (issue #331). This
434
- // used to re-read those bytes from storage to re-derive a category,
435
- // keyed on `category === uncategorized` — which is also the classifier's
436
- // own "nothing to say", so a message it declined cost one storage read
437
- // and two writes on every pass, forever, for the same answer. What the
438
- // classifier did is now recorded as `classificationState`, so a sync
439
- // pass never has to guess it from the row, and the cohort it has not
440
- // reached is selectable by a backfill instead of re-derived here.
441
- if (hasStoredBody(message.bodyStorageKey) && !force) {
442
- this.log.debug?.({ messageId }, "Body already stored, skipping");
443
- skippedCount++;
444
- continue;
445
- }
446
- if (message.bodyStorageKey) reStoredMessageIds.add(messageId);
447
- pending.set(message.uid, messageId);
448
- }
449
-
450
- if (pending.size === 0) {
451
- return this.buildResult(syncedMessageIds, skippedCount, []);
452
- }
453
-
454
- const connection = await getConnection();
455
- // Single SELECT for the whole batch. openBox is idempotent, so a warm
456
- // connection already on this mailbox skips the SELECT entirely.
457
- await connection.openBox(folder.fullPath);
458
-
459
- let connectionLost = false;
460
- try {
461
- for await (const { uid, source } of connection.fetchMessageBodies([
462
- ...pending.keys(),
463
- ])) {
464
- const messageId = pending.get(uid);
465
- if (!messageId) {
466
- // A UID we didn't ask for — drain the stream so the connection
467
- // stays usable, then ignore it.
468
- source.resume();
469
- continue;
470
- }
471
- try {
472
- await this.storeStreamedBody(
473
- messageId,
474
- accountId,
475
- accountConfigId,
476
- source,
477
- folder.placement,
478
- reStoredMessageIds.has(messageId),
479
- );
480
- } catch (error) {
481
- // The one error in this frame that is the message's own fault. It
482
- // reaches here from the parse call and nowhere else, so the S3, DB
483
- // and queue work this same frame wraps cannot be mistaken for it.
484
- // The row is durable before the UID leaves `pending`, so the
485
- // message is never let go of unrecorded.
486
- if (error instanceof BodyParseError && this.quarantineConfig) {
487
- await this.recordQuarantine(
488
- this.quarantineConfig,
489
- messageId,
490
- location,
491
- error,
492
- );
493
- pending.delete(uid);
494
- skippedCount++;
495
- source.resume();
496
- continue;
497
- }
498
-
499
- // Anything else is our own storage, database or queue failing. It
500
- // aborts the batch, which redelivers whole; what stored skips.
501
- throw error;
502
- }
503
- pending.delete(uid);
504
- syncedMessageIds.push(messageId);
505
- syncedUids.push(uid);
506
- }
507
- } catch (error) {
508
- this.log.error?.(
509
- { error: (error as Error).message },
510
- "Body fetch stream failed",
511
- );
512
- // Fail-fast: a dropped connection mid-stream leaves every not-yet-yielded
513
- // UID in `pending`; they fall through to failedMessageIds and re-enqueue.
514
- // Any other error is a real fault — let it crash.
515
- if (!isConnectionDrop(error)) {
516
- throw error;
517
- }
518
- connectionLost = true;
519
- this.log.info?.(
520
- { remainingCount: pending.size },
521
- "Connection lost, aborting batch",
522
- );
523
- }
524
-
525
- // Anything still pending was never yielded (mid-stream drop or a UID the
526
- // server silently omitted) — re-enqueue it.
527
- const failedMessageIds = [...pending.values()];
528
-
529
- await this.syncWarningService.settle(
530
- {
531
- accountConfigId,
532
- accountId,
533
- mailboxId: folder.mailboxId,
534
- uidValidity: folder.uidValidity,
535
- },
536
- SyncWarningStage.Body,
537
- {
538
- failedUids: [...pending.keys()],
539
- syncedUids,
540
- serverUids: { kind: "Unlisted" },
541
- },
542
- );
543
-
544
- this.log.info(
545
- {
546
- synced: syncedMessageIds.length,
547
- skipped: skippedCount,
548
- failed: failedMessageIds.length,
549
- total: messageIds.length,
550
- aborted: connectionLost,
551
- },
552
- "Body sync complete",
553
- );
554
-
555
- return this.buildResult(syncedMessageIds, skippedCount, failedMessageIds);
556
- }
557
-
558
- /**
559
- * Set a message aside because its body will not parse (issue #72). The
560
- * fingerprint comes from the BodyPart rows metadata sync already wrote: the
561
- * body path streams raw bytes and has no FETCH result to read.
562
- */
563
- private async recordQuarantine(
564
- config: QuarantineConfig,
565
- messageId: string,
566
- location: {
567
- accountId: string;
568
- accountConfigId: string;
569
- mailboxPath: string;
570
- },
571
- error: BodyParseError,
572
- ): Promise<void> {
573
- const message = await this.messageService.get(messageId);
574
- const messageData = await this.envelopeService.getMessageData(messageId);
575
- const threadMessage = await this.threadMessageService.findByMessageId(
576
- location.accountConfigId,
577
- messageId,
578
- );
579
-
580
- await config.quarantineService.record(
581
- {
582
- accountId: location.accountId,
583
- accountConfigId: location.accountConfigId,
584
- mailboxId: config.mailboxId,
585
- mailboxPath: location.mailboxPath,
586
- uidValidity: config.uidValidity,
587
- attempts: config.attempts,
588
- },
589
- message.uid,
590
- {
591
- stage: QuarantineFailureStage.BodyParse,
592
- code: error.failureCode,
593
- message: error.message,
594
- },
595
- shapeFromMessageData(
596
- message,
597
- messageData,
598
- threadMessage?.messageIdHeader,
599
- ),
600
- );
601
- }
602
-
603
- private buildResult(
604
- syncedMessageIds: string[],
605
- skippedCount: number,
606
- failedMessageIds: string[],
607
- ): SyncBodiesResult {
608
- return {
609
- syncedCount: syncedMessageIds.length,
610
- syncedMessageIds,
611
- skippedCount,
612
- failedCount: failedMessageIds.length,
613
- failedMessageIds,
614
- };
615
- }
616
-
617
- /**
618
- * Stream one message body straight to storage while teeing the bytes into a
619
- * buffer for the parse-dependent steps (snippet, classification, parsed-body
620
- * cache, per-part objects). The S3 upload never sees a whole-body concat —
621
- * the storage service streams it — but mailparser still needs the full bytes,
622
- * so we collect them in parallel. Later issues move parsing off the hot path.
623
- *
624
- * `isReStore` marks a re-fetch of a message whose row already carried
625
- * `bodyStorageKey` (the `force` re-arm in {@link syncBodies}): the
626
- * post-store steps then re-materialize the body artifacts without
627
- * re-running any decision (issue #1011).
628
- */
629
- private async storeStreamedBody(
630
- messageId: string,
631
- accountId: string,
632
- accountConfigId: string,
633
- source: Readable,
634
- placement: FolderPlacement,
635
- isReStore: boolean,
636
- ): Promise<void> {
637
- const toStorage = new PassThrough();
638
- const chunks: Buffer[] = [];
639
-
640
- // Tee the source: bytes flow to storage as a stream (no whole-body concat
641
- // on the upload path) while we also collect them for the parse-dependent
642
- // steps below, which still need the full body for mailparser. The store
643
- // and the tee are awaited together so neither rejection is orphaned.
644
- const tee = new Promise<void>((resolve, reject) => {
645
- source.on("data", (chunk: Buffer) => {
646
- chunks.push(chunk);
647
- toStorage.write(chunk);
648
- });
649
- source.on("end", () => {
650
- toStorage.end();
651
- resolve();
652
- });
653
- source.on("error", (err) => {
654
- toStorage.destroy(err);
655
- reject(err);
656
- });
657
- });
658
-
659
- const [ref] = await Promise.all([
660
- this.storageService.storeMessageBodyStream({
661
- accountConfigId,
662
- accountId,
663
- messageId,
664
- content: toStorage,
665
- }),
666
- tee,
667
- ]);
668
-
669
- const body = Buffer.concat(chunks);
670
-
671
- await this.applyPostStoreSteps(
672
- messageId,
673
- accountId,
674
- accountConfigId,
675
- body,
676
- {
677
- uri: ref.uri,
678
- },
679
- placement,
680
- isReStore,
681
- );
682
- }
683
-
684
- /**
685
- * The steps every synced body must go through regardless of which path
686
- * fetched it first — the sync path ({@link storeStreamedBody}) and the
687
- * read-path backfill ({@link fetchAndGetBody}) both call this instead of
688
- * duplicating (and drifting on) their own subset. Before issue #1271,
689
- * `fetchAndGetBody` skipped classification/placement entirely — a message
690
- * materialized by a read got different treatment than one synced in bulk.
691
- *
692
- * Order matters for two independent reasons, both load-bearing:
693
- * 1. The parsed-body cache must be durable before `bodyStorageKey` (the
694
- * skip-guard signal) — see the comment on the write below.
695
- * 2. A placement move — local mailboxId change, pending marker, IMAP-push
696
- * enqueue (`PlacementMoveService.moveMessage`, issue #1271) — must
697
- * complete, or fully fail, BEFORE `bodyStorageKey` is written. That
698
- * failure is never swallowed: if it throws, `bodyStorageKey` is not yet
699
- * durable, so a retry reprocesses the message from scratch (marker
700
- * included) instead of the skip-guard stranding a `movedByRemit` flag
701
- * with no marker behind it — the defect this issue fixes.
702
- *
703
- * Returns the parsed mail so callers that already need it (both do) don't
704
- * pay for mailparser twice.
705
- */
706
- private async applyPostStoreSteps(
707
- messageId: string,
708
- accountId: string,
709
- accountConfigId: string,
710
- body: Buffer,
711
- bodyRef: { uri: string },
712
- placement: FolderPlacement,
713
- /**
714
- * Whether this call is a re-store of an already-classified message
715
- * rather than a first store. The two re-entrant paths (`force: true`
716
- * in `syncBodies`, and `fetchAndGetBody`'s `NoSuchKey` IMAP fallback) are
717
- * both body-only re-fetches: a body re-sync fetches and stores bytes and
718
- * runs no decision logic (issue #1011). When `isReStore` is true the
719
- * placement/filter/category machinery is skipped entirely and the
720
- * message's existing category is carried forward unchanged.
721
- */
722
- isReStore: boolean,
723
- ): Promise<ParsedMail> {
724
- // The one operation on this path that can fail because of how the message
725
- // is built. Its own try block lives in `parseMessageBody`, so this frame —
726
- // which also wraps the S3 body write, the parsed-body cache, the placement
727
- // move, the label writes and the counter update — can tell the two apart
728
- // and quarantine only this one (issue #72). The ThreadMessage write these
729
- // feed happens later, alongside the Message write (see below), once the
730
- // write-once category is decided.
731
- const parsed = await parseMessageBody(body);
732
-
733
- // Body-only re-fetch (issue #1011): the decision pass already ran when
734
- // this message's body first landed, so a re-store re-materializes only
735
- // the body-derived artifacts — parsed cache, body parts, the storage
736
- // key — and re-decides nothing: no classification, placement, filters,
737
- // labels, read state, and no second engagement-counter increment.
738
- // Skipping the decisions here is what lets the write-once fields stay
739
- // safe without per-derivation re-entrancy guards.
740
- if (isReStore) {
741
- // Same ordering rule as the first-store path below: the parsed-body
742
- // cache must be durable before `bodyStorageKey` — the skip-guard
743
- // signal — is written, so a failure here requeues the message and
744
- // the retry genuinely re-attempts the artifact writes.
745
- await this.storeParsedBodyCache(
746
- accountConfigId,
747
- accountId,
748
- messageId,
749
- parsed,
750
- );
751
- if (!isBodyPartDeferralEnabled()) {
752
- await this.storeBodyPartContents(
753
- accountConfigId,
754
- accountId,
755
- messageId,
756
- await this.envelopeService.listBodyParts(messageId),
757
- body,
758
- );
759
- }
760
- await this.messageService.update(messageId, {
761
- bodyStorageKey: bodyRef.uri,
762
- });
763
- this.log.info(
764
- { messageId, storageKey: bodyRef.uri },
765
- "Body re-stored without re-running decisions",
766
- );
767
- return parsed;
768
- }
769
-
770
- const snippet = extractSnippet(parsed);
771
- const listId = extractListId(parsed);
772
-
773
- // Compute the header classification once, folding in the sender's
774
- // `flags.category` override when one applies (issue #299). The derived
775
- // fields are folded into the single Message update below — they are NOT
776
- // written here.
777
- const classification = await this.classifyMessage(
778
- accountConfigId,
779
- parsed,
780
- placement,
781
- );
782
-
783
- // Decide the placement move from the in-memory classification before the
784
- // write, so its `movedByRemit` flag and audit verdict join the same
785
- // UpdateItem. The verdict is recorded whenever Remit decided to act
786
- // (action != leave); the move is enqueued only for a confident,
787
- // actionable verdict.
788
- const resolved = await this.resolvePlacement(
789
- messageId,
790
- accountId,
791
- accountConfigId,
792
- parsed,
793
- classification,
794
- );
795
-
796
- // Index-time filter pass (RFC 034). Isolated the same way placement is —
797
- // FilterPipeline.evaluate swallows any read/decision failure and returns
798
- // no actions, so a schema-drifted filter query never fails the body store.
799
- // The DECISION is computed here; the applies below are unswallowed.
800
- const filterDecision = await this.evaluateFilters(
801
- accountConfigId,
802
- accountId,
803
- messageId,
804
- parsed,
805
- );
806
-
807
- // Store the parsed-body cache BEFORE persisting bodyStorageKey. The skip
808
- // guard in syncBodies treats a stored bodyStorageKey as "fully synced",
809
- // so if we wrote it first and the parsed-cache write then failed, the
810
- // requeued retry would skip the message — leaving parsedBody null forever.
811
- // A parsed-cache failure here throws and propagates before bodyStorageKey
812
- // is written, so the requeued message genuinely re-attempts the parsed
813
- // write.
814
- await this.storeParsedBodyCache(
815
- accountConfigId,
816
- accountId,
817
- messageId,
818
- parsed,
819
- );
820
-
821
- // Per-part S3 objects dominate S3 writes during bulk sync. When deferral
822
- // is enabled we skip them here; they are materialized lazily on the first
823
- // `contentUrl` read (see ensureBodyPartsStored), keeping bulk sync at 2
824
- // writes/message (body.eml + parsed.json.gz).
825
- if (!isBodyPartDeferralEnabled()) {
826
- await this.storeBodyPartContents(
827
- accountConfigId,
828
- accountId,
829
- messageId,
830
- await this.envelopeService.listBodyParts(messageId),
831
- body,
832
- );
833
- }
834
-
835
- // Local-first move (issue #1271): local mailboxId change + pending marker +
836
- // IMAP-push enqueue via PlacementMoveService, UNSWALLOWED — a throw here
837
- // propagates before bodyStorageKey is written (see the method doc above),
838
- // so a genuine SQS/DDB write failure requeues the message rather than being
839
- // absorbed. A matched filter's move is exclusive and outranks the
840
- // classifier's placement move (RFC 034 Decision 3.1) — an explicit user
841
- // rule wins the single mailbox a message occupies — so at most one move is
842
- // enqueued. A non-confident/leave verdict with no filter move is a no-op.
843
- let filterMove: MessageFilterMove | undefined;
844
- let filterMoved = false;
845
- if (filterDecision.move) {
846
- const destinationMailboxId = filterDecision.move.destinationMailboxId;
847
- // Capture the source mailbox BEFORE the move rewrites mailboxId, so the
848
- // auto-moved badge can name where the filter took the message from and
849
- // offer an undo back to it (issue #223). Snapshotted into the marker
850
- // rather than read from `originalMailboxId` at derive time, which a
851
- // later user move would overwrite — the filter's source must survive an
852
- // undo to stay derivable. A redelivery after a failed enqueue finds
853
- // the move already applied locally, so the source is the folder the
854
- // uid still names, not the one the row now points at.
855
- const sourceMailboxId = uidMailboxIdOf(
856
- await this.messageService.get(messageId),
857
- );
858
- const filed = await this.filterConfig?.placementMoveService.moveMessage(
859
- accountConfigId,
860
- messageId,
861
- destinationMailboxId,
862
- accountId,
863
- );
864
- // A filter matches by content, not location: a message delivered
865
- // straight into the destination (a server-side rule) or already filed
866
- // there matches but was not moved by Remit. `moveMessage` no-ops in that
867
- // case; record neither the marker nor `movedByRemit`, so the badge never
868
- // claims a move that did not happen and its Undo is never a no-op.
869
- if (sourceMailboxId !== destinationMailboxId && filed !== "refused") {
870
- filterMoved = true;
871
- filterMove = {
872
- filterId: filterDecision.move.filterId,
873
- sourceMailboxId,
874
- destinationMailboxId,
875
- decidedAt: Date.now(),
876
- };
877
- }
878
- } else if (resolved.move) {
879
- await this.placementConfig?.placementMoveService.moveMessage(
880
- accountConfigId,
881
- messageId,
882
- resolved.move.destinationMailboxId,
883
- accountId,
884
- );
885
- }
886
-
887
- // Additive label actions (RFC 034 Decision 3.1): every matching filter's
888
- // label applies. The deterministic MessageLabel upsert is idempotent, so a
889
- // requeue after a later failure re-applies safely; UNSWALLOWED, so a DDB
890
- // write failure propagates before bodyStorageKey is written.
891
- for (const label of filterDecision.labels) {
892
- await this.filterConfig?.messageLabelService.apply({
893
- messageId,
894
- labelId: label.labelId,
895
- accountConfigId,
896
- appliedByFilterId: label.filterId,
897
- });
898
- }
899
-
900
- // Read once for the two write-once guards below — the auto-read decision
901
- // and the category carry-forward. Taken here rather than earlier because
902
- // nothing between this point and the Message update writes either field;
903
- // the moves above touch `mailboxId` only.
904
- const existingMessage = await this.messageService.get(messageId);
905
-
906
- // `flags.unsubscribed` (issue #302, RFC 039 Decision 3): auto-mark-read,
907
- // reusing the same FlagQueueService.markAsRead round-trip a manual
908
- // mark-as-read goes through — idempotent on a retry (flipFlag no-ops when
909
- // the message is already \Seen), so a failure here safely re-fires on the
910
- // next attempt rather than being lost behind the bodyStorageKey skip guard.
911
- await this.applyUnsubscribedAutoRead(
912
- messageId,
913
- accountId,
914
- accountConfigId,
915
- parsed,
916
- existingMessage.bodyStorageKey,
917
- );
918
-
919
- await this.deriveCalendarSuggestions(
920
- messageId,
921
- accountConfigId,
922
- parsed,
923
- existingMessage.bodyStorageKey,
924
- );
925
-
926
- const moved = Boolean(resolved.move || filterMoved);
927
-
928
- // ONE Message UpdateItem per synced message: bodyStorageKey + every
929
- // classification/derived field + the move flag + the audit verdict,
930
- // collapsed into a single write.
931
- // The flag is folded in only when a move WAS applied (by the classifier or
932
- // a filter); the verdict is folded in whenever Remit decided to act.
933
- // Written LAST so bodyStorageKey — the skip-guard signal — is only durable
934
- // once the parsed cache AND the move (when any) are.
935
- //
936
- // `category` is RFC 034 D3.1's write-once field (RFC 030's message-list
937
- // GSI sort key depends on it never churning). The re-entrant re-store
938
- // paths (issue #1011) return before this write; the carry-forward keeps
939
- // the field write-once even when a first store lands on a row that
940
- // already carries a real category — a previously-decided category wins
941
- // over the just-recomputed one. It likewise protects a `flags.category`
942
- // override (issue #299): recomputing would let a *later* override silently
943
- // rewrite a category already decided, which is exactly the churn RFC
944
- // 030's GSI-safety argument forbids.
945
- //
946
- // `resolved.placementDecidedAt` (issue #383) is only ever present on
947
- // the pass that first decided it: the re-store paths skip placement
948
- // entirely (issue #1011), so `computePlacement` is reached only on a
949
- // first store.
950
- const finalCategory = hasDecidedCategory(existingMessage.category)
951
- ? existingMessage.category
952
- : classification.category;
953
-
954
- // Thread-list denormalization, using the same write-once category as the
955
- // Message update below — the two rows must never disagree on category.
956
- await denormalizeMessageCategory(
957
- { threadMessageService: this.threadMessageService },
958
- accountConfigId,
959
- messageId,
960
- { category: finalCategory, snippet, listId },
961
- );
962
-
963
- // `classificationState` records that the classifier ran, in the same
964
- // UpdateItem as the answer it gave (issue #331). Without it `uncategorized`
965
- // carries two meanings at once — not reached yet, and reached with nothing
966
- // to say — and only one of them should ever be looked at again.
967
- const update: UpdateMessageInput = {
968
- bodyStorageKey: bodyRef.uri,
969
- ...classification,
970
- category: finalCategory,
971
- classificationState: MessageClassificationState.Examined,
972
- ...(moved ? { movedByRemit: true } : {}),
973
- ...(resolved.verdict ? { placementVerdict: resolved.verdict } : {}),
974
- ...(filterMove ? { filterMove } : {}),
975
- ...(resolved.placementDecidedAt
976
- ? { placementDecidedAt: resolved.placementDecidedAt }
977
- : {}),
978
- };
979
- await this.messageService.update(messageId, update);
980
- this.log.info({ messageId, storageKey: bodyRef.uri }, "Body stored");
981
-
982
- // From-Address engagement counter (Address entity, not Message).
983
- await this.incrementInboundCount(
984
- messageId,
985
- accountConfigId,
986
- parsed,
987
- classification,
988
- );
989
-
990
- return parsed;
991
- }
992
-
993
- /**
994
- * Fetch a single message body, store it, and return the parsed content.
995
- *
996
- * If the body is already stored, retrieves it from storage.
997
- * Otherwise fetches from IMAP, stores it, and returns the parsed content.
998
- *
999
- * @param messageId - The message ID to fetch
1000
- * @param accountId - The account ID (for storage path)
1001
- * @param accountConfigId - The account config ID (for thread updates)
1002
- * @param folder - The IMAP mailbox path and its folder placement
1003
- * @param getConnection - Function to get a (lazy) IMAP connection
1004
- * @returns Parsed text and HTML content
1005
- */
1006
- async fetchAndGetBody(
1007
- messageId: string,
1008
- accountId: string,
1009
- accountConfigId: string,
1010
- folder: BodySyncFolder,
1011
- getConnection: ConnectionGetter,
1012
- ): Promise<FetchBodyResult> {
1013
- const message = await this.messageService.get(messageId);
1014
-
1015
- let body: Buffer;
1016
- let needsStore = false;
1017
-
1018
- if (message.bodyStorageKey) {
1019
- // Body already stored, try to retrieve from storage
1020
- this.log.debug?.({ messageId }, "Retrieving body from storage");
1021
- try {
1022
- body = await this.storageService.retrieve(message.bodyStorageKey);
1023
- } catch (err) {
1024
- // Only a genuinely-missing object (NoSuchKey) is a safe IMAP
1025
- // fallback — that's the cross-environment / never-stored case.
1026
- // A permission/infra error (AccessDenied, throttle) must NOT be
1027
- // masked as a missing object: let it crash so it's observable.
1028
- if (!isStorageNotFoundError(err)) {
1029
- this.log.error?.(
1030
- {
1031
- messageId,
1032
- storageKey: message.bodyStorageKey,
1033
- errorName: (err as { name?: string }).name,
1034
- errorCode: (err as { Code?: string }).Code,
1035
- error: inspect(err),
1036
- },
1037
- "Body storage retrieval failed (non-NoSuchKey); not falling back to IMAP",
1038
- );
1039
- throw err;
1040
- }
1041
- this.log.debug?.(
1042
- { messageId, error: (err as Error).message },
1043
- "Body object missing (NoSuchKey), falling back to IMAP",
1044
- );
1045
- needsStore = true;
1046
- body = await this.fetchFromImap(
1047
- message,
1048
- accountId,
1049
- accountConfigId,
1050
- folder.fullPath,
1051
- getConnection,
1052
- );
1053
- }
1054
- } else {
1055
- // Fetch from IMAP and store
1056
- needsStore = true;
1057
- body = await this.fetchFromImap(
1058
- message,
1059
- accountId,
1060
- accountConfigId,
1061
- folder.fullPath,
1062
- getConnection,
1063
- );
1064
- }
1065
-
1066
- let parsed: ParsedMail;
1067
-
1068
- if (needsStore) {
1069
- const ref = await this.storageService.storeMessageBody({
1070
- accountConfigId,
1071
- accountId,
1072
- messageId,
1073
- content: body,
1074
- });
1075
-
1076
- // Same shared step the sync path (storeStreamedBody) runs — issue
1077
- // #1271 unified the two body paths so classification/placement no
1078
- // longer depends on which one fetched the body first. A store from
1079
- // here is a re-store exactly when the row already claimed a stored
1080
- // body — the NoSuchKey fallback above; a row with no bodyStorageKey is
1081
- // a genuine first store and takes the full decision pass.
1082
- parsed = await this.applyPostStoreSteps(
1083
- messageId,
1084
- accountId,
1085
- accountConfigId,
1086
- body,
1087
- { uri: ref.uri },
1088
- folder.placement,
1089
- message.bodyStorageKey != null,
1090
- );
1091
- } else {
1092
- parsed = await simpleParser(body);
1093
- }
1094
-
1095
- return {
1096
- text: parsed.text ?? null,
1097
- html: typeof parsed.html === "string" ? parsed.html : null,
1098
- storedAt: message.bodyStorageKey ?? "newly-stored",
1099
- };
1100
- }
1101
-
1102
- private async fetchFromImap(
1103
- message: MessageItem,
1104
- accountId: string,
1105
- accountConfigId: string,
1106
- mailboxPath: string,
1107
- getConnection: ConnectionGetter,
1108
- ): Promise<Buffer> {
1109
- const { uid } = message;
1110
- this.log.debug?.({ uid }, "Fetching body from IMAP");
1111
- const connection = await getConnection();
1112
- await connection.openBox(mailboxPath);
1113
- const body = await connection.fetchMessageBody(uid);
1114
- if (body) return body;
1115
-
1116
- if (!(await isMessageGoneFromOpenMailbox(connection, uid))) {
1117
- throw new Error(
1118
- `IMAP FETCH returned no source for UID ${uid} in ${mailboxPath}, which the server still lists`,
1119
- );
1120
- }
1121
-
1122
- return this.reconcileGoneMessage(message, accountId, mailboxPath);
1123
- }
1124
-
1125
- /**
1126
- * The server no longer holds the message at the row's UID. A row with our
1127
- * own move or delete in flight belongs to that mutation (409). A settled row
1128
- * is removed with its thread entries, but only while its placement still
1129
- * matches what was read (imap-mutations R3); losing that predicate is a 409
1130
- * like any other lost placement transition.
1131
- */
1132
- private async reconcileGoneMessage(
1133
- message: MessageItem,
1134
- accountId: string,
1135
- mailboxPath: string,
1136
- ): Promise<never> {
1137
- const { messageId, uid } = message;
1138
- if (isPlacementUnsettled(message)) {
1139
- throw new MessagePlacementUnsettledError(
1140
- `Message ${messageId} is not at its UID while a move or delete is in flight`,
1141
- accountId,
1142
- messageId,
1143
- "in_flight",
1144
- );
1145
- }
1146
-
1147
- const reconciled = await this.messageService.deleteIfPlacement(messageId, {
1148
- status: message.status,
1149
- mailboxId: message.mailboxId,
1150
- uid,
1151
- });
1152
-
1153
- if (!reconciled) {
1154
- throw new MessagePlacementUnsettledError(
1155
- `Message ${messageId} changed placement while its absence from ${mailboxPath} was being reconciled`,
1156
- accountId,
1157
- messageId,
1158
- "in_flight",
1159
- );
1160
- }
1161
-
1162
- this.log.info(
1163
- {
1164
- metric: "message_gone_stale_row_reconciled",
1165
- messageId,
1166
- uid,
1167
- mailboxPath,
1168
- },
1169
- "Message no longer on the mail server (expunged or moved by another client); stale row reconciled",
1170
- );
1171
- throw new MessageGoneFromServerError(messageId);
1172
- }
1173
-
1174
- /**
1175
- * Header classification, with the sender's `Address.flags.category`
1176
- * override (issue #299, RFC 039 Decision 3) substituted for the
1177
- * header-derived category when one is set. Shared with the classification
1178
- * backfill (issue #1197) as {@link classifyParsedMessage}, so the one-time
1179
- * pass and the body-store path can never classify the same bytes
1180
- * differently.
1181
- */
1182
- private async classifyMessage(
1183
- accountConfigId: string,
1184
- parsed: ParsedMail,
1185
- placement: FolderPlacement,
1186
- ): Promise<UpdateMessageInput & { category: ThreadMessageCategory }> {
1187
- return classifyParsedMessage(
1188
- this.addressService,
1189
- accountConfigId,
1190
- parsed,
1191
- placement,
1192
- );
1193
- }
1194
-
1195
- private async incrementInboundCount(
1196
- messageId: string,
1197
- accountConfigId: string,
1198
- parsed: ParsedMail,
1199
- classification: UpdateMessageInput,
1200
- ): Promise<void> {
1201
- const fromEmail = extractPrimaryFromEmail(parsed);
1202
- if (!fromEmail) {
1203
- this.log.debug?.(
1204
- { messageId },
1205
- "No From address; skipping inbound counter",
1206
- );
1207
- return;
1208
- }
1209
-
1210
- const addressId = deriveAddressId(accountConfigId, fromEmail);
1211
- const bulk = isBulkSender(
1212
- classification.category,
1213
- classification.hasListUnsubscribe ?? false,
1214
- );
1215
- await this.addressService.incrementInboundCount(
1216
- accountConfigId,
1217
- addressId,
1218
- Date.now(),
1219
- bulk,
1220
- );
1221
- }
1222
-
1223
- /**
1224
- * The per-sender signals {@link computePlacement} needs, from ONE `Address`
1225
- * fetch (RFC 039 Decision 3/3a, issue #300): the trust reads exactly as
1226
- * `deriveSenderTrust` always did — `vip → wellknown → unknown`, untouched by
1227
- * `blocked` — plus `blocked`/`autoArchive` off the same row. `trustSetAt` is
1228
- * the `setAt` of whichever flag produced the trust value, needed by
1229
- * {@link resolveSenderPlacement}'s tie-break; it stays local to placement and
1230
- * never reaches `deriveSenderTrust`'s own contract (the trust badge).
1231
- */
1232
- private async deriveSenderPlacementSignals(
1233
- accountConfigId: string,
1234
- fromEmail: string,
1235
- ): Promise<{
1236
- trust: (typeof SenderTrust)[keyof typeof SenderTrust];
1237
- trustSetAt?: number;
1238
- blocked: boolean;
1239
- blockedSetAt?: number;
1240
- neverSpam: boolean;
1241
- autoArchive: boolean;
1242
- }> {
1243
- const unknown = {
1244
- trust: SenderTrust.Unknown,
1245
- blocked: false,
1246
- neverSpam: false,
1247
- autoArchive: false,
1248
- } as const;
1249
- try {
1250
- const addressId = deriveAddressId(accountConfigId, fromEmail);
1251
- const address = await this.addressService.getAddress(
1252
- accountConfigId,
1253
- addressId,
1254
- );
1255
- const flags = address.flags;
1256
- if (flags?.vip?.value === true) {
1257
- return {
1258
- trust: SenderTrust.Vip,
1259
- trustSetAt: flags.vip.setAt,
1260
- blocked: flags.blocked?.value === true,
1261
- blockedSetAt: flags.blocked?.setAt,
1262
- neverSpam: flags.neverSpam?.value === true,
1263
- autoArchive: flags.autoArchive?.value === true,
1264
- };
1265
- }
1266
- if (flags?.wellknown?.value === true) {
1267
- return {
1268
- trust: SenderTrust.Wellknown,
1269
- trustSetAt: flags.wellknown.setAt,
1270
- blocked: flags.blocked?.value === true,
1271
- blockedSetAt: flags.blocked?.setAt,
1272
- neverSpam: flags.neverSpam?.value === true,
1273
- autoArchive: flags.autoArchive?.value === true,
1274
- };
1275
- }
1276
- return {
1277
- ...unknown,
1278
- blocked: flags?.blocked?.value === true,
1279
- blockedSetAt: flags?.blocked?.setAt,
1280
- neverSpam: flags?.neverSpam?.value === true,
1281
- autoArchive: flags?.autoArchive?.value === true,
1282
- };
1283
- } catch (err) {
1284
- // A genuinely-absent address means "no signals". Any other failure
1285
- // (AccessDenied, throttle, infra) must NOT be silently downgraded — let
1286
- // it crash so the placement decision isn't made on bad data.
1287
- if (!(err instanceof NotFoundError)) throw err;
1288
- }
1289
- return unknown;
1290
- }
1291
-
1292
- /**
1293
- * `flags.unsubscribed` (issue #302, RFC 039 Decision 3): "auto-mark-read
1294
- * until sender stops" — fires on every new message from that sender for as
1295
- * long as the flag stays set, per the flag's own doc comment; there is no
1296
- * separate expiry mechanism, and no caching of the decision beyond this
1297
- * per-message `Address` read. A no-op when body sync was built without an
1298
- * {@link UnsubscribeConfig} or the message carries no `From` address.
1299
- *
1300
- * Write-once per message (issue #499), like the `category` and placement
1301
- * derivations alongside it: read state is applied on the pass that first
1302
- * classifies a message and never re-decided. A user who marks such a message
1303
- * unread afterwards owns it — a re-entrant pass over an already-classified
1304
- * body would otherwise silently undo that.
1305
- */
1306
- private async applyUnsubscribedAutoRead(
1307
- messageId: string,
1308
- accountId: string,
1309
- accountConfigId: string,
1310
- parsed: ParsedMail,
1311
- storedBodyKey: string | undefined,
1312
- ): Promise<void> {
1313
- if (!this.unsubscribeConfig) return;
1314
- if (hasStoredBody(storedBodyKey)) return;
1315
-
1316
- const fromEmail = extractPrimaryFromEmail(parsed);
1317
- if (!fromEmail) return;
1318
-
1319
- const unsubscribed = await this.deriveSenderUnsubscribed(
1320
- accountConfigId,
1321
- fromEmail,
1322
- );
1323
- if (!unsubscribed) return;
1324
-
1325
- await this.unsubscribeConfig.flagQueueService.markAsRead(
1326
- accountConfigId,
1327
- messageId,
1328
- accountId,
1329
- );
1330
- }
1331
-
1332
- /**
1333
- * Offer what a message's `text/calendar` parts propose, as cards beside the
1334
- * message (issue #1033). Nothing here reaches a calendar: a suggestion is
1335
- * written `Pending` and waits for a person, and a `METHOD:CANCEL` is a card
1336
- * of its own rather than a withdrawal of an event the user added.
1337
- *
1338
- * Write-once per message, like the `category`, placement and read-state
1339
- * derivations alongside it (issues #499, #1011): a card is offered on the
1340
- * pass that first classifies a message — the same moment the account's
1341
- * filters run — and never again. The two re-entrant paths a forced body
1342
- * re-sync goes through (`fetchAndGetBody`'s `NoSuchKey` fallback,
1343
- * `syncBodies(..., force: true)`) reach this with `bodyStorageKey` already
1344
- * set, and re-offering there would resurrect a card the user dismissed.
1345
- *
1346
- * Isolated the same way placement and filters are: a card is auxiliary to
1347
- * storing the mail, which is already durable by the time this runs. A
1348
- * repository failure here is logged loudly with the messageId rather than
1349
- * failing a body store that otherwise succeeded.
1350
- */
1351
- private async deriveCalendarSuggestions(
1352
- messageId: string,
1353
- accountConfigId: string,
1354
- parsed: ParsedMail,
1355
- storedBodyKey: string | undefined,
1356
- ): Promise<void> {
1357
- if (!this.calendarConfig) return;
1358
- if (hasStoredBody(storedBodyKey)) return;
1359
-
1360
- const parts = calendarParts(parsed);
1361
- if (parts.length === 0) return;
1362
-
1363
- const { calendarStore, calendarSuggestionService, filterService } =
1364
- this.calendarConfig;
1365
-
1366
- await Promise.resolve()
1367
- .then(async () => {
1368
- // `dismiss{muteSender:true}` wrote a standing rule naming this
1369
- // sender. Honouring it here is what makes that button do anything:
1370
- // the index-time filter pipeline skips a rule with no label and no
1371
- // move, so this is the only reader such a rule has.
1372
- const sender = extractPrimaryFromEmail(parsed);
1373
- if (sender) {
1374
- const active = await filterService.listByAccountAndState(
1375
- accountConfigId,
1376
- FilterState.Active,
1377
- );
1378
- if (isSenderMuted(active, sender)) {
1379
- this.log.debug?.(
1380
- { messageId, sender },
1381
- "Sender is muted; offering no calendar suggestion",
1382
- );
1383
- return;
1384
- }
1385
- }
1386
-
1387
- // Provisioned rather than looked up, through the one function every
1388
- // caller provisions with. A missing collection has no timezone, and
1389
- // a floating DTSTART read in UTC puts the meeting hours from where
1390
- // the organizer meant it. Idempotent: the id is derived, so a second
1391
- // first use returns the collection the first one made.
1392
- const collection = await provisionDefaultCalendar(
1393
- calendarStore,
1394
- accountConfigId,
1395
- );
1396
- for (const part of parts) {
1397
- const recorded = await recordCalendarSuggestion(
1398
- calendarSuggestionService,
1399
- {
1400
- accountConfigId,
1401
- messageId,
1402
- bodyPartId: deriveBodyPartId(messageId, part.partPath),
1403
- source: part.source,
1404
- icalData: part.icalData,
1405
- timezone: collection.timezone,
1406
- },
1407
- );
1408
- if (!recorded.ok) {
1409
- this.log.info(
1410
- {
1411
- messageId,
1412
- partPath: part.partPath,
1413
- code: recorded.error.code,
1414
- reason: recorded.error.message,
1415
- },
1416
- "Message carries iCalendar this cannot read; no suggestion offered",
1417
- );
1418
- }
1419
- }
1420
- })
1421
- .catch((error: unknown) => {
1422
- this.log.error?.(
1423
- { messageId, accountConfigId, error: inspect(error) },
1424
- "Calendar suggestion derivation failed; body is stored without a card",
1425
- );
1426
- });
1427
- }
1428
-
1429
- private async deriveSenderUnsubscribed(
1430
- accountConfigId: string,
1431
- fromEmail: string,
1432
- ): Promise<boolean> {
1433
- try {
1434
- const addressId = deriveAddressId(accountConfigId, fromEmail);
1435
- const address = await this.addressService.getAddress(
1436
- accountConfigId,
1437
- addressId,
1438
- );
1439
- return address.flags?.unsubscribed?.value === true;
1440
- } catch (err) {
1441
- // A genuinely-absent address means "not unsubscribed". Any other
1442
- // failure (AccessDenied, throttle, infra) must NOT be silently
1443
- // downgraded — let it crash so the read-state decision isn't made on
1444
- // bad data.
1445
- if (!(err instanceof NotFoundError)) throw err;
1446
- return false;
1447
- }
1448
- }
1449
-
1450
- /**
1451
- * Evaluate the account's active filters against a synced message (RFC 034),
1452
- * BEFORE the single Message update — so a filter's `movedByRemit` flag joins
1453
- * that same UpdateItem. Returns the actions to apply (labels + at most one
1454
- * move); the caller applies them. A no-op returning no actions when no
1455
- * {@link FilterConfig} is wired.
1456
- *
1457
- * Isolation lives in {@link FilterPipeline.evaluate}: this read/decision phase
1458
- * never fails the body store (the #1246 placement precedent). The applies of
1459
- * the returned decision are the caller's and are deliberately unswallowed.
1460
- */
1461
- private async evaluateFilters(
1462
- accountConfigId: string,
1463
- accountId: string,
1464
- messageId: string,
1465
- parsed: ParsedMail,
1466
- ): Promise<FilterDecision> {
1467
- if (!this.filterPipeline) return { labels: [] };
1468
- return this.filterPipeline.evaluate(
1469
- accountConfigId,
1470
- accountId,
1471
- messageId,
1472
- toFilterMessage(parsed),
1473
- );
1474
- }
1475
-
1476
- /**
1477
- * Resolve the placement decision for a message, BEFORE the single Message
1478
- * update — so both the `movedByRemit` flag and the audit verdict join that
1479
- * one UpdateItem instead of a second mutation. Drives both directions (junk → inbox rescue,
1480
- * inbox → junk demote) off the pure {@link classifyPlacement} verdict.
1481
- *
1482
- * Returns a {@link PlacementOutcome}: a `verdict` to persist whenever Remit
1483
- * decided to act (action != leave), confident and unsure alike, so the
1484
- * distribution is queryable on the message; a `move` to enqueue only for a
1485
- * confident verdict; and `placementDecidedAt` whenever a verdict — including
1486
- * "leave" — was genuinely computed (issue #383), so a re-entrant call short-
1487
- * circuits instead of re-deciding a placement the user may have since
1488
- * overridden by hand.
1489
- *
1490
- * Always logs a structured verdict line for confident, actionable verdicts so
1491
- * the real distribution is observable on a live mailbox.
1492
- *
1493
- * The verdict reads the just-computed classification rather than a persisted
1494
- * row, since those fields are not written until the single update. Placement
1495
- * is auxiliary to the body store — the primary artifact (body.eml) is already
1496
- * durable by the time this runs — so a failure here (e.g. a placement-repo
1497
- * query erroring on a schema-drifted DB) is caught and logged loudly with an
1498
- * alertable field instead of failing the surrounding message store. The empty
1499
- * {@link PlacementOutcome} means "no action", whether Remit genuinely decided
1500
- * to leave the message alone or placement itself failed; the alert log is
1501
- * what distinguishes the latter, and neither case marks the message decided.
1502
- */
1503
- private async resolvePlacement(
1504
- messageId: string,
1505
- accountId: string,
1506
- accountConfigId: string,
1507
- parsed: ParsedMail,
1508
- classification: UpdateMessageInput,
1509
- ): Promise<PlacementOutcome> {
1510
- if (!this.placementConfig) return {};
1511
- return this.computePlacement(
1512
- this.placementConfig,
1513
- messageId,
1514
- accountId,
1515
- accountConfigId,
1516
- parsed,
1517
- classification,
1518
- ).catch((error: unknown) => {
1519
- this.log.error?.(
1520
- {
1521
- alert: "body_sync_placement_failed",
1522
- messageId,
1523
- accountId,
1524
- accountConfigId,
1525
- errorName: (error as { name?: string })?.name,
1526
- error: inspect(error),
1527
- },
1528
- "Placement resolution failed; body already stored, continuing without placement (best-effort, non-fatal)",
1529
- );
1530
- return {};
1531
- });
1532
- }
1533
-
1534
- private async computePlacement(
1535
- placementConfig: PlacementConfig,
1536
- messageId: string,
1537
- accountId: string,
1538
- accountConfigId: string,
1539
- parsed: ParsedMail,
1540
- classification: UpdateMessageInput,
1541
- ): Promise<PlacementOutcome> {
1542
- const { mailboxSpecialUseService } = placementConfig;
1543
-
1544
- const message = await this.messageService.get(messageId);
1545
-
1546
- // Issue #383 / #1011: placement still runs once per message (RFC 039
1547
- // Non-goals), but the once-per-message guarantee now comes from the
1548
- // caller rather than a guard here — the re-store paths
1549
- // (`fetchAndGetBody`'s `NoSuchKey` fallback, `syncBodies(..., force:
1550
- // true)`) skip the decision steps entirely, so only the pass that first
1551
- // stores a body ever reaches this method.
1552
-
1553
- const roleMailboxes = await resolveFolderRoleMailboxes(
1554
- mailboxSpecialUseService,
1555
- accountId,
1556
- );
1557
- const { junkMailbox, inboxMailbox } = roleMailboxes;
1558
- const placement = placementOf(message.mailboxId, roleMailboxes);
1559
-
1560
- const fromEmail = extractPrimaryFromEmail(parsed);
1561
- const signals = fromEmail
1562
- ? await this.deriveSenderPlacementSignals(accountConfigId, fromEmail)
1563
- : {
1564
- trust: SenderTrust.Unknown,
1565
- blocked: false,
1566
- neverSpam: false,
1567
- autoArchive: false,
1568
- };
1569
-
1570
- const { senderTrust, senderOverride } = resolveSenderPlacement(
1571
- { trust: signals.trust, setAt: signals.trustSetAt },
1572
- { blocked: signals.blocked, setAt: signals.blockedSetAt },
1573
- signals.neverSpam,
1574
- );
1575
-
1576
- // The verdict needs the classification signals (providerSpam,
1577
- // authResult, authenticity) that this body-sync pass just derived; the
1578
- // stored row does not carry them yet, so overlay them onto the message.
1579
- const candidate = { ...message, ...classification };
1580
- const verdict = classifyPlacement(
1581
- candidate,
1582
- placement,
1583
- senderTrust,
1584
- senderOverride,
1585
- );
1586
-
1587
- // A `leave` verdict — including "nothing confident to say" — carries no
1588
- // audit record of its own. `flags.autoArchive` (issue #300) is a distinct,
1589
- // lower-priority filing preference: it only files a message away when
1590
- // `blocked`/DKIM/DMARC had nothing to say, never overriding a confident
1591
- // junk/inbox verdict computed above. Still marked decided (issue #383):
1592
- // "leave" is itself a verdict, not "not yet evaluated".
1593
- if (verdict.action === "leave") {
1594
- const outcome = autoArchiveApplies(verdict, placement)
1595
- ? await this.resolveAutoArchive(
1596
- mailboxSpecialUseService,
1597
- message,
1598
- accountId,
1599
- signals.autoArchive,
1600
- )
1601
- : {};
1602
- return { ...outcome, placementDecidedAt: Date.now() };
1603
- }
1604
-
1605
- // Audit verdict — recorded for every actionable verdict (both
1606
- // confidences), so the distribution is queryable on the message.
1607
- // Independent of whether a move is enqueued.
1608
- const audit: MessagePlacementVerdict = {
1609
- action:
1610
- verdict.action === "move-to-inbox"
1611
- ? PlacementAction.MoveToInbox
1612
- : PlacementAction.MoveToJunk,
1613
- confidence:
1614
- verdict.confidence === "confident"
1615
- ? PlacementConfidence.Confident
1616
- : PlacementConfidence.Unsure,
1617
- fromPlacement: placement,
1618
- reasons: verdict.reasons,
1619
- dryRun: false,
1620
- decidedAt: Date.now(),
1621
- };
1622
-
1623
- // Only a confident verdict moves mail. An unsure verdict is recorded
1624
- // but never enqueues a move.
1625
- if (verdict.confidence !== "confident") {
1626
- return { verdict: audit, placementDecidedAt: audit.decidedAt };
1627
- }
1628
-
1629
- const target =
1630
- verdict.action === "move-to-inbox" ? inboxMailbox : junkMailbox;
1631
- if (!target) return { verdict: audit, placementDecidedAt: audit.decidedAt };
1632
-
1633
- // Structured verdict line — emitted for confident, actionable verdicts
1634
- // so the real verdict distribution is observable on a live mailbox.
1635
- this.log.info(
1636
- {
1637
- messageId,
1638
- accountId,
1639
- placement,
1640
- action: verdict.action,
1641
- confidence: verdict.confidence,
1642
- reasons: verdict.reasons,
1643
- destinationMailboxId: target.mailboxId,
1644
- },
1645
- "Placement verdict",
1646
- );
1647
-
1648
- return {
1649
- verdict: audit,
1650
- move: {
1651
- destinationMailboxId: target.mailboxId,
1652
- destinationPath: target.fullPath,
1653
- },
1654
- placementDecidedAt: audit.decidedAt,
1655
- };
1656
- }
1657
-
1658
- /**
1659
- * `flags.autoArchive` (issue #300, RFC 039 Decision 3): file a message
1660
- * straight to Archive, skipping Inbox. Only reached from
1661
- * {@link computePlacement} when {@link classifyPlacement} had no confident
1662
- * junk/inbox verdict of its own — `blocked`/DKIM/DMARC always take priority
1663
- * over this filing preference — and only for the `leave` verdicts
1664
- * {@link autoArchiveApplies} admits.
1665
- *
1666
- * No {@link MessagePlacementVerdict} is recorded: `PlacementAction` (the
1667
- * audit enum) has only `MoveToInbox`/`MoveToJunk` — issue #300 is scoped to
1668
- * no TypeSpec change, so an archive move carries no audit verdict, same as
1669
- * a matched filter's move. The move itself reuses the same
1670
- * `placementMoveService.moveMessage` path as every other confident move.
1671
- *
1672
- * Idempotent the same way {@link classifyPlacement}'s own branches are: a
1673
- * message already sitting in Archive is left alone, not moved again.
1674
- */
1675
- private async resolveAutoArchive(
1676
- mailboxSpecialUseService: IMailboxSpecialUseRepository,
1677
- message: MessageItem,
1678
- accountId: string,
1679
- autoArchive: boolean,
1680
- ): Promise<PlacementOutcome> {
1681
- if (!autoArchive) return {};
1682
-
1683
- const archiveMailbox =
1684
- await mailboxSpecialUseService.findArchiveMailbox(accountId);
1685
- if (!archiveMailbox || message.mailboxId === archiveMailbox.mailboxId) {
1686
- return {};
1687
- }
1688
-
1689
- this.log.info(
1690
- {
1691
- messageId: message.messageId,
1692
- accountId,
1693
- destinationMailboxId: archiveMailbox.mailboxId,
1694
- },
1695
- "Auto-archive verdict",
1696
- );
1697
-
1698
- return {
1699
- move: {
1700
- destinationMailboxId: archiveMailbox.mailboxId,
1701
- destinationPath: archiveMailbox.fullPath,
1702
- },
1703
- };
1704
- }
1705
-
1706
- // The old `enqueuePlacementMove` (best-effort, catch-and-log) lived here.
1707
- // Issue #1271: it ran AFTER `bodyStorageKey` was already durable, so a
1708
- // failure was swallowed to avoid stranding the message behind the
1709
- // body-sync skip guard — but that meant `movedByRemit` could be true with
1710
- // no record the move ever reached IMAP. Replaced by the unswallowed call
1711
- // inside `applyPostStoreSteps`, sequenced BEFORE `bodyStorageKey` is
1712
- // written, backed by `PlacementMoveService`'s pending-move marker.
1713
-
1714
- /**
1715
- * Persist one S3 object per non-multipart leaf so the SPA can resolve
1716
- * `BodyPartResponse.contentUrl` (#298). Keys follow the layout
1717
- * `accounts/{accountConfigId}/{accountId}/messages/{messageId}/parts/{partPath}`
1718
- * so they line up with the URL shape `derive/contentUrl.ts` emits.
1719
- *
1720
- * The mapper is total (#395 PR B): every leaf gets a `BodyPartContentPair`,
1721
- * possibly with a zero-byte content for leaves that have no source bytes
1722
- * (genuinely empty parts, or pathological inputs the positional fallback
1723
- * couldn't pair). No try/catch is needed; the only failure surface here is
1724
- * an S3 write itself, which `pMap` surfaces directly.
1725
- *
1726
- * If `listBodyParts` returns an empty list (e.g. a legacy message synced
1727
- * before #133 populated BodyPart rows), this is a no-op.
1728
- */
1729
- private async storeBodyPartContents(
1730
- accountConfigId: string,
1731
- accountId: string,
1732
- messageId: string,
1733
- bodyParts: readonly BodyPartItem[],
1734
- body: Buffer,
1735
- options?: { skipExisting?: boolean },
1736
- ): Promise<StoreBodyPartContentsResult> {
1737
- if (bodyParts.length === 0) {
1738
- this.log.debug?.(
1739
- { messageId },
1740
- "No BodyPart rows; skipping per-part storage",
1741
- );
1742
- return { stored: 0 };
1743
- }
1744
-
1745
- const log = this.log;
1746
- // A parse of its own: classification and the snippet, which search reads,
1747
- // need an inline forwarded message's text, which `ignoreEmbedded` keeps out.
1748
- const pairs = await mapMessagePartsToContent(
1749
- bodyParts,
1750
- await parsePartSource(body),
1751
- {
1752
- messageId,
1753
- logger:
1754
- log.warn || log.debug
1755
- ? {
1756
- warn: (obj, msg) => {
1757
- log.warn?.(obj, msg);
1758
- },
1759
- debug: (obj, msg) => {
1760
- log.debug?.(obj, msg);
1761
- },
1762
- }
1763
- : undefined,
1764
- },
1765
- );
1766
-
1767
- if (pairs.length === 0) {
1768
- return { stored: 0 };
1769
- }
1770
-
1771
- let stored = 0;
1772
- await pMap(
1773
- pairs,
1774
- async (entry) => {
1775
- if (
1776
- options?.skipExisting &&
1777
- (await this.storageService.bodyPartExists(
1778
- accountConfigId,
1779
- accountId,
1780
- messageId,
1781
- entry.partPath,
1782
- ))
1783
- ) {
1784
- return;
1785
- }
1786
- await this.storageService.storeBodyPart({
1787
- accountConfigId,
1788
- accountId,
1789
- messageId,
1790
- partPath: entry.partPath,
1791
- content: entry.content,
1792
- contentType: entry.contentType,
1793
- });
1794
- stored++;
1795
- },
1796
- { concurrency: BODY_PART_STORE_CONCURRENCY },
1797
- );
1798
-
1799
- this.log.info(
1800
- { messageId, partCount: pairs.length, stored },
1801
- "Body parts stored",
1802
- );
1803
-
1804
- return { stored };
1805
- }
1806
-
1807
- /**
1808
- * Lazily materialize the per-part S3 objects for an already-synced message
1809
- * whose parts were deferred during bulk sync (DEFER_BODY_PARTS).
1810
- *
1811
- * Called from the read path (the API `describeMessage` handler) before the
1812
- * SPA fetches any `contentUrl`: the part bytes are served directly from S3 by
1813
- * CloudFront with no Lambda in the request path, so a missing object would
1814
- * surface to the SPA as a hard "body-missing" failure. Generating the parts
1815
- * here — re-parsing the stored `body.eml` with the same `mapBodyPartsToContent`
1816
- * logic body-sync uses — guarantees every `contentUrl` resolves, while bulk
1817
- * sync stays at 2 writes/message.
1818
- *
1819
- * Idempotent: skips any leaf already on S3, so repeat reads cost only HEAD
1820
- * checks, and a re-read after a partial failure fills the gaps. Requires the
1821
- * message body to be stored (`bodyStorageKey`); callers gate on that.
1822
- *
1823
- * Warm-open fast path: a single HEAD on the `.materialized` sentinel
1824
- * short-circuits the whole pass — no body.eml GET, no MIME re-parse, no
1825
- * per-leaf HEADs. The sentinel is written only after every leaf is confirmed
1826
- * stored, so its presence guarantees all `contentUrl`s resolve.
1827
- */
1828
- async ensureBodyPartsStored(
1829
- accountConfigId: string,
1830
- accountId: string,
1831
- messageId: string,
1832
- bodyStorageKey: string,
1833
- ): Promise<StoreBodyPartContentsResult> {
1834
- const bodyParts = await this.envelopeService.listBodyParts(messageId);
1835
- const carriesMessage = bodyParts.some(isEncapsulatedMessage);
1836
- const sentinel = carriesMessage
1837
- ? MATERIALIZED_SENTINEL_V2_PART_PATH
1838
- : MATERIALIZED_SENTINEL_PART_PATH;
1839
- if (
1840
- await this.storageService.bodyPartExists(
1841
- accountConfigId,
1842
- accountId,
1843
- messageId,
1844
- sentinel,
1845
- )
1846
- ) {
1847
- this.log.debug?.(
1848
- { messageId },
1849
- "Body parts already materialized (sentinel hit); skipping",
1850
- );
1851
- return { stored: 0 };
1852
- }
1853
-
1854
- const body = await this.storageService.retrieve(bodyStorageKey);
1855
- const result = await this.storeBodyPartContents(
1856
- accountConfigId,
1857
- accountId,
1858
- messageId,
1859
- bodyParts,
1860
- body,
1861
- { skipExisting: !carriesMessage },
1862
- );
1863
-
1864
- // Reaching here means storeBodyPartContents resolved — `pMap` rejects on
1865
- // any single failed write, so every leaf is now durably on S3. Drop the
1866
- // sentinel so subsequent opens take the HEAD-only fast path above.
1867
- await this.storageService.storeBodyPart({
1868
- accountConfigId,
1869
- accountId,
1870
- messageId,
1871
- partPath: sentinel,
1872
- content: Buffer.alloc(0),
1873
- });
1874
-
1875
- return result;
1876
- }
1877
-
1878
- private async storeParsedBodyCache(
1879
- accountConfigId: string,
1880
- accountId: string,
1881
- messageId: string,
1882
- parsed: ParsedMail,
1883
- ): Promise<void> {
1884
- await this.storageService.storeParsedBody({
1885
- accountConfigId,
1886
- accountId,
1887
- messageId,
1888
- parsed: toParsedBody(parsed),
1889
- });
1890
- }
1891
- }