@remit/mailbox-service 0.0.111 → 0.0.113

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (298) hide show
  1. package/dist/account-credentials.d.ts +83 -0
  2. package/dist/account-credentials.js +104 -0
  3. package/dist/attribute-mapper.d.ts +36 -0
  4. package/dist/attribute-mapper.js +119 -0
  5. package/dist/body-parse.d.ts +28 -0
  6. package/{src/body-parse.ts → dist/body-parse.js} +18 -26
  7. package/dist/body-part-mapper.d.ts +92 -0
  8. package/dist/body-part-mapper.js +301 -0
  9. package/dist/body-sync-queue.d.ts +59 -0
  10. package/dist/body-sync-queue.js +56 -0
  11. package/dist/body-sync-terminal.d.ts +58 -0
  12. package/dist/body-sync-terminal.js +95 -0
  13. package/dist/body-sync.d.ts +351 -0
  14. package/dist/body-sync.js +1204 -0
  15. package/dist/calendar-parts.d.ts +23 -0
  16. package/dist/calendar-parts.js +18 -0
  17. package/dist/classification-backfill.d.ts +91 -0
  18. package/dist/classification-backfill.js +180 -0
  19. package/dist/classify-message.d.ts +31 -0
  20. package/dist/classify-message.js +89 -0
  21. package/dist/connection-factory.d.ts +54 -0
  22. package/dist/connection-factory.js +54 -0
  23. package/dist/connection-test.d.ts +41 -0
  24. package/dist/connection-test.js +152 -0
  25. package/dist/external-move.d.ts +94 -0
  26. package/dist/external-move.js +173 -0
  27. package/dist/filter-operations.d.ts +42 -0
  28. package/dist/filter-operations.js +211 -0
  29. package/dist/filters/filter-config.d.ts +14 -0
  30. package/dist/filters/filter-config.js +10 -0
  31. package/{src/filters/list-id.ts → dist/filters/list-id.d.ts} +2 -16
  32. package/dist/filters/list-id.js +26 -0
  33. package/dist/filters/match.d.ts +69 -0
  34. package/dist/filters/match.js +165 -0
  35. package/dist/filters/pipeline.d.ts +64 -0
  36. package/dist/filters/pipeline.js +100 -0
  37. package/{src/flag-push-terminal.ts → dist/flag-push-terminal.d.ts} +19 -77
  38. package/dist/flag-push-terminal.js +63 -0
  39. package/dist/flag-push.d.ts +69 -0
  40. package/dist/flag-push.js +111 -0
  41. package/dist/flag-queue.d.ts +175 -0
  42. package/dist/flag-queue.js +291 -0
  43. package/dist/folder-operations.d.ts +39 -0
  44. package/dist/folder-operations.js +112 -0
  45. package/dist/folder-placement.d.ts +13 -0
  46. package/dist/folder-placement.js +23 -0
  47. package/dist/heuristics/authenticationResults.d.ts +10 -0
  48. package/dist/heuristics/authenticationResults.js +187 -0
  49. package/dist/heuristics/classifyByHeaders.d.ts +82 -0
  50. package/dist/heuristics/classifyByHeaders.js +365 -0
  51. package/dist/heuristics/classifyPlacement.d.ts +67 -0
  52. package/dist/heuristics/classifyPlacement.js +153 -0
  53. package/dist/heuristics/machineSenders.d.ts +21 -0
  54. package/{src/heuristics/machineSenders.ts → dist/heuristics/machineSenders.js} +34 -40
  55. package/dist/heuristics/senderMismatch.d.ts +81 -0
  56. package/dist/heuristics/senderMismatch.js +192 -0
  57. package/dist/heuristics/socialDomains.d.ts +14 -0
  58. package/{src/heuristics/socialDomains.ts → dist/heuristics/socialDomains.js} +9 -9
  59. package/dist/heuristics/transactionalDomains.d.ts +18 -0
  60. package/{src/heuristics/transactionalDomains.ts → dist/heuristics/transactionalDomains.js} +8 -8
  61. package/dist/imapflow-connection.d.ts +377 -0
  62. package/dist/imapflow-connection.js +1147 -0
  63. package/dist/index.d.ts +63 -0
  64. package/dist/index.js +65 -0
  65. package/dist/list-id-backfill.d.ts +83 -0
  66. package/dist/list-id-backfill.js +132 -0
  67. package/dist/mailbox-cursor-rebuild.d.ts +74 -0
  68. package/dist/mailbox-cursor-rebuild.js +71 -0
  69. package/{src/mailbox-cursor.ts → dist/mailbox-cursor.d.ts} +20 -98
  70. package/dist/mailbox-cursor.js +102 -0
  71. package/dist/mailbox-intent.d.ts +41 -0
  72. package/{src/mailbox-intent.ts → dist/mailbox-intent.js} +39 -63
  73. package/dist/mailbox-management.d.ts +240 -0
  74. package/dist/mailbox-management.js +576 -0
  75. package/dist/mailbox-presence.d.ts +52 -0
  76. package/{src/mailbox-presence.ts → dist/mailbox-presence.js} +19 -30
  77. package/dist/mailbox-queue.d.ts +126 -0
  78. package/dist/mailbox-queue.js +214 -0
  79. package/dist/mailbox-sync.d.ts +93 -0
  80. package/dist/mailbox-sync.js +490 -0
  81. package/dist/mailbox-upstream.d.ts +38 -0
  82. package/{src/mailbox-upstream.ts → dist/mailbox-upstream.js} +37 -47
  83. package/dist/message-category.d.ts +51 -0
  84. package/{src/message-category.ts → dist/message-category.js} +28 -71
  85. package/dist/message-move.d.ts +285 -0
  86. package/dist/message-move.js +858 -0
  87. package/dist/message-parser.d.ts +14 -0
  88. package/dist/message-parser.js +17 -0
  89. package/{src/message-presence.ts → dist/message-presence.d.ts} +1 -13
  90. package/dist/message-presence.js +29 -0
  91. package/dist/message-read.d.ts +51 -0
  92. package/dist/message-read.js +31 -0
  93. package/dist/message-sync.d.ts +345 -0
  94. package/dist/message-sync.js +1358 -0
  95. package/dist/mime-walker.d.ts +102 -0
  96. package/dist/mime-walker.js +198 -0
  97. package/dist/organize-job.d.ts +20 -0
  98. package/dist/organize-job.js +37 -0
  99. package/dist/outbox-attachment-content.d.ts +37 -0
  100. package/dist/outbox-attachment-content.js +50 -0
  101. package/dist/outbox-attachment-filename.d.ts +19 -0
  102. package/dist/outbox-attachment-filename.js +64 -0
  103. package/dist/outbox-attachment.d.ts +148 -0
  104. package/dist/outbox-attachment.js +273 -0
  105. package/dist/outbox-queue.d.ts +73 -0
  106. package/dist/outbox-queue.js +284 -0
  107. package/{src/outbox-status.ts → dist/outbox-status.d.ts} +1 -6
  108. package/dist/outbox-status.js +18 -0
  109. package/dist/ownership.d.ts +24 -0
  110. package/dist/ownership.js +83 -0
  111. package/dist/placement-move-terminal.d.ts +67 -0
  112. package/dist/placement-move-terminal.js +88 -0
  113. package/dist/placement-move.d.ts +83 -0
  114. package/dist/placement-move.js +216 -0
  115. package/{src/placement-settled.ts → dist/placement-settled.d.ts} +19 -50
  116. package/dist/placement-settled.js +75 -0
  117. package/dist/quarantine.d.ts +95 -0
  118. package/dist/quarantine.js +131 -0
  119. package/dist/refusal.d.ts +31 -0
  120. package/dist/refusal.js +3 -0
  121. package/{src/restore-source-placement.ts → dist/restore-source-placement.d.ts} +29 -101
  122. package/dist/restore-source-placement.js +79 -0
  123. package/dist/search-text-backfill.d.ts +15 -0
  124. package/dist/search-text-backfill.js +33 -0
  125. package/dist/snippet.d.ts +61 -0
  126. package/dist/snippet.js +129 -0
  127. package/dist/spam-report.d.ts +86 -0
  128. package/dist/spam-report.js +241 -0
  129. package/dist/sync-warning.d.ts +21 -0
  130. package/dist/sync-warning.js +26 -0
  131. package/dist/sync-watermarks.d.ts +138 -0
  132. package/dist/sync-watermarks.js +205 -0
  133. package/dist/test-helpers/fake-imap.d.ts +151 -0
  134. package/dist/test-helpers/fake-imap.js +508 -0
  135. package/dist/test-helpers/folder-roles.d.ts +34 -0
  136. package/dist/test-helpers/folder-roles.js +47 -0
  137. package/dist/test-helpers/mime-tree-generator.d.ts +82 -0
  138. package/dist/test-helpers/mime-tree-generator.js +329 -0
  139. package/dist/test-helpers/test-store.d.ts +69 -0
  140. package/dist/test-helpers/test-store.js +193 -0
  141. package/dist/text/normalizer.d.ts +20 -0
  142. package/dist/text/normalizer.js +102 -0
  143. package/dist/thread-message-rows.d.ts +69 -0
  144. package/dist/thread-message-rows.js +40 -0
  145. package/dist/thread-root.d.ts +7 -0
  146. package/dist/thread-root.js +25 -0
  147. package/dist/types.d.ts +343 -0
  148. package/dist/types.js +19 -0
  149. package/package.json +86 -73
  150. package/src/account-credentials.ts +0 -210
  151. package/src/adversarial-mime.e2e.test.ts +0 -307
  152. package/src/attribute-mapper.test.ts +0 -104
  153. package/src/attribute-mapper.ts +0 -148
  154. package/src/body-parse.test.ts +0 -55
  155. package/src/body-part-mapper.property.test.ts +0 -354
  156. package/src/body-part-mapper.ts +0 -360
  157. package/src/body-sync-auto-read-writeonce.test.ts +0 -219
  158. package/src/body-sync-calendar-suggestion.test.ts +0 -433
  159. package/src/body-sync-category-denormalize.test.ts +0 -352
  160. package/src/body-sync-category-override.test.ts +0 -501
  161. package/src/body-sync-category-writeonce.test.ts +0 -234
  162. package/src/body-sync-filter.test.ts +0 -237
  163. package/src/body-sync-message-gone.test.ts +0 -139
  164. package/src/body-sync-placement-flags.test.ts +0 -347
  165. package/src/body-sync-placement-writeonce.test.ts +0 -337
  166. package/src/body-sync-quarantine.test.ts +0 -276
  167. package/src/body-sync-queue.test.ts +0 -124
  168. package/src/body-sync-queue.ts +0 -131
  169. package/src/body-sync-resync.test.ts +0 -605
  170. package/src/body-sync-terminal.test.ts +0 -310
  171. package/src/body-sync-terminal.ts +0 -176
  172. package/src/body-sync-unclassifiable-once.test.ts +0 -260
  173. package/src/body-sync-unsubscribed-auto-read.test.ts +0 -176
  174. package/src/body-sync.ts +0 -1970
  175. package/src/calendar-parts.ts +0 -41
  176. package/src/classification-backfill.test.ts +0 -615
  177. package/src/classification-backfill.ts +0 -363
  178. package/src/classify-message.test.ts +0 -205
  179. package/src/classify-message.ts +0 -116
  180. package/src/connection-factory.ts +0 -102
  181. package/src/connection-test.ts +0 -208
  182. package/src/credentials.test.ts +0 -181
  183. package/src/external-move.test.ts +0 -369
  184. package/src/external-move.ts +0 -272
  185. package/src/filters/anchor-drift.ts +0 -55
  186. package/src/filters/filter-config.test.ts +0 -229
  187. package/src/filters/filter-config.ts +0 -69
  188. package/src/filters/list-id.test.ts +0 -50
  189. package/src/filters/match.test.ts +0 -429
  190. package/src/filters/match.ts +0 -274
  191. package/src/filters/message-embedder.test.ts +0 -39
  192. package/src/filters/message-embedder.ts +0 -33
  193. package/src/filters/pipeline.test.ts +0 -330
  194. package/src/filters/pipeline.ts +0 -284
  195. package/src/flag-push-terminal.test.ts +0 -209
  196. package/src/flag-push.test.ts +0 -408
  197. package/src/flag-push.ts +0 -198
  198. package/src/flag-queue-stars.test.ts +0 -171
  199. package/src/flag-queue.ts +0 -509
  200. package/src/folder-placement.ts +0 -56
  201. package/src/heuristics/authenticationResults.test.ts +0 -189
  202. package/src/heuristics/authenticationResults.ts +0 -212
  203. package/src/heuristics/classifyByHeaders.test.ts +0 -889
  204. package/src/heuristics/classifyByHeaders.ts +0 -467
  205. package/src/heuristics/classifyPlacement.test.ts +0 -366
  206. package/src/heuristics/classifyPlacement.ts +0 -211
  207. package/src/heuristics/classifyRealMail.test.ts +0 -242
  208. package/src/heuristics/machineSenders.test.ts +0 -101
  209. package/src/heuristics/senderMismatch.test.ts +0 -346
  210. package/src/heuristics/senderMismatch.ts +0 -259
  211. package/src/imapflow-connect-retry.test.ts +0 -134
  212. package/src/imapflow-connection.e2e.test.ts +0 -335
  213. package/src/imapflow-connection.integ.test.ts +0 -316
  214. package/src/imapflow-connection.test.ts +0 -469
  215. package/src/imapflow-connection.ts +0 -1475
  216. package/src/index.ts +0 -350
  217. package/src/list-id-backfill.test.ts +0 -573
  218. package/src/list-id-backfill.ts +0 -274
  219. package/src/mailbox-cursor-rebuild.test.ts +0 -171
  220. package/src/mailbox-cursor-rebuild.ts +0 -146
  221. package/src/mailbox-cursor.test.ts +0 -249
  222. package/src/mailbox-delete-intent.test.ts +0 -246
  223. package/src/mailbox-management.test.ts +0 -356
  224. package/src/mailbox-management.ts +0 -839
  225. package/src/mailbox-presence.test.ts +0 -72
  226. package/src/mailbox-queue.ts +0 -415
  227. package/src/mailbox-rename-intent.test.ts +0 -389
  228. package/src/mailbox-rename-subtree.test.ts +0 -664
  229. package/src/mailbox-sync.test.ts +0 -848
  230. package/src/mailbox-sync.ts +0 -724
  231. package/src/message-delete-junk-reconcile.test.ts +0 -117
  232. package/src/message-delete-no-trash.test.ts +0 -245
  233. package/src/message-delete-unsettled-placement.test.ts +0 -396
  234. package/src/message-move-copy.test.ts +0 -562
  235. package/src/message-move-empty-trash.test.ts +0 -326
  236. package/src/message-move-same-mailbox.test.ts +0 -139
  237. package/src/message-move-unsettled-placement.test.ts +0 -378
  238. package/src/message-move.ts +0 -1348
  239. package/src/message-parser.ts +0 -26
  240. package/src/message-presence.test.ts +0 -50
  241. package/src/message-sync-address-spoof.test.ts +0 -293
  242. package/src/message-sync-category.test.ts +0 -142
  243. package/src/message-sync-changedsince.test.ts +0 -840
  244. package/src/message-sync-flags.test.ts +0 -158
  245. package/src/message-sync-junk-harvest.test.ts +0 -244
  246. package/src/message-sync-quarantine.test.ts +0 -416
  247. package/src/message-sync-repoint.test.ts +0 -419
  248. package/src/message-sync.ts +0 -1963
  249. package/src/mime-walker.test.ts +0 -311
  250. package/src/mime-walker.ts +0 -305
  251. package/src/outbox-attachment-content.test.ts +0 -153
  252. package/src/outbox-attachment-content.ts +0 -104
  253. package/src/outbox-attachment-filename.test.ts +0 -132
  254. package/src/outbox-attachment-filename.ts +0 -76
  255. package/src/outbox-attachment.test.ts +0 -738
  256. package/src/outbox-attachment.ts +0 -536
  257. package/src/outbox-failed-recovery.test.ts +0 -370
  258. package/src/outbox-queue-settled-row.test.ts +0 -140
  259. package/src/outbox-queue.test.ts +0 -291
  260. package/src/outbox-queue.ts +0 -508
  261. package/src/pass-through-unit-of-work.ts +0 -17
  262. package/src/placement-move-terminal.test.ts +0 -300
  263. package/src/placement-move-terminal.ts +0 -148
  264. package/src/placement-move-unsettled.test.ts +0 -275
  265. package/src/placement-move.ts +0 -336
  266. package/src/placement-settled.test.ts +0 -147
  267. package/src/placement-transition-race.test.ts +0 -482
  268. package/src/quarantine.test.ts +0 -191
  269. package/src/quarantine.ts +0 -228
  270. package/src/snippet.test.ts +0 -213
  271. package/src/snippet.ts +0 -152
  272. package/src/spam-report.test.ts +0 -801
  273. package/src/spam-report.ts +0 -357
  274. package/src/stale-message-reconcile.test.ts +0 -82
  275. package/src/stale-message-reconcile.ts +0 -55
  276. package/src/sync-watermarks.test.ts +0 -409
  277. package/src/sync-watermarks.ts +0 -319
  278. package/src/system-flag-wire-format.test.ts +0 -77
  279. package/src/test-helpers/folder-roles.ts +0 -66
  280. package/src/test-helpers/isolated-mailbox.ts +0 -44
  281. package/src/test-helpers/mailfuzz-connection.ts +0 -27
  282. package/src/test-helpers/mime-tree-generator.ts +0 -461
  283. package/src/text/normalizer.test.ts +0 -223
  284. package/src/text/normalizer.ts +0 -140
  285. package/src/thread-message-rows.ts +0 -77
  286. package/src/types/mailparser-augment.d.ts +0 -15
  287. package/src/types/natural-porter-stemmer.d.ts +0 -50
  288. package/src/types/stopword.d.ts +0 -69
  289. package/src/types.ts +0 -371
  290. package/test/fixtures/mime-adversarial-e2e/01-octet-stream-pdf.eml +0 -21
  291. package/test/fixtures/mime-adversarial-e2e/02-nested-related-cid-image.eml +0 -31
  292. package/test/fixtures/mime-adversarial-e2e/03-calendar-invite.eml +0 -33
  293. package/test/fixtures/mime-adversarial-e2e/04-forwarded-rfc822-with-attachments.eml +0 -42
  294. package/test/fixtures/mime-adversarial-e2e/05-tnef-winmail.eml +0 -25
  295. package/test/fixtures/mime-adversarial-e2e/06-8bit-non-ascii.eml +0 -15
  296. package/test/fixtures/mime-adversarial-e2e/07-quoted-printable-soft-breaks.eml +0 -17
  297. package/test/fixtures/mime-adversarial-e2e/08-alternative-empty-text.eml +0 -18
  298. package/tsconfig.json +0 -8
@@ -0,0 +1,301 @@
1
+ /**
2
+ * Pair every persisted `BodyPart` row with the decoded bytes mailparser
3
+ * produced for the same MIME node. Total function: returns one
4
+ * `BodyPartContentPair` per non-multipart leaf, never undefined, never
5
+ * throws on well-formed input.
6
+ *
7
+ * Source of truth (per issue #395):
8
+ * - `partPath`, `contentType`, `contentId`, `dispositionFilename`,
9
+ * `sizeOctets` — owned by BODYSTRUCTURE (the `BodyPart` row).
10
+ * - Decoded bytes — owned by mailparser (`parsed.text`, `parsed.html`,
11
+ * `parsed.attachments[i].content`).
12
+ * - `parsed.attachments[i].partId` is also dot-numbered (set at runtime
13
+ * in `mail-parser.js:860`; missing from `@types/mailparser`; surfaced
14
+ * here via `./types/mailparser-augment.d.ts`). Structural pairing on
15
+ * `partId === partPath` is the dominant strategy for non-text leaves.
16
+ *
17
+ * Pipeline (per leaf in declaration order; each step consumes one
18
+ * attachment when it pairs, subsequent steps work on the residual set):
19
+ * 1. text/plain → first leaf gets `parsed.text` (empty Buffer if
20
+ * `sizeOctets === 0`); subsequent text/plain leaves get empty.
21
+ * 2. text/html → first leaf gets `parsed.html` (empty Buffer if
22
+ * `sizeOctets === 0`); subsequent text/html leaves get empty.
23
+ * 3. Non-text leaves with `sizeOctets === 0` short-circuit to an empty
24
+ * Buffer without consuming an attachment — symmetric with the text
25
+ * path, so the positional fallback can't hand a zero-sized leaf real
26
+ * bytes that belong to another leaf.
27
+ * 4. Remaining non-text leaves try, in order:
28
+ * a. `attachment.partId === bodyPart.partPath`
29
+ * b. contentId match (case + angle-insensitive)
30
+ * c. dispositionFilename match (case-insensitive)
31
+ * d. content-type compatibility class: exact → binary-family
32
+ * (any non-text/*) → text-family
33
+ * e. positional fallback: next unconsumed attachment in declaration
34
+ * order. Logs at `debug` so operators can see "we guessed"
35
+ * pairings without spamming higher levels.
36
+ * 5. Anything still unpaired → zero-byte Buffer + structured `warn` log.
37
+ *
38
+ * The mapper assumes well-formed input — malformed MIME is rejected
39
+ * upstream in `mime-walker.ts`. If a leaf truly has no source bytes
40
+ * available (e.g. mailparser tolerated a malformed message that
41
+ * BODYSTRUCTURE disagrees with on leaf count), it gets the empty-Buffer
42
+ * fallback rather than a throw, so the call site stays simple.
43
+ */
44
+ /// <reference path="./types/mailparser-augment.d.ts" />
45
+ import { Buffer } from "node:buffer";
46
+ import { encapsulatedParts, outermostMessages, ownParts, } from "@remit/data-ports/message-parts";
47
+ import { simpleParser } from "mailparser";
48
+ const EMPTY_BUFFER = Buffer.alloc(0);
49
+ const stripAngles = (value) => value.replace(/^<+|>+$/g, "");
50
+ const buildContentType = (part) => `${part.mediaType.toLowerCase()}/${part.mediaSubtype}`;
51
+ const textToBuffer = (value) => {
52
+ if (typeof value !== "string" || value.length === 0)
53
+ return EMPTY_BUFFER;
54
+ return Buffer.from(value, "utf8");
55
+ };
56
+ const isTextStar = (contentType) => contentType.toLowerCase().startsWith("text/");
57
+ /** Match `partId === partPath` against an unconsumed attachment. */
58
+ const findByPartId = (attachments, consumed, partPath) => {
59
+ for (let i = 0; i < attachments.length; i++) {
60
+ if (consumed.has(i))
61
+ continue;
62
+ const att = attachments[i];
63
+ if (typeof att.partId !== "string")
64
+ continue;
65
+ if (att.partId === partPath)
66
+ return i;
67
+ }
68
+ return null;
69
+ };
70
+ /** Match by contentId (case + angle-insensitive). */
71
+ const findByContentId = (attachments, consumed, contentId) => {
72
+ const target = stripAngles(contentId).toLowerCase();
73
+ for (let i = 0; i < attachments.length; i++) {
74
+ if (consumed.has(i))
75
+ continue;
76
+ const att = attachments[i];
77
+ const candidates = [att.cid, att.contentId];
78
+ for (const cand of candidates) {
79
+ if (typeof cand !== "string")
80
+ continue;
81
+ if (stripAngles(cand).toLowerCase() === target)
82
+ return i;
83
+ }
84
+ }
85
+ return null;
86
+ };
87
+ /** Match by dispositionFilename (case-insensitive). */
88
+ const findByFilename = (attachments, consumed, filename) => {
89
+ const target = filename.toLowerCase();
90
+ for (let i = 0; i < attachments.length; i++) {
91
+ if (consumed.has(i))
92
+ continue;
93
+ const att = attachments[i];
94
+ if (typeof att.filename !== "string")
95
+ continue;
96
+ if (att.filename.toLowerCase() === target)
97
+ return i;
98
+ }
99
+ return null;
100
+ };
101
+ /**
102
+ * Content-type compatibility class. The BodyPart row's content-type is the
103
+ * sender's verbatim label; mailparser's `att.contentType` is sniffed from
104
+ * filename extension. They often disagree (the Odido bug — postmortem
105
+ * #394 — is `application/octet-stream` vs `application/pdf`).
106
+ *
107
+ * Order:
108
+ * 1. Exact match (`application/pdf` ↔ `application/pdf`).
109
+ * 2. Binary family — leaf is non-`text/*`, attachment is non-`text/*`.
110
+ * Covers octet-stream ↔ pdf, octet-stream ↔ image, etc.
111
+ * 3. Text family — leaf is `text/*`, attachment is `text/*`. Catches
112
+ * `text/calendar` / `text/csv` after the inline `text/plain` and
113
+ * `text/html` slots are taken.
114
+ */
115
+ const findByContentTypeClass = (attachments, consumed, contentType) => {
116
+ const target = contentType.toLowerCase();
117
+ const leafIsText = isTextStar(target);
118
+ for (let i = 0; i < attachments.length; i++) {
119
+ if (consumed.has(i))
120
+ continue;
121
+ const att = attachments[i];
122
+ if (att.contentType?.toLowerCase() === target)
123
+ return i;
124
+ }
125
+ if (!leafIsText) {
126
+ for (let i = 0; i < attachments.length; i++) {
127
+ if (consumed.has(i))
128
+ continue;
129
+ const att = attachments[i];
130
+ const attType = att.contentType?.toLowerCase();
131
+ if (!attType)
132
+ continue;
133
+ if (!isTextStar(attType))
134
+ return i;
135
+ }
136
+ }
137
+ else {
138
+ for (let i = 0; i < attachments.length; i++) {
139
+ if (consumed.has(i))
140
+ continue;
141
+ const att = attachments[i];
142
+ const attType = att.contentType?.toLowerCase();
143
+ if (!attType)
144
+ continue;
145
+ if (isTextStar(attType))
146
+ return i;
147
+ }
148
+ }
149
+ return null;
150
+ };
151
+ /** Positional fallback: next unconsumed attachment in declaration order. */
152
+ const findPositional = (attachments, consumed) => {
153
+ for (let i = 0; i < attachments.length; i++) {
154
+ if (!consumed.has(i))
155
+ return i;
156
+ }
157
+ return null;
158
+ };
159
+ const pairNonTextLeaf = (part, attachments, consumed, context) => {
160
+ const contentType = buildContentType(part);
161
+ // Symmetry with the text-leaf path: a leaf whose BODYSTRUCTURE-declared
162
+ // size is zero has no bytes to find, so do not consume a residual
163
+ // attachment for it (the positional fallback would otherwise hand it
164
+ // real bytes belonging to another leaf — see PR D, review note b).
165
+ if (part.sizeOctets === 0) {
166
+ return { content: EMPTY_BUFFER, paired: true };
167
+ }
168
+ const byPartId = findByPartId(attachments, consumed, part.partPath);
169
+ if (byPartId !== null) {
170
+ consumed.add(byPartId);
171
+ return { content: attachments[byPartId].content, paired: true };
172
+ }
173
+ if (part.contentId) {
174
+ const byCid = findByContentId(attachments, consumed, part.contentId);
175
+ if (byCid !== null) {
176
+ consumed.add(byCid);
177
+ return { content: attachments[byCid].content, paired: true };
178
+ }
179
+ }
180
+ if (part.dispositionFilename) {
181
+ const byName = findByFilename(attachments, consumed, part.dispositionFilename);
182
+ if (byName !== null) {
183
+ consumed.add(byName);
184
+ return { content: attachments[byName].content, paired: true };
185
+ }
186
+ }
187
+ const byType = findByContentTypeClass(attachments, consumed, contentType);
188
+ if (byType !== null) {
189
+ consumed.add(byType);
190
+ return { content: attachments[byType].content, paired: true };
191
+ }
192
+ const positional = findPositional(attachments, consumed);
193
+ if (positional !== null) {
194
+ consumed.add(positional);
195
+ const chosen = attachments[positional];
196
+ // Audit log so operators can see "we guessed" pairings when a
197
+ // well-formed message takes the structural-fallback path. Helps
198
+ // triage cases where mailparser and BODYSTRUCTURE disagree more
199
+ // than the earlier matchers can handle.
200
+ context.logger?.debug?.({
201
+ messageId: context.messageId,
202
+ partPath: part.partPath,
203
+ leafContentType: contentType,
204
+ chosenAttachmentPartId: chosen.partId ?? null,
205
+ chosenAttachmentContentType: chosen.contentType ?? null,
206
+ chosenAttachmentFilename: chosen.filename ?? null,
207
+ }, "body-part-mapper: positional fallback paired leaf with next residual attachment");
208
+ return { content: chosen.content, paired: true };
209
+ }
210
+ return { paired: false };
211
+ };
212
+ /**
213
+ * Pair body-part rows with mailparser bytes. Returns one pair per
214
+ * non-multipart leaf, in the order the leaves appear in `bodyParts`.
215
+ *
216
+ * Never throws on well-formed input. Leaves with no source bytes get an
217
+ * empty Buffer; the call site does not need a try/catch.
218
+ */
219
+ export const mapBodyPartsToContent = (bodyParts, parsed, context = {}) => {
220
+ const pairs = [];
221
+ const attachments = parsed.attachments ?? [];
222
+ const consumed = new Set();
223
+ let textRouted = false;
224
+ let htmlRouted = false;
225
+ for (const part of bodyParts) {
226
+ if (part.isMultipart)
227
+ continue;
228
+ const contentType = buildContentType(part);
229
+ const isPlain = part.mediaType === "TEXT" && part.mediaSubtype === "plain";
230
+ const isHtml = part.mediaType === "TEXT" && part.mediaSubtype === "html";
231
+ // `sizeOctets === 0` is the explicit empty-leaf signal; an undefined
232
+ // sizeOctets (hand-built test input) is treated as non-empty so we
233
+ // fall through to the text-routing path.
234
+ const isEmptyLeaf = part.sizeOctets === 0;
235
+ if (isPlain) {
236
+ const content = !textRouted && !isEmptyLeaf ? textToBuffer(parsed.text) : EMPTY_BUFFER;
237
+ textRouted = true;
238
+ pairs.push({ partPath: part.partPath, contentType, content });
239
+ continue;
240
+ }
241
+ if (isHtml) {
242
+ const content = !htmlRouted && !isEmptyLeaf
243
+ ? textToBuffer(typeof parsed.html === "string" ? parsed.html : undefined)
244
+ : EMPTY_BUFFER;
245
+ htmlRouted = true;
246
+ pairs.push({ partPath: part.partPath, contentType, content });
247
+ continue;
248
+ }
249
+ const result = pairNonTextLeaf(part, attachments, consumed, context);
250
+ if (result.paired) {
251
+ pairs.push({
252
+ partPath: part.partPath,
253
+ contentType,
254
+ content: result.content,
255
+ });
256
+ continue;
257
+ }
258
+ context.logger?.warn({
259
+ messageId: context.messageId,
260
+ partPath: part.partPath,
261
+ contentType,
262
+ contentId: part.contentId,
263
+ filename: part.dispositionFilename,
264
+ }, "body-part-mapper: no source bytes for leaf; pairing with zero-byte buffer");
265
+ pairs.push({ partPath: part.partPath, contentType, content: EMPTY_BUFFER });
266
+ }
267
+ return pairs;
268
+ };
269
+ /**
270
+ * The parse every part's bytes are paired against. A message/rfc822 part stays
271
+ * one leaf holding the raw forwarded message whatever its disposition, so the
272
+ * forwarded message's own parts come only from parsing it in turn.
273
+ */
274
+ export const parsePartSource = (source) => simpleParser(source, { ignoreEmbedded: true });
275
+ /**
276
+ * `mapBodyPartsToContent`, extended to forwarded messages. mailparser hands a
277
+ * message/rfc822 part over as one attachment holding the raw forwarded
278
+ * message, so the parts inside it have no bytes in `parsed`. Each forwarded
279
+ * message is parsed on its own and its parts are paired against that parse,
280
+ * with their paths rebased so `<messagePath>.0` is the forwarded message's
281
+ * root — the shape `walkMimeStructure` stores.
282
+ */
283
+ export const mapMessagePartsToContent = async (bodyParts, parsed, context = {}) => {
284
+ const messages = outermostMessages(bodyParts);
285
+ const pairs = mapBodyPartsToContent(ownParts(bodyParts), parsed, context);
286
+ for (const message of messages) {
287
+ const raw = pairs.find((pair) => pair.partPath === message.partPath);
288
+ const inner = encapsulatedParts(bodyParts, message.partPath);
289
+ if (!raw || inner.length === 0)
290
+ continue;
291
+ const prefix = `${message.partPath}.`;
292
+ const nested = await mapMessagePartsToContent(inner.map((part) => ({
293
+ ...part,
294
+ partPath: part.partPath.slice(prefix.length),
295
+ })), await parsePartSource(raw.content), context);
296
+ for (const pair of nested) {
297
+ pairs.push({ ...pair, partPath: `${prefix}${pair.partPath}` });
298
+ }
299
+ }
300
+ return pairs;
301
+ };
@@ -0,0 +1,59 @@
1
+ import type { QueueProducer } from "@remit/data-ports";
2
+ /**
3
+ * SYNC_MESSAGE_BODY event structure (matches remit-imap-worker/events.ts). The
4
+ * consumer falls back to `messageIds` when `messages` (id+uid pairs) is absent.
5
+ */
6
+ export interface SyncMessageBodyTarget {
7
+ messageId: string;
8
+ uid: number;
9
+ }
10
+ export interface SyncMessageBodyEvent {
11
+ type: "SYNC_MESSAGE_BODY";
12
+ eventId: string;
13
+ timestamp: number;
14
+ accountId: string;
15
+ mailboxId: string;
16
+ messageIds: string[];
17
+ messages?: SyncMessageBodyTarget[];
18
+ /**
19
+ * Always true for this read-miss re-arm cue: the caller only reaches
20
+ * `requestBodySync` when a `/content` read found `bodyStorageKey` set but
21
+ * the storage object missing, so the worker must bypass its "already
22
+ * stored" skip guard and re-fetch even though the DB row is stale.
23
+ */
24
+ force: true;
25
+ }
26
+ export interface BodySyncQueueLogger {
27
+ info(obj: Record<string, unknown>, msg: string): void;
28
+ error(obj: Record<string, unknown>, msg: string): void;
29
+ }
30
+ export interface BodySyncQueueConfig {
31
+ queue: QueueProducer<SyncMessageBodyEvent>;
32
+ logger?: BodySyncQueueLogger;
33
+ }
34
+ export interface RequestBodySyncInput {
35
+ accountId: string;
36
+ mailboxId: string;
37
+ messageId: string;
38
+ /** UID resolved at read time so the worker issues one FETCH without a lookup. */
39
+ uid?: number;
40
+ }
41
+ /**
42
+ * Re-arms the body-sync cue for a single message from the read path.
43
+ *
44
+ * The worker emits SYNC_MESSAGE_BODY during metadata sync; this service lets the
45
+ * API re-emit that cue on demand when a body-fetch finds the storage object
46
+ * missing (never synced, or lost). The subsequent client retry then resolves
47
+ * once the worker has stored the body.
48
+ *
49
+ * A queue-send failure never rejects: the read path already returns a retryable
50
+ * 202 to the caller, and a rejection here would land on the shared API event
51
+ * loop and fail an unrelated in-flight request. It is logged loudly with an
52
+ * alertable field instead.
53
+ */
54
+ export declare class BodySyncQueueService {
55
+ private queue;
56
+ private log;
57
+ constructor(config: BodySyncQueueConfig);
58
+ requestBodySync: (input: RequestBodySyncInput) => Promise<void>;
59
+ }
@@ -0,0 +1,56 @@
1
+ import { randomUUID } from "node:crypto";
2
+ const noopLogger = {
3
+ info: () => { },
4
+ error: () => { },
5
+ };
6
+ /**
7
+ * Re-arms the body-sync cue for a single message from the read path.
8
+ *
9
+ * The worker emits SYNC_MESSAGE_BODY during metadata sync; this service lets the
10
+ * API re-emit that cue on demand when a body-fetch finds the storage object
11
+ * missing (never synced, or lost). The subsequent client retry then resolves
12
+ * once the worker has stored the body.
13
+ *
14
+ * A queue-send failure never rejects: the read path already returns a retryable
15
+ * 202 to the caller, and a rejection here would land on the shared API event
16
+ * loop and fail an unrelated in-flight request. It is logged loudly with an
17
+ * alertable field instead.
18
+ */
19
+ export class BodySyncQueueService {
20
+ queue;
21
+ log;
22
+ constructor(config) {
23
+ this.queue = config.queue;
24
+ this.log = config.logger ?? noopLogger;
25
+ }
26
+ requestBodySync = async (input) => {
27
+ const { accountId, mailboxId, messageId, uid } = input;
28
+ const event = {
29
+ type: "SYNC_MESSAGE_BODY",
30
+ eventId: randomUUID(),
31
+ timestamp: Date.now(),
32
+ accountId,
33
+ mailboxId,
34
+ messageIds: [messageId],
35
+ ...(uid !== undefined && { messages: [{ messageId, uid }] }),
36
+ force: true,
37
+ };
38
+ await this.queue
39
+ .send({ body: event })
40
+ .then(() => {
41
+ this.log.info({ eventId: event.eventId, accountId, mailboxId, messageId }, "Re-armed SYNC_MESSAGE_BODY cue");
42
+ })
43
+ .catch((error) => {
44
+ this.log.error({
45
+ alert: "body_sync_cue_enqueue_failed",
46
+ eventId: event.eventId,
47
+ accountId,
48
+ mailboxId,
49
+ messageId,
50
+ errorName: error?.name,
51
+ errorCode: error?.Code ??
52
+ error?.code,
53
+ }, "Failed to re-arm SYNC_MESSAGE_BODY cue (read path returned a retryable 202)");
54
+ });
55
+ };
56
+ }
@@ -0,0 +1,58 @@
1
+ import type { IMessageRepository } from "@remit/data-ports";
2
+ import type { StorageService } from "@remit/storage-service";
3
+ import type { BodySyncLogger } from "./body-sync.js";
4
+ import type { IImapConnection } from "./types.js";
5
+ export interface ResolveExhaustedBodySyncDeps {
6
+ messageService: Pick<IMessageRepository, "get" | "deleteIfPlacement">;
7
+ storageService: Pick<StorageService, "storeBodyPart" | "bodyPartExists">;
8
+ log: BodySyncLogger;
9
+ }
10
+ export interface ResolveExhaustedBodySyncInput {
11
+ accountId: string;
12
+ accountConfigId: string;
13
+ mailboxId: string;
14
+ mailboxPath: string;
15
+ failedMessageIds: string[];
16
+ getConnection: () => Promise<IImapConnection>;
17
+ }
18
+ export interface ResolveExhaustedBodySyncResult {
19
+ /** Outcome 1 (EXPECTED): message gone upstream, row deleted. */
20
+ reconciledMessageIds: string[];
21
+ /** Outcome 2 (BROKEN): message exists but is unfetchable/unparseable. */
22
+ brokenMessageIds: string[];
23
+ }
24
+ /**
25
+ * Whether a message's body-sync has already been marked permanently failed
26
+ * (outcome 2 of {@link resolveExhaustedBodySyncFailures}). The read path
27
+ * calls this before attempting or re-arming any fetch so a client that opens
28
+ * a broken message gets the explicit unrecoverable error immediately instead
29
+ * of looping through another 202 or another IMAP round-trip.
30
+ */
31
+ export declare const isMessageBodySyncBroken: (storageService: Pick<StorageService, "bodyPartExists">, accountConfigId: string, accountId: string, messageId: string) => Promise<boolean>;
32
+ /**
33
+ * Resolve every SYNC_MESSAGE_BODY failure that has exhausted the body
34
+ * queue's redelivery budget (see `BODY_SYNC_MAX_ATTEMPTS` in
35
+ * `sync-message-body.ts`) into exactly one of the two terminal outcomes
36
+ * issue #1270 / epic #1281 invariant 3 require. There is no third, softer
37
+ * outcome — every failed id lands in one of the two result lists.
38
+ *
39
+ * 1. EXPECTED — the message no longer exists on IMAP (expunged, or a
40
+ * UIDVALIDITY change moved it, #1272), confirmed by
41
+ * {@link isMessageGoneFromOpenMailbox}. This loop is the most exposed of
42
+ * the four sites that used to read an empty FETCH as absence: it issues
43
+ * back-to-back single-UID FETCHes on one connection, which is the
44
+ * condition imapflow #408 drops rows under. The stale row is deleted via
45
+ * `deleteIfPlacement` so the existing missing-row 404 path
46
+ * takes over. Callers should emit a metric only — this is routine, not an
47
+ * incident.
48
+ * 2. BROKEN — the message still exists on IMAP but its body could not be
49
+ * fetched or parsed. The failure is persisted as a body-part sentinel
50
+ * (survives across invocations and warm/cold starts) and logged with an
51
+ * `alert`-shaped entry so an operator alarm can key off it. Callers
52
+ * should treat this as loud: the message is unrecoverable until an
53
+ * operator investigates.
54
+ *
55
+ * Neither outcome re-enqueues: both are terminal, so the caller acks the SQS
56
+ * message either way — retrying a stale or broken message can never succeed.
57
+ */
58
+ export declare const resolveExhaustedBodySyncFailures: (deps: ResolveExhaustedBodySyncDeps, input: ResolveExhaustedBodySyncInput) => Promise<ResolveExhaustedBodySyncResult>;
@@ -0,0 +1,95 @@
1
+ import { isMessageGoneFromOpenMailbox } from "./message-presence.js";
2
+ /**
3
+ * Body-part sentinel path a permanently-broken message's failure context is
4
+ * written under, mirroring the `.materialized` deferred-parts sentinel in
5
+ * `body-sync.ts`. Using the existing per-message body-part storage (instead
6
+ * of a new Message-row column) means no schema change: the read path checks
7
+ * for it with the same `bodyPartExists` HEAD it already issues.
8
+ */
9
+ const BODY_SYNC_FAILED_MARKER_PATH = ".sync-failed";
10
+ /**
11
+ * Whether a message's body-sync has already been marked permanently failed
12
+ * (outcome 2 of {@link resolveExhaustedBodySyncFailures}). The read path
13
+ * calls this before attempting or re-arming any fetch so a client that opens
14
+ * a broken message gets the explicit unrecoverable error immediately instead
15
+ * of looping through another 202 or another IMAP round-trip.
16
+ */
17
+ export const isMessageBodySyncBroken = (storageService, accountConfigId, accountId, messageId) => storageService.bodyPartExists(accountConfigId, accountId, messageId, BODY_SYNC_FAILED_MARKER_PATH);
18
+ const markMessageBodySyncFailed = async (storageService, accountConfigId, accountId, messageId, context) => {
19
+ await storageService.storeBodyPart({
20
+ accountConfigId,
21
+ accountId,
22
+ messageId,
23
+ partPath: BODY_SYNC_FAILED_MARKER_PATH,
24
+ content: Buffer.from(JSON.stringify({ ...context, failedAt: Date.now() })),
25
+ contentType: "application/json",
26
+ });
27
+ };
28
+ /**
29
+ * Resolve every SYNC_MESSAGE_BODY failure that has exhausted the body
30
+ * queue's redelivery budget (see `BODY_SYNC_MAX_ATTEMPTS` in
31
+ * `sync-message-body.ts`) into exactly one of the two terminal outcomes
32
+ * issue #1270 / epic #1281 invariant 3 require. There is no third, softer
33
+ * outcome — every failed id lands in one of the two result lists.
34
+ *
35
+ * 1. EXPECTED — the message no longer exists on IMAP (expunged, or a
36
+ * UIDVALIDITY change moved it, #1272), confirmed by
37
+ * {@link isMessageGoneFromOpenMailbox}. This loop is the most exposed of
38
+ * the four sites that used to read an empty FETCH as absence: it issues
39
+ * back-to-back single-UID FETCHes on one connection, which is the
40
+ * condition imapflow #408 drops rows under. The stale row is deleted via
41
+ * `deleteIfPlacement` so the existing missing-row 404 path
42
+ * takes over. Callers should emit a metric only — this is routine, not an
43
+ * incident.
44
+ * 2. BROKEN — the message still exists on IMAP but its body could not be
45
+ * fetched or parsed. The failure is persisted as a body-part sentinel
46
+ * (survives across invocations and warm/cold starts) and logged with an
47
+ * `alert`-shaped entry so an operator alarm can key off it. Callers
48
+ * should treat this as loud: the message is unrecoverable until an
49
+ * operator investigates.
50
+ *
51
+ * Neither outcome re-enqueues: both are terminal, so the caller acks the SQS
52
+ * message either way — retrying a stale or broken message can never succeed.
53
+ */
54
+ export const resolveExhaustedBodySyncFailures = async (deps, input) => {
55
+ const { accountId, accountConfigId, mailboxId, mailboxPath, failedMessageIds, getConnection, } = input;
56
+ const reconciledMessageIds = [];
57
+ const brokenMessageIds = [];
58
+ if (failedMessageIds.length === 0) {
59
+ return { reconciledMessageIds, brokenMessageIds };
60
+ }
61
+ const connection = await getConnection();
62
+ await connection.openBox(mailboxPath);
63
+ for (const messageId of failedMessageIds) {
64
+ const message = await deps.messageService.get(messageId);
65
+ if (await isMessageGoneFromOpenMailbox(connection, message.uid)) {
66
+ const reconciled = await deps.messageService.deleteIfPlacement(messageId, {
67
+ status: message.status,
68
+ mailboxId: message.mailboxId,
69
+ uid: message.uid,
70
+ });
71
+ deps.log.info({
72
+ metric: "body_sync_stale_row_reconciled",
73
+ accountId,
74
+ accountConfigId,
75
+ mailboxId,
76
+ messageId,
77
+ uid: message.uid,
78
+ reconciled,
79
+ }, "Message no longer exists on IMAP after retry exhaustion; stale row reconciled");
80
+ reconciledMessageIds.push(messageId);
81
+ continue;
82
+ }
83
+ await markMessageBodySyncFailed(deps.storageService, accountConfigId, accountId, messageId, { accountId, accountConfigId, mailboxId, messageId, uid: message.uid });
84
+ deps.log.error?.({
85
+ alert: "body_sync_message_broken",
86
+ accountId,
87
+ accountConfigId,
88
+ mailboxId,
89
+ messageId,
90
+ uid: message.uid,
91
+ }, "Message body could not be fetched/parsed after retry exhaustion; message exists on IMAP but is unrecoverable");
92
+ brokenMessageIds.push(messageId);
93
+ }
94
+ return { reconciledMessageIds, brokenMessageIds };
95
+ };