@remit/mailbox-service 0.0.111 → 0.0.113

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (298) hide show
  1. package/dist/account-credentials.d.ts +83 -0
  2. package/dist/account-credentials.js +104 -0
  3. package/dist/attribute-mapper.d.ts +36 -0
  4. package/dist/attribute-mapper.js +119 -0
  5. package/dist/body-parse.d.ts +28 -0
  6. package/{src/body-parse.ts → dist/body-parse.js} +18 -26
  7. package/dist/body-part-mapper.d.ts +92 -0
  8. package/dist/body-part-mapper.js +301 -0
  9. package/dist/body-sync-queue.d.ts +59 -0
  10. package/dist/body-sync-queue.js +56 -0
  11. package/dist/body-sync-terminal.d.ts +58 -0
  12. package/dist/body-sync-terminal.js +95 -0
  13. package/dist/body-sync.d.ts +351 -0
  14. package/dist/body-sync.js +1204 -0
  15. package/dist/calendar-parts.d.ts +23 -0
  16. package/dist/calendar-parts.js +18 -0
  17. package/dist/classification-backfill.d.ts +91 -0
  18. package/dist/classification-backfill.js +180 -0
  19. package/dist/classify-message.d.ts +31 -0
  20. package/dist/classify-message.js +89 -0
  21. package/dist/connection-factory.d.ts +54 -0
  22. package/dist/connection-factory.js +54 -0
  23. package/dist/connection-test.d.ts +41 -0
  24. package/dist/connection-test.js +152 -0
  25. package/dist/external-move.d.ts +94 -0
  26. package/dist/external-move.js +173 -0
  27. package/dist/filter-operations.d.ts +42 -0
  28. package/dist/filter-operations.js +211 -0
  29. package/dist/filters/filter-config.d.ts +14 -0
  30. package/dist/filters/filter-config.js +10 -0
  31. package/{src/filters/list-id.ts → dist/filters/list-id.d.ts} +2 -16
  32. package/dist/filters/list-id.js +26 -0
  33. package/dist/filters/match.d.ts +69 -0
  34. package/dist/filters/match.js +165 -0
  35. package/dist/filters/pipeline.d.ts +64 -0
  36. package/dist/filters/pipeline.js +100 -0
  37. package/{src/flag-push-terminal.ts → dist/flag-push-terminal.d.ts} +19 -77
  38. package/dist/flag-push-terminal.js +63 -0
  39. package/dist/flag-push.d.ts +69 -0
  40. package/dist/flag-push.js +111 -0
  41. package/dist/flag-queue.d.ts +175 -0
  42. package/dist/flag-queue.js +291 -0
  43. package/dist/folder-operations.d.ts +39 -0
  44. package/dist/folder-operations.js +112 -0
  45. package/dist/folder-placement.d.ts +13 -0
  46. package/dist/folder-placement.js +23 -0
  47. package/dist/heuristics/authenticationResults.d.ts +10 -0
  48. package/dist/heuristics/authenticationResults.js +187 -0
  49. package/dist/heuristics/classifyByHeaders.d.ts +82 -0
  50. package/dist/heuristics/classifyByHeaders.js +365 -0
  51. package/dist/heuristics/classifyPlacement.d.ts +67 -0
  52. package/dist/heuristics/classifyPlacement.js +153 -0
  53. package/dist/heuristics/machineSenders.d.ts +21 -0
  54. package/{src/heuristics/machineSenders.ts → dist/heuristics/machineSenders.js} +34 -40
  55. package/dist/heuristics/senderMismatch.d.ts +81 -0
  56. package/dist/heuristics/senderMismatch.js +192 -0
  57. package/dist/heuristics/socialDomains.d.ts +14 -0
  58. package/{src/heuristics/socialDomains.ts → dist/heuristics/socialDomains.js} +9 -9
  59. package/dist/heuristics/transactionalDomains.d.ts +18 -0
  60. package/{src/heuristics/transactionalDomains.ts → dist/heuristics/transactionalDomains.js} +8 -8
  61. package/dist/imapflow-connection.d.ts +377 -0
  62. package/dist/imapflow-connection.js +1147 -0
  63. package/dist/index.d.ts +63 -0
  64. package/dist/index.js +65 -0
  65. package/dist/list-id-backfill.d.ts +83 -0
  66. package/dist/list-id-backfill.js +132 -0
  67. package/dist/mailbox-cursor-rebuild.d.ts +74 -0
  68. package/dist/mailbox-cursor-rebuild.js +71 -0
  69. package/{src/mailbox-cursor.ts → dist/mailbox-cursor.d.ts} +20 -98
  70. package/dist/mailbox-cursor.js +102 -0
  71. package/dist/mailbox-intent.d.ts +41 -0
  72. package/{src/mailbox-intent.ts → dist/mailbox-intent.js} +39 -63
  73. package/dist/mailbox-management.d.ts +240 -0
  74. package/dist/mailbox-management.js +576 -0
  75. package/dist/mailbox-presence.d.ts +52 -0
  76. package/{src/mailbox-presence.ts → dist/mailbox-presence.js} +19 -30
  77. package/dist/mailbox-queue.d.ts +126 -0
  78. package/dist/mailbox-queue.js +214 -0
  79. package/dist/mailbox-sync.d.ts +93 -0
  80. package/dist/mailbox-sync.js +490 -0
  81. package/dist/mailbox-upstream.d.ts +38 -0
  82. package/{src/mailbox-upstream.ts → dist/mailbox-upstream.js} +37 -47
  83. package/dist/message-category.d.ts +51 -0
  84. package/{src/message-category.ts → dist/message-category.js} +28 -71
  85. package/dist/message-move.d.ts +285 -0
  86. package/dist/message-move.js +858 -0
  87. package/dist/message-parser.d.ts +14 -0
  88. package/dist/message-parser.js +17 -0
  89. package/{src/message-presence.ts → dist/message-presence.d.ts} +1 -13
  90. package/dist/message-presence.js +29 -0
  91. package/dist/message-read.d.ts +51 -0
  92. package/dist/message-read.js +31 -0
  93. package/dist/message-sync.d.ts +345 -0
  94. package/dist/message-sync.js +1358 -0
  95. package/dist/mime-walker.d.ts +102 -0
  96. package/dist/mime-walker.js +198 -0
  97. package/dist/organize-job.d.ts +20 -0
  98. package/dist/organize-job.js +37 -0
  99. package/dist/outbox-attachment-content.d.ts +37 -0
  100. package/dist/outbox-attachment-content.js +50 -0
  101. package/dist/outbox-attachment-filename.d.ts +19 -0
  102. package/dist/outbox-attachment-filename.js +64 -0
  103. package/dist/outbox-attachment.d.ts +148 -0
  104. package/dist/outbox-attachment.js +273 -0
  105. package/dist/outbox-queue.d.ts +73 -0
  106. package/dist/outbox-queue.js +284 -0
  107. package/{src/outbox-status.ts → dist/outbox-status.d.ts} +1 -6
  108. package/dist/outbox-status.js +18 -0
  109. package/dist/ownership.d.ts +24 -0
  110. package/dist/ownership.js +83 -0
  111. package/dist/placement-move-terminal.d.ts +67 -0
  112. package/dist/placement-move-terminal.js +88 -0
  113. package/dist/placement-move.d.ts +83 -0
  114. package/dist/placement-move.js +216 -0
  115. package/{src/placement-settled.ts → dist/placement-settled.d.ts} +19 -50
  116. package/dist/placement-settled.js +75 -0
  117. package/dist/quarantine.d.ts +95 -0
  118. package/dist/quarantine.js +131 -0
  119. package/dist/refusal.d.ts +31 -0
  120. package/dist/refusal.js +3 -0
  121. package/{src/restore-source-placement.ts → dist/restore-source-placement.d.ts} +29 -101
  122. package/dist/restore-source-placement.js +79 -0
  123. package/dist/search-text-backfill.d.ts +15 -0
  124. package/dist/search-text-backfill.js +33 -0
  125. package/dist/snippet.d.ts +61 -0
  126. package/dist/snippet.js +129 -0
  127. package/dist/spam-report.d.ts +86 -0
  128. package/dist/spam-report.js +241 -0
  129. package/dist/sync-warning.d.ts +21 -0
  130. package/dist/sync-warning.js +26 -0
  131. package/dist/sync-watermarks.d.ts +138 -0
  132. package/dist/sync-watermarks.js +205 -0
  133. package/dist/test-helpers/fake-imap.d.ts +151 -0
  134. package/dist/test-helpers/fake-imap.js +508 -0
  135. package/dist/test-helpers/folder-roles.d.ts +34 -0
  136. package/dist/test-helpers/folder-roles.js +47 -0
  137. package/dist/test-helpers/mime-tree-generator.d.ts +82 -0
  138. package/dist/test-helpers/mime-tree-generator.js +329 -0
  139. package/dist/test-helpers/test-store.d.ts +69 -0
  140. package/dist/test-helpers/test-store.js +193 -0
  141. package/dist/text/normalizer.d.ts +20 -0
  142. package/dist/text/normalizer.js +102 -0
  143. package/dist/thread-message-rows.d.ts +69 -0
  144. package/dist/thread-message-rows.js +40 -0
  145. package/dist/thread-root.d.ts +7 -0
  146. package/dist/thread-root.js +25 -0
  147. package/dist/types.d.ts +343 -0
  148. package/dist/types.js +19 -0
  149. package/package.json +86 -73
  150. package/src/account-credentials.ts +0 -210
  151. package/src/adversarial-mime.e2e.test.ts +0 -307
  152. package/src/attribute-mapper.test.ts +0 -104
  153. package/src/attribute-mapper.ts +0 -148
  154. package/src/body-parse.test.ts +0 -55
  155. package/src/body-part-mapper.property.test.ts +0 -354
  156. package/src/body-part-mapper.ts +0 -360
  157. package/src/body-sync-auto-read-writeonce.test.ts +0 -219
  158. package/src/body-sync-calendar-suggestion.test.ts +0 -433
  159. package/src/body-sync-category-denormalize.test.ts +0 -352
  160. package/src/body-sync-category-override.test.ts +0 -501
  161. package/src/body-sync-category-writeonce.test.ts +0 -234
  162. package/src/body-sync-filter.test.ts +0 -237
  163. package/src/body-sync-message-gone.test.ts +0 -139
  164. package/src/body-sync-placement-flags.test.ts +0 -347
  165. package/src/body-sync-placement-writeonce.test.ts +0 -337
  166. package/src/body-sync-quarantine.test.ts +0 -276
  167. package/src/body-sync-queue.test.ts +0 -124
  168. package/src/body-sync-queue.ts +0 -131
  169. package/src/body-sync-resync.test.ts +0 -605
  170. package/src/body-sync-terminal.test.ts +0 -310
  171. package/src/body-sync-terminal.ts +0 -176
  172. package/src/body-sync-unclassifiable-once.test.ts +0 -260
  173. package/src/body-sync-unsubscribed-auto-read.test.ts +0 -176
  174. package/src/body-sync.ts +0 -1970
  175. package/src/calendar-parts.ts +0 -41
  176. package/src/classification-backfill.test.ts +0 -615
  177. package/src/classification-backfill.ts +0 -363
  178. package/src/classify-message.test.ts +0 -205
  179. package/src/classify-message.ts +0 -116
  180. package/src/connection-factory.ts +0 -102
  181. package/src/connection-test.ts +0 -208
  182. package/src/credentials.test.ts +0 -181
  183. package/src/external-move.test.ts +0 -369
  184. package/src/external-move.ts +0 -272
  185. package/src/filters/anchor-drift.ts +0 -55
  186. package/src/filters/filter-config.test.ts +0 -229
  187. package/src/filters/filter-config.ts +0 -69
  188. package/src/filters/list-id.test.ts +0 -50
  189. package/src/filters/match.test.ts +0 -429
  190. package/src/filters/match.ts +0 -274
  191. package/src/filters/message-embedder.test.ts +0 -39
  192. package/src/filters/message-embedder.ts +0 -33
  193. package/src/filters/pipeline.test.ts +0 -330
  194. package/src/filters/pipeline.ts +0 -284
  195. package/src/flag-push-terminal.test.ts +0 -209
  196. package/src/flag-push.test.ts +0 -408
  197. package/src/flag-push.ts +0 -198
  198. package/src/flag-queue-stars.test.ts +0 -171
  199. package/src/flag-queue.ts +0 -509
  200. package/src/folder-placement.ts +0 -56
  201. package/src/heuristics/authenticationResults.test.ts +0 -189
  202. package/src/heuristics/authenticationResults.ts +0 -212
  203. package/src/heuristics/classifyByHeaders.test.ts +0 -889
  204. package/src/heuristics/classifyByHeaders.ts +0 -467
  205. package/src/heuristics/classifyPlacement.test.ts +0 -366
  206. package/src/heuristics/classifyPlacement.ts +0 -211
  207. package/src/heuristics/classifyRealMail.test.ts +0 -242
  208. package/src/heuristics/machineSenders.test.ts +0 -101
  209. package/src/heuristics/senderMismatch.test.ts +0 -346
  210. package/src/heuristics/senderMismatch.ts +0 -259
  211. package/src/imapflow-connect-retry.test.ts +0 -134
  212. package/src/imapflow-connection.e2e.test.ts +0 -335
  213. package/src/imapflow-connection.integ.test.ts +0 -316
  214. package/src/imapflow-connection.test.ts +0 -469
  215. package/src/imapflow-connection.ts +0 -1475
  216. package/src/index.ts +0 -350
  217. package/src/list-id-backfill.test.ts +0 -573
  218. package/src/list-id-backfill.ts +0 -274
  219. package/src/mailbox-cursor-rebuild.test.ts +0 -171
  220. package/src/mailbox-cursor-rebuild.ts +0 -146
  221. package/src/mailbox-cursor.test.ts +0 -249
  222. package/src/mailbox-delete-intent.test.ts +0 -246
  223. package/src/mailbox-management.test.ts +0 -356
  224. package/src/mailbox-management.ts +0 -839
  225. package/src/mailbox-presence.test.ts +0 -72
  226. package/src/mailbox-queue.ts +0 -415
  227. package/src/mailbox-rename-intent.test.ts +0 -389
  228. package/src/mailbox-rename-subtree.test.ts +0 -664
  229. package/src/mailbox-sync.test.ts +0 -848
  230. package/src/mailbox-sync.ts +0 -724
  231. package/src/message-delete-junk-reconcile.test.ts +0 -117
  232. package/src/message-delete-no-trash.test.ts +0 -245
  233. package/src/message-delete-unsettled-placement.test.ts +0 -396
  234. package/src/message-move-copy.test.ts +0 -562
  235. package/src/message-move-empty-trash.test.ts +0 -326
  236. package/src/message-move-same-mailbox.test.ts +0 -139
  237. package/src/message-move-unsettled-placement.test.ts +0 -378
  238. package/src/message-move.ts +0 -1348
  239. package/src/message-parser.ts +0 -26
  240. package/src/message-presence.test.ts +0 -50
  241. package/src/message-sync-address-spoof.test.ts +0 -293
  242. package/src/message-sync-category.test.ts +0 -142
  243. package/src/message-sync-changedsince.test.ts +0 -840
  244. package/src/message-sync-flags.test.ts +0 -158
  245. package/src/message-sync-junk-harvest.test.ts +0 -244
  246. package/src/message-sync-quarantine.test.ts +0 -416
  247. package/src/message-sync-repoint.test.ts +0 -419
  248. package/src/message-sync.ts +0 -1963
  249. package/src/mime-walker.test.ts +0 -311
  250. package/src/mime-walker.ts +0 -305
  251. package/src/outbox-attachment-content.test.ts +0 -153
  252. package/src/outbox-attachment-content.ts +0 -104
  253. package/src/outbox-attachment-filename.test.ts +0 -132
  254. package/src/outbox-attachment-filename.ts +0 -76
  255. package/src/outbox-attachment.test.ts +0 -738
  256. package/src/outbox-attachment.ts +0 -536
  257. package/src/outbox-failed-recovery.test.ts +0 -370
  258. package/src/outbox-queue-settled-row.test.ts +0 -140
  259. package/src/outbox-queue.test.ts +0 -291
  260. package/src/outbox-queue.ts +0 -508
  261. package/src/pass-through-unit-of-work.ts +0 -17
  262. package/src/placement-move-terminal.test.ts +0 -300
  263. package/src/placement-move-terminal.ts +0 -148
  264. package/src/placement-move-unsettled.test.ts +0 -275
  265. package/src/placement-move.ts +0 -336
  266. package/src/placement-settled.test.ts +0 -147
  267. package/src/placement-transition-race.test.ts +0 -482
  268. package/src/quarantine.test.ts +0 -191
  269. package/src/quarantine.ts +0 -228
  270. package/src/snippet.test.ts +0 -213
  271. package/src/snippet.ts +0 -152
  272. package/src/spam-report.test.ts +0 -801
  273. package/src/spam-report.ts +0 -357
  274. package/src/stale-message-reconcile.test.ts +0 -82
  275. package/src/stale-message-reconcile.ts +0 -55
  276. package/src/sync-watermarks.test.ts +0 -409
  277. package/src/sync-watermarks.ts +0 -319
  278. package/src/system-flag-wire-format.test.ts +0 -77
  279. package/src/test-helpers/folder-roles.ts +0 -66
  280. package/src/test-helpers/isolated-mailbox.ts +0 -44
  281. package/src/test-helpers/mailfuzz-connection.ts +0 -27
  282. package/src/test-helpers/mime-tree-generator.ts +0 -461
  283. package/src/text/normalizer.test.ts +0 -223
  284. package/src/text/normalizer.ts +0 -140
  285. package/src/thread-message-rows.ts +0 -77
  286. package/src/types/mailparser-augment.d.ts +0 -15
  287. package/src/types/natural-porter-stemmer.d.ts +0 -50
  288. package/src/types/stopword.d.ts +0 -69
  289. package/src/types.ts +0 -371
  290. package/test/fixtures/mime-adversarial-e2e/01-octet-stream-pdf.eml +0 -21
  291. package/test/fixtures/mime-adversarial-e2e/02-nested-related-cid-image.eml +0 -31
  292. package/test/fixtures/mime-adversarial-e2e/03-calendar-invite.eml +0 -33
  293. package/test/fixtures/mime-adversarial-e2e/04-forwarded-rfc822-with-attachments.eml +0 -42
  294. package/test/fixtures/mime-adversarial-e2e/05-tnef-winmail.eml +0 -25
  295. package/test/fixtures/mime-adversarial-e2e/06-8bit-non-ascii.eml +0 -15
  296. package/test/fixtures/mime-adversarial-e2e/07-quoted-printable-soft-breaks.eml +0 -17
  297. package/test/fixtures/mime-adversarial-e2e/08-alternative-empty-text.eml +0 -18
  298. package/tsconfig.json +0 -8
@@ -0,0 +1,23 @@
1
+ import { CalendarSuggestionSource } from "@remit/domain-enums";
2
+ import type { ParsedMail } from "mailparser";
3
+ /**
4
+ * One `text/calendar` part of a message, and where it sat.
5
+ *
6
+ * The distinction between an inline part and an attached one is not cosmetic:
7
+ * a scheduling client sends the invitation inline as part of a
8
+ * `multipart/alternative`, while a client that cannot speak iTIP attaches an
9
+ * `.ics` file. Both carry the same bytes, and a card says which it read.
10
+ */
11
+ export interface CalendarPart {
12
+ partPath: string;
13
+ source: typeof CalendarSuggestionSource.IcalendarPart | typeof CalendarSuggestionSource.IcalendarAttachment;
14
+ icalData: string;
15
+ }
16
+ /**
17
+ * Every `text/calendar` part of a parsed message, in declaration order.
18
+ *
19
+ * Reads mailparser's `attachments[]`, which is where every non-`text/plain`,
20
+ * non-`text/html` leaf lands — the same list the header classifier already
21
+ * looks at to route an invitation to `transactional`.
22
+ */
23
+ export declare const calendarParts: (parsed: ParsedMail) => CalendarPart[];
@@ -0,0 +1,18 @@
1
+ /// <reference path="./types/mailparser-augment.d.ts" />
2
+ import { ROOT_PART_PATH } from "@remit/data-ports/id";
3
+ import { CalendarSuggestionSource } from "@remit/domain-enums";
4
+ const isCalendar = (attachment) => (attachment.contentType ?? "").toLowerCase().startsWith("text/calendar");
5
+ /**
6
+ * Every `text/calendar` part of a parsed message, in declaration order.
7
+ *
8
+ * Reads mailparser's `attachments[]`, which is where every non-`text/plain`,
9
+ * non-`text/html` leaf lands — the same list the header classifier already
10
+ * looks at to route an invitation to `transactional`.
11
+ */
12
+ export const calendarParts = (parsed) => (parsed.attachments ?? []).filter(isCalendar).map((attachment) => ({
13
+ partPath: attachment.partId ?? ROOT_PART_PATH,
14
+ source: attachment.contentDisposition === "attachment"
15
+ ? CalendarSuggestionSource.IcalendarAttachment
16
+ : CalendarSuggestionSource.IcalendarPart,
17
+ icalData: attachment.content.toString("utf8"),
18
+ }));
@@ -0,0 +1,91 @@
1
+ import type { IAccountConfigRepository, IAccountRepository, IAddressRepository, IMessageRepository, IThreadMessageRepository } from "@remit/data-ports";
2
+ import type { StorageService } from "@remit/storage-service";
3
+ import { type FolderPlacementLookup } from "./folder-placement.js";
4
+ /**
5
+ * Where the full-corpus pass left off: the identity of the account it was
6
+ * working through, and the page cursor within that account. Resuming looks the
7
+ * account up by id and re-opens it at its saved cursor, rather than re-scanning
8
+ * the whole corpus from the top after an interruption. A checkpoint naming no
9
+ * configured account restarts the pass.
10
+ */
11
+ export interface ClassificationBackfillCheckpoint {
12
+ accountConfigId: string;
13
+ continuationToken?: string;
14
+ }
15
+ /**
16
+ * Persistence for {@link ClassificationBackfillCheckpoint}, injected so the
17
+ * pass stays testable with an in-memory fake; the real entrypoint backs it
18
+ * with a file. `clear()` runs once the whole corpus has been scanned, so a
19
+ * later run starts fresh rather than reading a stale finished-run checkpoint.
20
+ */
21
+ export interface ClassificationBackfillCheckpointStore {
22
+ load(): Promise<ClassificationBackfillCheckpoint | undefined>;
23
+ save(checkpoint: ClassificationBackfillCheckpoint): Promise<void>;
24
+ clear(): Promise<void>;
25
+ }
26
+ export interface ClassificationBackfillDeps {
27
+ accountConfigService: Pick<IAccountConfigRepository, "listAll">;
28
+ accountService: Pick<IAccountRepository, "listAllByAccountConfig">;
29
+ mailboxSpecialUseService: FolderPlacementLookup;
30
+ /** Reads the sender's `Address.flags.category` override (issue #299). */
31
+ addressService: Pick<IAddressRepository, "getAddress">;
32
+ threadMessageService: Pick<IThreadMessageRepository, "listByAccount" | "findAllByMessageId" | "update">;
33
+ messageService: Pick<IMessageRepository, "get" | "update">;
34
+ storageService: Pick<StorageService, "retrieve">;
35
+ }
36
+ export interface ClassificationBackfillLogger {
37
+ info(obj: Record<string, unknown>, msg: string): void;
38
+ error?(obj: Record<string, unknown>, msg: string): void;
39
+ }
40
+ export interface ClassificationBackfillTotals {
41
+ scanned: number;
42
+ alreadyExamined: number;
43
+ alreadyCategorized: number;
44
+ skippedNoBody: number;
45
+ backfilled: number;
46
+ failed: number;
47
+ }
48
+ export interface ClassificationBackfillResult extends ClassificationBackfillTotals {
49
+ failedThreadMessageIds: string[];
50
+ }
51
+ export interface ClassificationBackfillProgress extends ClassificationBackfillTotals {
52
+ accountConfigId: string;
53
+ }
54
+ export interface ClassificationBackfillOptions {
55
+ /** Rows fetched per `listByAccount` page. */
56
+ batchSize?: number;
57
+ checkpointStore?: ClassificationBackfillCheckpointStore;
58
+ logger?: ClassificationBackfillLogger;
59
+ /** Called once per page, after that page's rows are settled. */
60
+ onProgress?: (progress: ClassificationBackfillProgress) => void;
61
+ }
62
+ /**
63
+ * One-time, resumable pass that classifies the stored-body cohort the
64
+ * classifier has never examined (issue #1197): rows whose bodies were synced
65
+ * before the classifier reached them, which carry the `NotExamined` sentinel
66
+ * on `Message.classificationState` and so stay `uncategorized` in the list —
67
+ * the field is what selects the cohort, and nothing on the sync path selects
68
+ * it. Read-only against the stored raw source: it never opens IMAP, and it
69
+ * has no placement or filter side effects — the same bounds the in-sync
70
+ * backfill #1173 removed had. Index-time moves and filter actions are
71
+ * decisions that already ran (or were declined) when the body first landed;
72
+ * re-running them would move mail the user has since filed by hand.
73
+ *
74
+ * A row is a candidate when its Message carries `NotExamined` — the named
75
+ * pending state, never inferred from `category` or from `bodyStorageKey` —
76
+ * and has a `bodyStorageKey`; a candidate without a stored body has nothing
77
+ * local to read, and the ordinary sync path will classify it once the body
78
+ * lands. A candidate whose category is already decided is marked `Examined`
79
+ * without re-deriving anything: those rows were classified by a pass that
80
+ * predates the field, and RFC 034 Decision 3.1's write-once category must not
81
+ * be recomputed (#355). The classification itself is
82
+ * {@link classifyParsedMessage} — the same rule table, and the same sender
83
+ * override, the body-store path classifies with.
84
+ *
85
+ * Chunked by `listByAccount`'s existing keyset pagination, one account at a
86
+ * time in account-id order, so an interrupted run and its resume walk the
87
+ * same sequence. A failure reading, parsing, or classifying one message is
88
+ * contained to that message — logged, counted, and the pass continues: one
89
+ * unreadable object must not strand the rest of the corpus.
90
+ */
91
+ export declare const backfillClassifications: (deps: ClassificationBackfillDeps, options?: ClassificationBackfillOptions) => Promise<ClassificationBackfillResult>;
@@ -0,0 +1,180 @@
1
+ import { MessageClassificationState } from "@remit/domain-enums";
2
+ import { parseMessageBody } from "./body-parse.js";
3
+ import { extractSnippet } from "./body-sync.js";
4
+ import { classifyParsedMessage } from "./classify-message.js";
5
+ import { extractListId } from "./filters/list-id.js";
6
+ import { resolveAccountFolderPlacements, } from "./folder-placement.js";
7
+ import { denormalizeMessageCategory, hasDecidedCategory, } from "./message-category.js";
8
+ const DEFAULT_BATCH_SIZE = 200;
9
+ const emptyTotals = () => ({
10
+ scanned: 0,
11
+ alreadyExamined: 0,
12
+ alreadyCategorized: 0,
13
+ skippedNoBody: 0,
14
+ backfilled: 0,
15
+ failed: 0,
16
+ });
17
+ /**
18
+ * Read the stored raw source, classify it once, and record that the
19
+ * classifier ran. Kept as its own function (rather than inline in a try/catch)
20
+ * so the caller can contain a failure with `.then(fulfilled, rejected)`
21
+ * instead of a block catch, which keeps the `try` around the one call whose
22
+ * failure this contains.
23
+ *
24
+ * The ThreadMessage rows are denormalized BEFORE the Message update, for the
25
+ * same reason the deleted in-sync backfill did it that way (issue #320): the
26
+ * signal this pass selects by — `classificationState` — is written in the
27
+ * Message update, so a failure between the two writes leaves both undone and
28
+ * the rerun redoes them. Writing the Message first would strand the
29
+ * denormalized rows at `uncategorized` forever — the row reads as examined and
30
+ * the retry returns early.
31
+ */
32
+ const classifyAndApply = async (deps, accountConfigId, row, bodyStorageKey, placement) => {
33
+ const body = await deps.storageService.retrieve(bodyStorageKey);
34
+ const parsed = await parseMessageBody(body);
35
+ const classification = await classifyParsedMessage(deps.addressService, accountConfigId, parsed, placement);
36
+ // The same denormalized fields the body-store path writes (see
37
+ // `applyPostStoreSteps`), not just the category: the snippet and `List-Id`
38
+ // are derived from the same bytes already in hand, and the rows this pass
39
+ // reaches never got them — the pass that stored the body is the pass that
40
+ // classifies, and it never ran for this cohort.
41
+ await denormalizeMessageCategory({ threadMessageService: deps.threadMessageService }, accountConfigId, row.messageId, {
42
+ category: classification.category,
43
+ snippet: extractSnippet(parsed),
44
+ listId: extractListId(parsed),
45
+ });
46
+ await deps.messageService.update(row.messageId, {
47
+ ...classification,
48
+ classificationState: MessageClassificationState.Examined,
49
+ });
50
+ };
51
+ /**
52
+ * One-time, resumable pass that classifies the stored-body cohort the
53
+ * classifier has never examined (issue #1197): rows whose bodies were synced
54
+ * before the classifier reached them, which carry the `NotExamined` sentinel
55
+ * on `Message.classificationState` and so stay `uncategorized` in the list —
56
+ * the field is what selects the cohort, and nothing on the sync path selects
57
+ * it. Read-only against the stored raw source: it never opens IMAP, and it
58
+ * has no placement or filter side effects — the same bounds the in-sync
59
+ * backfill #1173 removed had. Index-time moves and filter actions are
60
+ * decisions that already ran (or were declined) when the body first landed;
61
+ * re-running them would move mail the user has since filed by hand.
62
+ *
63
+ * A row is a candidate when its Message carries `NotExamined` — the named
64
+ * pending state, never inferred from `category` or from `bodyStorageKey` —
65
+ * and has a `bodyStorageKey`; a candidate without a stored body has nothing
66
+ * local to read, and the ordinary sync path will classify it once the body
67
+ * lands. A candidate whose category is already decided is marked `Examined`
68
+ * without re-deriving anything: those rows were classified by a pass that
69
+ * predates the field, and RFC 034 Decision 3.1's write-once category must not
70
+ * be recomputed (#355). The classification itself is
71
+ * {@link classifyParsedMessage} — the same rule table, and the same sender
72
+ * override, the body-store path classifies with.
73
+ *
74
+ * Chunked by `listByAccount`'s existing keyset pagination, one account at a
75
+ * time in account-id order, so an interrupted run and its resume walk the
76
+ * same sequence. A failure reading, parsing, or classifying one message is
77
+ * contained to that message — logged, counted, and the pass continues: one
78
+ * unreadable object must not strand the rest of the corpus.
79
+ */
80
+ export const backfillClassifications = async (deps, options = {}) => {
81
+ const batchSize = options.batchSize ?? DEFAULT_BATCH_SIZE;
82
+ const { logger, checkpointStore } = options;
83
+ const accounts = [...(await deps.accountConfigService.listAll())].sort((left, right) => left.accountConfigId.localeCompare(right.accountConfigId));
84
+ const startingCheckpoint = await checkpointStore?.load();
85
+ const checkpointedIndex = startingCheckpoint
86
+ ? accounts.findIndex((account) => account.accountConfigId === startingCheckpoint.accountConfigId)
87
+ : -1;
88
+ if (startingCheckpoint && checkpointedIndex === -1) {
89
+ logger?.info({ accountConfigId: startingCheckpoint.accountConfigId }, "Classification backfill checkpoint names no configured account; restarting the pass");
90
+ }
91
+ const startIndex = checkpointedIndex === -1 ? 0 : checkpointedIndex;
92
+ const startContinuationToken = checkpointedIndex === -1
93
+ ? undefined
94
+ : startingCheckpoint?.continuationToken;
95
+ const totals = emptyTotals();
96
+ const failedThreadMessageIds = [];
97
+ // Messages settled this run. More than one ThreadMessage row can index one
98
+ // Message, and the candidate fact lives on the Message, so without this a
99
+ // second row would re-read the same body and re-classify it within the
100
+ // same page — idempotent in outcome, wasteful in storage reads.
101
+ const examinedMessageIds = new Set();
102
+ for (let accountIndex = startIndex; accountIndex < accounts.length; accountIndex++) {
103
+ const account = accounts[accountIndex];
104
+ let continuationToken = accountIndex === startIndex ? startContinuationToken : undefined;
105
+ const mailAccounts = await deps.accountService.listAllByAccountConfig(account.accountConfigId);
106
+ const placements = await resolveAccountFolderPlacements(deps.mailboxSpecialUseService, mailAccounts.map((mailAccount) => mailAccount.accountId));
107
+ do {
108
+ const page = await deps.threadMessageService.listByAccount(account.accountConfigId, { limit: batchSize, continuationToken });
109
+ totals.scanned += page.items.length;
110
+ if (page.items.length > 0) {
111
+ const messages = await deps.messageService.get(page.items.map((row) => row.messageId));
112
+ const messageByMessageId = new Map(messages.map((message) => [message.messageId, message]));
113
+ for (const row of page.items) {
114
+ const message = messageByMessageId.get(row.messageId);
115
+ if (!message?.bodyStorageKey) {
116
+ totals.skippedNoBody++;
117
+ continue;
118
+ }
119
+ if (message.classificationState !==
120
+ MessageClassificationState.NotExamined ||
121
+ examinedMessageIds.has(row.messageId)) {
122
+ totals.alreadyExamined++;
123
+ continue;
124
+ }
125
+ // Classified by a pass that predates the field: record that it
126
+ // ran, and touch nothing else — the write-once category is not
127
+ // recomputed (#355), and the derived fields that pass wrote are
128
+ // already on the row.
129
+ if (hasDecidedCategory(message.category)) {
130
+ const outcome = await deps.messageService
131
+ .update(row.messageId, {
132
+ classificationState: MessageClassificationState.Examined,
133
+ })
134
+ .then(() => null, (error) => error);
135
+ if (outcome !== null) {
136
+ totals.failed++;
137
+ failedThreadMessageIds.push(row.threadMessageId);
138
+ logger?.error?.({
139
+ threadMessageId: row.threadMessageId,
140
+ messageId: row.messageId,
141
+ error: outcome instanceof Error
142
+ ? outcome.message
143
+ : String(outcome),
144
+ }, "Classification backfill failed to record an already-categorized message; leaving it for a later pass");
145
+ continue;
146
+ }
147
+ examinedMessageIds.add(row.messageId);
148
+ totals.alreadyCategorized++;
149
+ continue;
150
+ }
151
+ const outcome = await classifyAndApply(deps, account.accountConfigId, row, message.bodyStorageKey, placements.get(row.mailboxId) ?? "other").then(() => null, (error) => error);
152
+ if (outcome !== null) {
153
+ totals.failed++;
154
+ failedThreadMessageIds.push(row.threadMessageId);
155
+ logger?.error?.({
156
+ threadMessageId: row.threadMessageId,
157
+ messageId: row.messageId,
158
+ error: outcome instanceof Error ? outcome.message : String(outcome),
159
+ }, "Classification backfill failed for a message; leaving it for a later pass");
160
+ continue;
161
+ }
162
+ examinedMessageIds.add(row.messageId);
163
+ totals.backfilled++;
164
+ }
165
+ }
166
+ continuationToken = page.continuationToken;
167
+ await checkpointStore?.save({
168
+ accountConfigId: account.accountConfigId,
169
+ continuationToken,
170
+ });
171
+ logger?.info({ accountConfigId: account.accountConfigId, ...totals }, "Classification backfill progress");
172
+ options.onProgress?.({
173
+ accountConfigId: account.accountConfigId,
174
+ ...totals,
175
+ });
176
+ } while (continuationToken);
177
+ }
178
+ await checkpointStore?.clear();
179
+ return { ...totals, failedThreadMessageIds };
180
+ };
@@ -0,0 +1,31 @@
1
+ import type { IAddressRepository, ThreadMessageItem, UpdateMessageInput } from "@remit/data-ports";
2
+ import type { ParsedMail } from "mailparser";
3
+ import type { FolderPlacement } from "./heuristics/classifyPlacement.js";
4
+ type ThreadMessageCategory = ThreadMessageItem["category"];
5
+ /**
6
+ * The From address the classifier reads, when there is one to read. Shared by
7
+ * every path that classifies or counts a sender (body-sync's classification,
8
+ * inbound counter and placement signals, and the classification backfill) so
9
+ * they all read the same address off the same bytes.
10
+ */
11
+ export declare const extractPrimaryFromEmail: (parsed: ParsedMail) => string | null;
12
+ /**
13
+ * Header classification, with the sender's `Address.flags.category` override
14
+ * (issue #299, RFC 039 Decision 3) substituted for the header-derived
15
+ * category when one is set. Returns the subset of the Message update that
16
+ * carries the derived fields; the caller folds it into a single UpdateItem
17
+ * alongside `bodyStorageKey` (body-sync) or `classificationState` (the
18
+ * classification backfill, issue #1197). Optional signals are omitted when
19
+ * absent so we never overwrite an existing value with `undefined`.
20
+ *
21
+ * The override wins outright rather than blending with the heuristic — RFC
22
+ * 039 Decision 3 treats a direct reclassification as final, the same as
23
+ * `flags.blocked`/`vip` already override placement. Every caller gates on
24
+ * `hasDecidedCategory` before this result reaches a write, so a message that
25
+ * already carries a real category is never re-touched regardless of what
26
+ * this returns.
27
+ */
28
+ export declare const classifyParsedMessage: (addressService: Pick<IAddressRepository, "getAddress">, accountConfigId: string, parsed: ParsedMail, placement: FolderPlacement) => Promise<UpdateMessageInput & {
29
+ category: ThreadMessageCategory;
30
+ }>;
31
+ export {};
@@ -0,0 +1,89 @@
1
+ import { NotFoundError } from "@remit/data-ports/errors";
2
+ import { deriveAddressId } from "@remit/data-ports/id";
3
+ import { classifyByHeaders, extractAuthenticity, extractAuthResult, extractHasListUnsubscribe, extractProviderSpam, } from "./heuristics/classifyByHeaders.js";
4
+ import { extractSenderMismatch } from "./heuristics/senderMismatch.js";
5
+ /**
6
+ * The From address the classifier reads, when there is one to read. Shared by
7
+ * every path that classifies or counts a sender (body-sync's classification,
8
+ * inbound counter and placement signals, and the classification backfill) so
9
+ * they all read the same address off the same bytes.
10
+ */
11
+ export const extractPrimaryFromEmail = (parsed) => {
12
+ const from = parsed.from;
13
+ if (!from || !from.value || from.value.length === 0)
14
+ return null;
15
+ const address = from.value[0]?.address;
16
+ if (!address)
17
+ return null;
18
+ return address.toLowerCase();
19
+ };
20
+ /**
21
+ * The one-sided half of `Address.flags.category` (issue #299, RFC 039
22
+ * Decision 3): a real value here overrides `classifyByHeaders` outright.
23
+ * Only a genuinely-absent Address is "no override" — any other failure
24
+ * (throttle, infra) propagates, because this feeds the write-once
25
+ * `Message.category` at the moment it is decided, where a masked infra
26
+ * failure would silently classify by headers when the user asked for
27
+ * something else.
28
+ */
29
+ const resolveCategoryOverride = async (addressService, accountConfigId, fromEmail) => {
30
+ try {
31
+ const addressId = deriveAddressId(accountConfigId, fromEmail);
32
+ const address = await addressService.getAddress(accountConfigId, addressId);
33
+ return address.flags?.category?.value;
34
+ }
35
+ catch (err) {
36
+ if (!(err instanceof NotFoundError))
37
+ throw err;
38
+ return undefined;
39
+ }
40
+ };
41
+ /**
42
+ * Header classification, with the sender's `Address.flags.category` override
43
+ * (issue #299, RFC 039 Decision 3) substituted for the header-derived
44
+ * category when one is set. Returns the subset of the Message update that
45
+ * carries the derived fields; the caller folds it into a single UpdateItem
46
+ * alongside `bodyStorageKey` (body-sync) or `classificationState` (the
47
+ * classification backfill, issue #1197). Optional signals are omitted when
48
+ * absent so we never overwrite an existing value with `undefined`.
49
+ *
50
+ * The override wins outright rather than blending with the heuristic — RFC
51
+ * 039 Decision 3 treats a direct reclassification as final, the same as
52
+ * `flags.blocked`/`vip` already override placement. Every caller gates on
53
+ * `hasDecidedCategory` before this result reaches a write, so a message that
54
+ * already carries a real category is never re-touched regardless of what
55
+ * this returns.
56
+ */
57
+ export const classifyParsedMessage = async (addressService, accountConfigId, parsed, placement) => {
58
+ const headerCategory = classifyByHeaders(parsed);
59
+ const authenticity = extractAuthenticity(parsed);
60
+ const authResult = extractAuthResult(parsed);
61
+ const providerSpam = extractProviderSpam(parsed);
62
+ const hasListUnsubscribe = extractHasListUnsubscribe(parsed);
63
+ // A passing SPF/DKIM/DMARC check proves the sending domain, not the
64
+ // identity the message claims — on a shared-tenant host the verified
65
+ // subdomain belongs to whoever signed up. These two comparisons say
66
+ // whether the claim holds, and run only over mail the provider already
67
+ // called spam or that sits in the Junk folder.
68
+ const senderMismatch = authenticity === null
69
+ ? {}
70
+ : extractSenderMismatch(parsed, {
71
+ fromDomain: authenticity.fromDomain,
72
+ providerSpamClassified: providerSpam?.classified === true,
73
+ placement,
74
+ bulkSender: hasListUnsubscribe,
75
+ });
76
+ const fromEmail = extractPrimaryFromEmail(parsed);
77
+ const categoryOverride = fromEmail
78
+ ? await resolveCategoryOverride(addressService, accountConfigId, fromEmail)
79
+ : undefined;
80
+ return {
81
+ category: categoryOverride ?? headerCategory,
82
+ hasListUnsubscribe,
83
+ ...(authenticity !== null
84
+ ? { authenticity: { ...authenticity, ...senderMismatch } }
85
+ : {}),
86
+ ...(authResult !== null ? { authResult } : {}),
87
+ ...(providerSpam !== null ? { providerSpam } : {}),
88
+ };
89
+ };
@@ -0,0 +1,54 @@
1
+ /**
2
+ * Connection factory for IMAP implementations
3
+ *
4
+ * Uses ImapFlow for all IMAP connections.
5
+ */
6
+ import type { IImapConnection, ImapConnectionConfig, MailCredentials } from "./types.js";
7
+ /**
8
+ * Create an IMAP connection
9
+ */
10
+ export declare const createConnection: (config: ImapConnectionConfig) => IImapConnection;
11
+ /**
12
+ * Create an IMAP connection from account data using password credentials.
13
+ */
14
+ export declare const createConnectionFromAccount: (account: {
15
+ imapHost: string;
16
+ imapPort: number;
17
+ imapTls: boolean;
18
+ username: string;
19
+ }, password: string) => IImapConnection;
20
+ /**
21
+ * Create an IMAP connection from account data using a MailCredentials union.
22
+ * Use this instead of createConnectionFromAccount for OAuth accounts.
23
+ */
24
+ export declare const createConnectionWithCredentials: (account: {
25
+ imapHost: string;
26
+ imapPort: number;
27
+ imapTls: boolean;
28
+ username: string;
29
+ }, credentials: MailCredentials) => IImapConnection;
30
+ /**
31
+ * Managed connection factory that caches and reuses a single connection.
32
+ *
33
+ * The factory handles connection lifecycle - callers should not disconnect.
34
+ * Call close() on the factory to disconnect when done.
35
+ */
36
+ export interface ManagedConnectionFactory {
37
+ /** Get the cached connection (creates on first call) */
38
+ getConnection(): IImapConnection;
39
+ /** Disconnect and cleanup */
40
+ close(): Promise<void>;
41
+ }
42
+ /**
43
+ * Create a managed connection factory that caches a single connection.
44
+ *
45
+ * @example
46
+ * ```typescript
47
+ * const factory = createManagedConnectionFactory(config);
48
+ * const conn = factory.getConnection();
49
+ * await conn.connect();
50
+ * // ... reuse conn via factory.getConnection() ...
51
+ * await factory.close(); // Disconnect when done
52
+ * ```
53
+ */
54
+ export declare const createManagedConnectionFactory: (config: ImapConnectionConfig) => ManagedConnectionFactory;
@@ -0,0 +1,54 @@
1
+ /**
2
+ * Connection factory for IMAP implementations
3
+ *
4
+ * Uses ImapFlow for all IMAP connections.
5
+ */
6
+ import { createImapFlowConnectionFromAccount, createImapFlowConnectionWithCredentials, ImapFlowConnection, } from "./imapflow-connection.js";
7
+ /**
8
+ * Create an IMAP connection
9
+ */
10
+ export const createConnection = (config) => {
11
+ return new ImapFlowConnection(config);
12
+ };
13
+ /**
14
+ * Create an IMAP connection from account data using password credentials.
15
+ */
16
+ export const createConnectionFromAccount = (account, password) => {
17
+ return createImapFlowConnectionFromAccount(account, password);
18
+ };
19
+ /**
20
+ * Create an IMAP connection from account data using a MailCredentials union.
21
+ * Use this instead of createConnectionFromAccount for OAuth accounts.
22
+ */
23
+ export const createConnectionWithCredentials = (account, credentials) => {
24
+ return createImapFlowConnectionWithCredentials(account, credentials);
25
+ };
26
+ /**
27
+ * Create a managed connection factory that caches a single connection.
28
+ *
29
+ * @example
30
+ * ```typescript
31
+ * const factory = createManagedConnectionFactory(config);
32
+ * const conn = factory.getConnection();
33
+ * await conn.connect();
34
+ * // ... reuse conn via factory.getConnection() ...
35
+ * await factory.close(); // Disconnect when done
36
+ * ```
37
+ */
38
+ export const createManagedConnectionFactory = (config) => {
39
+ let connection = null;
40
+ return {
41
+ getConnection: () => {
42
+ if (!connection) {
43
+ connection = new ImapFlowConnection(config);
44
+ }
45
+ return connection;
46
+ },
47
+ close: async () => {
48
+ if (connection) {
49
+ await connection.disconnect();
50
+ connection = null;
51
+ }
52
+ },
53
+ };
54
+ };
@@ -0,0 +1,41 @@
1
+ /**
2
+ * Connection testing utilities for IMAP and SMTP servers
3
+ *
4
+ * Used to validate credentials before saving account configuration.
5
+ */
6
+ import type { MailCredentials } from "./types.js";
7
+ export interface TestResult {
8
+ success: boolean;
9
+ error?: string;
10
+ }
11
+ export interface ImapTestConfig {
12
+ host: string;
13
+ port: number;
14
+ secure: boolean;
15
+ startTls?: boolean;
16
+ credentials: MailCredentials;
17
+ user: string;
18
+ }
19
+ export interface SmtpTestConfig {
20
+ host: string;
21
+ port: number;
22
+ secure: boolean;
23
+ credentials: MailCredentials;
24
+ user: string;
25
+ }
26
+ /**
27
+ * Test IMAP connection with provided credentials.
28
+ *
29
+ * Uses a raw socket to send AUTH PLAIN directly — bypasses the CAPABILITY
30
+ * round-trip that IMAP clients typically perform first, avoiding deadlocks
31
+ * with servers that pipeline their tagged responses.
32
+ *
33
+ * Supports plaintext, direct TLS (secure: true), and STARTTLS (startTls: true).
34
+ */
35
+ export declare const testImapConnection: (config: ImapTestConfig) => Promise<TestResult>;
36
+ /**
37
+ * Test SMTP connection with provided credentials
38
+ *
39
+ * Uses nodemailer's verify() method to test authentication.
40
+ */
41
+ export declare const testSmtpConnection: (config: SmtpTestConfig) => Promise<TestResult>;