@remit/mailbox-service 0.0.112 → 0.0.114

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (270) hide show
  1. package/dist/account-credentials.d.ts +83 -0
  2. package/dist/account-credentials.js +104 -0
  3. package/dist/attribute-mapper.d.ts +36 -0
  4. package/dist/attribute-mapper.js +119 -0
  5. package/dist/body-parse.d.ts +28 -0
  6. package/{src/body-parse.ts → dist/body-parse.js} +18 -26
  7. package/dist/body-part-mapper.d.ts +92 -0
  8. package/dist/body-part-mapper.js +301 -0
  9. package/dist/body-sync-queue.d.ts +59 -0
  10. package/dist/body-sync-queue.js +56 -0
  11. package/dist/body-sync-terminal.d.ts +58 -0
  12. package/dist/body-sync-terminal.js +95 -0
  13. package/dist/body-sync.d.ts +351 -0
  14. package/dist/body-sync.js +1204 -0
  15. package/dist/calendar-parts.d.ts +23 -0
  16. package/dist/calendar-parts.js +18 -0
  17. package/dist/classification-backfill.d.ts +91 -0
  18. package/dist/classification-backfill.js +180 -0
  19. package/dist/classify-message.d.ts +31 -0
  20. package/dist/classify-message.js +89 -0
  21. package/dist/connection-factory.d.ts +54 -0
  22. package/dist/connection-factory.js +54 -0
  23. package/dist/connection-test.d.ts +41 -0
  24. package/dist/connection-test.js +152 -0
  25. package/dist/external-move.d.ts +94 -0
  26. package/dist/external-move.js +173 -0
  27. package/dist/filter-operations.d.ts +42 -0
  28. package/dist/filter-operations.js +211 -0
  29. package/dist/filters/filter-config.d.ts +14 -0
  30. package/dist/filters/filter-config.js +10 -0
  31. package/{src/filters/list-id.ts → dist/filters/list-id.d.ts} +2 -16
  32. package/dist/filters/list-id.js +26 -0
  33. package/dist/filters/match.d.ts +69 -0
  34. package/dist/filters/match.js +165 -0
  35. package/dist/filters/pipeline.d.ts +64 -0
  36. package/dist/filters/pipeline.js +100 -0
  37. package/{src/flag-push-terminal.ts → dist/flag-push-terminal.d.ts} +16 -72
  38. package/dist/flag-push-terminal.js +63 -0
  39. package/dist/flag-push.d.ts +69 -0
  40. package/dist/flag-push.js +111 -0
  41. package/dist/flag-queue.d.ts +175 -0
  42. package/dist/flag-queue.js +291 -0
  43. package/dist/folder-operations.d.ts +39 -0
  44. package/dist/folder-operations.js +112 -0
  45. package/dist/folder-placement.d.ts +13 -0
  46. package/dist/folder-placement.js +23 -0
  47. package/dist/heuristics/authenticationResults.d.ts +10 -0
  48. package/dist/heuristics/authenticationResults.js +187 -0
  49. package/dist/heuristics/classifyByHeaders.d.ts +82 -0
  50. package/dist/heuristics/classifyByHeaders.js +365 -0
  51. package/dist/heuristics/classifyPlacement.d.ts +67 -0
  52. package/dist/heuristics/classifyPlacement.js +153 -0
  53. package/dist/heuristics/machineSenders.d.ts +21 -0
  54. package/{src/heuristics/machineSenders.ts → dist/heuristics/machineSenders.js} +34 -40
  55. package/dist/heuristics/senderMismatch.d.ts +81 -0
  56. package/dist/heuristics/senderMismatch.js +192 -0
  57. package/dist/heuristics/socialDomains.d.ts +14 -0
  58. package/{src/heuristics/socialDomains.ts → dist/heuristics/socialDomains.js} +9 -9
  59. package/dist/heuristics/transactionalDomains.d.ts +18 -0
  60. package/{src/heuristics/transactionalDomains.ts → dist/heuristics/transactionalDomains.js} +8 -8
  61. package/dist/imapflow-connection.d.ts +377 -0
  62. package/dist/imapflow-connection.js +1147 -0
  63. package/dist/index.d.ts +63 -0
  64. package/dist/index.js +65 -0
  65. package/dist/list-id-backfill.d.ts +83 -0
  66. package/dist/list-id-backfill.js +132 -0
  67. package/dist/mailbox-cursor-rebuild.d.ts +74 -0
  68. package/dist/mailbox-cursor-rebuild.js +71 -0
  69. package/{src/mailbox-cursor.ts → dist/mailbox-cursor.d.ts} +20 -98
  70. package/dist/mailbox-cursor.js +102 -0
  71. package/dist/mailbox-intent.d.ts +41 -0
  72. package/{src/mailbox-intent.ts → dist/mailbox-intent.js} +38 -64
  73. package/dist/mailbox-management.d.ts +240 -0
  74. package/dist/mailbox-management.js +576 -0
  75. package/dist/mailbox-presence.d.ts +52 -0
  76. package/{src/mailbox-presence.ts → dist/mailbox-presence.js} +19 -30
  77. package/dist/mailbox-queue.d.ts +129 -0
  78. package/dist/mailbox-queue.js +216 -0
  79. package/dist/mailbox-sync.d.ts +93 -0
  80. package/dist/mailbox-sync.js +490 -0
  81. package/dist/mailbox-upstream.d.ts +38 -0
  82. package/{src/mailbox-upstream.ts → dist/mailbox-upstream.js} +37 -47
  83. package/dist/message-category.d.ts +51 -0
  84. package/{src/message-category.ts → dist/message-category.js} +28 -71
  85. package/dist/message-move.d.ts +285 -0
  86. package/dist/message-move.js +858 -0
  87. package/dist/message-parser.d.ts +14 -0
  88. package/dist/message-parser.js +17 -0
  89. package/{src/message-presence.ts → dist/message-presence.d.ts} +1 -13
  90. package/dist/message-presence.js +29 -0
  91. package/dist/message-read.d.ts +51 -0
  92. package/dist/message-read.js +31 -0
  93. package/dist/message-sync.d.ts +345 -0
  94. package/dist/message-sync.js +1358 -0
  95. package/dist/mime-walker.d.ts +102 -0
  96. package/dist/mime-walker.js +198 -0
  97. package/dist/organize-job.d.ts +20 -0
  98. package/dist/organize-job.js +37 -0
  99. package/dist/outbox-attachment-content.d.ts +37 -0
  100. package/dist/outbox-attachment-content.js +50 -0
  101. package/dist/outbox-attachment-filename.d.ts +19 -0
  102. package/dist/outbox-attachment-filename.js +64 -0
  103. package/dist/outbox-attachment.d.ts +148 -0
  104. package/dist/outbox-attachment.js +273 -0
  105. package/dist/outbox-queue.d.ts +73 -0
  106. package/dist/outbox-queue.js +280 -0
  107. package/{src/outbox-status.ts → dist/outbox-status.d.ts} +1 -6
  108. package/dist/outbox-status.js +18 -0
  109. package/dist/ownership.d.ts +24 -0
  110. package/dist/ownership.js +83 -0
  111. package/dist/placement-move-terminal.d.ts +67 -0
  112. package/dist/placement-move-terminal.js +88 -0
  113. package/dist/placement-move.d.ts +83 -0
  114. package/dist/placement-move.js +216 -0
  115. package/{src/placement-settled.ts → dist/placement-settled.d.ts} +13 -68
  116. package/dist/placement-settled.js +75 -0
  117. package/dist/quarantine.d.ts +95 -0
  118. package/dist/quarantine.js +131 -0
  119. package/dist/refusal.d.ts +31 -0
  120. package/dist/refusal.js +3 -0
  121. package/{src/restore-source-placement.ts → dist/restore-source-placement.d.ts} +29 -97
  122. package/dist/restore-source-placement.js +79 -0
  123. package/dist/search-text-backfill.d.ts +15 -0
  124. package/dist/search-text-backfill.js +33 -0
  125. package/dist/snippet.d.ts +61 -0
  126. package/dist/snippet.js +129 -0
  127. package/dist/spam-report.d.ts +86 -0
  128. package/dist/spam-report.js +241 -0
  129. package/dist/sync-warning.d.ts +21 -0
  130. package/dist/sync-warning.js +26 -0
  131. package/dist/sync-watermarks.d.ts +138 -0
  132. package/dist/sync-watermarks.js +205 -0
  133. package/dist/test-helpers/fake-imap.d.ts +151 -0
  134. package/dist/test-helpers/fake-imap.js +508 -0
  135. package/dist/test-helpers/folder-roles.d.ts +34 -0
  136. package/dist/test-helpers/folder-roles.js +47 -0
  137. package/dist/test-helpers/mime-tree-generator.d.ts +82 -0
  138. package/dist/test-helpers/mime-tree-generator.js +329 -0
  139. package/dist/test-helpers/test-store.d.ts +69 -0
  140. package/dist/test-helpers/test-store.js +193 -0
  141. package/dist/text/normalizer.d.ts +20 -0
  142. package/dist/text/normalizer.js +102 -0
  143. package/dist/thread-message-rows.d.ts +69 -0
  144. package/dist/thread-message-rows.js +40 -0
  145. package/dist/thread-root.d.ts +7 -0
  146. package/dist/thread-root.js +25 -0
  147. package/dist/types.d.ts +343 -0
  148. package/dist/types.js +19 -0
  149. package/package.json +86 -88
  150. package/src/account-credentials.ts +0 -210
  151. package/src/attribute-mapper.test.ts +0 -104
  152. package/src/attribute-mapper.ts +0 -148
  153. package/src/body-parse.test.ts +0 -55
  154. package/src/body-part-mapper.property.test.ts +0 -354
  155. package/src/body-part-mapper.ts +0 -410
  156. package/src/body-sync-queue.test.ts +0 -64
  157. package/src/body-sync-queue.ts +0 -131
  158. package/src/body-sync-terminal.test.ts +0 -148
  159. package/src/body-sync-terminal.ts +0 -174
  160. package/src/body-sync.test.ts +0 -925
  161. package/src/body-sync.ts +0 -1891
  162. package/src/calendar-parts.ts +0 -41
  163. package/src/classification-backfill.test.ts +0 -285
  164. package/src/classification-backfill.ts +0 -363
  165. package/src/classify-message.test.ts +0 -142
  166. package/src/classify-message.ts +0 -116
  167. package/src/connection-factory.ts +0 -102
  168. package/src/connection-test.ts +0 -208
  169. package/src/credentials.test.ts +0 -181
  170. package/src/external-move.test.ts +0 -262
  171. package/src/external-move.ts +0 -285
  172. package/src/filter-operations.test.ts +0 -319
  173. package/src/filter-operations.ts +0 -377
  174. package/src/filter-update.test.ts +0 -313
  175. package/src/filters/filter-config.test.ts +0 -87
  176. package/src/filters/filter-config.ts +0 -34
  177. package/src/filters/list-id.test.ts +0 -50
  178. package/src/filters/match.test.ts +0 -392
  179. package/src/filters/match.ts +0 -225
  180. package/src/filters/pipeline.test.ts +0 -199
  181. package/src/filters/pipeline.ts +0 -170
  182. package/src/flag-push.test.ts +0 -112
  183. package/src/flag-push.ts +0 -198
  184. package/src/flag-queue.test.ts +0 -197
  185. package/src/flag-queue.ts +0 -509
  186. package/src/folder-operations.test.ts +0 -273
  187. package/src/folder-operations.ts +0 -217
  188. package/src/folder-placement.ts +0 -56
  189. package/src/forwarded-message.test.ts +0 -122
  190. package/src/heuristics/authenticationResults.test.ts +0 -189
  191. package/src/heuristics/authenticationResults.ts +0 -212
  192. package/src/heuristics/classifyByHeaders.test.ts +0 -889
  193. package/src/heuristics/classifyByHeaders.ts +0 -467
  194. package/src/heuristics/classifyPlacement.test.ts +0 -366
  195. package/src/heuristics/classifyPlacement.ts +0 -211
  196. package/src/heuristics/classifyRealMail.test.ts +0 -242
  197. package/src/heuristics/machineSenders.test.ts +0 -101
  198. package/src/heuristics/senderMismatch.test.ts +0 -346
  199. package/src/heuristics/senderMismatch.ts +0 -259
  200. package/src/imapflow-connect-retry.test.ts +0 -97
  201. package/src/imapflow-connection.test.ts +0 -503
  202. package/src/imapflow-connection.ts +0 -1476
  203. package/src/index.ts +0 -397
  204. package/src/list-id-backfill.test.ts +0 -207
  205. package/src/list-id-backfill.ts +0 -274
  206. package/src/mailbox-cursor-rebuild.test.ts +0 -171
  207. package/src/mailbox-cursor-rebuild.ts +0 -146
  208. package/src/mailbox-cursor.test.ts +0 -141
  209. package/src/mailbox-delete-intent.test.ts +0 -119
  210. package/src/mailbox-management.test.ts +0 -243
  211. package/src/mailbox-management.ts +0 -838
  212. package/src/mailbox-presence.test.ts +0 -74
  213. package/src/mailbox-queue.ts +0 -397
  214. package/src/mailbox-rename-intent.test.ts +0 -220
  215. package/src/mailbox-rename-subtree.test.ts +0 -287
  216. package/src/mailbox-sync.test.ts +0 -286
  217. package/src/mailbox-sync.ts +0 -724
  218. package/src/message-move.test.ts +0 -726
  219. package/src/message-move.ts +0 -1518
  220. package/src/message-parser.ts +0 -26
  221. package/src/message-presence.test.ts +0 -50
  222. package/src/message-read.ts +0 -87
  223. package/src/message-save-retry.test.ts +0 -90
  224. package/src/message-sync-flags.test.ts +0 -167
  225. package/src/message-sync.test.ts +0 -762
  226. package/src/message-sync.ts +0 -2133
  227. package/src/mime-walker.test.ts +0 -311
  228. package/src/mime-walker.ts +0 -306
  229. package/src/organize-job.ts +0 -93
  230. package/src/outbox-attachment-content.test.ts +0 -153
  231. package/src/outbox-attachment-content.ts +0 -104
  232. package/src/outbox-attachment-filename.test.ts +0 -132
  233. package/src/outbox-attachment-filename.ts +0 -76
  234. package/src/outbox-attachment.test.ts +0 -626
  235. package/src/outbox-attachment.ts +0 -536
  236. package/src/outbox-failed-recovery.test.ts +0 -299
  237. package/src/outbox-queue.test.ts +0 -315
  238. package/src/outbox-queue.ts +0 -525
  239. package/src/ownership.ts +0 -140
  240. package/src/placement-move-terminal.ts +0 -145
  241. package/src/placement-move.ts +0 -363
  242. package/src/placement-settled.test.ts +0 -147
  243. package/src/placement.test.ts +0 -404
  244. package/src/quarantine.test.ts +0 -133
  245. package/src/quarantine.ts +0 -229
  246. package/src/refusal.ts +0 -29
  247. package/src/snippet.test.ts +0 -213
  248. package/src/snippet.ts +0 -152
  249. package/src/spam-report.test.ts +0 -329
  250. package/src/spam-report.ts +0 -357
  251. package/src/sync-warning.test.ts +0 -213
  252. package/src/sync-warning.ts +0 -56
  253. package/src/sync-watermarks.test.ts +0 -409
  254. package/src/sync-watermarks.ts +0 -319
  255. package/src/system-flag-wire-format.test.ts +0 -77
  256. package/src/test-helpers/fake-imap.ts +0 -690
  257. package/src/test-helpers/folder-roles.ts +0 -66
  258. package/src/test-helpers/mime-tree-generator.ts +0 -461
  259. package/src/test-helpers/test-queue.ts +0 -56
  260. package/src/test-helpers/test-store.ts +0 -299
  261. package/src/text/normalizer.test.ts +0 -223
  262. package/src/text/normalizer.ts +0 -140
  263. package/src/thread-message-rows.ts +0 -77
  264. package/src/thread-root.test.ts +0 -29
  265. package/src/thread-root.ts +0 -51
  266. package/src/types/mailparser-augment.d.ts +0 -23
  267. package/src/types/natural-porter-stemmer.d.ts +0 -50
  268. package/src/types/stopword.d.ts +0 -69
  269. package/src/types.ts +0 -371
  270. package/tsconfig.json +0 -8
@@ -0,0 +1,23 @@
1
+ import { CalendarSuggestionSource } from "@remit/domain-enums";
2
+ import type { ParsedMail } from "mailparser";
3
+ /**
4
+ * One `text/calendar` part of a message, and where it sat.
5
+ *
6
+ * The distinction between an inline part and an attached one is not cosmetic:
7
+ * a scheduling client sends the invitation inline as part of a
8
+ * `multipart/alternative`, while a client that cannot speak iTIP attaches an
9
+ * `.ics` file. Both carry the same bytes, and a card says which it read.
10
+ */
11
+ export interface CalendarPart {
12
+ partPath: string;
13
+ source: typeof CalendarSuggestionSource.IcalendarPart | typeof CalendarSuggestionSource.IcalendarAttachment;
14
+ icalData: string;
15
+ }
16
+ /**
17
+ * Every `text/calendar` part of a parsed message, in declaration order.
18
+ *
19
+ * Reads mailparser's `attachments[]`, which is where every non-`text/plain`,
20
+ * non-`text/html` leaf lands — the same list the header classifier already
21
+ * looks at to route an invitation to `transactional`.
22
+ */
23
+ export declare const calendarParts: (parsed: ParsedMail) => CalendarPart[];
@@ -0,0 +1,18 @@
1
+ /// <reference path="./types/mailparser-augment.d.ts" />
2
+ import { ROOT_PART_PATH } from "@remit/data-ports/id";
3
+ import { CalendarSuggestionSource } from "@remit/domain-enums";
4
+ const isCalendar = (attachment) => (attachment.contentType ?? "").toLowerCase().startsWith("text/calendar");
5
+ /**
6
+ * Every `text/calendar` part of a parsed message, in declaration order.
7
+ *
8
+ * Reads mailparser's `attachments[]`, which is where every non-`text/plain`,
9
+ * non-`text/html` leaf lands — the same list the header classifier already
10
+ * looks at to route an invitation to `transactional`.
11
+ */
12
+ export const calendarParts = (parsed) => (parsed.attachments ?? []).filter(isCalendar).map((attachment) => ({
13
+ partPath: attachment.partId ?? ROOT_PART_PATH,
14
+ source: attachment.contentDisposition === "attachment"
15
+ ? CalendarSuggestionSource.IcalendarAttachment
16
+ : CalendarSuggestionSource.IcalendarPart,
17
+ icalData: attachment.content.toString("utf8"),
18
+ }));
@@ -0,0 +1,91 @@
1
+ import type { IAccountConfigRepository, IAccountRepository, IAddressRepository, IMessageRepository, IThreadMessageRepository } from "@remit/data-ports";
2
+ import type { StorageService } from "@remit/storage-service";
3
+ import { type FolderPlacementLookup } from "./folder-placement.js";
4
+ /**
5
+ * Where the full-corpus pass left off: the identity of the account it was
6
+ * working through, and the page cursor within that account. Resuming looks the
7
+ * account up by id and re-opens it at its saved cursor, rather than re-scanning
8
+ * the whole corpus from the top after an interruption. A checkpoint naming no
9
+ * configured account restarts the pass.
10
+ */
11
+ export interface ClassificationBackfillCheckpoint {
12
+ accountConfigId: string;
13
+ continuationToken?: string;
14
+ }
15
+ /**
16
+ * Persistence for {@link ClassificationBackfillCheckpoint}, injected so the
17
+ * pass stays testable with an in-memory fake; the real entrypoint backs it
18
+ * with a file. `clear()` runs once the whole corpus has been scanned, so a
19
+ * later run starts fresh rather than reading a stale finished-run checkpoint.
20
+ */
21
+ export interface ClassificationBackfillCheckpointStore {
22
+ load(): Promise<ClassificationBackfillCheckpoint | undefined>;
23
+ save(checkpoint: ClassificationBackfillCheckpoint): Promise<void>;
24
+ clear(): Promise<void>;
25
+ }
26
+ export interface ClassificationBackfillDeps {
27
+ accountConfigService: Pick<IAccountConfigRepository, "listAll">;
28
+ accountService: Pick<IAccountRepository, "listAllByAccountConfig">;
29
+ mailboxSpecialUseService: FolderPlacementLookup;
30
+ /** Reads the sender's `Address.flags.category` override (issue #299). */
31
+ addressService: Pick<IAddressRepository, "getAddress">;
32
+ threadMessageService: Pick<IThreadMessageRepository, "listByAccount" | "findAllByMessageId" | "update">;
33
+ messageService: Pick<IMessageRepository, "get" | "update">;
34
+ storageService: Pick<StorageService, "retrieve">;
35
+ }
36
+ export interface ClassificationBackfillLogger {
37
+ info(obj: Record<string, unknown>, msg: string): void;
38
+ error?(obj: Record<string, unknown>, msg: string): void;
39
+ }
40
+ export interface ClassificationBackfillTotals {
41
+ scanned: number;
42
+ alreadyExamined: number;
43
+ alreadyCategorized: number;
44
+ skippedNoBody: number;
45
+ backfilled: number;
46
+ failed: number;
47
+ }
48
+ export interface ClassificationBackfillResult extends ClassificationBackfillTotals {
49
+ failedThreadMessageIds: string[];
50
+ }
51
+ export interface ClassificationBackfillProgress extends ClassificationBackfillTotals {
52
+ accountConfigId: string;
53
+ }
54
+ export interface ClassificationBackfillOptions {
55
+ /** Rows fetched per `listByAccount` page. */
56
+ batchSize?: number;
57
+ checkpointStore?: ClassificationBackfillCheckpointStore;
58
+ logger?: ClassificationBackfillLogger;
59
+ /** Called once per page, after that page's rows are settled. */
60
+ onProgress?: (progress: ClassificationBackfillProgress) => void;
61
+ }
62
+ /**
63
+ * One-time, resumable pass that classifies the stored-body cohort the
64
+ * classifier has never examined (issue #1197): rows whose bodies were synced
65
+ * before the classifier reached them, which carry the `NotExamined` sentinel
66
+ * on `Message.classificationState` and so stay `uncategorized` in the list —
67
+ * the field is what selects the cohort, and nothing on the sync path selects
68
+ * it. Read-only against the stored raw source: it never opens IMAP, and it
69
+ * has no placement or filter side effects — the same bounds the in-sync
70
+ * backfill #1173 removed had. Index-time moves and filter actions are
71
+ * decisions that already ran (or were declined) when the body first landed;
72
+ * re-running them would move mail the user has since filed by hand.
73
+ *
74
+ * A row is a candidate when its Message carries `NotExamined` — the named
75
+ * pending state, never inferred from `category` or from `bodyStorageKey` —
76
+ * and has a `bodyStorageKey`; a candidate without a stored body has nothing
77
+ * local to read, and the ordinary sync path will classify it once the body
78
+ * lands. A candidate whose category is already decided is marked `Examined`
79
+ * without re-deriving anything: those rows were classified by a pass that
80
+ * predates the field, and RFC 034 Decision 3.1's write-once category must not
81
+ * be recomputed (#355). The classification itself is
82
+ * {@link classifyParsedMessage} — the same rule table, and the same sender
83
+ * override, the body-store path classifies with.
84
+ *
85
+ * Chunked by `listByAccount`'s existing keyset pagination, one account at a
86
+ * time in account-id order, so an interrupted run and its resume walk the
87
+ * same sequence. A failure reading, parsing, or classifying one message is
88
+ * contained to that message — logged, counted, and the pass continues: one
89
+ * unreadable object must not strand the rest of the corpus.
90
+ */
91
+ export declare const backfillClassifications: (deps: ClassificationBackfillDeps, options?: ClassificationBackfillOptions) => Promise<ClassificationBackfillResult>;
@@ -0,0 +1,180 @@
1
+ import { MessageClassificationState } from "@remit/domain-enums";
2
+ import { parseMessageBody } from "./body-parse.js";
3
+ import { extractSnippet } from "./body-sync.js";
4
+ import { classifyParsedMessage } from "./classify-message.js";
5
+ import { extractListId } from "./filters/list-id.js";
6
+ import { resolveAccountFolderPlacements, } from "./folder-placement.js";
7
+ import { denormalizeMessageCategory, hasDecidedCategory, } from "./message-category.js";
8
+ const DEFAULT_BATCH_SIZE = 200;
9
+ const emptyTotals = () => ({
10
+ scanned: 0,
11
+ alreadyExamined: 0,
12
+ alreadyCategorized: 0,
13
+ skippedNoBody: 0,
14
+ backfilled: 0,
15
+ failed: 0,
16
+ });
17
+ /**
18
+ * Read the stored raw source, classify it once, and record that the
19
+ * classifier ran. Kept as its own function (rather than inline in a try/catch)
20
+ * so the caller can contain a failure with `.then(fulfilled, rejected)`
21
+ * instead of a block catch, which keeps the `try` around the one call whose
22
+ * failure this contains.
23
+ *
24
+ * The ThreadMessage rows are denormalized BEFORE the Message update, for the
25
+ * same reason the deleted in-sync backfill did it that way (issue #320): the
26
+ * signal this pass selects by — `classificationState` — is written in the
27
+ * Message update, so a failure between the two writes leaves both undone and
28
+ * the rerun redoes them. Writing the Message first would strand the
29
+ * denormalized rows at `uncategorized` forever — the row reads as examined and
30
+ * the retry returns early.
31
+ */
32
+ const classifyAndApply = async (deps, accountConfigId, row, bodyStorageKey, placement) => {
33
+ const body = await deps.storageService.retrieve(bodyStorageKey);
34
+ const parsed = await parseMessageBody(body);
35
+ const classification = await classifyParsedMessage(deps.addressService, accountConfigId, parsed, placement);
36
+ // The same denormalized fields the body-store path writes (see
37
+ // `applyPostStoreSteps`), not just the category: the snippet and `List-Id`
38
+ // are derived from the same bytes already in hand, and the rows this pass
39
+ // reaches never got them — the pass that stored the body is the pass that
40
+ // classifies, and it never ran for this cohort.
41
+ await denormalizeMessageCategory({ threadMessageService: deps.threadMessageService }, accountConfigId, row.messageId, {
42
+ category: classification.category,
43
+ snippet: extractSnippet(parsed),
44
+ listId: extractListId(parsed),
45
+ });
46
+ await deps.messageService.update(row.messageId, {
47
+ ...classification,
48
+ classificationState: MessageClassificationState.Examined,
49
+ });
50
+ };
51
+ /**
52
+ * One-time, resumable pass that classifies the stored-body cohort the
53
+ * classifier has never examined (issue #1197): rows whose bodies were synced
54
+ * before the classifier reached them, which carry the `NotExamined` sentinel
55
+ * on `Message.classificationState` and so stay `uncategorized` in the list —
56
+ * the field is what selects the cohort, and nothing on the sync path selects
57
+ * it. Read-only against the stored raw source: it never opens IMAP, and it
58
+ * has no placement or filter side effects — the same bounds the in-sync
59
+ * backfill #1173 removed had. Index-time moves and filter actions are
60
+ * decisions that already ran (or were declined) when the body first landed;
61
+ * re-running them would move mail the user has since filed by hand.
62
+ *
63
+ * A row is a candidate when its Message carries `NotExamined` — the named
64
+ * pending state, never inferred from `category` or from `bodyStorageKey` —
65
+ * and has a `bodyStorageKey`; a candidate without a stored body has nothing
66
+ * local to read, and the ordinary sync path will classify it once the body
67
+ * lands. A candidate whose category is already decided is marked `Examined`
68
+ * without re-deriving anything: those rows were classified by a pass that
69
+ * predates the field, and RFC 034 Decision 3.1's write-once category must not
70
+ * be recomputed (#355). The classification itself is
71
+ * {@link classifyParsedMessage} — the same rule table, and the same sender
72
+ * override, the body-store path classifies with.
73
+ *
74
+ * Chunked by `listByAccount`'s existing keyset pagination, one account at a
75
+ * time in account-id order, so an interrupted run and its resume walk the
76
+ * same sequence. A failure reading, parsing, or classifying one message is
77
+ * contained to that message — logged, counted, and the pass continues: one
78
+ * unreadable object must not strand the rest of the corpus.
79
+ */
80
+ export const backfillClassifications = async (deps, options = {}) => {
81
+ const batchSize = options.batchSize ?? DEFAULT_BATCH_SIZE;
82
+ const { logger, checkpointStore } = options;
83
+ const accounts = [...(await deps.accountConfigService.listAll())].sort((left, right) => left.accountConfigId.localeCompare(right.accountConfigId));
84
+ const startingCheckpoint = await checkpointStore?.load();
85
+ const checkpointedIndex = startingCheckpoint
86
+ ? accounts.findIndex((account) => account.accountConfigId === startingCheckpoint.accountConfigId)
87
+ : -1;
88
+ if (startingCheckpoint && checkpointedIndex === -1) {
89
+ logger?.info({ accountConfigId: startingCheckpoint.accountConfigId }, "Classification backfill checkpoint names no configured account; restarting the pass");
90
+ }
91
+ const startIndex = checkpointedIndex === -1 ? 0 : checkpointedIndex;
92
+ const startContinuationToken = checkpointedIndex === -1
93
+ ? undefined
94
+ : startingCheckpoint?.continuationToken;
95
+ const totals = emptyTotals();
96
+ const failedThreadMessageIds = [];
97
+ // Messages settled this run. More than one ThreadMessage row can index one
98
+ // Message, and the candidate fact lives on the Message, so without this a
99
+ // second row would re-read the same body and re-classify it within the
100
+ // same page — idempotent in outcome, wasteful in storage reads.
101
+ const examinedMessageIds = new Set();
102
+ for (let accountIndex = startIndex; accountIndex < accounts.length; accountIndex++) {
103
+ const account = accounts[accountIndex];
104
+ let continuationToken = accountIndex === startIndex ? startContinuationToken : undefined;
105
+ const mailAccounts = await deps.accountService.listAllByAccountConfig(account.accountConfigId);
106
+ const placements = await resolveAccountFolderPlacements(deps.mailboxSpecialUseService, mailAccounts.map((mailAccount) => mailAccount.accountId));
107
+ do {
108
+ const page = await deps.threadMessageService.listByAccount(account.accountConfigId, { limit: batchSize, continuationToken });
109
+ totals.scanned += page.items.length;
110
+ if (page.items.length > 0) {
111
+ const messages = await deps.messageService.get(page.items.map((row) => row.messageId));
112
+ const messageByMessageId = new Map(messages.map((message) => [message.messageId, message]));
113
+ for (const row of page.items) {
114
+ const message = messageByMessageId.get(row.messageId);
115
+ if (!message?.bodyStorageKey) {
116
+ totals.skippedNoBody++;
117
+ continue;
118
+ }
119
+ if (message.classificationState !==
120
+ MessageClassificationState.NotExamined ||
121
+ examinedMessageIds.has(row.messageId)) {
122
+ totals.alreadyExamined++;
123
+ continue;
124
+ }
125
+ // Classified by a pass that predates the field: record that it
126
+ // ran, and touch nothing else — the write-once category is not
127
+ // recomputed (#355), and the derived fields that pass wrote are
128
+ // already on the row.
129
+ if (hasDecidedCategory(message.category)) {
130
+ const outcome = await deps.messageService
131
+ .update(row.messageId, {
132
+ classificationState: MessageClassificationState.Examined,
133
+ })
134
+ .then(() => null, (error) => error);
135
+ if (outcome !== null) {
136
+ totals.failed++;
137
+ failedThreadMessageIds.push(row.threadMessageId);
138
+ logger?.error?.({
139
+ threadMessageId: row.threadMessageId,
140
+ messageId: row.messageId,
141
+ error: outcome instanceof Error
142
+ ? outcome.message
143
+ : String(outcome),
144
+ }, "Classification backfill failed to record an already-categorized message; leaving it for a later pass");
145
+ continue;
146
+ }
147
+ examinedMessageIds.add(row.messageId);
148
+ totals.alreadyCategorized++;
149
+ continue;
150
+ }
151
+ const outcome = await classifyAndApply(deps, account.accountConfigId, row, message.bodyStorageKey, placements.get(row.mailboxId) ?? "other").then(() => null, (error) => error);
152
+ if (outcome !== null) {
153
+ totals.failed++;
154
+ failedThreadMessageIds.push(row.threadMessageId);
155
+ logger?.error?.({
156
+ threadMessageId: row.threadMessageId,
157
+ messageId: row.messageId,
158
+ error: outcome instanceof Error ? outcome.message : String(outcome),
159
+ }, "Classification backfill failed for a message; leaving it for a later pass");
160
+ continue;
161
+ }
162
+ examinedMessageIds.add(row.messageId);
163
+ totals.backfilled++;
164
+ }
165
+ }
166
+ continuationToken = page.continuationToken;
167
+ await checkpointStore?.save({
168
+ accountConfigId: account.accountConfigId,
169
+ continuationToken,
170
+ });
171
+ logger?.info({ accountConfigId: account.accountConfigId, ...totals }, "Classification backfill progress");
172
+ options.onProgress?.({
173
+ accountConfigId: account.accountConfigId,
174
+ ...totals,
175
+ });
176
+ } while (continuationToken);
177
+ }
178
+ await checkpointStore?.clear();
179
+ return { ...totals, failedThreadMessageIds };
180
+ };
@@ -0,0 +1,31 @@
1
+ import type { IAddressRepository, ThreadMessageItem, UpdateMessageInput } from "@remit/data-ports";
2
+ import type { ParsedMail } from "mailparser";
3
+ import type { FolderPlacement } from "./heuristics/classifyPlacement.js";
4
+ type ThreadMessageCategory = ThreadMessageItem["category"];
5
+ /**
6
+ * The From address the classifier reads, when there is one to read. Shared by
7
+ * every path that classifies or counts a sender (body-sync's classification,
8
+ * inbound counter and placement signals, and the classification backfill) so
9
+ * they all read the same address off the same bytes.
10
+ */
11
+ export declare const extractPrimaryFromEmail: (parsed: ParsedMail) => string | null;
12
+ /**
13
+ * Header classification, with the sender's `Address.flags.category` override
14
+ * (issue #299, RFC 039 Decision 3) substituted for the header-derived
15
+ * category when one is set. Returns the subset of the Message update that
16
+ * carries the derived fields; the caller folds it into a single UpdateItem
17
+ * alongside `bodyStorageKey` (body-sync) or `classificationState` (the
18
+ * classification backfill, issue #1197). Optional signals are omitted when
19
+ * absent so we never overwrite an existing value with `undefined`.
20
+ *
21
+ * The override wins outright rather than blending with the heuristic — RFC
22
+ * 039 Decision 3 treats a direct reclassification as final, the same as
23
+ * `flags.blocked`/`vip` already override placement. Every caller gates on
24
+ * `hasDecidedCategory` before this result reaches a write, so a message that
25
+ * already carries a real category is never re-touched regardless of what
26
+ * this returns.
27
+ */
28
+ export declare const classifyParsedMessage: (addressService: Pick<IAddressRepository, "getAddress">, accountConfigId: string, parsed: ParsedMail, placement: FolderPlacement) => Promise<UpdateMessageInput & {
29
+ category: ThreadMessageCategory;
30
+ }>;
31
+ export {};
@@ -0,0 +1,89 @@
1
+ import { NotFoundError } from "@remit/data-ports/errors";
2
+ import { deriveAddressId } from "@remit/data-ports/id";
3
+ import { classifyByHeaders, extractAuthenticity, extractAuthResult, extractHasListUnsubscribe, extractProviderSpam, } from "./heuristics/classifyByHeaders.js";
4
+ import { extractSenderMismatch } from "./heuristics/senderMismatch.js";
5
+ /**
6
+ * The From address the classifier reads, when there is one to read. Shared by
7
+ * every path that classifies or counts a sender (body-sync's classification,
8
+ * inbound counter and placement signals, and the classification backfill) so
9
+ * they all read the same address off the same bytes.
10
+ */
11
+ export const extractPrimaryFromEmail = (parsed) => {
12
+ const from = parsed.from;
13
+ if (!from || !from.value || from.value.length === 0)
14
+ return null;
15
+ const address = from.value[0]?.address;
16
+ if (!address)
17
+ return null;
18
+ return address.toLowerCase();
19
+ };
20
+ /**
21
+ * The one-sided half of `Address.flags.category` (issue #299, RFC 039
22
+ * Decision 3): a real value here overrides `classifyByHeaders` outright.
23
+ * Only a genuinely-absent Address is "no override" — any other failure
24
+ * (throttle, infra) propagates, because this feeds the write-once
25
+ * `Message.category` at the moment it is decided, where a masked infra
26
+ * failure would silently classify by headers when the user asked for
27
+ * something else.
28
+ */
29
+ const resolveCategoryOverride = async (addressService, accountConfigId, fromEmail) => {
30
+ try {
31
+ const addressId = deriveAddressId(accountConfigId, fromEmail);
32
+ const address = await addressService.getAddress(accountConfigId, addressId);
33
+ return address.flags?.category?.value;
34
+ }
35
+ catch (err) {
36
+ if (!(err instanceof NotFoundError))
37
+ throw err;
38
+ return undefined;
39
+ }
40
+ };
41
+ /**
42
+ * Header classification, with the sender's `Address.flags.category` override
43
+ * (issue #299, RFC 039 Decision 3) substituted for the header-derived
44
+ * category when one is set. Returns the subset of the Message update that
45
+ * carries the derived fields; the caller folds it into a single UpdateItem
46
+ * alongside `bodyStorageKey` (body-sync) or `classificationState` (the
47
+ * classification backfill, issue #1197). Optional signals are omitted when
48
+ * absent so we never overwrite an existing value with `undefined`.
49
+ *
50
+ * The override wins outright rather than blending with the heuristic — RFC
51
+ * 039 Decision 3 treats a direct reclassification as final, the same as
52
+ * `flags.blocked`/`vip` already override placement. Every caller gates on
53
+ * `hasDecidedCategory` before this result reaches a write, so a message that
54
+ * already carries a real category is never re-touched regardless of what
55
+ * this returns.
56
+ */
57
+ export const classifyParsedMessage = async (addressService, accountConfigId, parsed, placement) => {
58
+ const headerCategory = classifyByHeaders(parsed);
59
+ const authenticity = extractAuthenticity(parsed);
60
+ const authResult = extractAuthResult(parsed);
61
+ const providerSpam = extractProviderSpam(parsed);
62
+ const hasListUnsubscribe = extractHasListUnsubscribe(parsed);
63
+ // A passing SPF/DKIM/DMARC check proves the sending domain, not the
64
+ // identity the message claims — on a shared-tenant host the verified
65
+ // subdomain belongs to whoever signed up. These two comparisons say
66
+ // whether the claim holds, and run only over mail the provider already
67
+ // called spam or that sits in the Junk folder.
68
+ const senderMismatch = authenticity === null
69
+ ? {}
70
+ : extractSenderMismatch(parsed, {
71
+ fromDomain: authenticity.fromDomain,
72
+ providerSpamClassified: providerSpam?.classified === true,
73
+ placement,
74
+ bulkSender: hasListUnsubscribe,
75
+ });
76
+ const fromEmail = extractPrimaryFromEmail(parsed);
77
+ const categoryOverride = fromEmail
78
+ ? await resolveCategoryOverride(addressService, accountConfigId, fromEmail)
79
+ : undefined;
80
+ return {
81
+ category: categoryOverride ?? headerCategory,
82
+ hasListUnsubscribe,
83
+ ...(authenticity !== null
84
+ ? { authenticity: { ...authenticity, ...senderMismatch } }
85
+ : {}),
86
+ ...(authResult !== null ? { authResult } : {}),
87
+ ...(providerSpam !== null ? { providerSpam } : {}),
88
+ };
89
+ };
@@ -0,0 +1,54 @@
1
+ /**
2
+ * Connection factory for IMAP implementations
3
+ *
4
+ * Uses ImapFlow for all IMAP connections.
5
+ */
6
+ import type { IImapConnection, ImapConnectionConfig, MailCredentials } from "./types.js";
7
+ /**
8
+ * Create an IMAP connection
9
+ */
10
+ export declare const createConnection: (config: ImapConnectionConfig) => IImapConnection;
11
+ /**
12
+ * Create an IMAP connection from account data using password credentials.
13
+ */
14
+ export declare const createConnectionFromAccount: (account: {
15
+ imapHost: string;
16
+ imapPort: number;
17
+ imapTls: boolean;
18
+ username: string;
19
+ }, password: string) => IImapConnection;
20
+ /**
21
+ * Create an IMAP connection from account data using a MailCredentials union.
22
+ * Use this instead of createConnectionFromAccount for OAuth accounts.
23
+ */
24
+ export declare const createConnectionWithCredentials: (account: {
25
+ imapHost: string;
26
+ imapPort: number;
27
+ imapTls: boolean;
28
+ username: string;
29
+ }, credentials: MailCredentials) => IImapConnection;
30
+ /**
31
+ * Managed connection factory that caches and reuses a single connection.
32
+ *
33
+ * The factory handles connection lifecycle - callers should not disconnect.
34
+ * Call close() on the factory to disconnect when done.
35
+ */
36
+ export interface ManagedConnectionFactory {
37
+ /** Get the cached connection (creates on first call) */
38
+ getConnection(): IImapConnection;
39
+ /** Disconnect and cleanup */
40
+ close(): Promise<void>;
41
+ }
42
+ /**
43
+ * Create a managed connection factory that caches a single connection.
44
+ *
45
+ * @example
46
+ * ```typescript
47
+ * const factory = createManagedConnectionFactory(config);
48
+ * const conn = factory.getConnection();
49
+ * await conn.connect();
50
+ * // ... reuse conn via factory.getConnection() ...
51
+ * await factory.close(); // Disconnect when done
52
+ * ```
53
+ */
54
+ export declare const createManagedConnectionFactory: (config: ImapConnectionConfig) => ManagedConnectionFactory;
@@ -0,0 +1,54 @@
1
+ /**
2
+ * Connection factory for IMAP implementations
3
+ *
4
+ * Uses ImapFlow for all IMAP connections.
5
+ */
6
+ import { createImapFlowConnectionFromAccount, createImapFlowConnectionWithCredentials, ImapFlowConnection, } from "./imapflow-connection.js";
7
+ /**
8
+ * Create an IMAP connection
9
+ */
10
+ export const createConnection = (config) => {
11
+ return new ImapFlowConnection(config);
12
+ };
13
+ /**
14
+ * Create an IMAP connection from account data using password credentials.
15
+ */
16
+ export const createConnectionFromAccount = (account, password) => {
17
+ return createImapFlowConnectionFromAccount(account, password);
18
+ };
19
+ /**
20
+ * Create an IMAP connection from account data using a MailCredentials union.
21
+ * Use this instead of createConnectionFromAccount for OAuth accounts.
22
+ */
23
+ export const createConnectionWithCredentials = (account, credentials) => {
24
+ return createImapFlowConnectionWithCredentials(account, credentials);
25
+ };
26
+ /**
27
+ * Create a managed connection factory that caches a single connection.
28
+ *
29
+ * @example
30
+ * ```typescript
31
+ * const factory = createManagedConnectionFactory(config);
32
+ * const conn = factory.getConnection();
33
+ * await conn.connect();
34
+ * // ... reuse conn via factory.getConnection() ...
35
+ * await factory.close(); // Disconnect when done
36
+ * ```
37
+ */
38
+ export const createManagedConnectionFactory = (config) => {
39
+ let connection = null;
40
+ return {
41
+ getConnection: () => {
42
+ if (!connection) {
43
+ connection = new ImapFlowConnection(config);
44
+ }
45
+ return connection;
46
+ },
47
+ close: async () => {
48
+ if (connection) {
49
+ await connection.disconnect();
50
+ connection = null;
51
+ }
52
+ },
53
+ };
54
+ };
@@ -0,0 +1,41 @@
1
+ /**
2
+ * Connection testing utilities for IMAP and SMTP servers
3
+ *
4
+ * Used to validate credentials before saving account configuration.
5
+ */
6
+ import type { MailCredentials } from "./types.js";
7
+ export interface TestResult {
8
+ success: boolean;
9
+ error?: string;
10
+ }
11
+ export interface ImapTestConfig {
12
+ host: string;
13
+ port: number;
14
+ secure: boolean;
15
+ startTls?: boolean;
16
+ credentials: MailCredentials;
17
+ user: string;
18
+ }
19
+ export interface SmtpTestConfig {
20
+ host: string;
21
+ port: number;
22
+ secure: boolean;
23
+ credentials: MailCredentials;
24
+ user: string;
25
+ }
26
+ /**
27
+ * Test IMAP connection with provided credentials.
28
+ *
29
+ * Uses a raw socket to send AUTH PLAIN directly — bypasses the CAPABILITY
30
+ * round-trip that IMAP clients typically perform first, avoiding deadlocks
31
+ * with servers that pipeline their tagged responses.
32
+ *
33
+ * Supports plaintext, direct TLS (secure: true), and STARTTLS (startTls: true).
34
+ */
35
+ export declare const testImapConnection: (config: ImapTestConfig) => Promise<TestResult>;
36
+ /**
37
+ * Test SMTP connection with provided credentials
38
+ *
39
+ * Uses nodemailer's verify() method to test authentication.
40
+ */
41
+ export declare const testSmtpConnection: (config: SmtpTestConfig) => Promise<TestResult>;