@remit/mailbox-service 0.0.111 → 0.0.113

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (298) hide show
  1. package/dist/account-credentials.d.ts +83 -0
  2. package/dist/account-credentials.js +104 -0
  3. package/dist/attribute-mapper.d.ts +36 -0
  4. package/dist/attribute-mapper.js +119 -0
  5. package/dist/body-parse.d.ts +28 -0
  6. package/{src/body-parse.ts → dist/body-parse.js} +18 -26
  7. package/dist/body-part-mapper.d.ts +92 -0
  8. package/dist/body-part-mapper.js +301 -0
  9. package/dist/body-sync-queue.d.ts +59 -0
  10. package/dist/body-sync-queue.js +56 -0
  11. package/dist/body-sync-terminal.d.ts +58 -0
  12. package/dist/body-sync-terminal.js +95 -0
  13. package/dist/body-sync.d.ts +351 -0
  14. package/dist/body-sync.js +1204 -0
  15. package/dist/calendar-parts.d.ts +23 -0
  16. package/dist/calendar-parts.js +18 -0
  17. package/dist/classification-backfill.d.ts +91 -0
  18. package/dist/classification-backfill.js +180 -0
  19. package/dist/classify-message.d.ts +31 -0
  20. package/dist/classify-message.js +89 -0
  21. package/dist/connection-factory.d.ts +54 -0
  22. package/dist/connection-factory.js +54 -0
  23. package/dist/connection-test.d.ts +41 -0
  24. package/dist/connection-test.js +152 -0
  25. package/dist/external-move.d.ts +94 -0
  26. package/dist/external-move.js +173 -0
  27. package/dist/filter-operations.d.ts +42 -0
  28. package/dist/filter-operations.js +211 -0
  29. package/dist/filters/filter-config.d.ts +14 -0
  30. package/dist/filters/filter-config.js +10 -0
  31. package/{src/filters/list-id.ts → dist/filters/list-id.d.ts} +2 -16
  32. package/dist/filters/list-id.js +26 -0
  33. package/dist/filters/match.d.ts +69 -0
  34. package/dist/filters/match.js +165 -0
  35. package/dist/filters/pipeline.d.ts +64 -0
  36. package/dist/filters/pipeline.js +100 -0
  37. package/{src/flag-push-terminal.ts → dist/flag-push-terminal.d.ts} +19 -77
  38. package/dist/flag-push-terminal.js +63 -0
  39. package/dist/flag-push.d.ts +69 -0
  40. package/dist/flag-push.js +111 -0
  41. package/dist/flag-queue.d.ts +175 -0
  42. package/dist/flag-queue.js +291 -0
  43. package/dist/folder-operations.d.ts +39 -0
  44. package/dist/folder-operations.js +112 -0
  45. package/dist/folder-placement.d.ts +13 -0
  46. package/dist/folder-placement.js +23 -0
  47. package/dist/heuristics/authenticationResults.d.ts +10 -0
  48. package/dist/heuristics/authenticationResults.js +187 -0
  49. package/dist/heuristics/classifyByHeaders.d.ts +82 -0
  50. package/dist/heuristics/classifyByHeaders.js +365 -0
  51. package/dist/heuristics/classifyPlacement.d.ts +67 -0
  52. package/dist/heuristics/classifyPlacement.js +153 -0
  53. package/dist/heuristics/machineSenders.d.ts +21 -0
  54. package/{src/heuristics/machineSenders.ts → dist/heuristics/machineSenders.js} +34 -40
  55. package/dist/heuristics/senderMismatch.d.ts +81 -0
  56. package/dist/heuristics/senderMismatch.js +192 -0
  57. package/dist/heuristics/socialDomains.d.ts +14 -0
  58. package/{src/heuristics/socialDomains.ts → dist/heuristics/socialDomains.js} +9 -9
  59. package/dist/heuristics/transactionalDomains.d.ts +18 -0
  60. package/{src/heuristics/transactionalDomains.ts → dist/heuristics/transactionalDomains.js} +8 -8
  61. package/dist/imapflow-connection.d.ts +377 -0
  62. package/dist/imapflow-connection.js +1147 -0
  63. package/dist/index.d.ts +63 -0
  64. package/dist/index.js +65 -0
  65. package/dist/list-id-backfill.d.ts +83 -0
  66. package/dist/list-id-backfill.js +132 -0
  67. package/dist/mailbox-cursor-rebuild.d.ts +74 -0
  68. package/dist/mailbox-cursor-rebuild.js +71 -0
  69. package/{src/mailbox-cursor.ts → dist/mailbox-cursor.d.ts} +20 -98
  70. package/dist/mailbox-cursor.js +102 -0
  71. package/dist/mailbox-intent.d.ts +41 -0
  72. package/{src/mailbox-intent.ts → dist/mailbox-intent.js} +39 -63
  73. package/dist/mailbox-management.d.ts +240 -0
  74. package/dist/mailbox-management.js +576 -0
  75. package/dist/mailbox-presence.d.ts +52 -0
  76. package/{src/mailbox-presence.ts → dist/mailbox-presence.js} +19 -30
  77. package/dist/mailbox-queue.d.ts +126 -0
  78. package/dist/mailbox-queue.js +214 -0
  79. package/dist/mailbox-sync.d.ts +93 -0
  80. package/dist/mailbox-sync.js +490 -0
  81. package/dist/mailbox-upstream.d.ts +38 -0
  82. package/{src/mailbox-upstream.ts → dist/mailbox-upstream.js} +37 -47
  83. package/dist/message-category.d.ts +51 -0
  84. package/{src/message-category.ts → dist/message-category.js} +28 -71
  85. package/dist/message-move.d.ts +285 -0
  86. package/dist/message-move.js +858 -0
  87. package/dist/message-parser.d.ts +14 -0
  88. package/dist/message-parser.js +17 -0
  89. package/{src/message-presence.ts → dist/message-presence.d.ts} +1 -13
  90. package/dist/message-presence.js +29 -0
  91. package/dist/message-read.d.ts +51 -0
  92. package/dist/message-read.js +31 -0
  93. package/dist/message-sync.d.ts +345 -0
  94. package/dist/message-sync.js +1358 -0
  95. package/dist/mime-walker.d.ts +102 -0
  96. package/dist/mime-walker.js +198 -0
  97. package/dist/organize-job.d.ts +20 -0
  98. package/dist/organize-job.js +37 -0
  99. package/dist/outbox-attachment-content.d.ts +37 -0
  100. package/dist/outbox-attachment-content.js +50 -0
  101. package/dist/outbox-attachment-filename.d.ts +19 -0
  102. package/dist/outbox-attachment-filename.js +64 -0
  103. package/dist/outbox-attachment.d.ts +148 -0
  104. package/dist/outbox-attachment.js +273 -0
  105. package/dist/outbox-queue.d.ts +73 -0
  106. package/dist/outbox-queue.js +284 -0
  107. package/{src/outbox-status.ts → dist/outbox-status.d.ts} +1 -6
  108. package/dist/outbox-status.js +18 -0
  109. package/dist/ownership.d.ts +24 -0
  110. package/dist/ownership.js +83 -0
  111. package/dist/placement-move-terminal.d.ts +67 -0
  112. package/dist/placement-move-terminal.js +88 -0
  113. package/dist/placement-move.d.ts +83 -0
  114. package/dist/placement-move.js +216 -0
  115. package/{src/placement-settled.ts → dist/placement-settled.d.ts} +19 -50
  116. package/dist/placement-settled.js +75 -0
  117. package/dist/quarantine.d.ts +95 -0
  118. package/dist/quarantine.js +131 -0
  119. package/dist/refusal.d.ts +31 -0
  120. package/dist/refusal.js +3 -0
  121. package/{src/restore-source-placement.ts → dist/restore-source-placement.d.ts} +29 -101
  122. package/dist/restore-source-placement.js +79 -0
  123. package/dist/search-text-backfill.d.ts +15 -0
  124. package/dist/search-text-backfill.js +33 -0
  125. package/dist/snippet.d.ts +61 -0
  126. package/dist/snippet.js +129 -0
  127. package/dist/spam-report.d.ts +86 -0
  128. package/dist/spam-report.js +241 -0
  129. package/dist/sync-warning.d.ts +21 -0
  130. package/dist/sync-warning.js +26 -0
  131. package/dist/sync-watermarks.d.ts +138 -0
  132. package/dist/sync-watermarks.js +205 -0
  133. package/dist/test-helpers/fake-imap.d.ts +151 -0
  134. package/dist/test-helpers/fake-imap.js +508 -0
  135. package/dist/test-helpers/folder-roles.d.ts +34 -0
  136. package/dist/test-helpers/folder-roles.js +47 -0
  137. package/dist/test-helpers/mime-tree-generator.d.ts +82 -0
  138. package/dist/test-helpers/mime-tree-generator.js +329 -0
  139. package/dist/test-helpers/test-store.d.ts +69 -0
  140. package/dist/test-helpers/test-store.js +193 -0
  141. package/dist/text/normalizer.d.ts +20 -0
  142. package/dist/text/normalizer.js +102 -0
  143. package/dist/thread-message-rows.d.ts +69 -0
  144. package/dist/thread-message-rows.js +40 -0
  145. package/dist/thread-root.d.ts +7 -0
  146. package/dist/thread-root.js +25 -0
  147. package/dist/types.d.ts +343 -0
  148. package/dist/types.js +19 -0
  149. package/package.json +86 -73
  150. package/src/account-credentials.ts +0 -210
  151. package/src/adversarial-mime.e2e.test.ts +0 -307
  152. package/src/attribute-mapper.test.ts +0 -104
  153. package/src/attribute-mapper.ts +0 -148
  154. package/src/body-parse.test.ts +0 -55
  155. package/src/body-part-mapper.property.test.ts +0 -354
  156. package/src/body-part-mapper.ts +0 -360
  157. package/src/body-sync-auto-read-writeonce.test.ts +0 -219
  158. package/src/body-sync-calendar-suggestion.test.ts +0 -433
  159. package/src/body-sync-category-denormalize.test.ts +0 -352
  160. package/src/body-sync-category-override.test.ts +0 -501
  161. package/src/body-sync-category-writeonce.test.ts +0 -234
  162. package/src/body-sync-filter.test.ts +0 -237
  163. package/src/body-sync-message-gone.test.ts +0 -139
  164. package/src/body-sync-placement-flags.test.ts +0 -347
  165. package/src/body-sync-placement-writeonce.test.ts +0 -337
  166. package/src/body-sync-quarantine.test.ts +0 -276
  167. package/src/body-sync-queue.test.ts +0 -124
  168. package/src/body-sync-queue.ts +0 -131
  169. package/src/body-sync-resync.test.ts +0 -605
  170. package/src/body-sync-terminal.test.ts +0 -310
  171. package/src/body-sync-terminal.ts +0 -176
  172. package/src/body-sync-unclassifiable-once.test.ts +0 -260
  173. package/src/body-sync-unsubscribed-auto-read.test.ts +0 -176
  174. package/src/body-sync.ts +0 -1970
  175. package/src/calendar-parts.ts +0 -41
  176. package/src/classification-backfill.test.ts +0 -615
  177. package/src/classification-backfill.ts +0 -363
  178. package/src/classify-message.test.ts +0 -205
  179. package/src/classify-message.ts +0 -116
  180. package/src/connection-factory.ts +0 -102
  181. package/src/connection-test.ts +0 -208
  182. package/src/credentials.test.ts +0 -181
  183. package/src/external-move.test.ts +0 -369
  184. package/src/external-move.ts +0 -272
  185. package/src/filters/anchor-drift.ts +0 -55
  186. package/src/filters/filter-config.test.ts +0 -229
  187. package/src/filters/filter-config.ts +0 -69
  188. package/src/filters/list-id.test.ts +0 -50
  189. package/src/filters/match.test.ts +0 -429
  190. package/src/filters/match.ts +0 -274
  191. package/src/filters/message-embedder.test.ts +0 -39
  192. package/src/filters/message-embedder.ts +0 -33
  193. package/src/filters/pipeline.test.ts +0 -330
  194. package/src/filters/pipeline.ts +0 -284
  195. package/src/flag-push-terminal.test.ts +0 -209
  196. package/src/flag-push.test.ts +0 -408
  197. package/src/flag-push.ts +0 -198
  198. package/src/flag-queue-stars.test.ts +0 -171
  199. package/src/flag-queue.ts +0 -509
  200. package/src/folder-placement.ts +0 -56
  201. package/src/heuristics/authenticationResults.test.ts +0 -189
  202. package/src/heuristics/authenticationResults.ts +0 -212
  203. package/src/heuristics/classifyByHeaders.test.ts +0 -889
  204. package/src/heuristics/classifyByHeaders.ts +0 -467
  205. package/src/heuristics/classifyPlacement.test.ts +0 -366
  206. package/src/heuristics/classifyPlacement.ts +0 -211
  207. package/src/heuristics/classifyRealMail.test.ts +0 -242
  208. package/src/heuristics/machineSenders.test.ts +0 -101
  209. package/src/heuristics/senderMismatch.test.ts +0 -346
  210. package/src/heuristics/senderMismatch.ts +0 -259
  211. package/src/imapflow-connect-retry.test.ts +0 -134
  212. package/src/imapflow-connection.e2e.test.ts +0 -335
  213. package/src/imapflow-connection.integ.test.ts +0 -316
  214. package/src/imapflow-connection.test.ts +0 -469
  215. package/src/imapflow-connection.ts +0 -1475
  216. package/src/index.ts +0 -350
  217. package/src/list-id-backfill.test.ts +0 -573
  218. package/src/list-id-backfill.ts +0 -274
  219. package/src/mailbox-cursor-rebuild.test.ts +0 -171
  220. package/src/mailbox-cursor-rebuild.ts +0 -146
  221. package/src/mailbox-cursor.test.ts +0 -249
  222. package/src/mailbox-delete-intent.test.ts +0 -246
  223. package/src/mailbox-management.test.ts +0 -356
  224. package/src/mailbox-management.ts +0 -839
  225. package/src/mailbox-presence.test.ts +0 -72
  226. package/src/mailbox-queue.ts +0 -415
  227. package/src/mailbox-rename-intent.test.ts +0 -389
  228. package/src/mailbox-rename-subtree.test.ts +0 -664
  229. package/src/mailbox-sync.test.ts +0 -848
  230. package/src/mailbox-sync.ts +0 -724
  231. package/src/message-delete-junk-reconcile.test.ts +0 -117
  232. package/src/message-delete-no-trash.test.ts +0 -245
  233. package/src/message-delete-unsettled-placement.test.ts +0 -396
  234. package/src/message-move-copy.test.ts +0 -562
  235. package/src/message-move-empty-trash.test.ts +0 -326
  236. package/src/message-move-same-mailbox.test.ts +0 -139
  237. package/src/message-move-unsettled-placement.test.ts +0 -378
  238. package/src/message-move.ts +0 -1348
  239. package/src/message-parser.ts +0 -26
  240. package/src/message-presence.test.ts +0 -50
  241. package/src/message-sync-address-spoof.test.ts +0 -293
  242. package/src/message-sync-category.test.ts +0 -142
  243. package/src/message-sync-changedsince.test.ts +0 -840
  244. package/src/message-sync-flags.test.ts +0 -158
  245. package/src/message-sync-junk-harvest.test.ts +0 -244
  246. package/src/message-sync-quarantine.test.ts +0 -416
  247. package/src/message-sync-repoint.test.ts +0 -419
  248. package/src/message-sync.ts +0 -1963
  249. package/src/mime-walker.test.ts +0 -311
  250. package/src/mime-walker.ts +0 -305
  251. package/src/outbox-attachment-content.test.ts +0 -153
  252. package/src/outbox-attachment-content.ts +0 -104
  253. package/src/outbox-attachment-filename.test.ts +0 -132
  254. package/src/outbox-attachment-filename.ts +0 -76
  255. package/src/outbox-attachment.test.ts +0 -738
  256. package/src/outbox-attachment.ts +0 -536
  257. package/src/outbox-failed-recovery.test.ts +0 -370
  258. package/src/outbox-queue-settled-row.test.ts +0 -140
  259. package/src/outbox-queue.test.ts +0 -291
  260. package/src/outbox-queue.ts +0 -508
  261. package/src/pass-through-unit-of-work.ts +0 -17
  262. package/src/placement-move-terminal.test.ts +0 -300
  263. package/src/placement-move-terminal.ts +0 -148
  264. package/src/placement-move-unsettled.test.ts +0 -275
  265. package/src/placement-move.ts +0 -336
  266. package/src/placement-settled.test.ts +0 -147
  267. package/src/placement-transition-race.test.ts +0 -482
  268. package/src/quarantine.test.ts +0 -191
  269. package/src/quarantine.ts +0 -228
  270. package/src/snippet.test.ts +0 -213
  271. package/src/snippet.ts +0 -152
  272. package/src/spam-report.test.ts +0 -801
  273. package/src/spam-report.ts +0 -357
  274. package/src/stale-message-reconcile.test.ts +0 -82
  275. package/src/stale-message-reconcile.ts +0 -55
  276. package/src/sync-watermarks.test.ts +0 -409
  277. package/src/sync-watermarks.ts +0 -319
  278. package/src/system-flag-wire-format.test.ts +0 -77
  279. package/src/test-helpers/folder-roles.ts +0 -66
  280. package/src/test-helpers/isolated-mailbox.ts +0 -44
  281. package/src/test-helpers/mailfuzz-connection.ts +0 -27
  282. package/src/test-helpers/mime-tree-generator.ts +0 -461
  283. package/src/text/normalizer.test.ts +0 -223
  284. package/src/text/normalizer.ts +0 -140
  285. package/src/thread-message-rows.ts +0 -77
  286. package/src/types/mailparser-augment.d.ts +0 -15
  287. package/src/types/natural-porter-stemmer.d.ts +0 -50
  288. package/src/types/stopword.d.ts +0 -69
  289. package/src/types.ts +0 -371
  290. package/test/fixtures/mime-adversarial-e2e/01-octet-stream-pdf.eml +0 -21
  291. package/test/fixtures/mime-adversarial-e2e/02-nested-related-cid-image.eml +0 -31
  292. package/test/fixtures/mime-adversarial-e2e/03-calendar-invite.eml +0 -33
  293. package/test/fixtures/mime-adversarial-e2e/04-forwarded-rfc822-with-attachments.eml +0 -42
  294. package/test/fixtures/mime-adversarial-e2e/05-tnef-winmail.eml +0 -25
  295. package/test/fixtures/mime-adversarial-e2e/06-8bit-non-ascii.eml +0 -15
  296. package/test/fixtures/mime-adversarial-e2e/07-quoted-printable-soft-breaks.eml +0 -17
  297. package/test/fixtures/mime-adversarial-e2e/08-alternative-empty-text.eml +0 -18
  298. package/tsconfig.json +0 -8
@@ -1,369 +0,0 @@
1
- import assert from "node:assert/strict";
2
- import { describe, it } from "node:test";
3
- import type {
4
- IMailboxRepository,
5
- IMessageRepository,
6
- MailboxItem,
7
- MessageItem,
8
- } from "@remit/data-ports";
9
- import { MessageStatus, MessageSyncStatus } from "@remit/domain-enums";
10
- import {
11
- confirmDepartures,
12
- placementKey,
13
- sightingContestsPlacement,
14
- } from "./external-move.js";
15
- import type { IImapConnection } from "./types.js";
16
-
17
- const mailboxAt = (
18
- fullPath: string,
19
- specialUse?: string[],
20
- uidValidity = 1,
21
- ): MailboxItem =>
22
- ({
23
- mailboxId: `mbx-${fullPath}`,
24
- fullPath,
25
- uidValidity,
26
- hierarchyDelimiter: "/",
27
- ...(specialUse ? { specialUse } : {}),
28
- }) as MailboxItem;
29
-
30
- const INBOX = mailboxAt("INBOX");
31
- const LABEL = mailboxAt("Receipts");
32
- const JUNK = mailboxAt("INBOX/Rubbish");
33
- const ALL_MAIL = mailboxAt("[Gmail]/All Mail", ["All"]);
34
-
35
- const storedIn = (
36
- mailbox: MailboxItem,
37
- overrides: Partial<MessageItem> = {},
38
- ): MessageItem =>
39
- ({
40
- messageId: "msg-1",
41
- mailboxId: mailbox.mailboxId,
42
- uid: 7,
43
- status: MessageStatus.active,
44
- syncStatus: MessageSyncStatus.pending,
45
- ...overrides,
46
- }) as MessageItem;
47
-
48
- interface Probe {
49
- opened: string[];
50
- searched: string[];
51
- departed: Set<string>;
52
- settled: Set<string>;
53
- }
54
-
55
- /**
56
- * Ask whether the messages this batch sighted in `sightedIn` have left the
57
- * folders their rows point at, against a server where `held` names the uids each
58
- * folder still holds and `served` the UIDVALIDITY it answers on SELECT.
59
- */
60
- const probe = async (
61
- sightedIn: MailboxItem,
62
- rows: MessageItem[],
63
- held: Record<string, number[]>,
64
- served: Record<string, number> = {},
65
- sources: MailboxItem[] = [INBOX],
66
- unreachable: string[] = [],
67
- ): Promise<Probe> => {
68
- const opened: string[] = [];
69
- const searched: string[] = [];
70
-
71
- const connection = {
72
- openBox: async (fullPath: string) => {
73
- if (unreachable.includes(fullPath)) {
74
- throw new Error(`no such mailbox on the server: ${fullPath}`);
75
- }
76
- opened.push(fullPath);
77
- const source = sources.find((box) => box.fullPath === fullPath);
78
- return { uidvalidity: served[fullPath] ?? source?.uidValidity ?? 1 };
79
- },
80
- search: async (criteria: unknown[]) => {
81
- const [[, uids]] = criteria as Array<[string, string]>;
82
- searched.push(uids);
83
- const current = opened[opened.length - 1];
84
- const asked = uids.split(",").map(Number);
85
- return (held[current] ?? []).filter((uid) => asked.includes(uid));
86
- },
87
- } as unknown as Pick<IImapConnection, "openBox" | "search">;
88
-
89
- const verdicts = await confirmDepartures(
90
- {
91
- connection,
92
- mailboxService: {
93
- get: async (_accountId: string, mailboxId: string) => {
94
- const source = sources.find((box) => box.mailboxId === mailboxId);
95
- if (!source) throw new Error(`no such mailbox: ${mailboxId}`);
96
- return source;
97
- },
98
- } as unknown as Pick<IMailboxRepository, "get">,
99
- messageService: {
100
- get: async () => rows,
101
- } as unknown as Pick<IMessageRepository, "get">,
102
- log: { info: () => {}, warn: () => {} },
103
- },
104
- "acct-1",
105
- sightedIn,
106
- rows.map((row) => row.messageId),
107
- );
108
-
109
- return { opened, searched, ...verdicts };
110
- };
111
-
112
- describe("whether a sighting in a second folder is a move", () => {
113
- /**
114
- * The reported bug (#1146). A Gmail user label, a Sieve `fileinto` beside a
115
- * `keep` and a Sent copy a list echoes back all put one message in two real
116
- * folders. Reading the second folder's sighting as a move took the message
117
- * out of the Inbox on the round that enumerated the label.
118
- */
119
- it("is not a move while the folder the row points at still holds it", async () => {
120
- const observed = await probe(LABEL, [storedIn(INBOX)], {
121
- INBOX: [7],
122
- });
123
-
124
- assert.deepEqual([...observed.departed], []);
125
- });
126
-
127
- it("is a move once the folder the row points at has let it go", async () => {
128
- const observed = await probe(LABEL, [storedIn(INBOX)], {
129
- INBOX: [3, 11],
130
- });
131
-
132
- assert.deepEqual(
133
- [...observed.departed],
134
- [placementKey("msg-1", INBOX.mailboxId, 7)],
135
- );
136
- });
137
-
138
- it("names the placement it was answered for, so a row that moved since is left alone", async () => {
139
- const observed = await probe(JUNK, [storedIn(INBOX, { uid: 42 })], {
140
- INBOX: [],
141
- });
142
-
143
- assert.equal(
144
- observed.departed.has(placementKey("msg-1", INBOX.mailboxId, 7)),
145
- false,
146
- );
147
- assert.equal(
148
- observed.departed.has(placementKey("msg-1", INBOX.mailboxId, 42)),
149
- true,
150
- );
151
- });
152
-
153
- it("asks each source folder once for the whole batch", async () => {
154
- const observed = await probe(
155
- LABEL,
156
- [
157
- storedIn(INBOX, { messageId: "msg-1", uid: 7 }),
158
- storedIn(INBOX, { messageId: "msg-2", uid: 8 }),
159
- storedIn(INBOX, { messageId: "msg-3", uid: 9 }),
160
- ],
161
- { INBOX: [7] },
162
- );
163
-
164
- assert.deepEqual(observed.opened, ["INBOX"]);
165
- assert.deepEqual(observed.searched, ["7,8,9"]);
166
- });
167
-
168
- /**
169
- * The answer needed is about the contested uids. A SEARCH ALL would pull a
170
- * page of uids per batch out of every source folder an account has, which on
171
- * a large Inbox is traffic proportional to the mailbox rather than to the
172
- * question.
173
- */
174
- it("asks about the contested uids, not the whole source folder", async () => {
175
- const observed = await probe(LABEL, [storedIn(INBOX, { uid: 42 })], {
176
- INBOX: [1, 2, 42],
177
- });
178
-
179
- assert.deepEqual(observed.searched, ["42"]);
180
- assert.deepEqual([...observed.departed], []);
181
- });
182
-
183
- it("asks nothing at all when no row is contested", async () => {
184
- const observed = await probe(INBOX, [storedIn(INBOX)], { INBOX: [7] });
185
-
186
- assert.deepEqual(observed.opened, []);
187
- assert.deepEqual([...observed.departed], []);
188
- });
189
-
190
- it("asks nothing for a folder that holds a copy of every message", async () => {
191
- const observed = await probe(ALL_MAIL, [storedIn(INBOX)], { INBOX: [] });
192
-
193
- assert.deepEqual(observed.opened, []);
194
- assert.deepEqual([...observed.departed], []);
195
- });
196
-
197
- it("asks nothing for a row whose own mutation has not settled", async () => {
198
- const observed = await probe(
199
- LABEL,
200
- [storedIn(INBOX, { status: MessageStatus.moving })],
201
- { INBOX: [] },
202
- );
203
-
204
- assert.deepEqual(observed.opened, []);
205
- assert.deepEqual([...observed.departed], []);
206
- });
207
-
208
- /**
209
- * All Mail, Starred and Important hold a copy of every message and release
210
- * nothing, so a row that landed on one — its sync ran before the real
211
- * folder's — would answer "still here" forever and freeze there. Reachable on
212
- * any Gmail account whose mail takes more than one batch.
213
- */
214
- it("frees a row whose source is a folder that copies every message", async () => {
215
- const observed = await probe(
216
- LABEL,
217
- [storedIn(ALL_MAIL)],
218
- { "[Gmail]/All Mail": [7] },
219
- {},
220
- [ALL_MAIL],
221
- );
222
-
223
- assert.deepEqual(observed.opened, []);
224
- assert.deepEqual(
225
- [...observed.departed],
226
- [placementKey("msg-1", ALL_MAIL.mailboxId, 7)],
227
- );
228
- });
229
-
230
- /**
231
- * Reader shares its mailboxes, so another client renaming or deleting the
232
- * source folder between the row being written and this round is ordinary. It
233
- * decides nothing about these rows, and it may not fail the sync of the
234
- * unrelated folder that happened to sight them.
235
- */
236
- it("leaves a source folder it cannot reach undecided rather than failing the round", async () => {
237
- const observed = await probe(
238
- LABEL,
239
- [storedIn(INBOX)],
240
- { INBOX: [] },
241
- {},
242
- [INBOX],
243
- ["INBOX"],
244
- );
245
-
246
- assert.deepEqual([...observed.departed], []);
247
- assert.deepEqual([...observed.settled], []);
248
- });
249
-
250
- it("settles a decided sighting, so nothing holds the watermark for it", async () => {
251
- const observed = await probe(LABEL, [storedIn(INBOX)], { INBOX: [7] });
252
-
253
- assert.deepEqual(
254
- [...observed.settled],
255
- [placementKey("msg-1", INBOX.mailboxId, 7)],
256
- );
257
- });
258
-
259
- /**
260
- * A uid means nothing without the axis it was issued on (RFC 9051 2.3.1.1),
261
- * so a source that answers a different UIDVALIDITY is evidence of nothing.
262
- * The cursor rebuild re-keys that folder; until it has, the sighting waits.
263
- */
264
- it("refuses a source folder that has been re-keyed under it", async () => {
265
- const observed = await probe(
266
- LABEL,
267
- [storedIn(INBOX)],
268
- { INBOX: [] },
269
- {
270
- INBOX: 99,
271
- },
272
- );
273
-
274
- assert.deepEqual([...observed.departed], []);
275
- assert.deepEqual([...observed.settled], []);
276
- });
277
- });
278
-
279
- describe("sightingContestsPlacement", () => {
280
- it("refuses a Gmail virtual folder the server never flagged", () => {
281
- assert.equal(
282
- sightingContestsPlacement(mailboxAt("[Gmail]/All Mail"), storedIn(INBOX)),
283
- false,
284
- );
285
- });
286
-
287
- it("accepts a folder the user named after a virtual one", () => {
288
- assert.equal(
289
- sightingContestsPlacement(mailboxAt("Starred ideas"), storedIn(INBOX)),
290
- true,
291
- );
292
- });
293
-
294
- it("accepts an ordinary inbound row the sync path left pending", () => {
295
- assert.equal(
296
- sightingContestsPlacement(
297
- JUNK,
298
- storedIn(INBOX, { syncStatus: MessageSyncStatus.pending }),
299
- ),
300
- true,
301
- );
302
- });
303
-
304
- it("accepts a row a settled mutation marked synced", () => {
305
- assert.equal(
306
- sightingContestsPlacement(
307
- JUNK,
308
- storedIn(INBOX, { syncStatus: MessageSyncStatus.synced }),
309
- ),
310
- true,
311
- );
312
- });
313
-
314
- /**
315
- * The row `abandonDelete` hands back: reader refused the delete, put the
316
- * message back where the server still has it, and nothing else is coming
317
- * for the row. Reader shares its mailboxes, so the user moving that same
318
- * message in another client is ordinary — and before R3 the sighting was
319
- * refused on `syncStatus` alone and the row never followed the move.
320
- */
321
- it("accepts a row whose delete was abandoned, so a move made elsewhere still lands", () => {
322
- assert.equal(
323
- sightingContestsPlacement(
324
- JUNK,
325
- storedIn(INBOX, {
326
- status: MessageStatus.active,
327
- syncStatus: MessageSyncStatus.abandoned,
328
- }),
329
- ),
330
- true,
331
- );
332
- });
333
-
334
- it("accepts a row left `failed` by a transient attempt that has since settled", () => {
335
- assert.equal(
336
- sightingContestsPlacement(
337
- JUNK,
338
- storedIn(INBOX, { syncStatus: MessageSyncStatus.failed }),
339
- ),
340
- true,
341
- );
342
- });
343
-
344
- it("refuses a row whose own move is still in flight", () => {
345
- assert.equal(
346
- sightingContestsPlacement(
347
- JUNK,
348
- storedIn(INBOX, {
349
- status: MessageStatus.moving,
350
- syncStatus: MessageSyncStatus.pending,
351
- }),
352
- ),
353
- false,
354
- );
355
- });
356
-
357
- it("refuses a row whose own delete is still in flight", () => {
358
- assert.equal(
359
- sightingContestsPlacement(
360
- JUNK,
361
- storedIn(INBOX, {
362
- status: MessageStatus.deleting,
363
- syncStatus: MessageSyncStatus.pending,
364
- }),
365
- ),
366
- false,
367
- );
368
- });
369
- });
@@ -1,272 +0,0 @@
1
- import type {
2
- IMailboxRepository,
3
- IMessageRepository,
4
- MailboxItem,
5
- MessageItem,
6
- } from "@remit/data-ports";
7
- import { isVirtualCopyMailbox } from "@remit/data-ports/virtual-copy";
8
- import { MessageStatus } from "@remit/domain-enums";
9
- import type { IImapConnection } from "./types.js";
10
-
11
- export interface ExternalMoveLogger {
12
- info(obj: Record<string, unknown>, msg: string): void;
13
- warn(obj: Record<string, unknown>, msg: string): void;
14
- }
15
-
16
- /**
17
- * Whether a sighting of an already-stored message in this mailbox CONTESTS the
18
- * placement the row holds (#859). Contesting is not deciding: the sighting is a
19
- * candidate move, and {@link confirmDepartures} is what turns it into one.
20
- *
21
- * Three sightings do not even contest. The same mailbox is the row standing
22
- * where it already is. Gmail's All Mail, Starred and Important hold every
23
- * message that also lives in a real folder, so they are views rather than filing
24
- * locations; following one would empty every other folder's listing. And a row
25
- * whose own mutation has not settled is in flight, not stale
26
- * (imap-mutations R1): the server's answer predates the move reader is still
27
- * waiting on, so taking it would undo the optimistic write. The sync path
28
- * reconciles rather than waits (imap-mutations R2): it never blocks on a
29
- * mutation, it re-reads the row on the next sighting after the move settles.
30
- *
31
- * In flight is `status`, and `status` alone (#1096, imap-mutations R3). Every
32
- * outbound mutation writes `moving` or `deleting` and only settling returns the
33
- * row to `active`, so `active` is precisely the set nothing is coming for —
34
- * whatever `syncStatus` says about how the row got there. A row put back by
35
- * `abandonDelete` carries `abandoned`, and it needs this repair more than any
36
- * other row does: nothing routine settles it, and reader shares its mailboxes,
37
- * so refusing it left a message whose delete was refused unable to follow a
38
- * move the user then made in another client.
39
- */
40
- export const sightingContestsPlacement = (
41
- mailbox: MailboxItem,
42
- message: Pick<MessageItem, "mailboxId" | "status">,
43
- ): boolean => {
44
- if (message.mailboxId === mailbox.mailboxId) return false;
45
- if (isVirtualCopyMailbox(mailbox)) return false;
46
- return message.status === MessageStatus.active;
47
- };
48
-
49
- /**
50
- * The exact placement a verdict was reached for. The probe reads the row outside
51
- * the transaction that acts on it, so the verdict names the mailbox and uid it
52
- * was reached for; a row that moved in between matches no verdict and is left
53
- * unsettled.
54
- */
55
- export const placementKey = (
56
- messageId: string,
57
- mailboxId: string,
58
- uid: number,
59
- ): string => `${messageId}:${mailboxId}:${uid}`;
60
-
61
- export interface DepartureVerdicts {
62
- /**
63
- * Placements the source folder no longer holds. A sighting elsewhere of one
64
- * of these is a move, and re-points the row.
65
- */
66
- departed: Set<string>;
67
- /**
68
- * Placements a verdict was reached for at all, departed or still held. A
69
- * contested sighting outside this set was not decided — the source could not
70
- * be asked, or the row has moved since it was — and its uid is held back from
71
- * the watermark so the next round serves the sighting again. Without that a
72
- * single unanswerable round would leave the row mispointed until a cursor
73
- * rebuild, which is the silent never-repair this whole change is about.
74
- */
75
- settled: Set<string>;
76
- }
77
-
78
- export interface DepartureProbeDeps {
79
- connection: Pick<IImapConnection, "openBox" | "search">;
80
- mailboxService: Pick<IMailboxRepository, "get">;
81
- messageService: Pick<IMessageRepository, "get">;
82
- log: ExternalMoveLogger;
83
- }
84
-
85
- /**
86
- * Which of this batch's messages have really left the folder their row points at
87
- * (#1146).
88
- *
89
- * A message sighted in a second real folder has not necessarily moved. Living in
90
- * two folders at once is ordinary mail: a Gmail user label, a Sieve `fileinto`
91
- * beside a `keep`, a Sent copy a mailing list echoes back. Reading every such
92
- * sighting as a move takes the message out of the folder it is still in, and
93
- * whichever folder the sync round enumerates last wins — which is how a labelled
94
- * message disappeared from the Inbox on the next sync.
95
- *
96
- * What separates a move from a copy is the source folder: a move empties it, a
97
- * copy leaves it alone. So the source is asked, and only a source that no longer
98
- * holds the uid the row claims makes the sighting a move. A move another client
99
- * made is still followed — that is the case where the source really is empty —
100
- * and reader stops fighting a second client over mail neither of them moved.
101
- *
102
- * The question is asked with a UID SEARCH rather than a FETCH, for the reason
103
- * `isMessageGoneFromOpenMailbox` documents: imapflow drops rows on back-to-back
104
- * FETCHes (#408), and absence read off a FETCH would file live mail as departed.
105
- * `IImapConnection.search` throws on a failed SEARCH, so an empty answer means
106
- * the server matched nothing. The SEARCH names the contested uids rather than
107
- * ALL: the answer needed is about those uids, and a folder's whole uid set is a
108
- * page of traffic per batch for an account with a large Inbox.
109
- *
110
- * One SEARCH per source folder per batch, and no IMAP at all when nothing is
111
- * contested — which is every round on an account whose folders do not overlap.
112
- */
113
- export const confirmDepartures = async (
114
- deps: DepartureProbeDeps,
115
- accountId: string,
116
- sightedIn: MailboxItem,
117
- messageIds: string[],
118
- ): Promise<DepartureVerdicts> => {
119
- const verdicts: DepartureVerdicts = {
120
- departed: new Set<string>(),
121
- settled: new Set<string>(),
122
- };
123
- if (messageIds.length === 0) return verdicts;
124
-
125
- const stored = await deps.messageService.get(messageIds);
126
- // A row carrying no uid names no placement the source can be asked about, so
127
- // it reaches no verdict here and its sighting is served again next round. It
128
- // is not reachable from a sync — `upsertWithStatus` writes the uid the FETCH
129
- // returned — and a folder that somehow produces one stalls its cursor and
130
- // raises the alert rather than repairing a row against nothing.
131
- const contested = stored.filter(
132
- (row) => row.uid > 0 && sightingContestsPlacement(sightedIn, row),
133
- );
134
- if (contested.length === 0) return verdicts;
135
-
136
- const bySource = new Map<string, MessageItem[]>();
137
- for (const row of contested) {
138
- const group = bySource.get(row.mailboxId);
139
- if (group === undefined) {
140
- bySource.set(row.mailboxId, [row]);
141
- continue;
142
- }
143
- group.push(row);
144
- }
145
-
146
- for (const [sourceMailboxId, rows] of bySource) {
147
- await askSource(
148
- deps,
149
- accountId,
150
- sightedIn,
151
- sourceMailboxId,
152
- rows,
153
- verdicts,
154
- );
155
- }
156
-
157
- return verdicts;
158
- };
159
-
160
- /**
161
- * What one source folder said about the rows pointed at it. Only `holds` decides
162
- * anything; the other three are the folder declining to be evidence, each for
163
- * its own reason and each leaving the sightings to come back next round.
164
- */
165
- type SourceAnswer =
166
- | { kind: "holds"; path: string; held: Set<number> }
167
- | { kind: "copiesEverything"; path: string }
168
- | { kind: "rekeyed"; path: string; stored: number; served: number }
169
- | { kind: "unreachable"; reason: string };
170
-
171
- const readSource = async (
172
- deps: DepartureProbeDeps,
173
- accountId: string,
174
- sourceMailboxId: string,
175
- rows: MessageItem[],
176
- ): Promise<SourceAnswer> => {
177
- const source = await deps.mailboxService.get(accountId, sourceMailboxId);
178
-
179
- // A row whose source is All Mail, Starred or Important is not filed there:
180
- // those folders hold a copy of everything and never release anything, so
181
- // asking them would answer "still here" forever and freeze the row on a view.
182
- if (isVirtualCopyMailbox(source)) {
183
- return { kind: "copiesEverything", path: source.fullPath };
184
- }
185
-
186
- const box = await deps.connection.openBox(source.fullPath, true);
187
-
188
- // A source folder on a different UIDVALIDITY axis holds no uid these rows can
189
- // be compared against, so its message set is evidence of nothing (RFC 9051
190
- // 2.3.1.1). The cursor rebuild re-keys that folder; until it has, the
191
- // sightings stay unsettled and come back next round.
192
- if (box.uidvalidity !== source.uidValidity) {
193
- return {
194
- kind: "rekeyed",
195
- path: source.fullPath,
196
- stored: source.uidValidity,
197
- served: box.uidvalidity,
198
- };
199
- }
200
-
201
- const held = await deps.connection.search([
202
- ["UID", rows.map((row) => row.uid).join(",")],
203
- ]);
204
- return { kind: "holds", path: source.fullPath, held: new Set(held) };
205
- };
206
-
207
- const askSource = async (
208
- deps: DepartureProbeDeps,
209
- accountId: string,
210
- sightedIn: MailboxItem,
211
- sourceMailboxId: string,
212
- rows: MessageItem[],
213
- verdicts: DepartureVerdicts,
214
- ): Promise<void> => {
215
- const settle = (row: MessageItem, departed: boolean): void => {
216
- const key = placementKey(row.messageId, row.mailboxId, row.uid);
217
- verdicts.settled.add(key);
218
- if (departed) verdicts.departed.add(key);
219
- };
220
-
221
- // A source folder reader cannot reach decides nothing, and does not fail the
222
- // round either. Another client renaming or deleting it between the row being
223
- // written and now is ordinary — reader shares its mailboxes — and the sync of
224
- // the unrelated folder that sighted these messages must survive it.
225
- const answer = await readSource(
226
- deps,
227
- accountId,
228
- sourceMailboxId,
229
- rows,
230
- ).catch<SourceAnswer>((error: unknown) => ({
231
- kind: "unreachable",
232
- reason: error instanceof Error ? error.message : String(error),
233
- }));
234
-
235
- const observed = {
236
- sourceMailboxId,
237
- sightedMailboxId: sightedIn.mailboxId,
238
- contested: rows.length,
239
- };
240
-
241
- if (answer.kind === "unreachable") {
242
- deps.log.warn(
243
- { ...observed, error: answer.reason },
244
- "Could not ask the source folder whether it still holds these messages; leaving the sightings unsettled this round",
245
- );
246
- return;
247
- }
248
-
249
- if (answer.kind === "rekeyed") {
250
- deps.log.warn(
251
- {
252
- ...observed,
253
- sourceMailboxPath: answer.path,
254
- storedUidValidity: answer.stored,
255
- servedUidValidity: answer.served,
256
- },
257
- "Source folder is on a different UIDVALIDITY axis; leaving these sightings unsettled this round",
258
- );
259
- return;
260
- }
261
-
262
- if (answer.kind === "copiesEverything") {
263
- for (const row of rows) settle(row, true);
264
- deps.log.info(
265
- { ...observed, sourceMailboxPath: answer.path },
266
- "Row points at a folder that copies every message; the sighting in a real folder is where it lives",
267
- );
268
- return;
269
- }
270
-
271
- for (const row of rows) settle(row, !answer.held.has(row.uid));
272
- };
@@ -1,55 +0,0 @@
1
- import type {
2
- FilterAnchorItem,
3
- IFilterAnchorRepository,
4
- } from "@remit/data-ports";
5
-
6
- /**
7
- * The half of an embedding service an anchor refresh needs: the text-to-vector
8
- * call and the `<modelId>@<dimensions>` identifier of the model behind it (the
9
- * same scheme `EmbeddingService.embeddingId` derives). Compared against
10
- * `FilterAnchor.anchorEmbeddingId` to detect a model drift a same-dimension
11
- * swap would otherwise pass through silently (RFC 039 Decision 1a, reader
12
- * #295).
13
- */
14
- export interface AnchorEmbedder {
15
- embed(text: string): Promise<number[]>;
16
- readonly embeddingId: string;
17
- }
18
-
19
- export interface AnchorDriftDeps {
20
- anchorRepository: Pick<IFilterAnchorRepository, "put">;
21
- /** Absent on a deployment with no embedder wired; then nothing can drift. */
22
- embedder: AnchorEmbedder | undefined;
23
- }
24
-
25
- /**
26
- * The anchor to score against, re-embedded in place when the embedding model
27
- * has drifted since it was written (RFC 039 Decision 1a). No migration job
28
- * walks these rows proactively, so the refresh is lazy: the first read that
29
- * notices the stamp no longer matches the configured model re-embeds the
30
- * already-persisted `anchorSourceText` and writes it back under the current
31
- * id. A failure here is never a terminal state — the row is left as it was,
32
- * so the next read that reaches this anchor retries.
33
- *
34
- * The single mechanism behind both index-time matching
35
- * ({@link FilterPipeline}) and the back-apply pass's cross-filter precedence
36
- * check, which must agree on what a filter currently matches (reader #399).
37
- */
38
- export const refreshAnchorForEmbedder = async (
39
- deps: AnchorDriftDeps,
40
- anchor: FilterAnchorItem,
41
- ): Promise<FilterAnchorItem> => {
42
- const { embedder } = deps;
43
- if (!embedder || anchor.anchorEmbeddingId === embedder.embeddingId) {
44
- return anchor;
45
- }
46
- const anchorEmbedding = await embedder.embed(anchor.anchorSourceText);
47
- return deps.anchorRepository.put({
48
- accountConfigId: anchor.accountConfigId,
49
- filterId: anchor.filterId,
50
- anchorEmbedding,
51
- anchorEmbeddingId: embedder.embeddingId,
52
- anchorSourceText: anchor.anchorSourceText,
53
- anchorMessageId: anchor.anchorMessageId,
54
- });
55
- };