@remit/mailbox-service 0.0.102 → 0.0.103

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@remit/mailbox-service",
3
- "version": "0.0.102",
3
+ "version": "0.0.103",
4
4
  "type": "module",
5
5
  "main": "src/index.ts",
6
6
  "types": "src/index.ts",
@@ -160,7 +160,7 @@ describe("unsubscribed auto-mark-read is decided once (issue #499)", () => {
160
160
  "m-1",
161
161
  "acc-1",
162
162
  "cfg-1",
163
- "INBOX",
163
+ { fullPath: "INBOX", placement: "inbox" },
164
164
  async () => bodyConnection(),
165
165
  );
166
166
 
@@ -181,7 +181,7 @@ describe("unsubscribed auto-mark-read is decided once (issue #499)", () => {
181
181
  "m-1",
182
182
  "acc-1",
183
183
  "cfg-1",
184
- "INBOX",
184
+ { fullPath: "INBOX", placement: "inbox" },
185
185
  async () => bodyConnection(),
186
186
  );
187
187
 
@@ -204,7 +204,7 @@ describe("unsubscribed auto-mark-read is decided once (issue #499)", () => {
204
204
  ["m-1"],
205
205
  "acc-1",
206
206
  "cfg-1",
207
- "INBOX",
207
+ { fullPath: "INBOX", placement: "inbox" },
208
208
  async () => bodyConnection(),
209
209
  true,
210
210
  );
@@ -346,7 +346,7 @@ const readBody = async (
346
346
  messageId,
347
347
  "acc-1",
348
348
  "cfg-1",
349
- "INBOX",
349
+ { fullPath: "INBOX", placement: "inbox" },
350
350
  async () => connection,
351
351
  );
352
352
  };
@@ -195,7 +195,7 @@ const syncBatch = (harness: Harness) =>
195
195
  ["m-1"],
196
196
  "acc-1",
197
197
  "cfg-1",
198
- "INBOX",
198
+ { fullPath: "INBOX", placement: "inbox" },
199
199
  async () =>
200
200
  ({
201
201
  openBox: async () => {},
@@ -212,7 +212,7 @@ const syncFromImap = (harness: Harness) =>
212
212
  "m-1",
213
213
  "acc-1",
214
214
  "cfg-1",
215
- "INBOX",
215
+ { fullPath: "INBOX", placement: "inbox" },
216
216
  async () =>
217
217
  ({
218
218
  openBox: async () => {},
@@ -199,7 +199,7 @@ describe("Address.flags.category overrides classification at sync time (issue #2
199
199
  "m-1",
200
200
  "acc-1",
201
201
  "cfg-1",
202
- "INBOX",
202
+ { fullPath: "INBOX", placement: "inbox" },
203
203
  async () => connection,
204
204
  );
205
205
 
@@ -229,7 +229,7 @@ describe("Address.flags.category overrides classification at sync time (issue #2
229
229
  "m-1",
230
230
  "acc-1",
231
231
  "cfg-1",
232
- "INBOX",
232
+ { fullPath: "INBOX", placement: "inbox" },
233
233
  async () => connection,
234
234
  );
235
235
 
@@ -251,7 +251,7 @@ describe("Address.flags.category overrides classification at sync time (issue #2
251
251
  "m-1",
252
252
  "acc-1",
253
253
  "cfg-1",
254
- "INBOX",
254
+ { fullPath: "INBOX", placement: "inbox" },
255
255
  async () => connection,
256
256
  );
257
257
 
@@ -276,7 +276,7 @@ describe("Address.flags.category overrides classification at sync time (issue #2
276
276
  "m-1",
277
277
  "acc-1",
278
278
  "cfg-1",
279
- "INBOX",
279
+ { fullPath: "INBOX", placement: "inbox" },
280
280
  async () => connection,
281
281
  );
282
282
 
@@ -296,7 +296,7 @@ describe("Address.flags.category overrides classification at sync time (issue #2
296
296
  ["m-1"],
297
297
  "acc-1",
298
298
  "cfg-1",
299
- "INBOX",
299
+ { fullPath: "INBOX", placement: "inbox" },
300
300
  async () =>
301
301
  ({
302
302
  openBox: async () => {},
@@ -336,7 +336,7 @@ describe("Address.flags.category overrides classification at sync time (issue #2
336
336
  ["m-1"],
337
337
  "acc-1",
338
338
  "cfg-1",
339
- "INBOX",
339
+ { fullPath: "INBOX", placement: "inbox" },
340
340
  async () => {
341
341
  throw new Error("an examined message must not be fetched again");
342
342
  },
@@ -376,7 +376,7 @@ describe("Address.flags.category overrides classification at sync time (issue #2
376
376
  "m-1",
377
377
  "acc-1",
378
378
  "cfg-1",
379
- "INBOX",
379
+ { fullPath: "INBOX", placement: "inbox" },
380
380
  async () => connection,
381
381
  );
382
382
 
@@ -413,7 +413,7 @@ describe("Address.flags.category overrides classification at sync time (issue #2
413
413
  ["m-1"],
414
414
  "acc-1",
415
415
  "cfg-1",
416
- "INBOX",
416
+ { fullPath: "INBOX", placement: "inbox" },
417
417
  async () => connection,
418
418
  true,
419
419
  );
@@ -492,7 +492,7 @@ describe("Address.flags.category overrides classification at sync time (issue #2
492
492
  "m-1",
493
493
  "acc-1",
494
494
  "cfg-1",
495
- "INBOX",
495
+ { fullPath: "INBOX", placement: "inbox" },
496
496
  async () => connection,
497
497
  ),
498
498
  /ProvisionedThroughputExceededException/,
@@ -150,7 +150,7 @@ describe("Message.category survives a re-entrant classification pass", () => {
150
150
  "m-1",
151
151
  "acc-1",
152
152
  "cfg-1",
153
- "INBOX",
153
+ { fullPath: "INBOX", placement: "inbox" },
154
154
  async () => connection,
155
155
  );
156
156
 
@@ -188,7 +188,7 @@ describe("Message.category survives a re-entrant classification pass", () => {
188
188
  ["m-1"],
189
189
  "acc-1",
190
190
  "cfg-1",
191
- "INBOX",
191
+ { fullPath: "INBOX", placement: "inbox" },
192
192
  async () => connection,
193
193
  true,
194
194
  );
@@ -220,7 +220,7 @@ describe("Message.category survives a re-entrant classification pass", () => {
220
220
  "m-1",
221
221
  "acc-1",
222
222
  "cfg-1",
223
- "INBOX",
223
+ { fullPath: "INBOX", placement: "inbox" },
224
224
  async () => connection,
225
225
  );
226
226
 
@@ -181,7 +181,7 @@ const readBody = async (service: BodySyncService, mailboxPath = "INBOX") => {
181
181
  "m-1",
182
182
  "acc-1",
183
183
  "cfg-1",
184
- mailboxPath,
184
+ { fullPath: mailboxPath, placement: "other" },
185
185
  async () => connection,
186
186
  );
187
187
  };
@@ -161,7 +161,7 @@ const readBody = async (
161
161
  "m-1",
162
162
  "acc-1",
163
163
  "cfg-1",
164
- mailboxPath,
164
+ { fullPath: mailboxPath, placement: "other" },
165
165
  async () => connection,
166
166
  );
167
167
  };
@@ -187,7 +187,7 @@ describe("placement survives a re-entrant computePlacement pass (issue #383)", (
187
187
  "m-1",
188
188
  "acc-1",
189
189
  "cfg-1",
190
- "INBOX",
190
+ { fullPath: "INBOX", placement: "inbox" },
191
191
  async () => connection,
192
192
  );
193
193
 
@@ -233,7 +233,7 @@ describe("placement survives a re-entrant computePlacement pass (issue #383)", (
233
233
  ["m-1"],
234
234
  "acc-1",
235
235
  "cfg-1",
236
- "INBOX",
236
+ { fullPath: "INBOX", placement: "inbox" },
237
237
  async () => connection,
238
238
  true,
239
239
  );
@@ -268,7 +268,7 @@ describe("placement survives a re-entrant computePlacement pass (issue #383)", (
268
268
  "m-1",
269
269
  "acc-1",
270
270
  "cfg-1",
271
- "INBOX",
271
+ { fullPath: "INBOX", placement: "inbox" },
272
272
  async () => connection,
273
273
  );
274
274
 
@@ -309,7 +309,7 @@ describe("placement survives a re-entrant computePlacement pass (issue #383)", (
309
309
  "m-1",
310
310
  "acc-1",
311
311
  "cfg-1",
312
- "INBOX",
312
+ { fullPath: "INBOX", placement: "inbox" },
313
313
  async () => connection,
314
314
  );
315
315
 
@@ -162,8 +162,12 @@ const connection = () =>
162
162
  }) as unknown as IImapConnection;
163
163
 
164
164
  const sync = (service: BodySyncService) =>
165
- service.syncBodies(["m-1"], "acc-1", "cfg-1", "INBOX", async () =>
166
- connection(),
165
+ service.syncBodies(
166
+ ["m-1"],
167
+ "acc-1",
168
+ "cfg-1",
169
+ { fullPath: "INBOX", placement: "inbox" },
170
+ async () => connection(),
167
171
  );
168
172
 
169
173
  const parseRefusal = () =>
@@ -258,7 +262,7 @@ describe("body sync skips what is already quarantined", () => {
258
262
  ["m-1"],
259
263
  "acc-1",
260
264
  "cfg-1",
261
- "INBOX",
265
+ { fullPath: "INBOX", placement: "inbox" },
262
266
  async () => {
263
267
  throw new Error("a quarantined message must not be fetched again");
264
268
  },
@@ -275,7 +275,7 @@ describe("issue #1011: forced re-store does not re-fire standing rules", () => {
275
275
  ["m-1"],
276
276
  "acc-1",
277
277
  "cfg-1",
278
- "INBOX",
278
+ { fullPath: "INBOX", placement: "inbox" },
279
279
  async () => connection,
280
280
  true,
281
281
  );
@@ -330,7 +330,7 @@ describe("issue #1011: forced re-store does not re-fire standing rules", () => {
330
330
  ["m-1"],
331
331
  "acc-1",
332
332
  "cfg-1",
333
- "INBOX",
333
+ { fullPath: "INBOX", placement: "inbox" },
334
334
  async () => connection,
335
335
  true,
336
336
  );
@@ -374,7 +374,7 @@ describe("issue #1011: forced re-store does not re-fire standing rules", () => {
374
374
  "m-1",
375
375
  "acc-1",
376
376
  "cfg-1",
377
- "INBOX",
377
+ { fullPath: "INBOX", placement: "inbox" },
378
378
  async () => noSuchKeyConn(DEMOTE_EML),
379
379
  );
380
380
 
@@ -415,7 +415,7 @@ describe("issue #1011: forced re-store does not re-fire standing rules", () => {
415
415
  "m-1",
416
416
  "acc-1",
417
417
  "cfg-1",
418
- "INBOX",
418
+ { fullPath: "INBOX", placement: "inbox" },
419
419
  async () => noSuchKeyConn(INVITATION_EML),
420
420
  );
421
421
 
@@ -569,7 +569,7 @@ describe("issue #1011: forced re-store does not re-fire standing rules", () => {
569
569
  ["m-stored", "m-new"],
570
570
  "acc-1",
571
571
  "cfg-1",
572
- "INBOX",
572
+ { fullPath: "INBOX", placement: "inbox" },
573
573
  async () => connection,
574
574
  true,
575
575
  );
@@ -167,8 +167,12 @@ const imapConnection = () =>
167
167
  }) as unknown as IImapConnection;
168
168
 
169
169
  const syncPass = (harness: Harness) =>
170
- harness.service.syncBodies(["m-1"], "acc-1", "cfg-1", "INBOX", async () =>
171
- imapConnection(),
170
+ harness.service.syncBodies(
171
+ ["m-1"],
172
+ "acc-1",
173
+ "cfg-1",
174
+ { fullPath: "INBOX", placement: "inbox" },
175
+ async () => imapConnection(),
172
176
  );
173
177
 
174
178
  describe("a body the classifier declined to categorize is examined once", () => {
@@ -129,7 +129,7 @@ const readBody = async (
129
129
  "m-1",
130
130
  "acc-1",
131
131
  "cfg-1",
132
- "INBOX",
132
+ { fullPath: "INBOX", placement: "inbox" },
133
133
  async () => connection,
134
134
  );
135
135
  };
package/src/body-sync.ts CHANGED
@@ -51,6 +51,7 @@ import {
51
51
  FilterPipeline,
52
52
  } from "./filters/pipeline.js";
53
53
  import type { FlagQueueService } from "./flag-queue.js";
54
+ import { placementOf, resolveFolderRoleMailboxes } from "./folder-placement.js";
54
55
  import {
55
56
  classifyPlacement,
56
57
  type FolderPlacement,
@@ -239,6 +240,11 @@ export interface StoreBodyPartContentsResult {
239
240
 
240
241
  export type ConnectionGetter = () => Promise<IImapConnection>;
241
242
 
243
+ export interface BodySyncFolder {
244
+ fullPath: string;
245
+ placement: FolderPlacement;
246
+ }
247
+
242
248
  export interface PlacementConfig {
243
249
  mailboxSpecialUseService: IMailboxSpecialUseRepository;
244
250
  /**
@@ -332,7 +338,7 @@ export class BodySyncService {
332
338
  * @param messageIds - The message IDs to sync bodies for
333
339
  * @param accountId - The account ID (for storage path)
334
340
  * @param accountConfigId - The account config ID (for thread updates)
335
- * @param mailboxPath - The IMAP mailbox path
341
+ * @param folder - The IMAP mailbox path and its folder placement
336
342
  * @param getConnection - Function to get a (lazy) IMAP connection
337
343
  * @param force - Bypass the "already stored" skip guard and re-fetch every
338
344
  * message in this batch even though `bodyStorageKey` is already set. Set
@@ -345,13 +351,17 @@ export class BodySyncService {
345
351
  messageIds: string[],
346
352
  accountId: string,
347
353
  accountConfigId: string,
348
- mailboxPath: string,
354
+ folder: BodySyncFolder,
349
355
  getConnection: ConnectionGetter,
350
356
  force = false,
351
357
  ): Promise<SyncBodiesResult> {
352
358
  const syncedMessageIds: string[] = [];
353
359
  let skippedCount = 0;
354
- const location = { accountId, accountConfigId, mailboxPath };
360
+ const location = {
361
+ accountId,
362
+ accountConfigId,
363
+ mailboxPath: folder.fullPath,
364
+ };
355
365
 
356
366
  // Resolve every message up front so we can issue ONE ranged FETCH for the
357
367
  // whole batch (the desktop-client pattern) instead of a SELECT + download
@@ -420,7 +430,7 @@ export class BodySyncService {
420
430
  const connection = await getConnection();
421
431
  // Single SELECT for the whole batch. openBox is idempotent, so a warm
422
432
  // connection already on this mailbox skips the SELECT entirely.
423
- await connection.openBox(mailboxPath);
433
+ await connection.openBox(folder.fullPath);
424
434
 
425
435
  let connectionLost = false;
426
436
  try {
@@ -440,6 +450,7 @@ export class BodySyncService {
440
450
  accountId,
441
451
  accountConfigId,
442
452
  source,
453
+ folder.placement,
443
454
  reStoredMessageIds.has(messageId),
444
455
  );
445
456
  } catch (error) {
@@ -628,6 +639,7 @@ export class BodySyncService {
628
639
  accountId: string,
629
640
  accountConfigId: string,
630
641
  source: Readable,
642
+ placement: FolderPlacement,
631
643
  isReStore: boolean,
632
644
  ): Promise<void> {
633
645
  const toStorage = new PassThrough();
@@ -672,6 +684,7 @@ export class BodySyncService {
672
684
  {
673
685
  uri: ref.uri,
674
686
  },
687
+ placement,
675
688
  isReStore,
676
689
  );
677
690
  }
@@ -704,6 +717,7 @@ export class BodySyncService {
704
717
  accountConfigId: string,
705
718
  body: Buffer,
706
719
  bodyRef: { uri: string },
720
+ placement: FolderPlacement,
707
721
  /**
708
722
  * Whether this call is a re-store of an already-classified message
709
723
  * rather than a first store. The two re-entrant paths (`force: true`
@@ -767,7 +781,11 @@ export class BodySyncService {
767
781
  // `flags.category` override when one applies (issue #299). The derived
768
782
  // fields are folded into the single Message update below — they are NOT
769
783
  // written here.
770
- const classification = await this.classifyMessage(accountConfigId, parsed);
784
+ const classification = await this.classifyMessage(
785
+ accountConfigId,
786
+ parsed,
787
+ placement,
788
+ );
771
789
 
772
790
  // Decide the placement move from the in-memory classification before the
773
791
  // write, so its `movedByRemit` flag and audit verdict join the same
@@ -984,7 +1002,7 @@ export class BodySyncService {
984
1002
  * @param messageId - The message ID to fetch
985
1003
  * @param accountId - The account ID (for storage path)
986
1004
  * @param accountConfigId - The account config ID (for thread updates)
987
- * @param mailboxPath - The IMAP mailbox path
1005
+ * @param folder - The IMAP mailbox path and its folder placement
988
1006
  * @param getConnection - Function to get a (lazy) IMAP connection
989
1007
  * @returns Parsed text and HTML content
990
1008
  */
@@ -992,7 +1010,7 @@ export class BodySyncService {
992
1010
  messageId: string,
993
1011
  accountId: string,
994
1012
  accountConfigId: string,
995
- mailboxPath: string,
1013
+ folder: BodySyncFolder,
996
1014
  getConnection: ConnectionGetter,
997
1015
  ): Promise<FetchBodyResult> {
998
1016
  const message = await this.messageService.get(messageId);
@@ -1030,14 +1048,18 @@ export class BodySyncService {
1030
1048
  needsStore = true;
1031
1049
  body = await this.fetchFromImap(
1032
1050
  message.uid,
1033
- mailboxPath,
1051
+ folder.fullPath,
1034
1052
  getConnection,
1035
1053
  );
1036
1054
  }
1037
1055
  } else {
1038
1056
  // Fetch from IMAP and store
1039
1057
  needsStore = true;
1040
- body = await this.fetchFromImap(message.uid, mailboxPath, getConnection);
1058
+ body = await this.fetchFromImap(
1059
+ message.uid,
1060
+ folder.fullPath,
1061
+ getConnection,
1062
+ );
1041
1063
  }
1042
1064
 
1043
1065
  let parsed: ParsedMail;
@@ -1062,6 +1084,7 @@ export class BodySyncService {
1062
1084
  accountConfigId,
1063
1085
  body,
1064
1086
  { uri: ref.uri },
1087
+ folder.placement,
1065
1088
  message.bodyStorageKey != null,
1066
1089
  );
1067
1090
  } else {
@@ -1097,8 +1120,14 @@ export class BodySyncService {
1097
1120
  private async classifyMessage(
1098
1121
  accountConfigId: string,
1099
1122
  parsed: ParsedMail,
1123
+ placement: FolderPlacement,
1100
1124
  ): Promise<UpdateMessageInput & { category: ThreadMessageCategory }> {
1101
- return classifyParsedMessage(this.addressService, accountConfigId, parsed);
1125
+ return classifyParsedMessage(
1126
+ this.addressService,
1127
+ accountConfigId,
1128
+ parsed,
1129
+ placement,
1130
+ );
1102
1131
  }
1103
1132
 
1104
1133
  private async incrementInboundCount(
@@ -1458,17 +1487,12 @@ export class BodySyncService {
1458
1487
  // true)`) skip the decision steps entirely, so only the pass that first
1459
1488
  // stores a body ever reaches this method.
1460
1489
 
1461
- const junkMailbox =
1462
- await mailboxSpecialUseService.findJunkMailbox(accountId);
1463
- const inboxMailbox =
1464
- await mailboxSpecialUseService.findInboxMailbox(accountId);
1465
-
1466
- const placement: FolderPlacement =
1467
- junkMailbox && message.mailboxId === junkMailbox.mailboxId
1468
- ? "junk"
1469
- : inboxMailbox && message.mailboxId === inboxMailbox.mailboxId
1470
- ? "inbox"
1471
- : "other";
1490
+ const roleMailboxes = await resolveFolderRoleMailboxes(
1491
+ mailboxSpecialUseService,
1492
+ accountId,
1493
+ );
1494
+ const { junkMailbox, inboxMailbox } = roleMailboxes;
1495
+ const placement = placementOf(message.mailboxId, roleMailboxes);
1472
1496
 
1473
1497
  const fromEmail = extractPrimaryFromEmail(parsed);
1474
1498
  const signals = fromEmail
@@ -13,8 +13,10 @@ import assert from "node:assert/strict";
13
13
  import { describe, it } from "node:test";
14
14
  import type {
15
15
  AccountConfigItem,
16
+ AccountItem,
16
17
  AddressItem,
17
18
  IAccountConfigRepository,
19
+ IAccountRepository,
18
20
  IAddressRepository,
19
21
  IMessageRepository,
20
22
  IThreadMessageRepository,
@@ -25,8 +27,10 @@ import type {
25
27
  UpdateThreadMessageInput,
26
28
  } from "@remit/data-ports";
27
29
  import { NotFoundError } from "@remit/data-ports/errors";
30
+ import { DisplayNameCorrespondence } from "@remit/domain-enums";
28
31
  import type { StorageService } from "@remit/storage-service";
29
32
  import { backfillClassifications } from "./classification-backfill.js";
33
+ import type { FolderPlacementLookup } from "./folder-placement.js";
30
34
 
31
35
  const PLAIN_EML = Buffer.from(
32
36
  [
@@ -52,6 +56,20 @@ const LIST_EML = Buffer.from(
52
56
  ].join("\r\n"),
53
57
  );
54
58
 
59
+ const IMPERSONATION_EML = Buffer.from(
60
+ [
61
+ "From: InfoMedics <billing@serviceupdatebank.atlassian.net>",
62
+ "To: me@example.com",
63
+ "Subject: Openstaande factuur",
64
+ "DKIM-Signature: v=1; a=rsa-sha256; d=atlassian.net; s=sel; b=xxx",
65
+ "Content-Type: text/plain",
66
+ "",
67
+ "Betaal uw factuur",
68
+ ].join("\r\n"),
69
+ );
70
+
71
+ const JUNK_MAILBOX_ID = "mb-junk";
72
+
55
73
  const row = (overrides: Partial<ThreadMessageItem>): ThreadMessageItem =>
56
74
  ({
57
75
  threadMessageId: "tm-1",
@@ -97,6 +115,8 @@ const asAccount = (accountConfigId: string): AccountConfigItem =>
97
115
 
98
116
  interface Harness {
99
117
  accountConfigService: Pick<IAccountConfigRepository, "listAll">;
118
+ accountService: Pick<IAccountRepository, "listAllByAccountConfig">;
119
+ mailboxSpecialUseService: FolderPlacementLookup;
100
120
  addressService: Pick<IAddressRepository, "getAddress">;
101
121
  threadMessageService: Pick<
102
122
  IThreadMessageRepository,
@@ -134,6 +154,20 @@ const buildHarness = (options: {
134
154
  listAll: async () => accounts,
135
155
  };
136
156
 
157
+ const accountService: Pick<IAccountRepository, "listAllByAccountConfig"> = {
158
+ listAllByAccountConfig: async (accountConfigId: string) => [
159
+ { accountId: `account-${accountConfigId}` } as unknown as AccountItem,
160
+ ],
161
+ };
162
+
163
+ const mailboxSpecialUseService: FolderPlacementLookup = {
164
+ findJunkMailbox: async () => ({
165
+ mailboxId: JUNK_MAILBOX_ID,
166
+ fullPath: "Junk",
167
+ }),
168
+ findInboxMailbox: async () => ({ mailboxId: "mb-1", fullPath: "INBOX" }),
169
+ };
170
+
137
171
  const addressService = {
138
172
  getAddress: (async () => {
139
173
  if (options.addressFlags === undefined)
@@ -203,6 +237,8 @@ const buildHarness = (options: {
203
237
 
204
238
  return {
205
239
  accountConfigService,
240
+ accountService,
241
+ mailboxSpecialUseService,
206
242
  addressService,
207
243
  threadMessageService,
208
244
  messageService,
@@ -237,6 +273,42 @@ describe("backfillClassifications", () => {
237
273
  );
238
274
  });
239
275
 
276
+ it("compares the sender of a Junk row the provider rated clean, and not of an inbox row", async () => {
277
+ const harness = buildHarness({
278
+ rows: [
279
+ row({
280
+ threadMessageId: "tm-junk",
281
+ messageId: "m-junk",
282
+ mailboxId: JUNK_MAILBOX_ID,
283
+ }),
284
+ row({ threadMessageId: "tm-inbox", messageId: "m-inbox" }),
285
+ ],
286
+ messages: [
287
+ message({
288
+ messageId: "m-junk",
289
+ mailboxId: JUNK_MAILBOX_ID,
290
+ bodyStorageKey: "s3://m-junk",
291
+ }),
292
+ message({ messageId: "m-inbox", bodyStorageKey: "s3://m-inbox" }),
293
+ ],
294
+ retrieve: async () => IMPERSONATION_EML,
295
+ });
296
+
297
+ await backfillClassifications(harness);
298
+
299
+ const correspondence = new Map(
300
+ harness.messageUpdates.map((update) => [
301
+ update.messageId,
302
+ update.input.authenticity?.displayNameCorrespondence,
303
+ ]),
304
+ );
305
+ assert.equal(
306
+ correspondence.get("m-junk"),
307
+ DisplayNameCorrespondence.Unrelated,
308
+ );
309
+ assert.equal(correspondence.get("m-inbox"), undefined);
310
+ });
311
+
240
312
  it("denormalizes the classifier's answer onto the thread row", async () => {
241
313
  const harness = buildHarness({
242
314
  rows: [row({ threadMessageId: "tm-1", messageId: "m-1" })],
@@ -1,5 +1,6 @@
1
1
  import type {
2
2
  IAccountConfigRepository,
3
+ IAccountRepository,
3
4
  IAddressRepository,
4
5
  IMessageRepository,
5
6
  IThreadMessageRepository,
@@ -11,6 +12,11 @@ import { parseMessageBody } from "./body-parse.js";
11
12
  import { extractSnippet } from "./body-sync.js";
12
13
  import { classifyParsedMessage } from "./classify-message.js";
13
14
  import { extractListId } from "./filters/list-id.js";
15
+ import {
16
+ type FolderPlacementLookup,
17
+ resolveAccountFolderPlacements,
18
+ } from "./folder-placement.js";
19
+ import type { FolderPlacement } from "./heuristics/classifyPlacement.js";
14
20
  import {
15
21
  denormalizeMessageCategory,
16
22
  hasDecidedCategory,
@@ -44,6 +50,8 @@ export interface ClassificationBackfillCheckpointStore {
44
50
 
45
51
  export interface ClassificationBackfillDeps {
46
52
  accountConfigService: Pick<IAccountConfigRepository, "listAll">;
53
+ accountService: Pick<IAccountRepository, "listAllByAccountConfig">;
54
+ mailboxSpecialUseService: FolderPlacementLookup;
47
55
  /** Reads the sender's `Address.flags.category` override (issue #299). */
48
56
  addressService: Pick<IAddressRepository, "getAddress">;
49
57
  threadMessageService: Pick<
@@ -116,6 +124,7 @@ const classifyAndApply = async (
116
124
  accountConfigId: string,
117
125
  row: ThreadMessageItem,
118
126
  bodyStorageKey: string,
127
+ placement: FolderPlacement,
119
128
  ): Promise<void> => {
120
129
  const body = await deps.storageService.retrieve(bodyStorageKey);
121
130
  const parsed = await parseMessageBody(body);
@@ -123,6 +132,7 @@ const classifyAndApply = async (
123
132
  deps.addressService,
124
133
  accountConfigId,
125
134
  parsed,
135
+ placement,
126
136
  );
127
137
 
128
138
  // The same denormalized fields the body-store path writes (see
@@ -223,6 +233,13 @@ export const backfillClassifications = async (
223
233
  const account = accounts[accountIndex];
224
234
  let continuationToken: string | undefined =
225
235
  accountIndex === startIndex ? startContinuationToken : undefined;
236
+ const mailAccounts = await deps.accountService.listAllByAccountConfig(
237
+ account.accountConfigId,
238
+ );
239
+ const placements = await resolveAccountFolderPlacements(
240
+ deps.mailboxSpecialUseService,
241
+ mailAccounts.map((mailAccount) => mailAccount.accountId),
242
+ );
226
243
 
227
244
  do {
228
245
  const page = await deps.threadMessageService.listByAccount(
@@ -297,6 +314,7 @@ export const backfillClassifications = async (
297
314
  account.accountConfigId,
298
315
  row,
299
316
  message.bodyStorageKey,
317
+ placements.get(row.mailboxId) ?? "other",
300
318
  ).then(
301
319
  () => null,
302
320
  (error: unknown) => error,
@@ -13,6 +13,7 @@ import assert from "node:assert/strict";
13
13
  import { describe, it } from "node:test";
14
14
  import type { AddressItem, IAddressRepository } from "@remit/data-ports";
15
15
  import { NotFoundError } from "@remit/data-ports/errors";
16
+ import { DisplayNameCorrespondence } from "@remit/domain-enums";
16
17
  import { parseMessageBody } from "./body-parse.js";
17
18
  import {
18
19
  classifyParsedMessage,
@@ -43,6 +44,18 @@ const LIST_EML = Buffer.from(
43
44
  ].join("\r\n"),
44
45
  );
45
46
 
47
+ const IMPERSONATION_EML = Buffer.from(
48
+ [
49
+ "From: InfoMedics <billing@serviceupdatebank.atlassian.net>",
50
+ "To: me@example.com",
51
+ "Subject: Openstaande factuur",
52
+ "DKIM-Signature: v=1; a=rsa-sha256; d=atlassian.net; s=sel; b=xxx",
53
+ "Content-Type: text/html",
54
+ "",
55
+ '<a href="https://betaal-vordering.example.org/pay">Betaal uw factuur</a>',
56
+ ].join("\r\n"),
57
+ );
58
+
46
59
  const addressService = (
47
60
  behavior: (accountConfigId: string, addressId: string) => unknown,
48
61
  ): Pick<IAddressRepository, "getAddress"> => ({
@@ -68,6 +81,7 @@ describe("classifyParsedMessage", () => {
68
81
  }),
69
82
  "acc-1",
70
83
  parsed,
84
+ "inbox",
71
85
  );
72
86
 
73
87
  assert.equal(classification.category, "newsletter");
@@ -81,6 +95,7 @@ describe("classifyParsedMessage", () => {
81
95
  addressService(withFlags({ category: { value: "marketing", setAt: 1 } })),
82
96
  "acc-1",
83
97
  parsed,
98
+ "inbox",
84
99
  );
85
100
 
86
101
  // The header table alone would answer `personal`; the override wins
@@ -98,6 +113,7 @@ describe("classifyParsedMessage", () => {
98
113
  }),
99
114
  "acc-1",
100
115
  parsed,
116
+ "inbox",
101
117
  ),
102
118
  /throttled/,
103
119
  );
@@ -124,6 +140,7 @@ describe("classifyParsedMessage", () => {
124
140
  }),
125
141
  "acc-1",
126
142
  parsed,
143
+ "inbox",
127
144
  );
128
145
 
129
146
  assert.equal(reads, 0);
@@ -131,6 +148,48 @@ describe("classifyParsedMessage", () => {
131
148
  });
132
149
  });
133
150
 
151
+ describe("classifyParsedMessage sender mismatch", () => {
152
+ const noAddress = addressService(async () => {
153
+ throw new NotFoundError("no Address row");
154
+ });
155
+
156
+ it("compares the sender of mail in Junk the provider rated clean", async () => {
157
+ const parsed = await parseMessageBody(IMPERSONATION_EML);
158
+
159
+ const classification = await classifyParsedMessage(
160
+ noAddress,
161
+ "acc-1",
162
+ parsed,
163
+ "junk",
164
+ );
165
+
166
+ assert.equal(
167
+ classification.authenticity?.displayNameCorrespondence,
168
+ DisplayNameCorrespondence.Unrelated,
169
+ );
170
+ assert.deepEqual(classification.authenticity?.offDomainLinkDomains, [
171
+ "example.org",
172
+ ]);
173
+ });
174
+
175
+ it("compares nothing for the same mail outside Junk without a provider verdict", async () => {
176
+ const parsed = await parseMessageBody(IMPERSONATION_EML);
177
+
178
+ const classification = await classifyParsedMessage(
179
+ noAddress,
180
+ "acc-1",
181
+ parsed,
182
+ "inbox",
183
+ );
184
+
185
+ assert.equal(
186
+ classification.authenticity?.displayNameCorrespondence,
187
+ undefined,
188
+ );
189
+ assert.equal(classification.authenticity?.offDomainLinkDomains, undefined);
190
+ });
191
+ });
192
+
134
193
  describe("extractPrimaryFromEmail", () => {
135
194
  it("reads the first From address", async () => {
136
195
  const parsed = await parseMessageBody(PLAIN_EML);
@@ -13,6 +13,7 @@ import {
13
13
  extractHasListUnsubscribe,
14
14
  extractProviderSpam,
15
15
  } from "./heuristics/classifyByHeaders.js";
16
+ import type { FolderPlacement } from "./heuristics/classifyPlacement.js";
16
17
  import { extractSenderMismatch } from "./heuristics/senderMismatch.js";
17
18
 
18
19
  type ThreadMessageCategory = ThreadMessageItem["category"];
@@ -75,6 +76,7 @@ export const classifyParsedMessage = async (
75
76
  addressService: Pick<IAddressRepository, "getAddress">,
76
77
  accountConfigId: string,
77
78
  parsed: ParsedMail,
79
+ placement: FolderPlacement,
78
80
  ): Promise<UpdateMessageInput & { category: ThreadMessageCategory }> => {
79
81
  const headerCategory = classifyByHeaders(parsed);
80
82
  const authenticity = extractAuthenticity(parsed);
@@ -86,13 +88,14 @@ export const classifyParsedMessage = async (
86
88
  // identity the message claims — on a shared-tenant host the verified
87
89
  // subdomain belongs to whoever signed up. These two comparisons say
88
90
  // whether the claim holds, and run only over mail the provider already
89
- // called spam.
91
+ // called spam or that sits in the Junk folder.
90
92
  const senderMismatch =
91
93
  authenticity === null
92
94
  ? {}
93
95
  : extractSenderMismatch(parsed, {
94
96
  fromDomain: authenticity.fromDomain,
95
- spamClassified: providerSpam?.classified === true,
97
+ providerSpamClassified: providerSpam?.classified === true,
98
+ placement,
96
99
  bulkSender: hasListUnsubscribe,
97
100
  });
98
101
 
@@ -0,0 +1,56 @@
1
+ import type { IMailboxSpecialUseRepository } from "@remit/data-ports";
2
+ import type { FolderPlacement } from "./heuristics/classifyPlacement.js";
3
+
4
+ export type FolderPlacementLookup = Pick<
5
+ IMailboxSpecialUseRepository,
6
+ "findJunkMailbox" | "findInboxMailbox"
7
+ >;
8
+
9
+ type RoleMailbox = Awaited<
10
+ ReturnType<IMailboxSpecialUseRepository["findJunkMailbox"]>
11
+ >;
12
+
13
+ export interface FolderRoleMailboxes {
14
+ junkMailbox: RoleMailbox;
15
+ inboxMailbox: RoleMailbox;
16
+ }
17
+
18
+ export const resolveFolderRoleMailboxes = async (
19
+ lookup: FolderPlacementLookup,
20
+ accountId: string,
21
+ ): Promise<FolderRoleMailboxes> => ({
22
+ junkMailbox: await lookup.findJunkMailbox(accountId),
23
+ inboxMailbox: await lookup.findInboxMailbox(accountId),
24
+ });
25
+
26
+ export const placementOf = (
27
+ mailboxId: string,
28
+ { junkMailbox, inboxMailbox }: FolderRoleMailboxes,
29
+ ): FolderPlacement => {
30
+ if (junkMailbox?.mailboxId === mailboxId) return "junk";
31
+ if (inboxMailbox?.mailboxId === mailboxId) return "inbox";
32
+ return "other";
33
+ };
34
+
35
+ export const resolveFolderPlacement = async (
36
+ lookup: FolderPlacementLookup,
37
+ accountId: string,
38
+ mailboxId: string,
39
+ ): Promise<FolderPlacement> =>
40
+ placementOf(mailboxId, await resolveFolderRoleMailboxes(lookup, accountId));
41
+
42
+ export const resolveAccountFolderPlacements = async (
43
+ lookup: FolderPlacementLookup,
44
+ accountIds: readonly string[],
45
+ ): Promise<ReadonlyMap<string, FolderPlacement>> => {
46
+ const placements = new Map<string, FolderPlacement>();
47
+ for (const accountId of accountIds) {
48
+ const { junkMailbox, inboxMailbox } = await resolveFolderRoleMailboxes(
49
+ lookup,
50
+ accountId,
51
+ );
52
+ if (inboxMailbox) placements.set(inboxMailbox.mailboxId, "inbox");
53
+ if (junkMailbox) placements.set(junkMailbox.mailboxId, "junk");
54
+ }
55
+ return placements;
56
+ };
@@ -245,7 +245,8 @@ describe("extractSenderMismatch", () => {
245
245
  assert.deepEqual(
246
246
  extractSenderMismatch(parsed, {
247
247
  fromDomain: "serviceupdatebank.atlassian.net",
248
- spamClassified: true,
248
+ providerSpamClassified: true,
249
+ placement: "other",
249
250
  bulkSender: false,
250
251
  }),
251
252
  {
@@ -255,18 +256,53 @@ describe("extractSenderMismatch", () => {
255
256
  );
256
257
  });
257
258
 
258
- it("compares nothing when the provider did not call the message spam", async () => {
259
+ it("flags a message sitting in Junk that the provider rated clean", async () => {
259
260
  const parsed = await parse(infoMedicsPhish);
260
261
  assert.deepEqual(
261
262
  extractSenderMismatch(parsed, {
262
263
  fromDomain: "serviceupdatebank.atlassian.net",
263
- spamClassified: false,
264
+ providerSpamClassified: false,
265
+ placement: "junk",
264
266
  bulkSender: false,
265
267
  }),
266
- {},
268
+ {
269
+ displayNameCorrespondence: DisplayNameCorrespondence.Unrelated,
270
+ offDomainLinkDomains: ["example.org"],
271
+ },
267
272
  );
268
273
  });
269
274
 
275
+ it("flags a provider-classified message outside Junk", async () => {
276
+ const parsed = await parse(infoMedicsPhish);
277
+ assert.deepEqual(
278
+ extractSenderMismatch(parsed, {
279
+ fromDomain: "serviceupdatebank.atlassian.net",
280
+ providerSpamClassified: true,
281
+ placement: "inbox",
282
+ bulkSender: false,
283
+ }),
284
+ {
285
+ displayNameCorrespondence: DisplayNameCorrespondence.Unrelated,
286
+ offDomainLinkDomains: ["example.org"],
287
+ },
288
+ );
289
+ });
290
+
291
+ for (const placement of ["inbox", "other"] as const) {
292
+ it(`compares nothing in ${placement} when the provider did not call the message spam`, async () => {
293
+ const parsed = await parse(infoMedicsPhish);
294
+ assert.deepEqual(
295
+ extractSenderMismatch(parsed, {
296
+ fromDomain: "serviceupdatebank.atlassian.net",
297
+ providerSpamClassified: false,
298
+ placement,
299
+ bulkSender: false,
300
+ }),
301
+ {},
302
+ );
303
+ });
304
+ }
305
+
270
306
  it("flags a display name spelling the recipient's own address over a foreign domain", async () => {
271
307
  const parsed = await parse([
272
308
  "From: matthijs@ischen.nl <aramirez@secresaludguaviare.gov.co>",
@@ -279,7 +315,8 @@ describe("extractSenderMismatch", () => {
279
315
  assert.equal(
280
316
  extractSenderMismatch(parsed, {
281
317
  fromDomain: "secresaludguaviare.gov.co",
282
- spamClassified: true,
318
+ providerSpamClassified: true,
319
+ placement: "other",
283
320
  bulkSender: false,
284
321
  }).displayNameCorrespondence,
285
322
  DisplayNameCorrespondence.ForeignAddress,
@@ -299,7 +336,8 @@ describe("extractSenderMismatch", () => {
299
336
  ]);
300
337
  const signals = extractSenderMismatch(parsed, {
301
338
  fromDomain: "mailer.esp.example",
302
- spamClassified: true,
339
+ providerSpamClassified: true,
340
+ placement: "other",
303
341
  bulkSender: true,
304
342
  });
305
343
  assert.equal(signals.displayNameCorrespondence, undefined);
@@ -1,6 +1,7 @@
1
1
  import { DisplayNameCorrespondence } from "@remit/domain-enums";
2
2
  import type { ParsedMail } from "mailparser";
3
3
  import { getDomain, parse as parseHost } from "tldts";
4
+ import type { FolderPlacement } from "./classifyPlacement.js";
4
5
 
5
6
  type CorrespondenceValue =
6
7
  (typeof DisplayNameCorrespondence)[keyof typeof DisplayNameCorrespondence];
@@ -10,7 +11,8 @@ type CorrespondenceValue =
10
11
  * that belongs to nobody at the sending domain, and body links that leave it.
11
12
  *
12
13
  * Both are deliberately aggressive, and both are computed only for mail the
13
- * provider's own filter already called spam (see {@link extractSenderMismatch}).
14
+ * provider's own filter already called spam or that sits in the Junk folder
15
+ * (see {@link extractSenderMismatch}).
14
16
  * On a shared-tenant host — a free Atlassian, Salesforce or Zendesk instance —
15
17
  * the verified subdomain is chosen by whoever signed up, so a passing signature
16
18
  * proves the domain and nothing about the identity the message claims.
@@ -22,7 +24,8 @@ export interface SenderMismatchSignals {
22
24
 
23
25
  export interface SenderMismatchContext {
24
26
  fromDomain: string;
25
- spamClassified: boolean;
27
+ providerSpamClassified: boolean;
28
+ placement: FolderPlacement;
26
29
  /**
27
30
  * List-Unsubscribe present. Bulk mail routinely shows a brand name over an
28
31
  * ESP's sending domain, so the display-name comparison says nothing there and
@@ -223,7 +226,8 @@ export const extractOffDomainLinkDomains = (
223
226
  };
224
227
 
225
228
  /**
226
- * Both signals, gated on the provider's own spam verdict.
229
+ * Both signals, gated on a spam verdict: the provider's own, or the message
230
+ * sitting in the Junk folder.
227
231
  *
228
232
  * The gate is what lets the checks be this aggressive: ordinary mail never
229
233
  * reaches them, so a brand name over an ESP domain or a newsletter full of
@@ -234,7 +238,9 @@ export const extractSenderMismatch = (
234
238
  parsed: ParsedMail,
235
239
  context: SenderMismatchContext,
236
240
  ): SenderMismatchSignals => {
237
- if (!context.spamClassified) return {};
241
+ if (!context.providerSpamClassified && context.placement !== "junk") {
242
+ return {};
243
+ }
238
244
 
239
245
  const offDomainLinkDomains = extractOffDomainLinkDomains(
240
246
  parsed,
package/src/index.ts CHANGED
@@ -25,6 +25,7 @@ export {
25
25
  parseMessageBody,
26
26
  } from "./body-parse.js";
27
27
  export {
28
+ type BodySyncFolder,
28
29
  type BodySyncLogger,
29
30
  BodySyncService,
30
31
  type ConnectionGetter,
@@ -125,6 +126,11 @@ export {
125
126
  type UpdateFlagsInput,
126
127
  type UpdateFlagsResult,
127
128
  } from "./flag-queue.js";
129
+ export {
130
+ type FolderPlacementLookup,
131
+ resolveAccountFolderPlacements,
132
+ resolveFolderPlacement,
133
+ } from "./folder-placement.js";
128
134
  export {
129
135
  classifyByHeaders,
130
136
  extractAuthenticity,