@remit/mailbox-service 0.0.102 → 0.0.103
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/src/body-sync-auto-read-writeonce.test.ts +3 -3
- package/src/body-sync-calendar-suggestion.test.ts +1 -1
- package/src/body-sync-category-denormalize.test.ts +2 -2
- package/src/body-sync-category-override.test.ts +9 -9
- package/src/body-sync-category-writeonce.test.ts +3 -3
- package/src/body-sync-filter.test.ts +1 -1
- package/src/body-sync-placement-flags.test.ts +1 -1
- package/src/body-sync-placement-writeonce.test.ts +4 -4
- package/src/body-sync-quarantine.test.ts +7 -3
- package/src/body-sync-resync.test.ts +5 -5
- package/src/body-sync-unclassifiable-once.test.ts +6 -2
- package/src/body-sync-unsubscribed-auto-read.test.ts +1 -1
- package/src/body-sync.ts +45 -21
- package/src/classification-backfill.test.ts +72 -0
- package/src/classification-backfill.ts +18 -0
- package/src/classify-message.test.ts +59 -0
- package/src/classify-message.ts +5 -2
- package/src/folder-placement.ts +56 -0
- package/src/heuristics/senderMismatch.test.ts +44 -6
- package/src/heuristics/senderMismatch.ts +10 -4
- package/src/index.ts +6 -0
package/package.json
CHANGED
|
@@ -160,7 +160,7 @@ describe("unsubscribed auto-mark-read is decided once (issue #499)", () => {
|
|
|
160
160
|
"m-1",
|
|
161
161
|
"acc-1",
|
|
162
162
|
"cfg-1",
|
|
163
|
-
"INBOX",
|
|
163
|
+
{ fullPath: "INBOX", placement: "inbox" },
|
|
164
164
|
async () => bodyConnection(),
|
|
165
165
|
);
|
|
166
166
|
|
|
@@ -181,7 +181,7 @@ describe("unsubscribed auto-mark-read is decided once (issue #499)", () => {
|
|
|
181
181
|
"m-1",
|
|
182
182
|
"acc-1",
|
|
183
183
|
"cfg-1",
|
|
184
|
-
"INBOX",
|
|
184
|
+
{ fullPath: "INBOX", placement: "inbox" },
|
|
185
185
|
async () => bodyConnection(),
|
|
186
186
|
);
|
|
187
187
|
|
|
@@ -204,7 +204,7 @@ describe("unsubscribed auto-mark-read is decided once (issue #499)", () => {
|
|
|
204
204
|
["m-1"],
|
|
205
205
|
"acc-1",
|
|
206
206
|
"cfg-1",
|
|
207
|
-
"INBOX",
|
|
207
|
+
{ fullPath: "INBOX", placement: "inbox" },
|
|
208
208
|
async () => bodyConnection(),
|
|
209
209
|
true,
|
|
210
210
|
);
|
|
@@ -195,7 +195,7 @@ const syncBatch = (harness: Harness) =>
|
|
|
195
195
|
["m-1"],
|
|
196
196
|
"acc-1",
|
|
197
197
|
"cfg-1",
|
|
198
|
-
"INBOX",
|
|
198
|
+
{ fullPath: "INBOX", placement: "inbox" },
|
|
199
199
|
async () =>
|
|
200
200
|
({
|
|
201
201
|
openBox: async () => {},
|
|
@@ -212,7 +212,7 @@ const syncFromImap = (harness: Harness) =>
|
|
|
212
212
|
"m-1",
|
|
213
213
|
"acc-1",
|
|
214
214
|
"cfg-1",
|
|
215
|
-
"INBOX",
|
|
215
|
+
{ fullPath: "INBOX", placement: "inbox" },
|
|
216
216
|
async () =>
|
|
217
217
|
({
|
|
218
218
|
openBox: async () => {},
|
|
@@ -199,7 +199,7 @@ describe("Address.flags.category overrides classification at sync time (issue #2
|
|
|
199
199
|
"m-1",
|
|
200
200
|
"acc-1",
|
|
201
201
|
"cfg-1",
|
|
202
|
-
"INBOX",
|
|
202
|
+
{ fullPath: "INBOX", placement: "inbox" },
|
|
203
203
|
async () => connection,
|
|
204
204
|
);
|
|
205
205
|
|
|
@@ -229,7 +229,7 @@ describe("Address.flags.category overrides classification at sync time (issue #2
|
|
|
229
229
|
"m-1",
|
|
230
230
|
"acc-1",
|
|
231
231
|
"cfg-1",
|
|
232
|
-
"INBOX",
|
|
232
|
+
{ fullPath: "INBOX", placement: "inbox" },
|
|
233
233
|
async () => connection,
|
|
234
234
|
);
|
|
235
235
|
|
|
@@ -251,7 +251,7 @@ describe("Address.flags.category overrides classification at sync time (issue #2
|
|
|
251
251
|
"m-1",
|
|
252
252
|
"acc-1",
|
|
253
253
|
"cfg-1",
|
|
254
|
-
"INBOX",
|
|
254
|
+
{ fullPath: "INBOX", placement: "inbox" },
|
|
255
255
|
async () => connection,
|
|
256
256
|
);
|
|
257
257
|
|
|
@@ -276,7 +276,7 @@ describe("Address.flags.category overrides classification at sync time (issue #2
|
|
|
276
276
|
"m-1",
|
|
277
277
|
"acc-1",
|
|
278
278
|
"cfg-1",
|
|
279
|
-
"INBOX",
|
|
279
|
+
{ fullPath: "INBOX", placement: "inbox" },
|
|
280
280
|
async () => connection,
|
|
281
281
|
);
|
|
282
282
|
|
|
@@ -296,7 +296,7 @@ describe("Address.flags.category overrides classification at sync time (issue #2
|
|
|
296
296
|
["m-1"],
|
|
297
297
|
"acc-1",
|
|
298
298
|
"cfg-1",
|
|
299
|
-
"INBOX",
|
|
299
|
+
{ fullPath: "INBOX", placement: "inbox" },
|
|
300
300
|
async () =>
|
|
301
301
|
({
|
|
302
302
|
openBox: async () => {},
|
|
@@ -336,7 +336,7 @@ describe("Address.flags.category overrides classification at sync time (issue #2
|
|
|
336
336
|
["m-1"],
|
|
337
337
|
"acc-1",
|
|
338
338
|
"cfg-1",
|
|
339
|
-
"INBOX",
|
|
339
|
+
{ fullPath: "INBOX", placement: "inbox" },
|
|
340
340
|
async () => {
|
|
341
341
|
throw new Error("an examined message must not be fetched again");
|
|
342
342
|
},
|
|
@@ -376,7 +376,7 @@ describe("Address.flags.category overrides classification at sync time (issue #2
|
|
|
376
376
|
"m-1",
|
|
377
377
|
"acc-1",
|
|
378
378
|
"cfg-1",
|
|
379
|
-
"INBOX",
|
|
379
|
+
{ fullPath: "INBOX", placement: "inbox" },
|
|
380
380
|
async () => connection,
|
|
381
381
|
);
|
|
382
382
|
|
|
@@ -413,7 +413,7 @@ describe("Address.flags.category overrides classification at sync time (issue #2
|
|
|
413
413
|
["m-1"],
|
|
414
414
|
"acc-1",
|
|
415
415
|
"cfg-1",
|
|
416
|
-
"INBOX",
|
|
416
|
+
{ fullPath: "INBOX", placement: "inbox" },
|
|
417
417
|
async () => connection,
|
|
418
418
|
true,
|
|
419
419
|
);
|
|
@@ -492,7 +492,7 @@ describe("Address.flags.category overrides classification at sync time (issue #2
|
|
|
492
492
|
"m-1",
|
|
493
493
|
"acc-1",
|
|
494
494
|
"cfg-1",
|
|
495
|
-
"INBOX",
|
|
495
|
+
{ fullPath: "INBOX", placement: "inbox" },
|
|
496
496
|
async () => connection,
|
|
497
497
|
),
|
|
498
498
|
/ProvisionedThroughputExceededException/,
|
|
@@ -150,7 +150,7 @@ describe("Message.category survives a re-entrant classification pass", () => {
|
|
|
150
150
|
"m-1",
|
|
151
151
|
"acc-1",
|
|
152
152
|
"cfg-1",
|
|
153
|
-
"INBOX",
|
|
153
|
+
{ fullPath: "INBOX", placement: "inbox" },
|
|
154
154
|
async () => connection,
|
|
155
155
|
);
|
|
156
156
|
|
|
@@ -188,7 +188,7 @@ describe("Message.category survives a re-entrant classification pass", () => {
|
|
|
188
188
|
["m-1"],
|
|
189
189
|
"acc-1",
|
|
190
190
|
"cfg-1",
|
|
191
|
-
"INBOX",
|
|
191
|
+
{ fullPath: "INBOX", placement: "inbox" },
|
|
192
192
|
async () => connection,
|
|
193
193
|
true,
|
|
194
194
|
);
|
|
@@ -220,7 +220,7 @@ describe("Message.category survives a re-entrant classification pass", () => {
|
|
|
220
220
|
"m-1",
|
|
221
221
|
"acc-1",
|
|
222
222
|
"cfg-1",
|
|
223
|
-
"INBOX",
|
|
223
|
+
{ fullPath: "INBOX", placement: "inbox" },
|
|
224
224
|
async () => connection,
|
|
225
225
|
);
|
|
226
226
|
|
|
@@ -187,7 +187,7 @@ describe("placement survives a re-entrant computePlacement pass (issue #383)", (
|
|
|
187
187
|
"m-1",
|
|
188
188
|
"acc-1",
|
|
189
189
|
"cfg-1",
|
|
190
|
-
"INBOX",
|
|
190
|
+
{ fullPath: "INBOX", placement: "inbox" },
|
|
191
191
|
async () => connection,
|
|
192
192
|
);
|
|
193
193
|
|
|
@@ -233,7 +233,7 @@ describe("placement survives a re-entrant computePlacement pass (issue #383)", (
|
|
|
233
233
|
["m-1"],
|
|
234
234
|
"acc-1",
|
|
235
235
|
"cfg-1",
|
|
236
|
-
"INBOX",
|
|
236
|
+
{ fullPath: "INBOX", placement: "inbox" },
|
|
237
237
|
async () => connection,
|
|
238
238
|
true,
|
|
239
239
|
);
|
|
@@ -268,7 +268,7 @@ describe("placement survives a re-entrant computePlacement pass (issue #383)", (
|
|
|
268
268
|
"m-1",
|
|
269
269
|
"acc-1",
|
|
270
270
|
"cfg-1",
|
|
271
|
-
"INBOX",
|
|
271
|
+
{ fullPath: "INBOX", placement: "inbox" },
|
|
272
272
|
async () => connection,
|
|
273
273
|
);
|
|
274
274
|
|
|
@@ -309,7 +309,7 @@ describe("placement survives a re-entrant computePlacement pass (issue #383)", (
|
|
|
309
309
|
"m-1",
|
|
310
310
|
"acc-1",
|
|
311
311
|
"cfg-1",
|
|
312
|
-
"INBOX",
|
|
312
|
+
{ fullPath: "INBOX", placement: "inbox" },
|
|
313
313
|
async () => connection,
|
|
314
314
|
);
|
|
315
315
|
|
|
@@ -162,8 +162,12 @@ const connection = () =>
|
|
|
162
162
|
}) as unknown as IImapConnection;
|
|
163
163
|
|
|
164
164
|
const sync = (service: BodySyncService) =>
|
|
165
|
-
service.syncBodies(
|
|
166
|
-
|
|
165
|
+
service.syncBodies(
|
|
166
|
+
["m-1"],
|
|
167
|
+
"acc-1",
|
|
168
|
+
"cfg-1",
|
|
169
|
+
{ fullPath: "INBOX", placement: "inbox" },
|
|
170
|
+
async () => connection(),
|
|
167
171
|
);
|
|
168
172
|
|
|
169
173
|
const parseRefusal = () =>
|
|
@@ -258,7 +262,7 @@ describe("body sync skips what is already quarantined", () => {
|
|
|
258
262
|
["m-1"],
|
|
259
263
|
"acc-1",
|
|
260
264
|
"cfg-1",
|
|
261
|
-
"INBOX",
|
|
265
|
+
{ fullPath: "INBOX", placement: "inbox" },
|
|
262
266
|
async () => {
|
|
263
267
|
throw new Error("a quarantined message must not be fetched again");
|
|
264
268
|
},
|
|
@@ -275,7 +275,7 @@ describe("issue #1011: forced re-store does not re-fire standing rules", () => {
|
|
|
275
275
|
["m-1"],
|
|
276
276
|
"acc-1",
|
|
277
277
|
"cfg-1",
|
|
278
|
-
"INBOX",
|
|
278
|
+
{ fullPath: "INBOX", placement: "inbox" },
|
|
279
279
|
async () => connection,
|
|
280
280
|
true,
|
|
281
281
|
);
|
|
@@ -330,7 +330,7 @@ describe("issue #1011: forced re-store does not re-fire standing rules", () => {
|
|
|
330
330
|
["m-1"],
|
|
331
331
|
"acc-1",
|
|
332
332
|
"cfg-1",
|
|
333
|
-
"INBOX",
|
|
333
|
+
{ fullPath: "INBOX", placement: "inbox" },
|
|
334
334
|
async () => connection,
|
|
335
335
|
true,
|
|
336
336
|
);
|
|
@@ -374,7 +374,7 @@ describe("issue #1011: forced re-store does not re-fire standing rules", () => {
|
|
|
374
374
|
"m-1",
|
|
375
375
|
"acc-1",
|
|
376
376
|
"cfg-1",
|
|
377
|
-
"INBOX",
|
|
377
|
+
{ fullPath: "INBOX", placement: "inbox" },
|
|
378
378
|
async () => noSuchKeyConn(DEMOTE_EML),
|
|
379
379
|
);
|
|
380
380
|
|
|
@@ -415,7 +415,7 @@ describe("issue #1011: forced re-store does not re-fire standing rules", () => {
|
|
|
415
415
|
"m-1",
|
|
416
416
|
"acc-1",
|
|
417
417
|
"cfg-1",
|
|
418
|
-
"INBOX",
|
|
418
|
+
{ fullPath: "INBOX", placement: "inbox" },
|
|
419
419
|
async () => noSuchKeyConn(INVITATION_EML),
|
|
420
420
|
);
|
|
421
421
|
|
|
@@ -569,7 +569,7 @@ describe("issue #1011: forced re-store does not re-fire standing rules", () => {
|
|
|
569
569
|
["m-stored", "m-new"],
|
|
570
570
|
"acc-1",
|
|
571
571
|
"cfg-1",
|
|
572
|
-
"INBOX",
|
|
572
|
+
{ fullPath: "INBOX", placement: "inbox" },
|
|
573
573
|
async () => connection,
|
|
574
574
|
true,
|
|
575
575
|
);
|
|
@@ -167,8 +167,12 @@ const imapConnection = () =>
|
|
|
167
167
|
}) as unknown as IImapConnection;
|
|
168
168
|
|
|
169
169
|
const syncPass = (harness: Harness) =>
|
|
170
|
-
harness.service.syncBodies(
|
|
171
|
-
|
|
170
|
+
harness.service.syncBodies(
|
|
171
|
+
["m-1"],
|
|
172
|
+
"acc-1",
|
|
173
|
+
"cfg-1",
|
|
174
|
+
{ fullPath: "INBOX", placement: "inbox" },
|
|
175
|
+
async () => imapConnection(),
|
|
172
176
|
);
|
|
173
177
|
|
|
174
178
|
describe("a body the classifier declined to categorize is examined once", () => {
|
package/src/body-sync.ts
CHANGED
|
@@ -51,6 +51,7 @@ import {
|
|
|
51
51
|
FilterPipeline,
|
|
52
52
|
} from "./filters/pipeline.js";
|
|
53
53
|
import type { FlagQueueService } from "./flag-queue.js";
|
|
54
|
+
import { placementOf, resolveFolderRoleMailboxes } from "./folder-placement.js";
|
|
54
55
|
import {
|
|
55
56
|
classifyPlacement,
|
|
56
57
|
type FolderPlacement,
|
|
@@ -239,6 +240,11 @@ export interface StoreBodyPartContentsResult {
|
|
|
239
240
|
|
|
240
241
|
export type ConnectionGetter = () => Promise<IImapConnection>;
|
|
241
242
|
|
|
243
|
+
export interface BodySyncFolder {
|
|
244
|
+
fullPath: string;
|
|
245
|
+
placement: FolderPlacement;
|
|
246
|
+
}
|
|
247
|
+
|
|
242
248
|
export interface PlacementConfig {
|
|
243
249
|
mailboxSpecialUseService: IMailboxSpecialUseRepository;
|
|
244
250
|
/**
|
|
@@ -332,7 +338,7 @@ export class BodySyncService {
|
|
|
332
338
|
* @param messageIds - The message IDs to sync bodies for
|
|
333
339
|
* @param accountId - The account ID (for storage path)
|
|
334
340
|
* @param accountConfigId - The account config ID (for thread updates)
|
|
335
|
-
* @param
|
|
341
|
+
* @param folder - The IMAP mailbox path and its folder placement
|
|
336
342
|
* @param getConnection - Function to get a (lazy) IMAP connection
|
|
337
343
|
* @param force - Bypass the "already stored" skip guard and re-fetch every
|
|
338
344
|
* message in this batch even though `bodyStorageKey` is already set. Set
|
|
@@ -345,13 +351,17 @@ export class BodySyncService {
|
|
|
345
351
|
messageIds: string[],
|
|
346
352
|
accountId: string,
|
|
347
353
|
accountConfigId: string,
|
|
348
|
-
|
|
354
|
+
folder: BodySyncFolder,
|
|
349
355
|
getConnection: ConnectionGetter,
|
|
350
356
|
force = false,
|
|
351
357
|
): Promise<SyncBodiesResult> {
|
|
352
358
|
const syncedMessageIds: string[] = [];
|
|
353
359
|
let skippedCount = 0;
|
|
354
|
-
const location = {
|
|
360
|
+
const location = {
|
|
361
|
+
accountId,
|
|
362
|
+
accountConfigId,
|
|
363
|
+
mailboxPath: folder.fullPath,
|
|
364
|
+
};
|
|
355
365
|
|
|
356
366
|
// Resolve every message up front so we can issue ONE ranged FETCH for the
|
|
357
367
|
// whole batch (the desktop-client pattern) instead of a SELECT + download
|
|
@@ -420,7 +430,7 @@ export class BodySyncService {
|
|
|
420
430
|
const connection = await getConnection();
|
|
421
431
|
// Single SELECT for the whole batch. openBox is idempotent, so a warm
|
|
422
432
|
// connection already on this mailbox skips the SELECT entirely.
|
|
423
|
-
await connection.openBox(
|
|
433
|
+
await connection.openBox(folder.fullPath);
|
|
424
434
|
|
|
425
435
|
let connectionLost = false;
|
|
426
436
|
try {
|
|
@@ -440,6 +450,7 @@ export class BodySyncService {
|
|
|
440
450
|
accountId,
|
|
441
451
|
accountConfigId,
|
|
442
452
|
source,
|
|
453
|
+
folder.placement,
|
|
443
454
|
reStoredMessageIds.has(messageId),
|
|
444
455
|
);
|
|
445
456
|
} catch (error) {
|
|
@@ -628,6 +639,7 @@ export class BodySyncService {
|
|
|
628
639
|
accountId: string,
|
|
629
640
|
accountConfigId: string,
|
|
630
641
|
source: Readable,
|
|
642
|
+
placement: FolderPlacement,
|
|
631
643
|
isReStore: boolean,
|
|
632
644
|
): Promise<void> {
|
|
633
645
|
const toStorage = new PassThrough();
|
|
@@ -672,6 +684,7 @@ export class BodySyncService {
|
|
|
672
684
|
{
|
|
673
685
|
uri: ref.uri,
|
|
674
686
|
},
|
|
687
|
+
placement,
|
|
675
688
|
isReStore,
|
|
676
689
|
);
|
|
677
690
|
}
|
|
@@ -704,6 +717,7 @@ export class BodySyncService {
|
|
|
704
717
|
accountConfigId: string,
|
|
705
718
|
body: Buffer,
|
|
706
719
|
bodyRef: { uri: string },
|
|
720
|
+
placement: FolderPlacement,
|
|
707
721
|
/**
|
|
708
722
|
* Whether this call is a re-store of an already-classified message
|
|
709
723
|
* rather than a first store. The two re-entrant paths (`force: true`
|
|
@@ -767,7 +781,11 @@ export class BodySyncService {
|
|
|
767
781
|
// `flags.category` override when one applies (issue #299). The derived
|
|
768
782
|
// fields are folded into the single Message update below — they are NOT
|
|
769
783
|
// written here.
|
|
770
|
-
const classification = await this.classifyMessage(
|
|
784
|
+
const classification = await this.classifyMessage(
|
|
785
|
+
accountConfigId,
|
|
786
|
+
parsed,
|
|
787
|
+
placement,
|
|
788
|
+
);
|
|
771
789
|
|
|
772
790
|
// Decide the placement move from the in-memory classification before the
|
|
773
791
|
// write, so its `movedByRemit` flag and audit verdict join the same
|
|
@@ -984,7 +1002,7 @@ export class BodySyncService {
|
|
|
984
1002
|
* @param messageId - The message ID to fetch
|
|
985
1003
|
* @param accountId - The account ID (for storage path)
|
|
986
1004
|
* @param accountConfigId - The account config ID (for thread updates)
|
|
987
|
-
* @param
|
|
1005
|
+
* @param folder - The IMAP mailbox path and its folder placement
|
|
988
1006
|
* @param getConnection - Function to get a (lazy) IMAP connection
|
|
989
1007
|
* @returns Parsed text and HTML content
|
|
990
1008
|
*/
|
|
@@ -992,7 +1010,7 @@ export class BodySyncService {
|
|
|
992
1010
|
messageId: string,
|
|
993
1011
|
accountId: string,
|
|
994
1012
|
accountConfigId: string,
|
|
995
|
-
|
|
1013
|
+
folder: BodySyncFolder,
|
|
996
1014
|
getConnection: ConnectionGetter,
|
|
997
1015
|
): Promise<FetchBodyResult> {
|
|
998
1016
|
const message = await this.messageService.get(messageId);
|
|
@@ -1030,14 +1048,18 @@ export class BodySyncService {
|
|
|
1030
1048
|
needsStore = true;
|
|
1031
1049
|
body = await this.fetchFromImap(
|
|
1032
1050
|
message.uid,
|
|
1033
|
-
|
|
1051
|
+
folder.fullPath,
|
|
1034
1052
|
getConnection,
|
|
1035
1053
|
);
|
|
1036
1054
|
}
|
|
1037
1055
|
} else {
|
|
1038
1056
|
// Fetch from IMAP and store
|
|
1039
1057
|
needsStore = true;
|
|
1040
|
-
body = await this.fetchFromImap(
|
|
1058
|
+
body = await this.fetchFromImap(
|
|
1059
|
+
message.uid,
|
|
1060
|
+
folder.fullPath,
|
|
1061
|
+
getConnection,
|
|
1062
|
+
);
|
|
1041
1063
|
}
|
|
1042
1064
|
|
|
1043
1065
|
let parsed: ParsedMail;
|
|
@@ -1062,6 +1084,7 @@ export class BodySyncService {
|
|
|
1062
1084
|
accountConfigId,
|
|
1063
1085
|
body,
|
|
1064
1086
|
{ uri: ref.uri },
|
|
1087
|
+
folder.placement,
|
|
1065
1088
|
message.bodyStorageKey != null,
|
|
1066
1089
|
);
|
|
1067
1090
|
} else {
|
|
@@ -1097,8 +1120,14 @@ export class BodySyncService {
|
|
|
1097
1120
|
private async classifyMessage(
|
|
1098
1121
|
accountConfigId: string,
|
|
1099
1122
|
parsed: ParsedMail,
|
|
1123
|
+
placement: FolderPlacement,
|
|
1100
1124
|
): Promise<UpdateMessageInput & { category: ThreadMessageCategory }> {
|
|
1101
|
-
return classifyParsedMessage(
|
|
1125
|
+
return classifyParsedMessage(
|
|
1126
|
+
this.addressService,
|
|
1127
|
+
accountConfigId,
|
|
1128
|
+
parsed,
|
|
1129
|
+
placement,
|
|
1130
|
+
);
|
|
1102
1131
|
}
|
|
1103
1132
|
|
|
1104
1133
|
private async incrementInboundCount(
|
|
@@ -1458,17 +1487,12 @@ export class BodySyncService {
|
|
|
1458
1487
|
// true)`) skip the decision steps entirely, so only the pass that first
|
|
1459
1488
|
// stores a body ever reaches this method.
|
|
1460
1489
|
|
|
1461
|
-
const
|
|
1462
|
-
|
|
1463
|
-
|
|
1464
|
-
|
|
1465
|
-
|
|
1466
|
-
const placement
|
|
1467
|
-
junkMailbox && message.mailboxId === junkMailbox.mailboxId
|
|
1468
|
-
? "junk"
|
|
1469
|
-
: inboxMailbox && message.mailboxId === inboxMailbox.mailboxId
|
|
1470
|
-
? "inbox"
|
|
1471
|
-
: "other";
|
|
1490
|
+
const roleMailboxes = await resolveFolderRoleMailboxes(
|
|
1491
|
+
mailboxSpecialUseService,
|
|
1492
|
+
accountId,
|
|
1493
|
+
);
|
|
1494
|
+
const { junkMailbox, inboxMailbox } = roleMailboxes;
|
|
1495
|
+
const placement = placementOf(message.mailboxId, roleMailboxes);
|
|
1472
1496
|
|
|
1473
1497
|
const fromEmail = extractPrimaryFromEmail(parsed);
|
|
1474
1498
|
const signals = fromEmail
|
|
@@ -13,8 +13,10 @@ import assert from "node:assert/strict";
|
|
|
13
13
|
import { describe, it } from "node:test";
|
|
14
14
|
import type {
|
|
15
15
|
AccountConfigItem,
|
|
16
|
+
AccountItem,
|
|
16
17
|
AddressItem,
|
|
17
18
|
IAccountConfigRepository,
|
|
19
|
+
IAccountRepository,
|
|
18
20
|
IAddressRepository,
|
|
19
21
|
IMessageRepository,
|
|
20
22
|
IThreadMessageRepository,
|
|
@@ -25,8 +27,10 @@ import type {
|
|
|
25
27
|
UpdateThreadMessageInput,
|
|
26
28
|
} from "@remit/data-ports";
|
|
27
29
|
import { NotFoundError } from "@remit/data-ports/errors";
|
|
30
|
+
import { DisplayNameCorrespondence } from "@remit/domain-enums";
|
|
28
31
|
import type { StorageService } from "@remit/storage-service";
|
|
29
32
|
import { backfillClassifications } from "./classification-backfill.js";
|
|
33
|
+
import type { FolderPlacementLookup } from "./folder-placement.js";
|
|
30
34
|
|
|
31
35
|
const PLAIN_EML = Buffer.from(
|
|
32
36
|
[
|
|
@@ -52,6 +56,20 @@ const LIST_EML = Buffer.from(
|
|
|
52
56
|
].join("\r\n"),
|
|
53
57
|
);
|
|
54
58
|
|
|
59
|
+
const IMPERSONATION_EML = Buffer.from(
|
|
60
|
+
[
|
|
61
|
+
"From: InfoMedics <billing@serviceupdatebank.atlassian.net>",
|
|
62
|
+
"To: me@example.com",
|
|
63
|
+
"Subject: Openstaande factuur",
|
|
64
|
+
"DKIM-Signature: v=1; a=rsa-sha256; d=atlassian.net; s=sel; b=xxx",
|
|
65
|
+
"Content-Type: text/plain",
|
|
66
|
+
"",
|
|
67
|
+
"Betaal uw factuur",
|
|
68
|
+
].join("\r\n"),
|
|
69
|
+
);
|
|
70
|
+
|
|
71
|
+
const JUNK_MAILBOX_ID = "mb-junk";
|
|
72
|
+
|
|
55
73
|
const row = (overrides: Partial<ThreadMessageItem>): ThreadMessageItem =>
|
|
56
74
|
({
|
|
57
75
|
threadMessageId: "tm-1",
|
|
@@ -97,6 +115,8 @@ const asAccount = (accountConfigId: string): AccountConfigItem =>
|
|
|
97
115
|
|
|
98
116
|
interface Harness {
|
|
99
117
|
accountConfigService: Pick<IAccountConfigRepository, "listAll">;
|
|
118
|
+
accountService: Pick<IAccountRepository, "listAllByAccountConfig">;
|
|
119
|
+
mailboxSpecialUseService: FolderPlacementLookup;
|
|
100
120
|
addressService: Pick<IAddressRepository, "getAddress">;
|
|
101
121
|
threadMessageService: Pick<
|
|
102
122
|
IThreadMessageRepository,
|
|
@@ -134,6 +154,20 @@ const buildHarness = (options: {
|
|
|
134
154
|
listAll: async () => accounts,
|
|
135
155
|
};
|
|
136
156
|
|
|
157
|
+
const accountService: Pick<IAccountRepository, "listAllByAccountConfig"> = {
|
|
158
|
+
listAllByAccountConfig: async (accountConfigId: string) => [
|
|
159
|
+
{ accountId: `account-${accountConfigId}` } as unknown as AccountItem,
|
|
160
|
+
],
|
|
161
|
+
};
|
|
162
|
+
|
|
163
|
+
const mailboxSpecialUseService: FolderPlacementLookup = {
|
|
164
|
+
findJunkMailbox: async () => ({
|
|
165
|
+
mailboxId: JUNK_MAILBOX_ID,
|
|
166
|
+
fullPath: "Junk",
|
|
167
|
+
}),
|
|
168
|
+
findInboxMailbox: async () => ({ mailboxId: "mb-1", fullPath: "INBOX" }),
|
|
169
|
+
};
|
|
170
|
+
|
|
137
171
|
const addressService = {
|
|
138
172
|
getAddress: (async () => {
|
|
139
173
|
if (options.addressFlags === undefined)
|
|
@@ -203,6 +237,8 @@ const buildHarness = (options: {
|
|
|
203
237
|
|
|
204
238
|
return {
|
|
205
239
|
accountConfigService,
|
|
240
|
+
accountService,
|
|
241
|
+
mailboxSpecialUseService,
|
|
206
242
|
addressService,
|
|
207
243
|
threadMessageService,
|
|
208
244
|
messageService,
|
|
@@ -237,6 +273,42 @@ describe("backfillClassifications", () => {
|
|
|
237
273
|
);
|
|
238
274
|
});
|
|
239
275
|
|
|
276
|
+
it("compares the sender of a Junk row the provider rated clean, and not of an inbox row", async () => {
|
|
277
|
+
const harness = buildHarness({
|
|
278
|
+
rows: [
|
|
279
|
+
row({
|
|
280
|
+
threadMessageId: "tm-junk",
|
|
281
|
+
messageId: "m-junk",
|
|
282
|
+
mailboxId: JUNK_MAILBOX_ID,
|
|
283
|
+
}),
|
|
284
|
+
row({ threadMessageId: "tm-inbox", messageId: "m-inbox" }),
|
|
285
|
+
],
|
|
286
|
+
messages: [
|
|
287
|
+
message({
|
|
288
|
+
messageId: "m-junk",
|
|
289
|
+
mailboxId: JUNK_MAILBOX_ID,
|
|
290
|
+
bodyStorageKey: "s3://m-junk",
|
|
291
|
+
}),
|
|
292
|
+
message({ messageId: "m-inbox", bodyStorageKey: "s3://m-inbox" }),
|
|
293
|
+
],
|
|
294
|
+
retrieve: async () => IMPERSONATION_EML,
|
|
295
|
+
});
|
|
296
|
+
|
|
297
|
+
await backfillClassifications(harness);
|
|
298
|
+
|
|
299
|
+
const correspondence = new Map(
|
|
300
|
+
harness.messageUpdates.map((update) => [
|
|
301
|
+
update.messageId,
|
|
302
|
+
update.input.authenticity?.displayNameCorrespondence,
|
|
303
|
+
]),
|
|
304
|
+
);
|
|
305
|
+
assert.equal(
|
|
306
|
+
correspondence.get("m-junk"),
|
|
307
|
+
DisplayNameCorrespondence.Unrelated,
|
|
308
|
+
);
|
|
309
|
+
assert.equal(correspondence.get("m-inbox"), undefined);
|
|
310
|
+
});
|
|
311
|
+
|
|
240
312
|
it("denormalizes the classifier's answer onto the thread row", async () => {
|
|
241
313
|
const harness = buildHarness({
|
|
242
314
|
rows: [row({ threadMessageId: "tm-1", messageId: "m-1" })],
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import type {
|
|
2
2
|
IAccountConfigRepository,
|
|
3
|
+
IAccountRepository,
|
|
3
4
|
IAddressRepository,
|
|
4
5
|
IMessageRepository,
|
|
5
6
|
IThreadMessageRepository,
|
|
@@ -11,6 +12,11 @@ import { parseMessageBody } from "./body-parse.js";
|
|
|
11
12
|
import { extractSnippet } from "./body-sync.js";
|
|
12
13
|
import { classifyParsedMessage } from "./classify-message.js";
|
|
13
14
|
import { extractListId } from "./filters/list-id.js";
|
|
15
|
+
import {
|
|
16
|
+
type FolderPlacementLookup,
|
|
17
|
+
resolveAccountFolderPlacements,
|
|
18
|
+
} from "./folder-placement.js";
|
|
19
|
+
import type { FolderPlacement } from "./heuristics/classifyPlacement.js";
|
|
14
20
|
import {
|
|
15
21
|
denormalizeMessageCategory,
|
|
16
22
|
hasDecidedCategory,
|
|
@@ -44,6 +50,8 @@ export interface ClassificationBackfillCheckpointStore {
|
|
|
44
50
|
|
|
45
51
|
export interface ClassificationBackfillDeps {
|
|
46
52
|
accountConfigService: Pick<IAccountConfigRepository, "listAll">;
|
|
53
|
+
accountService: Pick<IAccountRepository, "listAllByAccountConfig">;
|
|
54
|
+
mailboxSpecialUseService: FolderPlacementLookup;
|
|
47
55
|
/** Reads the sender's `Address.flags.category` override (issue #299). */
|
|
48
56
|
addressService: Pick<IAddressRepository, "getAddress">;
|
|
49
57
|
threadMessageService: Pick<
|
|
@@ -116,6 +124,7 @@ const classifyAndApply = async (
|
|
|
116
124
|
accountConfigId: string,
|
|
117
125
|
row: ThreadMessageItem,
|
|
118
126
|
bodyStorageKey: string,
|
|
127
|
+
placement: FolderPlacement,
|
|
119
128
|
): Promise<void> => {
|
|
120
129
|
const body = await deps.storageService.retrieve(bodyStorageKey);
|
|
121
130
|
const parsed = await parseMessageBody(body);
|
|
@@ -123,6 +132,7 @@ const classifyAndApply = async (
|
|
|
123
132
|
deps.addressService,
|
|
124
133
|
accountConfigId,
|
|
125
134
|
parsed,
|
|
135
|
+
placement,
|
|
126
136
|
);
|
|
127
137
|
|
|
128
138
|
// The same denormalized fields the body-store path writes (see
|
|
@@ -223,6 +233,13 @@ export const backfillClassifications = async (
|
|
|
223
233
|
const account = accounts[accountIndex];
|
|
224
234
|
let continuationToken: string | undefined =
|
|
225
235
|
accountIndex === startIndex ? startContinuationToken : undefined;
|
|
236
|
+
const mailAccounts = await deps.accountService.listAllByAccountConfig(
|
|
237
|
+
account.accountConfigId,
|
|
238
|
+
);
|
|
239
|
+
const placements = await resolveAccountFolderPlacements(
|
|
240
|
+
deps.mailboxSpecialUseService,
|
|
241
|
+
mailAccounts.map((mailAccount) => mailAccount.accountId),
|
|
242
|
+
);
|
|
226
243
|
|
|
227
244
|
do {
|
|
228
245
|
const page = await deps.threadMessageService.listByAccount(
|
|
@@ -297,6 +314,7 @@ export const backfillClassifications = async (
|
|
|
297
314
|
account.accountConfigId,
|
|
298
315
|
row,
|
|
299
316
|
message.bodyStorageKey,
|
|
317
|
+
placements.get(row.mailboxId) ?? "other",
|
|
300
318
|
).then(
|
|
301
319
|
() => null,
|
|
302
320
|
(error: unknown) => error,
|
|
@@ -13,6 +13,7 @@ import assert from "node:assert/strict";
|
|
|
13
13
|
import { describe, it } from "node:test";
|
|
14
14
|
import type { AddressItem, IAddressRepository } from "@remit/data-ports";
|
|
15
15
|
import { NotFoundError } from "@remit/data-ports/errors";
|
|
16
|
+
import { DisplayNameCorrespondence } from "@remit/domain-enums";
|
|
16
17
|
import { parseMessageBody } from "./body-parse.js";
|
|
17
18
|
import {
|
|
18
19
|
classifyParsedMessage,
|
|
@@ -43,6 +44,18 @@ const LIST_EML = Buffer.from(
|
|
|
43
44
|
].join("\r\n"),
|
|
44
45
|
);
|
|
45
46
|
|
|
47
|
+
const IMPERSONATION_EML = Buffer.from(
|
|
48
|
+
[
|
|
49
|
+
"From: InfoMedics <billing@serviceupdatebank.atlassian.net>",
|
|
50
|
+
"To: me@example.com",
|
|
51
|
+
"Subject: Openstaande factuur",
|
|
52
|
+
"DKIM-Signature: v=1; a=rsa-sha256; d=atlassian.net; s=sel; b=xxx",
|
|
53
|
+
"Content-Type: text/html",
|
|
54
|
+
"",
|
|
55
|
+
'<a href="https://betaal-vordering.example.org/pay">Betaal uw factuur</a>',
|
|
56
|
+
].join("\r\n"),
|
|
57
|
+
);
|
|
58
|
+
|
|
46
59
|
const addressService = (
|
|
47
60
|
behavior: (accountConfigId: string, addressId: string) => unknown,
|
|
48
61
|
): Pick<IAddressRepository, "getAddress"> => ({
|
|
@@ -68,6 +81,7 @@ describe("classifyParsedMessage", () => {
|
|
|
68
81
|
}),
|
|
69
82
|
"acc-1",
|
|
70
83
|
parsed,
|
|
84
|
+
"inbox",
|
|
71
85
|
);
|
|
72
86
|
|
|
73
87
|
assert.equal(classification.category, "newsletter");
|
|
@@ -81,6 +95,7 @@ describe("classifyParsedMessage", () => {
|
|
|
81
95
|
addressService(withFlags({ category: { value: "marketing", setAt: 1 } })),
|
|
82
96
|
"acc-1",
|
|
83
97
|
parsed,
|
|
98
|
+
"inbox",
|
|
84
99
|
);
|
|
85
100
|
|
|
86
101
|
// The header table alone would answer `personal`; the override wins
|
|
@@ -98,6 +113,7 @@ describe("classifyParsedMessage", () => {
|
|
|
98
113
|
}),
|
|
99
114
|
"acc-1",
|
|
100
115
|
parsed,
|
|
116
|
+
"inbox",
|
|
101
117
|
),
|
|
102
118
|
/throttled/,
|
|
103
119
|
);
|
|
@@ -124,6 +140,7 @@ describe("classifyParsedMessage", () => {
|
|
|
124
140
|
}),
|
|
125
141
|
"acc-1",
|
|
126
142
|
parsed,
|
|
143
|
+
"inbox",
|
|
127
144
|
);
|
|
128
145
|
|
|
129
146
|
assert.equal(reads, 0);
|
|
@@ -131,6 +148,48 @@ describe("classifyParsedMessage", () => {
|
|
|
131
148
|
});
|
|
132
149
|
});
|
|
133
150
|
|
|
151
|
+
describe("classifyParsedMessage sender mismatch", () => {
|
|
152
|
+
const noAddress = addressService(async () => {
|
|
153
|
+
throw new NotFoundError("no Address row");
|
|
154
|
+
});
|
|
155
|
+
|
|
156
|
+
it("compares the sender of mail in Junk the provider rated clean", async () => {
|
|
157
|
+
const parsed = await parseMessageBody(IMPERSONATION_EML);
|
|
158
|
+
|
|
159
|
+
const classification = await classifyParsedMessage(
|
|
160
|
+
noAddress,
|
|
161
|
+
"acc-1",
|
|
162
|
+
parsed,
|
|
163
|
+
"junk",
|
|
164
|
+
);
|
|
165
|
+
|
|
166
|
+
assert.equal(
|
|
167
|
+
classification.authenticity?.displayNameCorrespondence,
|
|
168
|
+
DisplayNameCorrespondence.Unrelated,
|
|
169
|
+
);
|
|
170
|
+
assert.deepEqual(classification.authenticity?.offDomainLinkDomains, [
|
|
171
|
+
"example.org",
|
|
172
|
+
]);
|
|
173
|
+
});
|
|
174
|
+
|
|
175
|
+
it("compares nothing for the same mail outside Junk without a provider verdict", async () => {
|
|
176
|
+
const parsed = await parseMessageBody(IMPERSONATION_EML);
|
|
177
|
+
|
|
178
|
+
const classification = await classifyParsedMessage(
|
|
179
|
+
noAddress,
|
|
180
|
+
"acc-1",
|
|
181
|
+
parsed,
|
|
182
|
+
"inbox",
|
|
183
|
+
);
|
|
184
|
+
|
|
185
|
+
assert.equal(
|
|
186
|
+
classification.authenticity?.displayNameCorrespondence,
|
|
187
|
+
undefined,
|
|
188
|
+
);
|
|
189
|
+
assert.equal(classification.authenticity?.offDomainLinkDomains, undefined);
|
|
190
|
+
});
|
|
191
|
+
});
|
|
192
|
+
|
|
134
193
|
describe("extractPrimaryFromEmail", () => {
|
|
135
194
|
it("reads the first From address", async () => {
|
|
136
195
|
const parsed = await parseMessageBody(PLAIN_EML);
|
package/src/classify-message.ts
CHANGED
|
@@ -13,6 +13,7 @@ import {
|
|
|
13
13
|
extractHasListUnsubscribe,
|
|
14
14
|
extractProviderSpam,
|
|
15
15
|
} from "./heuristics/classifyByHeaders.js";
|
|
16
|
+
import type { FolderPlacement } from "./heuristics/classifyPlacement.js";
|
|
16
17
|
import { extractSenderMismatch } from "./heuristics/senderMismatch.js";
|
|
17
18
|
|
|
18
19
|
type ThreadMessageCategory = ThreadMessageItem["category"];
|
|
@@ -75,6 +76,7 @@ export const classifyParsedMessage = async (
|
|
|
75
76
|
addressService: Pick<IAddressRepository, "getAddress">,
|
|
76
77
|
accountConfigId: string,
|
|
77
78
|
parsed: ParsedMail,
|
|
79
|
+
placement: FolderPlacement,
|
|
78
80
|
): Promise<UpdateMessageInput & { category: ThreadMessageCategory }> => {
|
|
79
81
|
const headerCategory = classifyByHeaders(parsed);
|
|
80
82
|
const authenticity = extractAuthenticity(parsed);
|
|
@@ -86,13 +88,14 @@ export const classifyParsedMessage = async (
|
|
|
86
88
|
// identity the message claims — on a shared-tenant host the verified
|
|
87
89
|
// subdomain belongs to whoever signed up. These two comparisons say
|
|
88
90
|
// whether the claim holds, and run only over mail the provider already
|
|
89
|
-
// called spam.
|
|
91
|
+
// called spam or that sits in the Junk folder.
|
|
90
92
|
const senderMismatch =
|
|
91
93
|
authenticity === null
|
|
92
94
|
? {}
|
|
93
95
|
: extractSenderMismatch(parsed, {
|
|
94
96
|
fromDomain: authenticity.fromDomain,
|
|
95
|
-
|
|
97
|
+
providerSpamClassified: providerSpam?.classified === true,
|
|
98
|
+
placement,
|
|
96
99
|
bulkSender: hasListUnsubscribe,
|
|
97
100
|
});
|
|
98
101
|
|
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
import type { IMailboxSpecialUseRepository } from "@remit/data-ports";
|
|
2
|
+
import type { FolderPlacement } from "./heuristics/classifyPlacement.js";
|
|
3
|
+
|
|
4
|
+
export type FolderPlacementLookup = Pick<
|
|
5
|
+
IMailboxSpecialUseRepository,
|
|
6
|
+
"findJunkMailbox" | "findInboxMailbox"
|
|
7
|
+
>;
|
|
8
|
+
|
|
9
|
+
type RoleMailbox = Awaited<
|
|
10
|
+
ReturnType<IMailboxSpecialUseRepository["findJunkMailbox"]>
|
|
11
|
+
>;
|
|
12
|
+
|
|
13
|
+
export interface FolderRoleMailboxes {
|
|
14
|
+
junkMailbox: RoleMailbox;
|
|
15
|
+
inboxMailbox: RoleMailbox;
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
export const resolveFolderRoleMailboxes = async (
|
|
19
|
+
lookup: FolderPlacementLookup,
|
|
20
|
+
accountId: string,
|
|
21
|
+
): Promise<FolderRoleMailboxes> => ({
|
|
22
|
+
junkMailbox: await lookup.findJunkMailbox(accountId),
|
|
23
|
+
inboxMailbox: await lookup.findInboxMailbox(accountId),
|
|
24
|
+
});
|
|
25
|
+
|
|
26
|
+
export const placementOf = (
|
|
27
|
+
mailboxId: string,
|
|
28
|
+
{ junkMailbox, inboxMailbox }: FolderRoleMailboxes,
|
|
29
|
+
): FolderPlacement => {
|
|
30
|
+
if (junkMailbox?.mailboxId === mailboxId) return "junk";
|
|
31
|
+
if (inboxMailbox?.mailboxId === mailboxId) return "inbox";
|
|
32
|
+
return "other";
|
|
33
|
+
};
|
|
34
|
+
|
|
35
|
+
export const resolveFolderPlacement = async (
|
|
36
|
+
lookup: FolderPlacementLookup,
|
|
37
|
+
accountId: string,
|
|
38
|
+
mailboxId: string,
|
|
39
|
+
): Promise<FolderPlacement> =>
|
|
40
|
+
placementOf(mailboxId, await resolveFolderRoleMailboxes(lookup, accountId));
|
|
41
|
+
|
|
42
|
+
export const resolveAccountFolderPlacements = async (
|
|
43
|
+
lookup: FolderPlacementLookup,
|
|
44
|
+
accountIds: readonly string[],
|
|
45
|
+
): Promise<ReadonlyMap<string, FolderPlacement>> => {
|
|
46
|
+
const placements = new Map<string, FolderPlacement>();
|
|
47
|
+
for (const accountId of accountIds) {
|
|
48
|
+
const { junkMailbox, inboxMailbox } = await resolveFolderRoleMailboxes(
|
|
49
|
+
lookup,
|
|
50
|
+
accountId,
|
|
51
|
+
);
|
|
52
|
+
if (inboxMailbox) placements.set(inboxMailbox.mailboxId, "inbox");
|
|
53
|
+
if (junkMailbox) placements.set(junkMailbox.mailboxId, "junk");
|
|
54
|
+
}
|
|
55
|
+
return placements;
|
|
56
|
+
};
|
|
@@ -245,7 +245,8 @@ describe("extractSenderMismatch", () => {
|
|
|
245
245
|
assert.deepEqual(
|
|
246
246
|
extractSenderMismatch(parsed, {
|
|
247
247
|
fromDomain: "serviceupdatebank.atlassian.net",
|
|
248
|
-
|
|
248
|
+
providerSpamClassified: true,
|
|
249
|
+
placement: "other",
|
|
249
250
|
bulkSender: false,
|
|
250
251
|
}),
|
|
251
252
|
{
|
|
@@ -255,18 +256,53 @@ describe("extractSenderMismatch", () => {
|
|
|
255
256
|
);
|
|
256
257
|
});
|
|
257
258
|
|
|
258
|
-
it("
|
|
259
|
+
it("flags a message sitting in Junk that the provider rated clean", async () => {
|
|
259
260
|
const parsed = await parse(infoMedicsPhish);
|
|
260
261
|
assert.deepEqual(
|
|
261
262
|
extractSenderMismatch(parsed, {
|
|
262
263
|
fromDomain: "serviceupdatebank.atlassian.net",
|
|
263
|
-
|
|
264
|
+
providerSpamClassified: false,
|
|
265
|
+
placement: "junk",
|
|
264
266
|
bulkSender: false,
|
|
265
267
|
}),
|
|
266
|
-
{
|
|
268
|
+
{
|
|
269
|
+
displayNameCorrespondence: DisplayNameCorrespondence.Unrelated,
|
|
270
|
+
offDomainLinkDomains: ["example.org"],
|
|
271
|
+
},
|
|
267
272
|
);
|
|
268
273
|
});
|
|
269
274
|
|
|
275
|
+
it("flags a provider-classified message outside Junk", async () => {
|
|
276
|
+
const parsed = await parse(infoMedicsPhish);
|
|
277
|
+
assert.deepEqual(
|
|
278
|
+
extractSenderMismatch(parsed, {
|
|
279
|
+
fromDomain: "serviceupdatebank.atlassian.net",
|
|
280
|
+
providerSpamClassified: true,
|
|
281
|
+
placement: "inbox",
|
|
282
|
+
bulkSender: false,
|
|
283
|
+
}),
|
|
284
|
+
{
|
|
285
|
+
displayNameCorrespondence: DisplayNameCorrespondence.Unrelated,
|
|
286
|
+
offDomainLinkDomains: ["example.org"],
|
|
287
|
+
},
|
|
288
|
+
);
|
|
289
|
+
});
|
|
290
|
+
|
|
291
|
+
for (const placement of ["inbox", "other"] as const) {
|
|
292
|
+
it(`compares nothing in ${placement} when the provider did not call the message spam`, async () => {
|
|
293
|
+
const parsed = await parse(infoMedicsPhish);
|
|
294
|
+
assert.deepEqual(
|
|
295
|
+
extractSenderMismatch(parsed, {
|
|
296
|
+
fromDomain: "serviceupdatebank.atlassian.net",
|
|
297
|
+
providerSpamClassified: false,
|
|
298
|
+
placement,
|
|
299
|
+
bulkSender: false,
|
|
300
|
+
}),
|
|
301
|
+
{},
|
|
302
|
+
);
|
|
303
|
+
});
|
|
304
|
+
}
|
|
305
|
+
|
|
270
306
|
it("flags a display name spelling the recipient's own address over a foreign domain", async () => {
|
|
271
307
|
const parsed = await parse([
|
|
272
308
|
"From: matthijs@ischen.nl <aramirez@secresaludguaviare.gov.co>",
|
|
@@ -279,7 +315,8 @@ describe("extractSenderMismatch", () => {
|
|
|
279
315
|
assert.equal(
|
|
280
316
|
extractSenderMismatch(parsed, {
|
|
281
317
|
fromDomain: "secresaludguaviare.gov.co",
|
|
282
|
-
|
|
318
|
+
providerSpamClassified: true,
|
|
319
|
+
placement: "other",
|
|
283
320
|
bulkSender: false,
|
|
284
321
|
}).displayNameCorrespondence,
|
|
285
322
|
DisplayNameCorrespondence.ForeignAddress,
|
|
@@ -299,7 +336,8 @@ describe("extractSenderMismatch", () => {
|
|
|
299
336
|
]);
|
|
300
337
|
const signals = extractSenderMismatch(parsed, {
|
|
301
338
|
fromDomain: "mailer.esp.example",
|
|
302
|
-
|
|
339
|
+
providerSpamClassified: true,
|
|
340
|
+
placement: "other",
|
|
303
341
|
bulkSender: true,
|
|
304
342
|
});
|
|
305
343
|
assert.equal(signals.displayNameCorrespondence, undefined);
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { DisplayNameCorrespondence } from "@remit/domain-enums";
|
|
2
2
|
import type { ParsedMail } from "mailparser";
|
|
3
3
|
import { getDomain, parse as parseHost } from "tldts";
|
|
4
|
+
import type { FolderPlacement } from "./classifyPlacement.js";
|
|
4
5
|
|
|
5
6
|
type CorrespondenceValue =
|
|
6
7
|
(typeof DisplayNameCorrespondence)[keyof typeof DisplayNameCorrespondence];
|
|
@@ -10,7 +11,8 @@ type CorrespondenceValue =
|
|
|
10
11
|
* that belongs to nobody at the sending domain, and body links that leave it.
|
|
11
12
|
*
|
|
12
13
|
* Both are deliberately aggressive, and both are computed only for mail the
|
|
13
|
-
* provider's own filter already called spam
|
|
14
|
+
* provider's own filter already called spam or that sits in the Junk folder
|
|
15
|
+
* (see {@link extractSenderMismatch}).
|
|
14
16
|
* On a shared-tenant host — a free Atlassian, Salesforce or Zendesk instance —
|
|
15
17
|
* the verified subdomain is chosen by whoever signed up, so a passing signature
|
|
16
18
|
* proves the domain and nothing about the identity the message claims.
|
|
@@ -22,7 +24,8 @@ export interface SenderMismatchSignals {
|
|
|
22
24
|
|
|
23
25
|
export interface SenderMismatchContext {
|
|
24
26
|
fromDomain: string;
|
|
25
|
-
|
|
27
|
+
providerSpamClassified: boolean;
|
|
28
|
+
placement: FolderPlacement;
|
|
26
29
|
/**
|
|
27
30
|
* List-Unsubscribe present. Bulk mail routinely shows a brand name over an
|
|
28
31
|
* ESP's sending domain, so the display-name comparison says nothing there and
|
|
@@ -223,7 +226,8 @@ export const extractOffDomainLinkDomains = (
|
|
|
223
226
|
};
|
|
224
227
|
|
|
225
228
|
/**
|
|
226
|
-
* Both signals, gated on the provider's own
|
|
229
|
+
* Both signals, gated on a spam verdict: the provider's own, or the message
|
|
230
|
+
* sitting in the Junk folder.
|
|
227
231
|
*
|
|
228
232
|
* The gate is what lets the checks be this aggressive: ordinary mail never
|
|
229
233
|
* reaches them, so a brand name over an ESP domain or a newsletter full of
|
|
@@ -234,7 +238,9 @@ export const extractSenderMismatch = (
|
|
|
234
238
|
parsed: ParsedMail,
|
|
235
239
|
context: SenderMismatchContext,
|
|
236
240
|
): SenderMismatchSignals => {
|
|
237
|
-
if (!context.
|
|
241
|
+
if (!context.providerSpamClassified && context.placement !== "junk") {
|
|
242
|
+
return {};
|
|
243
|
+
}
|
|
238
244
|
|
|
239
245
|
const offDomainLinkDomains = extractOffDomainLinkDomains(
|
|
240
246
|
parsed,
|
package/src/index.ts
CHANGED
|
@@ -25,6 +25,7 @@ export {
|
|
|
25
25
|
parseMessageBody,
|
|
26
26
|
} from "./body-parse.js";
|
|
27
27
|
export {
|
|
28
|
+
type BodySyncFolder,
|
|
28
29
|
type BodySyncLogger,
|
|
29
30
|
BodySyncService,
|
|
30
31
|
type ConnectionGetter,
|
|
@@ -125,6 +126,11 @@ export {
|
|
|
125
126
|
type UpdateFlagsInput,
|
|
126
127
|
type UpdateFlagsResult,
|
|
127
128
|
} from "./flag-queue.js";
|
|
129
|
+
export {
|
|
130
|
+
type FolderPlacementLookup,
|
|
131
|
+
resolveAccountFolderPlacements,
|
|
132
|
+
resolveFolderPlacement,
|
|
133
|
+
} from "./folder-placement.js";
|
|
128
134
|
export {
|
|
129
135
|
classifyByHeaders,
|
|
130
136
|
extractAuthenticity,
|