@remit/mailbox-service 0.0.42 → 0.0.44
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/package.json
CHANGED
|
@@ -0,0 +1,219 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Issue #499: `Address.flags.unsubscribed` auto-mark-read is a first-
|
|
3
|
+
* classification decision, like `Message.category` (#355) and the placement
|
|
4
|
+
* verdict (#383). Re-deriving it on the two shipped re-entrant paths —
|
|
5
|
+
* `fetchAndGetBody`'s `NoSuchKey` fallback and `syncBodies(..., force: true)`
|
|
6
|
+
* — undid a user who had deliberately marked such a message unread.
|
|
7
|
+
*
|
|
8
|
+
* The re-entrant fixtures below keep `flags.unsubscribed` set, so a regression
|
|
9
|
+
* that drops the guard shows up as a real `FlagQueueService.markAsRead`
|
|
10
|
+
* round-trip against a message the user owns, not as a fixture that happens to
|
|
11
|
+
* disagree with the flag.
|
|
12
|
+
*/
|
|
13
|
+
|
|
14
|
+
import assert from "node:assert/strict";
|
|
15
|
+
import { Readable } from "node:stream";
|
|
16
|
+
import { describe, it } from "node:test";
|
|
17
|
+
import type {
|
|
18
|
+
AddressItem,
|
|
19
|
+
IAddressRepository,
|
|
20
|
+
IEnvelopeRepository,
|
|
21
|
+
IMessageRepository,
|
|
22
|
+
IThreadMessageRepository,
|
|
23
|
+
MessageItem,
|
|
24
|
+
UpdateMessageInput,
|
|
25
|
+
} from "@remit/data-ports";
|
|
26
|
+
import type { StorageService } from "@remit/storage-service";
|
|
27
|
+
import { BodySyncService } from "./body-sync.js";
|
|
28
|
+
import type { FlagQueueService } from "./flag-queue.js";
|
|
29
|
+
import type { IImapConnection } from "./types.js";
|
|
30
|
+
|
|
31
|
+
const UNSUBSCRIBED_SENDER = "newsletter@example.com";
|
|
32
|
+
|
|
33
|
+
const PLAIN_EML = Buffer.from(
|
|
34
|
+
[
|
|
35
|
+
`From: Newsletter <${UNSUBSCRIBED_SENDER}>`,
|
|
36
|
+
"To: me@example.com",
|
|
37
|
+
"Subject: This week",
|
|
38
|
+
"List-Unsubscribe: <mailto:stop@example.com>",
|
|
39
|
+
"Content-Type: text/plain",
|
|
40
|
+
"",
|
|
41
|
+
"body",
|
|
42
|
+
].join("\r\n"),
|
|
43
|
+
);
|
|
44
|
+
|
|
45
|
+
interface MarkReadCall {
|
|
46
|
+
accountConfigId: string;
|
|
47
|
+
messageId: string;
|
|
48
|
+
accountId: string;
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
interface Harness {
|
|
52
|
+
service: BodySyncService;
|
|
53
|
+
markReadCalls: MarkReadCall[];
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
const buildHarness = (
|
|
57
|
+
message: Partial<MessageItem> & Pick<MessageItem, "messageId">,
|
|
58
|
+
retrieve: () => Promise<Buffer>,
|
|
59
|
+
): Harness => {
|
|
60
|
+
const markReadCalls: MarkReadCall[] = [];
|
|
61
|
+
|
|
62
|
+
const messageRow = {
|
|
63
|
+
uid: 1,
|
|
64
|
+
mailboxId: "mb-inbox",
|
|
65
|
+
...message,
|
|
66
|
+
} as unknown as MessageItem;
|
|
67
|
+
|
|
68
|
+
const messageService = {
|
|
69
|
+
get: async () => messageRow,
|
|
70
|
+
update: async (_messageId: string, input: UpdateMessageInput) => {
|
|
71
|
+
Object.assign(messageRow, input);
|
|
72
|
+
},
|
|
73
|
+
} as unknown as IMessageRepository;
|
|
74
|
+
|
|
75
|
+
const threadMessageService = {
|
|
76
|
+
findAllByMessageId: async () => [
|
|
77
|
+
{
|
|
78
|
+
threadMessageId: "tm-1",
|
|
79
|
+
messageId: message.messageId,
|
|
80
|
+
mailboxId: messageRow.mailboxId,
|
|
81
|
+
sentDate: 1,
|
|
82
|
+
isRead: false,
|
|
83
|
+
isDeleted: false,
|
|
84
|
+
hasStars: false,
|
|
85
|
+
hasAttachment: false,
|
|
86
|
+
},
|
|
87
|
+
],
|
|
88
|
+
update: async () => {},
|
|
89
|
+
} as unknown as IThreadMessageRepository;
|
|
90
|
+
|
|
91
|
+
const storageService = {
|
|
92
|
+
retrieve,
|
|
93
|
+
storeMessageBody: async () => ({ uri: `s3://bodies/${message.messageId}` }),
|
|
94
|
+
storeMessageBodyStream: async () => ({
|
|
95
|
+
uri: `s3://bodies/${message.messageId}`,
|
|
96
|
+
}),
|
|
97
|
+
storeParsedBody: async () => {},
|
|
98
|
+
listBodyParts: async () => [],
|
|
99
|
+
} as unknown as StorageService;
|
|
100
|
+
|
|
101
|
+
const addressService = {
|
|
102
|
+
getAddress: async () =>
|
|
103
|
+
({
|
|
104
|
+
flags: { unsubscribed: { value: true, setAt: 1 } },
|
|
105
|
+
}) as unknown as AddressItem,
|
|
106
|
+
incrementInboundCount: async () => {},
|
|
107
|
+
} as unknown as IAddressRepository;
|
|
108
|
+
|
|
109
|
+
const envelopeService = {
|
|
110
|
+
listBodyParts: async () => [],
|
|
111
|
+
} as unknown as IEnvelopeRepository;
|
|
112
|
+
|
|
113
|
+
const flagQueueService = {
|
|
114
|
+
markAsRead: async (
|
|
115
|
+
accountConfigId: string,
|
|
116
|
+
messageId: string,
|
|
117
|
+
accountId: string,
|
|
118
|
+
) => {
|
|
119
|
+
markReadCalls.push({ accountConfigId, messageId, accountId });
|
|
120
|
+
},
|
|
121
|
+
} as unknown as FlagQueueService;
|
|
122
|
+
|
|
123
|
+
const service = new BodySyncService(
|
|
124
|
+
messageService,
|
|
125
|
+
storageService,
|
|
126
|
+
threadMessageService,
|
|
127
|
+
addressService,
|
|
128
|
+
envelopeService,
|
|
129
|
+
{ info: () => {}, error: () => {} },
|
|
130
|
+
undefined,
|
|
131
|
+
undefined,
|
|
132
|
+
undefined,
|
|
133
|
+
{ flagQueueService },
|
|
134
|
+
);
|
|
135
|
+
|
|
136
|
+
return { service, markReadCalls };
|
|
137
|
+
};
|
|
138
|
+
|
|
139
|
+
const noSuchKeyError = () =>
|
|
140
|
+
Object.assign(new Error("missing"), { name: "NoSuchKey" });
|
|
141
|
+
|
|
142
|
+
const bodyConnection = () =>
|
|
143
|
+
({
|
|
144
|
+
openBox: async () => {},
|
|
145
|
+
fetchMessageBody: async () => PLAIN_EML,
|
|
146
|
+
async *fetchMessageBodies(uids: number[]) {
|
|
147
|
+
for (const uid of uids) {
|
|
148
|
+
yield { uid, source: Readable.from([PLAIN_EML]) };
|
|
149
|
+
}
|
|
150
|
+
},
|
|
151
|
+
}) as unknown as IImapConnection;
|
|
152
|
+
|
|
153
|
+
describe("unsubscribed auto-mark-read is decided once (issue #499)", () => {
|
|
154
|
+
it("marks a message read on the pass that first classifies it", async () => {
|
|
155
|
+
const harness = buildHarness({ messageId: "m-1" }, async () => {
|
|
156
|
+
throw new Error("no body stored yet; must not retrieve");
|
|
157
|
+
});
|
|
158
|
+
|
|
159
|
+
await harness.service.fetchAndGetBody(
|
|
160
|
+
"m-1",
|
|
161
|
+
"acc-1",
|
|
162
|
+
"cfg-1",
|
|
163
|
+
"INBOX",
|
|
164
|
+
async () => bodyConnection(),
|
|
165
|
+
);
|
|
166
|
+
|
|
167
|
+
assert.deepEqual(harness.markReadCalls, [
|
|
168
|
+
{ accountConfigId: "cfg-1", messageId: "m-1", accountId: "acc-1" },
|
|
169
|
+
]);
|
|
170
|
+
});
|
|
171
|
+
|
|
172
|
+
it("leaves a manually-unread message alone through the NoSuchKey IMAP re-fetch", async () => {
|
|
173
|
+
const harness = buildHarness(
|
|
174
|
+
{ messageId: "m-1", bodyStorageKey: "s3://bodies/m-1" },
|
|
175
|
+
async () => {
|
|
176
|
+
throw noSuchKeyError();
|
|
177
|
+
},
|
|
178
|
+
);
|
|
179
|
+
|
|
180
|
+
await harness.service.fetchAndGetBody(
|
|
181
|
+
"m-1",
|
|
182
|
+
"acc-1",
|
|
183
|
+
"cfg-1",
|
|
184
|
+
"INBOX",
|
|
185
|
+
async () => bodyConnection(),
|
|
186
|
+
);
|
|
187
|
+
|
|
188
|
+
assert.deepEqual(
|
|
189
|
+
harness.markReadCalls,
|
|
190
|
+
[],
|
|
191
|
+
"a re-entrant pass must not re-apply auto-read over the user's unread",
|
|
192
|
+
);
|
|
193
|
+
});
|
|
194
|
+
|
|
195
|
+
it("leaves a manually-unread message alone when syncBodies re-fetches with force", async () => {
|
|
196
|
+
const harness = buildHarness(
|
|
197
|
+
{ messageId: "m-1", bodyStorageKey: "s3://bodies/m-1" },
|
|
198
|
+
async () => {
|
|
199
|
+
throw new Error("force path must not retrieve from storage");
|
|
200
|
+
},
|
|
201
|
+
);
|
|
202
|
+
|
|
203
|
+
const result = await harness.service.syncBodies(
|
|
204
|
+
["m-1"],
|
|
205
|
+
"acc-1",
|
|
206
|
+
"cfg-1",
|
|
207
|
+
"INBOX",
|
|
208
|
+
async () => bodyConnection(),
|
|
209
|
+
true,
|
|
210
|
+
);
|
|
211
|
+
|
|
212
|
+
assert.deepEqual(result.syncedMessageIds, ["m-1"]);
|
|
213
|
+
assert.deepEqual(
|
|
214
|
+
harness.markReadCalls,
|
|
215
|
+
[],
|
|
216
|
+
"a forced re-sync must not re-apply auto-read over the user's unread",
|
|
217
|
+
);
|
|
218
|
+
});
|
|
219
|
+
});
|
package/src/body-sync.ts
CHANGED
|
@@ -51,6 +51,7 @@ import {
|
|
|
51
51
|
type PlacementVerdict,
|
|
52
52
|
resolveBlockedVsTrust,
|
|
53
53
|
} from "./heuristics/classifyPlacement.js";
|
|
54
|
+
import { extractSenderMismatch } from "./heuristics/senderMismatch.js";
|
|
54
55
|
import type { PlacementMoveService } from "./placement-move.js";
|
|
55
56
|
import { type QuarantineService, shapeFromMessageData } from "./quarantine.js";
|
|
56
57
|
import { extractSnippetFromEmail } from "./snippet.js";
|
|
@@ -200,6 +201,18 @@ const hasDecidedCategory = (
|
|
|
200
201
|
const hasDecidedPlacement = (placementDecidedAt: number | undefined): boolean =>
|
|
201
202
|
placementDecidedAt !== undefined;
|
|
202
203
|
|
|
204
|
+
/**
|
|
205
|
+
* Issue #499: whether a completed body-sync pass has already classified this
|
|
206
|
+
* message. `bodyStorageKey` is written last, once every derivation of that pass
|
|
207
|
+
* has run, so its presence means the pass that first classified the message
|
|
208
|
+
* finished. Reads it exactly as `syncBodies`' own skip guard does. The same two
|
|
209
|
+
* re-entrant paths `hasDecidedCategory` and `hasDecidedPlacement` guard
|
|
210
|
+
* (`fetchAndGetBody`'s `NoSuchKey` fallback, `syncBodies(..., force: true)`) are
|
|
211
|
+
* the ones that reach the derivations again with it already set.
|
|
212
|
+
*/
|
|
213
|
+
const hasClassifiedBody = (bodyStorageKey: string | undefined): boolean =>
|
|
214
|
+
Boolean(bodyStorageKey);
|
|
215
|
+
|
|
203
216
|
/**
|
|
204
217
|
* Issue #398: `flags.autoArchive` is a filing preference relative to the Inbox.
|
|
205
218
|
* A `leave` verdict reaches it for two unrelated reasons and only one of them
|
|
@@ -869,6 +882,12 @@ export class BodySyncService {
|
|
|
869
882
|
});
|
|
870
883
|
}
|
|
871
884
|
|
|
885
|
+
// Read once for the two write-once guards below — the auto-read decision
|
|
886
|
+
// and the category carry-forward. Taken here rather than earlier because
|
|
887
|
+
// nothing between this point and the Message update writes either field;
|
|
888
|
+
// the moves above touch `mailboxId` only.
|
|
889
|
+
const existingMessage = await this.messageService.get(messageId);
|
|
890
|
+
|
|
872
891
|
// `flags.unsubscribed` (issue #302, RFC 039 Decision 3): auto-mark-read,
|
|
873
892
|
// reusing the same FlagQueueService.markAsRead round-trip a manual
|
|
874
893
|
// mark-as-read goes through — idempotent on a retry (flipFlag no-ops when
|
|
@@ -879,6 +898,7 @@ export class BodySyncService {
|
|
|
879
898
|
accountId,
|
|
880
899
|
accountConfigId,
|
|
881
900
|
parsed,
|
|
901
|
+
existingMessage.bodyStorageKey,
|
|
882
902
|
);
|
|
883
903
|
|
|
884
904
|
const moved = Boolean(resolved.move || filterMoved);
|
|
@@ -907,7 +927,6 @@ export class BodySyncService {
|
|
|
907
927
|
// re-entrant paths for placement: `computePlacement` already declined to
|
|
908
928
|
// recompute a verdict once this field is set, so it is only ever present
|
|
909
929
|
// here on the pass that first decided it.
|
|
910
|
-
const existingMessage = await this.messageService.get(messageId);
|
|
911
930
|
const finalCategory = hasDecidedCategory(existingMessage.category)
|
|
912
931
|
? existingMessage.category
|
|
913
932
|
: classification.category;
|
|
@@ -1141,6 +1160,20 @@ export class BodySyncService {
|
|
|
1141
1160
|
const providerSpam = extractProviderSpam(parsed);
|
|
1142
1161
|
const hasListUnsubscribe = extractHasListUnsubscribe(parsed);
|
|
1143
1162
|
|
|
1163
|
+
// A passing SPF/DKIM/DMARC check proves the sending domain, not the
|
|
1164
|
+
// identity the message claims — on a shared-tenant host the verified
|
|
1165
|
+
// subdomain belongs to whoever signed up. These two comparisons say
|
|
1166
|
+
// whether the claim holds, and run only over mail the provider already
|
|
1167
|
+
// called spam.
|
|
1168
|
+
const senderMismatch =
|
|
1169
|
+
authenticity === null
|
|
1170
|
+
? {}
|
|
1171
|
+
: extractSenderMismatch(parsed, {
|
|
1172
|
+
fromDomain: authenticity.fromDomain,
|
|
1173
|
+
spamClassified: providerSpam?.classified === true,
|
|
1174
|
+
bulkSender: hasListUnsubscribe,
|
|
1175
|
+
});
|
|
1176
|
+
|
|
1144
1177
|
const fromEmail = extractPrimaryFromEmail(parsed);
|
|
1145
1178
|
const categoryOverride = fromEmail
|
|
1146
1179
|
? await this.resolveCategoryOverride(accountConfigId, fromEmail)
|
|
@@ -1149,7 +1182,9 @@ export class BodySyncService {
|
|
|
1149
1182
|
return {
|
|
1150
1183
|
category: categoryOverride ?? headerCategory,
|
|
1151
1184
|
hasListUnsubscribe,
|
|
1152
|
-
...(authenticity !== null
|
|
1185
|
+
...(authenticity !== null
|
|
1186
|
+
? { authenticity: { ...authenticity, ...senderMismatch } }
|
|
1187
|
+
: {}),
|
|
1153
1188
|
...(authResult !== null ? { authResult } : {}),
|
|
1154
1189
|
...(providerSpam !== null ? { providerSpam } : {}),
|
|
1155
1190
|
};
|
|
@@ -1284,14 +1319,22 @@ export class BodySyncService {
|
|
|
1284
1319
|
* separate expiry mechanism, and no caching of the decision beyond this
|
|
1285
1320
|
* per-message `Address` read. A no-op when body sync was built without an
|
|
1286
1321
|
* {@link UnsubscribeConfig} or the message carries no `From` address.
|
|
1322
|
+
*
|
|
1323
|
+
* Write-once per message (issue #499), like the `category` and placement
|
|
1324
|
+
* derivations alongside it: read state is applied on the pass that first
|
|
1325
|
+
* classifies a message and never re-decided. A user who marks such a message
|
|
1326
|
+
* unread afterwards owns it — a re-entrant pass over an already-classified
|
|
1327
|
+
* body would otherwise silently undo that.
|
|
1287
1328
|
*/
|
|
1288
1329
|
private async applyUnsubscribedAutoRead(
|
|
1289
1330
|
messageId: string,
|
|
1290
1331
|
accountId: string,
|
|
1291
1332
|
accountConfigId: string,
|
|
1292
1333
|
parsed: ParsedMail,
|
|
1334
|
+
storedBodyKey: string | undefined,
|
|
1293
1335
|
): Promise<void> {
|
|
1294
1336
|
if (!this.unsubscribeConfig) return;
|
|
1337
|
+
if (hasClassifiedBody(storedBodyKey)) return;
|
|
1295
1338
|
|
|
1296
1339
|
const fromEmail = extractPrimaryFromEmail(parsed);
|
|
1297
1340
|
if (!fromEmail) return;
|
|
@@ -0,0 +1,213 @@
|
|
|
1
|
+
import assert from "node:assert/strict";
|
|
2
|
+
import { describe, it } from "node:test";
|
|
3
|
+
import { DisplayNameCorrespondence } from "@remit/domain-enums";
|
|
4
|
+
import { simpleParser } from "mailparser";
|
|
5
|
+
import {
|
|
6
|
+
classifyDisplayNameCorrespondence,
|
|
7
|
+
extractOffDomainLinkDomains,
|
|
8
|
+
extractSenderMismatch,
|
|
9
|
+
} from "./senderMismatch.js";
|
|
10
|
+
|
|
11
|
+
const parse = async (lines: string[]) =>
|
|
12
|
+
simpleParser(Buffer.from(lines.join("\r\n")));
|
|
13
|
+
|
|
14
|
+
describe("classifyDisplayNameCorrespondence", () => {
|
|
15
|
+
it("corresponds when the name is a label of the From domain", () => {
|
|
16
|
+
assert.equal(
|
|
17
|
+
classifyDisplayNameCorrespondence("GitHub", "notifications.github.com"),
|
|
18
|
+
DisplayNameCorrespondence.Corresponds,
|
|
19
|
+
);
|
|
20
|
+
});
|
|
21
|
+
|
|
22
|
+
it("corresponds when a word of a decorated name is a label of the From domain", () => {
|
|
23
|
+
assert.equal(
|
|
24
|
+
classifyDisplayNameCorrespondence(
|
|
25
|
+
"GitHub Actions",
|
|
26
|
+
"notifications.github.com",
|
|
27
|
+
),
|
|
28
|
+
DisplayNameCorrespondence.Corresponds,
|
|
29
|
+
);
|
|
30
|
+
});
|
|
31
|
+
|
|
32
|
+
it("corresponds across a multi-part public suffix", () => {
|
|
33
|
+
assert.equal(
|
|
34
|
+
classifyDisplayNameCorrespondence("Sainsbury's", "mail.sainsburys.co.uk"),
|
|
35
|
+
DisplayNameCorrespondence.Corresponds,
|
|
36
|
+
);
|
|
37
|
+
});
|
|
38
|
+
|
|
39
|
+
it("is unrelated when the name appears nowhere in the From domain", () => {
|
|
40
|
+
assert.equal(
|
|
41
|
+
classifyDisplayNameCorrespondence(
|
|
42
|
+
"InfoMedics",
|
|
43
|
+
"serviceupdatebank.atlassian.net",
|
|
44
|
+
),
|
|
45
|
+
DisplayNameCorrespondence.Unrelated,
|
|
46
|
+
);
|
|
47
|
+
});
|
|
48
|
+
|
|
49
|
+
it("is a lookalike when a digit stands in for a letter of the domain", () => {
|
|
50
|
+
assert.equal(
|
|
51
|
+
classifyDisplayNameCorrespondence("InfoMedics", "1nfomedics.nl"),
|
|
52
|
+
DisplayNameCorrespondence.Lookalike,
|
|
53
|
+
);
|
|
54
|
+
});
|
|
55
|
+
|
|
56
|
+
it("is a lookalike for a short brand one character off", () => {
|
|
57
|
+
assert.equal(
|
|
58
|
+
classifyDisplayNameCorrespondence("PayPal", "paypa1.com"),
|
|
59
|
+
DisplayNameCorrespondence.Lookalike,
|
|
60
|
+
);
|
|
61
|
+
});
|
|
62
|
+
|
|
63
|
+
it("claims nothing when the display name is empty", () => {
|
|
64
|
+
assert.equal(
|
|
65
|
+
classifyDisplayNameCorrespondence("", "example.com"),
|
|
66
|
+
DisplayNameCorrespondence.NoClaim,
|
|
67
|
+
);
|
|
68
|
+
assert.equal(
|
|
69
|
+
classifyDisplayNameCorrespondence(undefined, "example.com"),
|
|
70
|
+
DisplayNameCorrespondence.NoClaim,
|
|
71
|
+
);
|
|
72
|
+
});
|
|
73
|
+
|
|
74
|
+
it("claims nothing when the display name is the address itself", () => {
|
|
75
|
+
assert.equal(
|
|
76
|
+
classifyDisplayNameCorrespondence(
|
|
77
|
+
"billing@serviceupdatebank.atlassian.net",
|
|
78
|
+
"serviceupdatebank.atlassian.net",
|
|
79
|
+
),
|
|
80
|
+
DisplayNameCorrespondence.NoClaim,
|
|
81
|
+
);
|
|
82
|
+
});
|
|
83
|
+
|
|
84
|
+
it("does not read the public suffix as a match for a brand containing it", () => {
|
|
85
|
+
assert.equal(
|
|
86
|
+
classifyDisplayNameCorrespondence("Netflix", "mailer.example.net"),
|
|
87
|
+
DisplayNameCorrespondence.Unrelated,
|
|
88
|
+
);
|
|
89
|
+
});
|
|
90
|
+
});
|
|
91
|
+
|
|
92
|
+
describe("extractOffDomainLinkDomains", () => {
|
|
93
|
+
it("names only the registrable domains that leave the sender's own", async () => {
|
|
94
|
+
const parsed = await parse([
|
|
95
|
+
"From: Jira <jira@serviceupdatebank.atlassian.net>",
|
|
96
|
+
"To: bob@example.com",
|
|
97
|
+
"Subject: Vordering",
|
|
98
|
+
"Content-Type: text/html",
|
|
99
|
+
"",
|
|
100
|
+
'<a href="https://serviceupdatebank.atlassian.net/browse/X">ticket</a>',
|
|
101
|
+
'<a href="https://betaal-vordering.example.org/pay">betaal nu</a>',
|
|
102
|
+
'<a href="https://cdn.betaal-vordering.example.org/logo.png">logo</a>',
|
|
103
|
+
]);
|
|
104
|
+
assert.deepEqual(
|
|
105
|
+
extractOffDomainLinkDomains(parsed, "serviceupdatebank.atlassian.net"),
|
|
106
|
+
["example.org"],
|
|
107
|
+
);
|
|
108
|
+
});
|
|
109
|
+
|
|
110
|
+
it("compares public-suffix-aware, so a co.uk sibling is not off-domain", async () => {
|
|
111
|
+
const parsed = await parse([
|
|
112
|
+
"From: Shop <shop@mail.example.co.uk>",
|
|
113
|
+
"To: bob@example.com",
|
|
114
|
+
"Subject: order",
|
|
115
|
+
"Content-Type: text/html",
|
|
116
|
+
"",
|
|
117
|
+
'<a href="https://www.example.co.uk/orders">orders</a>',
|
|
118
|
+
]);
|
|
119
|
+
assert.deepEqual(
|
|
120
|
+
extractOffDomainLinkDomains(parsed, "mail.example.co.uk"),
|
|
121
|
+
[],
|
|
122
|
+
);
|
|
123
|
+
});
|
|
124
|
+
|
|
125
|
+
it("ignores mailto, anchors and relative hrefs", async () => {
|
|
126
|
+
const parsed = await parse([
|
|
127
|
+
"From: Shop <shop@example.com>",
|
|
128
|
+
"To: bob@example.com",
|
|
129
|
+
"Subject: order",
|
|
130
|
+
"Content-Type: text/html",
|
|
131
|
+
"",
|
|
132
|
+
'<a href="mailto:help@elsewhere.example">mail us</a>',
|
|
133
|
+
'<a href="#top">top</a>',
|
|
134
|
+
'<a href="/orders">orders</a>',
|
|
135
|
+
'<a href="tel:+31201234567">call</a>',
|
|
136
|
+
]);
|
|
137
|
+
assert.deepEqual(extractOffDomainLinkDomains(parsed, "example.com"), []);
|
|
138
|
+
});
|
|
139
|
+
|
|
140
|
+
it("reads bare URLs out of a plain-text body", async () => {
|
|
141
|
+
const parsed = await parse([
|
|
142
|
+
"From: Shop <shop@example.com>",
|
|
143
|
+
"To: bob@example.com",
|
|
144
|
+
"Subject: order",
|
|
145
|
+
"",
|
|
146
|
+
"Betaal hier: https://betaal.elsewhere.example/pay?id=1",
|
|
147
|
+
]);
|
|
148
|
+
assert.deepEqual(extractOffDomainLinkDomains(parsed, "example.com"), [
|
|
149
|
+
"elsewhere.example",
|
|
150
|
+
]);
|
|
151
|
+
});
|
|
152
|
+
});
|
|
153
|
+
|
|
154
|
+
describe("extractSenderMismatch", () => {
|
|
155
|
+
const infoMedicsPhish = [
|
|
156
|
+
"From: InfoMedics <jira@serviceupdatebank.atlassian.net>",
|
|
157
|
+
"To: bob@example.com",
|
|
158
|
+
"Subject: Vordering",
|
|
159
|
+
"Authentication-Results: mx.example.com; dmarc=pass; spf=pass; dkim=pass",
|
|
160
|
+
"X-HalOne-Spam-Probability: 1",
|
|
161
|
+
"DKIM-Signature: v=1; a=rsa-sha256; d=custmx.one.com; s=sel; b=xxx",
|
|
162
|
+
"Content-Type: text/html",
|
|
163
|
+
"",
|
|
164
|
+
'<a href="https://betaal-vordering.example.org/pay">Betaal uw factuur</a>',
|
|
165
|
+
];
|
|
166
|
+
|
|
167
|
+
it("flags the display name and the links on a spam-classified message", async () => {
|
|
168
|
+
const parsed = await parse(infoMedicsPhish);
|
|
169
|
+
assert.deepEqual(
|
|
170
|
+
extractSenderMismatch(parsed, {
|
|
171
|
+
fromDomain: "serviceupdatebank.atlassian.net",
|
|
172
|
+
spamClassified: true,
|
|
173
|
+
bulkSender: false,
|
|
174
|
+
}),
|
|
175
|
+
{
|
|
176
|
+
displayNameCorrespondence: DisplayNameCorrespondence.Unrelated,
|
|
177
|
+
offDomainLinkDomains: ["example.org"],
|
|
178
|
+
},
|
|
179
|
+
);
|
|
180
|
+
});
|
|
181
|
+
|
|
182
|
+
it("compares nothing when the provider did not call the message spam", async () => {
|
|
183
|
+
const parsed = await parse(infoMedicsPhish);
|
|
184
|
+
assert.deepEqual(
|
|
185
|
+
extractSenderMismatch(parsed, {
|
|
186
|
+
fromDomain: "serviceupdatebank.atlassian.net",
|
|
187
|
+
spamClassified: false,
|
|
188
|
+
bulkSender: false,
|
|
189
|
+
}),
|
|
190
|
+
{},
|
|
191
|
+
);
|
|
192
|
+
});
|
|
193
|
+
|
|
194
|
+
it("leaves the display name uncompared for a bulk sender", async () => {
|
|
195
|
+
const parsed = await parse([
|
|
196
|
+
"From: Dutch Cycling Weekly <bounce-9f2@mailer.esp.example>",
|
|
197
|
+
"To: bob@example.com",
|
|
198
|
+
"Subject: This week in cycling",
|
|
199
|
+
"List-Unsubscribe: <https://mailer.esp.example/u/9f2>",
|
|
200
|
+
"X-HalOne-Spam-Probability: 1",
|
|
201
|
+
"Content-Type: text/html",
|
|
202
|
+
"",
|
|
203
|
+
'<a href="https://dutchcyclingweekly.example.org/issue/12">read</a>',
|
|
204
|
+
]);
|
|
205
|
+
const signals = extractSenderMismatch(parsed, {
|
|
206
|
+
fromDomain: "mailer.esp.example",
|
|
207
|
+
spamClassified: true,
|
|
208
|
+
bulkSender: true,
|
|
209
|
+
});
|
|
210
|
+
assert.equal(signals.displayNameCorrespondence, undefined);
|
|
211
|
+
assert.deepEqual(signals.offDomainLinkDomains, ["example.org"]);
|
|
212
|
+
});
|
|
213
|
+
});
|
|
@@ -0,0 +1,213 @@
|
|
|
1
|
+
import { DisplayNameCorrespondence } from "@remit/domain-enums";
|
|
2
|
+
import type { ParsedMail } from "mailparser";
|
|
3
|
+
import { getDomain, parse as parseHost } from "tldts";
|
|
4
|
+
|
|
5
|
+
type CorrespondenceValue =
|
|
6
|
+
(typeof DisplayNameCorrespondence)[keyof typeof DisplayNameCorrespondence];
|
|
7
|
+
|
|
8
|
+
/**
|
|
9
|
+
* The two signals that survive a passing SPF/DKIM/DMARC check: a display name
|
|
10
|
+
* that belongs to nobody at the sending domain, and body links that leave it.
|
|
11
|
+
*
|
|
12
|
+
* Both are deliberately aggressive, and both are computed only for mail the
|
|
13
|
+
* provider's own filter already called spam (see {@link extractSenderMismatch}).
|
|
14
|
+
* On a shared-tenant host — a free Atlassian, Salesforce or Zendesk instance —
|
|
15
|
+
* the verified subdomain is chosen by whoever signed up, so a passing signature
|
|
16
|
+
* proves the domain and nothing about the identity the message claims.
|
|
17
|
+
*/
|
|
18
|
+
export interface SenderMismatchSignals {
|
|
19
|
+
displayNameCorrespondence?: CorrespondenceValue;
|
|
20
|
+
offDomainLinkDomains?: string[];
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
export interface SenderMismatchContext {
|
|
24
|
+
fromDomain: string;
|
|
25
|
+
spamClassified: boolean;
|
|
26
|
+
/**
|
|
27
|
+
* List-Unsubscribe present. Bulk mail routinely shows a brand name over an
|
|
28
|
+
* ESP's sending domain, so the display-name comparison says nothing there and
|
|
29
|
+
* is not made.
|
|
30
|
+
*/
|
|
31
|
+
bulkSender: boolean;
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
const MAX_LINK_DOMAINS = 10;
|
|
35
|
+
|
|
36
|
+
const normalize = (value: string): string =>
|
|
37
|
+
value.toLowerCase().replace(/[^a-z0-9]/g, "");
|
|
38
|
+
|
|
39
|
+
/**
|
|
40
|
+
* The parts of a host a display name could honestly correspond to: the
|
|
41
|
+
* registrable name without its public suffix, every subdomain label, and the
|
|
42
|
+
* registrable domain run together. The suffix itself is dropped — `net` and
|
|
43
|
+
* `com` sit inside half the brand names in existence.
|
|
44
|
+
*/
|
|
45
|
+
const correspondenceCandidates = (host: string): string[] => {
|
|
46
|
+
const parsed = parseHost(host);
|
|
47
|
+
const labels = (parsed.subdomain ?? "").split(".");
|
|
48
|
+
if (parsed.domainWithoutSuffix !== null) {
|
|
49
|
+
labels.push(parsed.domainWithoutSuffix);
|
|
50
|
+
} else {
|
|
51
|
+
labels.push(...host.split(".").slice(0, -1));
|
|
52
|
+
}
|
|
53
|
+
if (parsed.domain !== null) labels.push(parsed.domain);
|
|
54
|
+
return labels.map(normalize).filter((label) => label.length >= 3);
|
|
55
|
+
};
|
|
56
|
+
|
|
57
|
+
const words = (displayName: string): string[] =>
|
|
58
|
+
displayName
|
|
59
|
+
.toLowerCase()
|
|
60
|
+
.split(/[^a-z0-9]+/)
|
|
61
|
+
.filter((word) => word.length >= 3);
|
|
62
|
+
|
|
63
|
+
const editDistance = (a: string, b: string): number => {
|
|
64
|
+
let previous = Array.from({ length: b.length + 1 }, (_, i) => i);
|
|
65
|
+
for (let i = 1; i <= a.length; i++) {
|
|
66
|
+
const current = [i];
|
|
67
|
+
for (let j = 1; j <= b.length; j++) {
|
|
68
|
+
current[j] = Math.min(
|
|
69
|
+
previous[j] + 1,
|
|
70
|
+
current[j - 1] + 1,
|
|
71
|
+
previous[j - 1] + (a[i - 1] === b[j - 1] ? 0 : 1),
|
|
72
|
+
);
|
|
73
|
+
}
|
|
74
|
+
previous = current;
|
|
75
|
+
}
|
|
76
|
+
return previous[b.length];
|
|
77
|
+
};
|
|
78
|
+
|
|
79
|
+
/**
|
|
80
|
+
* The distance a name of this length may be from a domain label and still be
|
|
81
|
+
* read as an imitation of it. Tight on purpose: an unrelated brand name and an
|
|
82
|
+
* ESP's domain are always far apart, so distance alone must never decide.
|
|
83
|
+
*/
|
|
84
|
+
const lookalikeThreshold = (length: number): number => {
|
|
85
|
+
if (length < 5) return 0;
|
|
86
|
+
if (length < 9) return 1;
|
|
87
|
+
return 2;
|
|
88
|
+
};
|
|
89
|
+
|
|
90
|
+
/**
|
|
91
|
+
* Whether the From display name corresponds to the From domain.
|
|
92
|
+
*
|
|
93
|
+
* Containment decides: the normalised name, or any word of it, appearing inside
|
|
94
|
+
* the registrable domain or one of the domain's labels. `GitHub` sits inside
|
|
95
|
+
* `notifications.github.com`; `InfoMedics` sits nowhere inside
|
|
96
|
+
* `serviceupdatebank.atlassian.net`.
|
|
97
|
+
*
|
|
98
|
+
* A bounded edit distance is the secondary test, and only reaches names that
|
|
99
|
+
* nearly match a label — `InfoMedics` against `1nfomedics.nl`. It cannot promote
|
|
100
|
+
* an unrelated name on its own.
|
|
101
|
+
*/
|
|
102
|
+
export const classifyDisplayNameCorrespondence = (
|
|
103
|
+
displayName: string | undefined,
|
|
104
|
+
fromDomain: string,
|
|
105
|
+
): CorrespondenceValue => {
|
|
106
|
+
const raw = (displayName ?? "").trim();
|
|
107
|
+
if (raw === "" || raw.includes("@")) {
|
|
108
|
+
return DisplayNameCorrespondence.NoClaim;
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
const name = normalize(raw);
|
|
112
|
+
if (name.length < 3) return DisplayNameCorrespondence.NoClaim;
|
|
113
|
+
|
|
114
|
+
const candidates = correspondenceCandidates(fromDomain);
|
|
115
|
+
if (candidates.length === 0) return DisplayNameCorrespondence.NoClaim;
|
|
116
|
+
|
|
117
|
+
const terms = [name, ...words(raw)];
|
|
118
|
+
for (const term of terms) {
|
|
119
|
+
for (const candidate of candidates) {
|
|
120
|
+
if (candidate.includes(term) || term.includes(candidate)) {
|
|
121
|
+
return DisplayNameCorrespondence.Corresponds;
|
|
122
|
+
}
|
|
123
|
+
}
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
for (const candidate of candidates) {
|
|
127
|
+
const threshold = lookalikeThreshold(
|
|
128
|
+
Math.min(name.length, candidate.length),
|
|
129
|
+
);
|
|
130
|
+
if (threshold === 0) continue;
|
|
131
|
+
if (Math.abs(name.length - candidate.length) > threshold) continue;
|
|
132
|
+
if (editDistance(name, candidate) <= threshold) {
|
|
133
|
+
return DisplayNameCorrespondence.Lookalike;
|
|
134
|
+
}
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
return DisplayNameCorrespondence.Unrelated;
|
|
138
|
+
};
|
|
139
|
+
|
|
140
|
+
const hrefsFromHtml = (html: string): string[] => {
|
|
141
|
+
const out: string[] = [];
|
|
142
|
+
const pattern = /href\s*=\s*(?:"([^"]*)"|'([^']*)'|([^\s>]+))/gi;
|
|
143
|
+
let match = pattern.exec(html);
|
|
144
|
+
while (match !== null) {
|
|
145
|
+
out.push(match[1] ?? match[2] ?? match[3] ?? "");
|
|
146
|
+
match = pattern.exec(html);
|
|
147
|
+
}
|
|
148
|
+
return out;
|
|
149
|
+
};
|
|
150
|
+
|
|
151
|
+
const urlsFromText = (text: string): string[] =>
|
|
152
|
+
text.match(/https?:\/\/[^\s<>"')\]]+/gi) ?? [];
|
|
153
|
+
|
|
154
|
+
/**
|
|
155
|
+
* Registrable domains the body's links point at, minus the sender's own.
|
|
156
|
+
*
|
|
157
|
+
* `mailto:`, `tel:`, in-page anchors and relative hrefs carry no destination
|
|
158
|
+
* domain and are skipped; anything else is resolved public-suffix-aware through
|
|
159
|
+
* tldts, so `co.uk` and `atlassian.net` both survive the comparison a naive dot
|
|
160
|
+
* split gets wrong.
|
|
161
|
+
*/
|
|
162
|
+
export const extractOffDomainLinkDomains = (
|
|
163
|
+
parsed: ParsedMail,
|
|
164
|
+
fromDomain: string,
|
|
165
|
+
): string[] => {
|
|
166
|
+
const senderDomain = getDomain(fromDomain);
|
|
167
|
+
const html = typeof parsed.html === "string" ? parsed.html : "";
|
|
168
|
+
const text = typeof parsed.text === "string" ? parsed.text : "";
|
|
169
|
+
|
|
170
|
+
const seen = new Set<string>();
|
|
171
|
+
const out: string[] = [];
|
|
172
|
+
for (const href of [...hrefsFromHtml(html), ...urlsFromText(text)]) {
|
|
173
|
+
const value = href.trim();
|
|
174
|
+
if (value === "" || !/^https?:\/\//i.test(value)) continue;
|
|
175
|
+
const domain = getDomain(value);
|
|
176
|
+
if (domain === null || domain === senderDomain) continue;
|
|
177
|
+
if (seen.has(domain)) continue;
|
|
178
|
+
seen.add(domain);
|
|
179
|
+
out.push(domain);
|
|
180
|
+
if (out.length === MAX_LINK_DOMAINS) break;
|
|
181
|
+
}
|
|
182
|
+
return out;
|
|
183
|
+
};
|
|
184
|
+
|
|
185
|
+
/**
|
|
186
|
+
* Both signals, gated on the provider's own spam verdict.
|
|
187
|
+
*
|
|
188
|
+
* The gate is what lets the checks be this aggressive: ordinary mail never
|
|
189
|
+
* reaches them, so a brand name over an ESP domain or a newsletter full of
|
|
190
|
+
* tracking links can never be flagged. Returns an empty object when the gate
|
|
191
|
+
* does not open — the fields stay absent, meaning "not compared".
|
|
192
|
+
*/
|
|
193
|
+
export const extractSenderMismatch = (
|
|
194
|
+
parsed: ParsedMail,
|
|
195
|
+
context: SenderMismatchContext,
|
|
196
|
+
): SenderMismatchSignals => {
|
|
197
|
+
if (!context.spamClassified) return {};
|
|
198
|
+
|
|
199
|
+
const offDomainLinkDomains = extractOffDomainLinkDomains(
|
|
200
|
+
parsed,
|
|
201
|
+
context.fromDomain,
|
|
202
|
+
);
|
|
203
|
+
|
|
204
|
+
if (context.bulkSender) return { offDomainLinkDomains };
|
|
205
|
+
|
|
206
|
+
return {
|
|
207
|
+
displayNameCorrespondence: classifyDisplayNameCorrespondence(
|
|
208
|
+
parsed.from?.value?.[0]?.name,
|
|
209
|
+
context.fromDomain,
|
|
210
|
+
),
|
|
211
|
+
offDomainLinkDomains,
|
|
212
|
+
};
|
|
213
|
+
};
|
package/src/index.ts
CHANGED
|
@@ -114,6 +114,13 @@ export {
|
|
|
114
114
|
type PlacementAction,
|
|
115
115
|
type PlacementVerdict,
|
|
116
116
|
} from "./heuristics/classifyPlacement.js";
|
|
117
|
+
export {
|
|
118
|
+
classifyDisplayNameCorrespondence,
|
|
119
|
+
extractOffDomainLinkDomains,
|
|
120
|
+
extractSenderMismatch,
|
|
121
|
+
type SenderMismatchContext,
|
|
122
|
+
type SenderMismatchSignals,
|
|
123
|
+
} from "./heuristics/senderMismatch.js";
|
|
117
124
|
export { SOCIAL_DOMAINS } from "./heuristics/socialDomains.js";
|
|
118
125
|
export { TRANSACTIONAL_DOMAINS } from "./heuristics/transactionalDomains.js";
|
|
119
126
|
// IMAP connection (ImapFlow-based)
|