@remit/mailbox-service 0.0.93 → 0.0.95
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/package.json
CHANGED
|
@@ -343,8 +343,9 @@ describe("Address.flags.category overrides classification at sync time (issue #2
|
|
|
343
343
|
);
|
|
344
344
|
|
|
345
345
|
// The harness's default `retrieve` throws, so a storage read would fail
|
|
346
|
-
// this outright.
|
|
347
|
-
//
|
|
346
|
+
// this outright. Reaching back is the back-apply job's work (#415,
|
|
347
|
+
// `backApplySenderCategory`), fired by the flag PATCH; a sync pass is
|
|
348
|
+
// still not it.
|
|
348
349
|
assert.equal(result.skippedCount, 1);
|
|
349
350
|
assert.deepEqual(result.failedMessageIds, []);
|
|
350
351
|
assert.deepEqual(harness.messageUpdates, []);
|
package/src/body-sync.ts
CHANGED
|
@@ -59,9 +59,10 @@ import {
|
|
|
59
59
|
classifyPlacement,
|
|
60
60
|
type FolderPlacement,
|
|
61
61
|
type PlacementVerdict,
|
|
62
|
-
|
|
62
|
+
resolveSenderPlacement,
|
|
63
63
|
} from "./heuristics/classifyPlacement.js";
|
|
64
64
|
import { extractSenderMismatch } from "./heuristics/senderMismatch.js";
|
|
65
|
+
import { denormalizeMessageCategory } from "./message-category.js";
|
|
65
66
|
import type { PlacementMoveService } from "./placement-move.js";
|
|
66
67
|
import { type QuarantineService, shapeFromMessageData } from "./quarantine.js";
|
|
67
68
|
import { extractSnippetFromEmail } from "./snippet.js";
|
|
@@ -165,23 +166,6 @@ const extractSnippet = (parsed: ParsedMail): string =>
|
|
|
165
166
|
SNIPPET_LENGTH,
|
|
166
167
|
);
|
|
167
168
|
|
|
168
|
-
/**
|
|
169
|
-
* A row is skipped only when every field the denormalization would write
|
|
170
|
-
* already matches. `snippet` and `listId` are absent from the update when the
|
|
171
|
-
* message has neither, and an absent field is not a mismatch.
|
|
172
|
-
*/
|
|
173
|
-
const alreadyDenormalized = (
|
|
174
|
-
row: ThreadMessageItem,
|
|
175
|
-
update: {
|
|
176
|
-
category: ThreadMessageCategory;
|
|
177
|
-
snippet?: string;
|
|
178
|
-
listId?: string;
|
|
179
|
-
},
|
|
180
|
-
): boolean =>
|
|
181
|
-
row.category === update.category &&
|
|
182
|
-
(update.snippet === undefined || row.snippet === update.snippet) &&
|
|
183
|
-
(update.listId === undefined || row.listId === update.listId);
|
|
184
|
-
|
|
185
169
|
/**
|
|
186
170
|
* RFC 034 Decision 3.1: `Message.category` is written once and never mutated
|
|
187
171
|
* after — RFC 030's message-list GSI sort key depends on it never churning.
|
|
@@ -928,12 +912,11 @@ export class BodySyncService {
|
|
|
928
912
|
|
|
929
913
|
// Thread-list denormalization, using the same write-once category as the
|
|
930
914
|
// Message update below — the two rows must never disagree on category.
|
|
931
|
-
await
|
|
915
|
+
await denormalizeMessageCategory(
|
|
916
|
+
{ threadMessageService: this.threadMessageService },
|
|
932
917
|
accountConfigId,
|
|
933
918
|
messageId,
|
|
934
|
-
finalCategory,
|
|
935
|
-
snippet,
|
|
936
|
-
listId,
|
|
919
|
+
{ category: finalCategory, snippet, listId },
|
|
937
920
|
);
|
|
938
921
|
|
|
939
922
|
// `classificationState` records that the classifier ran, in the same
|
|
@@ -1193,7 +1176,7 @@ export class BodySyncService {
|
|
|
1193
1176
|
* `deriveSenderTrust` always did — `vip → wellknown → unknown`, untouched by
|
|
1194
1177
|
* `blocked` — plus `blocked`/`autoArchive` off the same row. `trustSetAt` is
|
|
1195
1178
|
* the `setAt` of whichever flag produced the trust value, needed by
|
|
1196
|
-
* {@link
|
|
1179
|
+
* {@link resolveSenderPlacement}'s tie-break; it stays local to placement and
|
|
1197
1180
|
* never reaches `deriveSenderTrust`'s own contract (the trust badge).
|
|
1198
1181
|
*/
|
|
1199
1182
|
private async deriveSenderPlacementSignals(
|
|
@@ -1204,11 +1187,13 @@ export class BodySyncService {
|
|
|
1204
1187
|
trustSetAt?: number;
|
|
1205
1188
|
blocked: boolean;
|
|
1206
1189
|
blockedSetAt?: number;
|
|
1190
|
+
neverSpam: boolean;
|
|
1207
1191
|
autoArchive: boolean;
|
|
1208
1192
|
}> {
|
|
1209
1193
|
const unknown = {
|
|
1210
1194
|
trust: SenderTrust.Unknown,
|
|
1211
1195
|
blocked: false,
|
|
1196
|
+
neverSpam: false,
|
|
1212
1197
|
autoArchive: false,
|
|
1213
1198
|
} as const;
|
|
1214
1199
|
try {
|
|
@@ -1224,6 +1209,7 @@ export class BodySyncService {
|
|
|
1224
1209
|
trustSetAt: flags.vip.setAt,
|
|
1225
1210
|
blocked: flags.blocked?.value === true,
|
|
1226
1211
|
blockedSetAt: flags.blocked?.setAt,
|
|
1212
|
+
neverSpam: flags.neverSpam?.value === true,
|
|
1227
1213
|
autoArchive: flags.autoArchive?.value === true,
|
|
1228
1214
|
};
|
|
1229
1215
|
}
|
|
@@ -1233,6 +1219,7 @@ export class BodySyncService {
|
|
|
1233
1219
|
trustSetAt: flags.wellknown.setAt,
|
|
1234
1220
|
blocked: flags.blocked?.value === true,
|
|
1235
1221
|
blockedSetAt: flags.blocked?.setAt,
|
|
1222
|
+
neverSpam: flags.neverSpam?.value === true,
|
|
1236
1223
|
autoArchive: flags.autoArchive?.value === true,
|
|
1237
1224
|
};
|
|
1238
1225
|
}
|
|
@@ -1240,6 +1227,7 @@ export class BodySyncService {
|
|
|
1240
1227
|
...unknown,
|
|
1241
1228
|
blocked: flags?.blocked?.value === true,
|
|
1242
1229
|
blockedSetAt: flags?.blocked?.setAt,
|
|
1230
|
+
neverSpam: flags?.neverSpam?.value === true,
|
|
1243
1231
|
autoArchive: flags?.autoArchive?.value === true,
|
|
1244
1232
|
};
|
|
1245
1233
|
} catch (err) {
|
|
@@ -1531,12 +1519,14 @@ export class BodySyncService {
|
|
|
1531
1519
|
: {
|
|
1532
1520
|
trust: SenderTrust.Unknown,
|
|
1533
1521
|
blocked: false,
|
|
1522
|
+
neverSpam: false,
|
|
1534
1523
|
autoArchive: false,
|
|
1535
1524
|
};
|
|
1536
1525
|
|
|
1537
|
-
const { senderTrust,
|
|
1526
|
+
const { senderTrust, senderOverride } = resolveSenderPlacement(
|
|
1538
1527
|
{ trust: signals.trust, setAt: signals.trustSetAt },
|
|
1539
1528
|
{ blocked: signals.blocked, setAt: signals.blockedSetAt },
|
|
1529
|
+
signals.neverSpam,
|
|
1540
1530
|
);
|
|
1541
1531
|
|
|
1542
1532
|
// The verdict needs the classification signals (providerSpam,
|
|
@@ -1547,7 +1537,7 @@ export class BodySyncService {
|
|
|
1547
1537
|
candidate,
|
|
1548
1538
|
placement,
|
|
1549
1539
|
senderTrust,
|
|
1550
|
-
|
|
1540
|
+
senderOverride,
|
|
1551
1541
|
);
|
|
1552
1542
|
|
|
1553
1543
|
// A `leave` verdict — including "nothing confident to say" — carries no
|
|
@@ -1891,73 +1881,4 @@ export class BodySyncService {
|
|
|
1891
1881
|
throw err;
|
|
1892
1882
|
}
|
|
1893
1883
|
}
|
|
1894
|
-
|
|
1895
|
-
/**
|
|
1896
|
-
* Write the denormalized `category` (and optionally the snippet and the
|
|
1897
|
-
* normalized `List-Id`) onto EVERY ThreadMessage row the message has — the
|
|
1898
|
-
* copy the list/search read path serves without a per-row Message fetch.
|
|
1899
|
-
*
|
|
1900
|
-
* More than one row per messageId is schema-legal but not normally produced,
|
|
1901
|
-
* and this iterates for the same reason `message-move.ts` does (see the model
|
|
1902
|
-
* stated at its `deleteThreadMessagesForMessage`): the key permits it and
|
|
1903
|
-
* nothing enforces otherwise. It is NOT the second mailbox a message appears
|
|
1904
|
-
* in — `deriveMessageId` and `deriveThreadMessageId` are both
|
|
1905
|
-
* mailbox-independent, so INBOX and Archive resolve to one row, and a copy
|
|
1906
|
-
* gets its own messageId. The reachable case is thread-root drift: the same
|
|
1907
|
-
* message re-saved under different `References`, which mints a second
|
|
1908
|
-
* threadId and so a second row. Iterating is therefore hardening against a
|
|
1909
|
-
* legal state, not a repair for one the sync path manufactures, which is why
|
|
1910
|
-
* the tree's other single-row `messageId` lookups are correct as they stand.
|
|
1911
|
-
* `flag-queue.ts` iterates the same list.
|
|
1912
|
-
*
|
|
1913
|
-
* Rows are looked up by messageId, so this does not depend on the RFC822
|
|
1914
|
-
* Message-ID header — a headerless message still gets denormalized, matching
|
|
1915
|
-
* the unconditional Message.category write. The composite set is built per
|
|
1916
|
-
* row, never reused: `mailboxId` and `isRead` can differ between two rows for
|
|
1917
|
-
* one message, and it is passed at all so that a future key-attribute
|
|
1918
|
-
* addition touching the lsi3/lsi4/lsi5/gsi2 sort keys keeps the index rows
|
|
1919
|
-
* consistent.
|
|
1920
|
-
*/
|
|
1921
|
-
private async denormalizeCategory(
|
|
1922
|
-
accountConfigId: string,
|
|
1923
|
-
messageId: string,
|
|
1924
|
-
category: ThreadMessageCategory,
|
|
1925
|
-
snippet?: string,
|
|
1926
|
-
listId?: string,
|
|
1927
|
-
): Promise<void> {
|
|
1928
|
-
const rows = await this.threadMessageService.findAllByMessageId(
|
|
1929
|
-
accountConfigId,
|
|
1930
|
-
messageId,
|
|
1931
|
-
);
|
|
1932
|
-
if (rows.length === 0) {
|
|
1933
|
-
throw new NotFoundError(
|
|
1934
|
-
`ThreadMessage not found for message ${messageId}`,
|
|
1935
|
-
);
|
|
1936
|
-
}
|
|
1937
|
-
|
|
1938
|
-
const update = {
|
|
1939
|
-
category,
|
|
1940
|
-
...(snippet ? { snippet } : {}),
|
|
1941
|
-
...(listId ? { listId } : {}),
|
|
1942
|
-
};
|
|
1943
|
-
|
|
1944
|
-
for (const row of rows) {
|
|
1945
|
-
if (alreadyDenormalized(row, update)) continue;
|
|
1946
|
-
await this.threadMessageService.update(
|
|
1947
|
-
accountConfigId,
|
|
1948
|
-
row.threadMessageId,
|
|
1949
|
-
update,
|
|
1950
|
-
{
|
|
1951
|
-
composites: {
|
|
1952
|
-
sentDate: row.sentDate,
|
|
1953
|
-
mailboxId: row.mailboxId,
|
|
1954
|
-
isRead: row.isRead,
|
|
1955
|
-
isDeleted: row.isDeleted,
|
|
1956
|
-
hasStars: row.hasStars,
|
|
1957
|
-
hasAttachment: row.hasAttachment,
|
|
1958
|
-
},
|
|
1959
|
-
},
|
|
1960
|
-
);
|
|
1961
|
-
}
|
|
1962
|
-
}
|
|
1963
1884
|
}
|
|
@@ -1,20 +1,21 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* `classifyPlacement` had no dedicated unit test — only the DKIM/DMARC paths
|
|
3
3
|
* were exercised indirectly through realistic-mail fixtures elsewhere. Issue
|
|
4
|
-
* #300 (RFC 039 Decision 3/3a)
|
|
5
|
-
* independent of every DKIM/DMARC/provider signal, plus a
|
|
6
|
-
* against `vip`/`wellknown` (Decision 3a
|
|
7
|
-
*
|
|
8
|
-
* regression in
|
|
4
|
+
* #300 (RFC 039 Decision 3/3a) added the user's placement instruction as a
|
|
5
|
+
* confident move independent of every DKIM/DMARC/provider signal, plus a
|
|
6
|
+
* `setAt` tie-break against `vip`/`wellknown` (Decision 3a). Issue #605 adds
|
|
7
|
+
* the other direction, `neverSpam`. These tests cover both instruction
|
|
8
|
+
* branches and the pre-existing DKIM/DMARC branches, so a regression in any of
|
|
9
|
+
* them shows up here.
|
|
9
10
|
*/
|
|
10
11
|
|
|
11
12
|
import assert from "node:assert/strict";
|
|
12
13
|
import { describe, it } from "node:test";
|
|
13
14
|
import type { MessageItem } from "@remit/data-ports";
|
|
14
|
-
import { SenderTrust } from "@remit/domain-enums";
|
|
15
|
+
import { SenderOverride, SenderTrust } from "@remit/domain-enums";
|
|
15
16
|
import {
|
|
16
17
|
classifyPlacement,
|
|
17
|
-
|
|
18
|
+
resolveSenderPlacement,
|
|
18
19
|
} from "./classifyPlacement.js";
|
|
19
20
|
|
|
20
21
|
const baseMessage = (overrides: Partial<MessageItem> = {}): MessageItem =>
|
|
@@ -28,7 +29,7 @@ const baseMessage = (overrides: Partial<MessageItem> = {}): MessageItem =>
|
|
|
28
29
|
}) as unknown as MessageItem;
|
|
29
30
|
|
|
30
31
|
describe("classifyPlacement", () => {
|
|
31
|
-
describe("
|
|
32
|
+
describe("SenderOverride.Blocked (RFC 039 Decision 3)", () => {
|
|
32
33
|
it("demotes an inbox message from a blocked sender, independent of DKIM/DMARC", () => {
|
|
33
34
|
const message = baseMessage({
|
|
34
35
|
providerSpam: undefined,
|
|
@@ -38,7 +39,7 @@ describe("classifyPlacement", () => {
|
|
|
38
39
|
message,
|
|
39
40
|
"inbox",
|
|
40
41
|
SenderTrust.Unknown,
|
|
41
|
-
|
|
42
|
+
SenderOverride.Blocked,
|
|
42
43
|
);
|
|
43
44
|
assert.deepEqual(verdict, {
|
|
44
45
|
action: "move-to-junk",
|
|
@@ -52,7 +53,7 @@ describe("classifyPlacement", () => {
|
|
|
52
53
|
baseMessage(),
|
|
53
54
|
"other",
|
|
54
55
|
SenderTrust.Unknown,
|
|
55
|
-
|
|
56
|
+
SenderOverride.Blocked,
|
|
56
57
|
);
|
|
57
58
|
assert.equal(verdict.action, "move-to-junk");
|
|
58
59
|
assert.equal(verdict.confidence, "confident");
|
|
@@ -63,17 +64,17 @@ describe("classifyPlacement", () => {
|
|
|
63
64
|
baseMessage(),
|
|
64
65
|
"junk",
|
|
65
66
|
SenderTrust.Unknown,
|
|
66
|
-
|
|
67
|
+
SenderOverride.Blocked,
|
|
67
68
|
);
|
|
68
69
|
assert.notEqual(verdict.action, "move-to-junk");
|
|
69
70
|
});
|
|
70
71
|
|
|
71
|
-
it("does not demote when the sender
|
|
72
|
+
it("does not demote when the sender carries no instruction, all else equal", () => {
|
|
72
73
|
const verdict = classifyPlacement(
|
|
73
74
|
baseMessage({ providerSpam: undefined, authResult: undefined }),
|
|
74
75
|
"inbox",
|
|
75
76
|
SenderTrust.Unknown,
|
|
76
|
-
|
|
77
|
+
SenderOverride.None,
|
|
77
78
|
);
|
|
78
79
|
assert.deepEqual(verdict, {
|
|
79
80
|
action: "leave",
|
|
@@ -87,7 +88,7 @@ describe("classifyPlacement", () => {
|
|
|
87
88
|
baseMessage({ movedByRemit: true }),
|
|
88
89
|
"inbox",
|
|
89
90
|
SenderTrust.Unknown,
|
|
90
|
-
|
|
91
|
+
SenderOverride.Blocked,
|
|
91
92
|
);
|
|
92
93
|
assert.deepEqual(verdict, {
|
|
93
94
|
action: "leave",
|
|
@@ -97,70 +98,221 @@ describe("classifyPlacement", () => {
|
|
|
97
98
|
});
|
|
98
99
|
});
|
|
99
100
|
|
|
100
|
-
describe("
|
|
101
|
+
describe("SenderOverride.NeverSpam (issue #605)", () => {
|
|
102
|
+
it("rescues mail the provider rated clean from a domain that publishes no DMARC", () => {
|
|
103
|
+
const verdict = classifyPlacement(
|
|
104
|
+
baseMessage({
|
|
105
|
+
providerSpam: { classified: false },
|
|
106
|
+
authResult: { dmarc: "None" },
|
|
107
|
+
}),
|
|
108
|
+
"junk",
|
|
109
|
+
SenderTrust.Unknown,
|
|
110
|
+
SenderOverride.NeverSpam,
|
|
111
|
+
);
|
|
112
|
+
assert.deepEqual(verdict, {
|
|
113
|
+
action: "move-to-inbox",
|
|
114
|
+
confidence: "confident",
|
|
115
|
+
reasons: ["sender=never-spam"],
|
|
116
|
+
});
|
|
117
|
+
});
|
|
118
|
+
|
|
119
|
+
it("rescues before the missing-signals return, so a message with no parsed signals still comes out of junk", () => {
|
|
120
|
+
const verdict = classifyPlacement(
|
|
121
|
+
baseMessage({ providerSpam: undefined, authResult: undefined }),
|
|
122
|
+
"junk",
|
|
123
|
+
SenderTrust.Unknown,
|
|
124
|
+
SenderOverride.NeverSpam,
|
|
125
|
+
);
|
|
126
|
+
assert.equal(verdict.action, "move-to-inbox");
|
|
127
|
+
assert.equal(verdict.confidence, "confident");
|
|
128
|
+
});
|
|
129
|
+
|
|
130
|
+
it("leaves a never-spam sender's mail filed in a user folder where it is", () => {
|
|
131
|
+
const verdict = classifyPlacement(
|
|
132
|
+
baseMessage(),
|
|
133
|
+
"other",
|
|
134
|
+
SenderTrust.Unknown,
|
|
135
|
+
SenderOverride.NeverSpam,
|
|
136
|
+
);
|
|
137
|
+
assert.equal(verdict.action, "leave");
|
|
138
|
+
});
|
|
139
|
+
|
|
140
|
+
// The demote branch is reached only from `inbox`, so a spoof already in
|
|
141
|
+
// Junk never meets it. Without the guard on the rescue, the grant would
|
|
142
|
+
// carry mail impersonating the granted address out of Junk.
|
|
143
|
+
it("refuses to rescue a dkim-mismatch + dmarc-fail spoof sitting in junk", () => {
|
|
144
|
+
const verdict = classifyPlacement(
|
|
145
|
+
baseMessage({
|
|
146
|
+
providerSpam: { classified: true },
|
|
147
|
+
authResult: { dmarc: "Fail" },
|
|
148
|
+
authenticity: { fromDomain: "example.com", dkimMismatch: true },
|
|
149
|
+
}),
|
|
150
|
+
"junk",
|
|
151
|
+
SenderTrust.Unknown,
|
|
152
|
+
SenderOverride.NeverSpam,
|
|
153
|
+
);
|
|
154
|
+
assert.notEqual(verdict.action, "move-to-inbox");
|
|
155
|
+
});
|
|
156
|
+
|
|
157
|
+
it("refuses to rescue on a hard DMARC failure even with DKIM aligned", () => {
|
|
158
|
+
const verdict = classifyPlacement(
|
|
159
|
+
baseMessage({
|
|
160
|
+
providerSpam: { classified: true },
|
|
161
|
+
authResult: { dmarc: "Fail" },
|
|
162
|
+
}),
|
|
163
|
+
"junk",
|
|
164
|
+
SenderTrust.Unknown,
|
|
165
|
+
SenderOverride.NeverSpam,
|
|
166
|
+
);
|
|
167
|
+
assert.notEqual(verdict.action, "move-to-inbox");
|
|
168
|
+
});
|
|
169
|
+
|
|
170
|
+
it("refuses to rescue on a DKIM mismatch even where DMARC did not fail", () => {
|
|
171
|
+
const verdict = classifyPlacement(
|
|
172
|
+
baseMessage({
|
|
173
|
+
providerSpam: { classified: true },
|
|
174
|
+
authResult: { dmarc: "None" },
|
|
175
|
+
authenticity: { fromDomain: "example.com", dkimMismatch: true },
|
|
176
|
+
}),
|
|
177
|
+
"junk",
|
|
178
|
+
SenderTrust.Unknown,
|
|
179
|
+
SenderOverride.NeverSpam,
|
|
180
|
+
);
|
|
181
|
+
assert.notEqual(verdict.action, "move-to-inbox");
|
|
182
|
+
});
|
|
183
|
+
|
|
184
|
+
it("still demotes a never-spam sender's inbox mail on dkim-mismatch + dmarc-fail", () => {
|
|
185
|
+
const verdict = classifyPlacement(
|
|
186
|
+
baseMessage({
|
|
187
|
+
authResult: { dmarc: "Fail" },
|
|
188
|
+
authenticity: { fromDomain: "example.com", dkimMismatch: true },
|
|
189
|
+
}),
|
|
190
|
+
"inbox",
|
|
191
|
+
SenderTrust.Unknown,
|
|
192
|
+
SenderOverride.NeverSpam,
|
|
193
|
+
);
|
|
194
|
+
assert.deepEqual(verdict, {
|
|
195
|
+
action: "move-to-junk",
|
|
196
|
+
confidence: "confident",
|
|
197
|
+
reasons: ["dkim-mismatch", "dmarc=fail", "sender=untrusted"],
|
|
198
|
+
});
|
|
199
|
+
});
|
|
200
|
+
|
|
201
|
+
it("still leaves an already-Remit-moved message alone", () => {
|
|
202
|
+
const verdict = classifyPlacement(
|
|
203
|
+
baseMessage({ movedByRemit: true }),
|
|
204
|
+
"junk",
|
|
205
|
+
SenderTrust.Unknown,
|
|
206
|
+
SenderOverride.NeverSpam,
|
|
207
|
+
);
|
|
208
|
+
assert.deepEqual(verdict, {
|
|
209
|
+
action: "leave",
|
|
210
|
+
confidence: "confident",
|
|
211
|
+
reasons: ["already-moved-by-remit"],
|
|
212
|
+
});
|
|
213
|
+
});
|
|
214
|
+
});
|
|
215
|
+
|
|
216
|
+
describe("resolveSenderPlacement (Decision 3a tie-break)", () => {
|
|
101
217
|
it("blocked wins when set after vip", () => {
|
|
102
|
-
const result =
|
|
218
|
+
const result = resolveSenderPlacement(
|
|
103
219
|
{ trust: SenderTrust.Vip, setAt: 1_000 },
|
|
104
220
|
{ blocked: true, setAt: 5_000 },
|
|
221
|
+
false,
|
|
105
222
|
);
|
|
106
223
|
assert.deepEqual(result, {
|
|
107
224
|
senderTrust: SenderTrust.Unknown,
|
|
108
|
-
|
|
225
|
+
senderOverride: SenderOverride.Blocked,
|
|
109
226
|
});
|
|
110
227
|
});
|
|
111
228
|
|
|
112
229
|
it("vip wins when set after blocked", () => {
|
|
113
|
-
const result =
|
|
230
|
+
const result = resolveSenderPlacement(
|
|
114
231
|
{ trust: SenderTrust.Vip, setAt: 5_000 },
|
|
115
232
|
{ blocked: true, setAt: 1_000 },
|
|
233
|
+
false,
|
|
116
234
|
);
|
|
117
235
|
assert.deepEqual(result, {
|
|
118
236
|
senderTrust: SenderTrust.Vip,
|
|
119
|
-
|
|
237
|
+
senderOverride: SenderOverride.None,
|
|
120
238
|
});
|
|
121
239
|
});
|
|
122
240
|
|
|
123
241
|
it("breaks a same-second tie in favor of blocked", () => {
|
|
124
|
-
const result =
|
|
242
|
+
const result = resolveSenderPlacement(
|
|
125
243
|
{ trust: SenderTrust.Wellknown, setAt: 1_000 },
|
|
126
244
|
{ blocked: true, setAt: 1_400 },
|
|
245
|
+
false,
|
|
127
246
|
);
|
|
128
247
|
assert.deepEqual(result, {
|
|
129
248
|
senderTrust: SenderTrust.Unknown,
|
|
130
|
-
|
|
249
|
+
senderOverride: SenderOverride.Blocked,
|
|
131
250
|
});
|
|
132
251
|
});
|
|
133
252
|
|
|
134
253
|
it("blocked applies outright when there is no competing trust flag", () => {
|
|
135
|
-
const result =
|
|
254
|
+
const result = resolveSenderPlacement(
|
|
136
255
|
{ trust: SenderTrust.Unknown },
|
|
137
256
|
{ blocked: true, setAt: 1_000 },
|
|
257
|
+
false,
|
|
138
258
|
);
|
|
139
259
|
assert.deepEqual(result, {
|
|
140
260
|
senderTrust: SenderTrust.Unknown,
|
|
141
|
-
|
|
261
|
+
senderOverride: SenderOverride.Blocked,
|
|
142
262
|
});
|
|
143
263
|
});
|
|
144
264
|
|
|
145
|
-
it("passes trust through unchanged when the sender
|
|
146
|
-
const result =
|
|
265
|
+
it("passes trust through unchanged when the sender carries no instruction", () => {
|
|
266
|
+
const result = resolveSenderPlacement(
|
|
147
267
|
{ trust: SenderTrust.Wellknown, setAt: 1_000 },
|
|
148
268
|
{ blocked: false },
|
|
269
|
+
false,
|
|
149
270
|
);
|
|
150
271
|
assert.deepEqual(result, {
|
|
151
272
|
senderTrust: SenderTrust.Wellknown,
|
|
152
|
-
|
|
273
|
+
senderOverride: SenderOverride.None,
|
|
153
274
|
});
|
|
154
275
|
});
|
|
276
|
+
|
|
277
|
+
it("reads never-spam as the instruction, leaving trust untouched", () => {
|
|
278
|
+
const result = resolveSenderPlacement(
|
|
279
|
+
{ trust: SenderTrust.Unknown },
|
|
280
|
+
{ blocked: false },
|
|
281
|
+
true,
|
|
282
|
+
);
|
|
283
|
+
assert.deepEqual(result, {
|
|
284
|
+
senderTrust: SenderTrust.Unknown,
|
|
285
|
+
senderOverride: SenderOverride.NeverSpam,
|
|
286
|
+
});
|
|
287
|
+
});
|
|
288
|
+
|
|
289
|
+
it("does not promote a never-spam sender's trust to vip or wellknown", () => {
|
|
290
|
+
const result = resolveSenderPlacement(
|
|
291
|
+
{ trust: SenderTrust.Vip, setAt: 1_000 },
|
|
292
|
+
{ blocked: false },
|
|
293
|
+
true,
|
|
294
|
+
);
|
|
295
|
+
assert.equal(result.senderTrust, SenderTrust.Vip);
|
|
296
|
+
assert.equal(result.senderOverride, SenderOverride.NeverSpam);
|
|
297
|
+
});
|
|
298
|
+
|
|
299
|
+
it("resolves a row that somehow carries both instructions to blocked", () => {
|
|
300
|
+
const result = resolveSenderPlacement(
|
|
301
|
+
{ trust: SenderTrust.Unknown },
|
|
302
|
+
{ blocked: true, setAt: 1_000 },
|
|
303
|
+
true,
|
|
304
|
+
);
|
|
305
|
+
assert.equal(result.senderOverride, SenderOverride.Blocked);
|
|
306
|
+
});
|
|
155
307
|
});
|
|
156
308
|
|
|
157
|
-
describe("existing DKIM/DMARC paths (unaffected by
|
|
309
|
+
describe("existing DKIM/DMARC paths (unaffected by SenderOverride.None)", () => {
|
|
158
310
|
it("rescues a trusted sender's mail from junk on provider-spam + dmarc-pass", () => {
|
|
159
311
|
const verdict = classifyPlacement(
|
|
160
312
|
baseMessage({ providerSpam: { classified: true } }),
|
|
161
313
|
"junk",
|
|
162
314
|
SenderTrust.Vip,
|
|
163
|
-
|
|
315
|
+
SenderOverride.None,
|
|
164
316
|
);
|
|
165
317
|
assert.equal(verdict.action, "move-to-inbox");
|
|
166
318
|
assert.equal(verdict.confidence, "confident");
|
|
@@ -171,7 +323,7 @@ describe("classifyPlacement", () => {
|
|
|
171
323
|
baseMessage({ providerSpam: { classified: true } }),
|
|
172
324
|
"junk",
|
|
173
325
|
SenderTrust.Unknown,
|
|
174
|
-
|
|
326
|
+
SenderOverride.None,
|
|
175
327
|
);
|
|
176
328
|
assert.equal(verdict.action, "leave");
|
|
177
329
|
assert.equal(verdict.confidence, "unsure");
|
|
@@ -185,7 +337,7 @@ describe("classifyPlacement", () => {
|
|
|
185
337
|
}),
|
|
186
338
|
"inbox",
|
|
187
339
|
SenderTrust.Unknown,
|
|
188
|
-
|
|
340
|
+
SenderOverride.None,
|
|
189
341
|
);
|
|
190
342
|
assert.deepEqual(verdict, {
|
|
191
343
|
action: "move-to-junk",
|
|
@@ -202,7 +354,7 @@ describe("classifyPlacement", () => {
|
|
|
202
354
|
}),
|
|
203
355
|
"inbox",
|
|
204
356
|
SenderTrust.Unknown,
|
|
205
|
-
|
|
357
|
+
SenderOverride.None,
|
|
206
358
|
);
|
|
207
359
|
assert.deepEqual(verdict, {
|
|
208
360
|
action: "leave",
|
|
@@ -1,8 +1,10 @@
|
|
|
1
1
|
import type { MessageItem } from "@remit/data-ports";
|
|
2
|
-
import { SenderTrust } from "@remit/domain-enums";
|
|
2
|
+
import { SenderOverride, SenderTrust } from "@remit/domain-enums";
|
|
3
3
|
|
|
4
4
|
type SenderTrustValue = (typeof SenderTrust)[keyof typeof SenderTrust];
|
|
5
5
|
|
|
6
|
+
type SenderOverrideValue = (typeof SenderOverride)[keyof typeof SenderOverride];
|
|
7
|
+
|
|
6
8
|
export type FolderPlacement = "inbox" | "junk" | "other";
|
|
7
9
|
|
|
8
10
|
export type PlacementAction = "move-to-inbox" | "move-to-junk" | "leave";
|
|
@@ -39,7 +41,7 @@ export interface SenderBlockedSignal {
|
|
|
39
41
|
* fixed, arbitrary order: `blocked` before `vip` before `wellknown` — a
|
|
40
42
|
* determinism backstop, not a meaningful signal.
|
|
41
43
|
*/
|
|
42
|
-
|
|
44
|
+
const resolveBlockedVsTrust = (
|
|
43
45
|
trust: SenderTrustSignal,
|
|
44
46
|
blocked: SenderBlockedSignal,
|
|
45
47
|
): { senderTrust: SenderTrustValue; senderBlocked: boolean } => {
|
|
@@ -60,6 +62,32 @@ export const resolveBlockedVsTrust = (
|
|
|
60
62
|
return { senderTrust: trust.trust, senderBlocked: false };
|
|
61
63
|
};
|
|
62
64
|
|
|
65
|
+
/**
|
|
66
|
+
* Resolve the two axes {@link classifyPlacement} takes: the trust signal, and
|
|
67
|
+
* the user's standing instruction about placement.
|
|
68
|
+
*
|
|
69
|
+
* `blocked` and `neverSpam` are two directly contradictory instructions and
|
|
70
|
+
* cannot both stand — `AddressRepository.mergeFlags` drops the loser at write
|
|
71
|
+
* time. Reading a row that carries both anyway (written before that invariant)
|
|
72
|
+
* resolves to `Blocked`, the same direction the `blocked`/`vip` same-second tie
|
|
73
|
+
* already breaks in: the demote is the recoverable mistake.
|
|
74
|
+
*/
|
|
75
|
+
export const resolveSenderPlacement = (
|
|
76
|
+
trust: SenderTrustSignal,
|
|
77
|
+
blocked: SenderBlockedSignal,
|
|
78
|
+
neverSpam: boolean,
|
|
79
|
+
): {
|
|
80
|
+
senderTrust: SenderTrustValue;
|
|
81
|
+
senderOverride: SenderOverrideValue;
|
|
82
|
+
} => {
|
|
83
|
+
const { senderTrust, senderBlocked } = resolveBlockedVsTrust(trust, blocked);
|
|
84
|
+
if (senderBlocked)
|
|
85
|
+
return { senderTrust, senderOverride: SenderOverride.Blocked };
|
|
86
|
+
if (neverSpam)
|
|
87
|
+
return { senderTrust, senderOverride: SenderOverride.NeverSpam };
|
|
88
|
+
return { senderTrust, senderOverride: SenderOverride.None };
|
|
89
|
+
};
|
|
90
|
+
|
|
63
91
|
/**
|
|
64
92
|
* Tier 0 deterministic placement verdict (RFC 031, "Confident moves").
|
|
65
93
|
*
|
|
@@ -69,17 +97,32 @@ export const resolveBlockedVsTrust = (
|
|
|
69
97
|
* recall-biased — a confident move only fires when cheap, deterministic signals
|
|
70
98
|
* agree; everything else is left in place for a later LLM tier.
|
|
71
99
|
*
|
|
72
|
-
* `
|
|
73
|
-
*
|
|
74
|
-
*
|
|
75
|
-
*
|
|
100
|
+
* `senderOverride` (RFC 039 Decision 3, issue #605) is the user's standing
|
|
101
|
+
* instruction about this sender's placement, and it outranks every
|
|
102
|
+
* DKIM/DMARC/provider signal below — an explicit block or never-spam grant is
|
|
103
|
+
* not a heuristic. Already-tie-broken against `vip`/`wellknown` by
|
|
104
|
+
* {@link resolveSenderPlacement} in the caller; this function itself does not
|
|
76
105
|
* compare `setAt`.
|
|
106
|
+
*
|
|
107
|
+
* `NeverSpam` is checked above the `missing-signals` return and above the
|
|
108
|
+
* `providerSpam.classified` gate on purpose: the senders who need the grant are
|
|
109
|
+
* the ones the provider rated clean and whose domain publishes no DMARC record,
|
|
110
|
+
* so a check inside the rescue branch below would never fire for them. It
|
|
111
|
+
* rescues only from `junk` — a message a standing filter put in a user folder
|
|
112
|
+
* is not a spam question.
|
|
113
|
+
*
|
|
114
|
+
* It refuses outright on a `dmarc === "Fail"` or a DKIM mismatch. The demote
|
|
115
|
+
* branch below is reached only from `inbox`, so a spoof already sitting in Junk
|
|
116
|
+
* would never meet it: without this guard the grant would launder mail out of
|
|
117
|
+
* Junk on exactly the two signals that identify someone impersonating the
|
|
118
|
+
* address the user granted. A domain that publishes no DMARC record never
|
|
119
|
+
* scores `Fail`, so the case the grant exists for is untouched.
|
|
77
120
|
*/
|
|
78
121
|
export const classifyPlacement = (
|
|
79
122
|
message: MessageItem,
|
|
80
123
|
placement: FolderPlacement,
|
|
81
124
|
senderTrust: SenderTrustValue,
|
|
82
|
-
|
|
125
|
+
senderOverride: SenderOverrideValue,
|
|
83
126
|
): PlacementVerdict => {
|
|
84
127
|
if (message.movedByRemit === true) {
|
|
85
128
|
return {
|
|
@@ -89,7 +132,7 @@ export const classifyPlacement = (
|
|
|
89
132
|
};
|
|
90
133
|
}
|
|
91
134
|
|
|
92
|
-
if (
|
|
135
|
+
if (senderOverride === SenderOverride.Blocked && placement !== "junk") {
|
|
93
136
|
return {
|
|
94
137
|
action: "move-to-junk",
|
|
95
138
|
confidence: "confident",
|
|
@@ -97,6 +140,19 @@ export const classifyPlacement = (
|
|
|
97
140
|
};
|
|
98
141
|
}
|
|
99
142
|
|
|
143
|
+
if (
|
|
144
|
+
senderOverride === SenderOverride.NeverSpam &&
|
|
145
|
+
placement === "junk" &&
|
|
146
|
+
message.authResult?.dmarc !== "Fail" &&
|
|
147
|
+
message.authenticity?.dkimMismatch !== true
|
|
148
|
+
) {
|
|
149
|
+
return {
|
|
150
|
+
action: "move-to-inbox",
|
|
151
|
+
confidence: "confident",
|
|
152
|
+
reasons: ["sender=never-spam"],
|
|
153
|
+
};
|
|
154
|
+
}
|
|
155
|
+
|
|
100
156
|
if (!message.providerSpam || !message.authResult) {
|
|
101
157
|
return {
|
|
102
158
|
action: "leave",
|
package/src/index.ts
CHANGED
|
@@ -193,6 +193,12 @@ export {
|
|
|
193
193
|
MailboxSyncService,
|
|
194
194
|
type SyncAccountInfo,
|
|
195
195
|
} from "./mailbox-sync.js";
|
|
196
|
+
export {
|
|
197
|
+
denormalizeMessageCategory,
|
|
198
|
+
type MessageCategoryDenormalizeDeps,
|
|
199
|
+
type MessageCategoryDenormalizeInput,
|
|
200
|
+
type MessageCategoryValue,
|
|
201
|
+
} from "./message-category.js";
|
|
196
202
|
export {
|
|
197
203
|
type DeleteMessagesOutcome,
|
|
198
204
|
type DeleteOptions,
|
|
@@ -0,0 +1,100 @@
|
|
|
1
|
+
import type {
|
|
2
|
+
IThreadMessageRepository,
|
|
3
|
+
ThreadMessageItem,
|
|
4
|
+
} from "@remit/data-ports";
|
|
5
|
+
import { NotFoundError } from "@remit/data-ports/errors";
|
|
6
|
+
|
|
7
|
+
export type MessageCategoryValue = ThreadMessageItem["category"];
|
|
8
|
+
|
|
9
|
+
export interface MessageCategoryDenormalizeDeps {
|
|
10
|
+
threadMessageService: Pick<
|
|
11
|
+
IThreadMessageRepository,
|
|
12
|
+
"findAllByMessageId" | "update"
|
|
13
|
+
>;
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
export interface MessageCategoryDenormalizeInput {
|
|
17
|
+
category: MessageCategoryValue;
|
|
18
|
+
snippet?: string;
|
|
19
|
+
listId?: string;
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
/**
|
|
23
|
+
* A row is skipped only when every field the denormalization would write
|
|
24
|
+
* already matches. `snippet` and `listId` are absent from the update when the
|
|
25
|
+
* message has neither, and an absent field is not a mismatch.
|
|
26
|
+
*/
|
|
27
|
+
const alreadyDenormalized = (
|
|
28
|
+
row: ThreadMessageItem,
|
|
29
|
+
update: MessageCategoryDenormalizeInput,
|
|
30
|
+
): boolean =>
|
|
31
|
+
row.category === update.category &&
|
|
32
|
+
(update.snippet === undefined || row.snippet === update.snippet) &&
|
|
33
|
+
(update.listId === undefined || row.listId === update.listId);
|
|
34
|
+
|
|
35
|
+
/**
|
|
36
|
+
* Write a message's category (and, when the caller has them, its snippet and
|
|
37
|
+
* `List-Id`) onto every ThreadMessage row that indexes it. The thread row is
|
|
38
|
+
* what the message list renders and what the category filter matches, so it
|
|
39
|
+
* must never disagree with the Message row it mirrors.
|
|
40
|
+
*
|
|
41
|
+
* More than one row per messageId is schema-legal but not normally produced,
|
|
42
|
+
* and this iterates for the same reason `message-move.ts` does (see the model
|
|
43
|
+
* stated at its `deleteThreadMessagesForMessage`): the key permits it and
|
|
44
|
+
* nothing enforces otherwise. It is NOT the second mailbox a message appears
|
|
45
|
+
* in — `deriveMessageId` and `deriveThreadMessageId` are both
|
|
46
|
+
* mailbox-independent, so INBOX and Archive resolve to one row, and a copy
|
|
47
|
+
* gets its own messageId. The reachable case is thread-root drift: the same
|
|
48
|
+
* message re-saved under different `References`, which mints a second
|
|
49
|
+
* threadId and so a second row. Iterating is therefore hardening against a
|
|
50
|
+
* legal state, not a repair for one the sync path manufactures, which is why
|
|
51
|
+
* the tree's other single-row `messageId` lookups are correct as they stand.
|
|
52
|
+
* `flag-queue.ts` iterates the same list.
|
|
53
|
+
*
|
|
54
|
+
* Rows are looked up by messageId, so this does not depend on the RFC822
|
|
55
|
+
* Message-ID header — a headerless message still gets denormalized, matching
|
|
56
|
+
* the unconditional Message.category write. The composite set is built per
|
|
57
|
+
* row, never reused: `mailboxId` and `isRead` can differ between two rows for
|
|
58
|
+
* one message, and it is passed at all so that a future key-attribute
|
|
59
|
+
* addition touching the lsi3/lsi4/lsi5/gsi2 sort keys keeps the index rows
|
|
60
|
+
* consistent.
|
|
61
|
+
*/
|
|
62
|
+
export const denormalizeMessageCategory = async (
|
|
63
|
+
deps: MessageCategoryDenormalizeDeps,
|
|
64
|
+
accountConfigId: string,
|
|
65
|
+
messageId: string,
|
|
66
|
+
update: MessageCategoryDenormalizeInput,
|
|
67
|
+
): Promise<void> => {
|
|
68
|
+
const rows = await deps.threadMessageService.findAllByMessageId(
|
|
69
|
+
accountConfigId,
|
|
70
|
+
messageId,
|
|
71
|
+
);
|
|
72
|
+
if (rows.length === 0) {
|
|
73
|
+
throw new NotFoundError(`ThreadMessage not found for message ${messageId}`);
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
const written = {
|
|
77
|
+
category: update.category,
|
|
78
|
+
...(update.snippet ? { snippet: update.snippet } : {}),
|
|
79
|
+
...(update.listId ? { listId: update.listId } : {}),
|
|
80
|
+
};
|
|
81
|
+
|
|
82
|
+
for (const row of rows) {
|
|
83
|
+
if (alreadyDenormalized(row, written)) continue;
|
|
84
|
+
await deps.threadMessageService.update(
|
|
85
|
+
accountConfigId,
|
|
86
|
+
row.threadMessageId,
|
|
87
|
+
written,
|
|
88
|
+
{
|
|
89
|
+
composites: {
|
|
90
|
+
sentDate: row.sentDate,
|
|
91
|
+
mailboxId: row.mailboxId,
|
|
92
|
+
isRead: row.isRead,
|
|
93
|
+
isDeleted: row.isDeleted,
|
|
94
|
+
hasStars: row.hasStars,
|
|
95
|
+
hasAttachment: row.hasAttachment,
|
|
96
|
+
},
|
|
97
|
+
},
|
|
98
|
+
);
|
|
99
|
+
}
|
|
100
|
+
};
|