@remit/mailbox-service 0.0.34 → 0.0.35
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/src/body-sync-backfill.test.ts +6 -1
- package/src/body-sync-category-denormalize.test.ts +7 -1
- package/src/body-sync-category-override.test.ts +460 -0
- package/src/body-sync-category-writeonce.test.ts +7 -1
- package/src/body-sync-filter.test.ts +4 -0
- package/src/body-sync.ts +98 -66
package/package.json
CHANGED
|
@@ -20,6 +20,7 @@ import type {
|
|
|
20
20
|
UpdateMessageInput,
|
|
21
21
|
UpdateThreadMessageInput,
|
|
22
22
|
} from "@remit/data-ports";
|
|
23
|
+
import { NotFoundError } from "@remit/data-ports/errors";
|
|
23
24
|
import { MessageCategory } from "@remit/domain-enums";
|
|
24
25
|
import type { StorageService } from "@remit/storage-service";
|
|
25
26
|
import { BodySyncService } from "./body-sync.js";
|
|
@@ -103,7 +104,11 @@ const buildHarness = (
|
|
|
103
104
|
messageService,
|
|
104
105
|
storageService,
|
|
105
106
|
threadMessageService,
|
|
106
|
-
{
|
|
107
|
+
{
|
|
108
|
+
getAddress: async () => {
|
|
109
|
+
throw new NotFoundError("Address not found");
|
|
110
|
+
},
|
|
111
|
+
} as unknown as IAddressRepository,
|
|
107
112
|
{} as unknown as IEnvelopeRepository,
|
|
108
113
|
{
|
|
109
114
|
info: () => {},
|
|
@@ -29,6 +29,7 @@ import type {
|
|
|
29
29
|
UpdateMessageInput,
|
|
30
30
|
UpdateThreadMessageInput,
|
|
31
31
|
} from "@remit/data-ports";
|
|
32
|
+
import { NotFoundError } from "@remit/data-ports/errors";
|
|
32
33
|
import { MessageCategory } from "@remit/domain-enums";
|
|
33
34
|
import type { StorageService } from "@remit/storage-service";
|
|
34
35
|
import { BodySyncService } from "./body-sync.js";
|
|
@@ -171,7 +172,12 @@ const buildHarness = (
|
|
|
171
172
|
messageService,
|
|
172
173
|
storageService,
|
|
173
174
|
threadMessageService,
|
|
174
|
-
{
|
|
175
|
+
{
|
|
176
|
+
getAddress: async () => {
|
|
177
|
+
throw new NotFoundError("Address not found");
|
|
178
|
+
},
|
|
179
|
+
incrementInboundCount: async () => {},
|
|
180
|
+
} as unknown as IAddressRepository,
|
|
175
181
|
{ listBodyParts: async () => [] } as unknown as IEnvelopeRepository,
|
|
176
182
|
{ info: () => {}, error: () => {} },
|
|
177
183
|
);
|
|
@@ -0,0 +1,460 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `Address.flags.category` overrides classification at sync time (issue #299,
|
|
3
|
+
* RFC 039 Decision 3, closing item 3 of RFC 039). A user who has told Remit
|
|
4
|
+
* "this sender is actually Personal, not Marketing" via the Reclassify dialog
|
|
5
|
+
* (`PATCH /addresses/{id}`) gets that category on the sender's next message,
|
|
6
|
+
* instead of `classifyByHeaders` re-deriving the same header-driven verdict
|
|
7
|
+
* every time.
|
|
8
|
+
*
|
|
9
|
+
* The override sits behind the same `hasDecidedCategory` write-once gate #378
|
|
10
|
+
* added for `Message.category` (RFC 034 Decision 3.1, RFC 030's message-list
|
|
11
|
+
* GSI sort key depends on it never churning): a message already carrying a
|
|
12
|
+
* real category is never re-touched, override present or not. The two tests
|
|
13
|
+
* under "survives a re-entrant pass" feed a *differing* override to an
|
|
14
|
+
* already-classified message through both shipped re-entry paths — the
|
|
15
|
+
* `NoSuchKey` IMAP re-fetch and `syncBodies(..., force: true)` — so a
|
|
16
|
+
* regression that let the override bypass the guard shows up here, not just
|
|
17
|
+
* in an isolated unit check.
|
|
18
|
+
*/
|
|
19
|
+
|
|
20
|
+
import assert from "node:assert/strict";
|
|
21
|
+
import { Readable } from "node:stream";
|
|
22
|
+
import { describe, it } from "node:test";
|
|
23
|
+
import type {
|
|
24
|
+
AddressItem,
|
|
25
|
+
IAddressRepository,
|
|
26
|
+
IEnvelopeRepository,
|
|
27
|
+
IMessageRepository,
|
|
28
|
+
IThreadMessageRepository,
|
|
29
|
+
MessageItem,
|
|
30
|
+
ThreadMessageItem,
|
|
31
|
+
UpdateMessageInput,
|
|
32
|
+
UpdateThreadMessageInput,
|
|
33
|
+
} from "@remit/data-ports";
|
|
34
|
+
import { NotFoundError } from "@remit/data-ports/errors";
|
|
35
|
+
import { MessageCategory } from "@remit/domain-enums";
|
|
36
|
+
import type { StorageService } from "@remit/storage-service";
|
|
37
|
+
import { BodySyncService } from "./body-sync.js";
|
|
38
|
+
import type { IImapConnection } from "./types.js";
|
|
39
|
+
|
|
40
|
+
const LINKEDIN_EML = Buffer.from(
|
|
41
|
+
[
|
|
42
|
+
"From: LinkedIn <messages-noreply@linkedin.com>",
|
|
43
|
+
"To: me@example.com",
|
|
44
|
+
"Subject: You have a new invitation",
|
|
45
|
+
"List-Unsubscribe: <https://www.linkedin.com/e/unsub>",
|
|
46
|
+
"Content-Type: text/plain",
|
|
47
|
+
"",
|
|
48
|
+
"invitation",
|
|
49
|
+
].join("\r\n"),
|
|
50
|
+
);
|
|
51
|
+
|
|
52
|
+
interface ThreadUpdate {
|
|
53
|
+
threadMessageId: string;
|
|
54
|
+
input: UpdateThreadMessageInput;
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
interface Harness {
|
|
58
|
+
service: BodySyncService;
|
|
59
|
+
message: MessageItem;
|
|
60
|
+
rows: ThreadMessageItem[];
|
|
61
|
+
messageUpdates: Array<{ messageId: string; input: UpdateMessageInput }>;
|
|
62
|
+
threadUpdates: ThreadUpdate[];
|
|
63
|
+
getAddressCalls: number;
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
const buildHarness = (
|
|
67
|
+
message: Partial<MessageItem> & Pick<MessageItem, "messageId">,
|
|
68
|
+
flags: AddressItem["flags"] | undefined,
|
|
69
|
+
retrieve: () => Promise<Buffer> = async () => {
|
|
70
|
+
throw new Error("no body configured for retrieve()");
|
|
71
|
+
},
|
|
72
|
+
): Harness => {
|
|
73
|
+
const messageUpdates: Array<{
|
|
74
|
+
messageId: string;
|
|
75
|
+
input: UpdateMessageInput;
|
|
76
|
+
}> = [];
|
|
77
|
+
const threadUpdates: ThreadUpdate[] = [];
|
|
78
|
+
let getAddressCalls = 0;
|
|
79
|
+
|
|
80
|
+
const messageRow = {
|
|
81
|
+
uid: 1,
|
|
82
|
+
mailboxId: "mb-1",
|
|
83
|
+
category: MessageCategory.uncategorized,
|
|
84
|
+
...message,
|
|
85
|
+
} as unknown as MessageItem;
|
|
86
|
+
|
|
87
|
+
const rows: ThreadMessageItem[] = [
|
|
88
|
+
{
|
|
89
|
+
threadMessageId: "tm-1",
|
|
90
|
+
messageId: message.messageId,
|
|
91
|
+
mailboxId: "mb-1",
|
|
92
|
+
sentDate: 1,
|
|
93
|
+
isRead: false,
|
|
94
|
+
isDeleted: false,
|
|
95
|
+
hasStars: false,
|
|
96
|
+
hasAttachment: false,
|
|
97
|
+
category: messageRow.category,
|
|
98
|
+
} as unknown as ThreadMessageItem,
|
|
99
|
+
];
|
|
100
|
+
|
|
101
|
+
const messageService = {
|
|
102
|
+
get: async () => messageRow,
|
|
103
|
+
update: async (messageId: string, input: UpdateMessageInput) => {
|
|
104
|
+
messageUpdates.push({ messageId, input });
|
|
105
|
+
Object.assign(messageRow, input);
|
|
106
|
+
},
|
|
107
|
+
} as unknown as IMessageRepository;
|
|
108
|
+
|
|
109
|
+
const threadMessageService = {
|
|
110
|
+
findAllByMessageId: async () => rows,
|
|
111
|
+
update: async (
|
|
112
|
+
_accountConfigId: string,
|
|
113
|
+
threadMessageId: string,
|
|
114
|
+
input: UpdateThreadMessageInput,
|
|
115
|
+
) => {
|
|
116
|
+
threadUpdates.push({ threadMessageId, input });
|
|
117
|
+
const row = rows.find((r) => r.threadMessageId === threadMessageId);
|
|
118
|
+
if (row) Object.assign(row, input);
|
|
119
|
+
},
|
|
120
|
+
} as unknown as IThreadMessageRepository;
|
|
121
|
+
|
|
122
|
+
const storageService = {
|
|
123
|
+
retrieve,
|
|
124
|
+
storeMessageBody: async () => ({ uri: `s3://bodies/${message.messageId}` }),
|
|
125
|
+
storeMessageBodyStream: async () => ({
|
|
126
|
+
uri: `s3://bodies/${message.messageId}`,
|
|
127
|
+
}),
|
|
128
|
+
storeParsedBody: async () => {},
|
|
129
|
+
listBodyParts: async () => [],
|
|
130
|
+
} as unknown as StorageService;
|
|
131
|
+
|
|
132
|
+
const addressService = {
|
|
133
|
+
getAddress: async () => {
|
|
134
|
+
getAddressCalls++;
|
|
135
|
+
if (flags === undefined) {
|
|
136
|
+
throw new NotFoundError("Address not found");
|
|
137
|
+
}
|
|
138
|
+
return { flags } as unknown as AddressItem;
|
|
139
|
+
},
|
|
140
|
+
incrementInboundCount: async () => {},
|
|
141
|
+
} as unknown as IAddressRepository;
|
|
142
|
+
|
|
143
|
+
const envelopeService = {
|
|
144
|
+
listBodyParts: async () => [],
|
|
145
|
+
} as unknown as IEnvelopeRepository;
|
|
146
|
+
|
|
147
|
+
const service = new BodySyncService(
|
|
148
|
+
messageService,
|
|
149
|
+
storageService,
|
|
150
|
+
threadMessageService,
|
|
151
|
+
addressService,
|
|
152
|
+
envelopeService,
|
|
153
|
+
{ info: () => {}, error: () => {} },
|
|
154
|
+
);
|
|
155
|
+
|
|
156
|
+
return {
|
|
157
|
+
service,
|
|
158
|
+
message: messageRow,
|
|
159
|
+
rows,
|
|
160
|
+
messageUpdates,
|
|
161
|
+
threadUpdates,
|
|
162
|
+
get getAddressCalls() {
|
|
163
|
+
return getAddressCalls;
|
|
164
|
+
},
|
|
165
|
+
};
|
|
166
|
+
};
|
|
167
|
+
|
|
168
|
+
const noSuchKeyError = () =>
|
|
169
|
+
Object.assign(new Error("missing"), { name: "NoSuchKey" });
|
|
170
|
+
|
|
171
|
+
const overrideFlags = (
|
|
172
|
+
category: (typeof MessageCategory)[keyof typeof MessageCategory],
|
|
173
|
+
): AddressItem["flags"] =>
|
|
174
|
+
({
|
|
175
|
+
category: { value: category, setAt: 1_000 },
|
|
176
|
+
}) as unknown as AddressItem["flags"];
|
|
177
|
+
|
|
178
|
+
describe("Address.flags.category overrides classification at sync time (issue #299)", () => {
|
|
179
|
+
it("classifies a new message by the sender's override, not the header-derived category", async () => {
|
|
180
|
+
const harness = buildHarness(
|
|
181
|
+
{ messageId: "m-1", category: MessageCategory.uncategorized },
|
|
182
|
+
overrideFlags(MessageCategory.personal),
|
|
183
|
+
);
|
|
184
|
+
|
|
185
|
+
const connection = {
|
|
186
|
+
openBox: async () => {},
|
|
187
|
+
fetchMessageBody: async () => LINKEDIN_EML,
|
|
188
|
+
} as unknown as IImapConnection;
|
|
189
|
+
|
|
190
|
+
await harness.service.fetchAndGetBody(
|
|
191
|
+
"m-1",
|
|
192
|
+
"acc-1",
|
|
193
|
+
"cfg-1",
|
|
194
|
+
"INBOX",
|
|
195
|
+
async () => connection,
|
|
196
|
+
);
|
|
197
|
+
|
|
198
|
+
assert.equal(harness.messageUpdates.length, 1);
|
|
199
|
+
assert.equal(
|
|
200
|
+
harness.messageUpdates[0].input.category,
|
|
201
|
+
MessageCategory.personal,
|
|
202
|
+
);
|
|
203
|
+
assert.equal(harness.message.category, MessageCategory.personal);
|
|
204
|
+
// No placementConfig is wired in this harness, so the override's own
|
|
205
|
+
// Address read is the only one classification makes per message.
|
|
206
|
+
assert.equal(harness.getAddressCalls, 1);
|
|
207
|
+
});
|
|
208
|
+
|
|
209
|
+
it("denormalizes the override onto the ThreadMessage, matching the Message row", async () => {
|
|
210
|
+
const harness = buildHarness(
|
|
211
|
+
{ messageId: "m-1", category: MessageCategory.uncategorized },
|
|
212
|
+
overrideFlags(MessageCategory.personal),
|
|
213
|
+
);
|
|
214
|
+
|
|
215
|
+
const connection = {
|
|
216
|
+
openBox: async () => {},
|
|
217
|
+
fetchMessageBody: async () => LINKEDIN_EML,
|
|
218
|
+
} as unknown as IImapConnection;
|
|
219
|
+
|
|
220
|
+
await harness.service.fetchAndGetBody(
|
|
221
|
+
"m-1",
|
|
222
|
+
"acc-1",
|
|
223
|
+
"cfg-1",
|
|
224
|
+
"INBOX",
|
|
225
|
+
async () => connection,
|
|
226
|
+
);
|
|
227
|
+
|
|
228
|
+
assert.equal(harness.rows[0].category, MessageCategory.personal);
|
|
229
|
+
});
|
|
230
|
+
|
|
231
|
+
it("classifies by headers as usual when the sender has no category override", async () => {
|
|
232
|
+
const harness = buildHarness(
|
|
233
|
+
{ messageId: "m-1", category: MessageCategory.uncategorized },
|
|
234
|
+
undefined,
|
|
235
|
+
);
|
|
236
|
+
|
|
237
|
+
const connection = {
|
|
238
|
+
openBox: async () => {},
|
|
239
|
+
fetchMessageBody: async () => LINKEDIN_EML,
|
|
240
|
+
} as unknown as IImapConnection;
|
|
241
|
+
|
|
242
|
+
await harness.service.fetchAndGetBody(
|
|
243
|
+
"m-1",
|
|
244
|
+
"acc-1",
|
|
245
|
+
"cfg-1",
|
|
246
|
+
"INBOX",
|
|
247
|
+
async () => connection,
|
|
248
|
+
);
|
|
249
|
+
|
|
250
|
+
assert.equal(
|
|
251
|
+
harness.messageUpdates[0].input.category,
|
|
252
|
+
MessageCategory.social,
|
|
253
|
+
);
|
|
254
|
+
});
|
|
255
|
+
|
|
256
|
+
it("classifies by headers as usual when the sender has flags but no category entry", async () => {
|
|
257
|
+
const harness = buildHarness(
|
|
258
|
+
{ messageId: "m-1", category: MessageCategory.uncategorized },
|
|
259
|
+
{ vip: { value: true, setAt: 1 } } as unknown as AddressItem["flags"],
|
|
260
|
+
);
|
|
261
|
+
|
|
262
|
+
const connection = {
|
|
263
|
+
openBox: async () => {},
|
|
264
|
+
fetchMessageBody: async () => LINKEDIN_EML,
|
|
265
|
+
} as unknown as IImapConnection;
|
|
266
|
+
|
|
267
|
+
await harness.service.fetchAndGetBody(
|
|
268
|
+
"m-1",
|
|
269
|
+
"acc-1",
|
|
270
|
+
"cfg-1",
|
|
271
|
+
"INBOX",
|
|
272
|
+
async () => connection,
|
|
273
|
+
);
|
|
274
|
+
|
|
275
|
+
assert.equal(
|
|
276
|
+
harness.messageUpdates[0].input.category,
|
|
277
|
+
MessageCategory.social,
|
|
278
|
+
);
|
|
279
|
+
});
|
|
280
|
+
|
|
281
|
+
it("backfillClassification picks up the same override for a body stored before classification existed", async () => {
|
|
282
|
+
const harness = buildHarness(
|
|
283
|
+
{
|
|
284
|
+
messageId: "m-1",
|
|
285
|
+
bodyStorageKey: "s3://bodies/m-1",
|
|
286
|
+
category: MessageCategory.uncategorized,
|
|
287
|
+
},
|
|
288
|
+
overrideFlags(MessageCategory.transactional),
|
|
289
|
+
async () => LINKEDIN_EML,
|
|
290
|
+
);
|
|
291
|
+
|
|
292
|
+
const result = await harness.service.syncBodies(
|
|
293
|
+
["m-1"],
|
|
294
|
+
"acc-1",
|
|
295
|
+
"cfg-1",
|
|
296
|
+
"INBOX",
|
|
297
|
+
async () => {
|
|
298
|
+
throw new Error("backfill must not open IMAP");
|
|
299
|
+
},
|
|
300
|
+
);
|
|
301
|
+
|
|
302
|
+
assert.equal(result.skippedCount, 1);
|
|
303
|
+
assert.equal(harness.messageUpdates.length, 1);
|
|
304
|
+
assert.equal(
|
|
305
|
+
harness.messageUpdates[0].input.category,
|
|
306
|
+
MessageCategory.transactional,
|
|
307
|
+
);
|
|
308
|
+
assert.equal(harness.rows[0].category, MessageCategory.transactional);
|
|
309
|
+
});
|
|
310
|
+
|
|
311
|
+
describe("survives a re-entrant pass with a differing override already set", () => {
|
|
312
|
+
it("keeps the already-decided category through the NoSuchKey IMAP re-fetch", async () => {
|
|
313
|
+
const harness = buildHarness(
|
|
314
|
+
{
|
|
315
|
+
messageId: "m-1",
|
|
316
|
+
bodyStorageKey: "s3://bodies/m-1",
|
|
317
|
+
category: MessageCategory.marketing,
|
|
318
|
+
},
|
|
319
|
+
overrideFlags(MessageCategory.personal),
|
|
320
|
+
async () => {
|
|
321
|
+
throw noSuchKeyError();
|
|
322
|
+
},
|
|
323
|
+
);
|
|
324
|
+
|
|
325
|
+
const connection = {
|
|
326
|
+
openBox: async () => {},
|
|
327
|
+
fetchMessageBody: async () => LINKEDIN_EML,
|
|
328
|
+
} as unknown as IImapConnection;
|
|
329
|
+
|
|
330
|
+
await harness.service.fetchAndGetBody(
|
|
331
|
+
"m-1",
|
|
332
|
+
"acc-1",
|
|
333
|
+
"cfg-1",
|
|
334
|
+
"INBOX",
|
|
335
|
+
async () => connection,
|
|
336
|
+
);
|
|
337
|
+
|
|
338
|
+
assert.equal(harness.messageUpdates.length, 1);
|
|
339
|
+
assert.equal(
|
|
340
|
+
harness.messageUpdates[0].input.category,
|
|
341
|
+
MessageCategory.marketing,
|
|
342
|
+
);
|
|
343
|
+
assert.equal(harness.message.category, MessageCategory.marketing);
|
|
344
|
+
assert.equal(harness.rows[0].category, MessageCategory.marketing);
|
|
345
|
+
});
|
|
346
|
+
|
|
347
|
+
it("keeps the already-decided category when syncBodies re-fetches with force", async () => {
|
|
348
|
+
const harness = buildHarness(
|
|
349
|
+
{
|
|
350
|
+
messageId: "m-1",
|
|
351
|
+
bodyStorageKey: "s3://bodies/m-1",
|
|
352
|
+
category: MessageCategory.marketing,
|
|
353
|
+
},
|
|
354
|
+
overrideFlags(MessageCategory.personal),
|
|
355
|
+
async () => {
|
|
356
|
+
throw new Error("force path must not retrieve from storage");
|
|
357
|
+
},
|
|
358
|
+
);
|
|
359
|
+
|
|
360
|
+
const connection = {
|
|
361
|
+
openBox: async () => {},
|
|
362
|
+
async *fetchMessageBodies(uids: number[]) {
|
|
363
|
+
for (const uid of uids) {
|
|
364
|
+
yield { uid, source: Readable.from([LINKEDIN_EML]) };
|
|
365
|
+
}
|
|
366
|
+
},
|
|
367
|
+
} as unknown as IImapConnection;
|
|
368
|
+
|
|
369
|
+
const result = await harness.service.syncBodies(
|
|
370
|
+
["m-1"],
|
|
371
|
+
"acc-1",
|
|
372
|
+
"cfg-1",
|
|
373
|
+
"INBOX",
|
|
374
|
+
async () => connection,
|
|
375
|
+
true,
|
|
376
|
+
);
|
|
377
|
+
|
|
378
|
+
assert.deepEqual(result.syncedMessageIds, ["m-1"]);
|
|
379
|
+
assert.equal(harness.messageUpdates.length, 1);
|
|
380
|
+
assert.equal(
|
|
381
|
+
harness.messageUpdates[0].input.category,
|
|
382
|
+
MessageCategory.marketing,
|
|
383
|
+
);
|
|
384
|
+
assert.equal(harness.message.category, MessageCategory.marketing);
|
|
385
|
+
assert.equal(harness.rows[0].category, MessageCategory.marketing);
|
|
386
|
+
});
|
|
387
|
+
});
|
|
388
|
+
|
|
389
|
+
it("propagates a non-NotFound Address lookup failure instead of silently classifying by headers", async () => {
|
|
390
|
+
const message = {
|
|
391
|
+
messageId: "m-1",
|
|
392
|
+
category: MessageCategory.uncategorized,
|
|
393
|
+
};
|
|
394
|
+
const messageRow = { uid: 1, mailboxId: "mb-1", ...message } as MessageItem;
|
|
395
|
+
|
|
396
|
+
const messageService = {
|
|
397
|
+
get: async () => messageRow,
|
|
398
|
+
update: async () => {},
|
|
399
|
+
} as unknown as IMessageRepository;
|
|
400
|
+
|
|
401
|
+
const threadMessageService = {
|
|
402
|
+
findAllByMessageId: async () => [
|
|
403
|
+
{
|
|
404
|
+
threadMessageId: "tm-1",
|
|
405
|
+
mailboxId: "mb-1",
|
|
406
|
+
sentDate: 1,
|
|
407
|
+
isRead: false,
|
|
408
|
+
isDeleted: false,
|
|
409
|
+
hasStars: false,
|
|
410
|
+
hasAttachment: false,
|
|
411
|
+
},
|
|
412
|
+
],
|
|
413
|
+
update: async () => {},
|
|
414
|
+
} as unknown as IThreadMessageRepository;
|
|
415
|
+
|
|
416
|
+
const storageService = {
|
|
417
|
+
storeMessageBody: async () => ({ uri: "s3://bodies/m-1" }),
|
|
418
|
+
storeParsedBody: async () => {},
|
|
419
|
+
listBodyParts: async () => [],
|
|
420
|
+
} as unknown as StorageService;
|
|
421
|
+
|
|
422
|
+
const addressService = {
|
|
423
|
+
getAddress: async () => {
|
|
424
|
+
const error = new Error("ProvisionedThroughputExceededException");
|
|
425
|
+
error.name = "ProvisionedThroughputExceededException";
|
|
426
|
+
throw error;
|
|
427
|
+
},
|
|
428
|
+
incrementInboundCount: async () => {},
|
|
429
|
+
} as unknown as IAddressRepository;
|
|
430
|
+
|
|
431
|
+
const envelopeService = {
|
|
432
|
+
listBodyParts: async () => [],
|
|
433
|
+
} as unknown as IEnvelopeRepository;
|
|
434
|
+
|
|
435
|
+
const service = new BodySyncService(
|
|
436
|
+
messageService,
|
|
437
|
+
storageService,
|
|
438
|
+
threadMessageService,
|
|
439
|
+
addressService,
|
|
440
|
+
envelopeService,
|
|
441
|
+
{ info: () => {}, error: () => {} },
|
|
442
|
+
);
|
|
443
|
+
|
|
444
|
+
const connection = {
|
|
445
|
+
openBox: async () => {},
|
|
446
|
+
fetchMessageBody: async () => LINKEDIN_EML,
|
|
447
|
+
} as unknown as IImapConnection;
|
|
448
|
+
|
|
449
|
+
await assert.rejects(
|
|
450
|
+
service.fetchAndGetBody(
|
|
451
|
+
"m-1",
|
|
452
|
+
"acc-1",
|
|
453
|
+
"cfg-1",
|
|
454
|
+
"INBOX",
|
|
455
|
+
async () => connection,
|
|
456
|
+
),
|
|
457
|
+
/ProvisionedThroughputExceededException/,
|
|
458
|
+
);
|
|
459
|
+
});
|
|
460
|
+
});
|
|
@@ -24,6 +24,7 @@ import type {
|
|
|
24
24
|
MessageItem,
|
|
25
25
|
UpdateMessageInput,
|
|
26
26
|
} from "@remit/data-ports";
|
|
27
|
+
import { NotFoundError } from "@remit/data-ports/errors";
|
|
27
28
|
import { MessageCategory } from "@remit/domain-enums";
|
|
28
29
|
import type { StorageService } from "@remit/storage-service";
|
|
29
30
|
import { BodySyncService } from "./body-sync.js";
|
|
@@ -111,7 +112,12 @@ const buildHarness = (
|
|
|
111
112
|
messageService,
|
|
112
113
|
storageService,
|
|
113
114
|
threadMessageService,
|
|
114
|
-
{
|
|
115
|
+
{
|
|
116
|
+
getAddress: async () => {
|
|
117
|
+
throw new NotFoundError("Address not found");
|
|
118
|
+
},
|
|
119
|
+
incrementInboundCount: async () => {},
|
|
120
|
+
} as unknown as IAddressRepository,
|
|
115
121
|
{ listBodyParts: async () => [] } as unknown as IEnvelopeRepository,
|
|
116
122
|
{ info: () => {}, error: () => {} },
|
|
117
123
|
);
|
|
@@ -28,6 +28,7 @@ import type {
|
|
|
28
28
|
MessageItem,
|
|
29
29
|
UpdateMessageInput,
|
|
30
30
|
} from "@remit/data-ports";
|
|
31
|
+
import { NotFoundError } from "@remit/data-ports/errors";
|
|
31
32
|
import {
|
|
32
33
|
FilterClauseField,
|
|
33
34
|
FilterMatchOperator,
|
|
@@ -123,6 +124,9 @@ const buildHarness = (
|
|
|
123
124
|
} as unknown as StorageService;
|
|
124
125
|
|
|
125
126
|
const addressService = {
|
|
127
|
+
getAddress: async () => {
|
|
128
|
+
throw new NotFoundError("Address not found");
|
|
129
|
+
},
|
|
126
130
|
incrementInboundCount: async () => {},
|
|
127
131
|
} as unknown as IAddressRepository;
|
|
128
132
|
|
package/src/body-sync.ts
CHANGED
|
@@ -705,14 +705,22 @@ export class BodySyncService {
|
|
|
705
705
|
body: Buffer,
|
|
706
706
|
bodyRef: { uri: string },
|
|
707
707
|
): Promise<ParsedMail> {
|
|
708
|
-
//
|
|
709
|
-
//
|
|
710
|
-
//
|
|
711
|
-
|
|
708
|
+
// The one operation on this path that can fail because of how the message
|
|
709
|
+
// is built. Its own try block lives in `parseMessageBody`, so this frame —
|
|
710
|
+
// which also wraps the S3 body write, the parsed-body cache, the placement
|
|
711
|
+
// move, the label writes and the counter update — can tell the two apart
|
|
712
|
+
// and quarantine only this one (issue #72). The ThreadMessage write these
|
|
713
|
+
// feed happens later, alongside the Message write (see below), once the
|
|
714
|
+
// write-once category is decided.
|
|
715
|
+
const parsed = await parseMessageBody(body);
|
|
716
|
+
const snippet = extractSnippet(parsed);
|
|
717
|
+
const listId = extractListId(parsed);
|
|
712
718
|
|
|
713
|
-
// Compute the header classification once
|
|
714
|
-
//
|
|
715
|
-
|
|
719
|
+
// Compute the header classification once, folding in the sender's
|
|
720
|
+
// `flags.category` override when one applies (issue #299). The derived
|
|
721
|
+
// fields are folded into the single Message update below — they are NOT
|
|
722
|
+
// written here.
|
|
723
|
+
const classification = await this.classifyMessage(accountConfigId, parsed);
|
|
716
724
|
|
|
717
725
|
// Decide the placement move from the in-memory classification before the
|
|
718
726
|
// write, so its `movedByRemit` flag and audit verdict join the same
|
|
@@ -855,13 +863,29 @@ export class BodySyncService {
|
|
|
855
863
|
// fallback in `fetchAndGetBody` and `syncBodies(..., force: true)` — so a
|
|
856
864
|
// real, previously-decided category is carried forward unchanged instead
|
|
857
865
|
// of the just-recomputed one, the same rule `backfillClassification` uses.
|
|
866
|
+
// This also protects a `flags.category` override (issue #299): without
|
|
867
|
+
// this guard a re-entrant pass would let a *later* override silently
|
|
868
|
+
// rewrite a category already decided on an earlier message, which is
|
|
869
|
+
// exactly the churn RFC 030's GSI-safety argument forbids.
|
|
858
870
|
const existingMessage = await this.messageService.get(messageId);
|
|
871
|
+
const finalCategory = hasDecidedCategory(existingMessage.category)
|
|
872
|
+
? existingMessage.category
|
|
873
|
+
: classification.category;
|
|
874
|
+
|
|
875
|
+
// Thread-list denormalization, using the same write-once category as the
|
|
876
|
+
// Message update below — the two rows must never disagree on category.
|
|
877
|
+
await this.denormalizeCategory(
|
|
878
|
+
accountConfigId,
|
|
879
|
+
messageId,
|
|
880
|
+
finalCategory,
|
|
881
|
+
snippet,
|
|
882
|
+
listId,
|
|
883
|
+
);
|
|
884
|
+
|
|
859
885
|
const update: UpdateMessageInput = {
|
|
860
886
|
bodyStorageKey: bodyRef.uri,
|
|
861
887
|
...classification,
|
|
862
|
-
category:
|
|
863
|
-
? existingMessage.category
|
|
864
|
-
: classification.category,
|
|
888
|
+
category: finalCategory,
|
|
865
889
|
...(moved ? { movedByRemit: true } : {}),
|
|
866
890
|
...(resolved.verdict ? { placementVerdict: resolved.verdict } : {}),
|
|
867
891
|
...(filterMove ? { filterMove } : {}),
|
|
@@ -1017,7 +1041,7 @@ export class BodySyncService {
|
|
|
1017
1041
|
|
|
1018
1042
|
const body = await this.storageService.retrieve(message.bodyStorageKey);
|
|
1019
1043
|
const parsed = await parseMessageBody(body);
|
|
1020
|
-
const classification = this.classifyMessage(parsed);
|
|
1044
|
+
const classification = await this.classifyMessage(accountConfigId, parsed);
|
|
1021
1045
|
|
|
1022
1046
|
// Same order as {@link applyPostStoreSteps}, for the same reason: the
|
|
1023
1047
|
// signal the skip guard reads is written last. `message.category` is that
|
|
@@ -1025,12 +1049,13 @@ export class BodySyncService {
|
|
|
1025
1049
|
// the requeued retry redoes both. Writing the Message first strands the
|
|
1026
1050
|
// denormalized row at `uncategorized` forever — the guard is satisfied and
|
|
1027
1051
|
// the retry returns early (issue #320).
|
|
1028
|
-
// The same three denormalized fields
|
|
1029
|
-
//
|
|
1030
|
-
// `bodyStorageKey` and a decided category from its source, so it
|
|
1031
|
-
// neither that path nor this one's classification — but nothing
|
|
1032
|
-
// writes `listId`, so leaving it out here made a copy's
|
|
1033
|
-
// permanently NULL. Both are derived from the same bytes
|
|
1052
|
+
// The same three denormalized fields the full body-store path writes (see
|
|
1053
|
+
// `applyPostStoreSteps`), not just the category. A copied message
|
|
1054
|
+
// inherits `bodyStorageKey` and a decided category from its source, so it
|
|
1055
|
+
// reaches neither that path nor this one's classification — but nothing
|
|
1056
|
+
// else ever writes `listId`, so leaving it out here made a copy's
|
|
1057
|
+
// `list_id` permanently NULL. Both are derived from the same bytes
|
|
1058
|
+
// already in hand.
|
|
1034
1059
|
await this.denormalizeCategory(
|
|
1035
1060
|
accountConfigId,
|
|
1036
1061
|
message.messageId,
|
|
@@ -1047,21 +1072,39 @@ export class BodySyncService {
|
|
|
1047
1072
|
}
|
|
1048
1073
|
|
|
1049
1074
|
/**
|
|
1050
|
-
*
|
|
1051
|
-
*
|
|
1052
|
-
*
|
|
1053
|
-
*
|
|
1075
|
+
* Header classification, with the sender's `Address.flags.category`
|
|
1076
|
+
* override (issue #299, RFC 039 Decision 3) substituted for the
|
|
1077
|
+
* header-derived category when one is set. Returns the subset of the
|
|
1078
|
+
* Message update that carries the derived fields; the caller folds it into
|
|
1079
|
+
* a single UpdateItem alongside `bodyStorageKey`. Optional signals are
|
|
1080
|
+
* omitted when absent so we never overwrite an existing value with
|
|
1081
|
+
* `undefined`.
|
|
1082
|
+
*
|
|
1083
|
+
* The override wins outright rather than blending with the heuristic — RFC
|
|
1084
|
+
* 039 Decision 3 treats a direct reclassification as final, the same as
|
|
1085
|
+
* `flags.blocked`/`vip` already override placement. Both callers
|
|
1086
|
+
* (`applyPostStoreSteps`, `backfillClassification`) already gate on
|
|
1087
|
+
* `hasDecidedCategory` before this result reaches a write, so a message
|
|
1088
|
+
* that already carries a real category is never re-touched regardless of
|
|
1089
|
+
* what this returns.
|
|
1054
1090
|
*/
|
|
1055
|
-
private classifyMessage(
|
|
1091
|
+
private async classifyMessage(
|
|
1092
|
+
accountConfigId: string,
|
|
1056
1093
|
parsed: ParsedMail,
|
|
1057
|
-
): UpdateMessageInput & { category: ThreadMessageCategory } {
|
|
1058
|
-
const
|
|
1094
|
+
): Promise<UpdateMessageInput & { category: ThreadMessageCategory }> {
|
|
1095
|
+
const headerCategory = classifyByHeaders(parsed);
|
|
1059
1096
|
const authenticity = extractAuthenticity(parsed);
|
|
1060
1097
|
const authResult = extractAuthResult(parsed);
|
|
1061
1098
|
const providerSpam = extractProviderSpam(parsed);
|
|
1062
1099
|
const hasListUnsubscribe = extractHasListUnsubscribe(parsed);
|
|
1100
|
+
|
|
1101
|
+
const fromEmail = extractPrimaryFromEmail(parsed);
|
|
1102
|
+
const categoryOverride = fromEmail
|
|
1103
|
+
? await this.resolveCategoryOverride(accountConfigId, fromEmail)
|
|
1104
|
+
: undefined;
|
|
1105
|
+
|
|
1063
1106
|
return {
|
|
1064
|
-
category,
|
|
1107
|
+
category: categoryOverride ?? headerCategory,
|
|
1065
1108
|
hasListUnsubscribe,
|
|
1066
1109
|
...(authenticity !== null ? { authenticity } : {}),
|
|
1067
1110
|
...(authResult !== null ? { authResult } : {}),
|
|
@@ -1069,6 +1112,36 @@ export class BodySyncService {
|
|
|
1069
1112
|
};
|
|
1070
1113
|
}
|
|
1071
1114
|
|
|
1115
|
+
/**
|
|
1116
|
+
* The one-sided half of `Address.flags.category` (issue #299, RFC 039
|
|
1117
|
+
* Decision 3): a real value here overrides `classifyByHeaders` outright.
|
|
1118
|
+
* A second, separate `Address` read from `deriveSenderPlacementSignals`'s
|
|
1119
|
+
* (issue #300) — sharing one fetch was not practical, because the two have
|
|
1120
|
+
* different failure contracts: placement's read is best-effort (a failure
|
|
1121
|
+
* is caught and logged, never failing the sync), while this one feeds the
|
|
1122
|
+
* write-once `Message.category` at the moment it is decided, where a
|
|
1123
|
+
* masked infra failure would silently classify by headers when the user
|
|
1124
|
+
* asked for something else. Only a genuinely-absent Address is "no
|
|
1125
|
+
* override" — any other failure (throttle, infra) propagates, matching the
|
|
1126
|
+
* existing `deriveSenderPlacementSignals` convention.
|
|
1127
|
+
*/
|
|
1128
|
+
private async resolveCategoryOverride(
|
|
1129
|
+
accountConfigId: string,
|
|
1130
|
+
fromEmail: string,
|
|
1131
|
+
): Promise<ThreadMessageCategory | undefined> {
|
|
1132
|
+
try {
|
|
1133
|
+
const addressId = deriveAddressId(accountConfigId, fromEmail);
|
|
1134
|
+
const address = await this.addressService.getAddress(
|
|
1135
|
+
accountConfigId,
|
|
1136
|
+
addressId,
|
|
1137
|
+
);
|
|
1138
|
+
return address.flags?.category?.value;
|
|
1139
|
+
} catch (err) {
|
|
1140
|
+
if (!(err instanceof NotFoundError)) throw err;
|
|
1141
|
+
return undefined;
|
|
1142
|
+
}
|
|
1143
|
+
}
|
|
1144
|
+
|
|
1072
1145
|
private async incrementInboundCount(
|
|
1073
1146
|
messageId: string,
|
|
1074
1147
|
accountConfigId: string,
|
|
@@ -1682,47 +1755,6 @@ export class BodySyncService {
|
|
|
1682
1755
|
}
|
|
1683
1756
|
}
|
|
1684
1757
|
|
|
1685
|
-
/**
|
|
1686
|
-
* Extract snippet, header category and the normalized `List-Id` from the body
|
|
1687
|
-
* and denormalize them onto the ThreadMessage. `category` mirrors the Message:
|
|
1688
|
-
* created as `uncategorized` at metadata-sync and set to the classified value
|
|
1689
|
-
* here; `listId` is written so the back-apply corpus projection can match a
|
|
1690
|
-
* `ListId` clause vector-free, off the same row the list/search path reads.
|
|
1691
|
-
* Returns the parsed mail so callers can reuse it (e.g., to write the
|
|
1692
|
-
* parsed-body cache) without paying for mailparser twice.
|
|
1693
|
-
*/
|
|
1694
|
-
private async updateSnippets(
|
|
1695
|
-
messageId: string,
|
|
1696
|
-
accountConfigId: string,
|
|
1697
|
-
body: Buffer,
|
|
1698
|
-
): Promise<ParsedMail> {
|
|
1699
|
-
// The one operation on this path that can fail because of how the message
|
|
1700
|
-
// is built. Its own try block lives in `parseMessageBody`, so the caller's
|
|
1701
|
-
// frame — which also wraps the S3 body write, the parsed-body cache, the
|
|
1702
|
-
// placement move, the label writes and the counter update — can tell the
|
|
1703
|
-
// two apart and quarantine only this one (issue #72).
|
|
1704
|
-
const parsed = await parseMessageBody(body);
|
|
1705
|
-
|
|
1706
|
-
const snippet = extractSnippet(parsed);
|
|
1707
|
-
const category = classifyByHeaders(parsed);
|
|
1708
|
-
const listId = extractListId(parsed);
|
|
1709
|
-
|
|
1710
|
-
await this.denormalizeCategory(
|
|
1711
|
-
accountConfigId,
|
|
1712
|
-
messageId,
|
|
1713
|
-
category,
|
|
1714
|
-
snippet,
|
|
1715
|
-
listId,
|
|
1716
|
-
);
|
|
1717
|
-
|
|
1718
|
-
this.log.debug?.(
|
|
1719
|
-
{ messageId, category, snippetLength: snippet?.length ?? 0 },
|
|
1720
|
-
"ThreadMessage snippet + category updated",
|
|
1721
|
-
);
|
|
1722
|
-
|
|
1723
|
-
return parsed;
|
|
1724
|
-
}
|
|
1725
|
-
|
|
1726
1758
|
/**
|
|
1727
1759
|
* Write the denormalized `category` (and optionally the snippet and the
|
|
1728
1760
|
* normalized `List-Id`) onto EVERY ThreadMessage row the message has — the
|