@remit/mailbox-service 0.0.84 → 0.0.86
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +5 -1
- package/src/body-sync-category-denormalize.test.ts +41 -36
- package/src/body-sync-category-override.test.ts +54 -10
- package/src/body-sync-quarantine.test.ts +86 -35
- package/src/body-sync-unclassifiable-once.test.ts +256 -0
- package/src/body-sync.ts +43 -140
- package/src/imapflow-connect-retry.test.ts +26 -0
- package/src/imapflow-connection.ts +21 -1
- package/src/list-id-backfill.ts +8 -5
- package/src/message-move-copy.test.ts +39 -0
- package/src/message-move.ts +9 -0
- package/src/body-sync-backfill.test.ts +0 -292
|
@@ -0,0 +1,256 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* A body the classifier had nothing to say about is examined once (issue #331).
|
|
3
|
+
*
|
|
4
|
+
* `Message.category` is NOT NULL with an `uncategorized` default, so that value
|
|
5
|
+
* means two things at once: a message the classifier has not reached, and one it
|
|
6
|
+
* reached with nothing to say. Keying "needs classifying" on it made every later
|
|
7
|
+
* sync pass re-read the body from storage and rewrite both rows for the same
|
|
8
|
+
* answer — a read amplification that scales with sync frequency and never
|
|
9
|
+
* converges.
|
|
10
|
+
*
|
|
11
|
+
* `Message.classificationState` is the fact itself, written in the same
|
|
12
|
+
* UpdateItem as the answer. It is persistence-only — no response model carries
|
|
13
|
+
* it — and it is what makes the declined cohort addressable later without a
|
|
14
|
+
* sync pass guessing at it.
|
|
15
|
+
*
|
|
16
|
+
* The sender in these tests carries a `flags.category` override of
|
|
17
|
+
* `uncategorized` — the Reclassify dialog's own value for "no category" — which
|
|
18
|
+
* is how a message ends up stored and `uncategorized` at the same time.
|
|
19
|
+
*/
|
|
20
|
+
|
|
21
|
+
import assert from "node:assert/strict";
|
|
22
|
+
import { Readable } from "node:stream";
|
|
23
|
+
import { describe, it } from "node:test";
|
|
24
|
+
import type {
|
|
25
|
+
AddressItem,
|
|
26
|
+
IAddressRepository,
|
|
27
|
+
IEnvelopeRepository,
|
|
28
|
+
IMessageRepository,
|
|
29
|
+
IThreadMessageRepository,
|
|
30
|
+
MessageItem,
|
|
31
|
+
ThreadMessageItem,
|
|
32
|
+
UpdateMessageInput,
|
|
33
|
+
UpdateThreadMessageInput,
|
|
34
|
+
} from "@remit/data-ports";
|
|
35
|
+
import { NotFoundError } from "@remit/data-ports/errors";
|
|
36
|
+
import { MessageCategory } from "@remit/domain-enums";
|
|
37
|
+
import type { StorageService } from "@remit/storage-service";
|
|
38
|
+
import { BodySyncService } from "./body-sync.js";
|
|
39
|
+
import type { IImapConnection } from "./types.js";
|
|
40
|
+
|
|
41
|
+
const LINKEDIN_EML = Buffer.from(
|
|
42
|
+
[
|
|
43
|
+
"From: LinkedIn <messages-noreply@linkedin.com>",
|
|
44
|
+
"To: me@example.com",
|
|
45
|
+
"Subject: You have a new invitation",
|
|
46
|
+
"List-Unsubscribe: <https://www.linkedin.com/e/unsub>",
|
|
47
|
+
"Content-Type: text/plain",
|
|
48
|
+
"",
|
|
49
|
+
"invitation",
|
|
50
|
+
].join("\r\n"),
|
|
51
|
+
);
|
|
52
|
+
|
|
53
|
+
interface Harness {
|
|
54
|
+
service: BodySyncService;
|
|
55
|
+
message: MessageItem;
|
|
56
|
+
rows: ThreadMessageItem[];
|
|
57
|
+
messageUpdates: Array<{ messageId: string; input: UpdateMessageInput }>;
|
|
58
|
+
threadUpdates: UpdateThreadMessageInput[];
|
|
59
|
+
retrieved: string[];
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
const buildHarness = (
|
|
63
|
+
message: Partial<MessageItem> & Pick<MessageItem, "messageId">,
|
|
64
|
+
overrideCategory?: (typeof MessageCategory)[keyof typeof MessageCategory],
|
|
65
|
+
): Harness => {
|
|
66
|
+
const messageUpdates: Array<{
|
|
67
|
+
messageId: string;
|
|
68
|
+
input: UpdateMessageInput;
|
|
69
|
+
}> = [];
|
|
70
|
+
const threadUpdates: UpdateThreadMessageInput[] = [];
|
|
71
|
+
const retrieved: string[] = [];
|
|
72
|
+
|
|
73
|
+
const messageRow = {
|
|
74
|
+
uid: 1,
|
|
75
|
+
mailboxId: "mb-1",
|
|
76
|
+
category: MessageCategory.uncategorized,
|
|
77
|
+
...message,
|
|
78
|
+
} as unknown as MessageItem;
|
|
79
|
+
|
|
80
|
+
const rows: ThreadMessageItem[] = [
|
|
81
|
+
{
|
|
82
|
+
threadMessageId: "tm-1",
|
|
83
|
+
messageId: message.messageId,
|
|
84
|
+
mailboxId: "mb-1",
|
|
85
|
+
sentDate: 1,
|
|
86
|
+
isRead: false,
|
|
87
|
+
isDeleted: false,
|
|
88
|
+
hasStars: false,
|
|
89
|
+
hasAttachment: false,
|
|
90
|
+
category: messageRow.category,
|
|
91
|
+
} as unknown as ThreadMessageItem,
|
|
92
|
+
];
|
|
93
|
+
|
|
94
|
+
const messageService = {
|
|
95
|
+
get: async () => messageRow,
|
|
96
|
+
update: async (messageId: string, input: UpdateMessageInput) => {
|
|
97
|
+
messageUpdates.push({ messageId, input });
|
|
98
|
+
Object.assign(messageRow, input);
|
|
99
|
+
},
|
|
100
|
+
} as unknown as IMessageRepository;
|
|
101
|
+
|
|
102
|
+
const threadMessageService = {
|
|
103
|
+
findAllByMessageId: async () => rows,
|
|
104
|
+
update: async (
|
|
105
|
+
_accountConfigId: string,
|
|
106
|
+
threadMessageId: string,
|
|
107
|
+
input: UpdateThreadMessageInput,
|
|
108
|
+
) => {
|
|
109
|
+
threadUpdates.push(input);
|
|
110
|
+
const row = rows.find((r) => r.threadMessageId === threadMessageId);
|
|
111
|
+
if (row) Object.assign(row, input);
|
|
112
|
+
},
|
|
113
|
+
} as unknown as IThreadMessageRepository;
|
|
114
|
+
|
|
115
|
+
const storageService = {
|
|
116
|
+
retrieve: async (key: string) => {
|
|
117
|
+
retrieved.push(key);
|
|
118
|
+
return LINKEDIN_EML;
|
|
119
|
+
},
|
|
120
|
+
storeMessageBody: async () => ({ uri: `s3://bodies/${message.messageId}` }),
|
|
121
|
+
storeMessageBodyStream: async () => ({
|
|
122
|
+
uri: `s3://bodies/${message.messageId}`,
|
|
123
|
+
}),
|
|
124
|
+
storeParsedBody: async () => {},
|
|
125
|
+
listBodyParts: async () => [],
|
|
126
|
+
} as unknown as StorageService;
|
|
127
|
+
|
|
128
|
+
const addressService = {
|
|
129
|
+
getAddress: async () => {
|
|
130
|
+
if (overrideCategory === undefined) {
|
|
131
|
+
throw new NotFoundError("Address not found");
|
|
132
|
+
}
|
|
133
|
+
return {
|
|
134
|
+
flags: { category: { value: overrideCategory, setAt: 1_000 } },
|
|
135
|
+
} as unknown as AddressItem;
|
|
136
|
+
},
|
|
137
|
+
incrementInboundCount: async () => {},
|
|
138
|
+
} as unknown as IAddressRepository;
|
|
139
|
+
|
|
140
|
+
const service = new BodySyncService(
|
|
141
|
+
messageService,
|
|
142
|
+
storageService,
|
|
143
|
+
threadMessageService,
|
|
144
|
+
addressService,
|
|
145
|
+
{ listBodyParts: async () => [] } as unknown as IEnvelopeRepository,
|
|
146
|
+
{ info: () => {}, error: () => {} },
|
|
147
|
+
);
|
|
148
|
+
|
|
149
|
+
return {
|
|
150
|
+
service,
|
|
151
|
+
message: messageRow,
|
|
152
|
+
rows,
|
|
153
|
+
messageUpdates,
|
|
154
|
+
threadUpdates,
|
|
155
|
+
retrieved,
|
|
156
|
+
};
|
|
157
|
+
};
|
|
158
|
+
|
|
159
|
+
const imapConnection = () =>
|
|
160
|
+
({
|
|
161
|
+
openBox: async () => {},
|
|
162
|
+
async *fetchMessageBodies(uids: number[]) {
|
|
163
|
+
for (const uid of uids) {
|
|
164
|
+
yield { uid, source: Readable.from([LINKEDIN_EML]) };
|
|
165
|
+
}
|
|
166
|
+
},
|
|
167
|
+
}) as unknown as IImapConnection;
|
|
168
|
+
|
|
169
|
+
const syncPass = (harness: Harness) =>
|
|
170
|
+
harness.service.syncBodies(["m-1"], "acc-1", "cfg-1", "INBOX", async () =>
|
|
171
|
+
imapConnection(),
|
|
172
|
+
);
|
|
173
|
+
|
|
174
|
+
describe("a body the classifier declined to categorize is examined once", () => {
|
|
175
|
+
it("does not read or rewrite the stored body on the pass after the one that stored it", async () => {
|
|
176
|
+
const harness = buildHarness(
|
|
177
|
+
{ messageId: "m-1" },
|
|
178
|
+
MessageCategory.uncategorized,
|
|
179
|
+
);
|
|
180
|
+
|
|
181
|
+
const first = await syncPass(harness);
|
|
182
|
+
|
|
183
|
+
assert.deepEqual(first.syncedMessageIds, ["m-1"]);
|
|
184
|
+
assert.equal(harness.message.category, MessageCategory.uncategorized);
|
|
185
|
+
assert.equal(harness.message.bodyStorageKey, "s3://bodies/m-1");
|
|
186
|
+
assert.equal(harness.messageUpdates.length, 1);
|
|
187
|
+
// The answer and the record that it was given, in one write.
|
|
188
|
+
assert.equal(
|
|
189
|
+
harness.messageUpdates[0].input.classificationState,
|
|
190
|
+
"Examined",
|
|
191
|
+
);
|
|
192
|
+
|
|
193
|
+
const second = await syncPass(harness);
|
|
194
|
+
|
|
195
|
+
assert.equal(second.skippedCount, 1);
|
|
196
|
+
assert.deepEqual(second.failedMessageIds, []);
|
|
197
|
+
assert.deepEqual(harness.retrieved, []);
|
|
198
|
+
assert.equal(harness.messageUpdates.length, 1);
|
|
199
|
+
assert.equal(harness.threadUpdates.length, 1);
|
|
200
|
+
});
|
|
201
|
+
|
|
202
|
+
it("leaves a message that is already stored and examined alone", async () => {
|
|
203
|
+
const harness = buildHarness(
|
|
204
|
+
{
|
|
205
|
+
messageId: "m-1",
|
|
206
|
+
bodyStorageKey: "s3://bodies/m-1",
|
|
207
|
+
category: MessageCategory.uncategorized,
|
|
208
|
+
classificationState: "Examined",
|
|
209
|
+
} as Partial<MessageItem> & Pick<MessageItem, "messageId">,
|
|
210
|
+
MessageCategory.uncategorized,
|
|
211
|
+
);
|
|
212
|
+
|
|
213
|
+
const result = await syncPass(harness);
|
|
214
|
+
|
|
215
|
+
assert.equal(result.skippedCount, 1);
|
|
216
|
+
assert.deepEqual(harness.retrieved, []);
|
|
217
|
+
assert.deepEqual(harness.messageUpdates, []);
|
|
218
|
+
assert.deepEqual(harness.threadUpdates, []);
|
|
219
|
+
});
|
|
220
|
+
|
|
221
|
+
it("leaves mail stored before the classifier ran marked as never examined", async () => {
|
|
222
|
+
const harness = buildHarness(
|
|
223
|
+
{
|
|
224
|
+
messageId: "m-1",
|
|
225
|
+
bodyStorageKey: "s3://bodies/m-1",
|
|
226
|
+
category: MessageCategory.uncategorized,
|
|
227
|
+
classificationState: "NotExamined",
|
|
228
|
+
} as Partial<MessageItem> & Pick<MessageItem, "messageId">,
|
|
229
|
+
MessageCategory.uncategorized,
|
|
230
|
+
);
|
|
231
|
+
|
|
232
|
+
const result = await syncPass(harness);
|
|
233
|
+
|
|
234
|
+
// A sync pass does not re-derive this cohort — that is the amplification
|
|
235
|
+
// #331 removes. It leaves the row saying so, which is what lets a
|
|
236
|
+
// deliberate backfill select it later.
|
|
237
|
+
assert.equal(result.skippedCount, 1);
|
|
238
|
+
assert.deepEqual(harness.retrieved, []);
|
|
239
|
+
assert.deepEqual(harness.messageUpdates, []);
|
|
240
|
+
assert.equal(harness.message.classificationState, "NotExamined");
|
|
241
|
+
});
|
|
242
|
+
|
|
243
|
+
it("still fetches and classifies a message whose body has never been stored", async () => {
|
|
244
|
+
const harness = buildHarness({ messageId: "m-1" });
|
|
245
|
+
|
|
246
|
+
const result = await syncPass(harness);
|
|
247
|
+
|
|
248
|
+
assert.deepEqual(result.syncedMessageIds, ["m-1"]);
|
|
249
|
+
assert.equal(harness.messageUpdates.length, 1);
|
|
250
|
+
assert.equal(
|
|
251
|
+
harness.messageUpdates[0].input.category,
|
|
252
|
+
MessageCategory.social,
|
|
253
|
+
);
|
|
254
|
+
assert.equal(harness.rows[0].category, MessageCategory.social);
|
|
255
|
+
});
|
|
256
|
+
});
|
package/src/body-sync.ts
CHANGED
|
@@ -24,6 +24,7 @@ import { isBulkSender } from "@remit/data-ports/wellknown";
|
|
|
24
24
|
import {
|
|
25
25
|
FilterState,
|
|
26
26
|
MessageCategory,
|
|
27
|
+
MessageClassificationState,
|
|
27
28
|
PlacementAction,
|
|
28
29
|
PlacementConfidence,
|
|
29
30
|
QuarantineFailureStage,
|
|
@@ -155,9 +156,7 @@ const toFilterMessage = (parsed: ParsedMail): FilterMessage => ({
|
|
|
155
156
|
const SNIPPET_LENGTH = 256;
|
|
156
157
|
|
|
157
158
|
/**
|
|
158
|
-
* The snippet the list row shows, from whichever body part carries text.
|
|
159
|
-
* by both paths that denormalize onto the ThreadMessage so they cannot derive it
|
|
160
|
-
* differently.
|
|
159
|
+
* The snippet the list row shows, from whichever body part carries text.
|
|
161
160
|
*/
|
|
162
161
|
const extractSnippet = (parsed: ParsedMail): string =>
|
|
163
162
|
extractSnippetFromEmail(
|
|
@@ -186,11 +185,14 @@ const alreadyDenormalized = (
|
|
|
186
185
|
/**
|
|
187
186
|
* RFC 034 Decision 3.1: `Message.category` is written once and never mutated
|
|
188
187
|
* after — RFC 030's message-list GSI sort key depends on it never churning.
|
|
189
|
-
* "Already decided" is any real category
|
|
190
|
-
*
|
|
191
|
-
*
|
|
192
|
-
*
|
|
193
|
-
*
|
|
188
|
+
* "Already decided" is any real category, so a message that carries one is
|
|
189
|
+
* never re-categorized by a re-entrant pass.
|
|
190
|
+
*
|
|
191
|
+
* `uncategorized` fails this test whether or not the classifier has run, which
|
|
192
|
+
* is deliberate: this asks what the row holds, not what was done to it. Whether
|
|
193
|
+
* the classifier has run is {@link Message.classificationState}, and it is the
|
|
194
|
+
* skip guard in {@link BodySyncService.syncBodies} — not this one — that keeps
|
|
195
|
+
* a declined message from being examined twice.
|
|
194
196
|
*/
|
|
195
197
|
const hasDecidedCategory = (
|
|
196
198
|
category: ThreadMessageCategory | undefined,
|
|
@@ -211,15 +213,18 @@ const hasDecidedPlacement = (placementDecidedAt: number | undefined): boolean =>
|
|
|
211
213
|
placementDecidedAt !== undefined;
|
|
212
214
|
|
|
213
215
|
/**
|
|
214
|
-
* Issue #499: whether a completed body-sync pass has already
|
|
215
|
-
* message. `bodyStorageKey` is written last, once every derivation of
|
|
216
|
-
* has run, so its presence means
|
|
217
|
-
* finished. Reads it exactly as `syncBodies`' own skip guard does. The same two
|
|
216
|
+
* Issue #499: whether a completed body-sync pass has already stored this
|
|
217
|
+
* message's body. `bodyStorageKey` is written last, once every derivation of
|
|
218
|
+
* that pass has run, so its presence means that pass finished. The same two
|
|
218
219
|
* re-entrant paths `hasDecidedCategory` and `hasDecidedPlacement` guard
|
|
219
|
-
* (`fetchAndGetBody`'s `NoSuchKey` fallback, `syncBodies(..., force: true)`)
|
|
220
|
-
* the ones that reach the derivations again with it already set.
|
|
220
|
+
* (`fetchAndGetBody`'s `NoSuchKey` fallback, `syncBodies(..., force: true)`)
|
|
221
|
+
* are the ones that reach the derivations again with it already set.
|
|
222
|
+
*
|
|
223
|
+
* Key presence, and nothing more. What the classifier decided is
|
|
224
|
+
* `Message.category`; whether it ran at all is `Message.classificationState`.
|
|
225
|
+
* Neither is inferred from here — the field this reads is the storage fact.
|
|
221
226
|
*/
|
|
222
|
-
const
|
|
227
|
+
const hasStoredBody = (bodyStorageKey: string | undefined): boolean =>
|
|
223
228
|
Boolean(bodyStorageKey);
|
|
224
229
|
|
|
225
230
|
/**
|
|
@@ -408,10 +413,6 @@ export class BodySyncService {
|
|
|
408
413
|
// messageId so we can match FETCH rows back and re-enqueue any UID the
|
|
409
414
|
// server never returns.
|
|
410
415
|
const pending = new Map<number, string>();
|
|
411
|
-
// Messages that were skipped (body already stored) but whose backfill
|
|
412
|
-
// classification failed. They are NOT in `pending` — nothing about them
|
|
413
|
-
// needs fetching — so they are merged into failedMessageIds separately.
|
|
414
|
-
const backfillFailedMessageIds: string[] = [];
|
|
415
416
|
|
|
416
417
|
// One read per round, not per message (issue #72). The list is small by
|
|
417
418
|
// design and almost always empty, so a lookup per message would put a
|
|
@@ -440,54 +441,16 @@ export class BodySyncService {
|
|
|
440
441
|
continue;
|
|
441
442
|
}
|
|
442
443
|
|
|
443
|
-
|
|
444
|
+
// The body is stored, so there is nothing to fetch (issue #331). This
|
|
445
|
+
// used to re-read those bytes from storage to re-derive a category,
|
|
446
|
+
// keyed on `category === uncategorized` — which is also the classifier's
|
|
447
|
+
// own "nothing to say", so a message it declined cost one storage read
|
|
448
|
+
// and two writes on every pass, forever, for the same answer. What the
|
|
449
|
+
// classifier did is now recorded as `classificationState`, so a sync
|
|
450
|
+
// pass never has to guess it from the row, and the cohort it has not
|
|
451
|
+
// reached is selectable by a backfill instead of re-derived here.
|
|
452
|
+
if (hasStoredBody(message.bodyStorageKey) && !force) {
|
|
444
453
|
this.log.debug?.({ messageId }, "Body already stored, skipping");
|
|
445
|
-
// The skip guard keys on the body, but classification is a separate
|
|
446
|
-
// derived field written by the same pass. A message that got its body
|
|
447
|
-
// before it got a classifier — or whose classifying pass failed after
|
|
448
|
-
// the body landed — is skipped here forever and stays `uncategorized`
|
|
449
|
-
// (issue #45). Classify it from the stored bytes: no IMAP, no
|
|
450
|
-
// placement/filter side effects, and it skips cleanly once done.
|
|
451
|
-
//
|
|
452
|
-
// Contained per-message: one unreadable body object must not abort a
|
|
453
|
-
// batch that has not fetched anything yet. The failure is loud and
|
|
454
|
-
// the id is requeued, but the other messages still get their bodies.
|
|
455
|
-
const backfillError = await this.backfillClassification(
|
|
456
|
-
message,
|
|
457
|
-
accountConfigId,
|
|
458
|
-
).then(
|
|
459
|
-
() => null,
|
|
460
|
-
(error: unknown) => error,
|
|
461
|
-
);
|
|
462
|
-
// The stored bytes will not parse, and they will not start parsing on
|
|
463
|
-
// a later attempt. Requeueing forever is the stall; set the message
|
|
464
|
-
// aside instead. Everything else this call can throw is storage or
|
|
465
|
-
// database work and stays on the requeue path below.
|
|
466
|
-
if (
|
|
467
|
-
backfillError instanceof BodyParseError &&
|
|
468
|
-
(await this.quarantineBodyParse(
|
|
469
|
-
message.messageId,
|
|
470
|
-
location,
|
|
471
|
-
backfillError,
|
|
472
|
-
))
|
|
473
|
-
) {
|
|
474
|
-
skippedCount++;
|
|
475
|
-
continue;
|
|
476
|
-
}
|
|
477
|
-
if (backfillError !== null) {
|
|
478
|
-
this.log.error?.(
|
|
479
|
-
{
|
|
480
|
-
messageId,
|
|
481
|
-
storageKey: message.bodyStorageKey,
|
|
482
|
-
errorName: (backfillError as { name?: string }).name,
|
|
483
|
-
errorCode: (backfillError as { Code?: string }).Code,
|
|
484
|
-
error: inspect(backfillError),
|
|
485
|
-
},
|
|
486
|
-
"Classification backfill failed for an already-stored body; leaving for requeue",
|
|
487
|
-
);
|
|
488
|
-
backfillFailedMessageIds.push(messageId);
|
|
489
|
-
continue;
|
|
490
|
-
}
|
|
491
454
|
skippedCount++;
|
|
492
455
|
continue;
|
|
493
456
|
}
|
|
@@ -495,11 +458,7 @@ export class BodySyncService {
|
|
|
495
458
|
}
|
|
496
459
|
|
|
497
460
|
if (pending.size === 0) {
|
|
498
|
-
return this.buildResult(
|
|
499
|
-
syncedMessageIds,
|
|
500
|
-
skippedCount,
|
|
501
|
-
backfillFailedMessageIds,
|
|
502
|
-
);
|
|
461
|
+
return this.buildResult(syncedMessageIds, skippedCount, []);
|
|
503
462
|
}
|
|
504
463
|
|
|
505
464
|
const connection = await getConnection();
|
|
@@ -585,7 +544,7 @@ export class BodySyncService {
|
|
|
585
544
|
|
|
586
545
|
// Anything still pending was never yielded (mid-stream drop or a UID the
|
|
587
546
|
// server silently omitted) — re-enqueue it.
|
|
588
|
-
const failedMessageIds = [...pending.values()
|
|
547
|
+
const failedMessageIds = [...pending.values()];
|
|
589
548
|
|
|
590
549
|
this.log.info(
|
|
591
550
|
{
|
|
@@ -953,7 +912,7 @@ export class BodySyncService {
|
|
|
953
912
|
// already-classified message through two shipped paths — the `NoSuchKey`
|
|
954
913
|
// fallback in `fetchAndGetBody` and `syncBodies(..., force: true)` — so a
|
|
955
914
|
// real, previously-decided category is carried forward unchanged instead
|
|
956
|
-
// of the just-recomputed one
|
|
915
|
+
// of the just-recomputed one.
|
|
957
916
|
// This also protects a `flags.category` override (issue #299): without
|
|
958
917
|
// this guard a re-entrant pass would let a *later* override silently
|
|
959
918
|
// rewrite a category already decided on an earlier message, which is
|
|
@@ -977,10 +936,15 @@ export class BodySyncService {
|
|
|
977
936
|
listId,
|
|
978
937
|
);
|
|
979
938
|
|
|
939
|
+
// `classificationState` records that the classifier ran, in the same
|
|
940
|
+
// UpdateItem as the answer it gave (issue #331). Without it `uncategorized`
|
|
941
|
+
// carries two meanings at once — not reached yet, and reached with nothing
|
|
942
|
+
// to say — and only one of them should ever be looked at again.
|
|
980
943
|
const update: UpdateMessageInput = {
|
|
981
944
|
bodyStorageKey: bodyRef.uri,
|
|
982
945
|
...classification,
|
|
983
946
|
category: finalCategory,
|
|
947
|
+
classificationState: MessageClassificationState.Examined,
|
|
984
948
|
...(moved ? { movedByRemit: true } : {}),
|
|
985
949
|
...(resolved.verdict ? { placementVerdict: resolved.verdict } : {}),
|
|
986
950
|
...(filterMove ? { filterMove } : {}),
|
|
@@ -1109,66 +1073,6 @@ export class BodySyncService {
|
|
|
1109
1073
|
return connection.fetchMessageBody(uid);
|
|
1110
1074
|
}
|
|
1111
1075
|
|
|
1112
|
-
/**
|
|
1113
|
-
* Classify a message whose body is already stored but which carries no
|
|
1114
|
-
* decided category, reading the body from storage instead of IMAP.
|
|
1115
|
-
*
|
|
1116
|
-
* "No decided category" is `uncategorized` OR the field being absent: rows
|
|
1117
|
-
* written before the column existed have no value at all, and treating that
|
|
1118
|
-
* as already-classified would strand exactly the oldest mail this backfill
|
|
1119
|
-
* exists to reach.
|
|
1120
|
-
*
|
|
1121
|
-
* Deliberately narrower than {@link applyPostStoreSteps}: it writes the
|
|
1122
|
-
* derived classification fields and the denormalized ThreadMessage category,
|
|
1123
|
-
* and nothing else. Placement moves and filter actions are index-time
|
|
1124
|
-
* decisions that already ran (or were declined) when the body first landed;
|
|
1125
|
-
* re-running them here would move mail the user has since filed by hand.
|
|
1126
|
-
*
|
|
1127
|
-
* A storage or write failure propagates to the caller, which contains it per
|
|
1128
|
-
* message: the id lands in `failedMessageIds` and SQS requeues it, while the
|
|
1129
|
-
* rest of the batch still gets its bodies. An unreadable body object is an
|
|
1130
|
-
* infra fault, never absorbed — but it is also not a reason to abort a batch
|
|
1131
|
-
* that has fetched nothing yet.
|
|
1132
|
-
*/
|
|
1133
|
-
private async backfillClassification(
|
|
1134
|
-
message: MessageItem,
|
|
1135
|
-
accountConfigId: string,
|
|
1136
|
-
): Promise<void> {
|
|
1137
|
-
if (!message.bodyStorageKey) return;
|
|
1138
|
-
if (hasDecidedCategory(message.category)) return;
|
|
1139
|
-
|
|
1140
|
-
const body = await this.storageService.retrieve(message.bodyStorageKey);
|
|
1141
|
-
const parsed = await parseMessageBody(body);
|
|
1142
|
-
const classification = await this.classifyMessage(accountConfigId, parsed);
|
|
1143
|
-
|
|
1144
|
-
// Same order as {@link applyPostStoreSteps}, for the same reason: the
|
|
1145
|
-
// signal the skip guard reads is written last. `message.category` is that
|
|
1146
|
-
// signal here, so a failure between the two writes leaves both undone and
|
|
1147
|
-
// the requeued retry redoes both. Writing the Message first strands the
|
|
1148
|
-
// denormalized row at `uncategorized` forever — the guard is satisfied and
|
|
1149
|
-
// the retry returns early (issue #320).
|
|
1150
|
-
// The same three denormalized fields the full body-store path writes (see
|
|
1151
|
-
// `applyPostStoreSteps`), not just the category. A copied message
|
|
1152
|
-
// inherits `bodyStorageKey` and a decided category from its source, so it
|
|
1153
|
-
// reaches neither that path nor this one's classification — but nothing
|
|
1154
|
-
// else ever writes `listId`, so leaving it out here made a copy's
|
|
1155
|
-
// `list_id` permanently NULL. Both are derived from the same bytes
|
|
1156
|
-
// already in hand.
|
|
1157
|
-
await this.denormalizeCategory(
|
|
1158
|
-
accountConfigId,
|
|
1159
|
-
message.messageId,
|
|
1160
|
-
classification.category,
|
|
1161
|
-
extractSnippet(parsed),
|
|
1162
|
-
extractListId(parsed),
|
|
1163
|
-
);
|
|
1164
|
-
await this.messageService.update(message.messageId, classification);
|
|
1165
|
-
|
|
1166
|
-
this.log.info(
|
|
1167
|
-
{ messageId: message.messageId, category: classification.category },
|
|
1168
|
-
"Backfilled classification for an already-stored body",
|
|
1169
|
-
);
|
|
1170
|
-
}
|
|
1171
|
-
|
|
1172
1076
|
/**
|
|
1173
1077
|
* Header classification, with the sender's `Address.flags.category`
|
|
1174
1078
|
* override (issue #299, RFC 039 Decision 3) substituted for the
|
|
@@ -1180,11 +1084,10 @@ export class BodySyncService {
|
|
|
1180
1084
|
*
|
|
1181
1085
|
* The override wins outright rather than blending with the heuristic — RFC
|
|
1182
1086
|
* 039 Decision 3 treats a direct reclassification as final, the same as
|
|
1183
|
-
* `flags.blocked`/`vip` already override placement.
|
|
1184
|
-
* (`applyPostStoreSteps
|
|
1185
|
-
*
|
|
1186
|
-
*
|
|
1187
|
-
* what this returns.
|
|
1087
|
+
* `flags.blocked`/`vip` already override placement. The caller
|
|
1088
|
+
* (`applyPostStoreSteps`) gates on `hasDecidedCategory` before this result
|
|
1089
|
+
* reaches a write, so a message that already carries a real category is
|
|
1090
|
+
* never re-touched regardless of what this returns.
|
|
1188
1091
|
*/
|
|
1189
1092
|
private async classifyMessage(
|
|
1190
1093
|
accountConfigId: string,
|
|
@@ -1370,7 +1273,7 @@ export class BodySyncService {
|
|
|
1370
1273
|
storedBodyKey: string | undefined,
|
|
1371
1274
|
): Promise<void> {
|
|
1372
1275
|
if (!this.unsubscribeConfig) return;
|
|
1373
|
-
if (
|
|
1276
|
+
if (hasStoredBody(storedBodyKey)) return;
|
|
1374
1277
|
|
|
1375
1278
|
const fromEmail = extractPrimaryFromEmail(parsed);
|
|
1376
1279
|
if (!fromEmail) return;
|
|
@@ -1414,7 +1317,7 @@ export class BodySyncService {
|
|
|
1414
1317
|
storedBodyKey: string | undefined,
|
|
1415
1318
|
): Promise<void> {
|
|
1416
1319
|
if (!this.calendarConfig) return;
|
|
1417
|
-
if (
|
|
1320
|
+
if (hasStoredBody(storedBodyKey)) return;
|
|
1418
1321
|
|
|
1419
1322
|
const parts = calendarParts(parsed);
|
|
1420
1323
|
if (parts.length === 0) return;
|
|
@@ -105,4 +105,30 @@ describe("ImapFlowConnection.connect() retry policy", () => {
|
|
|
105
105
|
assert.equal(attempt.mock.callCount(), 1);
|
|
106
106
|
assert.equal(sleep.mock.callCount(), 0);
|
|
107
107
|
});
|
|
108
|
+
|
|
109
|
+
it("carries the server's own refusal, which is not always a re-auth", async () => {
|
|
110
|
+
const connection = new ImapFlowConnection(baseConfig);
|
|
111
|
+
const priv = connection as unknown as WithPrivates;
|
|
112
|
+
|
|
113
|
+
// imapflow's shape for a tagged NO: the message is its own, the server's
|
|
114
|
+
// words are on `responseText`. Signing in again fixes nothing here — the
|
|
115
|
+
// tenant has IMAP switched off — so the text has to reach the user.
|
|
116
|
+
mock.method(priv, "attemptConnect", async () => {
|
|
117
|
+
throw Object.assign(new Error("Command failed"), {
|
|
118
|
+
authenticationFailed: true,
|
|
119
|
+
responseText:
|
|
120
|
+
"LOGIN failed. User is authenticated but not connected. SmtpClientAuthentication is disabled for the Tenant.",
|
|
121
|
+
});
|
|
122
|
+
});
|
|
123
|
+
|
|
124
|
+
await assert.rejects(
|
|
125
|
+
() => connection.connect(),
|
|
126
|
+
(error: unknown) => {
|
|
127
|
+
assert.ok(error instanceof MailConnectionError);
|
|
128
|
+
assert.equal(error.kind, "auth");
|
|
129
|
+
assert.match(error.message, /SmtpClientAuthentication is disabled/);
|
|
130
|
+
return true;
|
|
131
|
+
},
|
|
132
|
+
);
|
|
133
|
+
});
|
|
108
134
|
});
|
|
@@ -1391,6 +1391,20 @@ const buildImapAuth = (
|
|
|
1391
1391
|
throw new Error(`Unknown credential kind: ${JSON.stringify(_exhaustive)}`);
|
|
1392
1392
|
};
|
|
1393
1393
|
|
|
1394
|
+
/**
|
|
1395
|
+
* The server's own words for the failure: the text of the tagged NO/BAD, which
|
|
1396
|
+
* imapflow hangs off the error as `responseText`. It is the server's side of
|
|
1397
|
+
* the exchange only — never the command we sent, so no credential can ride out
|
|
1398
|
+
* on it. "SmtpClientAuthentication is disabled" and "User is authenticated but
|
|
1399
|
+
* not connected" arrive this way, and neither is fixed by re-authenticating, so
|
|
1400
|
+
* dropping the text leaves the account card telling the user to do the one
|
|
1401
|
+
* thing that cannot help.
|
|
1402
|
+
*/
|
|
1403
|
+
const serverResponseText = (error: Error): string => {
|
|
1404
|
+
const text = (error as { responseText?: unknown }).responseText;
|
|
1405
|
+
return typeof text === "string" ? text.trim() : "";
|
|
1406
|
+
};
|
|
1407
|
+
|
|
1394
1408
|
/**
|
|
1395
1409
|
* Classify a raw IMAP error into a MailConnectionError, or return null if
|
|
1396
1410
|
* the error is not recognisable (let it bubble as-is).
|
|
@@ -1416,7 +1430,13 @@ const classifyImapError = (
|
|
|
1416
1430
|
msg.includes("AUTHENTICATE") ||
|
|
1417
1431
|
(error as { authenticationFailed?: boolean }).authenticationFailed === true
|
|
1418
1432
|
) {
|
|
1419
|
-
|
|
1433
|
+
const serverText = serverResponseText(error);
|
|
1434
|
+
return new MailConnectionError(
|
|
1435
|
+
"auth",
|
|
1436
|
+
serverText
|
|
1437
|
+
? `IMAP authentication failed: ${serverText}`
|
|
1438
|
+
: "IMAP authentication failed",
|
|
1439
|
+
);
|
|
1420
1440
|
}
|
|
1421
1441
|
|
|
1422
1442
|
// Network-level failures — NEVER include the original message as it may
|
package/src/list-id-backfill.ts
CHANGED
|
@@ -88,8 +88,8 @@ const emptyTotals = (): ListIdBackfillTotals => ({
|
|
|
88
88
|
* Read the stored raw source, derive `List-Id`, and — only when one is
|
|
89
89
|
* present — write it. Kept as its own function (rather than inline in a
|
|
90
90
|
* try/catch) so the caller can contain a failure with `.then(fulfilled,
|
|
91
|
-
* rejected)` instead of a block catch,
|
|
92
|
-
*
|
|
91
|
+
* rejected)` instead of a block catch, which keeps the `try` around the one
|
|
92
|
+
* call whose failure this contains.
|
|
93
93
|
*/
|
|
94
94
|
const deriveAndApplyListId = async (
|
|
95
95
|
deps: ListIdBackfillDeps,
|
|
@@ -141,9 +141,12 @@ const deriveAndApplyListId = async (
|
|
|
141
141
|
* time in account-id order, so an interrupted run and its resume walk the same
|
|
142
142
|
* sequence. A failure reading or parsing one message's
|
|
143
143
|
* stored body is contained to that message — logged, counted, and the pass
|
|
144
|
-
* continues
|
|
145
|
-
*
|
|
146
|
-
*
|
|
144
|
+
* continues: one unreadable object must not strand the rest of the corpus.
|
|
145
|
+
*
|
|
146
|
+
* The shape a later pass over `Message.classificationState` should take
|
|
147
|
+
* (issue #331): a one-time, resumable, storage-only sweep that selects its
|
|
148
|
+
* cohort by an explicit field, so it can be run once, deliberately, after a
|
|
149
|
+
* classifier release — rather than a re-derivation folded into every sync pass.
|
|
147
150
|
*/
|
|
148
151
|
export const backfillListIds = async (
|
|
149
152
|
deps: ListIdBackfillDeps,
|