@remit/mailbox-service 0.0.111 → 0.0.112
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +21 -6
- package/src/body-part-mapper.ts +51 -1
- package/src/body-sync-queue.test.ts +39 -99
- package/src/body-sync-terminal.test.ts +115 -277
- package/src/body-sync-terminal.ts +10 -12
- package/src/body-sync.test.ts +925 -0
- package/src/body-sync.ts +156 -235
- package/src/classification-backfill.test.ts +197 -527
- package/src/classify-message.test.ts +11 -74
- package/src/external-move.test.ts +102 -209
- package/src/external-move.ts +13 -0
- package/src/filter-operations.test.ts +319 -0
- package/src/filter-operations.ts +377 -0
- package/src/filter-update.test.ts +313 -0
- package/src/filters/filter-config.test.ts +55 -197
- package/src/filters/filter-config.ts +4 -39
- package/src/filters/match.test.ts +1 -38
- package/src/filters/match.ts +3 -52
- package/src/filters/pipeline.test.ts +158 -289
- package/src/filters/pipeline.ts +56 -170
- package/src/flag-push-terminal.ts +9 -11
- package/src/flag-push.test.ts +76 -372
- package/src/flag-queue.test.ts +197 -0
- package/src/folder-operations.test.ts +273 -0
- package/src/folder-operations.ts +217 -0
- package/src/forwarded-message.test.ts +122 -0
- package/src/imapflow-connect-retry.test.ts +76 -113
- package/src/imapflow-connection.test.ts +34 -0
- package/src/imapflow-connection.ts +5 -4
- package/src/index.ts +59 -12
- package/src/list-id-backfill.test.ts +133 -499
- package/src/mailbox-cursor.test.ts +81 -189
- package/src/mailbox-delete-intent.test.ts +78 -205
- package/src/mailbox-intent.ts +5 -3
- package/src/mailbox-management.test.ts +107 -220
- package/src/mailbox-management.ts +1 -2
- package/src/mailbox-presence.test.ts +36 -34
- package/src/mailbox-queue.ts +2 -20
- package/src/mailbox-rename-intent.test.ts +156 -325
- package/src/mailbox-rename-subtree.test.ts +205 -582
- package/src/mailbox-sync.test.ts +182 -744
- package/src/message-move.test.ts +726 -0
- package/src/message-move.ts +227 -57
- package/src/message-read.ts +87 -0
- package/src/message-save-retry.test.ts +90 -0
- package/src/message-sync-flags.test.ts +10 -1
- package/src/message-sync.test.ts +762 -0
- package/src/message-sync.ts +409 -239
- package/src/mime-walker.ts +16 -15
- package/src/organize-job.ts +93 -0
- package/src/outbox-attachment.test.ts +97 -209
- package/src/outbox-failed-recovery.test.ts +198 -269
- package/src/outbox-queue.test.ts +24 -0
- package/src/outbox-queue.ts +25 -8
- package/src/ownership.ts +140 -0
- package/src/placement-move-terminal.ts +9 -12
- package/src/placement-move.ts +33 -6
- package/src/placement-settled.ts +28 -4
- package/src/placement.test.ts +404 -0
- package/src/quarantine.test.ts +90 -148
- package/src/quarantine.ts +3 -2
- package/src/refusal.ts +29 -0
- package/src/restore-source-placement.ts +0 -4
- package/src/spam-report.test.ts +238 -710
- package/src/sync-warning.test.ts +213 -0
- package/src/sync-warning.ts +56 -0
- package/src/test-helpers/fake-imap.ts +690 -0
- package/src/test-helpers/test-queue.ts +56 -0
- package/src/test-helpers/test-store.ts +299 -0
- package/src/thread-root.test.ts +29 -0
- package/src/thread-root.ts +51 -0
- package/src/types/mailparser-augment.d.ts +8 -0
- package/src/adversarial-mime.e2e.test.ts +0 -307
- package/src/body-sync-auto-read-writeonce.test.ts +0 -219
- package/src/body-sync-calendar-suggestion.test.ts +0 -433
- package/src/body-sync-category-denormalize.test.ts +0 -352
- package/src/body-sync-category-override.test.ts +0 -501
- package/src/body-sync-category-writeonce.test.ts +0 -234
- package/src/body-sync-filter.test.ts +0 -237
- package/src/body-sync-message-gone.test.ts +0 -139
- package/src/body-sync-placement-flags.test.ts +0 -347
- package/src/body-sync-placement-writeonce.test.ts +0 -337
- package/src/body-sync-quarantine.test.ts +0 -276
- package/src/body-sync-resync.test.ts +0 -605
- package/src/body-sync-unclassifiable-once.test.ts +0 -260
- package/src/body-sync-unsubscribed-auto-read.test.ts +0 -176
- package/src/filters/anchor-drift.ts +0 -55
- package/src/filters/message-embedder.test.ts +0 -39
- package/src/filters/message-embedder.ts +0 -33
- package/src/flag-push-terminal.test.ts +0 -209
- package/src/flag-queue-stars.test.ts +0 -171
- package/src/imapflow-connection.e2e.test.ts +0 -335
- package/src/imapflow-connection.integ.test.ts +0 -316
- package/src/message-delete-junk-reconcile.test.ts +0 -117
- package/src/message-delete-no-trash.test.ts +0 -245
- package/src/message-delete-unsettled-placement.test.ts +0 -396
- package/src/message-move-copy.test.ts +0 -562
- package/src/message-move-empty-trash.test.ts +0 -326
- package/src/message-move-same-mailbox.test.ts +0 -139
- package/src/message-move-unsettled-placement.test.ts +0 -378
- package/src/message-sync-address-spoof.test.ts +0 -293
- package/src/message-sync-category.test.ts +0 -142
- package/src/message-sync-changedsince.test.ts +0 -840
- package/src/message-sync-junk-harvest.test.ts +0 -244
- package/src/message-sync-quarantine.test.ts +0 -416
- package/src/message-sync-repoint.test.ts +0 -419
- package/src/outbox-queue-settled-row.test.ts +0 -140
- package/src/pass-through-unit-of-work.ts +0 -17
- package/src/placement-move-terminal.test.ts +0 -300
- package/src/placement-move-unsettled.test.ts +0 -275
- package/src/placement-transition-race.test.ts +0 -482
- package/src/stale-message-reconcile.test.ts +0 -82
- package/src/stale-message-reconcile.ts +0 -55
- package/src/test-helpers/isolated-mailbox.ts +0 -44
- package/src/test-helpers/mailfuzz-connection.ts +0 -27
- package/test/fixtures/mime-adversarial-e2e/01-octet-stream-pdf.eml +0 -21
- package/test/fixtures/mime-adversarial-e2e/02-nested-related-cid-image.eml +0 -31
- package/test/fixtures/mime-adversarial-e2e/03-calendar-invite.eml +0 -33
- package/test/fixtures/mime-adversarial-e2e/04-forwarded-rfc822-with-attachments.eml +0 -42
- package/test/fixtures/mime-adversarial-e2e/05-tnef-winmail.eml +0 -25
- package/test/fixtures/mime-adversarial-e2e/06-8bit-non-ascii.eml +0 -15
- package/test/fixtures/mime-adversarial-e2e/07-quoted-printable-soft-breaks.eml +0 -17
- package/test/fixtures/mime-adversarial-e2e/08-alternative-empty-text.eml +0 -18
package/src/body-sync.ts
CHANGED
|
@@ -5,15 +5,15 @@ import {
|
|
|
5
5
|
recordCalendarSuggestion,
|
|
6
6
|
} from "@remit/calendar-service";
|
|
7
7
|
import {
|
|
8
|
+
type BodyPartItem,
|
|
8
9
|
type IAddressRepository,
|
|
10
|
+
type ICalendarStore,
|
|
9
11
|
type ICalendarSuggestionRepository,
|
|
10
|
-
type ICalendarUnitOfWork,
|
|
11
12
|
type IEnvelopeRepository,
|
|
12
13
|
type IFilterRepository,
|
|
13
14
|
type IMailboxSpecialUseRepository,
|
|
14
15
|
type IMessageRepository,
|
|
15
16
|
type IThreadMessageRepository,
|
|
16
|
-
type IUnitOfWork,
|
|
17
17
|
isSenderMuted,
|
|
18
18
|
type MessageItem,
|
|
19
19
|
type ThreadMessageItem,
|
|
@@ -25,6 +25,7 @@ import {
|
|
|
25
25
|
NotFoundError,
|
|
26
26
|
} from "@remit/data-ports/errors";
|
|
27
27
|
import { deriveAddressId, deriveBodyPartId } from "@remit/data-ports/id";
|
|
28
|
+
import { isEncapsulatedMessage } from "@remit/data-ports/message-parts";
|
|
28
29
|
import { isBulkSender } from "@remit/data-ports/wellknown";
|
|
29
30
|
import {
|
|
30
31
|
FilterState,
|
|
@@ -33,6 +34,7 @@ import {
|
|
|
33
34
|
PlacementConfidence,
|
|
34
35
|
QuarantineFailureStage,
|
|
35
36
|
SenderTrust,
|
|
37
|
+
SyncWarningStage,
|
|
36
38
|
} from "@remit/domain-enums";
|
|
37
39
|
import {
|
|
38
40
|
isStorageNotFoundError,
|
|
@@ -42,12 +44,16 @@ import {
|
|
|
42
44
|
import { type ParsedMail, simpleParser } from "mailparser";
|
|
43
45
|
import pMap from "p-map";
|
|
44
46
|
import { BodyParseError, parseMessageBody } from "./body-parse.js";
|
|
45
|
-
import {
|
|
47
|
+
import {
|
|
48
|
+
mapMessagePartsToContent,
|
|
49
|
+
parsePartSource,
|
|
50
|
+
} from "./body-part-mapper.js";
|
|
46
51
|
import { calendarParts } from "./calendar-parts.js";
|
|
47
52
|
import {
|
|
48
53
|
classifyParsedMessage,
|
|
49
54
|
extractPrimaryFromEmail,
|
|
50
55
|
} from "./classify-message.js";
|
|
56
|
+
|
|
51
57
|
import { extractListId } from "./filters/list-id.js";
|
|
52
58
|
import type { FilterMessage } from "./filters/match.js";
|
|
53
59
|
import {
|
|
@@ -68,11 +74,11 @@ import {
|
|
|
68
74
|
hasDecidedCategory,
|
|
69
75
|
} from "./message-category.js";
|
|
70
76
|
import { isMessageGoneFromOpenMailbox } from "./message-presence.js";
|
|
71
|
-
import { PassThroughUnitOfWork } from "./pass-through-unit-of-work.js";
|
|
72
77
|
import type { PlacementMoveService } from "./placement-move.js";
|
|
73
|
-
import { isPlacementUnsettled } from "./placement-settled.js";
|
|
78
|
+
import { isPlacementUnsettled, uidMailboxIdOf } from "./placement-settled.js";
|
|
74
79
|
import { type QuarantineService, shapeFromMessageData } from "./quarantine.js";
|
|
75
80
|
import { extractSnippetFromEmail } from "./snippet.js";
|
|
81
|
+
import type { SyncWarningService } from "./sync-warning.js";
|
|
76
82
|
import { type IImapConnection, MailConnectionError } from "./types.js";
|
|
77
83
|
|
|
78
84
|
const BODY_PART_STORE_CONCURRENCY = 4;
|
|
@@ -120,12 +126,20 @@ export const isBodyPartDeferralEnabled = (): boolean => {
|
|
|
120
126
|
* single HEAD and early-returns, skipping the body.eml GET, the MIME re-parse,
|
|
121
127
|
* and the per-leaf HEADs on every warm re-open.
|
|
122
128
|
*
|
|
123
|
-
* Deliberately an S3-only marker (no Message-row flag)
|
|
124
|
-
* re-fire the redundant vector upsert that #607 removed. A leading dot keeps it
|
|
129
|
+
* Deliberately an S3-only marker (no Message-row flag). A leading dot keeps it
|
|
125
130
|
* out of any real IMAP section path (those are dot-separated digits).
|
|
126
131
|
*/
|
|
127
132
|
const MATERIALIZED_SENTINEL_PART_PATH = ".materialized";
|
|
128
133
|
|
|
134
|
+
/**
|
|
135
|
+
* The sentinel for a message carrying a forwarded message. Those parts were
|
|
136
|
+
* once stored under a layout in which the forwarded message's root overwrote
|
|
137
|
+
* its own row, so a `.materialized` written then vouches for objects at the
|
|
138
|
+
* wrong paths. Its own name makes every such message materialize once more,
|
|
139
|
+
* overwriting what is there; no other message is touched.
|
|
140
|
+
*/
|
|
141
|
+
const MATERIALIZED_SENTINEL_V2_PART_PATH = ".materialized.2";
|
|
142
|
+
|
|
129
143
|
/**
|
|
130
144
|
* A mid-stream socket drop during the ranged body fetch surfaces as a typed
|
|
131
145
|
* `MailConnectionError` (the connection layer classifies imapflow's
|
|
@@ -142,7 +156,7 @@ const isConnectionDrop = (error: unknown): boolean => {
|
|
|
142
156
|
|
|
143
157
|
/**
|
|
144
158
|
* Project a parsed message onto the fields a filter matches against (RFC 034) —
|
|
145
|
-
* the literal-clause targets
|
|
159
|
+
* the literal-clause targets. Kept
|
|
146
160
|
* separate from `ParsedMail` so {@link FilterPipeline} stays parser-agnostic.
|
|
147
161
|
*/
|
|
148
162
|
const toFilterMessage = (parsed: ParsedMail): FilterMessage => ({
|
|
@@ -249,10 +263,15 @@ export interface StoreBodyPartContentsResult {
|
|
|
249
263
|
export type ConnectionGetter = () => Promise<IImapConnection>;
|
|
250
264
|
|
|
251
265
|
export interface BodySyncFolder {
|
|
266
|
+
mailboxId: string;
|
|
252
267
|
fullPath: string;
|
|
253
268
|
placement: FolderPlacement;
|
|
254
269
|
}
|
|
255
270
|
|
|
271
|
+
export interface BodySyncRoundFolder extends BodySyncFolder {
|
|
272
|
+
uidValidity: number;
|
|
273
|
+
}
|
|
274
|
+
|
|
256
275
|
export interface PlacementConfig {
|
|
257
276
|
mailboxSpecialUseService: IMailboxSpecialUseRepository;
|
|
258
277
|
/**
|
|
@@ -297,7 +316,7 @@ export interface UnsubscribeConfig {
|
|
|
297
316
|
/**
|
|
298
317
|
* What body sync needs to offer a mail's invitation as a card (issue #1033).
|
|
299
318
|
*
|
|
300
|
-
* The
|
|
319
|
+
* The calendar store is here to provision the account's default collection on
|
|
301
320
|
* first use, through the one function every other caller provisions with. That
|
|
302
321
|
* is not bookkeeping: an invitation whose DTSTART names no zone is RFC 5545
|
|
303
322
|
* floating time, and reading it anywhere but where the user's calendar lives
|
|
@@ -309,14 +328,13 @@ export interface UnsubscribeConfig {
|
|
|
309
328
|
*/
|
|
310
329
|
export interface CalendarSuggestionConfig {
|
|
311
330
|
calendarSuggestionService: ICalendarSuggestionRepository;
|
|
312
|
-
|
|
331
|
+
calendarStore: ICalendarStore;
|
|
313
332
|
filterService: Pick<IFilterRepository, "listByAccountAndState">;
|
|
314
333
|
}
|
|
315
334
|
|
|
316
335
|
export class BodySyncService {
|
|
317
336
|
private log: BodySyncLogger;
|
|
318
337
|
private readonly filterPipeline?: FilterPipeline;
|
|
319
|
-
private readonly unitOfWork: IUnitOfWork;
|
|
320
338
|
|
|
321
339
|
constructor(
|
|
322
340
|
private messageService: IMessageRepository,
|
|
@@ -324,56 +342,30 @@ export class BodySyncService {
|
|
|
324
342
|
private threadMessageService: IThreadMessageRepository,
|
|
325
343
|
private addressService: IAddressRepository,
|
|
326
344
|
private envelopeService: IEnvelopeRepository,
|
|
345
|
+
private readonly syncWarningService: SyncWarningService,
|
|
327
346
|
logger?: BodySyncLogger,
|
|
328
347
|
private readonly placementConfig?: PlacementConfig,
|
|
329
348
|
private readonly filterConfig?: FilterConfig,
|
|
330
349
|
private readonly quarantineConfig?: QuarantineConfig,
|
|
331
350
|
private readonly unsubscribeConfig?: UnsubscribeConfig,
|
|
332
351
|
private readonly calendarConfig?: CalendarSuggestionConfig,
|
|
333
|
-
unitOfWork?: IUnitOfWork,
|
|
334
352
|
) {
|
|
335
353
|
this.log = logger ?? noopLogger;
|
|
336
|
-
this.unitOfWork =
|
|
337
|
-
unitOfWork ??
|
|
338
|
-
new PassThroughUnitOfWork({
|
|
339
|
-
message: messageService,
|
|
340
|
-
envelope: envelopeService,
|
|
341
|
-
address: addressService,
|
|
342
|
-
threadMessage: threadMessageService,
|
|
343
|
-
});
|
|
344
354
|
this.filterPipeline = filterConfig
|
|
345
355
|
? new FilterPipeline(filterConfig, this.log)
|
|
346
356
|
: undefined;
|
|
347
357
|
}
|
|
348
358
|
|
|
349
|
-
/**
|
|
350
|
-
* Sync message bodies for a batch of messages.
|
|
351
|
-
*
|
|
352
|
-
* Fail-fast on connection errors: if the IMAP connection is lost,
|
|
353
|
-
* immediately stop processing and return all remaining messages as failed
|
|
354
|
-
* so they can be requeued for later retry.
|
|
355
|
-
*
|
|
356
|
-
* @param messageIds - The message IDs to sync bodies for
|
|
357
|
-
* @param accountId - The account ID (for storage path)
|
|
358
|
-
* @param accountConfigId - The account config ID (for thread updates)
|
|
359
|
-
* @param folder - The IMAP mailbox path and its folder placement
|
|
360
|
-
* @param getConnection - Function to get a (lazy) IMAP connection
|
|
361
|
-
* @param force - Bypass the "already stored" skip guard and re-fetch every
|
|
362
|
-
* message in this batch even though `bodyStorageKey` is already set. Set
|
|
363
|
-
* only by the read-miss re-arm cue (a `/content` read found the storage
|
|
364
|
-
* object missing despite the DB row saying otherwise); bulk metadata-sync
|
|
365
|
-
* never sets this, so it keeps skipping on `bodyStorageKey` with no
|
|
366
|
-
* existence check.
|
|
367
|
-
*/
|
|
368
359
|
async syncBodies(
|
|
369
360
|
messageIds: string[],
|
|
370
361
|
accountId: string,
|
|
371
362
|
accountConfigId: string,
|
|
372
|
-
folder:
|
|
363
|
+
folder: BodySyncRoundFolder,
|
|
373
364
|
getConnection: ConnectionGetter,
|
|
374
365
|
force = false,
|
|
375
366
|
): Promise<SyncBodiesResult> {
|
|
376
367
|
const syncedMessageIds: string[] = [];
|
|
368
|
+
const syncedUids: number[] = [];
|
|
377
369
|
let skippedCount = 0;
|
|
378
370
|
const location = {
|
|
379
371
|
accountId,
|
|
@@ -403,8 +395,22 @@ export class BodySyncService {
|
|
|
403
395
|
const quarantined =
|
|
404
396
|
await this.quarantineConfig?.quarantineService.load(accountConfigId);
|
|
405
397
|
|
|
398
|
+
const rows = new Map(
|
|
399
|
+
(await this.messageService.get(messageIds)).map((row) => [
|
|
400
|
+
row.messageId,
|
|
401
|
+
row,
|
|
402
|
+
]),
|
|
403
|
+
);
|
|
404
|
+
|
|
406
405
|
for (const messageId of messageIds) {
|
|
407
|
-
const message =
|
|
406
|
+
const message = rows.get(messageId);
|
|
407
|
+
|
|
408
|
+
// A row deleted since the batch was cut, or moved out of this folder
|
|
409
|
+
// for good, has no uid here; its new uid would fetch another message.
|
|
410
|
+
if (!message || uidMailboxIdOf(message) !== folder.mailboxId) {
|
|
411
|
+
skippedCount++;
|
|
412
|
+
continue;
|
|
413
|
+
}
|
|
408
414
|
|
|
409
415
|
// A quarantined message has no stored body, so nothing else here would
|
|
410
416
|
// stop it being fetched and re-parsed on every redelivery, forever.
|
|
@@ -472,43 +478,31 @@ export class BodySyncService {
|
|
|
472
478
|
reStoredMessageIds.has(messageId),
|
|
473
479
|
);
|
|
474
480
|
} catch (error) {
|
|
475
|
-
// A per-message store failure (e.g. the parsed-body S3 write) is a
|
|
476
|
-
// real fault, but it must not silently mark the message synced —
|
|
477
|
-
// that would leave parsedBody null and the message unindexed. Leave
|
|
478
|
-
// the UID in `pending` so it lands in failedMessageIds and the SQS
|
|
479
|
-
// batch requeues just this message. A dropped connection is handled
|
|
480
|
-
// by the outer catch (fail-fast on the whole remaining batch).
|
|
481
|
-
if (isConnectionDrop(error)) throw error;
|
|
482
|
-
|
|
483
481
|
// The one error in this frame that is the message's own fault. It
|
|
484
|
-
// reaches here from the parse call and nowhere else, so the S3,
|
|
485
|
-
// and
|
|
486
|
-
//
|
|
487
|
-
//
|
|
488
|
-
if (
|
|
489
|
-
|
|
490
|
-
|
|
491
|
-
|
|
482
|
+
// reaches here from the parse call and nowhere else, so the S3, DB
|
|
483
|
+
// and queue work this same frame wraps cannot be mistaken for it.
|
|
484
|
+
// The row is durable before the UID leaves `pending`, so the
|
|
485
|
+
// message is never let go of unrecorded.
|
|
486
|
+
if (error instanceof BodyParseError && this.quarantineConfig) {
|
|
487
|
+
await this.recordQuarantine(
|
|
488
|
+
this.quarantineConfig,
|
|
489
|
+
messageId,
|
|
490
|
+
location,
|
|
491
|
+
error,
|
|
492
|
+
);
|
|
492
493
|
pending.delete(uid);
|
|
493
494
|
skippedCount++;
|
|
494
495
|
source.resume();
|
|
495
496
|
continue;
|
|
496
497
|
}
|
|
497
498
|
|
|
498
|
-
|
|
499
|
-
|
|
500
|
-
|
|
501
|
-
errorName: (error as { name?: string }).name,
|
|
502
|
-
error: (error as Error).message,
|
|
503
|
-
},
|
|
504
|
-
"Body store failed for message; leaving for requeue",
|
|
505
|
-
);
|
|
506
|
-
// Drain the stream so the connection stays usable for the next UID.
|
|
507
|
-
source.resume();
|
|
508
|
-
continue;
|
|
499
|
+
// Anything else is our own storage, database or queue failing. It
|
|
500
|
+
// aborts the batch, which redelivers whole; what stored skips.
|
|
501
|
+
throw error;
|
|
509
502
|
}
|
|
510
503
|
pending.delete(uid);
|
|
511
504
|
syncedMessageIds.push(messageId);
|
|
505
|
+
syncedUids.push(uid);
|
|
512
506
|
}
|
|
513
507
|
} catch (error) {
|
|
514
508
|
this.log.error?.(
|
|
@@ -532,6 +526,21 @@ export class BodySyncService {
|
|
|
532
526
|
// server silently omitted) — re-enqueue it.
|
|
533
527
|
const failedMessageIds = [...pending.values()];
|
|
534
528
|
|
|
529
|
+
await this.syncWarningService.settle(
|
|
530
|
+
{
|
|
531
|
+
accountConfigId,
|
|
532
|
+
accountId,
|
|
533
|
+
mailboxId: folder.mailboxId,
|
|
534
|
+
uidValidity: folder.uidValidity,
|
|
535
|
+
},
|
|
536
|
+
SyncWarningStage.Body,
|
|
537
|
+
{
|
|
538
|
+
failedUids: [...pending.keys()],
|
|
539
|
+
syncedUids,
|
|
540
|
+
serverUids: { kind: "Unlisted" },
|
|
541
|
+
},
|
|
542
|
+
);
|
|
543
|
+
|
|
535
544
|
this.log.info(
|
|
536
545
|
{
|
|
537
546
|
synced: syncedMessageIds.length,
|
|
@@ -547,53 +556,10 @@ export class BodySyncService {
|
|
|
547
556
|
}
|
|
548
557
|
|
|
549
558
|
/**
|
|
550
|
-
* Set a message aside because its body will not parse (issue #72).
|
|
551
|
-
*
|
|
552
|
-
*
|
|
553
|
-
* built without a quarantine writer, or because writing it failed. The
|
|
554
|
-
* caller then falls back to requeueing rather than dropping a message no
|
|
555
|
-
* record survives. The failure the row carries is the parser's, and only the
|
|
556
|
-
* parser's — see {@link BodyParseError}.
|
|
557
|
-
*
|
|
558
|
-
* Reading and writing the record is storage and database work, so its own
|
|
559
|
-
* failure is contained per message the same way the save it replaces is: it
|
|
560
|
-
* must not take down a batch that has other messages to fetch, and in the
|
|
561
|
-
* stream loop a throw would also skip `source.resume()` and leave the
|
|
562
|
-
* literal undrained on an otherwise usable connection.
|
|
563
|
-
*
|
|
564
|
-
* The fingerprint comes from the BodyPart rows metadata sync already wrote:
|
|
565
|
-
* the body path streams raw bytes and has no FETCH result to read, and
|
|
566
|
-
* re-reading headers the parser has just refused would be reading the very
|
|
567
|
-
* thing that is broken.
|
|
559
|
+
* Set a message aside because its body will not parse (issue #72). The
|
|
560
|
+
* fingerprint comes from the BodyPart rows metadata sync already wrote: the
|
|
561
|
+
* body path streams raw bytes and has no FETCH result to read.
|
|
568
562
|
*/
|
|
569
|
-
private async quarantineBodyParse(
|
|
570
|
-
messageId: string,
|
|
571
|
-
location: {
|
|
572
|
-
accountId: string;
|
|
573
|
-
accountConfigId: string;
|
|
574
|
-
mailboxPath: string;
|
|
575
|
-
},
|
|
576
|
-
error: BodyParseError,
|
|
577
|
-
): Promise<boolean> {
|
|
578
|
-
const config = this.quarantineConfig;
|
|
579
|
-
if (!config) return false;
|
|
580
|
-
|
|
581
|
-
return this.recordQuarantine(config, messageId, location, error).then(
|
|
582
|
-
() => true,
|
|
583
|
-
(writeError: unknown) => {
|
|
584
|
-
this.log.error?.(
|
|
585
|
-
{
|
|
586
|
-
messageId,
|
|
587
|
-
errorName: (writeError as { name?: string }).name,
|
|
588
|
-
error: inspect(writeError),
|
|
589
|
-
},
|
|
590
|
-
"Could not record quarantine; leaving the message for requeue",
|
|
591
|
-
);
|
|
592
|
-
return false;
|
|
593
|
-
},
|
|
594
|
-
);
|
|
595
|
-
}
|
|
596
|
-
|
|
597
563
|
private async recordQuarantine(
|
|
598
564
|
config: QuarantineConfig,
|
|
599
565
|
messageId: string,
|
|
@@ -606,6 +572,10 @@ export class BodySyncService {
|
|
|
606
572
|
): Promise<void> {
|
|
607
573
|
const message = await this.messageService.get(messageId);
|
|
608
574
|
const messageData = await this.envelopeService.getMessageData(messageId);
|
|
575
|
+
const threadMessage = await this.threadMessageService.findByMessageId(
|
|
576
|
+
location.accountConfigId,
|
|
577
|
+
messageId,
|
|
578
|
+
);
|
|
609
579
|
|
|
610
580
|
await config.quarantineService.record(
|
|
611
581
|
{
|
|
@@ -622,7 +592,11 @@ export class BodySyncService {
|
|
|
622
592
|
code: error.failureCode,
|
|
623
593
|
message: error.message,
|
|
624
594
|
},
|
|
625
|
-
shapeFromMessageData(
|
|
595
|
+
shapeFromMessageData(
|
|
596
|
+
message,
|
|
597
|
+
messageData,
|
|
598
|
+
threadMessage?.messageIdHeader,
|
|
599
|
+
),
|
|
626
600
|
);
|
|
627
601
|
}
|
|
628
602
|
|
|
@@ -779,7 +753,8 @@ export class BodySyncService {
|
|
|
779
753
|
accountConfigId,
|
|
780
754
|
accountId,
|
|
781
755
|
messageId,
|
|
782
|
-
|
|
756
|
+
await this.envelopeService.listBodyParts(messageId),
|
|
757
|
+
body,
|
|
783
758
|
);
|
|
784
759
|
}
|
|
785
760
|
await this.messageService.update(messageId, {
|
|
@@ -824,6 +799,7 @@ export class BodySyncService {
|
|
|
824
799
|
// The DECISION is computed here; the applies below are unswallowed.
|
|
825
800
|
const filterDecision = await this.evaluateFilters(
|
|
826
801
|
accountConfigId,
|
|
802
|
+
accountId,
|
|
827
803
|
messageId,
|
|
828
804
|
parsed,
|
|
829
805
|
);
|
|
@@ -831,10 +807,10 @@ export class BodySyncService {
|
|
|
831
807
|
// Store the parsed-body cache BEFORE persisting bodyStorageKey. The skip
|
|
832
808
|
// guard in syncBodies treats a stored bodyStorageKey as "fully synced",
|
|
833
809
|
// so if we wrote it first and the parsed-cache write then failed, the
|
|
834
|
-
// requeued retry would skip the message — leaving parsedBody null
|
|
835
|
-
//
|
|
836
|
-
//
|
|
837
|
-
//
|
|
810
|
+
// requeued retry would skip the message — leaving parsedBody null forever.
|
|
811
|
+
// A parsed-cache failure here throws and propagates before bodyStorageKey
|
|
812
|
+
// is written, so the requeued message genuinely re-attempts the parsed
|
|
813
|
+
// write.
|
|
838
814
|
await this.storeParsedBodyCache(
|
|
839
815
|
accountConfigId,
|
|
840
816
|
accountId,
|
|
@@ -851,7 +827,8 @@ export class BodySyncService {
|
|
|
851
827
|
accountConfigId,
|
|
852
828
|
accountId,
|
|
853
829
|
messageId,
|
|
854
|
-
|
|
830
|
+
await this.envelopeService.listBodyParts(messageId),
|
|
831
|
+
body,
|
|
855
832
|
);
|
|
856
833
|
}
|
|
857
834
|
|
|
@@ -872,10 +849,13 @@ export class BodySyncService {
|
|
|
872
849
|
// offer an undo back to it (issue #223). Snapshotted into the marker
|
|
873
850
|
// rather than read from `originalMailboxId` at derive time, which a
|
|
874
851
|
// later user move would overwrite — the filter's source must survive an
|
|
875
|
-
// undo to stay derivable.
|
|
876
|
-
|
|
877
|
-
|
|
878
|
-
|
|
852
|
+
// undo to stay derivable. A redelivery after a failed enqueue finds
|
|
853
|
+
// the move already applied locally, so the source is the folder the
|
|
854
|
+
// uid still names, not the one the row now points at.
|
|
855
|
+
const sourceMailboxId = uidMailboxIdOf(
|
|
856
|
+
await this.messageService.get(messageId),
|
|
857
|
+
);
|
|
858
|
+
const filed = await this.filterConfig?.placementMoveService.moveMessage(
|
|
879
859
|
accountConfigId,
|
|
880
860
|
messageId,
|
|
881
861
|
destinationMailboxId,
|
|
@@ -886,7 +866,7 @@ export class BodySyncService {
|
|
|
886
866
|
// there matches but was not moved by Remit. `moveMessage` no-ops in that
|
|
887
867
|
// case; record neither the marker nor `movedByRemit`, so the badge never
|
|
888
868
|
// claims a move that did not happen and its Undo is never a no-op.
|
|
889
|
-
if (sourceMailboxId !== destinationMailboxId) {
|
|
869
|
+
if (sourceMailboxId !== destinationMailboxId && filed !== "refused") {
|
|
890
870
|
filterMoved = true;
|
|
891
871
|
filterMove = {
|
|
892
872
|
filterId: filterDecision.move.filterId,
|
|
@@ -946,9 +926,8 @@ export class BodySyncService {
|
|
|
946
926
|
const moved = Boolean(resolved.move || filterMoved);
|
|
947
927
|
|
|
948
928
|
// ONE Message UpdateItem per synced message: bodyStorageKey + every
|
|
949
|
-
// classification/derived field + the move flag + the audit verdict
|
|
950
|
-
//
|
|
951
|
-
// redundant S3-Vectors upsert, so we collapse them into a single write.
|
|
929
|
+
// classification/derived field + the move flag + the audit verdict,
|
|
930
|
+
// collapsed into a single write.
|
|
952
931
|
// The flag is folded in only when a move WAS applied (by the classifier or
|
|
953
932
|
// a filter); the verdict is folded in whenever Remit decided to act.
|
|
954
933
|
// Written LAST so bodyStorageKey — the skip-guard signal — is only durable
|
|
@@ -1140,12 +1119,7 @@ export class BodySyncService {
|
|
|
1140
1119
|
);
|
|
1141
1120
|
}
|
|
1142
1121
|
|
|
1143
|
-
return this.reconcileGoneMessage(
|
|
1144
|
-
message,
|
|
1145
|
-
accountId,
|
|
1146
|
-
accountConfigId,
|
|
1147
|
-
mailboxPath,
|
|
1148
|
-
);
|
|
1122
|
+
return this.reconcileGoneMessage(message, accountId, mailboxPath);
|
|
1149
1123
|
}
|
|
1150
1124
|
|
|
1151
1125
|
/**
|
|
@@ -1158,7 +1132,6 @@ export class BodySyncService {
|
|
|
1158
1132
|
private async reconcileGoneMessage(
|
|
1159
1133
|
message: MessageItem,
|
|
1160
1134
|
accountId: string,
|
|
1161
|
-
accountConfigId: string,
|
|
1162
1135
|
mailboxPath: string,
|
|
1163
1136
|
): Promise<never> {
|
|
1164
1137
|
const { messageId, uid } = message;
|
|
@@ -1171,26 +1144,10 @@ export class BodySyncService {
|
|
|
1171
1144
|
);
|
|
1172
1145
|
}
|
|
1173
1146
|
|
|
1174
|
-
const reconciled = await this.
|
|
1175
|
-
|
|
1176
|
-
|
|
1177
|
-
|
|
1178
|
-
uid,
|
|
1179
|
-
});
|
|
1180
|
-
if (!deleted) return false;
|
|
1181
|
-
const threadMessages = await repos.threadMessage.findAllByMessageId(
|
|
1182
|
-
accountConfigId,
|
|
1183
|
-
messageId,
|
|
1184
|
-
);
|
|
1185
|
-
if (threadMessages.length > 0) {
|
|
1186
|
-
await repos.threadMessage.deleteMany(
|
|
1187
|
-
threadMessages.map((row) => ({
|
|
1188
|
-
accountConfigId: row.accountConfigId,
|
|
1189
|
-
threadMessageId: row.threadMessageId,
|
|
1190
|
-
})),
|
|
1191
|
-
);
|
|
1192
|
-
}
|
|
1193
|
-
return true;
|
|
1147
|
+
const reconciled = await this.messageService.deleteIfPlacement(messageId, {
|
|
1148
|
+
status: message.status,
|
|
1149
|
+
mailboxId: message.mailboxId,
|
|
1150
|
+
uid,
|
|
1194
1151
|
});
|
|
1195
1152
|
|
|
1196
1153
|
if (!reconciled) {
|
|
@@ -1403,7 +1360,7 @@ export class BodySyncService {
|
|
|
1403
1360
|
const parts = calendarParts(parsed);
|
|
1404
1361
|
if (parts.length === 0) return;
|
|
1405
1362
|
|
|
1406
|
-
const {
|
|
1363
|
+
const { calendarStore, calendarSuggestionService, filterService } =
|
|
1407
1364
|
this.calendarConfig;
|
|
1408
1365
|
|
|
1409
1366
|
await Promise.resolve()
|
|
@@ -1433,7 +1390,7 @@ export class BodySyncService {
|
|
|
1433
1390
|
// the organizer meant it. Idempotent: the id is derived, so a second
|
|
1434
1391
|
// first use returns the collection the first one made.
|
|
1435
1392
|
const collection = await provisionDefaultCalendar(
|
|
1436
|
-
|
|
1393
|
+
calendarStore,
|
|
1437
1394
|
accountConfigId,
|
|
1438
1395
|
);
|
|
1439
1396
|
for (const part of parts) {
|
|
@@ -1503,12 +1460,14 @@ export class BodySyncService {
|
|
|
1503
1460
|
*/
|
|
1504
1461
|
private async evaluateFilters(
|
|
1505
1462
|
accountConfigId: string,
|
|
1463
|
+
accountId: string,
|
|
1506
1464
|
messageId: string,
|
|
1507
1465
|
parsed: ParsedMail,
|
|
1508
1466
|
): Promise<FilterDecision> {
|
|
1509
1467
|
if (!this.filterPipeline) return { labels: [] };
|
|
1510
1468
|
return this.filterPipeline.evaluate(
|
|
1511
1469
|
accountConfigId,
|
|
1470
|
+
accountId,
|
|
1512
1471
|
messageId,
|
|
1513
1472
|
toFilterMessage(parsed),
|
|
1514
1473
|
);
|
|
@@ -1517,8 +1476,7 @@ export class BodySyncService {
|
|
|
1517
1476
|
/**
|
|
1518
1477
|
* Resolve the placement decision for a message, BEFORE the single Message
|
|
1519
1478
|
* update — so both the `movedByRemit` flag and the audit verdict join that
|
|
1520
|
-
* one UpdateItem instead of a second mutation
|
|
1521
|
-
* redundant S3-Vectors upsert. Drives both directions (junk → inbox rescue,
|
|
1479
|
+
* one UpdateItem instead of a second mutation. Drives both directions (junk → inbox rescue,
|
|
1522
1480
|
* inbox → junk demote) off the pure {@link classifyPlacement} verdict.
|
|
1523
1481
|
*
|
|
1524
1482
|
* Returns a {@link PlacementOutcome}: a `verdict` to persist whenever Remit
|
|
@@ -1772,10 +1730,10 @@ export class BodySyncService {
|
|
|
1772
1730
|
accountConfigId: string,
|
|
1773
1731
|
accountId: string,
|
|
1774
1732
|
messageId: string,
|
|
1775
|
-
|
|
1733
|
+
bodyParts: readonly BodyPartItem[],
|
|
1734
|
+
body: Buffer,
|
|
1776
1735
|
options?: { skipExisting?: boolean },
|
|
1777
1736
|
): Promise<StoreBodyPartContentsResult> {
|
|
1778
|
-
const bodyParts = await this.envelopeService.listBodyParts(messageId);
|
|
1779
1737
|
if (bodyParts.length === 0) {
|
|
1780
1738
|
this.log.debug?.(
|
|
1781
1739
|
{ messageId },
|
|
@@ -1785,31 +1743,32 @@ export class BodySyncService {
|
|
|
1785
1743
|
}
|
|
1786
1744
|
|
|
1787
1745
|
const log = this.log;
|
|
1788
|
-
|
|
1789
|
-
|
|
1790
|
-
|
|
1791
|
-
|
|
1792
|
-
|
|
1793
|
-
|
|
1794
|
-
|
|
1795
|
-
|
|
1796
|
-
|
|
1797
|
-
|
|
1798
|
-
|
|
1799
|
-
|
|
1800
|
-
|
|
1801
|
-
|
|
1746
|
+
// A parse of its own: classification and the snippet, which search reads,
|
|
1747
|
+
// need an inline forwarded message's text, which `ignoreEmbedded` keeps out.
|
|
1748
|
+
const pairs = await mapMessagePartsToContent(
|
|
1749
|
+
bodyParts,
|
|
1750
|
+
await parsePartSource(body),
|
|
1751
|
+
{
|
|
1752
|
+
messageId,
|
|
1753
|
+
logger:
|
|
1754
|
+
log.warn || log.debug
|
|
1755
|
+
? {
|
|
1756
|
+
warn: (obj, msg) => {
|
|
1757
|
+
log.warn?.(obj, msg);
|
|
1758
|
+
},
|
|
1759
|
+
debug: (obj, msg) => {
|
|
1760
|
+
log.debug?.(obj, msg);
|
|
1761
|
+
},
|
|
1762
|
+
}
|
|
1763
|
+
: undefined,
|
|
1764
|
+
},
|
|
1765
|
+
);
|
|
1802
1766
|
|
|
1803
1767
|
if (pairs.length === 0) {
|
|
1804
1768
|
return { stored: 0 };
|
|
1805
1769
|
}
|
|
1806
1770
|
|
|
1807
|
-
const bodyPartIdByPath = new Map(
|
|
1808
|
-
bodyParts.map((bp) => [bp.partPath, bp.bodyPartId]),
|
|
1809
|
-
);
|
|
1810
|
-
|
|
1811
1771
|
let stored = 0;
|
|
1812
|
-
const textContents: { bodyPartId: string; content: string }[] = [];
|
|
1813
1772
|
await pMap(
|
|
1814
1773
|
pairs,
|
|
1815
1774
|
async (entry) => {
|
|
@@ -1833,30 +1792,10 @@ export class BodySyncService {
|
|
|
1833
1792
|
contentType: entry.contentType,
|
|
1834
1793
|
});
|
|
1835
1794
|
stored++;
|
|
1836
|
-
|
|
1837
|
-
if (
|
|
1838
|
-
entry.contentType.toLowerCase().startsWith("text/") &&
|
|
1839
|
-
entry.content.length > 0
|
|
1840
|
-
) {
|
|
1841
|
-
const bodyPartId = bodyPartIdByPath.get(entry.partPath);
|
|
1842
|
-
if (bodyPartId) {
|
|
1843
|
-
textContents.push({
|
|
1844
|
-
bodyPartId,
|
|
1845
|
-
content: entry.content.toString("utf8"),
|
|
1846
|
-
});
|
|
1847
|
-
}
|
|
1848
|
-
}
|
|
1849
1795
|
},
|
|
1850
1796
|
{ concurrency: BODY_PART_STORE_CONCURRENCY },
|
|
1851
1797
|
);
|
|
1852
1798
|
|
|
1853
|
-
if (textContents.length > 0) {
|
|
1854
|
-
await this.envelopeService.upsertBodyPartContents(
|
|
1855
|
-
messageId,
|
|
1856
|
-
textContents,
|
|
1857
|
-
);
|
|
1858
|
-
}
|
|
1859
|
-
|
|
1860
1799
|
this.log.info(
|
|
1861
1800
|
{ messageId, partCount: pairs.length, stored },
|
|
1862
1801
|
"Body parts stored",
|
|
@@ -1892,12 +1831,17 @@ export class BodySyncService {
|
|
|
1892
1831
|
messageId: string,
|
|
1893
1832
|
bodyStorageKey: string,
|
|
1894
1833
|
): Promise<StoreBodyPartContentsResult> {
|
|
1834
|
+
const bodyParts = await this.envelopeService.listBodyParts(messageId);
|
|
1835
|
+
const carriesMessage = bodyParts.some(isEncapsulatedMessage);
|
|
1836
|
+
const sentinel = carriesMessage
|
|
1837
|
+
? MATERIALIZED_SENTINEL_V2_PART_PATH
|
|
1838
|
+
: MATERIALIZED_SENTINEL_PART_PATH;
|
|
1895
1839
|
if (
|
|
1896
1840
|
await this.storageService.bodyPartExists(
|
|
1897
1841
|
accountConfigId,
|
|
1898
1842
|
accountId,
|
|
1899
1843
|
messageId,
|
|
1900
|
-
|
|
1844
|
+
sentinel,
|
|
1901
1845
|
)
|
|
1902
1846
|
) {
|
|
1903
1847
|
this.log.debug?.(
|
|
@@ -1908,13 +1852,13 @@ export class BodySyncService {
|
|
|
1908
1852
|
}
|
|
1909
1853
|
|
|
1910
1854
|
const body = await this.storageService.retrieve(bodyStorageKey);
|
|
1911
|
-
const parsed = await simpleParser(body);
|
|
1912
1855
|
const result = await this.storeBodyPartContents(
|
|
1913
1856
|
accountConfigId,
|
|
1914
1857
|
accountId,
|
|
1915
1858
|
messageId,
|
|
1916
|
-
|
|
1917
|
-
|
|
1859
|
+
bodyParts,
|
|
1860
|
+
body,
|
|
1861
|
+
{ skipExisting: !carriesMessage },
|
|
1918
1862
|
);
|
|
1919
1863
|
|
|
1920
1864
|
// Reaching here means storeBodyPartContents resolved — `pMap` rejects on
|
|
@@ -1924,47 +1868,24 @@ export class BodySyncService {
|
|
|
1924
1868
|
accountConfigId,
|
|
1925
1869
|
accountId,
|
|
1926
1870
|
messageId,
|
|
1927
|
-
partPath:
|
|
1871
|
+
partPath: sentinel,
|
|
1928
1872
|
content: Buffer.alloc(0),
|
|
1929
1873
|
});
|
|
1930
1874
|
|
|
1931
1875
|
return result;
|
|
1932
1876
|
}
|
|
1933
1877
|
|
|
1934
|
-
/**
|
|
1935
|
-
* Persist the pre-parsed body cache. A failure here MUST fail the
|
|
1936
|
-
* surrounding body-sync: the parsed-body object is what the search-index
|
|
1937
|
-
* pipeline reads, so swallowing a write failure leaves the message marked
|
|
1938
|
-
* synced while `parsedBody` is null and it never gets indexed — a silent
|
|
1939
|
-
* search gap. Propagate so the message lands in failedMessageIds and the
|
|
1940
|
-
* SQS batch requeues it.
|
|
1941
|
-
*/
|
|
1942
1878
|
private async storeParsedBodyCache(
|
|
1943
1879
|
accountConfigId: string,
|
|
1944
1880
|
accountId: string,
|
|
1945
1881
|
messageId: string,
|
|
1946
1882
|
parsed: ParsedMail,
|
|
1947
1883
|
): Promise<void> {
|
|
1948
|
-
|
|
1949
|
-
|
|
1950
|
-
|
|
1951
|
-
|
|
1952
|
-
|
|
1953
|
-
|
|
1954
|
-
parsed: parsedBody,
|
|
1955
|
-
});
|
|
1956
|
-
this.log.debug?.({ messageId }, "Parsed body cache stored");
|
|
1957
|
-
} catch (err: unknown) {
|
|
1958
|
-
this.log.error?.(
|
|
1959
|
-
{
|
|
1960
|
-
messageId,
|
|
1961
|
-
errorName: (err as { name?: string }).name,
|
|
1962
|
-
errorCode: (err as { Code?: string }).Code,
|
|
1963
|
-
error: inspect(err),
|
|
1964
|
-
},
|
|
1965
|
-
"Failed to store parsed body cache; failing sync to requeue",
|
|
1966
|
-
);
|
|
1967
|
-
throw err;
|
|
1968
|
-
}
|
|
1884
|
+
await this.storageService.storeParsedBody({
|
|
1885
|
+
accountConfigId,
|
|
1886
|
+
accountId,
|
|
1887
|
+
messageId,
|
|
1888
|
+
parsed: toParsedBody(parsed),
|
|
1889
|
+
});
|
|
1969
1890
|
}
|
|
1970
1891
|
}
|