@remit/mailbox-service 0.0.111 → 0.0.113
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/account-credentials.d.ts +83 -0
- package/dist/account-credentials.js +104 -0
- package/dist/attribute-mapper.d.ts +36 -0
- package/dist/attribute-mapper.js +119 -0
- package/dist/body-parse.d.ts +28 -0
- package/{src/body-parse.ts → dist/body-parse.js} +18 -26
- package/dist/body-part-mapper.d.ts +92 -0
- package/dist/body-part-mapper.js +301 -0
- package/dist/body-sync-queue.d.ts +59 -0
- package/dist/body-sync-queue.js +56 -0
- package/dist/body-sync-terminal.d.ts +58 -0
- package/dist/body-sync-terminal.js +95 -0
- package/dist/body-sync.d.ts +351 -0
- package/dist/body-sync.js +1204 -0
- package/dist/calendar-parts.d.ts +23 -0
- package/dist/calendar-parts.js +18 -0
- package/dist/classification-backfill.d.ts +91 -0
- package/dist/classification-backfill.js +180 -0
- package/dist/classify-message.d.ts +31 -0
- package/dist/classify-message.js +89 -0
- package/dist/connection-factory.d.ts +54 -0
- package/dist/connection-factory.js +54 -0
- package/dist/connection-test.d.ts +41 -0
- package/dist/connection-test.js +152 -0
- package/dist/external-move.d.ts +94 -0
- package/dist/external-move.js +173 -0
- package/dist/filter-operations.d.ts +42 -0
- package/dist/filter-operations.js +211 -0
- package/dist/filters/filter-config.d.ts +14 -0
- package/dist/filters/filter-config.js +10 -0
- package/{src/filters/list-id.ts → dist/filters/list-id.d.ts} +2 -16
- package/dist/filters/list-id.js +26 -0
- package/dist/filters/match.d.ts +69 -0
- package/dist/filters/match.js +165 -0
- package/dist/filters/pipeline.d.ts +64 -0
- package/dist/filters/pipeline.js +100 -0
- package/{src/flag-push-terminal.ts → dist/flag-push-terminal.d.ts} +19 -77
- package/dist/flag-push-terminal.js +63 -0
- package/dist/flag-push.d.ts +69 -0
- package/dist/flag-push.js +111 -0
- package/dist/flag-queue.d.ts +175 -0
- package/dist/flag-queue.js +291 -0
- package/dist/folder-operations.d.ts +39 -0
- package/dist/folder-operations.js +112 -0
- package/dist/folder-placement.d.ts +13 -0
- package/dist/folder-placement.js +23 -0
- package/dist/heuristics/authenticationResults.d.ts +10 -0
- package/dist/heuristics/authenticationResults.js +187 -0
- package/dist/heuristics/classifyByHeaders.d.ts +82 -0
- package/dist/heuristics/classifyByHeaders.js +365 -0
- package/dist/heuristics/classifyPlacement.d.ts +67 -0
- package/dist/heuristics/classifyPlacement.js +153 -0
- package/dist/heuristics/machineSenders.d.ts +21 -0
- package/{src/heuristics/machineSenders.ts → dist/heuristics/machineSenders.js} +34 -40
- package/dist/heuristics/senderMismatch.d.ts +81 -0
- package/dist/heuristics/senderMismatch.js +192 -0
- package/dist/heuristics/socialDomains.d.ts +14 -0
- package/{src/heuristics/socialDomains.ts → dist/heuristics/socialDomains.js} +9 -9
- package/dist/heuristics/transactionalDomains.d.ts +18 -0
- package/{src/heuristics/transactionalDomains.ts → dist/heuristics/transactionalDomains.js} +8 -8
- package/dist/imapflow-connection.d.ts +377 -0
- package/dist/imapflow-connection.js +1147 -0
- package/dist/index.d.ts +63 -0
- package/dist/index.js +65 -0
- package/dist/list-id-backfill.d.ts +83 -0
- package/dist/list-id-backfill.js +132 -0
- package/dist/mailbox-cursor-rebuild.d.ts +74 -0
- package/dist/mailbox-cursor-rebuild.js +71 -0
- package/{src/mailbox-cursor.ts → dist/mailbox-cursor.d.ts} +20 -98
- package/dist/mailbox-cursor.js +102 -0
- package/dist/mailbox-intent.d.ts +41 -0
- package/{src/mailbox-intent.ts → dist/mailbox-intent.js} +39 -63
- package/dist/mailbox-management.d.ts +240 -0
- package/dist/mailbox-management.js +576 -0
- package/dist/mailbox-presence.d.ts +52 -0
- package/{src/mailbox-presence.ts → dist/mailbox-presence.js} +19 -30
- package/dist/mailbox-queue.d.ts +126 -0
- package/dist/mailbox-queue.js +214 -0
- package/dist/mailbox-sync.d.ts +93 -0
- package/dist/mailbox-sync.js +490 -0
- package/dist/mailbox-upstream.d.ts +38 -0
- package/{src/mailbox-upstream.ts → dist/mailbox-upstream.js} +37 -47
- package/dist/message-category.d.ts +51 -0
- package/{src/message-category.ts → dist/message-category.js} +28 -71
- package/dist/message-move.d.ts +285 -0
- package/dist/message-move.js +858 -0
- package/dist/message-parser.d.ts +14 -0
- package/dist/message-parser.js +17 -0
- package/{src/message-presence.ts → dist/message-presence.d.ts} +1 -13
- package/dist/message-presence.js +29 -0
- package/dist/message-read.d.ts +51 -0
- package/dist/message-read.js +31 -0
- package/dist/message-sync.d.ts +345 -0
- package/dist/message-sync.js +1358 -0
- package/dist/mime-walker.d.ts +102 -0
- package/dist/mime-walker.js +198 -0
- package/dist/organize-job.d.ts +20 -0
- package/dist/organize-job.js +37 -0
- package/dist/outbox-attachment-content.d.ts +37 -0
- package/dist/outbox-attachment-content.js +50 -0
- package/dist/outbox-attachment-filename.d.ts +19 -0
- package/dist/outbox-attachment-filename.js +64 -0
- package/dist/outbox-attachment.d.ts +148 -0
- package/dist/outbox-attachment.js +273 -0
- package/dist/outbox-queue.d.ts +73 -0
- package/dist/outbox-queue.js +284 -0
- package/{src/outbox-status.ts → dist/outbox-status.d.ts} +1 -6
- package/dist/outbox-status.js +18 -0
- package/dist/ownership.d.ts +24 -0
- package/dist/ownership.js +83 -0
- package/dist/placement-move-terminal.d.ts +67 -0
- package/dist/placement-move-terminal.js +88 -0
- package/dist/placement-move.d.ts +83 -0
- package/dist/placement-move.js +216 -0
- package/{src/placement-settled.ts → dist/placement-settled.d.ts} +19 -50
- package/dist/placement-settled.js +75 -0
- package/dist/quarantine.d.ts +95 -0
- package/dist/quarantine.js +131 -0
- package/dist/refusal.d.ts +31 -0
- package/dist/refusal.js +3 -0
- package/{src/restore-source-placement.ts → dist/restore-source-placement.d.ts} +29 -101
- package/dist/restore-source-placement.js +79 -0
- package/dist/search-text-backfill.d.ts +15 -0
- package/dist/search-text-backfill.js +33 -0
- package/dist/snippet.d.ts +61 -0
- package/dist/snippet.js +129 -0
- package/dist/spam-report.d.ts +86 -0
- package/dist/spam-report.js +241 -0
- package/dist/sync-warning.d.ts +21 -0
- package/dist/sync-warning.js +26 -0
- package/dist/sync-watermarks.d.ts +138 -0
- package/dist/sync-watermarks.js +205 -0
- package/dist/test-helpers/fake-imap.d.ts +151 -0
- package/dist/test-helpers/fake-imap.js +508 -0
- package/dist/test-helpers/folder-roles.d.ts +34 -0
- package/dist/test-helpers/folder-roles.js +47 -0
- package/dist/test-helpers/mime-tree-generator.d.ts +82 -0
- package/dist/test-helpers/mime-tree-generator.js +329 -0
- package/dist/test-helpers/test-store.d.ts +69 -0
- package/dist/test-helpers/test-store.js +193 -0
- package/dist/text/normalizer.d.ts +20 -0
- package/dist/text/normalizer.js +102 -0
- package/dist/thread-message-rows.d.ts +69 -0
- package/dist/thread-message-rows.js +40 -0
- package/dist/thread-root.d.ts +7 -0
- package/dist/thread-root.js +25 -0
- package/dist/types.d.ts +343 -0
- package/dist/types.js +19 -0
- package/package.json +86 -73
- package/src/account-credentials.ts +0 -210
- package/src/adversarial-mime.e2e.test.ts +0 -307
- package/src/attribute-mapper.test.ts +0 -104
- package/src/attribute-mapper.ts +0 -148
- package/src/body-parse.test.ts +0 -55
- package/src/body-part-mapper.property.test.ts +0 -354
- package/src/body-part-mapper.ts +0 -360
- package/src/body-sync-auto-read-writeonce.test.ts +0 -219
- package/src/body-sync-calendar-suggestion.test.ts +0 -433
- package/src/body-sync-category-denormalize.test.ts +0 -352
- package/src/body-sync-category-override.test.ts +0 -501
- package/src/body-sync-category-writeonce.test.ts +0 -234
- package/src/body-sync-filter.test.ts +0 -237
- package/src/body-sync-message-gone.test.ts +0 -139
- package/src/body-sync-placement-flags.test.ts +0 -347
- package/src/body-sync-placement-writeonce.test.ts +0 -337
- package/src/body-sync-quarantine.test.ts +0 -276
- package/src/body-sync-queue.test.ts +0 -124
- package/src/body-sync-queue.ts +0 -131
- package/src/body-sync-resync.test.ts +0 -605
- package/src/body-sync-terminal.test.ts +0 -310
- package/src/body-sync-terminal.ts +0 -176
- package/src/body-sync-unclassifiable-once.test.ts +0 -260
- package/src/body-sync-unsubscribed-auto-read.test.ts +0 -176
- package/src/body-sync.ts +0 -1970
- package/src/calendar-parts.ts +0 -41
- package/src/classification-backfill.test.ts +0 -615
- package/src/classification-backfill.ts +0 -363
- package/src/classify-message.test.ts +0 -205
- package/src/classify-message.ts +0 -116
- package/src/connection-factory.ts +0 -102
- package/src/connection-test.ts +0 -208
- package/src/credentials.test.ts +0 -181
- package/src/external-move.test.ts +0 -369
- package/src/external-move.ts +0 -272
- package/src/filters/anchor-drift.ts +0 -55
- package/src/filters/filter-config.test.ts +0 -229
- package/src/filters/filter-config.ts +0 -69
- package/src/filters/list-id.test.ts +0 -50
- package/src/filters/match.test.ts +0 -429
- package/src/filters/match.ts +0 -274
- package/src/filters/message-embedder.test.ts +0 -39
- package/src/filters/message-embedder.ts +0 -33
- package/src/filters/pipeline.test.ts +0 -330
- package/src/filters/pipeline.ts +0 -284
- package/src/flag-push-terminal.test.ts +0 -209
- package/src/flag-push.test.ts +0 -408
- package/src/flag-push.ts +0 -198
- package/src/flag-queue-stars.test.ts +0 -171
- package/src/flag-queue.ts +0 -509
- package/src/folder-placement.ts +0 -56
- package/src/heuristics/authenticationResults.test.ts +0 -189
- package/src/heuristics/authenticationResults.ts +0 -212
- package/src/heuristics/classifyByHeaders.test.ts +0 -889
- package/src/heuristics/classifyByHeaders.ts +0 -467
- package/src/heuristics/classifyPlacement.test.ts +0 -366
- package/src/heuristics/classifyPlacement.ts +0 -211
- package/src/heuristics/classifyRealMail.test.ts +0 -242
- package/src/heuristics/machineSenders.test.ts +0 -101
- package/src/heuristics/senderMismatch.test.ts +0 -346
- package/src/heuristics/senderMismatch.ts +0 -259
- package/src/imapflow-connect-retry.test.ts +0 -134
- package/src/imapflow-connection.e2e.test.ts +0 -335
- package/src/imapflow-connection.integ.test.ts +0 -316
- package/src/imapflow-connection.test.ts +0 -469
- package/src/imapflow-connection.ts +0 -1475
- package/src/index.ts +0 -350
- package/src/list-id-backfill.test.ts +0 -573
- package/src/list-id-backfill.ts +0 -274
- package/src/mailbox-cursor-rebuild.test.ts +0 -171
- package/src/mailbox-cursor-rebuild.ts +0 -146
- package/src/mailbox-cursor.test.ts +0 -249
- package/src/mailbox-delete-intent.test.ts +0 -246
- package/src/mailbox-management.test.ts +0 -356
- package/src/mailbox-management.ts +0 -839
- package/src/mailbox-presence.test.ts +0 -72
- package/src/mailbox-queue.ts +0 -415
- package/src/mailbox-rename-intent.test.ts +0 -389
- package/src/mailbox-rename-subtree.test.ts +0 -664
- package/src/mailbox-sync.test.ts +0 -848
- package/src/mailbox-sync.ts +0 -724
- package/src/message-delete-junk-reconcile.test.ts +0 -117
- package/src/message-delete-no-trash.test.ts +0 -245
- package/src/message-delete-unsettled-placement.test.ts +0 -396
- package/src/message-move-copy.test.ts +0 -562
- package/src/message-move-empty-trash.test.ts +0 -326
- package/src/message-move-same-mailbox.test.ts +0 -139
- package/src/message-move-unsettled-placement.test.ts +0 -378
- package/src/message-move.ts +0 -1348
- package/src/message-parser.ts +0 -26
- package/src/message-presence.test.ts +0 -50
- package/src/message-sync-address-spoof.test.ts +0 -293
- package/src/message-sync-category.test.ts +0 -142
- package/src/message-sync-changedsince.test.ts +0 -840
- package/src/message-sync-flags.test.ts +0 -158
- package/src/message-sync-junk-harvest.test.ts +0 -244
- package/src/message-sync-quarantine.test.ts +0 -416
- package/src/message-sync-repoint.test.ts +0 -419
- package/src/message-sync.ts +0 -1963
- package/src/mime-walker.test.ts +0 -311
- package/src/mime-walker.ts +0 -305
- package/src/outbox-attachment-content.test.ts +0 -153
- package/src/outbox-attachment-content.ts +0 -104
- package/src/outbox-attachment-filename.test.ts +0 -132
- package/src/outbox-attachment-filename.ts +0 -76
- package/src/outbox-attachment.test.ts +0 -738
- package/src/outbox-attachment.ts +0 -536
- package/src/outbox-failed-recovery.test.ts +0 -370
- package/src/outbox-queue-settled-row.test.ts +0 -140
- package/src/outbox-queue.test.ts +0 -291
- package/src/outbox-queue.ts +0 -508
- package/src/pass-through-unit-of-work.ts +0 -17
- package/src/placement-move-terminal.test.ts +0 -300
- package/src/placement-move-terminal.ts +0 -148
- package/src/placement-move-unsettled.test.ts +0 -275
- package/src/placement-move.ts +0 -336
- package/src/placement-settled.test.ts +0 -147
- package/src/placement-transition-race.test.ts +0 -482
- package/src/quarantine.test.ts +0 -191
- package/src/quarantine.ts +0 -228
- package/src/snippet.test.ts +0 -213
- package/src/snippet.ts +0 -152
- package/src/spam-report.test.ts +0 -801
- package/src/spam-report.ts +0 -357
- package/src/stale-message-reconcile.test.ts +0 -82
- package/src/stale-message-reconcile.ts +0 -55
- package/src/sync-watermarks.test.ts +0 -409
- package/src/sync-watermarks.ts +0 -319
- package/src/system-flag-wire-format.test.ts +0 -77
- package/src/test-helpers/folder-roles.ts +0 -66
- package/src/test-helpers/isolated-mailbox.ts +0 -44
- package/src/test-helpers/mailfuzz-connection.ts +0 -27
- package/src/test-helpers/mime-tree-generator.ts +0 -461
- package/src/text/normalizer.test.ts +0 -223
- package/src/text/normalizer.ts +0 -140
- package/src/thread-message-rows.ts +0 -77
- package/src/types/mailparser-augment.d.ts +0 -15
- package/src/types/natural-porter-stemmer.d.ts +0 -50
- package/src/types/stopword.d.ts +0 -69
- package/src/types.ts +0 -371
- package/test/fixtures/mime-adversarial-e2e/01-octet-stream-pdf.eml +0 -21
- package/test/fixtures/mime-adversarial-e2e/02-nested-related-cid-image.eml +0 -31
- package/test/fixtures/mime-adversarial-e2e/03-calendar-invite.eml +0 -33
- package/test/fixtures/mime-adversarial-e2e/04-forwarded-rfc822-with-attachments.eml +0 -42
- package/test/fixtures/mime-adversarial-e2e/05-tnef-winmail.eml +0 -25
- package/test/fixtures/mime-adversarial-e2e/06-8bit-non-ascii.eml +0 -15
- package/test/fixtures/mime-adversarial-e2e/07-quoted-printable-soft-breaks.eml +0 -17
- package/test/fixtures/mime-adversarial-e2e/08-alternative-empty-text.eml +0 -18
- package/tsconfig.json +0 -8
package/src/body-sync.ts
DELETED
|
@@ -1,1970 +0,0 @@
|
|
|
1
|
-
import { PassThrough, type Readable } from "node:stream";
|
|
2
|
-
import { inspect } from "node:util";
|
|
3
|
-
import {
|
|
4
|
-
provisionDefaultCalendar,
|
|
5
|
-
recordCalendarSuggestion,
|
|
6
|
-
} from "@remit/calendar-service";
|
|
7
|
-
import {
|
|
8
|
-
type IAddressRepository,
|
|
9
|
-
type ICalendarSuggestionRepository,
|
|
10
|
-
type ICalendarUnitOfWork,
|
|
11
|
-
type IEnvelopeRepository,
|
|
12
|
-
type IFilterRepository,
|
|
13
|
-
type IMailboxSpecialUseRepository,
|
|
14
|
-
type IMessageRepository,
|
|
15
|
-
type IThreadMessageRepository,
|
|
16
|
-
type IUnitOfWork,
|
|
17
|
-
isSenderMuted,
|
|
18
|
-
type MessageItem,
|
|
19
|
-
type ThreadMessageItem,
|
|
20
|
-
type UpdateMessageInput,
|
|
21
|
-
} from "@remit/data-ports";
|
|
22
|
-
import {
|
|
23
|
-
MessageGoneFromServerError,
|
|
24
|
-
MessagePlacementUnsettledError,
|
|
25
|
-
NotFoundError,
|
|
26
|
-
} from "@remit/data-ports/errors";
|
|
27
|
-
import { deriveAddressId, deriveBodyPartId } from "@remit/data-ports/id";
|
|
28
|
-
import { isBulkSender } from "@remit/data-ports/wellknown";
|
|
29
|
-
import {
|
|
30
|
-
FilterState,
|
|
31
|
-
MessageClassificationState,
|
|
32
|
-
PlacementAction,
|
|
33
|
-
PlacementConfidence,
|
|
34
|
-
QuarantineFailureStage,
|
|
35
|
-
SenderTrust,
|
|
36
|
-
} from "@remit/domain-enums";
|
|
37
|
-
import {
|
|
38
|
-
isStorageNotFoundError,
|
|
39
|
-
type ParsedBody,
|
|
40
|
-
type StorageService,
|
|
41
|
-
} from "@remit/storage-service";
|
|
42
|
-
import { type ParsedMail, simpleParser } from "mailparser";
|
|
43
|
-
import pMap from "p-map";
|
|
44
|
-
import { BodyParseError, parseMessageBody } from "./body-parse.js";
|
|
45
|
-
import { mapBodyPartsToContent } from "./body-part-mapper.js";
|
|
46
|
-
import { calendarParts } from "./calendar-parts.js";
|
|
47
|
-
import {
|
|
48
|
-
classifyParsedMessage,
|
|
49
|
-
extractPrimaryFromEmail,
|
|
50
|
-
} from "./classify-message.js";
|
|
51
|
-
import { extractListId } from "./filters/list-id.js";
|
|
52
|
-
import type { FilterMessage } from "./filters/match.js";
|
|
53
|
-
import {
|
|
54
|
-
type FilterConfig,
|
|
55
|
-
type FilterDecision,
|
|
56
|
-
FilterPipeline,
|
|
57
|
-
} from "./filters/pipeline.js";
|
|
58
|
-
import type { FlagQueueService } from "./flag-queue.js";
|
|
59
|
-
import { placementOf, resolveFolderRoleMailboxes } from "./folder-placement.js";
|
|
60
|
-
import {
|
|
61
|
-
classifyPlacement,
|
|
62
|
-
type FolderPlacement,
|
|
63
|
-
type PlacementVerdict,
|
|
64
|
-
resolveSenderPlacement,
|
|
65
|
-
} from "./heuristics/classifyPlacement.js";
|
|
66
|
-
import {
|
|
67
|
-
denormalizeMessageCategory,
|
|
68
|
-
hasDecidedCategory,
|
|
69
|
-
} from "./message-category.js";
|
|
70
|
-
import { isMessageGoneFromOpenMailbox } from "./message-presence.js";
|
|
71
|
-
import { PassThroughUnitOfWork } from "./pass-through-unit-of-work.js";
|
|
72
|
-
import type { PlacementMoveService } from "./placement-move.js";
|
|
73
|
-
import { isPlacementUnsettled } from "./placement-settled.js";
|
|
74
|
-
import { type QuarantineService, shapeFromMessageData } from "./quarantine.js";
|
|
75
|
-
import { extractSnippetFromEmail } from "./snippet.js";
|
|
76
|
-
import { type IImapConnection, MailConnectionError } from "./types.js";
|
|
77
|
-
|
|
78
|
-
const BODY_PART_STORE_CONCURRENCY = 4;
|
|
79
|
-
|
|
80
|
-
type MessagePlacementVerdict = NonNullable<
|
|
81
|
-
UpdateMessageInput["placementVerdict"]
|
|
82
|
-
>;
|
|
83
|
-
|
|
84
|
-
type MessageFilterMove = NonNullable<UpdateMessageInput["filterMove"]>;
|
|
85
|
-
|
|
86
|
-
type ThreadMessageCategory = ThreadMessageItem["category"];
|
|
87
|
-
|
|
88
|
-
/**
|
|
89
|
-
* Outcome of {@link BodySyncService.resolvePlacement}: the audit `verdict` to
|
|
90
|
-
* persist on the Message (present whenever Remit decided to act, i.e. action
|
|
91
|
-
* != leave), and the IMAP `move` to enqueue (present only for a confident,
|
|
92
|
-
* actionable verdict). An unsure verdict carries a `verdict` but no `move`.
|
|
93
|
-
*/
|
|
94
|
-
interface PlacementOutcome {
|
|
95
|
-
verdict?: MessagePlacementVerdict;
|
|
96
|
-
move?: { destinationMailboxId: string; destinationPath: string };
|
|
97
|
-
placementDecidedAt?: number;
|
|
98
|
-
}
|
|
99
|
-
|
|
100
|
-
/**
|
|
101
|
-
* Whether the body-sync hot path defers per-part S3 objects. Each MIME leaf is
|
|
102
|
-
* its own PutObject, so during bulk sync these per-part writes dominate S3 write
|
|
103
|
-
* count and cost. With deferral ON a synced message writes only `body.eml` +
|
|
104
|
-
* `parsed.json.gz` (2 writes); the per-part objects are materialized lazily on
|
|
105
|
-
* the first `contentUrl` read via {@link BodySyncService.ensureBodyPartsStored}.
|
|
106
|
-
*
|
|
107
|
-
* Default ON (cost savings). Set `DEFER_BODY_PARTS=false`/`0`/`off` to restore
|
|
108
|
-
* the original eager per-part writes — a safe, reversible escape hatch.
|
|
109
|
-
*/
|
|
110
|
-
export const isBodyPartDeferralEnabled = (): boolean => {
|
|
111
|
-
const raw = process.env.DEFER_BODY_PARTS;
|
|
112
|
-
if (raw === undefined) return true;
|
|
113
|
-
const normalized = raw.trim().toLowerCase();
|
|
114
|
-
return normalized !== "false" && normalized !== "0" && normalized !== "off";
|
|
115
|
-
};
|
|
116
|
-
|
|
117
|
-
/**
|
|
118
|
-
* Sentinel object written under a message's parts prefix once lazy
|
|
119
|
-
* materialization has stored EVERY leaf. The read path checks it first with a
|
|
120
|
-
* single HEAD and early-returns, skipping the body.eml GET, the MIME re-parse,
|
|
121
|
-
* and the per-leaf HEADs on every warm re-open.
|
|
122
|
-
*
|
|
123
|
-
* Deliberately an S3-only marker (no Message-row flag): a DDB write would
|
|
124
|
-
* re-fire the redundant vector upsert that #607 removed. A leading dot keeps it
|
|
125
|
-
* out of any real IMAP section path (those are dot-separated digits).
|
|
126
|
-
*/
|
|
127
|
-
const MATERIALIZED_SENTINEL_PART_PATH = ".materialized";
|
|
128
|
-
|
|
129
|
-
/**
|
|
130
|
-
* A mid-stream socket drop during the ranged body fetch surfaces as a typed
|
|
131
|
-
* `MailConnectionError` (the connection layer classifies imapflow's
|
|
132
|
-
* `EConnectionClosed`/`NoConnection`). Detect by type/code, never by message
|
|
133
|
-
* text — the underlying library strings vary.
|
|
134
|
-
*/
|
|
135
|
-
const isConnectionDrop = (error: unknown): boolean => {
|
|
136
|
-
if (error instanceof MailConnectionError) {
|
|
137
|
-
return error.kind === "network";
|
|
138
|
-
}
|
|
139
|
-
const code = (error as { code?: string }).code;
|
|
140
|
-
return code === "EConnectionClosed" || code === "NoConnection";
|
|
141
|
-
};
|
|
142
|
-
|
|
143
|
-
/**
|
|
144
|
-
* Project a parsed message onto the fields a filter matches against (RFC 034) —
|
|
145
|
-
* the literal-clause targets plus the text a semantic anchor embeds. Kept
|
|
146
|
-
* separate from `ParsedMail` so {@link FilterPipeline} stays parser-agnostic.
|
|
147
|
-
*/
|
|
148
|
-
const toFilterMessage = (parsed: ParsedMail): FilterMessage => ({
|
|
149
|
-
from: extractPrimaryFromEmail(parsed) ?? "",
|
|
150
|
-
fromName: parsed.from?.value?.[0]?.name ?? "",
|
|
151
|
-
subject: parsed.subject ?? "",
|
|
152
|
-
text: parsed.text ?? "",
|
|
153
|
-
listId: extractListId(parsed),
|
|
154
|
-
});
|
|
155
|
-
|
|
156
|
-
const SNIPPET_LENGTH = 256;
|
|
157
|
-
|
|
158
|
-
/**
|
|
159
|
-
* The snippet the list row shows, from whichever body part carries text.
|
|
160
|
-
* Exported for the classification backfill (issue #1197), which denormalizes
|
|
161
|
-
* the same fields the body-store path does, from the same bytes.
|
|
162
|
-
*/
|
|
163
|
-
export const extractSnippet = (parsed: ParsedMail): string =>
|
|
164
|
-
extractSnippetFromEmail(
|
|
165
|
-
parsed.text,
|
|
166
|
-
typeof parsed.html === "string" ? parsed.html : undefined,
|
|
167
|
-
SNIPPET_LENGTH,
|
|
168
|
-
);
|
|
169
|
-
|
|
170
|
-
/**
|
|
171
|
-
* Issue #499: whether a completed body-sync pass has already stored this
|
|
172
|
-
* message's body. `bodyStorageKey` is written last, once every derivation of
|
|
173
|
-
* that pass has run, so its presence means that pass finished.
|
|
174
|
-
*
|
|
175
|
-
* Key presence, and nothing more. What the classifier decided is
|
|
176
|
-
* `Message.category`; whether it ran at all is `Message.classificationState`.
|
|
177
|
-
* Neither is inferred from here — the field this reads is the storage fact.
|
|
178
|
-
*
|
|
179
|
-
* The re-fetch paths that re-enter with the key already set (`syncBodies(...,
|
|
180
|
-
* force: true)` and `fetchAndGetBody`'s `NoSuchKey` fallback) are re-stores
|
|
181
|
-
* that skip the derivations entirely (issue #1011), so this guard is what
|
|
182
|
-
* keeps a first store that lands on an already-stored row from re-deciding
|
|
183
|
-
* anything.
|
|
184
|
-
*/
|
|
185
|
-
const hasStoredBody = (bodyStorageKey: string | undefined): boolean =>
|
|
186
|
-
Boolean(bodyStorageKey);
|
|
187
|
-
|
|
188
|
-
/**
|
|
189
|
-
* Issue #398: `flags.autoArchive` is a filing preference relative to the Inbox.
|
|
190
|
-
* A `leave` verdict reaches it for two unrelated reasons and only one of them
|
|
191
|
-
* means "nothing confident to say":
|
|
192
|
-
*
|
|
193
|
-
* - `already-moved-by-remit` is {@link classifyPlacement}'s guard against
|
|
194
|
-
* re-deciding a settled placement, and auto-archive is a re-decision. It
|
|
195
|
-
* fires on messages Remit itself placed, or that a user moved back
|
|
196
|
-
* afterwards.
|
|
197
|
-
* - A message already in Junk gets an unsure `leave` whatever the sender's
|
|
198
|
-
* `blocked` flag says, since the demote branch only fires outside Junk.
|
|
199
|
-
* Moving mail out of Junk is a rescue, which this file reserves for a
|
|
200
|
-
* confident signal plus sender trust.
|
|
201
|
-
*/
|
|
202
|
-
const autoArchiveApplies = (
|
|
203
|
-
verdict: PlacementVerdict,
|
|
204
|
-
placement: FolderPlacement,
|
|
205
|
-
): boolean =>
|
|
206
|
-
placement !== "junk" && !verdict.reasons.includes("already-moved-by-remit");
|
|
207
|
-
|
|
208
|
-
export const toParsedBody = (parsed: ParsedMail): ParsedBody => ({
|
|
209
|
-
text: parsed.text ?? null,
|
|
210
|
-
html: typeof parsed.html === "string" ? parsed.html : null,
|
|
211
|
-
attachments: (parsed.attachments ?? []).map((a) => ({
|
|
212
|
-
filename: a.filename ?? null,
|
|
213
|
-
contentType: a.contentType,
|
|
214
|
-
contentDisposition: a.contentDisposition ?? null,
|
|
215
|
-
contentId: a.contentId ?? null,
|
|
216
|
-
size: a.size,
|
|
217
|
-
})),
|
|
218
|
-
});
|
|
219
|
-
|
|
220
|
-
export interface BodySyncLogger {
|
|
221
|
-
info(obj: Record<string, unknown>, msg: string): void;
|
|
222
|
-
debug?(obj: Record<string, unknown>, msg: string): void;
|
|
223
|
-
warn?(obj: Record<string, unknown>, msg: string): void;
|
|
224
|
-
error?(obj: Record<string, unknown>, msg: string): void;
|
|
225
|
-
}
|
|
226
|
-
|
|
227
|
-
const noopLogger: BodySyncLogger = {
|
|
228
|
-
info: () => {},
|
|
229
|
-
};
|
|
230
|
-
|
|
231
|
-
export interface SyncBodiesResult {
|
|
232
|
-
syncedCount: number;
|
|
233
|
-
syncedMessageIds: string[];
|
|
234
|
-
skippedCount: number;
|
|
235
|
-
failedCount: number;
|
|
236
|
-
failedMessageIds: string[];
|
|
237
|
-
}
|
|
238
|
-
|
|
239
|
-
export interface FetchBodyResult {
|
|
240
|
-
text: string | null;
|
|
241
|
-
html: string | null;
|
|
242
|
-
storedAt: string;
|
|
243
|
-
}
|
|
244
|
-
|
|
245
|
-
export interface StoreBodyPartContentsResult {
|
|
246
|
-
stored: number;
|
|
247
|
-
}
|
|
248
|
-
|
|
249
|
-
export type ConnectionGetter = () => Promise<IImapConnection>;
|
|
250
|
-
|
|
251
|
-
export interface BodySyncFolder {
|
|
252
|
-
fullPath: string;
|
|
253
|
-
placement: FolderPlacement;
|
|
254
|
-
}
|
|
255
|
-
|
|
256
|
-
export interface PlacementConfig {
|
|
257
|
-
mailboxSpecialUseService: IMailboxSpecialUseRepository;
|
|
258
|
-
/**
|
|
259
|
-
* Local-first mover for a confident placement verdict (issue #1271). Same
|
|
260
|
-
* `moveMessage(accountConfigId, messageId, destinationMailboxId, accountId)`
|
|
261
|
-
* signature as the general-purpose `MessageMoveService` it replaced here,
|
|
262
|
-
* but backed by a pending-move marker + its own reconciler queue instead of
|
|
263
|
-
* the general MESSAGE_MOVE event (see `placement-move.ts`).
|
|
264
|
-
*/
|
|
265
|
-
placementMoveService: PlacementMoveService;
|
|
266
|
-
}
|
|
267
|
-
|
|
268
|
-
/**
|
|
269
|
-
* What body sync needs to set a message aside (issue #72). The mailbox facts
|
|
270
|
-
* are supplied by the caller because they are properties of the round, not of
|
|
271
|
-
* any one message.
|
|
272
|
-
*/
|
|
273
|
-
export interface QuarantineConfig {
|
|
274
|
-
quarantineService: QuarantineService;
|
|
275
|
-
mailboxId: string;
|
|
276
|
-
/**
|
|
277
|
-
* The mailbox's stored UIDVALIDITY. Taken from the mailbox row rather than
|
|
278
|
-
* from the open box because a round whose two disagree is paused by the
|
|
279
|
-
* cursor guard before it reaches any UID.
|
|
280
|
-
*/
|
|
281
|
-
uidValidity: number;
|
|
282
|
-
/** Redeliveries of this batch so far, so the record says how hard we tried. */
|
|
283
|
-
attempts: number;
|
|
284
|
-
}
|
|
285
|
-
|
|
286
|
-
/**
|
|
287
|
-
* What body sync needs to auto-mark-read a message from an unsubscribed
|
|
288
|
-
* sender (issue #302, RFC 039 Decision 3). Reuses the same
|
|
289
|
-
* `FlagQueueService.markAsRead` a manual mark-as-read already goes through —
|
|
290
|
-
* local `\Seen` + `ThreadMessage.isRead` + a durable pending IMAP flag-push
|
|
291
|
-
* marker — so this fires the same round-trip, not a second primitive.
|
|
292
|
-
*/
|
|
293
|
-
export interface UnsubscribeConfig {
|
|
294
|
-
flagQueueService: FlagQueueService;
|
|
295
|
-
}
|
|
296
|
-
|
|
297
|
-
/**
|
|
298
|
-
* What body sync needs to offer a mail's invitation as a card (issue #1033).
|
|
299
|
-
*
|
|
300
|
-
* The unit of work is here to provision the account's default collection on
|
|
301
|
-
* first use, through the one function every other caller provisions with. That
|
|
302
|
-
* is not bookkeeping: an invitation whose DTSTART names no zone is RFC 5545
|
|
303
|
-
* floating time, and reading it anywhere but where the user's calendar lives
|
|
304
|
-
* shows the meeting at the wrong hour. Without a collection there is no zone
|
|
305
|
-
* to read it in, and every such event silently lands in UTC.
|
|
306
|
-
*
|
|
307
|
-
* `filterService` is read to honour `dismiss{muteSender:true}`: a sender the
|
|
308
|
-
* user muted gets no card at all.
|
|
309
|
-
*/
|
|
310
|
-
export interface CalendarSuggestionConfig {
|
|
311
|
-
calendarSuggestionService: ICalendarSuggestionRepository;
|
|
312
|
-
calendarUnitOfWork: ICalendarUnitOfWork;
|
|
313
|
-
filterService: Pick<IFilterRepository, "listByAccountAndState">;
|
|
314
|
-
}
|
|
315
|
-
|
|
316
|
-
export class BodySyncService {
|
|
317
|
-
private log: BodySyncLogger;
|
|
318
|
-
private readonly filterPipeline?: FilterPipeline;
|
|
319
|
-
private readonly unitOfWork: IUnitOfWork;
|
|
320
|
-
|
|
321
|
-
constructor(
|
|
322
|
-
private messageService: IMessageRepository,
|
|
323
|
-
private storageService: StorageService,
|
|
324
|
-
private threadMessageService: IThreadMessageRepository,
|
|
325
|
-
private addressService: IAddressRepository,
|
|
326
|
-
private envelopeService: IEnvelopeRepository,
|
|
327
|
-
logger?: BodySyncLogger,
|
|
328
|
-
private readonly placementConfig?: PlacementConfig,
|
|
329
|
-
private readonly filterConfig?: FilterConfig,
|
|
330
|
-
private readonly quarantineConfig?: QuarantineConfig,
|
|
331
|
-
private readonly unsubscribeConfig?: UnsubscribeConfig,
|
|
332
|
-
private readonly calendarConfig?: CalendarSuggestionConfig,
|
|
333
|
-
unitOfWork?: IUnitOfWork,
|
|
334
|
-
) {
|
|
335
|
-
this.log = logger ?? noopLogger;
|
|
336
|
-
this.unitOfWork =
|
|
337
|
-
unitOfWork ??
|
|
338
|
-
new PassThroughUnitOfWork({
|
|
339
|
-
message: messageService,
|
|
340
|
-
envelope: envelopeService,
|
|
341
|
-
address: addressService,
|
|
342
|
-
threadMessage: threadMessageService,
|
|
343
|
-
});
|
|
344
|
-
this.filterPipeline = filterConfig
|
|
345
|
-
? new FilterPipeline(filterConfig, this.log)
|
|
346
|
-
: undefined;
|
|
347
|
-
}
|
|
348
|
-
|
|
349
|
-
/**
|
|
350
|
-
* Sync message bodies for a batch of messages.
|
|
351
|
-
*
|
|
352
|
-
* Fail-fast on connection errors: if the IMAP connection is lost,
|
|
353
|
-
* immediately stop processing and return all remaining messages as failed
|
|
354
|
-
* so they can be requeued for later retry.
|
|
355
|
-
*
|
|
356
|
-
* @param messageIds - The message IDs to sync bodies for
|
|
357
|
-
* @param accountId - The account ID (for storage path)
|
|
358
|
-
* @param accountConfigId - The account config ID (for thread updates)
|
|
359
|
-
* @param folder - The IMAP mailbox path and its folder placement
|
|
360
|
-
* @param getConnection - Function to get a (lazy) IMAP connection
|
|
361
|
-
* @param force - Bypass the "already stored" skip guard and re-fetch every
|
|
362
|
-
* message in this batch even though `bodyStorageKey` is already set. Set
|
|
363
|
-
* only by the read-miss re-arm cue (a `/content` read found the storage
|
|
364
|
-
* object missing despite the DB row saying otherwise); bulk metadata-sync
|
|
365
|
-
* never sets this, so it keeps skipping on `bodyStorageKey` with no
|
|
366
|
-
* existence check.
|
|
367
|
-
*/
|
|
368
|
-
async syncBodies(
|
|
369
|
-
messageIds: string[],
|
|
370
|
-
accountId: string,
|
|
371
|
-
accountConfigId: string,
|
|
372
|
-
folder: BodySyncFolder,
|
|
373
|
-
getConnection: ConnectionGetter,
|
|
374
|
-
force = false,
|
|
375
|
-
): Promise<SyncBodiesResult> {
|
|
376
|
-
const syncedMessageIds: string[] = [];
|
|
377
|
-
let skippedCount = 0;
|
|
378
|
-
const location = {
|
|
379
|
-
accountId,
|
|
380
|
-
accountConfigId,
|
|
381
|
-
mailboxPath: folder.fullPath,
|
|
382
|
-
};
|
|
383
|
-
|
|
384
|
-
// Resolve every message up front so we can issue ONE ranged FETCH for the
|
|
385
|
-
// whole batch (the desktop-client pattern) instead of a SELECT + download
|
|
386
|
-
// per message. Messages whose body is already stored are skipped here and
|
|
387
|
-
// never hit the wire — UNLESS `force` is set, in which case every message
|
|
388
|
-
// is re-fetched regardless of `bodyStorageKey` (the read-miss cue already
|
|
389
|
-
// confirmed the stored object is gone). `pending` maps each UID to its
|
|
390
|
-
// messageId so we can match FETCH rows back and re-enqueue any UID the
|
|
391
|
-
// server never returns.
|
|
392
|
-
const pending = new Map<number, string>();
|
|
393
|
-
// Message IDs in this batch whose row already carried `bodyStorageKey`
|
|
394
|
-
// when `force` re-added them to `pending`: a forced re-fetch of an
|
|
395
|
-
// already-stored body is a re-store (issue #1011) — bytes only, no
|
|
396
|
-
// decision re-run. A message with no stored body inside a forced batch
|
|
397
|
-
// is still a first store and takes the full decision pass.
|
|
398
|
-
const reStoredMessageIds = new Set<string>();
|
|
399
|
-
|
|
400
|
-
// One read per round, not per message (issue #72). The list is small by
|
|
401
|
-
// design and almost always empty, so a lookup per message would put a
|
|
402
|
-
// query on the hot path for a state that is nearly never set.
|
|
403
|
-
const quarantined =
|
|
404
|
-
await this.quarantineConfig?.quarantineService.load(accountConfigId);
|
|
405
|
-
|
|
406
|
-
for (const messageId of messageIds) {
|
|
407
|
-
const message = await this.messageService.get(messageId);
|
|
408
|
-
|
|
409
|
-
// A quarantined message has no stored body, so nothing else here would
|
|
410
|
-
// stop it being fetched and re-parsed on every redelivery, forever.
|
|
411
|
-
if (
|
|
412
|
-
this.quarantineConfig &&
|
|
413
|
-
quarantined?.has(
|
|
414
|
-
this.quarantineConfig.mailboxId,
|
|
415
|
-
this.quarantineConfig.uidValidity,
|
|
416
|
-
message.uid,
|
|
417
|
-
)
|
|
418
|
-
) {
|
|
419
|
-
this.log.debug?.(
|
|
420
|
-
{ messageId, uid: message.uid },
|
|
421
|
-
"Message is quarantined, skipping",
|
|
422
|
-
);
|
|
423
|
-
skippedCount++;
|
|
424
|
-
continue;
|
|
425
|
-
}
|
|
426
|
-
|
|
427
|
-
// The body is stored, so there is nothing to fetch (issue #331). This
|
|
428
|
-
// used to re-read those bytes from storage to re-derive a category,
|
|
429
|
-
// keyed on `category === uncategorized` — which is also the classifier's
|
|
430
|
-
// own "nothing to say", so a message it declined cost one storage read
|
|
431
|
-
// and two writes on every pass, forever, for the same answer. What the
|
|
432
|
-
// classifier did is now recorded as `classificationState`, so a sync
|
|
433
|
-
// pass never has to guess it from the row, and the cohort it has not
|
|
434
|
-
// reached is selectable by a backfill instead of re-derived here.
|
|
435
|
-
if (hasStoredBody(message.bodyStorageKey) && !force) {
|
|
436
|
-
this.log.debug?.({ messageId }, "Body already stored, skipping");
|
|
437
|
-
skippedCount++;
|
|
438
|
-
continue;
|
|
439
|
-
}
|
|
440
|
-
if (message.bodyStorageKey) reStoredMessageIds.add(messageId);
|
|
441
|
-
pending.set(message.uid, messageId);
|
|
442
|
-
}
|
|
443
|
-
|
|
444
|
-
if (pending.size === 0) {
|
|
445
|
-
return this.buildResult(syncedMessageIds, skippedCount, []);
|
|
446
|
-
}
|
|
447
|
-
|
|
448
|
-
const connection = await getConnection();
|
|
449
|
-
// Single SELECT for the whole batch. openBox is idempotent, so a warm
|
|
450
|
-
// connection already on this mailbox skips the SELECT entirely.
|
|
451
|
-
await connection.openBox(folder.fullPath);
|
|
452
|
-
|
|
453
|
-
let connectionLost = false;
|
|
454
|
-
try {
|
|
455
|
-
for await (const { uid, source } of connection.fetchMessageBodies([
|
|
456
|
-
...pending.keys(),
|
|
457
|
-
])) {
|
|
458
|
-
const messageId = pending.get(uid);
|
|
459
|
-
if (!messageId) {
|
|
460
|
-
// A UID we didn't ask for — drain the stream so the connection
|
|
461
|
-
// stays usable, then ignore it.
|
|
462
|
-
source.resume();
|
|
463
|
-
continue;
|
|
464
|
-
}
|
|
465
|
-
try {
|
|
466
|
-
await this.storeStreamedBody(
|
|
467
|
-
messageId,
|
|
468
|
-
accountId,
|
|
469
|
-
accountConfigId,
|
|
470
|
-
source,
|
|
471
|
-
folder.placement,
|
|
472
|
-
reStoredMessageIds.has(messageId),
|
|
473
|
-
);
|
|
474
|
-
} catch (error) {
|
|
475
|
-
// A per-message store failure (e.g. the parsed-body S3 write) is a
|
|
476
|
-
// real fault, but it must not silently mark the message synced —
|
|
477
|
-
// that would leave parsedBody null and the message unindexed. Leave
|
|
478
|
-
// the UID in `pending` so it lands in failedMessageIds and the SQS
|
|
479
|
-
// batch requeues just this message. A dropped connection is handled
|
|
480
|
-
// by the outer catch (fail-fast on the whole remaining batch).
|
|
481
|
-
if (isConnectionDrop(error)) throw error;
|
|
482
|
-
|
|
483
|
-
// The one error in this frame that is the message's own fault. It
|
|
484
|
-
// reaches here from the parse call and nowhere else, so the S3, DDB
|
|
485
|
-
// and SQS work this same frame wraps cannot be mistaken for it and
|
|
486
|
-
// keeps requeueing below. The row is durable before the UID leaves
|
|
487
|
-
// `pending`, so the message is never let go of unrecorded.
|
|
488
|
-
if (
|
|
489
|
-
error instanceof BodyParseError &&
|
|
490
|
-
(await this.quarantineBodyParse(messageId, location, error))
|
|
491
|
-
) {
|
|
492
|
-
pending.delete(uid);
|
|
493
|
-
skippedCount++;
|
|
494
|
-
source.resume();
|
|
495
|
-
continue;
|
|
496
|
-
}
|
|
497
|
-
|
|
498
|
-
this.log.error?.(
|
|
499
|
-
{
|
|
500
|
-
messageId,
|
|
501
|
-
errorName: (error as { name?: string }).name,
|
|
502
|
-
error: (error as Error).message,
|
|
503
|
-
},
|
|
504
|
-
"Body store failed for message; leaving for requeue",
|
|
505
|
-
);
|
|
506
|
-
// Drain the stream so the connection stays usable for the next UID.
|
|
507
|
-
source.resume();
|
|
508
|
-
continue;
|
|
509
|
-
}
|
|
510
|
-
pending.delete(uid);
|
|
511
|
-
syncedMessageIds.push(messageId);
|
|
512
|
-
}
|
|
513
|
-
} catch (error) {
|
|
514
|
-
this.log.error?.(
|
|
515
|
-
{ error: (error as Error).message },
|
|
516
|
-
"Body fetch stream failed",
|
|
517
|
-
);
|
|
518
|
-
// Fail-fast: a dropped connection mid-stream leaves every not-yet-yielded
|
|
519
|
-
// UID in `pending`; they fall through to failedMessageIds and re-enqueue.
|
|
520
|
-
// Any other error is a real fault — let it crash.
|
|
521
|
-
if (!isConnectionDrop(error)) {
|
|
522
|
-
throw error;
|
|
523
|
-
}
|
|
524
|
-
connectionLost = true;
|
|
525
|
-
this.log.info?.(
|
|
526
|
-
{ remainingCount: pending.size },
|
|
527
|
-
"Connection lost, aborting batch",
|
|
528
|
-
);
|
|
529
|
-
}
|
|
530
|
-
|
|
531
|
-
// Anything still pending was never yielded (mid-stream drop or a UID the
|
|
532
|
-
// server silently omitted) — re-enqueue it.
|
|
533
|
-
const failedMessageIds = [...pending.values()];
|
|
534
|
-
|
|
535
|
-
this.log.info(
|
|
536
|
-
{
|
|
537
|
-
synced: syncedMessageIds.length,
|
|
538
|
-
skipped: skippedCount,
|
|
539
|
-
failed: failedMessageIds.length,
|
|
540
|
-
total: messageIds.length,
|
|
541
|
-
aborted: connectionLost,
|
|
542
|
-
},
|
|
543
|
-
"Body sync complete",
|
|
544
|
-
);
|
|
545
|
-
|
|
546
|
-
return this.buildResult(syncedMessageIds, skippedCount, failedMessageIds);
|
|
547
|
-
}
|
|
548
|
-
|
|
549
|
-
/**
|
|
550
|
-
* Set a message aside because its body will not parse (issue #72).
|
|
551
|
-
*
|
|
552
|
-
* Returns false when no record was written — either because this service was
|
|
553
|
-
* built without a quarantine writer, or because writing it failed. The
|
|
554
|
-
* caller then falls back to requeueing rather than dropping a message no
|
|
555
|
-
* record survives. The failure the row carries is the parser's, and only the
|
|
556
|
-
* parser's — see {@link BodyParseError}.
|
|
557
|
-
*
|
|
558
|
-
* Reading and writing the record is storage and database work, so its own
|
|
559
|
-
* failure is contained per message the same way the save it replaces is: it
|
|
560
|
-
* must not take down a batch that has other messages to fetch, and in the
|
|
561
|
-
* stream loop a throw would also skip `source.resume()` and leave the
|
|
562
|
-
* literal undrained on an otherwise usable connection.
|
|
563
|
-
*
|
|
564
|
-
* The fingerprint comes from the BodyPart rows metadata sync already wrote:
|
|
565
|
-
* the body path streams raw bytes and has no FETCH result to read, and
|
|
566
|
-
* re-reading headers the parser has just refused would be reading the very
|
|
567
|
-
* thing that is broken.
|
|
568
|
-
*/
|
|
569
|
-
private async quarantineBodyParse(
|
|
570
|
-
messageId: string,
|
|
571
|
-
location: {
|
|
572
|
-
accountId: string;
|
|
573
|
-
accountConfigId: string;
|
|
574
|
-
mailboxPath: string;
|
|
575
|
-
},
|
|
576
|
-
error: BodyParseError,
|
|
577
|
-
): Promise<boolean> {
|
|
578
|
-
const config = this.quarantineConfig;
|
|
579
|
-
if (!config) return false;
|
|
580
|
-
|
|
581
|
-
return this.recordQuarantine(config, messageId, location, error).then(
|
|
582
|
-
() => true,
|
|
583
|
-
(writeError: unknown) => {
|
|
584
|
-
this.log.error?.(
|
|
585
|
-
{
|
|
586
|
-
messageId,
|
|
587
|
-
errorName: (writeError as { name?: string }).name,
|
|
588
|
-
error: inspect(writeError),
|
|
589
|
-
},
|
|
590
|
-
"Could not record quarantine; leaving the message for requeue",
|
|
591
|
-
);
|
|
592
|
-
return false;
|
|
593
|
-
},
|
|
594
|
-
);
|
|
595
|
-
}
|
|
596
|
-
|
|
597
|
-
private async recordQuarantine(
|
|
598
|
-
config: QuarantineConfig,
|
|
599
|
-
messageId: string,
|
|
600
|
-
location: {
|
|
601
|
-
accountId: string;
|
|
602
|
-
accountConfigId: string;
|
|
603
|
-
mailboxPath: string;
|
|
604
|
-
},
|
|
605
|
-
error: BodyParseError,
|
|
606
|
-
): Promise<void> {
|
|
607
|
-
const message = await this.messageService.get(messageId);
|
|
608
|
-
const messageData = await this.envelopeService.getMessageData(messageId);
|
|
609
|
-
|
|
610
|
-
await config.quarantineService.record(
|
|
611
|
-
{
|
|
612
|
-
accountId: location.accountId,
|
|
613
|
-
accountConfigId: location.accountConfigId,
|
|
614
|
-
mailboxId: config.mailboxId,
|
|
615
|
-
mailboxPath: location.mailboxPath,
|
|
616
|
-
uidValidity: config.uidValidity,
|
|
617
|
-
attempts: config.attempts,
|
|
618
|
-
},
|
|
619
|
-
message.uid,
|
|
620
|
-
{
|
|
621
|
-
stage: QuarantineFailureStage.BodyParse,
|
|
622
|
-
code: error.failureCode,
|
|
623
|
-
message: error.message,
|
|
624
|
-
},
|
|
625
|
-
shapeFromMessageData(message, messageData),
|
|
626
|
-
);
|
|
627
|
-
}
|
|
628
|
-
|
|
629
|
-
private buildResult(
|
|
630
|
-
syncedMessageIds: string[],
|
|
631
|
-
skippedCount: number,
|
|
632
|
-
failedMessageIds: string[],
|
|
633
|
-
): SyncBodiesResult {
|
|
634
|
-
return {
|
|
635
|
-
syncedCount: syncedMessageIds.length,
|
|
636
|
-
syncedMessageIds,
|
|
637
|
-
skippedCount,
|
|
638
|
-
failedCount: failedMessageIds.length,
|
|
639
|
-
failedMessageIds,
|
|
640
|
-
};
|
|
641
|
-
}
|
|
642
|
-
|
|
643
|
-
/**
|
|
644
|
-
* Stream one message body straight to storage while teeing the bytes into a
|
|
645
|
-
* buffer for the parse-dependent steps (snippet, classification, parsed-body
|
|
646
|
-
* cache, per-part objects). The S3 upload never sees a whole-body concat —
|
|
647
|
-
* the storage service streams it — but mailparser still needs the full bytes,
|
|
648
|
-
* so we collect them in parallel. Later issues move parsing off the hot path.
|
|
649
|
-
*
|
|
650
|
-
* `isReStore` marks a re-fetch of a message whose row already carried
|
|
651
|
-
* `bodyStorageKey` (the `force` re-arm in {@link syncBodies}): the
|
|
652
|
-
* post-store steps then re-materialize the body artifacts without
|
|
653
|
-
* re-running any decision (issue #1011).
|
|
654
|
-
*/
|
|
655
|
-
private async storeStreamedBody(
|
|
656
|
-
messageId: string,
|
|
657
|
-
accountId: string,
|
|
658
|
-
accountConfigId: string,
|
|
659
|
-
source: Readable,
|
|
660
|
-
placement: FolderPlacement,
|
|
661
|
-
isReStore: boolean,
|
|
662
|
-
): Promise<void> {
|
|
663
|
-
const toStorage = new PassThrough();
|
|
664
|
-
const chunks: Buffer[] = [];
|
|
665
|
-
|
|
666
|
-
// Tee the source: bytes flow to storage as a stream (no whole-body concat
|
|
667
|
-
// on the upload path) while we also collect them for the parse-dependent
|
|
668
|
-
// steps below, which still need the full body for mailparser. The store
|
|
669
|
-
// and the tee are awaited together so neither rejection is orphaned.
|
|
670
|
-
const tee = new Promise<void>((resolve, reject) => {
|
|
671
|
-
source.on("data", (chunk: Buffer) => {
|
|
672
|
-
chunks.push(chunk);
|
|
673
|
-
toStorage.write(chunk);
|
|
674
|
-
});
|
|
675
|
-
source.on("end", () => {
|
|
676
|
-
toStorage.end();
|
|
677
|
-
resolve();
|
|
678
|
-
});
|
|
679
|
-
source.on("error", (err) => {
|
|
680
|
-
toStorage.destroy(err);
|
|
681
|
-
reject(err);
|
|
682
|
-
});
|
|
683
|
-
});
|
|
684
|
-
|
|
685
|
-
const [ref] = await Promise.all([
|
|
686
|
-
this.storageService.storeMessageBodyStream({
|
|
687
|
-
accountConfigId,
|
|
688
|
-
accountId,
|
|
689
|
-
messageId,
|
|
690
|
-
content: toStorage,
|
|
691
|
-
}),
|
|
692
|
-
tee,
|
|
693
|
-
]);
|
|
694
|
-
|
|
695
|
-
const body = Buffer.concat(chunks);
|
|
696
|
-
|
|
697
|
-
await this.applyPostStoreSteps(
|
|
698
|
-
messageId,
|
|
699
|
-
accountId,
|
|
700
|
-
accountConfigId,
|
|
701
|
-
body,
|
|
702
|
-
{
|
|
703
|
-
uri: ref.uri,
|
|
704
|
-
},
|
|
705
|
-
placement,
|
|
706
|
-
isReStore,
|
|
707
|
-
);
|
|
708
|
-
}
|
|
709
|
-
|
|
710
|
-
/**
|
|
711
|
-
* The steps every synced body must go through regardless of which path
|
|
712
|
-
* fetched it first — the sync path ({@link storeStreamedBody}) and the
|
|
713
|
-
* read-path backfill ({@link fetchAndGetBody}) both call this instead of
|
|
714
|
-
* duplicating (and drifting on) their own subset. Before issue #1271,
|
|
715
|
-
* `fetchAndGetBody` skipped classification/placement entirely — a message
|
|
716
|
-
* materialized by a read got different treatment than one synced in bulk.
|
|
717
|
-
*
|
|
718
|
-
* Order matters for two independent reasons, both load-bearing:
|
|
719
|
-
* 1. The parsed-body cache must be durable before `bodyStorageKey` (the
|
|
720
|
-
* skip-guard signal) — see the comment on the write below.
|
|
721
|
-
* 2. A placement move — local mailboxId change, pending marker, IMAP-push
|
|
722
|
-
* enqueue (`PlacementMoveService.moveMessage`, issue #1271) — must
|
|
723
|
-
* complete, or fully fail, BEFORE `bodyStorageKey` is written. That
|
|
724
|
-
* failure is never swallowed: if it throws, `bodyStorageKey` is not yet
|
|
725
|
-
* durable, so a retry reprocesses the message from scratch (marker
|
|
726
|
-
* included) instead of the skip-guard stranding a `movedByRemit` flag
|
|
727
|
-
* with no marker behind it — the defect this issue fixes.
|
|
728
|
-
*
|
|
729
|
-
* Returns the parsed mail so callers that already need it (both do) don't
|
|
730
|
-
* pay for mailparser twice.
|
|
731
|
-
*/
|
|
732
|
-
private async applyPostStoreSteps(
|
|
733
|
-
messageId: string,
|
|
734
|
-
accountId: string,
|
|
735
|
-
accountConfigId: string,
|
|
736
|
-
body: Buffer,
|
|
737
|
-
bodyRef: { uri: string },
|
|
738
|
-
placement: FolderPlacement,
|
|
739
|
-
/**
|
|
740
|
-
* Whether this call is a re-store of an already-classified message
|
|
741
|
-
* rather than a first store. The two re-entrant paths (`force: true`
|
|
742
|
-
* in `syncBodies`, and `fetchAndGetBody`'s `NoSuchKey` IMAP fallback) are
|
|
743
|
-
* both body-only re-fetches: a body re-sync fetches and stores bytes and
|
|
744
|
-
* runs no decision logic (issue #1011). When `isReStore` is true the
|
|
745
|
-
* placement/filter/category machinery is skipped entirely and the
|
|
746
|
-
* message's existing category is carried forward unchanged.
|
|
747
|
-
*/
|
|
748
|
-
isReStore: boolean,
|
|
749
|
-
): Promise<ParsedMail> {
|
|
750
|
-
// The one operation on this path that can fail because of how the message
|
|
751
|
-
// is built. Its own try block lives in `parseMessageBody`, so this frame —
|
|
752
|
-
// which also wraps the S3 body write, the parsed-body cache, the placement
|
|
753
|
-
// move, the label writes and the counter update — can tell the two apart
|
|
754
|
-
// and quarantine only this one (issue #72). The ThreadMessage write these
|
|
755
|
-
// feed happens later, alongside the Message write (see below), once the
|
|
756
|
-
// write-once category is decided.
|
|
757
|
-
const parsed = await parseMessageBody(body);
|
|
758
|
-
|
|
759
|
-
// Body-only re-fetch (issue #1011): the decision pass already ran when
|
|
760
|
-
// this message's body first landed, so a re-store re-materializes only
|
|
761
|
-
// the body-derived artifacts — parsed cache, body parts, the storage
|
|
762
|
-
// key — and re-decides nothing: no classification, placement, filters,
|
|
763
|
-
// labels, read state, and no second engagement-counter increment.
|
|
764
|
-
// Skipping the decisions here is what lets the write-once fields stay
|
|
765
|
-
// safe without per-derivation re-entrancy guards.
|
|
766
|
-
if (isReStore) {
|
|
767
|
-
// Same ordering rule as the first-store path below: the parsed-body
|
|
768
|
-
// cache must be durable before `bodyStorageKey` — the skip-guard
|
|
769
|
-
// signal — is written, so a failure here requeues the message and
|
|
770
|
-
// the retry genuinely re-attempts the artifact writes.
|
|
771
|
-
await this.storeParsedBodyCache(
|
|
772
|
-
accountConfigId,
|
|
773
|
-
accountId,
|
|
774
|
-
messageId,
|
|
775
|
-
parsed,
|
|
776
|
-
);
|
|
777
|
-
if (!isBodyPartDeferralEnabled()) {
|
|
778
|
-
await this.storeBodyPartContents(
|
|
779
|
-
accountConfigId,
|
|
780
|
-
accountId,
|
|
781
|
-
messageId,
|
|
782
|
-
parsed,
|
|
783
|
-
);
|
|
784
|
-
}
|
|
785
|
-
await this.messageService.update(messageId, {
|
|
786
|
-
bodyStorageKey: bodyRef.uri,
|
|
787
|
-
});
|
|
788
|
-
this.log.info(
|
|
789
|
-
{ messageId, storageKey: bodyRef.uri },
|
|
790
|
-
"Body re-stored without re-running decisions",
|
|
791
|
-
);
|
|
792
|
-
return parsed;
|
|
793
|
-
}
|
|
794
|
-
|
|
795
|
-
const snippet = extractSnippet(parsed);
|
|
796
|
-
const listId = extractListId(parsed);
|
|
797
|
-
|
|
798
|
-
// Compute the header classification once, folding in the sender's
|
|
799
|
-
// `flags.category` override when one applies (issue #299). The derived
|
|
800
|
-
// fields are folded into the single Message update below — they are NOT
|
|
801
|
-
// written here.
|
|
802
|
-
const classification = await this.classifyMessage(
|
|
803
|
-
accountConfigId,
|
|
804
|
-
parsed,
|
|
805
|
-
placement,
|
|
806
|
-
);
|
|
807
|
-
|
|
808
|
-
// Decide the placement move from the in-memory classification before the
|
|
809
|
-
// write, so its `movedByRemit` flag and audit verdict join the same
|
|
810
|
-
// UpdateItem. The verdict is recorded whenever Remit decided to act
|
|
811
|
-
// (action != leave); the move is enqueued only for a confident,
|
|
812
|
-
// actionable verdict.
|
|
813
|
-
const resolved = await this.resolvePlacement(
|
|
814
|
-
messageId,
|
|
815
|
-
accountId,
|
|
816
|
-
accountConfigId,
|
|
817
|
-
parsed,
|
|
818
|
-
classification,
|
|
819
|
-
);
|
|
820
|
-
|
|
821
|
-
// Index-time filter pass (RFC 034). Isolated the same way placement is —
|
|
822
|
-
// FilterPipeline.evaluate swallows any read/decision failure and returns
|
|
823
|
-
// no actions, so a schema-drifted filter query never fails the body store.
|
|
824
|
-
// The DECISION is computed here; the applies below are unswallowed.
|
|
825
|
-
const filterDecision = await this.evaluateFilters(
|
|
826
|
-
accountConfigId,
|
|
827
|
-
messageId,
|
|
828
|
-
parsed,
|
|
829
|
-
);
|
|
830
|
-
|
|
831
|
-
// Store the parsed-body cache BEFORE persisting bodyStorageKey. The skip
|
|
832
|
-
// guard in syncBodies treats a stored bodyStorageKey as "fully synced",
|
|
833
|
-
// so if we wrote it first and the parsed-cache write then failed, the
|
|
834
|
-
// requeued retry would skip the message — leaving parsedBody null and the
|
|
835
|
-
// search-index upsert bodyless forever. A parsed-cache failure here throws
|
|
836
|
-
// and propagates before bodyStorageKey is written, so the requeued message
|
|
837
|
-
// genuinely re-attempts the parsed write and gets indexed.
|
|
838
|
-
await this.storeParsedBodyCache(
|
|
839
|
-
accountConfigId,
|
|
840
|
-
accountId,
|
|
841
|
-
messageId,
|
|
842
|
-
parsed,
|
|
843
|
-
);
|
|
844
|
-
|
|
845
|
-
// Per-part S3 objects dominate S3 writes during bulk sync. When deferral
|
|
846
|
-
// is enabled we skip them here; they are materialized lazily on the first
|
|
847
|
-
// `contentUrl` read (see ensureBodyPartsStored), keeping bulk sync at 2
|
|
848
|
-
// writes/message (body.eml + parsed.json.gz).
|
|
849
|
-
if (!isBodyPartDeferralEnabled()) {
|
|
850
|
-
await this.storeBodyPartContents(
|
|
851
|
-
accountConfigId,
|
|
852
|
-
accountId,
|
|
853
|
-
messageId,
|
|
854
|
-
parsed,
|
|
855
|
-
);
|
|
856
|
-
}
|
|
857
|
-
|
|
858
|
-
// Local-first move (issue #1271): local mailboxId change + pending marker +
|
|
859
|
-
// IMAP-push enqueue via PlacementMoveService, UNSWALLOWED — a throw here
|
|
860
|
-
// propagates before bodyStorageKey is written (see the method doc above),
|
|
861
|
-
// so a genuine SQS/DDB write failure requeues the message rather than being
|
|
862
|
-
// absorbed. A matched filter's move is exclusive and outranks the
|
|
863
|
-
// classifier's placement move (RFC 034 Decision 3.1) — an explicit user
|
|
864
|
-
// rule wins the single mailbox a message occupies — so at most one move is
|
|
865
|
-
// enqueued. A non-confident/leave verdict with no filter move is a no-op.
|
|
866
|
-
let filterMove: MessageFilterMove | undefined;
|
|
867
|
-
let filterMoved = false;
|
|
868
|
-
if (filterDecision.move) {
|
|
869
|
-
const destinationMailboxId = filterDecision.move.destinationMailboxId;
|
|
870
|
-
// Capture the source mailbox BEFORE the move rewrites mailboxId, so the
|
|
871
|
-
// auto-moved badge can name where the filter took the message from and
|
|
872
|
-
// offer an undo back to it (issue #223). Snapshotted into the marker
|
|
873
|
-
// rather than read from `originalMailboxId` at derive time, which a
|
|
874
|
-
// later user move would overwrite — the filter's source must survive an
|
|
875
|
-
// undo to stay derivable.
|
|
876
|
-
const sourceMailboxId = (await this.messageService.get(messageId))
|
|
877
|
-
.mailboxId;
|
|
878
|
-
await this.filterConfig?.placementMoveService.moveMessage(
|
|
879
|
-
accountConfigId,
|
|
880
|
-
messageId,
|
|
881
|
-
destinationMailboxId,
|
|
882
|
-
accountId,
|
|
883
|
-
);
|
|
884
|
-
// A filter matches by content, not location: a message delivered
|
|
885
|
-
// straight into the destination (a server-side rule) or already filed
|
|
886
|
-
// there matches but was not moved by Remit. `moveMessage` no-ops in that
|
|
887
|
-
// case; record neither the marker nor `movedByRemit`, so the badge never
|
|
888
|
-
// claims a move that did not happen and its Undo is never a no-op.
|
|
889
|
-
if (sourceMailboxId !== destinationMailboxId) {
|
|
890
|
-
filterMoved = true;
|
|
891
|
-
filterMove = {
|
|
892
|
-
filterId: filterDecision.move.filterId,
|
|
893
|
-
sourceMailboxId,
|
|
894
|
-
destinationMailboxId,
|
|
895
|
-
decidedAt: Date.now(),
|
|
896
|
-
};
|
|
897
|
-
}
|
|
898
|
-
} else if (resolved.move) {
|
|
899
|
-
await this.placementConfig?.placementMoveService.moveMessage(
|
|
900
|
-
accountConfigId,
|
|
901
|
-
messageId,
|
|
902
|
-
resolved.move.destinationMailboxId,
|
|
903
|
-
accountId,
|
|
904
|
-
);
|
|
905
|
-
}
|
|
906
|
-
|
|
907
|
-
// Additive label actions (RFC 034 Decision 3.1): every matching filter's
|
|
908
|
-
// label applies. The deterministic MessageLabel upsert is idempotent, so a
|
|
909
|
-
// requeue after a later failure re-applies safely; UNSWALLOWED, so a DDB
|
|
910
|
-
// write failure propagates before bodyStorageKey is written.
|
|
911
|
-
for (const label of filterDecision.labels) {
|
|
912
|
-
await this.filterConfig?.messageLabelService.apply({
|
|
913
|
-
messageId,
|
|
914
|
-
labelId: label.labelId,
|
|
915
|
-
accountConfigId,
|
|
916
|
-
appliedByFilterId: label.filterId,
|
|
917
|
-
});
|
|
918
|
-
}
|
|
919
|
-
|
|
920
|
-
// Read once for the two write-once guards below — the auto-read decision
|
|
921
|
-
// and the category carry-forward. Taken here rather than earlier because
|
|
922
|
-
// nothing between this point and the Message update writes either field;
|
|
923
|
-
// the moves above touch `mailboxId` only.
|
|
924
|
-
const existingMessage = await this.messageService.get(messageId);
|
|
925
|
-
|
|
926
|
-
// `flags.unsubscribed` (issue #302, RFC 039 Decision 3): auto-mark-read,
|
|
927
|
-
// reusing the same FlagQueueService.markAsRead round-trip a manual
|
|
928
|
-
// mark-as-read goes through — idempotent on a retry (flipFlag no-ops when
|
|
929
|
-
// the message is already \Seen), so a failure here safely re-fires on the
|
|
930
|
-
// next attempt rather than being lost behind the bodyStorageKey skip guard.
|
|
931
|
-
await this.applyUnsubscribedAutoRead(
|
|
932
|
-
messageId,
|
|
933
|
-
accountId,
|
|
934
|
-
accountConfigId,
|
|
935
|
-
parsed,
|
|
936
|
-
existingMessage.bodyStorageKey,
|
|
937
|
-
);
|
|
938
|
-
|
|
939
|
-
await this.deriveCalendarSuggestions(
|
|
940
|
-
messageId,
|
|
941
|
-
accountConfigId,
|
|
942
|
-
parsed,
|
|
943
|
-
existingMessage.bodyStorageKey,
|
|
944
|
-
);
|
|
945
|
-
|
|
946
|
-
const moved = Boolean(resolved.move || filterMoved);
|
|
947
|
-
|
|
948
|
-
// ONE Message UpdateItem per synced message: bodyStorageKey + every
|
|
949
|
-
// classification/derived field + the move flag + the audit verdict. Each
|
|
950
|
-
// extra Message mutation emits a DDB stream record that fans out to a
|
|
951
|
-
// redundant S3-Vectors upsert, so we collapse them into a single write.
|
|
952
|
-
// The flag is folded in only when a move WAS applied (by the classifier or
|
|
953
|
-
// a filter); the verdict is folded in whenever Remit decided to act.
|
|
954
|
-
// Written LAST so bodyStorageKey — the skip-guard signal — is only durable
|
|
955
|
-
// once the parsed cache AND the move (when any) are.
|
|
956
|
-
//
|
|
957
|
-
// `category` is RFC 034 D3.1's write-once field (RFC 030's message-list
|
|
958
|
-
// GSI sort key depends on it never churning). The re-entrant re-store
|
|
959
|
-
// paths (issue #1011) return before this write; the carry-forward keeps
|
|
960
|
-
// the field write-once even when a first store lands on a row that
|
|
961
|
-
// already carries a real category — a previously-decided category wins
|
|
962
|
-
// over the just-recomputed one. It likewise protects a `flags.category`
|
|
963
|
-
// override (issue #299): recomputing would let a *later* override silently
|
|
964
|
-
// rewrite a category already decided, which is exactly the churn RFC
|
|
965
|
-
// 030's GSI-safety argument forbids.
|
|
966
|
-
//
|
|
967
|
-
// `resolved.placementDecidedAt` (issue #383) is only ever present on
|
|
968
|
-
// the pass that first decided it: the re-store paths skip placement
|
|
969
|
-
// entirely (issue #1011), so `computePlacement` is reached only on a
|
|
970
|
-
// first store.
|
|
971
|
-
const finalCategory = hasDecidedCategory(existingMessage.category)
|
|
972
|
-
? existingMessage.category
|
|
973
|
-
: classification.category;
|
|
974
|
-
|
|
975
|
-
// Thread-list denormalization, using the same write-once category as the
|
|
976
|
-
// Message update below — the two rows must never disagree on category.
|
|
977
|
-
await denormalizeMessageCategory(
|
|
978
|
-
{ threadMessageService: this.threadMessageService },
|
|
979
|
-
accountConfigId,
|
|
980
|
-
messageId,
|
|
981
|
-
{ category: finalCategory, snippet, listId },
|
|
982
|
-
);
|
|
983
|
-
|
|
984
|
-
// `classificationState` records that the classifier ran, in the same
|
|
985
|
-
// UpdateItem as the answer it gave (issue #331). Without it `uncategorized`
|
|
986
|
-
// carries two meanings at once — not reached yet, and reached with nothing
|
|
987
|
-
// to say — and only one of them should ever be looked at again.
|
|
988
|
-
const update: UpdateMessageInput = {
|
|
989
|
-
bodyStorageKey: bodyRef.uri,
|
|
990
|
-
...classification,
|
|
991
|
-
category: finalCategory,
|
|
992
|
-
classificationState: MessageClassificationState.Examined,
|
|
993
|
-
...(moved ? { movedByRemit: true } : {}),
|
|
994
|
-
...(resolved.verdict ? { placementVerdict: resolved.verdict } : {}),
|
|
995
|
-
...(filterMove ? { filterMove } : {}),
|
|
996
|
-
...(resolved.placementDecidedAt
|
|
997
|
-
? { placementDecidedAt: resolved.placementDecidedAt }
|
|
998
|
-
: {}),
|
|
999
|
-
};
|
|
1000
|
-
await this.messageService.update(messageId, update);
|
|
1001
|
-
this.log.info({ messageId, storageKey: bodyRef.uri }, "Body stored");
|
|
1002
|
-
|
|
1003
|
-
// From-Address engagement counter (Address entity, not Message).
|
|
1004
|
-
await this.incrementInboundCount(
|
|
1005
|
-
messageId,
|
|
1006
|
-
accountConfigId,
|
|
1007
|
-
parsed,
|
|
1008
|
-
classification,
|
|
1009
|
-
);
|
|
1010
|
-
|
|
1011
|
-
return parsed;
|
|
1012
|
-
}
|
|
1013
|
-
|
|
1014
|
-
/**
|
|
1015
|
-
* Fetch a single message body, store it, and return the parsed content.
|
|
1016
|
-
*
|
|
1017
|
-
* If the body is already stored, retrieves it from storage.
|
|
1018
|
-
* Otherwise fetches from IMAP, stores it, and returns the parsed content.
|
|
1019
|
-
*
|
|
1020
|
-
* @param messageId - The message ID to fetch
|
|
1021
|
-
* @param accountId - The account ID (for storage path)
|
|
1022
|
-
* @param accountConfigId - The account config ID (for thread updates)
|
|
1023
|
-
* @param folder - The IMAP mailbox path and its folder placement
|
|
1024
|
-
* @param getConnection - Function to get a (lazy) IMAP connection
|
|
1025
|
-
* @returns Parsed text and HTML content
|
|
1026
|
-
*/
|
|
1027
|
-
async fetchAndGetBody(
|
|
1028
|
-
messageId: string,
|
|
1029
|
-
accountId: string,
|
|
1030
|
-
accountConfigId: string,
|
|
1031
|
-
folder: BodySyncFolder,
|
|
1032
|
-
getConnection: ConnectionGetter,
|
|
1033
|
-
): Promise<FetchBodyResult> {
|
|
1034
|
-
const message = await this.messageService.get(messageId);
|
|
1035
|
-
|
|
1036
|
-
let body: Buffer;
|
|
1037
|
-
let needsStore = false;
|
|
1038
|
-
|
|
1039
|
-
if (message.bodyStorageKey) {
|
|
1040
|
-
// Body already stored, try to retrieve from storage
|
|
1041
|
-
this.log.debug?.({ messageId }, "Retrieving body from storage");
|
|
1042
|
-
try {
|
|
1043
|
-
body = await this.storageService.retrieve(message.bodyStorageKey);
|
|
1044
|
-
} catch (err) {
|
|
1045
|
-
// Only a genuinely-missing object (NoSuchKey) is a safe IMAP
|
|
1046
|
-
// fallback — that's the cross-environment / never-stored case.
|
|
1047
|
-
// A permission/infra error (AccessDenied, throttle) must NOT be
|
|
1048
|
-
// masked as a missing object: let it crash so it's observable.
|
|
1049
|
-
if (!isStorageNotFoundError(err)) {
|
|
1050
|
-
this.log.error?.(
|
|
1051
|
-
{
|
|
1052
|
-
messageId,
|
|
1053
|
-
storageKey: message.bodyStorageKey,
|
|
1054
|
-
errorName: (err as { name?: string }).name,
|
|
1055
|
-
errorCode: (err as { Code?: string }).Code,
|
|
1056
|
-
error: inspect(err),
|
|
1057
|
-
},
|
|
1058
|
-
"Body storage retrieval failed (non-NoSuchKey); not falling back to IMAP",
|
|
1059
|
-
);
|
|
1060
|
-
throw err;
|
|
1061
|
-
}
|
|
1062
|
-
this.log.debug?.(
|
|
1063
|
-
{ messageId, error: (err as Error).message },
|
|
1064
|
-
"Body object missing (NoSuchKey), falling back to IMAP",
|
|
1065
|
-
);
|
|
1066
|
-
needsStore = true;
|
|
1067
|
-
body = await this.fetchFromImap(
|
|
1068
|
-
message,
|
|
1069
|
-
accountId,
|
|
1070
|
-
accountConfigId,
|
|
1071
|
-
folder.fullPath,
|
|
1072
|
-
getConnection,
|
|
1073
|
-
);
|
|
1074
|
-
}
|
|
1075
|
-
} else {
|
|
1076
|
-
// Fetch from IMAP and store
|
|
1077
|
-
needsStore = true;
|
|
1078
|
-
body = await this.fetchFromImap(
|
|
1079
|
-
message,
|
|
1080
|
-
accountId,
|
|
1081
|
-
accountConfigId,
|
|
1082
|
-
folder.fullPath,
|
|
1083
|
-
getConnection,
|
|
1084
|
-
);
|
|
1085
|
-
}
|
|
1086
|
-
|
|
1087
|
-
let parsed: ParsedMail;
|
|
1088
|
-
|
|
1089
|
-
if (needsStore) {
|
|
1090
|
-
const ref = await this.storageService.storeMessageBody({
|
|
1091
|
-
accountConfigId,
|
|
1092
|
-
accountId,
|
|
1093
|
-
messageId,
|
|
1094
|
-
content: body,
|
|
1095
|
-
});
|
|
1096
|
-
|
|
1097
|
-
// Same shared step the sync path (storeStreamedBody) runs — issue
|
|
1098
|
-
// #1271 unified the two body paths so classification/placement no
|
|
1099
|
-
// longer depends on which one fetched the body first. A store from
|
|
1100
|
-
// here is a re-store exactly when the row already claimed a stored
|
|
1101
|
-
// body — the NoSuchKey fallback above; a row with no bodyStorageKey is
|
|
1102
|
-
// a genuine first store and takes the full decision pass.
|
|
1103
|
-
parsed = await this.applyPostStoreSteps(
|
|
1104
|
-
messageId,
|
|
1105
|
-
accountId,
|
|
1106
|
-
accountConfigId,
|
|
1107
|
-
body,
|
|
1108
|
-
{ uri: ref.uri },
|
|
1109
|
-
folder.placement,
|
|
1110
|
-
message.bodyStorageKey != null,
|
|
1111
|
-
);
|
|
1112
|
-
} else {
|
|
1113
|
-
parsed = await simpleParser(body);
|
|
1114
|
-
}
|
|
1115
|
-
|
|
1116
|
-
return {
|
|
1117
|
-
text: parsed.text ?? null,
|
|
1118
|
-
html: typeof parsed.html === "string" ? parsed.html : null,
|
|
1119
|
-
storedAt: message.bodyStorageKey ?? "newly-stored",
|
|
1120
|
-
};
|
|
1121
|
-
}
|
|
1122
|
-
|
|
1123
|
-
private async fetchFromImap(
|
|
1124
|
-
message: MessageItem,
|
|
1125
|
-
accountId: string,
|
|
1126
|
-
accountConfigId: string,
|
|
1127
|
-
mailboxPath: string,
|
|
1128
|
-
getConnection: ConnectionGetter,
|
|
1129
|
-
): Promise<Buffer> {
|
|
1130
|
-
const { uid } = message;
|
|
1131
|
-
this.log.debug?.({ uid }, "Fetching body from IMAP");
|
|
1132
|
-
const connection = await getConnection();
|
|
1133
|
-
await connection.openBox(mailboxPath);
|
|
1134
|
-
const body = await connection.fetchMessageBody(uid);
|
|
1135
|
-
if (body) return body;
|
|
1136
|
-
|
|
1137
|
-
if (!(await isMessageGoneFromOpenMailbox(connection, uid))) {
|
|
1138
|
-
throw new Error(
|
|
1139
|
-
`IMAP FETCH returned no source for UID ${uid} in ${mailboxPath}, which the server still lists`,
|
|
1140
|
-
);
|
|
1141
|
-
}
|
|
1142
|
-
|
|
1143
|
-
return this.reconcileGoneMessage(
|
|
1144
|
-
message,
|
|
1145
|
-
accountId,
|
|
1146
|
-
accountConfigId,
|
|
1147
|
-
mailboxPath,
|
|
1148
|
-
);
|
|
1149
|
-
}
|
|
1150
|
-
|
|
1151
|
-
/**
|
|
1152
|
-
* The server no longer holds the message at the row's UID. A row with our
|
|
1153
|
-
* own move or delete in flight belongs to that mutation (409). A settled row
|
|
1154
|
-
* is removed with its thread entries, but only while its placement still
|
|
1155
|
-
* matches what was read (imap-mutations R3); losing that predicate is a 409
|
|
1156
|
-
* like any other lost placement transition.
|
|
1157
|
-
*/
|
|
1158
|
-
private async reconcileGoneMessage(
|
|
1159
|
-
message: MessageItem,
|
|
1160
|
-
accountId: string,
|
|
1161
|
-
accountConfigId: string,
|
|
1162
|
-
mailboxPath: string,
|
|
1163
|
-
): Promise<never> {
|
|
1164
|
-
const { messageId, uid } = message;
|
|
1165
|
-
if (isPlacementUnsettled(message)) {
|
|
1166
|
-
throw new MessagePlacementUnsettledError(
|
|
1167
|
-
`Message ${messageId} is not at its UID while a move or delete is in flight`,
|
|
1168
|
-
accountId,
|
|
1169
|
-
messageId,
|
|
1170
|
-
"in_flight",
|
|
1171
|
-
);
|
|
1172
|
-
}
|
|
1173
|
-
|
|
1174
|
-
const reconciled = await this.unitOfWork.transaction(async (repos) => {
|
|
1175
|
-
const deleted = await repos.message.deleteIfPlacement(messageId, {
|
|
1176
|
-
status: message.status,
|
|
1177
|
-
mailboxId: message.mailboxId,
|
|
1178
|
-
uid,
|
|
1179
|
-
});
|
|
1180
|
-
if (!deleted) return false;
|
|
1181
|
-
const threadMessages = await repos.threadMessage.findAllByMessageId(
|
|
1182
|
-
accountConfigId,
|
|
1183
|
-
messageId,
|
|
1184
|
-
);
|
|
1185
|
-
if (threadMessages.length > 0) {
|
|
1186
|
-
await repos.threadMessage.deleteMany(
|
|
1187
|
-
threadMessages.map((row) => ({
|
|
1188
|
-
accountConfigId: row.accountConfigId,
|
|
1189
|
-
threadMessageId: row.threadMessageId,
|
|
1190
|
-
})),
|
|
1191
|
-
);
|
|
1192
|
-
}
|
|
1193
|
-
return true;
|
|
1194
|
-
});
|
|
1195
|
-
|
|
1196
|
-
if (!reconciled) {
|
|
1197
|
-
throw new MessagePlacementUnsettledError(
|
|
1198
|
-
`Message ${messageId} changed placement while its absence from ${mailboxPath} was being reconciled`,
|
|
1199
|
-
accountId,
|
|
1200
|
-
messageId,
|
|
1201
|
-
"in_flight",
|
|
1202
|
-
);
|
|
1203
|
-
}
|
|
1204
|
-
|
|
1205
|
-
this.log.info(
|
|
1206
|
-
{
|
|
1207
|
-
metric: "message_gone_stale_row_reconciled",
|
|
1208
|
-
messageId,
|
|
1209
|
-
uid,
|
|
1210
|
-
mailboxPath,
|
|
1211
|
-
},
|
|
1212
|
-
"Message no longer on the mail server (expunged or moved by another client); stale row reconciled",
|
|
1213
|
-
);
|
|
1214
|
-
throw new MessageGoneFromServerError(messageId);
|
|
1215
|
-
}
|
|
1216
|
-
|
|
1217
|
-
/**
|
|
1218
|
-
* Header classification, with the sender's `Address.flags.category`
|
|
1219
|
-
* override (issue #299, RFC 039 Decision 3) substituted for the
|
|
1220
|
-
* header-derived category when one is set. Shared with the classification
|
|
1221
|
-
* backfill (issue #1197) as {@link classifyParsedMessage}, so the one-time
|
|
1222
|
-
* pass and the body-store path can never classify the same bytes
|
|
1223
|
-
* differently.
|
|
1224
|
-
*/
|
|
1225
|
-
private async classifyMessage(
|
|
1226
|
-
accountConfigId: string,
|
|
1227
|
-
parsed: ParsedMail,
|
|
1228
|
-
placement: FolderPlacement,
|
|
1229
|
-
): Promise<UpdateMessageInput & { category: ThreadMessageCategory }> {
|
|
1230
|
-
return classifyParsedMessage(
|
|
1231
|
-
this.addressService,
|
|
1232
|
-
accountConfigId,
|
|
1233
|
-
parsed,
|
|
1234
|
-
placement,
|
|
1235
|
-
);
|
|
1236
|
-
}
|
|
1237
|
-
|
|
1238
|
-
private async incrementInboundCount(
|
|
1239
|
-
messageId: string,
|
|
1240
|
-
accountConfigId: string,
|
|
1241
|
-
parsed: ParsedMail,
|
|
1242
|
-
classification: UpdateMessageInput,
|
|
1243
|
-
): Promise<void> {
|
|
1244
|
-
const fromEmail = extractPrimaryFromEmail(parsed);
|
|
1245
|
-
if (!fromEmail) {
|
|
1246
|
-
this.log.debug?.(
|
|
1247
|
-
{ messageId },
|
|
1248
|
-
"No From address; skipping inbound counter",
|
|
1249
|
-
);
|
|
1250
|
-
return;
|
|
1251
|
-
}
|
|
1252
|
-
|
|
1253
|
-
const addressId = deriveAddressId(accountConfigId, fromEmail);
|
|
1254
|
-
const bulk = isBulkSender(
|
|
1255
|
-
classification.category,
|
|
1256
|
-
classification.hasListUnsubscribe ?? false,
|
|
1257
|
-
);
|
|
1258
|
-
await this.addressService.incrementInboundCount(
|
|
1259
|
-
accountConfigId,
|
|
1260
|
-
addressId,
|
|
1261
|
-
Date.now(),
|
|
1262
|
-
bulk,
|
|
1263
|
-
);
|
|
1264
|
-
}
|
|
1265
|
-
|
|
1266
|
-
/**
|
|
1267
|
-
* The per-sender signals {@link computePlacement} needs, from ONE `Address`
|
|
1268
|
-
* fetch (RFC 039 Decision 3/3a, issue #300): the trust reads exactly as
|
|
1269
|
-
* `deriveSenderTrust` always did — `vip → wellknown → unknown`, untouched by
|
|
1270
|
-
* `blocked` — plus `blocked`/`autoArchive` off the same row. `trustSetAt` is
|
|
1271
|
-
* the `setAt` of whichever flag produced the trust value, needed by
|
|
1272
|
-
* {@link resolveSenderPlacement}'s tie-break; it stays local to placement and
|
|
1273
|
-
* never reaches `deriveSenderTrust`'s own contract (the trust badge).
|
|
1274
|
-
*/
|
|
1275
|
-
private async deriveSenderPlacementSignals(
|
|
1276
|
-
accountConfigId: string,
|
|
1277
|
-
fromEmail: string,
|
|
1278
|
-
): Promise<{
|
|
1279
|
-
trust: (typeof SenderTrust)[keyof typeof SenderTrust];
|
|
1280
|
-
trustSetAt?: number;
|
|
1281
|
-
blocked: boolean;
|
|
1282
|
-
blockedSetAt?: number;
|
|
1283
|
-
neverSpam: boolean;
|
|
1284
|
-
autoArchive: boolean;
|
|
1285
|
-
}> {
|
|
1286
|
-
const unknown = {
|
|
1287
|
-
trust: SenderTrust.Unknown,
|
|
1288
|
-
blocked: false,
|
|
1289
|
-
neverSpam: false,
|
|
1290
|
-
autoArchive: false,
|
|
1291
|
-
} as const;
|
|
1292
|
-
try {
|
|
1293
|
-
const addressId = deriveAddressId(accountConfigId, fromEmail);
|
|
1294
|
-
const address = await this.addressService.getAddress(
|
|
1295
|
-
accountConfigId,
|
|
1296
|
-
addressId,
|
|
1297
|
-
);
|
|
1298
|
-
const flags = address.flags;
|
|
1299
|
-
if (flags?.vip?.value === true) {
|
|
1300
|
-
return {
|
|
1301
|
-
trust: SenderTrust.Vip,
|
|
1302
|
-
trustSetAt: flags.vip.setAt,
|
|
1303
|
-
blocked: flags.blocked?.value === true,
|
|
1304
|
-
blockedSetAt: flags.blocked?.setAt,
|
|
1305
|
-
neverSpam: flags.neverSpam?.value === true,
|
|
1306
|
-
autoArchive: flags.autoArchive?.value === true,
|
|
1307
|
-
};
|
|
1308
|
-
}
|
|
1309
|
-
if (flags?.wellknown?.value === true) {
|
|
1310
|
-
return {
|
|
1311
|
-
trust: SenderTrust.Wellknown,
|
|
1312
|
-
trustSetAt: flags.wellknown.setAt,
|
|
1313
|
-
blocked: flags.blocked?.value === true,
|
|
1314
|
-
blockedSetAt: flags.blocked?.setAt,
|
|
1315
|
-
neverSpam: flags.neverSpam?.value === true,
|
|
1316
|
-
autoArchive: flags.autoArchive?.value === true,
|
|
1317
|
-
};
|
|
1318
|
-
}
|
|
1319
|
-
return {
|
|
1320
|
-
...unknown,
|
|
1321
|
-
blocked: flags?.blocked?.value === true,
|
|
1322
|
-
blockedSetAt: flags?.blocked?.setAt,
|
|
1323
|
-
neverSpam: flags?.neverSpam?.value === true,
|
|
1324
|
-
autoArchive: flags?.autoArchive?.value === true,
|
|
1325
|
-
};
|
|
1326
|
-
} catch (err) {
|
|
1327
|
-
// A genuinely-absent address means "no signals". Any other failure
|
|
1328
|
-
// (AccessDenied, throttle, infra) must NOT be silently downgraded — let
|
|
1329
|
-
// it crash so the placement decision isn't made on bad data.
|
|
1330
|
-
if (!(err instanceof NotFoundError)) throw err;
|
|
1331
|
-
}
|
|
1332
|
-
return unknown;
|
|
1333
|
-
}
|
|
1334
|
-
|
|
1335
|
-
/**
|
|
1336
|
-
* `flags.unsubscribed` (issue #302, RFC 039 Decision 3): "auto-mark-read
|
|
1337
|
-
* until sender stops" — fires on every new message from that sender for as
|
|
1338
|
-
* long as the flag stays set, per the flag's own doc comment; there is no
|
|
1339
|
-
* separate expiry mechanism, and no caching of the decision beyond this
|
|
1340
|
-
* per-message `Address` read. A no-op when body sync was built without an
|
|
1341
|
-
* {@link UnsubscribeConfig} or the message carries no `From` address.
|
|
1342
|
-
*
|
|
1343
|
-
* Write-once per message (issue #499), like the `category` and placement
|
|
1344
|
-
* derivations alongside it: read state is applied on the pass that first
|
|
1345
|
-
* classifies a message and never re-decided. A user who marks such a message
|
|
1346
|
-
* unread afterwards owns it — a re-entrant pass over an already-classified
|
|
1347
|
-
* body would otherwise silently undo that.
|
|
1348
|
-
*/
|
|
1349
|
-
private async applyUnsubscribedAutoRead(
|
|
1350
|
-
messageId: string,
|
|
1351
|
-
accountId: string,
|
|
1352
|
-
accountConfigId: string,
|
|
1353
|
-
parsed: ParsedMail,
|
|
1354
|
-
storedBodyKey: string | undefined,
|
|
1355
|
-
): Promise<void> {
|
|
1356
|
-
if (!this.unsubscribeConfig) return;
|
|
1357
|
-
if (hasStoredBody(storedBodyKey)) return;
|
|
1358
|
-
|
|
1359
|
-
const fromEmail = extractPrimaryFromEmail(parsed);
|
|
1360
|
-
if (!fromEmail) return;
|
|
1361
|
-
|
|
1362
|
-
const unsubscribed = await this.deriveSenderUnsubscribed(
|
|
1363
|
-
accountConfigId,
|
|
1364
|
-
fromEmail,
|
|
1365
|
-
);
|
|
1366
|
-
if (!unsubscribed) return;
|
|
1367
|
-
|
|
1368
|
-
await this.unsubscribeConfig.flagQueueService.markAsRead(
|
|
1369
|
-
accountConfigId,
|
|
1370
|
-
messageId,
|
|
1371
|
-
accountId,
|
|
1372
|
-
);
|
|
1373
|
-
}
|
|
1374
|
-
|
|
1375
|
-
/**
|
|
1376
|
-
* Offer what a message's `text/calendar` parts propose, as cards beside the
|
|
1377
|
-
* message (issue #1033). Nothing here reaches a calendar: a suggestion is
|
|
1378
|
-
* written `Pending` and waits for a person, and a `METHOD:CANCEL` is a card
|
|
1379
|
-
* of its own rather than a withdrawal of an event the user added.
|
|
1380
|
-
*
|
|
1381
|
-
* Write-once per message, like the `category`, placement and read-state
|
|
1382
|
-
* derivations alongside it (issues #499, #1011): a card is offered on the
|
|
1383
|
-
* pass that first classifies a message — the same moment the account's
|
|
1384
|
-
* filters run — and never again. The two re-entrant paths a forced body
|
|
1385
|
-
* re-sync goes through (`fetchAndGetBody`'s `NoSuchKey` fallback,
|
|
1386
|
-
* `syncBodies(..., force: true)`) reach this with `bodyStorageKey` already
|
|
1387
|
-
* set, and re-offering there would resurrect a card the user dismissed.
|
|
1388
|
-
*
|
|
1389
|
-
* Isolated the same way placement and filters are: a card is auxiliary to
|
|
1390
|
-
* storing the mail, which is already durable by the time this runs. A
|
|
1391
|
-
* repository failure here is logged loudly with the messageId rather than
|
|
1392
|
-
* failing a body store that otherwise succeeded.
|
|
1393
|
-
*/
|
|
1394
|
-
private async deriveCalendarSuggestions(
|
|
1395
|
-
messageId: string,
|
|
1396
|
-
accountConfigId: string,
|
|
1397
|
-
parsed: ParsedMail,
|
|
1398
|
-
storedBodyKey: string | undefined,
|
|
1399
|
-
): Promise<void> {
|
|
1400
|
-
if (!this.calendarConfig) return;
|
|
1401
|
-
if (hasStoredBody(storedBodyKey)) return;
|
|
1402
|
-
|
|
1403
|
-
const parts = calendarParts(parsed);
|
|
1404
|
-
if (parts.length === 0) return;
|
|
1405
|
-
|
|
1406
|
-
const { calendarUnitOfWork, calendarSuggestionService, filterService } =
|
|
1407
|
-
this.calendarConfig;
|
|
1408
|
-
|
|
1409
|
-
await Promise.resolve()
|
|
1410
|
-
.then(async () => {
|
|
1411
|
-
// `dismiss{muteSender:true}` wrote a standing rule naming this
|
|
1412
|
-
// sender. Honouring it here is what makes that button do anything:
|
|
1413
|
-
// the index-time filter pipeline skips a rule with no label and no
|
|
1414
|
-
// move, so this is the only reader such a rule has.
|
|
1415
|
-
const sender = extractPrimaryFromEmail(parsed);
|
|
1416
|
-
if (sender) {
|
|
1417
|
-
const active = await filterService.listByAccountAndState(
|
|
1418
|
-
accountConfigId,
|
|
1419
|
-
FilterState.Active,
|
|
1420
|
-
);
|
|
1421
|
-
if (isSenderMuted(active, sender)) {
|
|
1422
|
-
this.log.debug?.(
|
|
1423
|
-
{ messageId, sender },
|
|
1424
|
-
"Sender is muted; offering no calendar suggestion",
|
|
1425
|
-
);
|
|
1426
|
-
return;
|
|
1427
|
-
}
|
|
1428
|
-
}
|
|
1429
|
-
|
|
1430
|
-
// Provisioned rather than looked up, through the one function every
|
|
1431
|
-
// caller provisions with. A missing collection has no timezone, and
|
|
1432
|
-
// a floating DTSTART read in UTC puts the meeting hours from where
|
|
1433
|
-
// the organizer meant it. Idempotent: the id is derived, so a second
|
|
1434
|
-
// first use returns the collection the first one made.
|
|
1435
|
-
const collection = await provisionDefaultCalendar(
|
|
1436
|
-
calendarUnitOfWork,
|
|
1437
|
-
accountConfigId,
|
|
1438
|
-
);
|
|
1439
|
-
for (const part of parts) {
|
|
1440
|
-
const recorded = await recordCalendarSuggestion(
|
|
1441
|
-
calendarSuggestionService,
|
|
1442
|
-
{
|
|
1443
|
-
accountConfigId,
|
|
1444
|
-
messageId,
|
|
1445
|
-
bodyPartId: deriveBodyPartId(messageId, part.partPath),
|
|
1446
|
-
source: part.source,
|
|
1447
|
-
icalData: part.icalData,
|
|
1448
|
-
timezone: collection.timezone,
|
|
1449
|
-
},
|
|
1450
|
-
);
|
|
1451
|
-
if (!recorded.ok) {
|
|
1452
|
-
this.log.info(
|
|
1453
|
-
{
|
|
1454
|
-
messageId,
|
|
1455
|
-
partPath: part.partPath,
|
|
1456
|
-
code: recorded.error.code,
|
|
1457
|
-
reason: recorded.error.message,
|
|
1458
|
-
},
|
|
1459
|
-
"Message carries iCalendar this cannot read; no suggestion offered",
|
|
1460
|
-
);
|
|
1461
|
-
}
|
|
1462
|
-
}
|
|
1463
|
-
})
|
|
1464
|
-
.catch((error: unknown) => {
|
|
1465
|
-
this.log.error?.(
|
|
1466
|
-
{ messageId, accountConfigId, error: inspect(error) },
|
|
1467
|
-
"Calendar suggestion derivation failed; body is stored without a card",
|
|
1468
|
-
);
|
|
1469
|
-
});
|
|
1470
|
-
}
|
|
1471
|
-
|
|
1472
|
-
private async deriveSenderUnsubscribed(
|
|
1473
|
-
accountConfigId: string,
|
|
1474
|
-
fromEmail: string,
|
|
1475
|
-
): Promise<boolean> {
|
|
1476
|
-
try {
|
|
1477
|
-
const addressId = deriveAddressId(accountConfigId, fromEmail);
|
|
1478
|
-
const address = await this.addressService.getAddress(
|
|
1479
|
-
accountConfigId,
|
|
1480
|
-
addressId,
|
|
1481
|
-
);
|
|
1482
|
-
return address.flags?.unsubscribed?.value === true;
|
|
1483
|
-
} catch (err) {
|
|
1484
|
-
// A genuinely-absent address means "not unsubscribed". Any other
|
|
1485
|
-
// failure (AccessDenied, throttle, infra) must NOT be silently
|
|
1486
|
-
// downgraded — let it crash so the read-state decision isn't made on
|
|
1487
|
-
// bad data.
|
|
1488
|
-
if (!(err instanceof NotFoundError)) throw err;
|
|
1489
|
-
return false;
|
|
1490
|
-
}
|
|
1491
|
-
}
|
|
1492
|
-
|
|
1493
|
-
/**
|
|
1494
|
-
* Evaluate the account's active filters against a synced message (RFC 034),
|
|
1495
|
-
* BEFORE the single Message update — so a filter's `movedByRemit` flag joins
|
|
1496
|
-
* that same UpdateItem. Returns the actions to apply (labels + at most one
|
|
1497
|
-
* move); the caller applies them. A no-op returning no actions when no
|
|
1498
|
-
* {@link FilterConfig} is wired.
|
|
1499
|
-
*
|
|
1500
|
-
* Isolation lives in {@link FilterPipeline.evaluate}: this read/decision phase
|
|
1501
|
-
* never fails the body store (the #1246 placement precedent). The applies of
|
|
1502
|
-
* the returned decision are the caller's and are deliberately unswallowed.
|
|
1503
|
-
*/
|
|
1504
|
-
private async evaluateFilters(
|
|
1505
|
-
accountConfigId: string,
|
|
1506
|
-
messageId: string,
|
|
1507
|
-
parsed: ParsedMail,
|
|
1508
|
-
): Promise<FilterDecision> {
|
|
1509
|
-
if (!this.filterPipeline) return { labels: [] };
|
|
1510
|
-
return this.filterPipeline.evaluate(
|
|
1511
|
-
accountConfigId,
|
|
1512
|
-
messageId,
|
|
1513
|
-
toFilterMessage(parsed),
|
|
1514
|
-
);
|
|
1515
|
-
}
|
|
1516
|
-
|
|
1517
|
-
/**
|
|
1518
|
-
* Resolve the placement decision for a message, BEFORE the single Message
|
|
1519
|
-
* update — so both the `movedByRemit` flag and the audit verdict join that
|
|
1520
|
-
* one UpdateItem instead of a second mutation that would fan out another
|
|
1521
|
-
* redundant S3-Vectors upsert. Drives both directions (junk → inbox rescue,
|
|
1522
|
-
* inbox → junk demote) off the pure {@link classifyPlacement} verdict.
|
|
1523
|
-
*
|
|
1524
|
-
* Returns a {@link PlacementOutcome}: a `verdict` to persist whenever Remit
|
|
1525
|
-
* decided to act (action != leave), confident and unsure alike, so the
|
|
1526
|
-
* distribution is queryable on the message; a `move` to enqueue only for a
|
|
1527
|
-
* confident verdict; and `placementDecidedAt` whenever a verdict — including
|
|
1528
|
-
* "leave" — was genuinely computed (issue #383), so a re-entrant call short-
|
|
1529
|
-
* circuits instead of re-deciding a placement the user may have since
|
|
1530
|
-
* overridden by hand.
|
|
1531
|
-
*
|
|
1532
|
-
* Always logs a structured verdict line for confident, actionable verdicts so
|
|
1533
|
-
* the real distribution is observable on a live mailbox.
|
|
1534
|
-
*
|
|
1535
|
-
* The verdict reads the just-computed classification rather than a persisted
|
|
1536
|
-
* row, since those fields are not written until the single update. Placement
|
|
1537
|
-
* is auxiliary to the body store — the primary artifact (body.eml) is already
|
|
1538
|
-
* durable by the time this runs — so a failure here (e.g. a placement-repo
|
|
1539
|
-
* query erroring on a schema-drifted DB) is caught and logged loudly with an
|
|
1540
|
-
* alertable field instead of failing the surrounding message store. The empty
|
|
1541
|
-
* {@link PlacementOutcome} means "no action", whether Remit genuinely decided
|
|
1542
|
-
* to leave the message alone or placement itself failed; the alert log is
|
|
1543
|
-
* what distinguishes the latter, and neither case marks the message decided.
|
|
1544
|
-
*/
|
|
1545
|
-
private async resolvePlacement(
|
|
1546
|
-
messageId: string,
|
|
1547
|
-
accountId: string,
|
|
1548
|
-
accountConfigId: string,
|
|
1549
|
-
parsed: ParsedMail,
|
|
1550
|
-
classification: UpdateMessageInput,
|
|
1551
|
-
): Promise<PlacementOutcome> {
|
|
1552
|
-
if (!this.placementConfig) return {};
|
|
1553
|
-
return this.computePlacement(
|
|
1554
|
-
this.placementConfig,
|
|
1555
|
-
messageId,
|
|
1556
|
-
accountId,
|
|
1557
|
-
accountConfigId,
|
|
1558
|
-
parsed,
|
|
1559
|
-
classification,
|
|
1560
|
-
).catch((error: unknown) => {
|
|
1561
|
-
this.log.error?.(
|
|
1562
|
-
{
|
|
1563
|
-
alert: "body_sync_placement_failed",
|
|
1564
|
-
messageId,
|
|
1565
|
-
accountId,
|
|
1566
|
-
accountConfigId,
|
|
1567
|
-
errorName: (error as { name?: string })?.name,
|
|
1568
|
-
error: inspect(error),
|
|
1569
|
-
},
|
|
1570
|
-
"Placement resolution failed; body already stored, continuing without placement (best-effort, non-fatal)",
|
|
1571
|
-
);
|
|
1572
|
-
return {};
|
|
1573
|
-
});
|
|
1574
|
-
}
|
|
1575
|
-
|
|
1576
|
-
private async computePlacement(
|
|
1577
|
-
placementConfig: PlacementConfig,
|
|
1578
|
-
messageId: string,
|
|
1579
|
-
accountId: string,
|
|
1580
|
-
accountConfigId: string,
|
|
1581
|
-
parsed: ParsedMail,
|
|
1582
|
-
classification: UpdateMessageInput,
|
|
1583
|
-
): Promise<PlacementOutcome> {
|
|
1584
|
-
const { mailboxSpecialUseService } = placementConfig;
|
|
1585
|
-
|
|
1586
|
-
const message = await this.messageService.get(messageId);
|
|
1587
|
-
|
|
1588
|
-
// Issue #383 / #1011: placement still runs once per message (RFC 039
|
|
1589
|
-
// Non-goals), but the once-per-message guarantee now comes from the
|
|
1590
|
-
// caller rather than a guard here — the re-store paths
|
|
1591
|
-
// (`fetchAndGetBody`'s `NoSuchKey` fallback, `syncBodies(..., force:
|
|
1592
|
-
// true)`) skip the decision steps entirely, so only the pass that first
|
|
1593
|
-
// stores a body ever reaches this method.
|
|
1594
|
-
|
|
1595
|
-
const roleMailboxes = await resolveFolderRoleMailboxes(
|
|
1596
|
-
mailboxSpecialUseService,
|
|
1597
|
-
accountId,
|
|
1598
|
-
);
|
|
1599
|
-
const { junkMailbox, inboxMailbox } = roleMailboxes;
|
|
1600
|
-
const placement = placementOf(message.mailboxId, roleMailboxes);
|
|
1601
|
-
|
|
1602
|
-
const fromEmail = extractPrimaryFromEmail(parsed);
|
|
1603
|
-
const signals = fromEmail
|
|
1604
|
-
? await this.deriveSenderPlacementSignals(accountConfigId, fromEmail)
|
|
1605
|
-
: {
|
|
1606
|
-
trust: SenderTrust.Unknown,
|
|
1607
|
-
blocked: false,
|
|
1608
|
-
neverSpam: false,
|
|
1609
|
-
autoArchive: false,
|
|
1610
|
-
};
|
|
1611
|
-
|
|
1612
|
-
const { senderTrust, senderOverride } = resolveSenderPlacement(
|
|
1613
|
-
{ trust: signals.trust, setAt: signals.trustSetAt },
|
|
1614
|
-
{ blocked: signals.blocked, setAt: signals.blockedSetAt },
|
|
1615
|
-
signals.neverSpam,
|
|
1616
|
-
);
|
|
1617
|
-
|
|
1618
|
-
// The verdict needs the classification signals (providerSpam,
|
|
1619
|
-
// authResult, authenticity) that this body-sync pass just derived; the
|
|
1620
|
-
// stored row does not carry them yet, so overlay them onto the message.
|
|
1621
|
-
const candidate = { ...message, ...classification };
|
|
1622
|
-
const verdict = classifyPlacement(
|
|
1623
|
-
candidate,
|
|
1624
|
-
placement,
|
|
1625
|
-
senderTrust,
|
|
1626
|
-
senderOverride,
|
|
1627
|
-
);
|
|
1628
|
-
|
|
1629
|
-
// A `leave` verdict — including "nothing confident to say" — carries no
|
|
1630
|
-
// audit record of its own. `flags.autoArchive` (issue #300) is a distinct,
|
|
1631
|
-
// lower-priority filing preference: it only files a message away when
|
|
1632
|
-
// `blocked`/DKIM/DMARC had nothing to say, never overriding a confident
|
|
1633
|
-
// junk/inbox verdict computed above. Still marked decided (issue #383):
|
|
1634
|
-
// "leave" is itself a verdict, not "not yet evaluated".
|
|
1635
|
-
if (verdict.action === "leave") {
|
|
1636
|
-
const outcome = autoArchiveApplies(verdict, placement)
|
|
1637
|
-
? await this.resolveAutoArchive(
|
|
1638
|
-
mailboxSpecialUseService,
|
|
1639
|
-
message,
|
|
1640
|
-
accountId,
|
|
1641
|
-
signals.autoArchive,
|
|
1642
|
-
)
|
|
1643
|
-
: {};
|
|
1644
|
-
return { ...outcome, placementDecidedAt: Date.now() };
|
|
1645
|
-
}
|
|
1646
|
-
|
|
1647
|
-
// Audit verdict — recorded for every actionable verdict (both
|
|
1648
|
-
// confidences), so the distribution is queryable on the message.
|
|
1649
|
-
// Independent of whether a move is enqueued.
|
|
1650
|
-
const audit: MessagePlacementVerdict = {
|
|
1651
|
-
action:
|
|
1652
|
-
verdict.action === "move-to-inbox"
|
|
1653
|
-
? PlacementAction.MoveToInbox
|
|
1654
|
-
: PlacementAction.MoveToJunk,
|
|
1655
|
-
confidence:
|
|
1656
|
-
verdict.confidence === "confident"
|
|
1657
|
-
? PlacementConfidence.Confident
|
|
1658
|
-
: PlacementConfidence.Unsure,
|
|
1659
|
-
fromPlacement: placement,
|
|
1660
|
-
reasons: verdict.reasons,
|
|
1661
|
-
dryRun: false,
|
|
1662
|
-
decidedAt: Date.now(),
|
|
1663
|
-
};
|
|
1664
|
-
|
|
1665
|
-
// Only a confident verdict moves mail. An unsure verdict is recorded
|
|
1666
|
-
// but never enqueues a move.
|
|
1667
|
-
if (verdict.confidence !== "confident") {
|
|
1668
|
-
return { verdict: audit, placementDecidedAt: audit.decidedAt };
|
|
1669
|
-
}
|
|
1670
|
-
|
|
1671
|
-
const target =
|
|
1672
|
-
verdict.action === "move-to-inbox" ? inboxMailbox : junkMailbox;
|
|
1673
|
-
if (!target) return { verdict: audit, placementDecidedAt: audit.decidedAt };
|
|
1674
|
-
|
|
1675
|
-
// Structured verdict line — emitted for confident, actionable verdicts
|
|
1676
|
-
// so the real verdict distribution is observable on a live mailbox.
|
|
1677
|
-
this.log.info(
|
|
1678
|
-
{
|
|
1679
|
-
messageId,
|
|
1680
|
-
accountId,
|
|
1681
|
-
placement,
|
|
1682
|
-
action: verdict.action,
|
|
1683
|
-
confidence: verdict.confidence,
|
|
1684
|
-
reasons: verdict.reasons,
|
|
1685
|
-
destinationMailboxId: target.mailboxId,
|
|
1686
|
-
},
|
|
1687
|
-
"Placement verdict",
|
|
1688
|
-
);
|
|
1689
|
-
|
|
1690
|
-
return {
|
|
1691
|
-
verdict: audit,
|
|
1692
|
-
move: {
|
|
1693
|
-
destinationMailboxId: target.mailboxId,
|
|
1694
|
-
destinationPath: target.fullPath,
|
|
1695
|
-
},
|
|
1696
|
-
placementDecidedAt: audit.decidedAt,
|
|
1697
|
-
};
|
|
1698
|
-
}
|
|
1699
|
-
|
|
1700
|
-
/**
|
|
1701
|
-
* `flags.autoArchive` (issue #300, RFC 039 Decision 3): file a message
|
|
1702
|
-
* straight to Archive, skipping Inbox. Only reached from
|
|
1703
|
-
* {@link computePlacement} when {@link classifyPlacement} had no confident
|
|
1704
|
-
* junk/inbox verdict of its own — `blocked`/DKIM/DMARC always take priority
|
|
1705
|
-
* over this filing preference — and only for the `leave` verdicts
|
|
1706
|
-
* {@link autoArchiveApplies} admits.
|
|
1707
|
-
*
|
|
1708
|
-
* No {@link MessagePlacementVerdict} is recorded: `PlacementAction` (the
|
|
1709
|
-
* audit enum) has only `MoveToInbox`/`MoveToJunk` — issue #300 is scoped to
|
|
1710
|
-
* no TypeSpec change, so an archive move carries no audit verdict, same as
|
|
1711
|
-
* a matched filter's move. The move itself reuses the same
|
|
1712
|
-
* `placementMoveService.moveMessage` path as every other confident move.
|
|
1713
|
-
*
|
|
1714
|
-
* Idempotent the same way {@link classifyPlacement}'s own branches are: a
|
|
1715
|
-
* message already sitting in Archive is left alone, not moved again.
|
|
1716
|
-
*/
|
|
1717
|
-
private async resolveAutoArchive(
|
|
1718
|
-
mailboxSpecialUseService: IMailboxSpecialUseRepository,
|
|
1719
|
-
message: MessageItem,
|
|
1720
|
-
accountId: string,
|
|
1721
|
-
autoArchive: boolean,
|
|
1722
|
-
): Promise<PlacementOutcome> {
|
|
1723
|
-
if (!autoArchive) return {};
|
|
1724
|
-
|
|
1725
|
-
const archiveMailbox =
|
|
1726
|
-
await mailboxSpecialUseService.findArchiveMailbox(accountId);
|
|
1727
|
-
if (!archiveMailbox || message.mailboxId === archiveMailbox.mailboxId) {
|
|
1728
|
-
return {};
|
|
1729
|
-
}
|
|
1730
|
-
|
|
1731
|
-
this.log.info(
|
|
1732
|
-
{
|
|
1733
|
-
messageId: message.messageId,
|
|
1734
|
-
accountId,
|
|
1735
|
-
destinationMailboxId: archiveMailbox.mailboxId,
|
|
1736
|
-
},
|
|
1737
|
-
"Auto-archive verdict",
|
|
1738
|
-
);
|
|
1739
|
-
|
|
1740
|
-
return {
|
|
1741
|
-
move: {
|
|
1742
|
-
destinationMailboxId: archiveMailbox.mailboxId,
|
|
1743
|
-
destinationPath: archiveMailbox.fullPath,
|
|
1744
|
-
},
|
|
1745
|
-
};
|
|
1746
|
-
}
|
|
1747
|
-
|
|
1748
|
-
// The old `enqueuePlacementMove` (best-effort, catch-and-log) lived here.
|
|
1749
|
-
// Issue #1271: it ran AFTER `bodyStorageKey` was already durable, so a
|
|
1750
|
-
// failure was swallowed to avoid stranding the message behind the
|
|
1751
|
-
// body-sync skip guard — but that meant `movedByRemit` could be true with
|
|
1752
|
-
// no record the move ever reached IMAP. Replaced by the unswallowed call
|
|
1753
|
-
// inside `applyPostStoreSteps`, sequenced BEFORE `bodyStorageKey` is
|
|
1754
|
-
// written, backed by `PlacementMoveService`'s pending-move marker.
|
|
1755
|
-
|
|
1756
|
-
/**
|
|
1757
|
-
* Persist one S3 object per non-multipart leaf so the SPA can resolve
|
|
1758
|
-
* `BodyPartResponse.contentUrl` (#298). Keys follow the layout
|
|
1759
|
-
* `accounts/{accountConfigId}/{accountId}/messages/{messageId}/parts/{partPath}`
|
|
1760
|
-
* so they line up with the URL shape `derive/contentUrl.ts` emits.
|
|
1761
|
-
*
|
|
1762
|
-
* The mapper is total (#395 PR B): every leaf gets a `BodyPartContentPair`,
|
|
1763
|
-
* possibly with a zero-byte content for leaves that have no source bytes
|
|
1764
|
-
* (genuinely empty parts, or pathological inputs the positional fallback
|
|
1765
|
-
* couldn't pair). No try/catch is needed; the only failure surface here is
|
|
1766
|
-
* an S3 write itself, which `pMap` surfaces directly.
|
|
1767
|
-
*
|
|
1768
|
-
* If `listBodyParts` returns an empty list (e.g. a legacy message synced
|
|
1769
|
-
* before #133 populated BodyPart rows), this is a no-op.
|
|
1770
|
-
*/
|
|
1771
|
-
private async storeBodyPartContents(
|
|
1772
|
-
accountConfigId: string,
|
|
1773
|
-
accountId: string,
|
|
1774
|
-
messageId: string,
|
|
1775
|
-
parsed: ParsedMail,
|
|
1776
|
-
options?: { skipExisting?: boolean },
|
|
1777
|
-
): Promise<StoreBodyPartContentsResult> {
|
|
1778
|
-
const bodyParts = await this.envelopeService.listBodyParts(messageId);
|
|
1779
|
-
if (bodyParts.length === 0) {
|
|
1780
|
-
this.log.debug?.(
|
|
1781
|
-
{ messageId },
|
|
1782
|
-
"No BodyPart rows; skipping per-part storage",
|
|
1783
|
-
);
|
|
1784
|
-
return { stored: 0 };
|
|
1785
|
-
}
|
|
1786
|
-
|
|
1787
|
-
const log = this.log;
|
|
1788
|
-
const pairs = mapBodyPartsToContent(bodyParts, parsed, {
|
|
1789
|
-
messageId,
|
|
1790
|
-
logger:
|
|
1791
|
-
log.warn || log.debug
|
|
1792
|
-
? {
|
|
1793
|
-
warn: (obj, msg) => {
|
|
1794
|
-
log.warn?.(obj, msg);
|
|
1795
|
-
},
|
|
1796
|
-
debug: (obj, msg) => {
|
|
1797
|
-
log.debug?.(obj, msg);
|
|
1798
|
-
},
|
|
1799
|
-
}
|
|
1800
|
-
: undefined,
|
|
1801
|
-
});
|
|
1802
|
-
|
|
1803
|
-
if (pairs.length === 0) {
|
|
1804
|
-
return { stored: 0 };
|
|
1805
|
-
}
|
|
1806
|
-
|
|
1807
|
-
const bodyPartIdByPath = new Map(
|
|
1808
|
-
bodyParts.map((bp) => [bp.partPath, bp.bodyPartId]),
|
|
1809
|
-
);
|
|
1810
|
-
|
|
1811
|
-
let stored = 0;
|
|
1812
|
-
const textContents: { bodyPartId: string; content: string }[] = [];
|
|
1813
|
-
await pMap(
|
|
1814
|
-
pairs,
|
|
1815
|
-
async (entry) => {
|
|
1816
|
-
if (
|
|
1817
|
-
options?.skipExisting &&
|
|
1818
|
-
(await this.storageService.bodyPartExists(
|
|
1819
|
-
accountConfigId,
|
|
1820
|
-
accountId,
|
|
1821
|
-
messageId,
|
|
1822
|
-
entry.partPath,
|
|
1823
|
-
))
|
|
1824
|
-
) {
|
|
1825
|
-
return;
|
|
1826
|
-
}
|
|
1827
|
-
await this.storageService.storeBodyPart({
|
|
1828
|
-
accountConfigId,
|
|
1829
|
-
accountId,
|
|
1830
|
-
messageId,
|
|
1831
|
-
partPath: entry.partPath,
|
|
1832
|
-
content: entry.content,
|
|
1833
|
-
contentType: entry.contentType,
|
|
1834
|
-
});
|
|
1835
|
-
stored++;
|
|
1836
|
-
|
|
1837
|
-
if (
|
|
1838
|
-
entry.contentType.toLowerCase().startsWith("text/") &&
|
|
1839
|
-
entry.content.length > 0
|
|
1840
|
-
) {
|
|
1841
|
-
const bodyPartId = bodyPartIdByPath.get(entry.partPath);
|
|
1842
|
-
if (bodyPartId) {
|
|
1843
|
-
textContents.push({
|
|
1844
|
-
bodyPartId,
|
|
1845
|
-
content: entry.content.toString("utf8"),
|
|
1846
|
-
});
|
|
1847
|
-
}
|
|
1848
|
-
}
|
|
1849
|
-
},
|
|
1850
|
-
{ concurrency: BODY_PART_STORE_CONCURRENCY },
|
|
1851
|
-
);
|
|
1852
|
-
|
|
1853
|
-
if (textContents.length > 0) {
|
|
1854
|
-
await this.envelopeService.upsertBodyPartContents(
|
|
1855
|
-
messageId,
|
|
1856
|
-
textContents,
|
|
1857
|
-
);
|
|
1858
|
-
}
|
|
1859
|
-
|
|
1860
|
-
this.log.info(
|
|
1861
|
-
{ messageId, partCount: pairs.length, stored },
|
|
1862
|
-
"Body parts stored",
|
|
1863
|
-
);
|
|
1864
|
-
|
|
1865
|
-
return { stored };
|
|
1866
|
-
}
|
|
1867
|
-
|
|
1868
|
-
/**
|
|
1869
|
-
* Lazily materialize the per-part S3 objects for an already-synced message
|
|
1870
|
-
* whose parts were deferred during bulk sync (DEFER_BODY_PARTS).
|
|
1871
|
-
*
|
|
1872
|
-
* Called from the read path (the API `describeMessage` handler) before the
|
|
1873
|
-
* SPA fetches any `contentUrl`: the part bytes are served directly from S3 by
|
|
1874
|
-
* CloudFront with no Lambda in the request path, so a missing object would
|
|
1875
|
-
* surface to the SPA as a hard "body-missing" failure. Generating the parts
|
|
1876
|
-
* here — re-parsing the stored `body.eml` with the same `mapBodyPartsToContent`
|
|
1877
|
-
* logic body-sync uses — guarantees every `contentUrl` resolves, while bulk
|
|
1878
|
-
* sync stays at 2 writes/message.
|
|
1879
|
-
*
|
|
1880
|
-
* Idempotent: skips any leaf already on S3, so repeat reads cost only HEAD
|
|
1881
|
-
* checks, and a re-read after a partial failure fills the gaps. Requires the
|
|
1882
|
-
* message body to be stored (`bodyStorageKey`); callers gate on that.
|
|
1883
|
-
*
|
|
1884
|
-
* Warm-open fast path: a single HEAD on the `.materialized` sentinel
|
|
1885
|
-
* short-circuits the whole pass — no body.eml GET, no MIME re-parse, no
|
|
1886
|
-
* per-leaf HEADs. The sentinel is written only after every leaf is confirmed
|
|
1887
|
-
* stored, so its presence guarantees all `contentUrl`s resolve.
|
|
1888
|
-
*/
|
|
1889
|
-
async ensureBodyPartsStored(
|
|
1890
|
-
accountConfigId: string,
|
|
1891
|
-
accountId: string,
|
|
1892
|
-
messageId: string,
|
|
1893
|
-
bodyStorageKey: string,
|
|
1894
|
-
): Promise<StoreBodyPartContentsResult> {
|
|
1895
|
-
if (
|
|
1896
|
-
await this.storageService.bodyPartExists(
|
|
1897
|
-
accountConfigId,
|
|
1898
|
-
accountId,
|
|
1899
|
-
messageId,
|
|
1900
|
-
MATERIALIZED_SENTINEL_PART_PATH,
|
|
1901
|
-
)
|
|
1902
|
-
) {
|
|
1903
|
-
this.log.debug?.(
|
|
1904
|
-
{ messageId },
|
|
1905
|
-
"Body parts already materialized (sentinel hit); skipping",
|
|
1906
|
-
);
|
|
1907
|
-
return { stored: 0 };
|
|
1908
|
-
}
|
|
1909
|
-
|
|
1910
|
-
const body = await this.storageService.retrieve(bodyStorageKey);
|
|
1911
|
-
const parsed = await simpleParser(body);
|
|
1912
|
-
const result = await this.storeBodyPartContents(
|
|
1913
|
-
accountConfigId,
|
|
1914
|
-
accountId,
|
|
1915
|
-
messageId,
|
|
1916
|
-
parsed,
|
|
1917
|
-
{ skipExisting: true },
|
|
1918
|
-
);
|
|
1919
|
-
|
|
1920
|
-
// Reaching here means storeBodyPartContents resolved — `pMap` rejects on
|
|
1921
|
-
// any single failed write, so every leaf is now durably on S3. Drop the
|
|
1922
|
-
// sentinel so subsequent opens take the HEAD-only fast path above.
|
|
1923
|
-
await this.storageService.storeBodyPart({
|
|
1924
|
-
accountConfigId,
|
|
1925
|
-
accountId,
|
|
1926
|
-
messageId,
|
|
1927
|
-
partPath: MATERIALIZED_SENTINEL_PART_PATH,
|
|
1928
|
-
content: Buffer.alloc(0),
|
|
1929
|
-
});
|
|
1930
|
-
|
|
1931
|
-
return result;
|
|
1932
|
-
}
|
|
1933
|
-
|
|
1934
|
-
/**
|
|
1935
|
-
* Persist the pre-parsed body cache. A failure here MUST fail the
|
|
1936
|
-
* surrounding body-sync: the parsed-body object is what the search-index
|
|
1937
|
-
* pipeline reads, so swallowing a write failure leaves the message marked
|
|
1938
|
-
* synced while `parsedBody` is null and it never gets indexed — a silent
|
|
1939
|
-
* search gap. Propagate so the message lands in failedMessageIds and the
|
|
1940
|
-
* SQS batch requeues it.
|
|
1941
|
-
*/
|
|
1942
|
-
private async storeParsedBodyCache(
|
|
1943
|
-
accountConfigId: string,
|
|
1944
|
-
accountId: string,
|
|
1945
|
-
messageId: string,
|
|
1946
|
-
parsed: ParsedMail,
|
|
1947
|
-
): Promise<void> {
|
|
1948
|
-
const parsedBody = toParsedBody(parsed);
|
|
1949
|
-
try {
|
|
1950
|
-
await this.storageService.storeParsedBody({
|
|
1951
|
-
accountConfigId,
|
|
1952
|
-
accountId,
|
|
1953
|
-
messageId,
|
|
1954
|
-
parsed: parsedBody,
|
|
1955
|
-
});
|
|
1956
|
-
this.log.debug?.({ messageId }, "Parsed body cache stored");
|
|
1957
|
-
} catch (err: unknown) {
|
|
1958
|
-
this.log.error?.(
|
|
1959
|
-
{
|
|
1960
|
-
messageId,
|
|
1961
|
-
errorName: (err as { name?: string }).name,
|
|
1962
|
-
errorCode: (err as { Code?: string }).Code,
|
|
1963
|
-
error: inspect(err),
|
|
1964
|
-
},
|
|
1965
|
-
"Failed to store parsed body cache; failing sync to requeue",
|
|
1966
|
-
);
|
|
1967
|
-
throw err;
|
|
1968
|
-
}
|
|
1969
|
-
}
|
|
1970
|
-
}
|