@remit/mailbox-service 0.0.112 → 0.0.113

package/dist/body-sync.d.ts

@@ -0,0 +1,351 @@
+import { type IAddressRepository, type ICalendarStore, type ICalendarSuggestionRepository, type IEnvelopeRepository, type IFilterRepository, type IMailboxSpecialUseRepository, type IMessageRepository, type IThreadMessageRepository } from "@remit/data-ports";
+import { type ParsedBody, type StorageService } from "@remit/storage-service";
+import { type ParsedMail } from "mailparser";
+import { type FilterConfig } from "./filters/pipeline.js";
+import type { FlagQueueService } from "./flag-queue.js";
+import { type FolderPlacement } from "./heuristics/classifyPlacement.js";
+import type { PlacementMoveService } from "./placement-move.js";
+import { type QuarantineService } from "./quarantine.js";
+import type { SyncWarningService } from "./sync-warning.js";
+import { type IImapConnection } from "./types.js";
+/**
+ * Whether the body-sync hot path defers per-part S3 objects. Each MIME leaf is
+ * its own PutObject, so during bulk sync these per-part writes dominate S3 write
+ * count and cost. With deferral ON a synced message writes only `body.eml` +
+ * `parsed.json.gz` (2 writes); the per-part objects are materialized lazily on
+ * the first `contentUrl` read via {@link BodySyncService.ensureBodyPartsStored}.
+ *
+ * Default ON (cost savings). Set `DEFER_BODY_PARTS=false`/`0`/`off` to restore
+ * the original eager per-part writes — a safe, reversible escape hatch.
+ */
+export declare const isBodyPartDeferralEnabled: () => boolean;
+/**
+ * The snippet the list row shows, from whichever body part carries text.
+ * Exported for the classification backfill (issue #1197), which denormalizes
+ * the same fields the body-store path does, from the same bytes.
+ */
+export declare const extractSnippet: (parsed: ParsedMail) => string;
+export declare const toParsedBody: (parsed: ParsedMail) => ParsedBody;
+export interface BodySyncLogger {
+    info(obj: Record<string, unknown>, msg: string): void;
+    debug?(obj: Record<string, unknown>, msg: string): void;
+    warn?(obj: Record<string, unknown>, msg: string): void;
+    error?(obj: Record<string, unknown>, msg: string): void;
+}
+export interface SyncBodiesResult {
+    syncedCount: number;
+    syncedMessageIds: string[];
+    skippedCount: number;
+    failedCount: number;
+    failedMessageIds: string[];
+}
+export interface FetchBodyResult {
+    text: string | null;
+    html: string | null;
+    storedAt: string;
+}
+export interface StoreBodyPartContentsResult {
+    stored: number;
+}
+export type ConnectionGetter = () => Promise<IImapConnection>;
+export interface BodySyncFolder {
+    mailboxId: string;
+    fullPath: string;
+    placement: FolderPlacement;
+}
+export interface BodySyncRoundFolder extends BodySyncFolder {
+    uidValidity: number;
+}
+export interface PlacementConfig {
+    mailboxSpecialUseService: IMailboxSpecialUseRepository;
+    /**
+     * Local-first mover for a confident placement verdict (issue #1271). Same
+     * `moveMessage(accountConfigId, messageId, destinationMailboxId, accountId)`
+     * signature as the general-purpose `MessageMoveService` it replaced here,
+     * but backed by a pending-move marker + its own reconciler queue instead of
+     * the general MESSAGE_MOVE event (see `placement-move.ts`).
+     */
+    placementMoveService: PlacementMoveService;
+}
+/**
+ * What body sync needs to set a message aside (issue #72). The mailbox facts
+ * are supplied by the caller because they are properties of the round, not of
+ * any one message.
+ */
+export interface QuarantineConfig {
+    quarantineService: QuarantineService;
+    mailboxId: string;
+    /**
+     * The mailbox's stored UIDVALIDITY. Taken from the mailbox row rather than
+     * from the open box because a round whose two disagree is paused by the
+     * cursor guard before it reaches any UID.
+     */
+    uidValidity: number;
+    /** Redeliveries of this batch so far, so the record says how hard we tried. */
+    attempts: number;
+}
+/**
+ * What body sync needs to auto-mark-read a message from an unsubscribed
+ * sender (issue #302, RFC 039 Decision 3). Reuses the same
+ * `FlagQueueService.markAsRead` a manual mark-as-read already goes through —
+ * local `\Seen` + `ThreadMessage.isRead` + a durable pending IMAP flag-push
+ * marker — so this fires the same round-trip, not a second primitive.
+ */
+export interface UnsubscribeConfig {
+    flagQueueService: FlagQueueService;
+}
+/**
+ * What body sync needs to offer a mail's invitation as a card (issue #1033).
+ *
+ * The calendar store is here to provision the account's default collection on
+ * first use, through the one function every other caller provisions with. That
+ * is not bookkeeping: an invitation whose DTSTART names no zone is RFC 5545
+ * floating time, and reading it anywhere but where the user's calendar lives
+ * shows the meeting at the wrong hour. Without a collection there is no zone
+ * to read it in, and every such event silently lands in UTC.
+ *
+ * `filterService` is read to honour `dismiss{muteSender:true}`: a sender the
+ * user muted gets no card at all.
+ */
+export interface CalendarSuggestionConfig {
+    calendarSuggestionService: ICalendarSuggestionRepository;
+    calendarStore: ICalendarStore;
+    filterService: Pick<IFilterRepository, "listByAccountAndState">;
+}
+export declare class BodySyncService {
+    private messageService;
+    private storageService;
+    private threadMessageService;
+    private addressService;
+    private envelopeService;
+    private readonly syncWarningService;
+    private readonly placementConfig?;
+    private readonly filterConfig?;
+    private readonly quarantineConfig?;
+    private readonly unsubscribeConfig?;
+    private readonly calendarConfig?;
+    private log;
+    private readonly filterPipeline?;
+    constructor(messageService: IMessageRepository, storageService: StorageService, threadMessageService: IThreadMessageRepository, addressService: IAddressRepository, envelopeService: IEnvelopeRepository, syncWarningService: SyncWarningService, logger?: BodySyncLogger, placementConfig?: PlacementConfig | undefined, filterConfig?: FilterConfig | undefined, quarantineConfig?: QuarantineConfig | undefined, unsubscribeConfig?: UnsubscribeConfig | undefined, calendarConfig?: CalendarSuggestionConfig | undefined);
+    syncBodies(messageIds: string[], accountId: string, accountConfigId: string, folder: BodySyncRoundFolder, getConnection: ConnectionGetter, force?: boolean): Promise<SyncBodiesResult>;
+    /**
+     * Set a message aside because its body will not parse (issue #72). The
+     * fingerprint comes from the BodyPart rows metadata sync already wrote: the
+     * body path streams raw bytes and has no FETCH result to read.
+     */
+    private recordQuarantine;
+    private buildResult;
+    /**
+     * Stream one message body straight to storage while teeing the bytes into a
+     * buffer for the parse-dependent steps (snippet, classification, parsed-body
+     * cache, per-part objects). The S3 upload never sees a whole-body concat —
+     * the storage service streams it — but mailparser still needs the full bytes,
+     * so we collect them in parallel. Later issues move parsing off the hot path.
+     *
+     * `isReStore` marks a re-fetch of a message whose row already carried
+     * `bodyStorageKey` (the `force` re-arm in {@link syncBodies}): the
+     * post-store steps then re-materialize the body artifacts without
+     * re-running any decision (issue #1011).
+     */
+    private storeStreamedBody;
+    /**
+     * The steps every synced body must go through regardless of which path
+     * fetched it first — the sync path ({@link storeStreamedBody}) and the
+     * read-path backfill ({@link fetchAndGetBody}) both call this instead of
+     * duplicating (and drifting on) their own subset. Before issue #1271,
+     * `fetchAndGetBody` skipped classification/placement entirely — a message
+     * materialized by a read got different treatment than one synced in bulk.
+     *
+     * Order matters for two independent reasons, both load-bearing:
+     * 1. The parsed-body cache must be durable before `bodyStorageKey` (the
+     *    skip-guard signal) — see the comment on the write below.
+     * 2. A placement move — local mailboxId change, pending marker, IMAP-push
+     *    enqueue (`PlacementMoveService.moveMessage`, issue #1271) — must
+     *    complete, or fully fail, BEFORE `bodyStorageKey` is written. That
+     *    failure is never swallowed: if it throws, `bodyStorageKey` is not yet
+     *    durable, so a retry reprocesses the message from scratch (marker
+     *    included) instead of the skip-guard stranding a `movedByRemit` flag
+     *    with no marker behind it — the defect this issue fixes.
+     *
+     * Returns the parsed mail so callers that already need it (both do) don't
+     * pay for mailparser twice.
+     */
+    private applyPostStoreSteps;
+    /**
+     * Fetch a single message body, store it, and return the parsed content.
+     *
+     * If the body is already stored, retrieves it from storage.
+     * Otherwise fetches from IMAP, stores it, and returns the parsed content.
+     *
+     * @param messageId - The message ID to fetch
+     * @param accountId - The account ID (for storage path)
+     * @param accountConfigId - The account config ID (for thread updates)
+     * @param folder - The IMAP mailbox path and its folder placement
+     * @param getConnection - Function to get a (lazy) IMAP connection
+     * @returns Parsed text and HTML content
+     */
+    fetchAndGetBody(messageId: string, accountId: string, accountConfigId: string, folder: BodySyncFolder, getConnection: ConnectionGetter): Promise<FetchBodyResult>;
+    private fetchFromImap;
+    /**
+     * The server no longer holds the message at the row's UID. A row with our
+     * own move or delete in flight belongs to that mutation (409). A settled row
+     * is removed with its thread entries, but only while its placement still
+     * matches what was read (imap-mutations R3); losing that predicate is a 409
+     * like any other lost placement transition.
+     */
+    private reconcileGoneMessage;
+    /**
+     * Header classification, with the sender's `Address.flags.category`
+     * override (issue #299, RFC 039 Decision 3) substituted for the
+     * header-derived category when one is set. Shared with the classification
+     * backfill (issue #1197) as {@link classifyParsedMessage}, so the one-time
+     * pass and the body-store path can never classify the same bytes
+     * differently.
+     */
+    private classifyMessage;
+    private incrementInboundCount;
+    /**
+     * The per-sender signals {@link computePlacement} needs, from ONE `Address`
+     * fetch (RFC 039 Decision 3/3a, issue #300): the trust reads exactly as
+     * `deriveSenderTrust` always did — `vip → wellknown → unknown`, untouched by
+     * `blocked` — plus `blocked`/`autoArchive` off the same row. `trustSetAt` is
+     * the `setAt` of whichever flag produced the trust value, needed by
+     * {@link resolveSenderPlacement}'s tie-break; it stays local to placement and
+     * never reaches `deriveSenderTrust`'s own contract (the trust badge).
+     */
+    private deriveSenderPlacementSignals;
+    /**
+     * `flags.unsubscribed` (issue #302, RFC 039 Decision 3): "auto-mark-read
+     * until sender stops" — fires on every new message from that sender for as
+     * long as the flag stays set, per the flag's own doc comment; there is no
+     * separate expiry mechanism, and no caching of the decision beyond this
+     * per-message `Address` read. A no-op when body sync was built without an
+     * {@link UnsubscribeConfig} or the message carries no `From` address.
+     *
+     * Write-once per message (issue #499), like the `category` and placement
+     * derivations alongside it: read state is applied on the pass that first
+     * classifies a message and never re-decided. A user who marks such a message
+     * unread afterwards owns it — a re-entrant pass over an already-classified
+     * body would otherwise silently undo that.
+     */
+    private applyUnsubscribedAutoRead;
+    /**
+     * Offer what a message's `text/calendar` parts propose, as cards beside the
+     * message (issue #1033). Nothing here reaches a calendar: a suggestion is
+     * written `Pending` and waits for a person, and a `METHOD:CANCEL` is a card
+     * of its own rather than a withdrawal of an event the user added.
+     *
+     * Write-once per message, like the `category`, placement and read-state
+     * derivations alongside it (issues #499, #1011): a card is offered on the
+     * pass that first classifies a message — the same moment the account's
+     * filters run — and never again. The two re-entrant paths a forced body
+     * re-sync goes through (`fetchAndGetBody`'s `NoSuchKey` fallback,
+     * `syncBodies(..., force: true)`) reach this with `bodyStorageKey` already
+     * set, and re-offering there would resurrect a card the user dismissed.
+     *
+     * Isolated the same way placement and filters are: a card is auxiliary to
+     * storing the mail, which is already durable by the time this runs. A
+     * repository failure here is logged loudly with the messageId rather than
+     * failing a body store that otherwise succeeded.
+     */
+    private deriveCalendarSuggestions;
+    private deriveSenderUnsubscribed;
+    /**
+     * Evaluate the account's active filters against a synced message (RFC 034),
+     * BEFORE the single Message update — so a filter's `movedByRemit` flag joins
+     * that same UpdateItem. Returns the actions to apply (labels + at most one
+     * move); the caller applies them. A no-op returning no actions when no
+     * {@link FilterConfig} is wired.
+     *
+     * Isolation lives in {@link FilterPipeline.evaluate}: this read/decision phase
+     * never fails the body store (the #1246 placement precedent). The applies of
+     * the returned decision are the caller's and are deliberately unswallowed.
+     */
+    private evaluateFilters;
+    /**
+     * Resolve the placement decision for a message, BEFORE the single Message
+     * update — so both the `movedByRemit` flag and the audit verdict join that
+     * one UpdateItem instead of a second mutation. Drives both directions (junk → inbox rescue,
+     * inbox → junk demote) off the pure {@link classifyPlacement} verdict.
+     *
+     * Returns a {@link PlacementOutcome}: a `verdict` to persist whenever Remit
+     * decided to act (action != leave), confident and unsure alike, so the
+     * distribution is queryable on the message; a `move` to enqueue only for a
+     * confident verdict; and `placementDecidedAt` whenever a verdict — including
+     * "leave" — was genuinely computed (issue #383), so a re-entrant call short-
+     * circuits instead of re-deciding a placement the user may have since
+     * overridden by hand.
+     *
+     * Always logs a structured verdict line for confident, actionable verdicts so
+     * the real distribution is observable on a live mailbox.
+     *
+     * The verdict reads the just-computed classification rather than a persisted
+     * row, since those fields are not written until the single update. Placement
+     * is auxiliary to the body store — the primary artifact (body.eml) is already
+     * durable by the time this runs — so a failure here (e.g. a placement-repo
+     * query erroring on a schema-drifted DB) is caught and logged loudly with an
+     * alertable field instead of failing the surrounding message store. The empty
+     * {@link PlacementOutcome} means "no action", whether Remit genuinely decided
+     * to leave the message alone or placement itself failed; the alert log is
+     * what distinguishes the latter, and neither case marks the message decided.
+     */
+    private resolvePlacement;
+    private computePlacement;
+    /**
+     * `flags.autoArchive` (issue #300, RFC 039 Decision 3): file a message
+     * straight to Archive, skipping Inbox. Only reached from
+     * {@link computePlacement} when {@link classifyPlacement} had no confident
+     * junk/inbox verdict of its own — `blocked`/DKIM/DMARC always take priority
+     * over this filing preference — and only for the `leave` verdicts
+     * {@link autoArchiveApplies} admits.
+     *
+     * No {@link MessagePlacementVerdict} is recorded: `PlacementAction` (the
+     * audit enum) has only `MoveToInbox`/`MoveToJunk` — issue #300 is scoped to
+     * no TypeSpec change, so an archive move carries no audit verdict, same as
+     * a matched filter's move. The move itself reuses the same
+     * `placementMoveService.moveMessage` path as every other confident move.
+     *
+     * Idempotent the same way {@link classifyPlacement}'s own branches are: a
+     * message already sitting in Archive is left alone, not moved again.
+     */
+    private resolveAutoArchive;
+    /**
+     * Persist one S3 object per non-multipart leaf so the SPA can resolve
+     * `BodyPartResponse.contentUrl` (#298). Keys follow the layout
+     * `accounts/{accountConfigId}/{accountId}/messages/{messageId}/parts/{partPath}`
+     * so they line up with the URL shape `derive/contentUrl.ts` emits.
+     *
+     * The mapper is total (#395 PR B): every leaf gets a `BodyPartContentPair`,
+     * possibly with a zero-byte content for leaves that have no source bytes
+     * (genuinely empty parts, or pathological inputs the positional fallback
+     * couldn't pair). No try/catch is needed; the only failure surface here is
+     * an S3 write itself, which `pMap` surfaces directly.
+     *
+     * If `listBodyParts` returns an empty list (e.g. a legacy message synced
+     * before #133 populated BodyPart rows), this is a no-op.
+     */
+    private storeBodyPartContents;
+    /**
+     * Lazily materialize the per-part S3 objects for an already-synced message
+     * whose parts were deferred during bulk sync (DEFER_BODY_PARTS).
+     *
+     * Called from the read path (the API `describeMessage` handler) before the
+     * SPA fetches any `contentUrl`: the part bytes are served directly from S3 by
+     * CloudFront with no Lambda in the request path, so a missing object would
+     * surface to the SPA as a hard "body-missing" failure. Generating the parts
+     * here — re-parsing the stored `body.eml` with the same `mapBodyPartsToContent`
+     * logic body-sync uses — guarantees every `contentUrl` resolves, while bulk
+     * sync stays at 2 writes/message.
+     *
+     * Idempotent: skips any leaf already on S3, so repeat reads cost only HEAD
+     * checks, and a re-read after a partial failure fills the gaps. Requires the
+     * message body to be stored (`bodyStorageKey`); callers gate on that.
+     *
+     * Warm-open fast path: a single HEAD on the `.materialized` sentinel
+     * short-circuits the whole pass — no body.eml GET, no MIME re-parse, no
+     * per-leaf HEADs. The sentinel is written only after every leaf is confirmed
+     * stored, so its presence guarantees all `contentUrl`s resolve.
+     */
+    ensureBodyPartsStored(accountConfigId: string, accountId: string, messageId: string, bodyStorageKey: string): Promise<StoreBodyPartContentsResult>;
+    private storeParsedBodyCache;
+}