@remit/mailbox-service 0.0.111 → 0.0.113

package/dist/message-parser.d.ts

@@ -0,0 +1,14 @@
+/**
+ * Message parsing utilities for extracting content from raw RFC822 messages.
+ */
+export interface ParsedMessageContent {
+    text: string | null;
+    html: string | null;
+}
+/**
+ * Parse a raw RFC822 message buffer and extract text/html content.
+ *
+ * @param buffer - The raw message content (RFC822 format)
+ * @returns Parsed content with text and html fields
+ */
+export declare const parseMessageContent: (buffer: Buffer) => Promise<ParsedMessageContent>;

package/dist/message-parser.js

@@ -0,0 +1,17 @@
+/**
+ * Message parsing utilities for extracting content from raw RFC822 messages.
+ */
+import { simpleParser } from "mailparser";
+/**
+ * Parse a raw RFC822 message buffer and extract text/html content.
+ *
+ * @param buffer - The raw message content (RFC822 format)
+ * @returns Parsed content with text and html fields
+ */
+export const parseMessageContent = async (buffer) => {
+    const parsed = await simpleParser(buffer);
+    return {
+        text: parsed.text ?? null,
+        html: typeof parsed.html === "string" ? parsed.html : null,
+    };
+};

package/{src/message-presence.ts → dist/message-presence.d.ts}

@@ -1,5 +1,4 @@
 import type { IImapConnection } from "./types.js";
-
 /**
  * Whether a UID is confirmed gone from the currently open mailbox.
  *
@@ -21,15 +20,4 @@ import type { IImapConnection } from "./types.js";
  * hands this function a deletion authority it cannot tell apart from a genuine
  * miss.
  */
-export const isMessageGoneFromOpenMailbox = async (
-	connection: Pick<IImapConnection, "fetchMessages" | "search">,
-	uid: number,
-): Promise<boolean> => {
-	const found = await connection.fetchMessages([uid]);
-	if (found.length > 0) {
-		return false;
-	}
-
-	const matched = await connection.search([["UID", String(uid)]]);
-	return !matched.includes(uid);
-};
+export declare const isMessageGoneFromOpenMailbox: (connection: Pick<IImapConnection, "fetchMessages" | "search">, uid: number) => Promise<boolean>;

package/dist/message-presence.js

@@ -0,0 +1,29 @@
+/**
+ * Whether a UID is confirmed gone from the currently open mailbox.
+ *
+ * A FETCH that yields no row is not proof of absence. imapflow drops rows on
+ * back-to-back FETCHes (#408) — the same client glitch #100 had to stop
+ * reading as authoritative on the sync path — so a transient blip returns an
+ * empty result for a message that is still on the server. Anything that
+ * deletes local rows on that reading destroys live mail.
+ *
+ * Absence is therefore confirmed with a UID SEARCH, which the server answers
+ * with a plain UID set rather than a stream of message rows, and which
+ * `placement-move-push.ts` already uses as its verification probe. Only a
+ * SEARCH that does not list the UID counts as gone; an empty FETCH the SEARCH
+ * contradicts leaves the message present, and the caller treats it as such.
+ *
+ * The absence verdict rests on an empty SEARCH meaning the server matched
+ * nothing, so `IImapConnection.search` must throw when a SEARCH fails rather
+ * than answer `[]` — an implementation that reports failure as an empty array
+ * hands this function a deletion authority it cannot tell apart from a genuine
+ * miss.
+ */
+export const isMessageGoneFromOpenMailbox = async (connection, uid) => {
+    const found = await connection.fetchMessages([uid]);
+    if (found.length > 0) {
+        return false;
+    }
+    const matched = await connection.search([["UID", String(uid)]]);
+    return !matched.includes(uid);
+};

package/dist/message-read.d.ts

@@ -0,0 +1,51 @@
+/**
+ * The slice of `BodySyncService` the read path needs to lazily materialize
+ * deferred per-part objects.
+ */
+export interface BodyPartMaterializer {
+    ensureBodyPartsStored(accountConfigId: string, accountId: string, messageId: string, bodyStorageKey: string): Promise<unknown>;
+}
+export interface MaterializeLogger {
+    warn(obj: Record<string, unknown>, msg: string): void;
+    error(obj: Record<string, unknown>, msg: string): void;
+}
+/**
+ * Re-arms the body-sync cue for one message (the read-path slice of
+ * `BodySyncQueueService`).
+ */
+export interface BodySyncCue {
+    requestBodySync(input: {
+        accountId: string;
+        mailboxId: string;
+        messageId: string;
+        uid?: number;
+    }): Promise<void>;
+}
+export interface MaterializeResult {
+    /** `false` when the body object was missing — the caller must not serve the body. */
+    ready: boolean;
+}
+/**
+ * Materialize the deferred per-part storage objects for a message before any
+ * part is read. Distinguishes the two failure modes the body contract cares
+ * about:
+ *
+ * - Missing storage object (never synced or lost) — re-arm the SYNC_MESSAGE_BODY
+ *   cue so the worker (re)stores the body, and report `ready: false`. Not a
+ *   500: a retry succeeds once the worker catches up.
+ * - Any other error — rethrow so the request 500s and the bug is observable.
+ *   Never swallowed, never "best-effort".
+ *
+ * No-op (`ready: true`) when the body isn't stored yet (no `bodyStorageKey`) or
+ * when the parts were written eagerly.
+ */
+export declare const materializeBodyParts: (materializer: BodyPartMaterializer, args: {
+    accountConfigId: string;
+    accountId: string;
+    mailboxId: string;
+    messageId: string;
+    uid?: number;
+    bodyStorageKey: string | undefined;
+    cue: BodySyncCue;
+    logger: MaterializeLogger;
+}) => Promise<MaterializeResult>;

package/dist/message-read.js

@@ -0,0 +1,31 @@
+import { isStorageNotFoundError } from "@remit/storage-service";
+/**
+ * Materialize the deferred per-part storage objects for a message before any
+ * part is read. Distinguishes the two failure modes the body contract cares
+ * about:
+ *
+ * - Missing storage object (never synced or lost) — re-arm the SYNC_MESSAGE_BODY
+ *   cue so the worker (re)stores the body, and report `ready: false`. Not a
+ *   500: a retry succeeds once the worker catches up.
+ * - Any other error — rethrow so the request 500s and the bug is observable.
+ *   Never swallowed, never "best-effort".
+ *
+ * No-op (`ready: true`) when the body isn't stored yet (no `bodyStorageKey`) or
+ * when the parts were written eagerly.
+ */
+export const materializeBodyParts = async (materializer, args) => {
+    const { accountConfigId, accountId, mailboxId, messageId, uid } = args;
+    if (!args.bodyStorageKey)
+        return { ready: true };
+    try {
+        await materializer.ensureBodyPartsStored(accountConfigId, accountId, messageId, args.bodyStorageKey);
+        return { ready: true };
+    }
+    catch (error) {
+        if (!isStorageNotFoundError(error))
+            throw error;
+        args.logger.warn({ messageId, accountId, mailboxId }, "Body object missing on read; re-arming body-sync cue and signalling retry");
+        await args.cue.requestBodySync({ accountId, mailboxId, messageId, uid });
+        return { ready: false };
+    }
+};

package/dist/message-sync.d.ts

@@ -0,0 +1,345 @@
+import type { IAddressRepository, IEnvelopeRepository, IMailboxRepository, IMailboxSpecialUseRepository, IMessageFlagPushRepository, IMessageFlagRepository, IMessageRepository, IThreadMessageRepository } from "@remit/data-ports";
+import type { JunkRoleMailboxes } from "@remit/data-ports/folder-role";
+import type { ManagedConnectionFactory } from "./connection-factory.js";
+import type { QuarantineService } from "./quarantine.js";
+import type { SyncWarningService } from "./sync-warning.js";
+import type { ImapAddress } from "./types.js";
+/**
+ * A From address is only usable when it looks like a real mailbox: both parts
+ * present, neither is a known "could not parse" sentinel, and the host carries
+ * an actual domain (at least one dot). Detect this structurally so a fabricated
+ * string like `missing_mailbox@missing_domain` is never persisted as a sender.
+ */
+export declare const isParseableEmailAddress: (address: ImapAddress | undefined) => boolean;
+export type AddressSighting = "junk" | "discarded" | "correspondent";
+/** The account's Junk and Trash mailboxes, resolved once per batch. */
+export interface AccountFolderRoles {
+    readonly junkMailboxId: string | null;
+    readonly trashMailboxId: string | null;
+    /**
+     * The same two roles across every account under the config, for the
+     * reconcile that follows a save into Junk. The address book is keyed by
+     * config, so a predicate over an address weighs sightings from all of them —
+     * resolving only this account withholds a sender on one and restores it on
+     * the next.
+     */
+    readonly configJunkRoles: JunkRoleMailboxes;
+}
+/**
+ * What a sighting of an address in this mailbox is worth. Junk withholds the
+ * sender from autocomplete (#822), Trash decides nothing either way, and
+ * anywhere else is a correspondent.
+ *
+ * Decided by mailbox IDENTITY against the account's resolved roles, not by the
+ * folder's flag and name: a role belongs to exactly one folder (RFC 032), and
+ * the folder it belongs to is whichever one the user appointed. Reading the
+ * name here meant a second folder called `Spam` also withheld its senders,
+ * while the folder the user actually appointed harvested every spammer in it as
+ * a correspondent (#837).
+ */
+export declare const addressSightingIn: (mailboxId: string, roles: AccountFolderRoles) => AddressSighting;
+/**
+ * The name an envelope carries for an address, as it should be stored: any
+ * claim to be some other address removed (issue #826), the rest of the name
+ * kept. Refused wherever a name lands — the address book, the envelope the
+ * message header renders, and the sender label on the thread row.
+ */
+export declare const harvestedDisplayName: (address: ImapAddress | undefined) => string;
+/**
+ * Parse an external `Date:` header into an epoch-millisecond integer.
+ *
+ * The IMAP envelope `date` is a raw RFC 2822 header copied verbatim from the
+ * message. It can be missing, malformed, or in a format `Date` cannot parse —
+ * `new Date(raw).getTime()` then yields `NaN`. `NaN` is not a valid integer
+ * and ElectroDB rejects it, which previously threw on the envelope upsert and
+ * (because the batch aborted on the first rejection) stalled the whole mailbox.
+ *
+ * When the header is unparseable we fall back to `fallbackMs` — the IMAP
+ * server's own INTERNALDATE receive time, always a valid integer. The raw
+ * header is preserved separately in `dateRaw`, so nothing is lost.
+ */
+export declare const parseHeaderDate: (raw: string | undefined, fallbackMs: number) => {
+    value: number;
+    usedFallback: boolean;
+};
+/**
+ * @deprecated Use ManagedConnectionFactory instead
+ */
+export type ImapConnectionFactory = () => {
+    connect(): Promise<void>;
+    disconnect(): Promise<void>;
+};
+export interface SyncLogger {
+    info(obj: Record<string, unknown>, msg: string): void;
+    warn(obj: Record<string, unknown>, msg: string): void;
+    error(obj: Record<string, unknown>, msg: string): void;
+}
+export interface SyncedMessage {
+    messageId: string;
+    uid: number;
+}
+export interface SyncMessagesResult {
+    syncedCount: number;
+    syncedMessageIds: string[];
+    syncedMessages: SyncedMessage[];
+    hasMore: boolean;
+    remainingCount: number;
+    /**
+     * The round had work to do and moved no cursor. Nothing throws on this
+     * path — a message that cannot be applied is caught and held back — so
+     * without this flag a mailbox that stops syncing looks exactly like one
+     * with nothing to sync. Callers must surface it.
+     */
+    cursorStalled: boolean;
+}
+/**
+ * Pick the UIDs a full-enumeration round should sync, newest first.
+ *
+ * 1. New messages: UIDs above the forward watermark.
+ * 2. Backfill: UIDs below the lowest one synced so far.
+ * 3. A mailbox with no watermarks at all syncs everything.
+ */
+export declare const selectUidsToSync: (allUids: number[], lastSyncUid: number, highWaterMarkUid: number) => number[];
+export declare class MessageSyncService {
+    private connectionFactory;
+    private mailboxService;
+    /**
+     * The account’s folder-role map. Required, not optional: which folder
+     * holds the Junk role decides whether a sighting withholds its sender
+     * from autocomplete, and a construction without it would quietly harvest
+     * spammers as correspondents.
+     */
+    private mailboxSpecialUseService;
+    private messageService;
+    private envelopeService;
+    private addressService;
+    private threadMessageService;
+    private syncWarningService;
+    /**
+     * Pending outbound flag-push markers (#1273). Supplied, an inbound
+     * metadata change never overwrites a local flip that IMAP has not been
+     * told about yet; omitted, the server always wins.
+     */
+    private flagPushMarkerService?;
+    /**
+     * The canonical flag record. Supplied, an inbound metadata change lands
+     * on the same record the outbound flip path reads, so a user's next
+     * flip is never dismissed as redundant.
+     */
+    private messageFlagService?;
+    /**
+     * The set of messages already set aside (issue #72). Message sync never
+     * writes a record — only the body path can attribute a failure to the
+     * message — but it reads the set so a uid the body path quarantined is
+     * not re-fetched on every round.
+     */
+    private quarantineService?;
+    private log;
+    constructor(connectionFactory: ManagedConnectionFactory, mailboxService: IMailboxRepository, 
+    /**
+     * The account’s folder-role map. Required, not optional: which folder
+     * holds the Junk role decides whether a sighting withholds its sender
+     * from autocomplete, and a construction without it would quietly harvest
+     * spammers as correspondents.
+     */
+    mailboxSpecialUseService: IMailboxSpecialUseRepository, messageService: IMessageRepository, envelopeService: IEnvelopeRepository, addressService: IAddressRepository, threadMessageService: IThreadMessageRepository, syncWarningService: SyncWarningService, logger?: SyncLogger, 
+    /**
+     * Pending outbound flag-push markers (#1273). Supplied, an inbound
+     * metadata change never overwrites a local flip that IMAP has not been
+     * told about yet; omitted, the server always wins.
+     */
+    flagPushMarkerService?: IMessageFlagPushRepository | undefined, 
+    /**
+     * The canonical flag record. Supplied, an inbound metadata change lands
+     * on the same record the outbound flip path reads, so a user's next
+     * flip is never dismissed as redundant.
+     */
+    messageFlagService?: IMessageFlagRepository | undefined, 
+    /**
+     * The set of messages already set aside (issue #72). Message sync never
+     * writes a record — only the body path can attribute a failure to the
+     * message — but it reads the set so a uid the body path quarantined is
+     * not re-fetched on every round.
+     */
+    quarantineService?: QuarantineService | undefined);
+    /**
+     * Sync ONE batch of messages for a mailbox using newest-first strategy.
+     *
+     * Uses dual-watermark tracking:
+     * - highWaterMarkUid: highest UID ever seen (detects new messages)
+     * - lastSyncUid: lowest UID processed (tracks backfill progress)
+     *
+     * Returns hasMore=true if there are more messages to sync. The caller
+     * should re-enqueue another sync event to continue processing.
+     *
+     * @param mailboxId - The database mailbox ID
+     * @param accountId - The account ID (scopes message/thread identity)
+     * @param accountConfigId - The account config ID (used for address linking)
+     * @param batchSize - Number of messages to process per batch
+     */
+    syncMessages(mailboxId: string, accountId: string, accountConfigId: string, batchSize?: number): Promise<SyncMessagesResult>;
+    /**
+     * Re-key a mailbox's stored UIDs against a new UIDVALIDITY axis (#1272).
+     *
+     * One envelope-level pass (UID + Message-ID + INTERNALDATE, no body
+     * fetches — {@link matchCursorRebuild}) matched against the rows already
+     * stored for this mailbox:
+     * - Match → rewrite the row's UID mapping in place; bodies and threads are
+     *   untouched.
+     * - Server message with no row → normal new-message sync (the same
+     *   `trySaveMessage` pipeline the regular batch sync uses).
+     * - Row with no counterpart → expunged; removed through `deleteIfPlacement`
+     *   on the uid the rebuild read (#1283 — the exact same "gone upstream"
+     *   outcome as a body-sync retry exhaustion finding a stale row), so a row
+     *   a move re-pointed in the meantime is left alone.
+     *
+     * Idempotent by construction: re-entering mid-rebuild (crash recovery —
+     * the mailbox was left `rebuilding`) simply redoes the same match/rewrite
+     * pass, which converges on the same result. `cursorState` is stamped
+     * `rebuilding` before any write and only cleared to `normal` after the
+     * watermarks are rebuilt, so a crash anywhere in between leaves the
+     * mailbox paused rather than falling back to the stale axis.
+     */
+    private rebuildCursor;
+    /**
+     * Page through every ThreadMessage row for this mailbox, projecting just
+     * the fields {@link matchCursorRebuild} needs. ThreadMessage (not
+     * Message) is the read source: it already denormalizes `messageIdHeader`
+     * and `internalDate` alongside `uid`, so this needs no per-row Envelope
+     * lookup.
+     */
+    private listExistingCursorRows;
+    /**
+     * One CHANGEDSINCE round (issue #20).
+     *
+     * A single `FETCH ... (CHANGEDSINCE <modseq>)` over the whole UID space
+     * returns both the messages that arrived and the messages whose metadata
+     * changed since the stored watermark, so a flag flipped on another client
+     * is picked up without enumerating the folder. Rows already present in
+     * this folder take the metadata path — the envelope, body structure and
+     * addresses are immutable, so re-writing them for a read-state change would
+     * be pure waste; everything else, a row another client moved here from a
+     * folder it points at included, takes the normal save pipeline (#1386).
+     *
+     * Expunges are invisible here (CONDSTORE without QRESYNC never reports
+     * them, RFC 7162 Section 3.1.2.1) — they remain the reconcile path's job.
+     */
+    private syncChangedSince;
+    private settleHeaderWarnings;
+    /**
+     * Apply one message from a CHANGEDSINCE result without ever rejecting —
+     * same contract as {@link trySaveMessage}, so a single unapplicable change
+     * holds the watermark back instead of failing the round.
+     */
+    private tryApplyChange;
+    private applyChange;
+    /**
+     * Bring a stored row's read and star state, and the $Junk/$NotJunk
+     * keywords on the canonical MessageFlag record, in line with the
+     * server's flags.
+     *
+     * A field with a pending outbound push is left alone: the user flipped it
+     * locally, IMAP has not been told yet, and the server's answer is
+     * therefore known-stale for that field. Writing it back would revert the
+     * flip in front of the user and then push the reverted value.
+     */
+    private applyServerFlags;
+    /**
+     * Set or clear one flag on the canonical record. Both repository calls are
+     * idempotent, so a re-applied change is a no-op rather than a conflict.
+     */
+    private setMessageFlag;
+    private hasPendingPush;
+    /**
+     * Mirror one RFC 5788 keyword ($Junk / $NotJunk) from the server into the
+     * canonical MessageFlag record, guarded the same way as the system flags:
+     * a pending local push wins over the server's known-stale answer.
+     *
+     * This mailbox is never remit-only — Apple Mail or another IMAP client is
+     * routinely connected too — so the keyword is message state, nothing
+     * more. It never reaches the Address model: reading another client's
+     * keyword as a standing rule about the sender is the exact fight this
+     * mirror must not pick.
+     */
+    private mirrorKeywordFlag;
+    /**
+     * Messages the forwarded-message repair queued — their part rows dropped,
+     * the lost row only on the mail server — are fetched again by UID and saved
+     * through the same path as any other sighting, which writes their parts. No
+     * watermark moves: these UIDs are already behind them. Each leaves the queue
+     * once saved or once the server returns no usable row for it, whatever
+     * parts came back; only a failed save keeps it for the next round. Resolves
+     * `true` when it touched the connection, which the probe of departures may
+     * have left on another folder.
+     */
+    private restoreMissingStructure;
+    /**
+     * Fetch a batch of messages using the managed connection.
+     * Assumes mailbox is already open from fetchUidsToSync.
+     */
+    private fetchMessageBatch;
+    /**
+     * Save a single message without ever rejecting. Any error is caught and
+     * reported as a `failed` outcome so it cannot abort the surrounding `pMap`
+     * batch — the failed UID is held back from the watermark and retried next
+     * cycle. This is the guardrail that stops a single unsaveable message from
+     * permanently freezing the mailbox (#817).
+     */
+    /**
+     * Which folders hold Junk and Trash for this account, resolved once per
+     * batch rather than per message: the lookup reads an appointment row and
+     * the account’s mailbox list, and every message in a batch shares both.
+     */
+    private folderRolesFor;
+    /**
+     * Ask the folders this batch's rows point at whether they still hold them
+     * (#1146).
+     *
+     * Between the batch FETCH and the save pass, in all three rounds, and in that
+     * position on purpose. It SELECTs another folder on the one connection this
+     * sync has, so it may not run while this mailbox's own fetches are still
+     * outstanding; and the save pass has to have the answer already, because the
+     * transaction that re-points a row is where the answer is spent. Nothing
+     * after the save pass touches IMAP, so leaving another folder selected costs
+     * the round nothing.
+     */
+    private probeDepartures;
+    private trySaveMessage;
+    private saveMessage;
+    /**
+     * Whether this folder already holds a copy the user made of this message
+     * (#75). A copy is a second row under its own derived id, so the sighting is
+     * that copy standing where it was put — the original has not moved, and
+     * following it would take the mail out of the folder the user copied it from.
+     */
+    private holdsCopyOf;
+    private saveAddresses;
+    /**
+     * The thread a synced message files under. A message keeps the thread it is
+     * already stored in; a reply joins the thread its parent is stored in;
+     * otherwise the thread is derived from the root `threadRootOf` names.
+     */
+    private threadFor;
+    /**
+     * Joins the rows written on either side of a parent and its reply, after
+     * this message's own row exists. Replies that arrived before this message,
+     * and so were keyed on its own id, follow it into its thread; and a reply
+     * whose parent was saved while it was being saved follows the parent, but
+     * only from the thread it derived for itself — a thread it joined or was
+     * already stored in is left as it is.
+     *
+     * Both checks run after the create, so of a parent and a reply saved
+     * concurrently at least one sees the other's row: neither is left in a
+     * thread of its own.
+     */
+    private joinThreads;
+    /**
+     * Create the ThreadMessage for a synced message, then join it to its
+     * parent and replies.
+     *
+     * `category` is the value the caller already holds from the Message it just
+     * wrote, so the row starts out agreeing with it rather than defaulting to
+     * `uncategorized` behind an already-classified message.
+     */
+    private createThreadForMessage;
+}