@remit/mailbox-service 0.0.118 → 0.0.119

package/dist/body-sync.d.ts

@@ -32,6 +32,11 @@ export interface BodySyncLogger {
     warn?(obj: Record<string, unknown>, msg: string): void;
     error?(obj: Record<string, unknown>, msg: string): void;
 }
+export interface SyncBodiesOptions {
+    force: boolean;
+    concurrency: number;
+    bytesInFlight: number;
+}
 export interface SyncBodiesResult {
     syncedCount: number;
     syncedMessageIds: string[];
@@ -127,7 +132,7 @@ export declare class BodySyncService {
     private log;
     private readonly filterPipeline?;
     constructor(messageService: IMessageRepository, storageService: StorageService, threadMessageService: IThreadMessageRepository, addressService: IAddressRepository, envelopeService: IEnvelopeRepository, syncWarningService: SyncWarningService, logger?: BodySyncLogger, placementConfig?: PlacementConfig | undefined, filterConfig?: FilterConfig | undefined, quarantineConfig?: QuarantineConfig | undefined, unsubscribeConfig?: UnsubscribeConfig | undefined, calendarConfig?: CalendarSuggestionConfig | undefined);
-    syncBodies(messageIds: string[], accountId: string, accountConfigId: string, folder: BodySyncRoundFolder, getConnection: ConnectionGetter, force?: boolean): Promise<SyncBodiesResult>;
+    syncBodies(messageIds: string[], accountId: string, accountConfigId: string, folder: BodySyncRoundFolder, getConnection: ConnectionGetter, { force, concurrency, bytesInFlight }: SyncBodiesOptions): Promise<SyncBodiesResult>;
     /**
      * Set a message aside because its body will not parse (issue #72). The
      * fingerprint comes from the BodyPart rows metadata sync already wrote: the
@@ -135,22 +140,10 @@ export declare class BodySyncService {
      */
     private recordQuarantine;
     private buildResult;
-    /**
-     * Stream one message body straight to storage while teeing the bytes into a
-     * buffer for the parse-dependent steps (snippet, classification, parsed-body
-     * cache, per-part objects). The S3 upload never sees a whole-body concat —
-     * the storage service streams it — but mailparser still needs the full bytes,
-     * so we collect them in parallel. Later issues move parsing off the hot path.
-     *
-     * `isReStore` marks a re-fetch of a message whose row already carried
-     * `bodyStorageKey` (the `force` re-arm in {@link syncBodies}): the
-     * post-store steps then re-materialize the body artifacts without
-     * re-running any decision (issue #1011).
-     */
-    private storeStreamedBody;
+    private storeFetchedBody;
     /**
      * The steps every synced body must go through regardless of which path
-     * fetched it first — the sync path ({@link storeStreamedBody}) and the
+     * fetched it first — the sync path ({@link storeFetchedBody}) and the
      * read-path backfill ({@link fetchAndGetBody}) both call this instead of
      * duplicating (and drifting on) their own subset. Before issue #1271,
      * `fetchAndGetBody` skipped classification/placement entirely — a message
@@ -257,11 +250,12 @@ export declare class BodySyncService {
      * move); the caller applies them. A no-op returning no actions when no
      * {@link FilterConfig} is wired.
      *
-     * Isolation lives in {@link FilterPipeline.evaluate}: this read/decision phase
-     * never fails the body store (the #1246 placement precedent). The applies of
-     * the returned decision are the caller's and are deliberately unswallowed.
+     * A failed filter read never fails the body store (the #1246 placement
+     * precedent). The applies of the returned decision are the caller's and are
+     * deliberately unswallowed.
      */
     private evaluateFilters;
+    private batchLookups;
     /**
      * Resolve the placement decision for a message, BEFORE the single Message
      * update — so both the `movedByRemit` flag and the audit verdict join that

package/dist/body-sync.js

@@ -1,4 +1,3 @@
-import { PassThrough } from "node:stream";
 import { inspect } from "node:util";
 import { provisionDefaultCalendar, recordCalendarSuggestion, } from "@remit/calendar-service";
 import { isSenderMuted, } from "@remit/data-ports";
@@ -8,6 +7,7 @@ import { isEncapsulatedMessage } from "@remit/data-ports/message-parts";
 import { isBulkSender } from "@remit/data-ports/wellknown";
 import { FilterState, MessageClassificationState, PlacementAction, PlacementConfidence, QuarantineFailureStage, SenderTrust, SyncWarningStage, } from "@remit/domain-enums";
 import { isStorageNotFoundError, } from "@remit/storage-service";
+import { Semaphore } from "async-mutex";
 import { simpleParser } from "mailparser";
 import pMap from "p-map";
 import { BodyParseError, parseMessageBody } from "./body-parse.js";
@@ -16,7 +16,7 @@ import { calendarParts } from "./calendar-parts.js";
 import { classifyParsedMessage, extractPrimaryFromEmail, } from "./classify-message.js";
 import { extractListId } from "./filters/list-id.js";
 import { FilterPipeline, } from "./filters/pipeline.js";
-import { placementOf, resolveFolderRoleMailboxes } from "./folder-placement.js";
+import { placementOf, resolveFolderRoleMailboxes, } from "./folder-placement.js";
 import { classifyPlacement, resolveSenderPlacement, } from "./heuristics/classifyPlacement.js";
 import { denormalizeMessageCategory, hasDecidedCategory, } from "./message-category.js";
 import { isMessageGoneFromOpenMailbox } from "./message-presence.js";
@@ -137,6 +137,22 @@ export const toParsedBody = (parsed) => ({
 const noopLogger = {
     info: () => { },
 };
+// A failed load is forgotten, so the next message retries it rather than the
+// rest of the batch inheriting the failure.
+const once = (load) => {
+    let loaded;
+    return () => {
+        if (loaded)
+            return loaded;
+        const attempt = load();
+        loaded = attempt;
+        attempt.then(undefined, () => {
+            if (loaded === attempt)
+                loaded = undefined;
+        });
+        return attempt;
+    };
+};
 export class BodySyncService {
     messageService;
     storageService;
@@ -168,9 +184,7 @@ export class BodySyncService {
             ? new FilterPipeline(filterConfig, this.log)
             : undefined;
     }
-    async syncBodies(messageIds, accountId, accountConfigId, folder, getConnection, force = false) {
-        const syncedMessageIds = [];
-        const syncedUids = [];
+    async syncBodies(messageIds, accountId, accountConfigId, folder, getConnection, { force, concurrency, bytesInFlight }) {
         let skippedCount = 0;
         const location = {
             accountId,
@@ -234,63 +248,81 @@ export class BodySyncService {
             pending.set(message.uid, messageId);
         }
         if (pending.size === 0) {
-            return this.buildResult(syncedMessageIds, skippedCount, []);
+            return this.buildResult([], skippedCount, []);
         }
         const connection = await getConnection();
         // Single SELECT for the whole batch. openBox is idempotent, so a warm
         // connection already on this mailbox skips the SELECT entirely.
         await connection.openBox(folder.fullPath);
-        let connectionLost = false;
-        try {
-            for await (const { uid, source } of connection.fetchMessageBodies([
-                ...pending.keys(),
-            ])) {
+        const lookups = this.batchLookups(accountConfigId, accountId);
+        const log = this.log;
+        const stream = { lost: false, fault: null, failed: false };
+        const budget = new Semaphore(bytesInFlight);
+        const endStream = (error) => {
+            log.error?.({ error: error.message }, "Body fetch stream failed");
+            // A dropped connection leaves every not-yet-yielded UID in `pending`;
+            // they fall through to failedMessageIds and re-enqueue. Any other error
+            // is a real fault, raised once the stores already running have finished.
+            if (isConnectionDrop(error)) {
+                stream.lost = true;
+                log.info?.({ remainingCount: pending.size }, "Connection lost, aborting batch");
+                return null;
+            }
+            stream.fault = { error };
+            return null;
+        };
+        // The FETCH stays one command on the one connection, read in order; no
+        // row is pulled after a store has failed.
+        async function* fetchedBodies() {
+            const rows = connection.fetchMessageBodies([...pending.keys()]);
+            for (;;) {
+                const step = await rows.next().catch(endStream);
+                if (!step || step.done)
+                    return;
+                const { uid, source } = step.value;
                 const messageId = pending.get(uid);
-                if (!messageId) {
-                    // A UID we didn't ask for — drain the stream so the connection
-                    // stays usable, then ignore it.
-                    source.resume();
+                if (!messageId)
                     continue;
+                if (stream.failed) {
+                    await rows.return(undefined);
+                    return;
                 }
-                try {
-                    await this.storeStreamedBody(messageId, accountId, accountConfigId, source, folder.placement, reStoredMessageIds.has(messageId));
-                }
-                catch (error) {
-                    // The one error in this frame that is the message's own fault. It
-                    // reaches here from the parse call and nowhere else, so the S3, DB
-                    // and queue work this same frame wraps cannot be mistaken for it.
-                    // The row is durable before the UID leaves `pending`, so the
-                    // message is never let go of unrecorded.
-                    if (error instanceof BodyParseError && this.quarantineConfig) {
-                        await this.recordQuarantine(this.quarantineConfig, messageId, location, error);
-                        pending.delete(uid);
-                        skippedCount++;
-                        source.resume();
-                        continue;
-                    }
-                    // Anything else is our own storage, database or queue failing. It
-                    // aborts the batch, which redelivers whole; what stored skips.
-                    throw error;
-                }
+                yield { uid, messageId, body: source };
+            }
+        }
+        const stored = await pMap(fetchedBodies(), async ({ uid, messageId, body }) => {
+            // A body larger than the whole budget takes all of it and goes alone.
+            const outcome = await budget.runExclusive(() => this.storeFetchedBody(messageId, accountId, accountConfigId, body, folder.placement, reStoredMessageIds.has(messageId), lookups, location), Math.min(body.length, bytesInFlight));
+            if (outcome.kind === "Failed")
+                stream.failed = true;
+            return { uid, messageId, outcome };
+        }, { concurrency });
+        const syncedMessageIds = [];
+        const syncedUids = [];
+        const faults = [];
+        for (const { uid, messageId, outcome } of stored) {
+            if (outcome.kind === "Stored") {
                 pending.delete(uid);
                 syncedMessageIds.push(messageId);
                 syncedUids.push(uid);
             }
-        }
-        catch (error) {
-            this.log.error?.({ error: error.message }, "Body fetch stream failed");
-            // Fail-fast: a dropped connection mid-stream leaves every not-yet-yielded
-            // UID in `pending`; they fall through to failedMessageIds and re-enqueue.
-            // Any other error is a real fault — let it crash.
-            if (!isConnectionDrop(error)) {
-                throw error;
+            if (outcome.kind === "Quarantined") {
+                pending.delete(uid);
+                skippedCount++;
             }
-            connectionLost = true;
-            this.log.info?.({ remainingCount: pending.size }, "Connection lost, aborting batch");
+            if (outcome.kind === "Failed")
+                faults.push(outcome.error);
         }
+        // A store failing on our own storage, database or queue fails the batch,
+        // which redelivers whole; what stored skips on the redelivery.
+        if (faults.length > 0)
+            throw faults[0];
+        if (stream.fault)
+            throw stream.fault.error;
         // Anything still pending was never yielded (mid-stream drop or a UID the
         // server silently omitted) — re-enqueue it.
         const failedMessageIds = [...pending.values()];
+        const connectionLost = stream.lost;
         await this.syncWarningService.settle({
             accountConfigId,
             accountId,
@@ -341,56 +373,36 @@ export class BodySyncService {
             failedMessageIds,
         };
     }
-    /**
-     * Stream one message body straight to storage while teeing the bytes into a
-     * buffer for the parse-dependent steps (snippet, classification, parsed-body
-     * cache, per-part objects). The S3 upload never sees a whole-body concat —
-     * the storage service streams it — but mailparser still needs the full bytes,
-     * so we collect them in parallel. Later issues move parsing off the hot path.
-     *
-     * `isReStore` marks a re-fetch of a message whose row already carried
-     * `bodyStorageKey` (the `force` re-arm in {@link syncBodies}): the
-     * post-store steps then re-materialize the body artifacts without
-     * re-running any decision (issue #1011).
-     */
-    async storeStreamedBody(messageId, accountId, accountConfigId, source, placement, isReStore) {
-        const toStorage = new PassThrough();
-        const chunks = [];
-        // Tee the source: bytes flow to storage as a stream (no whole-body concat
-        // on the upload path) while we also collect them for the parse-dependent
-        // steps below, which still need the full body for mailparser. The store
-        // and the tee are awaited together so neither rejection is orphaned.
-        const tee = new Promise((resolve, reject) => {
-            source.on("data", (chunk) => {
-                chunks.push(chunk);
-                toStorage.write(chunk);
-            });
-            source.on("end", () => {
-                toStorage.end();
-                resolve();
-            });
-            source.on("error", (err) => {
-                toStorage.destroy(err);
-                reject(err);
-            });
-        });
-        const [ref] = await Promise.all([
-            this.storageService.storeMessageBodyStream({
-                accountConfigId,
-                accountId,
-                messageId,
-                content: toStorage,
-            }),
-            tee,
-        ]);
-        const body = Buffer.concat(chunks);
-        await this.applyPostStoreSteps(messageId, accountId, accountConfigId, body, {
-            uri: ref.uri,
-        }, placement, isReStore);
+    async storeFetchedBody(messageId, accountId, accountConfigId, body, placement, isReStore, lookups, location) {
+        const stored = await this.storageService
+            .storeMessageBody({
+            accountConfigId,
+            accountId,
+            messageId,
+            content: body,
+        })
+            .then((ref) => this.applyPostStoreSteps(messageId, accountId, accountConfigId, body, { uri: ref.uri }, placement, isReStore, lookups))
+            .then(() => ({ kind: "Stored" }), (error) => ({ kind: "Failed", error }));
+        if (stored.kind !== "Failed")
+            return stored;
+        const { error } = stored;
+        // The one error in this frame that is the message's own fault. It
+        // reaches here from the parse call and nowhere else, so the S3, DB and
+        // queue work this same frame wraps cannot be mistaken for it. The row is
+        // durable before the UID leaves `pending`, so the message is never let go
+        // of unrecorded.
+        if (!(error instanceof BodyParseError) || !this.quarantineConfig) {
+            this.log.error?.({ messageId, accountId, error: inspect(error) }, "Body store failed");
+            return stored;
+        }
+        return this.recordQuarantine(this.quarantineConfig, messageId, location, error).then(() => ({ kind: "Quarantined" }), (recordError) => ({
+            kind: "Failed",
+            error: recordError,
+        }));
     }
     /**
      * The steps every synced body must go through regardless of which path
-     * fetched it first — the sync path ({@link storeStreamedBody}) and the
+     * fetched it first — the sync path ({@link storeFetchedBody}) and the
      * read-path backfill ({@link fetchAndGetBody}) both call this instead of
      * duplicating (and drifting on) their own subset. Before issue #1271,
      * `fetchAndGetBody` skipped classification/placement entirely — a message
@@ -420,7 +432,7 @@ export class BodySyncService {
      * placement/filter/category machinery is skipped entirely and the
      * message's existing category is carried forward unchanged.
      */
-    isReStore) {
+    isReStore, lookups) {
         // The one operation on this path that can fail because of how the message
         // is built. Its own try block lives in `parseMessageBody`, so this frame —
         // which also wraps the S3 body write, the parsed-body cache, the placement
@@ -465,12 +477,8 @@ export class BodySyncService {
         // UpdateItem. The verdict is recorded whenever Remit decided to act
         // (action != leave); the move is enqueued only for a confident,
         // actionable verdict.
-        const resolved = await this.resolvePlacement(messageId, accountId, accountConfigId, parsed, classification);
-        // Index-time filter pass (RFC 034). Isolated the same way placement is —
-        // FilterPipeline.evaluate swallows any read/decision failure and returns
-        // no actions, so a schema-drifted filter query never fails the body store.
-        // The DECISION is computed here; the applies below are unswallowed.
-        const filterDecision = await this.evaluateFilters(accountConfigId, accountId, messageId, parsed);
+        const resolved = await this.resolvePlacement(lookups, messageId, accountId, accountConfigId, parsed, classification);
+        const filterDecision = await this.evaluateFilters(lookups, messageId, parsed);
         // Store the parsed-body cache BEFORE persisting bodyStorageKey. The skip
         // guard in syncBodies treats a stored bodyStorageKey as "fully synced",
         // so if we wrote it first and the parsed-cache write then failed, the
@@ -549,7 +557,7 @@ export class BodySyncService {
         // the message is already \Seen), so a failure here safely re-fires on the
         // next attempt rather than being lost behind the bodyStorageKey skip guard.
         await this.applyUnsubscribedAutoRead(messageId, accountId, accountConfigId, parsed, existingMessage.bodyStorageKey);
-        await this.deriveCalendarSuggestions(messageId, accountConfigId, parsed, existingMessage.bodyStorageKey);
+        await this.deriveCalendarSuggestions(lookups, messageId, accountConfigId, parsed, existingMessage.bodyStorageKey);
         const moved = Boolean(resolved.move || filterMoved);
         // ONE Message UpdateItem per synced message: bodyStorageKey + every
         // classification/derived field + the move flag + the audit verdict,
@@ -657,13 +665,11 @@ export class BodySyncService {
                 messageId,
                 content: body,
             });
-            // Same shared step the sync path (storeStreamedBody) runs — issue
-            // #1271 unified the two body paths so classification/placement no
-            // longer depends on which one fetched the body first. A store from
-            // here is a re-store exactly when the row already claimed a stored
-            // body — the NoSuchKey fallback above; a row with no bodyStorageKey is
-            // a genuine first store and takes the full decision pass.
-            parsed = await this.applyPostStoreSteps(messageId, accountId, accountConfigId, body, { uri: ref.uri }, folder.placement, message.bodyStorageKey != null);
+            // A store from here is a re-store exactly when the row already claimed
+            // a stored body — the NoSuchKey fallback above; a row with no
+            // bodyStorageKey is a genuine first store and takes the full decision
+            // pass.
+            parsed = await this.applyPostStoreSteps(messageId, accountId, accountConfigId, body, { uri: ref.uri }, folder.placement, message.bodyStorageKey != null, this.batchLookups(accountConfigId, accountId));
         }
         else {
             parsed = await simpleParser(body);
@@ -839,7 +845,7 @@ export class BodySyncService {
      * repository failure here is logged loudly with the messageId rather than
      * failing a body store that otherwise succeeded.
      */
-    async deriveCalendarSuggestions(messageId, accountConfigId, parsed, storedBodyKey) {
+    async deriveCalendarSuggestions(lookups, messageId, accountConfigId, parsed, storedBodyKey) {
         if (!this.calendarConfig)
             return;
         if (hasStoredBody(storedBodyKey))
@@ -847,7 +853,7 @@ export class BodySyncService {
         const parts = calendarParts(parsed);
         if (parts.length === 0)
             return;
-        const { calendarStore, calendarSuggestionService, filterService } = this.calendarConfig;
+        const { calendarStore, calendarSuggestionService } = this.calendarConfig;
         await Promise.resolve()
             .then(async () => {
             // `dismiss{muteSender:true}` wrote a standing rule naming this
@@ -856,7 +862,7 @@ export class BodySyncService {
             // move, so this is the only reader such a rule has.
             const sender = extractPrimaryFromEmail(parsed);
             if (sender) {
-                const active = await filterService.listByAccountAndState(accountConfigId, FilterState.Active);
+                const active = await lookups.activeFilters();
                 if (isSenderMuted(active, sender)) {
                     this.log.debug?.({ messageId, sender }, "Sender is muted; offering no calendar suggestion");
                     return;
@@ -914,14 +920,42 @@ export class BodySyncService {
      * move); the caller applies them. A no-op returning no actions when no
      * {@link FilterConfig} is wired.
      *
-     * Isolation lives in {@link FilterPipeline.evaluate}: this read/decision phase
-     * never fails the body store (the #1246 placement precedent). The applies of
-     * the returned decision are the caller's and are deliberately unswallowed.
+     * A failed filter read never fails the body store (the #1246 placement
+     * precedent). The applies of the returned decision are the caller's and are
+     * deliberately unswallowed.
      */
-    async evaluateFilters(accountConfigId, accountId, messageId, parsed) {
-        if (!this.filterPipeline)
+    async evaluateFilters(lookups, messageId, parsed) {
+        const { filterPipeline } = this;
+        if (!filterPipeline)
             return { labels: [] };
-        return this.filterPipeline.evaluate(accountConfigId, accountId, messageId, toFilterMessage(parsed));
+        return lookups
+            .usableFilters()
+            .then((usable) => filterPipeline.decide(usable, toFilterMessage(parsed)))
+            .catch((error) => {
+            this.log.error?.({
+                alert: "body_sync_filter_match_failed",
+                messageId,
+                errorName: error?.name,
+                error: inspect(error),
+            }, "Filter matching failed; body already stored, continuing without filter actions (best-effort, non-fatal)");
+            return { labels: [] };
+        });
+    }
+    batchLookups(accountConfigId, accountId) {
+        const { placementConfig, filterPipeline } = this;
+        const filterService = this.filterConfig?.filterService ?? this.calendarConfig?.filterService;
+        const activeFilters = once(async () => filterService
+            ? filterService.listByAccountAndState(accountConfigId, FilterState.Active)
+            : []);
+        return {
+            roleMailboxes: once(async () => placementConfig
+                ? resolveFolderRoleMailboxes(placementConfig.mailboxSpecialUseService, accountId)
+                : { junkMailbox: null, inboxMailbox: null }),
+            activeFilters,
+            usableFilters: once(async () => filterPipeline
+                ? filterPipeline.prepare(accountConfigId, accountId, await activeFilters())
+                : []),
+        };
     }
     /**
      * Resolve the placement decision for a message, BEFORE the single Message
@@ -950,10 +984,10 @@ export class BodySyncService {
      * to leave the message alone or placement itself failed; the alert log is
      * what distinguishes the latter, and neither case marks the message decided.
      */
-    async resolvePlacement(messageId, accountId, accountConfigId, parsed, classification) {
+    async resolvePlacement(lookups, messageId, accountId, accountConfigId, parsed, classification) {
         if (!this.placementConfig)
             return {};
-        return this.computePlacement(this.placementConfig, messageId, accountId, accountConfigId, parsed, classification).catch((error) => {
+        return this.computePlacement(this.placementConfig, lookups, messageId, accountId, accountConfigId, parsed, classification).catch((error) => {
             this.log.error?.({
                 alert: "body_sync_placement_failed",
                 messageId,
@@ -965,7 +999,7 @@ export class BodySyncService {
             return {};
         });
     }
-    async computePlacement(placementConfig, messageId, accountId, accountConfigId, parsed, classification) {
+    async computePlacement(placementConfig, lookups, messageId, accountId, accountConfigId, parsed, classification) {
         const { mailboxSpecialUseService } = placementConfig;
         const message = await this.messageService.get(messageId);
         // Issue #383 / #1011: placement still runs once per message (RFC 039
@@ -974,7 +1008,7 @@ export class BodySyncService {
         // (`fetchAndGetBody`'s `NoSuchKey` fallback, `syncBodies(..., force:
         // true)`) skip the decision steps entirely, so only the pass that first
         // stores a body ever reaches this method.
-        const roleMailboxes = await resolveFolderRoleMailboxes(mailboxSpecialUseService, accountId);
+        const roleMailboxes = await lookups.roleMailboxes();
         const { junkMailbox, inboxMailbox } = roleMailboxes;
         const placement = placementOf(message.mailboxId, roleMailboxes);
         const fromEmail = extractPrimaryFromEmail(parsed);

package/dist/filters/pipeline.d.ts

@@ -1,4 +1,4 @@
-import type { IFilterRepository, IMailboxRepository, IMessageLabelRepository } from "@remit/data-ports";
+import type { FilterItem, IFilterRepository, IMailboxRepository, IMessageLabelRepository } from "@remit/data-ports";
 import type { PlacementMoveService } from "../placement-move.js";
 import { type FilterMessage } from "./match.js";
 export interface FilterLogger {
@@ -34,25 +34,23 @@ export interface FilterDecision {
         filterId: string;
     };
 }
+export type PreparedFilters = readonly FilterItem[];
 /**
  * Evaluates a synced message against the account's active filters and resolves
  * the actions to apply (RFC 034). Reads only — listing filters and refreshing
  * lazy expiry; the caller applies the
  * resulting {@link FilterDecision}.
  *
- * `evaluate` isolates every failure in this read/decision phase from body sync,
- * matching the placement-resolution precedent (#1246): a schema-drifted query
- * must never fail the surrounding message store, since the body is
- * already durable by the time this runs. The apply of the decision — label
- * writes and the move enqueue — is the caller's responsibility and is *not*
- * swallowed there, so a genuine DDB/SQS write failure still propagates and
- * requeues the message.
+ * `prepare` narrows the account's active filters to the ones that can act on
+ * one account's mail; `decide` matches a message against them without a read.
+ * The apply of the decision — label writes and the move enqueue — is the
+ * caller's responsibility.
  */
 export declare class FilterPipeline {
     private readonly config;
     private readonly log;
     constructor(config: FilterConfig, log: FilterLogger);
-    evaluate(accountConfigId: string, accountId: string, messageId: string, msg: FilterMessage): Promise<FilterDecision>;
+    decide(prepared: PreparedFilters, msg: FilterMessage): FilterDecision;
     /**
      * A filter sits on the account config and sees every account's mail, but its
      * folder is one account's: mail of another account is not this filter's to
@@ -60,5 +58,5 @@ export declare class FilterPipeline {
      * more turns the filter off, with the reason the filter list shows.
      */
     private appliesTo;
-    private match;
+    prepare(accountConfigId: string, accountId: string, active: readonly FilterItem[]): Promise<PreparedFilters>;
 }

package/dist/filters/pipeline.js

@@ -1,4 +1,3 @@
-import { inspect } from "node:util";
 import { FilterDisabledReason, FilterState } from "@remit/domain-enums";
 import { literalClausesMatch, NO_ACTION, selectMoveWinner, } from "./match.js";
 const EMPTY_DECISION = { labels: [] };
@@ -8,13 +7,10 @@ const EMPTY_DECISION = { labels: [] };
  * lazy expiry; the caller applies the
  * resulting {@link FilterDecision}.
  *
- * `evaluate` isolates every failure in this read/decision phase from body sync,
- * matching the placement-resolution precedent (#1246): a schema-drifted query
- * must never fail the surrounding message store, since the body is
- * already durable by the time this runs. The apply of the decision — label
- * writes and the move enqueue — is the caller's responsibility and is *not*
- * swallowed there, so a genuine DDB/SQS write failure still propagates and
- * requeues the message.
+ * `prepare` narrows the account's active filters to the ones that can act on
+ * one account's mail; `decide` matches a message against them without a read.
+ * The apply of the decision — label writes and the move enqueue — is the
+ * caller's responsibility.
  */
 export class FilterPipeline {
     config;
@@ -23,17 +19,32 @@ export class FilterPipeline {
         this.config = config;
         this.log = log;
     }
-    async evaluate(accountConfigId, accountId, messageId, msg) {
-        return this.match(accountConfigId, accountId, msg).catch((error) => {
-            this.log.error?.({
-                alert: "body_sync_filter_match_failed",
-                messageId,
-                accountConfigId,
-                errorName: error?.name,
-                error: inspect(error),
-            }, "Filter matching failed; body already stored, continuing without filter actions (best-effort, non-fatal)");
+    decide(prepared, msg) {
+        const matched = prepared.filter((filter) => filterMatches(filter, msg));
+        if (matched.length === 0)
             return EMPTY_DECISION;
-        });
+        const labels = [];
+        const seenLabels = new Set();
+        for (const filter of matched) {
+            if (filter.actionLabelId === NO_ACTION)
+                continue;
+            if (seenLabels.has(filter.actionLabelId))
+                continue;
+            seenLabels.add(filter.actionLabelId);
+            labels.push({ labelId: filter.actionLabelId, filterId: filter.filterId });
+        }
+        const winner = selectMoveWinner(matched.filter((filter) => filter.actionMailboxId !== NO_ACTION));
+        return {
+            labels,
+            ...(winner
+                ? {
+                    move: {
+                        destinationMailboxId: winner.actionMailboxId,
+                        filterId: winner.filterId,
+                    },
+                }
+                : {}),
+        };
     }
     /**
      * A filter sits on the account config and sees every account's mail, but its
@@ -54,11 +65,8 @@ export class FilterPipeline {
         this.log.warn?.({ filterId: filter.filterId, accountConfigId }, "Filter turned off: the folder it moves mail into no longer exists");
         return false;
     }
-    async match(accountConfigId, accountId, msg) {
-        const active = await this.config.filterService.listByAccountAndState(accountConfigId, FilterState.Active);
-        if (active.length === 0)
-            return EMPTY_DECISION;
-        const matched = [];
+    async prepare(accountConfigId, accountId, active) {
+        const filters = [];
         for (const filter of active) {
             // Lazy expiry (RFC 034 Decision 1.2): reading a Temporary filter past
             // its expiresAt patches it to Expired on this read and drops it from
@@ -68,33 +76,9 @@ export class FilterPipeline {
                 continue;
             if (!(await this.appliesTo(accountConfigId, accountId, usable)))
                 continue;
-            if (filterMatches(usable, msg))
-                matched.push(usable);
+            filters.push(usable);
         }
-        if (matched.length === 0)
-            return EMPTY_DECISION;
-        const labels = [];
-        const seenLabels = new Set();
-        for (const filter of matched) {
-            if (filter.actionLabelId === NO_ACTION)
-                continue;
-            if (seenLabels.has(filter.actionLabelId))
-                continue;
-            seenLabels.add(filter.actionLabelId);
-            labels.push({ labelId: filter.actionLabelId, filterId: filter.filterId });
-        }
-        const winner = selectMoveWinner(matched.filter((filter) => filter.actionMailboxId !== NO_ACTION));
-        return {
-            labels,
-            ...(winner
-                ? {
-                    move: {
-                        destinationMailboxId: winner.actionMailboxId,
-                        filterId: winner.filterId,
-                    },
-                }
-                : {}),
-        };
+        return filters;
     }
 }
 const filterMatches = (filter, msg) => literalClausesMatch(filter.literalClauses, filter.matchOperator, msg);

package/dist/imapflow-connection.d.ts

@@ -4,7 +4,6 @@
  * Modern async/await replacement for the node-imap based ImapConnection.
  * Provides the same interface but uses ImapFlow under the hood.
  */
-import { Readable } from "node:stream";
 import type { FlatMailboxInfo, ImapBoxStatus, ImapConnectionConfig, ImapConnectionState, ImapEnvelopeSnapshot, ImapMailboxStatus, ImapMessage, ImapNamespaces } from "./types.js";
 /**
  * Convert an envelope date into an ISO string, tolerating non-Date values.
@@ -196,21 +195,9 @@ export declare class ImapFlowConnection {
      * @returns The raw message body, or null when the UID FETCH matched no message
      */
     fetchMessageBody: (uid: number) => Promise<Buffer | null>;
-    /**
-     * Fetch full message bodies (RFC822 source) for many UIDs in ONE pipelined
-     * ranged UID FETCH on a single connection — the desktop-client pattern.
-     *
-     * Mirrors `fetchMessages`: one comma-joined UID range, one `client.fetch`,
-     * one SELECT (the caller opens the box once for the whole batch). Yields
-     * `{ uid, source }` as each message arrives so the caller can stream each
-     * body straight to storage without buffering the whole batch.
-     *
-     * `source` is a readable stream over the message bytes — callers must
-     * consume it (e.g. pipe to an upload) before requesting the next item.
-     */
     fetchMessageBodies(uids: number[]): AsyncGenerator<{
         uid: number;
-        source: Readable;
+        source: Buffer;
     }>;
     /**
      * Parse the References header from IMAP headers buffer.

package/dist/imapflow-connection.js

@@ -4,7 +4,6 @@
  * Modern async/await replacement for the node-imap based ImapConnection.
  * Provides the same interface but uses ImapFlow under the hood.
  */
-import { Readable } from "node:stream";
 import { ImapFlow } from "imapflow";
 import { simpleParser } from "mailparser";
 import { MailConnectionError } from "./types.js";
@@ -613,18 +612,6 @@ export class ImapFlowConnection {
         }
         return null;
     };
-    /**
-     * Fetch full message bodies (RFC822 source) for many UIDs in ONE pipelined
-     * ranged UID FETCH on a single connection — the desktop-client pattern.
-     *
-     * Mirrors `fetchMessages`: one comma-joined UID range, one `client.fetch`,
-     * one SELECT (the caller opens the box once for the whole batch). Yields
-     * `{ uid, source }` as each message arrives so the caller can stream each
-     * body straight to storage without buffering the whole batch.
-     *
-     * `source` is a readable stream over the message bytes — callers must
-     * consume it (e.g. pipe to an upload) before requesting the next item.
-     */
     async *fetchMessageBodies(uids) {
         this.ensureConnected();
         const { client } = this;
@@ -652,7 +639,7 @@ export class ImapFlowConnection {
                 if (msg.uid == null || !msg.source) {
                     continue;
                 }
-                yield { uid: msg.uid, source: Readable.from(msg.source) };
+                yield { uid: msg.uid, source: msg.source };
             }
         }
         catch (error) {

package/dist/index.d.ts

@@ -16,7 +16,7 @@ export { type ImapTestConfig, testImapConnection, } from "./connection-test.js";
 export { type CreatedFilter, createFilter, type FilterPage, type FilterPatch, listFilters, type NewFilter, resolveFilterUpdate, updateFilter, } from "./filter-operations.js";
 export { extractListId, normalizeListId } from "./filters/list-id.js";
 export { clauseMatches, type FilterMessage, type LiteralClauseNarrowing, literalClausesMatch, literalClauseTerms, NO_ACTION, selectMoveWinner, } from "./filters/match.js";
-export { type FilterConfig, type FilterDecision, type FilterLogger, FilterPipeline, } from "./filters/pipeline.js";
+export { type FilterConfig, type FilterDecision, type FilterLogger, FilterPipeline, type PreparedFilters, } from "./filters/pipeline.js";
 export { type FlagPushConfig, type FlagPushEvent, type FlagPushLogger, type FlagPushOperationValue, FlagPushService, } from "./flag-push.js";
 export { type FlagPushTerminalOutcome, type ResolveExhaustedFlagPushDeps, type ResolveExhaustedFlagPushInput, type ResolveExhaustedFlagPushResult, resolveExhaustedFlagPushFailure, } from "./flag-push-terminal.js";
 export { type FlagQueueConfig, type FlagQueueLogger, FlagQueueService, type UpdateFlagsInput, type UpdateFlagsResult, } from "./flag-queue.js";

package/dist/test-helpers/fake-imap.d.ts

@@ -1,4 +1,3 @@
-import { Readable } from "node:stream";
 import type { FlatMailboxInfo, IImapConnection, ImapBodyStructure, ImapBoxStatus, ImapConnectionState, ImapEnvelopeSnapshot, ImapMailboxStatus, ImapMessage, ImapNamespaces } from "../types.js";
 interface StoredMessage {
     raw: Buffer;
@@ -112,7 +111,7 @@ export declare class FakeImapConnection implements IImapConnection {
     fetchMessageBody: (uid: number) => Promise<Buffer | null>;
     fetchMessageBodies(uids: number[]): AsyncGenerator<{
         uid: number;
-        source: Readable;
+        source: Buffer;
     }>;
     private touch;
     addFlags: (uids: number[], flags: string[]) => Promise<void>;

package/dist/test-helpers/fake-imap.js

@@ -1,4 +1,3 @@
-import { Readable } from "node:stream";
 import { simpleParser } from "mailparser";
 /** An error shaped the way ImapFlow reports a tagged NO from the server. */
 export const imapError = (code, text) => Object.assign(new Error(`Command failed: [${code}] ${text}`), {
@@ -393,7 +392,7 @@ export class FakeImapConnection {
         for (const uid of uids) {
             const stored = folder.messages.get(uid);
             if (stored)
-                yield { uid, source: Readable.from([stored.raw]) };
+                yield { uid, source: stored.raw };
         }
     }
     touch = (detail, uids, change) => {

package/dist/types.d.ts

@@ -1,7 +1,6 @@
 /**
  * Types for the remit-mailbox-service
  */
-import type { Readable } from "node:stream";
 import type { ImapTransport } from "imapflow";
 export type { ImapTransport };
 /**
@@ -233,13 +232,12 @@ export interface IImapConnection {
     fetchMessageBody(uid: number): Promise<Buffer | null>;
     /**
      * Fetch full message bodies for many UIDs in ONE pipelined ranged UID FETCH.
-     * Yields `{ uid, source }` per message; `source` is a readable stream the
-     * caller must consume before requesting the next item. Requires a mailbox to
-     * be open (one SELECT for the whole batch).
+     * Yields `{ uid, source }` per message. Requires a mailbox to be open (one
+     * SELECT for the whole batch).
      */
     fetchMessageBodies(uids: number[]): AsyncGenerator<{
         uid: number;
-        source: Readable;
+        source: Buffer;
     }>;
     /**
      * Add flags to messages by UID.

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@remit/mailbox-service",
-	"version": "0.0.118",
+	"version": "0.0.119",
 	"type": "module",
 	"main": "dist/index.js",
 	"types": "dist/index.d.ts",
@@ -62,6 +62,7 @@
 		"imapflow": "^2.9.1",
 		"mailparser": "3.12.2",
 		"p-map": "^7.0.4",
+		"async-mutex": "0.5.0",
 		"tldts": "^7.4.9",
 		"@remit/data-ports": "*",
 		"@remit/domain-enums": "*",