@remit/mailbox-service 1.0.0 → 2.0.0

package/dist/body-sync.js

@@ -15,7 +15,7 @@ import { mapMessagePartsToContent, parsePartSource, } from "./body-part-mapper.j
 import { calendarParts } from "./calendar-parts.js";
 import { classifyParsedMessage, extractPrimaryFromEmail, } from "./classify-message.js";
 import { extractListId } from "./filters/list-id.js";
-import { FilterPipeline, } from "./filters/pipeline.js";
+import { FilterPipeline, filtersForArrival, } from "./filters/pipeline.js";
 import { placementOf, resolveFolderRoleMailboxes, } from "./folder-placement.js";
 import { classifyPlacement, resolveSenderPlacement, } from "./heuristics/classifyPlacement.js";
 import { denormalizeMessageCategory, hasDecidedCategory, } from "./message-category.js";
@@ -119,6 +119,13 @@ export const toParsedBody = (parsed) => ({
 const noopLogger = {
     info: () => { },
 };
+const internalDateOf = (internalDates, messageId) => {
+    const internalDate = internalDates.get(messageId);
+    if (internalDate === undefined) {
+        throw new Error(`${messageId} was fetched but never read from the batch`);
+    }
+    return internalDate;
+};
 // A failed load is forgotten, so the next message retries it rather than the
 // rest of the batch inheriting the failure.
 const once = (load) => {
@@ -188,6 +195,7 @@ export class BodySyncService {
         // decision re-run. A message with no stored body inside a forced batch
         // is still a first store and takes the full decision pass.
         const reStoredMessageIds = new Set();
+        const internalDates = new Map();
         // One read per round, not per message (issue #72). The list is small by
         // design and almost always empty, so a lookup per message would put a
         // query on the hot path for a state that is nearly never set.
@@ -228,6 +236,7 @@ export class BodySyncService {
             if (message.bodyStorageKey)
                 reStoredMessageIds.add(messageId);
             pending.set(message.uid, messageId);
+            internalDates.set(messageId, message.internalDate);
         }
         if (pending.size === 0) {
             return this.buildResult([], skippedCount, []);
@@ -274,7 +283,10 @@ export class BodySyncService {
         }
         const stored = await pMap(fetchedBodies(), async ({ uid, messageId, body }) => {
             // A body larger than the whole budget takes all of it and goes alone.
-            const outcome = await budget.runExclusive(() => this.storeFetchedBody(messageId, accountId, accountConfigId, body, folder.placement, reStoredMessageIds.has(messageId), lookups, location), Math.min(body.length, bytesInFlight));
+            const outcome = await budget.runExclusive(() => this.storeFetchedBody(messageId, accountId, accountConfigId, body, {
+                placement: folder.placement,
+                internalDate: internalDateOf(internalDates, messageId),
+            }, reStoredMessageIds.has(messageId), lookups, location), Math.min(body.length, bytesInFlight));
             if (outcome.kind === "Failed")
                 stream.failed = true;
             return { uid, messageId, outcome };
@@ -355,7 +367,7 @@ export class BodySyncService {
             failedMessageIds,
         };
     }
-    async storeFetchedBody(messageId, accountId, accountConfigId, body, placement, isReStore, lookups, location) {
+    async storeFetchedBody(messageId, accountId, accountConfigId, body, arrival, isReStore, lookups, location) {
         const stored = await this.storageService
             .storeMessageBody({
             accountConfigId,
@@ -363,7 +375,7 @@ export class BodySyncService {
             messageId,
             content: body,
         })
-            .then((ref) => this.applyPostStoreSteps(messageId, accountId, accountConfigId, body, { uri: ref.uri }, placement, isReStore, lookups))
+            .then((ref) => this.applyPostStoreSteps(messageId, accountId, accountConfigId, body, { uri: ref.uri }, arrival, isReStore, lookups))
             .then(() => ({ kind: "Stored" }), (error) => ({ kind: "Failed", error }));
         if (stored.kind !== "Failed")
             return stored;
@@ -404,7 +416,7 @@ export class BodySyncService {
      * Returns the parsed mail so callers that already need it (both do) don't
      * pay for mailparser twice.
      */
-    async applyPostStoreSteps(messageId, accountId, accountConfigId, body, bodyRef, placement, 
+    async applyPostStoreSteps(messageId, accountId, accountConfigId, body, bodyRef, arrival, 
     /**
      * Whether this call is a re-store of an already-classified message
      * rather than a first store. The two re-entrant paths (`force: true`
@@ -453,14 +465,14 @@ export class BodySyncService {
         // `flags.category` override when one applies (issue #299). The derived
         // fields are folded into the single Message update below — they are NOT
         // written here.
-        const classification = await this.classifyMessage(accountConfigId, parsed, placement);
+        const classification = await this.classifyMessage(accountConfigId, parsed, arrival.placement);
         // Decide the placement move from the in-memory classification before the
         // write, so its `movedByRemit` flag and audit verdict join the same
         // UpdateItem. The verdict is recorded whenever Remit decided to act
         // (action != leave); the move is enqueued only for a confident,
         // actionable verdict.
         const resolved = await this.resolvePlacement(lookups, messageId, accountId, accountConfigId, parsed, classification);
-        const filterDecision = await this.evaluateFilters(lookups, messageId, parsed);
+        const filterDecision = await this.evaluateFilters(lookups, messageId, parsed, arrival);
         // Store the parsed-body cache BEFORE persisting bodyStorageKey. The skip
         // guard in syncBodies treats a stored bodyStorageKey as "fully synced",
         // so if we wrote it first and the parsed-cache write then failed, the
@@ -651,7 +663,7 @@ export class BodySyncService {
             // a stored body — the NoSuchKey fallback above; a row with no
             // bodyStorageKey is a genuine first store and takes the full decision
             // pass.
-            parsed = await this.applyPostStoreSteps(messageId, accountId, accountConfigId, body, { uri: ref.uri }, folder.placement, message.bodyStorageKey != null, this.batchLookups(accountConfigId, accountId));
+            parsed = await this.applyPostStoreSteps(messageId, accountId, accountConfigId, body, { uri: ref.uri }, { placement: folder.placement, internalDate: message.internalDate }, message.bodyStorageKey != null, this.batchLookups(accountConfigId, accountId));
         }
         else {
             parsed = await simpleParser(body);
@@ -917,13 +929,13 @@ export class BodySyncService {
      * precedent). The applies of the returned decision are the caller's and are
      * deliberately unswallowed.
      */
-    async evaluateFilters(lookups, messageId, parsed) {
+    async evaluateFilters(lookups, messageId, parsed, arrival) {
         const { filterPipeline } = this;
         if (!filterPipeline)
             return { labels: [] };
         return lookups
             .usableFilters()
-            .then((usable) => filterPipeline.decide(usable, toFilterMessage(parsed)))
+            .then((usable) => filterPipeline.decide(filtersForArrival(usable, arrival), toFilterMessage(parsed)))
             .catch((error) => {
             this.log.error?.({
                 alert: "body_sync_filter_match_failed",

package/dist/filters/match.d.ts

@@ -27,9 +27,9 @@ export interface FilterMessage {
  */
 export declare const clauseMatches: (clause: FilterClause, msg: FilterMessage) => boolean;
 /**
- * Whether the literal clauses match under their combine operator. Empty clauses
- * carry no literal constraint and pass vacuously; a caller that needs a
- * predicate checks for clauses itself.
+ * Whether the literal clauses match under their combine operator. A filter with
+ * no clauses states nothing about the message, so it matches nothing: read as
+ * a vacuous pass it would file every message in every folder.
  */
 export declare const literalClausesMatch: (clauses: readonly FilterClause[], operator: FilterItem["matchOperator"], msg: FilterMessage) => boolean;
 /**

package/dist/filters/match.js

@@ -65,13 +65,13 @@ export const clauseMatches = (clause, msg) => {
     }
 };
 /**
- * Whether the literal clauses match under their combine operator. Empty clauses
- * carry no literal constraint and pass vacuously; a caller that needs a
- * predicate checks for clauses itself.
+ * Whether the literal clauses match under their combine operator. A filter with
+ * no clauses states nothing about the message, so it matches nothing: read as
+ * a vacuous pass it would file every message in every folder.
  */
 export const literalClausesMatch = (clauses, operator, msg) => {
     if (clauses.length === 0)
-        return true;
+        return false;
     return operator === FilterMatchOperator.Or
         ? clauses.some((clause) => clauseMatches(clause, msg))
         : clauses.every((clause) => clauseMatches(clause, msg));
@@ -118,7 +118,7 @@ const clauseNarrowing = (clause) => {
 export const literalClauseTerms = (clauses, operator) => {
     const combine = operator === FilterMatchOperator.Or ? "or" : "and";
     if (clauses.length === 0)
-        return { terms: [], operator: combine };
+        return null;
     const narrowings = clauses.map(clauseNarrowing);
     const terms = narrowings.flatMap((narrowing) => narrowing.kind === "Term" ? [narrowing.term] : []);
     if (combine === "and") {

package/dist/filters/pipeline.d.ts

@@ -1,4 +1,5 @@
 import type { FilterItem, IFilterRepository, IMailboxRepository, IMessageLabelRepository } from "@remit/data-ports";
+import type { FolderPlacement } from "../heuristics/classifyPlacement.js";
 import type { PlacementMoveService } from "../placement-move.js";
 import { type FilterMessage } from "./match.js";
 export interface FilterLogger {
@@ -60,3 +61,15 @@ export declare class FilterPipeline {
     private appliesTo;
     prepare(accountConfigId: string, accountId: string, active: readonly FilterItem[]): Promise<PreparedFilters>;
 }
+/**
+ * The filters a stored body is matched against. A filter acts on new mail:
+ * mail that arrived in Inbox or Junk, the folders mail is delivered to, after
+ * its rule or action was last set. Mail filed anywhere else, or delivered
+ * before the rule existed, is never moved by a body sync, which also runs for
+ * every old message a resync fetches again; applying a rule to existing mail
+ * is the organize job's.
+ */
+export declare const filtersForArrival: (prepared: PreparedFilters, arrival: {
+    placement: FolderPlacement;
+    internalDate: number;
+}) => PreparedFilters;

package/dist/filters/pipeline.js

@@ -81,4 +81,17 @@ export class FilterPipeline {
         return filters;
     }
 }
+/**
+ * The filters a stored body is matched against. A filter acts on new mail:
+ * mail that arrived in Inbox or Junk, the folders mail is delivered to, after
+ * its rule or action was last set. Mail filed anywhere else, or delivered
+ * before the rule existed, is never moved by a body sync, which also runs for
+ * every old message a resync fetches again; applying a rule to existing mail
+ * is the organize job's.
+ */
+export const filtersForArrival = (prepared, arrival) => {
+    if (arrival.placement === "other")
+        return [];
+    return prepared.filter((filter) => arrival.internalDate >= filter.actionChangedAt * 1000);
+};
 const filterMatches = (filter, msg) => literalClausesMatch(filter.literalClauses, filter.matchOperator, msg);

package/dist/test-helpers/fake-imap.d.ts

@@ -55,6 +55,7 @@ export declare class FakeImapServer {
         flags?: string[];
         uid?: number;
         bodyStructure?: ImapBodyStructure | null;
+        internalDate?: Date;
     }): number;
     /** Another client expunged the message. */
     expunge(path: string, uid: number): void;

package/dist/test-helpers/fake-imap.js

@@ -101,7 +101,7 @@ export class FakeImapServer {
             raw,
             flags: new Set(options.flags ?? []),
             modseq: this.modseq++,
-            internalDate: new Date(1_700_000_000_000 + uid * 1000),
+            internalDate: options.internalDate ?? new Date(1_700_000_000_000 + uid * 1000),
             bodyStructure: options.bodyStructure,
         });
         return uid;

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@remit/mailbox-service",
-	"version": "1.0.0",
+	"version": "2.0.0",
 	"type": "module",
 	"main": "dist/index.js",
 	"types": "dist/index.d.ts",