@remit/mailbox-service 0.0.1

package/src/placement-move.ts

@@ -0,0 +1,224 @@
+import { randomUUID } from "node:crypto";
+import { SendMessageCommand, type SQSClient } from "@aws-sdk/client-sqs";
+import type {
+	IMessagePlacementMoveRepository,
+	IMessageRepository,
+	IThreadMessageRepository,
+} from "@remit/data-ports";
+import { MessageStatus, MessageSyncStatus } from "@remit/domain-enums";
+import { createQueueProducer } from "@remit/sqs-client/producer";
+
+/**
+ * Event the reconciler (imap-worker `handlePlacementMovePush`) drains. Carries
+ * ONLY our own message id — never a UID (epic #1281 invariant 1). The UID is
+ * resolved fresh from the Message row at push time, so the event stays valid
+ * across any amount of queue delay or an unrelated UIDVALIDITY rebuild
+ * (#1272).
+ */
+export interface PlacementMovePushEvent {
+	type: "PLACEMENT_MOVE_PUSH";
+	eventId: string;
+	timestamp: number;
+	accountId: string;
+	accountConfigId: string;
+	messageId: string;
+}
+
+export interface PlacementMoveLogger {
+	info(obj: Record<string, unknown>, msg: string): void;
+	error(obj: Record<string, unknown>, msg: string): void;
+}
+
+const noopLogger: PlacementMoveLogger = {
+	info: () => {},
+	error: () => {},
+};
+
+export interface PlacementMoveConfig {
+	messageService: IMessageRepository;
+	threadMessageService: IThreadMessageRepository;
+	markerService: IMessagePlacementMoveRepository;
+	sqsQueueUrl: string;
+	sqsEndpoint?: string;
+	logger?: PlacementMoveLogger;
+}
+
+/**
+ * Local-first mover for a classification-driven placement move (issue #1271,
+ * epic #1281). Distinct from {@link MessageMoveService} (user-initiated
+ * moves/deletes/copies), which is untouched by this fix — the epic's UID
+ * staleness invariant (1) only applies to the NEW reconciler this class
+ * feeds; generalizing it to every move type is out of scope here.
+ *
+ * `moveMessage` matches {@link MessageMoveService.moveMessage}'s signature so
+ * `BodySyncService`'s existing rescue/demote call site and its tests need no
+ * shape changes beyond the dependency's name.
+ *
+ * Ordering is the fix: every step below (marker, ThreadMessage, Message row,
+ * SQS enqueue) completes — or the whole call rejects — BEFORE the caller
+ * (`BodySyncService.storeStreamedBody`) writes `bodyStorageKey`. A rejection
+ * here therefore always happens before the body-sync skip-guard can trip, so
+ * a retry reprocesses the message from scratch. Nothing here is swallowed —
+ * let it crash, same as any other infra failure (an SQS enqueue failure is a
+ * serious operational failure, not a routine blip: it propagates so a
+ * repeated failure reaches the alarmed message-mgmt DLQ, never absorbed).
+ *
+ * Recovery is driven by the marker's explicit state engine (`pending ->
+ * queued -> processing -> processed`, `MessagePlacementMoveState`), not by
+ * comparing the message's current location against the requested
+ * destination — that comparison alone cannot tell "already confirmed" apart
+ * from "local move landed but the enqueue never did" (the defect this issue
+ * originally fixes). A retry always re-fetches the marker and drives it
+ * forward from whichever state it is actually in.
+ */
+export class PlacementMoveService {
+	private messageService: IMessageRepository;
+	private threadMessageService: IThreadMessageRepository;
+	private markerService: IMessagePlacementMoveRepository;
+	private sqs: SQSClient;
+	private queueUrl: string;
+	private log: PlacementMoveLogger;
+
+	constructor(config: PlacementMoveConfig) {
+		this.messageService = config.messageService;
+		this.threadMessageService = config.threadMessageService;
+		this.markerService = config.markerService;
+		this.queueUrl = config.sqsQueueUrl;
+		this.log = config.logger ?? noopLogger;
+		this.sqs = createQueueProducer({
+			queueUrl: config.sqsQueueUrl,
+			endpoint: config.sqsEndpoint,
+		});
+	}
+
+	moveMessage = async (
+		accountConfigId: string,
+		messageId: string,
+		destinationMailboxId: string,
+		accountId: string,
+	): Promise<void> => {
+		const message = await this.messageService.get(messageId);
+		const sourceMailboxId = message.mailboxId;
+
+		// Recovery: a marker already exists for THIS message and destination —
+		// from an earlier (possibly partially-failed) call. The marker's STATE,
+		// not a comparison against the message's current location, decides what
+		// happens next (PR #1289 review finding 1 + design amendment: an
+		// explicit pending -> queued -> processing -> processed engine).
+		const existingMarker = await this.markerService.find(messageId);
+		if (
+			existingMarker &&
+			existingMarker.destinationMailboxId === destinationMailboxId
+		) {
+			if (existingMarker.state === "pending") {
+				// Local move + marker already committed; only the enqueue is
+				// missing (or a further retry after the enqueue itself failed
+				// again). Drive the SAME marker forward — never re-derive the
+				// local move, which already happened.
+				await this.enqueuePush({ accountId, accountConfigId, messageId });
+				await this.markerService.updateState(messageId, "queued");
+				this.log.info(
+					{ messageId, accountId, destinationMailboxId },
+					"Pending marker survived an earlier enqueue failure; re-enqueued and advanced to queued",
+				);
+			}
+			// queued / processing / processed already has its own driver in
+			// flight (the SQS message, or the reconciler currently running) —
+			// nothing to do here either way.
+			return;
+		}
+
+		// Genuine no-op: nothing pending for this destination, and the message
+		// is already there (a duplicate verdict recomputed after a confirmed
+		// move).
+		if (sourceMailboxId === destinationMailboxId) return;
+
+		// Fresh move. Strictly sequential — nothing here is parallel: the local
+		// move (marker + ThreadMessage + Message row) fully commits BEFORE the
+		// queue kick runs. `put` is idempotent and always resets `state` back to
+		// `pending`, since a fresh decision starts a new lifecycle regardless of
+		// what a stale row (different destination) held.
+		await this.markerService.put({
+			messageId,
+			accountId,
+			accountConfigId,
+			sourceMailboxId,
+			destinationMailboxId,
+		});
+
+		await this.updateThreadMessageMailbox(
+			accountConfigId,
+			messageId,
+			destinationMailboxId,
+		);
+
+		await this.messageService.updateForMove(messageId, {
+			mailboxId: destinationMailboxId,
+			status: MessageStatus.moving,
+			syncStatus: MessageSyncStatus.pending,
+			originalMailboxId: sourceMailboxId,
+			originalUid: message.uid,
+		});
+
+		// The queue kick is a serious operational step, not a routine one — a
+		// failure here MUST propagate (never swallowed): the marker stays
+		// `pending`, the surrounding body-sync call fails, SQS redelivers, and
+		// a repeated failure reaches the alarmed message-mgmt DLQ. The state
+		// engine makes this *recoverable* (the recovery branch above drives a
+		// surviving `pending` marker forward on the next call) — it must never
+		// make it *quiet*.
+		await this.enqueuePush({ accountId, accountConfigId, messageId });
+		await this.markerService.updateState(messageId, "queued");
+
+		this.log.info(
+			{ messageId, accountId, sourceMailboxId, destinationMailboxId },
+			"Placement move applied locally; pending marker queued for IMAP push",
+		);
+	};
+
+	private updateThreadMessageMailbox = async (
+		accountConfigId: string,
+		messageId: string,
+		destinationMailboxId: string,
+	): Promise<void> => {
+		const threadMessage = await this.threadMessageService.getByMessageId(
+			accountConfigId,
+			messageId,
+		);
+
+		await this.threadMessageService.update(
+			threadMessage.accountConfigId,
+			threadMessage.threadMessageId,
+			{ mailboxId: destinationMailboxId },
+			{
+				composites: {
+					sentDate: threadMessage.sentDate,
+					mailboxId: threadMessage.mailboxId,
+					isRead: threadMessage.isRead,
+					isDeleted: threadMessage.isDeleted,
+					hasStars: threadMessage.hasStars,
+					hasAttachment: threadMessage.hasAttachment,
+				},
+			},
+		);
+	};
+
+	private enqueuePush = async (params: {
+		accountId: string;
+		accountConfigId: string;
+		messageId: string;
+	}): Promise<void> => {
+		const event: PlacementMovePushEvent = {
+			type: "PLACEMENT_MOVE_PUSH",
+			eventId: randomUUID(),
+			timestamp: Date.now(),
+			...params,
+		};
+		await this.sqs.send(
+			new SendMessageCommand({
+				QueueUrl: this.queueUrl,
+				MessageBody: JSON.stringify(event),
+			}),
+		);
+	};
+}

package/src/snippet.test.ts

@@ -0,0 +1,213 @@
+import assert from "node:assert";
+import { describe, it } from "node:test";
+import {
+	extractSnippetFromEmail,
+	generateSnippet,
+	normalizeSubject,
+	removeQuotedContent,
+} from "./snippet.js";
+
+describe("removeQuotedContent", () => {
+	it("removes content after > quote marker", () => {
+		const text = "Hello\nWorld\n> quoted line\nmore quoted";
+		assert.strictEqual(removeQuotedContent(text), "Hello\nWorld");
+	});
+
+	it("removes content after 'On ... wrote:' pattern", () => {
+		const text = "Hello\nOn Monday, John wrote:\nQuoted content";
+		assert.strictEqual(removeQuotedContent(text), "Hello");
+	});
+
+	it("removes content after original message separator", () => {
+		const text = "Hello\n--- Original Message ---\nQuoted content";
+		assert.strictEqual(removeQuotedContent(text), "Hello");
+	});
+
+	it("removes content after Outlook underline separator", () => {
+		const text = "Hello\n___\nQuoted content";
+		assert.strictEqual(removeQuotedContent(text), "Hello");
+	});
+
+	it("removes content after Outlook forward header", () => {
+		const text = "Hello\nFrom: John Sent: Today To: Jane\nQuoted content";
+		assert.strictEqual(removeQuotedContent(text), "Hello");
+	});
+
+	it("returns full text when no quote markers", () => {
+		const text = "Hello\nWorld\nNo quotes here";
+		assert.strictEqual(removeQuotedContent(text), text);
+	});
+});
+
+describe("normalizeSubject", () => {
+	describe("prefix removal", () => {
+		it("removes Re: prefix", () => {
+			const result = normalizeSubject("Re: Meeting tomorrow");
+			assert.ok(!result.includes("re"));
+			assert.ok(result.includes("meet"));
+		});
+
+		it("removes Fwd: prefix", () => {
+			const result = normalizeSubject("Fwd: Important document");
+			assert.ok(!result.includes("fwd"));
+		});
+
+		it("removes numbered Re: prefix", () => {
+			const result = normalizeSubject("Re[3]: Meeting tomorrow");
+			assert.ok(!result.includes("re"));
+		});
+
+		it("removes multiple prefixes", () => {
+			const result = normalizeSubject("Re: Fwd: Re: Original subject");
+			assert.ok(!result.includes("re"));
+			assert.ok(!result.includes("fwd"));
+		});
+
+		it("removes German Aw: prefix", () => {
+			const result = normalizeSubject("Aw: Besprechung morgen");
+			assert.ok(!result.includes("aw"));
+		});
+
+		it("removes Portuguese Res: prefix", () => {
+			const result = normalizeSubject("Res: Reunião amanhã");
+			assert.ok(!result.includes("res"));
+		});
+	});
+
+	describe("language detection", () => {
+		it("auto-detects English and uses English stemming", () => {
+			const result = normalizeSubject(
+				"Meeting tomorrow about running the project quickly",
+			);
+			// Should stem "running" to "run", "quickly" to "quick"
+			assert.ok(result.includes("run") || result.includes("quick"));
+		});
+
+		it("auto-detects German and uses German processing", () => {
+			const result = normalizeSubject(
+				"Besprechung über die Verbindung und den Projektstand morgen",
+			);
+			// Should process with German stemmer
+			assert.ok(result.length > 0);
+		});
+
+		it("auto-detects French and uses French processing", () => {
+			const result = normalizeSubject(
+				"Réunion demain matin pour discuter du projet",
+			);
+			// Should preserve French accented characters and process
+			assert.ok(result.includes("réunion") || result.includes("reunion"));
+		});
+
+		it("uses provided language override", () => {
+			const result = normalizeSubject("Meeting tomorrow", "en");
+			assert.ok(result.includes("meet"));
+		});
+	});
+
+	describe("normalization", () => {
+		it("removes stopwords", () => {
+			const result = normalizeSubject("The meeting is about the project");
+			// "the" and "is" should be removed
+			assert.ok(!result.split(" ").includes("the"));
+		});
+
+		it("handles Unicode characters", () => {
+			const result = normalizeSubject("Réunion à Paris");
+			// Should not strip accented characters
+			assert.ok(result.includes("réunion") || result.includes("reunion"));
+			assert.ok(result.includes("pari"));
+		});
+
+		it("deduplicates consecutive words", () => {
+			const result = normalizeSubject("Meeting meeting meeting update");
+			// After stemming, should dedupe
+			const words = result.split(" ");
+			for (let i = 1; i < words.length; i++) {
+				assert.notStrictEqual(
+					words[i],
+					words[i - 1],
+					"Should not have consecutive duplicates",
+				);
+			}
+		});
+	});
+});
+
+describe("generateSnippet", () => {
+	it("generates snippet from text", () => {
+		const text = "Hello, this is a test message.";
+		assert.strictEqual(generateSnippet(text), text);
+	});
+
+	it("removes quoted content", () => {
+		const text = "Hello\n> quoted line";
+		assert.strictEqual(generateSnippet(text), "Hello");
+	});
+
+	it("normalizes whitespace", () => {
+		const text = "Hello   world\n\nmultiple  spaces";
+		assert.strictEqual(generateSnippet(text), "Hello world multiple spaces");
+	});
+
+	it("truncates long text at word boundary", () => {
+		const text = "The quick brown fox jumps over the lazy dog repeatedly";
+		const result = generateSnippet(text, 30);
+		assert.ok(result.length <= 30);
+		assert.ok(result.endsWith("..."));
+	});
+
+	it("truncates single long word", () => {
+		const text = "supercalifragilisticexpialidocious";
+		const result = generateSnippet(text, 20);
+		assert.ok(result.endsWith("..."));
+		// When no word boundary found, truncates at maxLength-1 + "..."
+		// so result is at most maxLength + 2 (for the extra chars in "...")
+		assert.ok(result.length <= 22);
+	});
+});
+
+describe("extractSnippetFromEmail", () => {
+	it("prefers plain text over HTML", () => {
+		const result = extractSnippetFromEmail("Plain text", "<p>HTML</p>");
+		assert.strictEqual(result, "Plain text");
+	});
+
+	it("falls back to HTML when no plain text", () => {
+		const result = extractSnippetFromEmail(undefined, "<p>HTML content</p>");
+		assert.ok(result.includes("HTML content"));
+	});
+
+	it("strips HTML tags", () => {
+		const result = extractSnippetFromEmail(
+			undefined,
+			"<p><strong>Bold</strong> text</p>",
+		);
+		assert.ok(!result.includes("<"));
+		assert.ok(!result.includes(">"));
+		assert.ok(result.includes("Bold"));
+		assert.ok(result.includes("text"));
+	});
+
+	it("removes script and style tags", () => {
+		const html = "<style>body{}</style><script>alert(1)</script><p>Content</p>";
+		const result = extractSnippetFromEmail(undefined, html);
+		assert.ok(!result.includes("body"));
+		assert.ok(!result.includes("alert"));
+		assert.ok(result.includes("Content"));
+	});
+
+	it("decodes HTML entities", () => {
+		const html = "<p>&amp; &lt; &gt; &quot; &nbsp;</p>";
+		const result = extractSnippetFromEmail(undefined, html);
+		assert.ok(result.includes("&"));
+		assert.ok(result.includes("<"));
+		assert.ok(result.includes(">"));
+		assert.ok(result.includes('"'));
+	});
+
+	it("returns empty string when no content", () => {
+		const result = extractSnippetFromEmail(undefined, undefined);
+		assert.strictEqual(result, "");
+	});
+});

package/src/snippet.ts

@@ -0,0 +1,152 @@
+/**
+ * Snippet generation utilities for email threading.
+ *
+ * Extracts preview text from email bodies by removing quoted content
+ * and normalizing whitespace.
+ */
+
+import {
+	createTextNormalizer,
+	type SupportedLanguage,
+} from "./text/normalizer.js";
+
+/**
+ * Remove quoted reply content from email body.
+ *
+ * Stops at common quote indicators:
+ * - Lines starting with ">"
+ * - "On ... wrote:" patterns
+ * - "--- Original Message ---" separators
+ * - Outlook-style underline separators
+ */
+export const removeQuotedContent = (text: string): string => {
+	const lines = text.split("\n");
+	const result: string[] = [];
+
+	for (const line of lines) {
+		// Stop at quote indicators
+		if (/^>/.test(line)) break;
+		if (/^On .+ wrote:$/i.test(line)) break;
+		if (/^-{3,}\s*Original Message\s*-{3,}$/i.test(line)) break;
+		if (/^_{3,}$/.test(line)) break; // Outlook separator
+		if (/^From:.*Sent:.*To:/i.test(line)) break; // Outlook forward header
+
+		result.push(line);
+	}
+
+	return result.join("\n");
+};
+
+const normalizer = createTextNormalizer();
+
+/**
+ * Normalize subject for thread grouping and search.
+ *
+ * Processing steps:
+ * 1. Remove Re:/Fwd:/etc prefixes (multi-language support)
+ * 2. Auto-detect language or use provided language
+ * 3. Normalize with stemming and stopword removal
+ * 4. Deduplicate consecutive words
+ *
+ * Handles common prefixes across languages:
+ * - Re, Fwd, Fw (English)
+ * - Aw, Sv, Vs (German, Swedish, Danish)
+ * - Ref, Rif (Italian, Spanish)
+ * - Odp (Polish)
+ * - Ynt (Turkish)
+ * - Antw (Dutch)
+ * - Res (Portuguese)
+ *
+ * @param subject - The email subject line
+ * @param language - Optional language override (auto-detected if not provided)
+ */
+export const normalizeSubject = (
+	subject: string,
+	language?: SupportedLanguage,
+): string => {
+	const SUBJECT_PREFIX_PATTERN =
+		/^(\s*(Re|Fwd|Fw|Aw|Sv|Vs|Ref|Rif|Odp|Ynt|Antw|Res)(\[\d+\])?:\s*)+/i;
+
+	// Step 1: Remove prefixes
+	const withoutPrefix = subject.replace(SUBJECT_PREFIX_PATTERN, "").trim();
+
+	// Step 2: Auto-detect language if not provided
+	const detectedLanguage = language ?? normalizer.detectLanguage(withoutPrefix);
+
+	// Step 3: Normalize with stemming and stopword removal
+	return normalizer.normalize(withoutPrefix, {
+		language: detectedLanguage,
+		stem: true,
+		removeStopwords: true,
+	});
+};
+
+/**
+ * Generate a snippet from email text content.
+ *
+ * @param text - The text content (plain text or stripped HTML)
+ * @param maxLength - Maximum snippet length (default 256)
+ * @returns Truncated snippet with ellipsis if needed
+ */
+export const generateSnippet = (text: string, maxLength = 256): string => {
+	// Remove quoted content
+	const unquoted = removeQuotedContent(text);
+
+	// Normalize whitespace: collapse multiple spaces/newlines to single space
+	const normalized = unquoted.replace(/\s+/g, " ").trim();
+
+	if (normalized.length <= maxLength) {
+		return normalized;
+	}
+
+	// Truncate at word boundary
+	const truncated = normalized.slice(0, maxLength);
+	const lastSpace = truncated.lastIndexOf(" ");
+
+	// If no space found, just truncate
+	if (lastSpace === -1) {
+		return `${truncated.slice(0, maxLength - 1)}...`;
+	}
+
+	// Truncate at word boundary and add ellipsis
+	return `${truncated.slice(0, lastSpace)}...`;
+};
+
+/**
+ * Extract text content from parsed email for snippet generation.
+ *
+ * Prefers plain text over HTML. If only HTML is available,
+ * the caller should strip HTML tags before calling generateSnippet.
+ *
+ * @param textContent - Plain text content (may be undefined)
+ * @param htmlContent - HTML content (may be undefined)
+ * @param maxLength - Maximum snippet length
+ * @returns Generated snippet or empty string
+ */
+export const extractSnippetFromEmail = (
+	textContent: string | undefined,
+	htmlContent: string | undefined,
+	maxLength = 256,
+): string => {
+	// Prefer plain text
+	if (textContent) {
+		return generateSnippet(textContent, maxLength);
+	}
+
+	// Fall back to HTML (caller should strip tags)
+	if (htmlContent) {
+		// Basic HTML tag stripping - for more robust handling, use html-to-text
+		const stripped = htmlContent
+			.replace(/<style[^>]*>[\s\S]*?<\/style>/gi, "")
+			.replace(/<script[^>]*>[\s\S]*?<\/script>/gi, "")
+			.replace(/<[^>]+>/g, " ")
+			.replace(/&nbsp;/g, " ")
+			.replace(/&amp;/g, "&")
+			.replace(/&lt;/g, "<")
+			.replace(/&gt;/g, ">")
+			.replace(/&quot;/g, '"');
+		return generateSnippet(stripped, maxLength);
+	}
+
+	return "";
+};

package/src/stale-message-reconcile.test.ts

@@ -0,0 +1,82 @@
+import assert from "node:assert/strict";
+import { describe, it } from "node:test";
+import type {
+	IMessageRepository,
+	IThreadMessageRepository,
+} from "@remit/data-ports";
+import {
+	reconcileStaleMessage,
+	type StaleMessageReconcileDeps,
+} from "./stale-message-reconcile.js";
+
+describe("reconcileStaleMessage", () => {
+	it("deletes the Message row and every ThreadMessage copy that points at it", async () => {
+		const deletedMessages: string[] = [];
+		const deletedThreadMessageKeys: Array<{
+			accountConfigId: string;
+			threadMessageId: string;
+		}> = [];
+
+		const deps: StaleMessageReconcileDeps = {
+			messageService: {
+				delete: async (messageId: string) => {
+					deletedMessages.push(messageId);
+				},
+			} as unknown as Pick<IMessageRepository, "delete">,
+			threadMessageService: {
+				findAllByMessageId: async (
+					accountConfigId: string,
+					messageId: string,
+				) => [
+					{ accountConfigId, threadMessageId: `tm-inbox-${messageId}` },
+					{ accountConfigId, threadMessageId: `tm-label-${messageId}` },
+				],
+				deleteMany: async (
+					keys: Array<{ accountConfigId: string; threadMessageId: string }>,
+				) => {
+					deletedThreadMessageKeys.push(...keys);
+				},
+			} as unknown as Pick<
+				IThreadMessageRepository,
+				"findAllByMessageId" | "deleteMany"
+			>,
+		};
+
+		const result = await reconcileStaleMessage(deps, "cfg-1", "msg-1");
+
+		assert.equal(result.threadMessagesDeleted, 2);
+		assert.deepEqual(deletedMessages, ["msg-1"]);
+		assert.deepEqual(deletedThreadMessageKeys, [
+			{ accountConfigId: "cfg-1", threadMessageId: "tm-inbox-msg-1" },
+			{ accountConfigId: "cfg-1", threadMessageId: "tm-label-msg-1" },
+		]);
+	});
+
+	it("skips the batch delete (never calls deleteMany with an empty array) when no copies exist", async () => {
+		const deletedMessages: string[] = [];
+		let deleteManyCalls = 0;
+
+		const deps: StaleMessageReconcileDeps = {
+			messageService: {
+				delete: async (messageId: string) => {
+					deletedMessages.push(messageId);
+				},
+			} as unknown as Pick<IMessageRepository, "delete">,
+			threadMessageService: {
+				findAllByMessageId: async () => [],
+				deleteMany: async () => {
+					deleteManyCalls++;
+				},
+			} as unknown as Pick<
+				IThreadMessageRepository,
+				"findAllByMessageId" | "deleteMany"
+			>,
+		};
+
+		const result = await reconcileStaleMessage(deps, "cfg-1", "msg-solo");
+
+		assert.equal(result.threadMessagesDeleted, 0);
+		assert.deepEqual(deletedMessages, ["msg-solo"]);
+		assert.equal(deleteManyCalls, 0);
+	});
+});