@remit/mailbox-service 0.0.1

package/src/index.ts

@@ -0,0 +1,214 @@
+/**
+ * remit-mailbox-service
+ *
+ * IMAP mailbox synchronization service for Remit
+ */
+
+export {
+	type AccountCredentialsDeps,
+	encryptRefreshToken,
+	resolveConnectionCredentials,
+} from "./account-credentials.js";
+export {
+	hasChildren,
+	isNoSelect,
+	type ParsedAttributes,
+	parseImapAttributes,
+} from "./attribute-mapper.js";
+export {
+	type BodySyncLogger,
+	BodySyncService,
+	type ConnectionGetter,
+	extractPrimaryFromEmail,
+	type FetchBodyResult,
+	type PlacementConfig,
+	type SyncBodiesResult,
+	toParsedBody,
+} from "./body-sync.js";
+export {
+	type BodySyncQueueConfig,
+	type BodySyncQueueLogger,
+	BodySyncQueueService,
+	type RequestBodySyncInput,
+} from "./body-sync-queue.js";
+export {
+	isMessageBodySyncBroken,
+	type ResolveExhaustedBodySyncDeps,
+	type ResolveExhaustedBodySyncInput,
+	type ResolveExhaustedBodySyncResult,
+	resolveExhaustedBodySyncFailures,
+} from "./body-sync-terminal.js";
+// Connection factory
+export {
+	createConnection,
+	createConnectionFromAccount,
+	createConnectionWithCredentials,
+	createManagedConnectionFactory,
+	type ManagedConnectionFactory,
+} from "./connection-factory.js";
+// Connection testing
+export {
+	type ImapTestConfig,
+	type SmtpTestConfig,
+	type TestResult,
+	testImapConnection,
+	testSmtpConnection,
+} from "./connection-test.js";
+export {
+	buildMatchText,
+	clauseMatches,
+	cosineSimilarity,
+	DEFAULT_SEMANTIC_MATCH_THRESHOLD,
+	type FilterMessage,
+	literalClausesMatch,
+	NO_ACTION,
+	selectMoveWinner,
+} from "./filters/match.js";
+export {
+	type FilterConfig,
+	type FilterDecision,
+	type FilterLogger,
+	FilterPipeline,
+	type MessageEmbedder,
+} from "./filters/pipeline.js";
+export {
+	type FlagPushConfig,
+	type FlagPushEvent,
+	type FlagPushLogger,
+	type FlagPushOperationValue,
+	FlagPushService,
+} from "./flag-push.js";
+export {
+	type FlagPushTerminalOutcome,
+	type ResolveExhaustedFlagPushDeps,
+	type ResolveExhaustedFlagPushInput,
+	type ResolveExhaustedFlagPushResult,
+	resolveExhaustedFlagPushFailure,
+} from "./flag-push-terminal.js";
+export {
+	type FlagQueueConfig,
+	type FlagQueueLogger,
+	FlagQueueService,
+	type UpdateFlagsInput,
+	type UpdateFlagsResult,
+} from "./flag-queue.js";
+export {
+	classifyByHeaders,
+	extractAuthenticity,
+	extractAuthResult,
+	extractHasListUnsubscribe,
+	extractProviderSpam,
+	type MessageAuthenticity,
+	type MessageAuthResult,
+	type MessageProviderSpam,
+} from "./heuristics/classifyByHeaders.js";
+export {
+	classifyPlacement,
+	type FolderPlacement,
+	type PlacementAction,
+	type PlacementVerdict,
+} from "./heuristics/classifyPlacement.js";
+export { SOCIAL_DOMAINS } from "./heuristics/socialDomains.js";
+export { TRANSACTIONAL_DOMAINS } from "./heuristics/transactionalDomains.js";
+// IMAP connection (ImapFlow-based)
+export {
+	createImapFlowConnectionFromAccount,
+	createImapFlowConnectionWithCredentials,
+	ImapFlowConnection,
+} from "./imapflow-connection.js";
+export {
+	guardConnectionCursor,
+	guardMailboxCursor,
+	isCursorRebuildNeeded,
+	type MailboxCursorCheck,
+	type MailboxCursorGuardDeps,
+	MailboxCursorPausedError,
+} from "./mailbox-cursor.js";
+export {
+	type CursorRebuildMatch,
+	type CursorRebuildMatchResult,
+	type CursorRebuildRow,
+	type CursorRebuildSnapshot,
+	matchCursorRebuild,
+} from "./mailbox-cursor-rebuild.js";
+export {
+	type CreateMailboxInput,
+	type DeleteMailboxInput,
+	type MailboxManagementLogger,
+	MailboxManagementService,
+	type MailboxManagementSyncResult,
+	parseMailboxPath,
+	type RenameMailboxInput,
+	validateMailboxOperation,
+	validateMailboxPath,
+} from "./mailbox-management.js";
+export {
+	type CreateMailboxQueueInput,
+	type MailboxQueueConfig,
+	type MailboxQueueLogger,
+	MailboxQueueService,
+} from "./mailbox-queue.js";
+export {
+	MailboxSyncService,
+	type SyncAccountInfo,
+} from "./mailbox-sync.js";
+export {
+	type DeleteOptions,
+	type MessageMoveConfig,
+	type MessageMoveLogger,
+	MessageMoveService,
+} from "./message-move.js";
+export {
+	type ParsedMessageContent,
+	parseMessageContent,
+} from "./message-parser.js";
+export {
+	type ImapConnectionFactory,
+	MessageSyncService,
+	type SyncedMessage,
+	type SyncMessagesResult,
+} from "./message-sync.js";
+export {
+	type CreateDraftInput,
+	type OutboxQueueConfig,
+	type OutboxQueueLogger,
+	OutboxQueueService,
+	type UpdateDraftInput,
+} from "./outbox-queue.js";
+export {
+	type PlacementMoveConfig,
+	type PlacementMoveLogger,
+	type PlacementMovePushEvent,
+	PlacementMoveService,
+} from "./placement-move.js";
+export {
+	type PlacementMoveTerminalOutcome,
+	type ResolveExhaustedPlacementMoveDeps,
+	type ResolveExhaustedPlacementMoveInput,
+	type ResolveExhaustedPlacementMoveResult,
+	resolveExhaustedPlacementMoveFailure,
+} from "./placement-move-terminal.js";
+export {
+	extractSnippetFromEmail,
+	generateSnippet,
+	normalizeSubject,
+	removeQuotedContent,
+} from "./snippet.js";
+export {
+	reconcileStaleMessage,
+	type StaleMessageReconcileDeps,
+	type StaleMessageReconcileResult,
+} from "./stale-message-reconcile.js";
+export type {
+	FlatMailboxInfo,
+	IImapConnection,
+	ImapBoxStatus,
+	ImapConnectionConfig,
+	ImapConnectionState,
+	ImapNamespace,
+	ImapNamespaces,
+	MailboxSyncResult,
+	MailConnectionErrorKind,
+	MailCredentials,
+} from "./types.js";
+export { MailConnectionError } from "./types.js";

package/src/mailbox-cursor-rebuild.test.ts

@@ -0,0 +1,171 @@
+import assert from "node:assert/strict";
+import { describe, it } from "node:test";
+import {
+	type CursorRebuildRow,
+	type CursorRebuildSnapshot,
+	matchCursorRebuild,
+} from "./mailbox-cursor-rebuild.js";
+
+const row = (over: Partial<CursorRebuildRow>): CursorRebuildRow => ({
+	messageId: "msg-1",
+	messageIdHeader: "<a@example.com>",
+	internalDate: 1_700_000_000_000,
+	uid: 10,
+	...over,
+});
+
+const snapshot = (
+	over: Partial<CursorRebuildSnapshot>,
+): CursorRebuildSnapshot => ({
+	uid: 10,
+	messageId: "<a@example.com>",
+	internalDate: 1_700_000_000_000,
+	...over,
+});
+
+describe("matchCursorRebuild", () => {
+	it("matches by Message-ID and rewrites the UID mapping when the UID changed", () => {
+		const existingRows = [row({ messageId: "msg-1", uid: 10 })];
+		const serverSnapshots = [snapshot({ uid: 55 })];
+
+		const result = matchCursorRebuild(serverSnapshots, existingRows);
+
+		assert.deepEqual(result.matched, [
+			{ messageId: "msg-1", oldUid: 10, newUid: 55 },
+		]);
+		assert.deepEqual(result.newUids, []);
+		assert.deepEqual(result.staleMessageIds, []);
+	});
+
+	it("does not emit a rewrite when the matched UID is unchanged (frugal — no-op write)", () => {
+		const existingRows = [row({ messageId: "msg-1", uid: 10 })];
+		const serverSnapshots = [snapshot({ uid: 10 })];
+
+		const result = matchCursorRebuild(serverSnapshots, existingRows);
+
+		assert.deepEqual(result.matched, []);
+		assert.deepEqual(result.staleMessageIds, []);
+	});
+
+	it("classifies a server message with no matching row as new — normal new-message sync", () => {
+		const existingRows: CursorRebuildRow[] = [];
+		const serverSnapshots = [
+			snapshot({ uid: 99, messageId: "<never-seen@example.com>" }),
+		];
+
+		const result = matchCursorRebuild(serverSnapshots, existingRows);
+
+		assert.deepEqual(result.matched, []);
+		assert.deepEqual(result.newUids, [99]);
+		assert.deepEqual(result.staleMessageIds, []);
+	});
+
+	it("classifies a row with no matching server message as stale — expunged, reconcile", () => {
+		const existingRows = [
+			row({ messageId: "msg-gone", messageIdHeader: "<gone@example.com>" }),
+		];
+		const serverSnapshots: CursorRebuildSnapshot[] = [];
+
+		const result = matchCursorRebuild(serverSnapshots, existingRows);
+
+		assert.deepEqual(result.matched, []);
+		assert.deepEqual(result.newUids, []);
+		assert.deepEqual(result.staleMessageIds, ["msg-gone"]);
+	});
+
+	it("handles a mixed batch: one match, one new, one stale", () => {
+		const existingRows = [
+			row({
+				messageId: "msg-keep",
+				messageIdHeader: "<keep@example.com>",
+				uid: 5,
+			}),
+			row({
+				messageId: "msg-gone",
+				messageIdHeader: "<gone@example.com>",
+				uid: 6,
+			}),
+		];
+		const serverSnapshots = [
+			snapshot({ uid: 105, messageId: "<keep@example.com>" }),
+			snapshot({ uid: 200, messageId: "<new@example.com>" }),
+		];
+
+		const result = matchCursorRebuild(serverSnapshots, existingRows);
+
+		assert.deepEqual(result.matched, [
+			{ messageId: "msg-keep", oldUid: 5, newUid: 105 },
+		]);
+		assert.deepEqual(result.newUids, [200]);
+		assert.deepEqual(result.staleMessageIds, ["msg-gone"]);
+	});
+
+	it("falls back to an exact INTERNALDATE match for headerless messages on both sides", () => {
+		const existingRows = [
+			row({
+				messageId: "msg-headerless",
+				messageIdHeader: "",
+				internalDate: 42,
+				uid: 3,
+			}),
+		];
+		const serverSnapshots = [
+			snapshot({ uid: 77, messageId: "", internalDate: 42 }),
+		];
+
+		const result = matchCursorRebuild(serverSnapshots, existingRows);
+
+		assert.deepEqual(result.matched, [
+			{ messageId: "msg-headerless", oldUid: 3, newUid: 77 },
+		]);
+		assert.deepEqual(result.staleMessageIds, []);
+	});
+
+	it("picks the closest INTERNALDATE among multiple rows sharing one Message-ID (legitimate resends)", () => {
+		const existingRows = [
+			row({ messageId: "msg-early", internalDate: 1000, uid: 1 }),
+			row({ messageId: "msg-late", internalDate: 5000, uid: 2 }),
+		];
+		const serverSnapshots = [snapshot({ uid: 50, internalDate: 4900 })];
+
+		const result = matchCursorRebuild(serverSnapshots, existingRows);
+
+		assert.deepEqual(result.matched, [
+			{ messageId: "msg-late", oldUid: 2, newUid: 50 },
+		]);
+		assert.deepEqual(result.staleMessageIds, ["msg-early"]);
+	});
+
+	it("carries the row's threadMessage ref through to the match, so the caller can rewrite ThreadMessage.uid alongside Message.uid", () => {
+		const threadMessage = {
+			accountConfigId: "acfg-1",
+			threadMessageId: "tm-1",
+			sentDate: 1_700_000_000_000,
+			mailboxId: "mbx-1",
+			isRead: true,
+			isDeleted: false,
+			hasStars: false,
+			hasAttachment: false,
+		};
+		const existingRows = [row({ messageId: "msg-1", uid: 10, threadMessage })];
+		const serverSnapshots = [snapshot({ uid: 99 })];
+
+		const result = matchCursorRebuild(serverSnapshots, existingRows);
+
+		assert.deepEqual(result.matched, [
+			{ messageId: "msg-1", oldUid: 10, newUid: 99, threadMessage },
+		]);
+	});
+
+	it("omits threadMessage from the match when the row carries none (test fixtures that don't need it)", () => {
+		const existingRows = [row({ messageId: "msg-1", uid: 10 })];
+		const serverSnapshots = [snapshot({ uid: 99 })];
+
+		const result = matchCursorRebuild(serverSnapshots, existingRows);
+
+		assert.deepEqual(result.matched, [
+			{ messageId: "msg-1", oldUid: 10, newUid: 99 },
+		]);
+		assert.equal("threadMessage" in result.matched[0], false);
+	});
+});

package/src/mailbox-cursor-rebuild.ts

@@ -0,0 +1,146 @@
+/**
+ * Pure matching logic for the UIDVALIDITY cursor rebuild (issue #1272).
+ *
+ * Message identity is the Message-ID header; UIDs are cursor positions on
+ * the server's axis. When UIDVALIDITY changes, the axis is gone but the
+ * data isn't — this reconciles a fresh envelope-level snapshot (UID +
+ * Message-ID + INTERNALDATE, no body fetch) against the rows already stored
+ * for the mailbox, by identity rather than by position.
+ */
+
+/** One server message from the cheap envelope-only rebuild FETCH. */
+export interface CursorRebuildSnapshot {
+	uid: number;
+	/** Raw RFC 822 Message-ID header; empty string when absent/unparseable. */
+	messageId: string;
+	/** INTERNALDATE in epoch milliseconds. */
+	internalDate: number;
+}
+
+/**
+ * ThreadMessage identity + every composite-key attribute ElectroDB needs to
+ * run a conditional update against the row (see `buildThreadMessageMoveUpdate`
+ * in `message-move.ts` for the same pattern on a normal move). Carried
+ * alongside a {@link CursorRebuildRow} purely so a match can rewrite
+ * ThreadMessage.uid together with Message.uid — matching itself never reads
+ * these fields.
+ */
+export interface CursorRebuildThreadMessageRef {
+	accountConfigId: string;
+	threadMessageId: string;
+	sentDate: number;
+	mailboxId: string;
+	isRead: boolean;
+	isDeleted: boolean;
+	hasStars: boolean;
+	hasAttachment: boolean;
+}
+
+/** One existing row for the mailbox, as needed to match it against a snapshot. */
+export interface CursorRebuildRow {
+	/** Our internal messageId (the Message row's primary key). */
+	messageId: string;
+	/** Raw RFC 822 Message-ID header the row was created with. */
+	messageIdHeader: string;
+	/** INTERNALDATE in epoch milliseconds. */
+	internalDate: number;
+	/** The row's currently-stored UID (on the old, invalid axis). */
+	uid: number;
+	/** Present when the caller wants a match to also rewrite ThreadMessage.uid. */
+	threadMessage?: CursorRebuildThreadMessageRef;
+}
+
+export interface CursorRebuildMatch {
+	messageId: string;
+	oldUid: number;
+	newUid: number;
+	/** Carried through from the matched row so the caller can rewrite
+	 * ThreadMessage.uid alongside Message.uid without a second lookup. */
+	threadMessage?: CursorRebuildThreadMessageRef;
+}
+
+export interface CursorRebuildMatchResult {
+	/** Rows whose Message-ID matched a server message under a different UID — rewrite in place. */
+	matched: CursorRebuildMatch[];
+	/** Server UIDs with no matching row — run through normal new-message sync. */
+	newUids: number[];
+	/** Rows with no matching server message — expunged; reconcile (#1283). */
+	staleMessageIds: string[];
+}
+
+const pickClosestByInternalDate = (
+	snapshot: CursorRebuildSnapshot,
+	candidates: CursorRebuildRow[],
+	consumed: Set<string>,
+): CursorRebuildRow | undefined => {
+	const available = candidates.filter((row) => !consumed.has(row.messageId));
+	if (available.length === 0) return undefined;
+	if (available.length === 1) return available[0];
+	return available.reduce((best, row) =>
+		Math.abs(row.internalDate - snapshot.internalDate) <
+		Math.abs(best.internalDate - snapshot.internalDate)
+			? row
+			: best,
+	);
+};
+
+/**
+ * Match a fresh server snapshot against the mailbox's existing rows by
+ * Message-ID, falling back to an exact INTERNALDATE match for the (rare)
+ * messages with no usable Message-ID header on either side — mirrors the
+ * same headerless fallback `message-sync.ts` already uses for threading.
+ *
+ * Pure and side-effect free: the caller is responsible for acting on the
+ * three outcomes (rewrite / new-message sync / reconcile).
+ */
+export const matchCursorRebuild = (
+	serverSnapshots: CursorRebuildSnapshot[],
+	existingRows: CursorRebuildRow[],
+): CursorRebuildMatchResult => {
+	const rowsByHeader = new Map<string, CursorRebuildRow[]>();
+	const rowsByInternalDateOnly = new Map<number, CursorRebuildRow[]>();
+
+	for (const row of existingRows) {
+		if (row.messageIdHeader) {
+			const list = rowsByHeader.get(row.messageIdHeader) ?? [];
+			list.push(row);
+			rowsByHeader.set(row.messageIdHeader, list);
+		} else {
+			const list = rowsByInternalDateOnly.get(row.internalDate) ?? [];
+			list.push(row);
+			rowsByInternalDateOnly.set(row.internalDate, list);
+		}
+	}
+
+	const consumed = new Set<string>();
+	const matched: CursorRebuildMatch[] = [];
+	const newUids: number[] = [];
+
+	for (const snapshot of serverSnapshots) {
+		const candidates = snapshot.messageId
+			? (rowsByHeader.get(snapshot.messageId) ?? [])
+			: (rowsByInternalDateOnly.get(snapshot.internalDate) ?? []);
+		const row = pickClosestByInternalDate(snapshot, candidates, consumed);
+
+		if (!row) {
+			newUids.push(snapshot.uid);
+			continue;
+		}
+
+		consumed.add(row.messageId);
+		if (row.uid !== snapshot.uid) {
+			matched.push({
+				messageId: row.messageId,
+				oldUid: row.uid,
+				newUid: snapshot.uid,
+				...(row.threadMessage ? { threadMessage: row.threadMessage } : {}),
+			});
+		}
+	}
+
+	const staleMessageIds = existingRows
+		.filter((row) => !consumed.has(row.messageId))
+		.map((row) => row.messageId);
+
+	return { matched, newUids, staleMessageIds };
+};