@remit/mailbox-service 0.0.1

package/README.md

@@ -0,0 +1,147 @@
+# @remit/mailbox-service
+
+IMAP mailbox synchronization service for Remit. Provides connection management, mailbox discovery, and message sync with DynamoDB persistence.
+
+## Features
+
+- **ImapFlow-based**: Modern async/await IMAP client with native envelope parsing
+- **Mailbox Sync**: Discovers and syncs mailbox metadata from IMAP to DynamoDB
+- **Message Sync**: Newest-first sync strategy with dual-watermark tracking
+- **Address Extraction**: Parses and stores envelope addresses with role tracking
+
+## Installation
+
+```bash
+npm install
+```
+
+## Usage
+
+### IMAP Connection
+
+```typescript
+import {
+  ImapFlowConnection,
+  createImapFlowConnectionFromAccount,
+} from "@remit/mailbox-service";
+
+// From account data
+const connection = createImapFlowConnectionFromAccount(
+  {
+    imapHost: "imap.example.com",
+    imapPort: 993,
+    imapTls: true,
+    username: "user",
+  },
+  "password",
+);
+
+// Or direct configuration
+const connection = new ImapFlowConnection({
+  host: "imap.example.com",
+  port: 993,
+  tls: true,
+  user: "user",
+  password: "password",
+});
+
+await connection.connect();
+const mailboxes = await connection.listMailboxes();
+await connection.disconnect();
+```
+
+### Mailbox Sync
+
+```typescript
+import { MailboxSyncService } from "@remit/mailbox-service";
+
+const syncService = new MailboxSyncService({
+  client: dynamoDBClient,
+  table: "remit-table",
+});
+
+const result = await syncService.syncMailboxes(
+  { accountId: "acc-123" },
+  connection,
+);
+// result: { created: 5, updated: 2, deleted: 0 }
+```
+
+### Message Sync
+
+```typescript
+import { MessageSyncService } from "@remit/mailbox-service";
+
+const messageSyncService = new MessageSyncService(
+  () => createConnection(), // connection factory
+  mailboxService,
+  messageService,
+  envelopeService,
+  addressService,
+  logger,
+);
+
+const synced = await messageSyncService.syncMessages(
+  mailboxId,
+  accountConfigId,
+  50, // batch size
+);
+```
+
+## Exports
+
+### Connections
+
+| Export                                | Description                          |
+| ------------------------------------- | ------------------------------------ |
+| `ImapFlowConnection`                  | ImapFlow-based IMAP connection class |
+| `createImapFlowConnectionFromAccount` | Factory from account data            |
+| `createConnection`                    | Connection factory                   |
+| `createConnectionFromAccount`         | Account-based connection factory     |
+
+### Services
+
+| Export               | Description                               |
+| -------------------- | ----------------------------------------- |
+| `MailboxSyncService` | Syncs mailbox list from IMAP to DynamoDB  |
+| `MessageSyncService` | Syncs messages with newest-first strategy |
+
+### Types
+
+| Export                 | Description                  |
+| ---------------------- | ---------------------------- |
+| `IImapConnection`      | Connection interface         |
+| `ImapConnectionConfig` | Connection configuration     |
+| `ImapConnectionState`  | Connection state enum        |
+| `ImapBoxStatus`        | Mailbox status after opening |
+| `FlatMailboxInfo`      | Flattened mailbox info       |
+| `MailboxSyncResult`    | Sync operation result        |
+| `ImapNamespaces`       | IMAP namespace info          |
+
+### Utilities
+
+| Export                | Description                        |
+| --------------------- | ---------------------------------- |
+| `parseImapAttributes` | Parse IMAP mailbox attributes      |
+| `hasChildren`         | Check if mailbox has children      |
+| `isNoSelect`          | Check if mailbox is non-selectable |
+
+## Sync Strategy
+
+Message sync uses a dual-watermark approach:
+
+```
+highWaterMarkUid: 150  ─────────────────────────────────────┐
+                                                            │ New messages
+lastSyncUid: 100       ─────────────────────────────────────┤ (synced first)
+                       │                                    │
+                       │ Already synced                     │
+                       │                                    │
+lastSyncUid progress   ─────────────────────────────────────┤
+                       │ Backfill                           │
+UID 1                  ─────────────────────────────────────┘
+```
+
+1. **New messages** (UID > highWaterMarkUid) are synced first
+2. **Backfill** (UID < lastSyncUid) continues in subsequent batches
+3. Both watermarks update after each batch for resumability

package/package.json

@@ -0,0 +1,65 @@
+{
+	"name": "@remit/mailbox-service",
+	"version": "0.0.1",
+	"type": "module",
+	"main": "src/index.ts",
+	"types": "src/index.ts",
+	"exports": {
+		".": {
+			"types": "./src/index.ts",
+			"default": "./src/index.ts"
+		},
+		"./body-sync-queue": {
+			"types": "./src/body-sync-queue.ts",
+			"default": "./src/body-sync-queue.ts"
+		},
+		"./account-credentials": {
+			"types": "./src/account-credentials.ts",
+			"default": "./src/account-credentials.ts"
+		}
+	},
+	"scripts": {
+		"test:typecheck": "tsgo --noEmit",
+		"test:run": "node --env-file=../../localhost-test-unit.env --import tsx --test 'src/**/*.test.ts'",
+		"test:integ": "RUN_INTEG_TESTS=1 node --env-file=../../localhost-test-unit.env --import tsx --test 'src/**/*.integ.test.ts'",
+		"test:e2e": "RUN_E2E_TESTS=1 node --env-file=../../localhost-test-e2e.env --import tsx --test --test-concurrency=1 'src/**/*.e2e.test.ts'",
+		"test": "npm run test:typecheck && npm run test:run"
+	},
+	"devDependencies": {
+		"@aws-sdk/client-dynamodb": "*",
+		"@aws-sdk/client-sqs": "*",
+		"@aws-sdk/lib-dynamodb": "*",
+		"@remit/data-ports": "*",
+		"@remit/electrodb-entities": "*",
+		"@remit/domain-enums": "*",
+		"@remit/mail-oauth-service": "*",
+		"@remit/secrets-service": "*",
+		"@remit/sqs-client": "*",
+		"@remit/storage-service": "*",
+		"@types/mailparser": "*",
+		"@types/nodemailer": "*",
+		"@types/stopword": "^2.0.3",
+		"@types/uuid": "^10.0.0",
+		"electrodb": "*",
+		"expect-env": "*",
+		"franc": "*",
+		"natural": "^8.1.1",
+		"short-uuid": "*",
+		"stopword": "*"
+	},
+	"dependencies": {
+		"imapflow": "^1.3.3",
+		"mailparser": "^3.9.14",
+		"nodemailer": "^9.0.3",
+		"p-map": "^7.0.4"
+	},
+	"license": "MIT",
+	"publishConfig": {
+		"access": "public"
+	},
+	"repository": {
+		"type": "git",
+		"url": "git+https://github.com/remit-mail/remit.git",
+		"directory": "packages/mailbox-service"
+	}
+}

package/src/account-credentials.ts

@@ -0,0 +1,128 @@
+/**
+ * account-credentials.ts
+ *
+ * Single place in the codebase that branches on account.authType.
+ * All sync handlers (IMAP, SMTP) must call resolveConnectionCredentials
+ * rather than duplicating credential logic.
+ *
+ * Grep-verifiable constraint: `account.authType` must only appear here.
+ */
+
+import type { AccountItem } from "@remit/data-ports";
+import { AccountAuthType, ConnectionState } from "@remit/domain-enums";
+import type { MailOAuthService } from "@remit/mail-oauth-service";
+import { RefreshTokenError } from "@remit/mail-oauth-service";
+import {
+	deserializeEncryptedPayload,
+	type SecretsService,
+	serializeEncryptedPayload,
+} from "@remit/secrets-service";
+import type { MailCredentials } from "./types.js";
+
+export interface AccountCredentialsDeps {
+	secrets: Pick<SecretsService, "decrypt" | "encrypt">;
+	tokenService: Pick<MailOAuthService, "getAccessToken">;
+	/**
+	 * Called when the OAuth provider rotated the refresh token.
+	 * Must be awaited BEFORE the credentials are returned so the new token
+	 * is persisted before any IMAP command is issued.
+	 */
+	persistRotatedToken: (
+		accountId: string,
+		encryptedHash: string,
+		updatedAt: number,
+	) => Promise<void>;
+}
+
+/**
+ * Resolve IMAP credentials for an account.
+ *
+ * - Password accounts: decrypt passwordHash → return password credential.
+ * - OAuth accounts: decrypt oauthRefreshTokenHash → mint access token via
+ *   tokenService.  If the provider rotated the refresh token, persist the
+ *   new token via `deps.persistRotatedToken` BEFORE returning credentials.
+ *
+ * Throws `RefreshTokenError` on OAuth failures; callers should handle:
+ * - `kind === "reauth-required"` → set connectionState to reauth_required, ACK
+ * - `kind === "transient"` → rethrow (SQS retry)
+ * - `kind === "config"` → rethrow (SQS retry / alert)
+ */
+export const resolveConnectionCredentials = async (
+	account: AccountItem,
+	deps: AccountCredentialsDeps,
+): Promise<MailCredentials> => {
+	const authType = account.authType ?? AccountAuthType.Password;
+
+	if (authType === AccountAuthType.OauthMicrosoft) {
+		return resolveOauthCredentials(account, deps);
+	}
+
+	// Default: password auth
+	return resolvePasswordCredentials(account, deps.secrets);
+};
+
+const resolvePasswordCredentials = async (
+	account: AccountItem,
+	secrets: Pick<SecretsService, "decrypt">,
+): Promise<MailCredentials> => {
+	if (!account.passwordHash) {
+		throw new Error(
+			`Account ${account.accountId} has authType=password but no passwordHash`,
+		);
+	}
+	const password = await secrets.decrypt(
+		deserializeEncryptedPayload(JSON.parse(account.passwordHash)),
+	);
+	return { kind: "password", password };
+};
+
+const resolveOauthCredentials = async (
+	account: AccountItem,
+	deps: AccountCredentialsDeps,
+): Promise<MailCredentials> => {
+	if (!account.oauthRefreshTokenHash) {
+		throw new Error(
+			`Account ${account.accountId} has authType=oauthMicrosoft but no oauthRefreshTokenHash`,
+		);
+	}
+
+	const refreshToken = await deps.secrets.decrypt(
+		deserializeEncryptedPayload(JSON.parse(account.oauthRefreshTokenHash)),
+	);
+
+	// getAccessToken throws RefreshTokenError on failure
+	const tokenSet = await deps.tokenService.getAccessToken(
+		account.accountId,
+		refreshToken,
+	);
+
+	// If the provider rotated the refresh token, persist it BEFORE returning.
+	// This guarantees the new token is stored even if a subsequent error occurs.
+	if (tokenSet.refreshToken) {
+		const encryptedPayload = await deps.secrets.encrypt(tokenSet.refreshToken);
+		const encryptedHash = JSON.stringify(
+			serializeEncryptedPayload(encryptedPayload),
+		);
+		await deps.persistRotatedToken(
+			account.accountId,
+			encryptedHash,
+			Date.now(),
+		);
+	}
+
+	return { kind: "accessToken", accessToken: tokenSet.accessToken };
+};
+
+/**
+ * Build the encrypted hash string for an OAuth refresh token.
+ * Helper used when first storing an OAuth token (e.g. after OAuth callback).
+ */
+export const encryptRefreshToken = async (
+	refreshToken: string,
+	secrets: Pick<SecretsService, "encrypt">,
+): Promise<string> => {
+	const payload = await secrets.encrypt(refreshToken);
+	return JSON.stringify(serializeEncryptedPayload(payload));
+};
+
+export { RefreshTokenError, ConnectionState };

package/src/adversarial-mime.e2e.test.ts

@@ -0,0 +1,311 @@
+/**
+ * E2E adversarial MIME corpus (#402).
+ *
+ * Each .eml fixture under `test/fixtures/mime-adversarial-e2e/` is APPENDed to
+ * the mailfuzz Dovecot INBOX, fetched back via IMAP, then parsed end-to-end
+ * with mailparser. The suite asserts the two behaviours called out in #402:
+ *
+ *   1. body renders            — `parsed.text` or `parsed.html` is non-empty
+ *   2. attachments are listable — when the fixture declares attachments, every
+ *                                 expected filename appears in `parsed.attachments`
+ *                                 with non-zero content length
+ *
+ * The shapes here are content-driven adversarial cases that real-world senders
+ * actually emit (PDF as application/octet-stream, TNEF, nested multipart/related,
+ * forwarded message/rfc822, calendar invites, quoted-printable soft breaks,
+ * 8-bit UTF-8 bodies, empty text/plain alternatives) — the same class of shape
+ * that caused the body-sync regression in #394.
+ *
+ * Mapper unit-level defence lives in `test/fixtures/mime/`; this suite is the
+ * integration-layer belt-and-braces against the same regression class.
+ */
+
+import assert from "node:assert/strict";
+import { readdirSync, readFileSync } from "node:fs";
+import { dirname, join, resolve } from "node:path";
+import { after, before, describe, test } from "node:test";
+import { fileURLToPath } from "node:url";
+import { type ParsedMail, simpleParser } from "mailparser";
+import { uniqueMailboxName } from "./test-helpers/isolated-mailbox.js";
+import {
+	createMailfuzzConnection,
+	withMailfuzzConnection,
+} from "./test-helpers/mailfuzz-connection.js";
+
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = dirname(__filename);
+const FIXTURES_DIR = resolve(
+	__dirname,
+	"../test/fixtures/mime-adversarial-e2e",
+);
+
+interface AttachmentExpectation {
+	filename: string;
+	contentType?: string;
+}
+
+interface FixtureExpectation {
+	/** Subject must round-trip — proves the message we fetched is the one we appended. */
+	subjectIncludes: string;
+	/** At least one of `text` or `html` must be non-empty after parsing. */
+	bodyRenders: true;
+	/** Attachments expected after parsing. Empty array means "no attachments". */
+	attachments: AttachmentExpectation[];
+	/** Optional substring that must appear in parsed.text or parsed.html. */
+	bodyContains?: string;
+	/**
+	 * Skip the BODYSTRUCTURE re-fetch for this fixture.
+	 * Set when the fixture shape is synthetic and doesn't produce a meaningful
+	 * BODYSTRUCTURE response (e.g. bare TNEF blob — see fixture 05 and #408).
+	 */
+	skipBodyStructure?: true;
+}
+
+/**
+ * Expectations indexed by .eml basename. Each fixture must have one entry.
+ *
+ * Filename is the contract — match it exactly to the file on disk.
+ */
+const EXPECTATIONS: Record<string, FixtureExpectation> = {
+	"01-octet-stream-pdf.eml": {
+		subjectIncludes: "Adversarial fixture 01",
+		bodyRenders: true,
+		attachments: [{ filename: "contract.pdf" }],
+		bodyContains: "contract PDF",
+	},
+	"02-nested-related-cid-image.eml": {
+		subjectIncludes: "Adversarial fixture 02",
+		bodyRenders: true,
+		attachments: [
+			{ filename: "logo.png", contentType: "image/png" },
+			{ filename: "invoice.pdf" },
+		],
+		bodyContains: "Inline logo",
+	},
+	"03-calendar-invite.eml": {
+		subjectIncludes: "Adversarial fixture 03",
+		bodyRenders: true,
+		attachments: [{ filename: "invite.ics" }],
+		bodyContains: "Meeting invite",
+	},
+	"04-forwarded-rfc822-with-attachments.eml": {
+		subjectIncludes: "Adversarial fixture 04",
+		bodyRenders: true,
+		// mailparser surfaces the forwarded rfc822 as an attachment whose name
+		// derives from the Content-Disposition filename. The inner pdf is not
+		// surfaced as a top-level attachment because it lives inside the nested
+		// rfc822 — listing the rfc822 itself is the integration-layer contract.
+		attachments: [{ filename: "forwarded.eml" }],
+		bodyContains: "Forwarding",
+	},
+	"05-tnef-winmail.eml": {
+		subjectIncludes: "Adversarial fixture 05",
+		bodyRenders: true,
+		attachments: [{ filename: "winmail.dat" }],
+		bodyContains: "TNEF",
+		// skipBodyStructure: The TNEF fixture uses a synthetic magic-byte stub
+		// rather than a real-world winmail.dat capture, so the BODYSTRUCTURE
+		// returned by Dovecot for this fixture is not meaningful. The re-fetch
+		// is skipped here to avoid asserting on a shape that will change once
+		// the fixture is replaced with a real capture. See #408.
+		skipBodyStructure: true,
+	},
+	"06-8bit-non-ascii.eml": {
+		subjectIncludes: "Café façade",
+		bodyRenders: true,
+		attachments: [],
+		bodyContains: "café",
+	},
+	"07-quoted-printable-soft-breaks.eml": {
+		subjectIncludes: "Adversarial fixture 07",
+		bodyRenders: true,
+		attachments: [],
+		// "single" is split across a soft line break in the encoded source;
+		// only an actually-decoded quoted-printable reassembles it.
+		bodyContains: "single logical line",
+	},
+	"08-alternative-empty-text.eml": {
+		subjectIncludes: "Adversarial fixture 08",
+		bodyRenders: true,
+		attachments: [],
+		// text/plain is empty; rendering must fall back to html.
+		bodyContains: "intentionally empty",
+	},
+};
+
+const listEmlFixtures = (): string[] =>
+	readdirSync(FIXTURES_DIR)
+		.filter((name) => name.endsWith(".eml"))
+		.sort();
+
+const loadFixtureRaw = (name: string): string => {
+	const path = join(FIXTURES_DIR, name);
+	// Read as utf8 — fixture 06 includes raw 8-bit UTF-8 bytes which IMAP
+	// servers accept on APPEND as long as they're valid UTF-8 octets.
+	return readFileSync(path, "utf8");
+};
+
+const bodyText = (parsed: ParsedMail): string => {
+	const text = parsed.text ?? "";
+	const html = typeof parsed.html === "string" ? parsed.html : "";
+	return `${text}\n${html}`;
+};
+
+describe(
+	"Adversarial MIME corpus (Dovecot) — #402",
+	{ skip: !process.env.RUN_E2E_TESTS },
+	() => {
+		// Own mailbox instead of the shared INBOX so appending fixtures never
+		// collides with another e2e file's count/flag assertions (#508).
+		const inbox = uniqueMailboxName("E2E_adversarial");
+
+		before(async () => {
+			await withMailfuzzConnection(async (connection) => {
+				await connection.createMailbox(inbox);
+			});
+		});
+
+		after(async () => {
+			const connection = createMailfuzzConnection();
+			await connection.connect();
+			await connection.deleteMailbox(inbox).catch(() => {});
+			if (connection.isConnected) {
+				await connection.disconnect();
+			}
+		});
+
+		// Sanity: every .eml on disk has an expectation entry, and every
+		// expectation entry points at an existing file. Catches a fixture being
+		// added without an assertion (or vice versa).
+		test("every fixture has an expectation entry", () => {
+			const onDisk = new Set(listEmlFixtures());
+			const expected = new Set(Object.keys(EXPECTATIONS));
+			for (const name of onDisk) {
+				assert.ok(
+					expected.has(name),
+					`Fixture ${name} has no expectation entry in EXPECTATIONS`,
+				);
+			}
+			for (const name of expected) {
+				assert.ok(
+					onDisk.has(name),
+					`Expectation ${name} has no .eml file on disk`,
+				);
+			}
+			// Pin the count: #402 calls for 8 adversarial shapes.
+			assert.equal(
+				onDisk.size,
+				8,
+				"Expected exactly 8 adversarial MIME fixtures",
+			);
+		});
+
+		for (const name of Object.keys(EXPECTATIONS).sort()) {
+			const expectation = EXPECTATIONS[name];
+			if (!expectation) continue;
+
+			test(`fixture ${name} round-trips through IMAP and parses`, async () => {
+				const raw = loadFixtureRaw(name);
+
+				await withMailfuzzConnection(async (connection) => {
+					// APPEND the fixture into this file's own mailbox. Mailfuzz uses
+					// Dovecot which returns a proper UID on APPEND (mokapi does not —
+					// see outbox-roundtrip.e2e.test.ts and the RFC 025 notes).
+					const appendResult = await connection.append(inbox, raw);
+					assert.ok(
+						appendResult.uid > 0,
+						`APPEND should return a UID for ${name}`,
+					);
+
+					await connection.openBox(inbox, true);
+
+					// Fetch the message we just appended back. fetchMessageBody
+					// returns the raw RFC822 source; mailparser handles the rest
+					// (multipart walking, base64/qp decoding, charset folding).
+					const rfc822 = await connection.fetchMessageBody(appendResult.uid);
+					assert.ok(
+						rfc822.length > 0,
+						`fetchMessageBody should return bytes for ${name}`,
+					);
+
+					// BODYSTRUCTURE re-fetch: exercises the IMAP mapper path at the
+					// integration boundary (the exact regression class from #394).
+					// Skipped for fixtures with synthetic shapes that don't produce
+					// meaningful BODYSTRUCTURE (see skipBodyStructure flag + #408).
+					// The guard in fetchMessages skips rows with undefined uid/internalDate
+					// so back-to-back FETCH is safe on Dovecot (see imapflow-connection.ts).
+					if (!expectation.skipBodyStructure) {
+						const refetched = await connection.fetchMessages([
+							appendResult.uid,
+						]);
+						assert.ok(
+							refetched.length > 0,
+							`BODYSTRUCTURE re-fetch should return a message for ${name}`,
+						);
+						assert.ok(
+							refetched[0]?.bodyStructure != null,
+							`BODYSTRUCTURE should be present for ${name}`,
+						);
+					}
+
+					const parsed = await simpleParser(rfc822);
+
+					// Subject round-trips (proves we fetched the right message and
+					// any encoded-word headers decoded correctly).
+					assert.ok(
+						parsed.subject,
+						`Parsed subject should be present for ${name}`,
+					);
+					assert.ok(
+						parsed.subject?.includes(expectation.subjectIncludes),
+						`Subject "${parsed.subject}" should include "${expectation.subjectIncludes}" for ${name}`,
+					);
+
+					// Body renders — at least one of text/html is non-empty. This
+					// is the headline "body renders" assertion from #402.
+					const text = parsed.text ?? "";
+					const html = typeof parsed.html === "string" ? parsed.html : "";
+					assert.ok(
+						text.trim().length > 0 || html.trim().length > 0,
+						`Body should render (text or html non-empty) for ${name}`,
+					);
+
+					if (expectation.bodyContains) {
+						const haystack = bodyText(parsed).toLowerCase();
+						const needle = expectation.bodyContains.toLowerCase();
+						assert.ok(
+							haystack.includes(needle),
+							`Body should contain "${expectation.bodyContains}" for ${name}`,
+						);
+					}
+
+					// Attachments are listable — every expected attachment appears
+					// in parsed.attachments with a matching filename and non-zero
+					// content length. This is the "attachments listable" half of
+					// the #402 contract.
+					const attachments = parsed.attachments ?? [];
+					for (const expectedAtt of expectation.attachments) {
+						const found = attachments.find(
+							(a) => a.filename === expectedAtt.filename,
+						);
+						assert.ok(
+							found,
+							`Attachment "${expectedAtt.filename}" should be listable for ${name}; got [${attachments.map((a) => a.filename).join(", ")}]`,
+						);
+						assert.ok(
+							(found.size ?? 0) > 0,
+							`Attachment "${expectedAtt.filename}" should have non-zero size for ${name}`,
+						);
+						if (expectedAtt.contentType) {
+							assert.equal(
+								found.contentType,
+								expectedAtt.contentType,
+								`Attachment "${expectedAtt.filename}" should have contentType "${expectedAtt.contentType}" for ${name}`,
+							);
+						}
+					}
+				});
+			});
+		}
+	},
+);