@remit/imap-worker 0.0.14 → 0.0.16

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@remit/imap-worker",
-  "version": "0.0.14",
+  "version": "0.0.16",
   "type": "module",
   "main": "dist/index.js",
   "exports": {
@@ -20,7 +20,7 @@
     "bundle": "esbuild src/index.ts --sourcemap --bundle --platform=node --format=esm --outfile=dist/index.js",
     "cli": "node --env-file=../../localhost-dev-aws.env src/cli.ts",
     "test:typecheck": "tsgo --noEmit",
-    "test:run": "node --env-file=../../localhost-test-unit.env --experimental-test-coverage --test-coverage-include='src/**' --test-coverage-exclude='src/**/*.test.ts' --test-coverage-exclude='src/**/*.test.tsx' --test-coverage-lines=50 --test 'src/**/*.test.ts'",
+    "test:run": "node --env-file=../../localhost-test-unit.env --experimental-test-coverage --test-coverage-include='src/**' --test-coverage-exclude='src/**/*.test.ts' --test-coverage-exclude='src/**/*.test.tsx' --test-coverage-lines=75 --test 'src/**/*.test.ts'",
     "test": "npm run test:typecheck && npm run test:run",
     "dev": "node --import tsx src/e2e-processor-shim.ts"
   },

package/src/handlers/append-sent-message.test.ts

@@ -0,0 +1,216 @@
+import assert from "node:assert/strict";
+import { beforeEach, describe, it } from "node:test";
+import type { Logger } from "@remit/logger-lambda";
+import type { AppendSentMessageEvent } from "../events.js";
+import {
+	type AppendSentMessageDeps,
+	handleAppendSentMessage,
+} from "./append-sent-message.js";
+
+const noopLog = {
+	info: () => {},
+	warn: () => {},
+	error: () => {},
+	debug: () => {},
+	fatal: () => {},
+	trace: () => {},
+	child: () => noopLog,
+} as unknown as Logger;
+
+interface Call {
+	method: string;
+	args: unknown[];
+}
+
+interface Harness {
+	calls: Call[];
+	account: {
+		accountId: string;
+		accountConfigId: string;
+		deletedAt?: number;
+	};
+	outbox: Record<string, unknown>;
+	specialUseSent: { mailboxId: string; fullPath: string } | null;
+	mailboxes: { mailboxId: string; fullPath: string }[];
+	append: (
+		path: string,
+		raw: Buffer,
+		flags: string[],
+	) => Promise<{ uid: number; uidValidity: number }>;
+	disconnectCount: number;
+}
+
+let h: Harness;
+
+const record =
+	(method: string) =>
+	async (...args: unknown[]) => {
+		h.calls.push({ method, args });
+	};
+
+const fresh = (): Harness => ({
+	calls: [],
+	account: { accountId: "acc-1", accountConfigId: "cfg-1" },
+	outbox: {
+		status: "sent",
+		fromName: "Alice",
+		fromAddress: "alice@example.com",
+		toAddresses: ["bob@example.com"],
+		ccAddresses: ["carol@example.com"],
+		subject: "Quarterly numbers",
+		textBody: "See attached.",
+		messageIdValue: "generated-id@example.com",
+		references: ["parent@example.com"],
+		inReplyTo: "parent@example.com",
+		sentAt: 1700000000000,
+	},
+	specialUseSent: { mailboxId: "sent-mbx", fullPath: "Sent" },
+	mailboxes: [
+		{ mailboxId: "inbox-mbx", fullPath: "INBOX" },
+		{ mailboxId: "sent-items-mbx", fullPath: "Sent Items" },
+	],
+	append: async (path, raw, flags) => {
+		h.calls.push({ method: "connection.append", args: [path, raw, flags] });
+		return { uid: 55, uidValidity: 7 };
+	},
+	disconnectCount: 0,
+});
+
+const deps = (): AppendSentMessageDeps =>
+	({
+		getClient: async () => ({
+			account: {
+				get: async (accountId: string) => {
+					h.calls.push({ method: "account.get", args: [accountId] });
+					return h.account;
+				},
+			},
+			outboxMessage: {
+				get: async () => h.outbox,
+				delete: record("outboxMessage.delete"),
+			},
+			mailboxSpecialUse: {
+				findBySpecialUse: async () => h.specialUseSent,
+			},
+			mailbox: {
+				listByAccount: async () => ({ items: h.mailboxes }),
+			},
+			secrets: {},
+		}),
+		buildLifecycleDeps: () => ({}),
+		withOAuthLifecycle: async (
+			_deps: unknown,
+			_account: unknown,
+			_log: unknown,
+			cb: (credentials: unknown) => Promise<void>,
+		) => cb({}),
+		createConnectionScope: () => ({
+			getConnection: async () => ({
+				append: (path: string, raw: Buffer, flags: string[]) =>
+					h.append(path, raw, flags),
+			}),
+			disconnect: async () => {
+				h.disconnectCount += 1;
+			},
+		}),
+	}) as unknown as AppendSentMessageDeps;
+
+const event: AppendSentMessageEvent = {
+	type: "APPEND_SENT_MESSAGE",
+	accountId: "acc-1",
+	outboxMessageId: "out-1",
+} as AppendSentMessageEvent;
+
+const called = (method: string): Call[] =>
+	h.calls.filter((c) => c.method === method);
+
+describe("handleAppendSentMessage", () => {
+	beforeEach(() => {
+		h = fresh();
+	});
+
+	it("appends a seen RFC822 copy to the Sent folder and drops the outbox row", async () => {
+		await handleAppendSentMessage(event, noopLog, deps());
+
+		const append = called("connection.append")[0];
+		assert.equal(append?.args[0], "Sent");
+		assert.deepEqual(append?.args[2], ["\\Seen"]);
+		assert.deepEqual(called("outboxMessage.delete")[0]?.args, [
+			"cfg-1",
+			"out-1",
+		]);
+		assert.equal(h.disconnectCount, 1);
+	});
+
+	it("builds the message from the outbox row's own headers", async () => {
+		await handleAppendSentMessage(event, noopLog, deps());
+
+		const raw = String(called("connection.append")[0]?.args[1] as Buffer);
+		assert.match(raw, /^From: Alice <alice@example\.com>$/m);
+		assert.match(raw, /^To: bob@example\.com$/m);
+		assert.match(raw, /^Cc: carol@example\.com$/m);
+		assert.match(raw, /^Subject: Quarterly numbers$/m);
+		assert.match(raw, /^Message-ID: <generated-id@example\.com>$/m);
+		assert.match(raw, /^In-Reply-To: <parent@example\.com>$/m);
+		assert.ok(raw.includes("See attached."));
+	});
+
+	it("uses a bare address when the outbox row carries no display name", async () => {
+		h.outbox = { ...h.outbox, fromName: undefined };
+
+		await handleAppendSentMessage(event, noopLog, deps());
+
+		const raw = String(called("connection.append")[0]?.args[1] as Buffer);
+		assert.match(raw, /^From: alice@example\.com$/m);
+	});
+
+	it("falls back to a conventionally-named Sent folder when no special-use flag is set", async () => {
+		h.specialUseSent = null;
+
+		await handleAppendSentMessage(event, noopLog, deps());
+
+		assert.equal(called("connection.append")[0]?.args[0], "Sent Items");
+	});
+
+	it("skips the append when the account has no Sent folder at all", async () => {
+		h.specialUseSent = null;
+		h.mailboxes = [{ mailboxId: "inbox-mbx", fullPath: "INBOX" }];
+
+		await handleAppendSentMessage(event, noopLog, deps());
+
+		assert.equal(called("connection.append").length, 0);
+		assert.equal(called("outboxMessage.delete").length, 0);
+	});
+
+	it("skips the append while the outbox row is not yet sent", async () => {
+		h.outbox = { ...h.outbox, status: "pending" };
+
+		await handleAppendSentMessage(event, noopLog, deps());
+
+		assert.equal(called("connection.append").length, 0);
+		assert.equal(called("outboxMessage.delete").length, 0);
+	});
+
+	it("returns early without touching the outbox when the account is soft-deleted", async () => {
+		h.account = { ...h.account, deletedAt: Date.now() };
+
+		await handleAppendSentMessage(event, noopLog, deps());
+
+		assert.equal(called("connection.append").length, 0);
+		assert.equal(called("outboxMessage.delete").length, 0);
+	});
+
+	it("keeps the outbox row when the APPEND fails so the send can be retried", async () => {
+		h.append = async () => {
+			throw new Error("server exploded");
+		};
+
+		await assert.rejects(
+			handleAppendSentMessage(event, noopLog, deps()),
+			/server exploded/,
+		);
+
+		assert.equal(called("outboxMessage.delete").length, 0);
+		assert.equal(h.disconnectCount, 1);
+	});
+});

package/src/handlers/append-sent-message.ts

@@ -75,10 +75,32 @@ const buildRawMessage = async (outbox: OutboxMessageItem): Promise<Buffer> => {
 	return Buffer.concat(chunks);
 };
 
+export interface AppendSentMessageDeps {
+	getClient: typeof getClient;
+	buildLifecycleDeps: typeof buildLifecycleDeps;
+	withOAuthLifecycle: typeof withOAuthLifecycle;
+	createConnectionScope: typeof createConnectionScopeWithCredentials;
+}
+
+const defaultDeps: AppendSentMessageDeps = {
+	getClient,
+	buildLifecycleDeps,
+	withOAuthLifecycle,
+	createConnectionScope: createConnectionScopeWithCredentials,
+};
+
 export const handleAppendSentMessage = async (
 	event: AppendSentMessageEvent,
 	log: Logger,
+	deps: AppendSentMessageDeps = defaultDeps,
 ): Promise<void> => {
+	const {
+		getClient,
+		buildLifecycleDeps,
+		withOAuthLifecycle,
+		createConnectionScope: createConnectionScopeWithCredentials,
+	} = deps;
+
 	const {
 		account: accountService,
 		outboxMessage: outboxMessageService,

package/src/handlers/delete-account-objects.test.ts

@@ -1,81 +1,129 @@
 import assert from "node:assert/strict";
-import { describe, it } from "node:test";
-import type { DeleteAccountObjectsEvent } from "./delete-account-objects.js";
-
-describe("DeleteAccountObjects handler", () => {
-	it("event shape is correct", () => {
-		const event: DeleteAccountObjectsEvent = {
-			type: "DELETE_ACCOUNT_OBJECTS",
-			accountConfigId: "test-account-config-id-12345",
-		};
-
-		assert.equal(event.type, "DELETE_ACCOUNT_OBJECTS");
-		assert.equal(event.accountConfigId, "test-account-config-id-12345");
-		assert.equal(event.continuationToken, undefined);
+import { afterEach, describe, it } from "node:test";
+import {
+	DeleteObjectsCommand,
+	ListObjectsV2Command,
+	S3Client,
+} from "@aws-sdk/client-s3";
+import { SendMessageCommand, SQSClient } from "@aws-sdk/client-sqs";
+import type { Logger } from "@remit/logger-lambda";
+import { mockClient } from "aws-sdk-client-mock";
+import {
+	type DeleteAccountObjectsEvent,
+	handleDeleteAccountObjects,
+} from "./delete-account-objects.js";
+
+const noopLog = {
+	info: () => {},
+	warn: () => {},
+	error: () => {},
+	debug: () => {},
+	fatal: () => {},
+	trace: () => {},
+	child: () => noopLog,
+} as unknown as Logger;
+
+const s3Mock = mockClient(S3Client);
+const sqsMock = mockClient(SQSClient);
+
+const event: DeleteAccountObjectsEvent = {
+	type: "DELETE_ACCOUNT_OBJECTS",
+	accountConfigId: "cfg-1",
+};
+
+describe("handleDeleteAccountObjects", () => {
+	afterEach(() => {
+		s3Mock.reset();
+		sqsMock.reset();
 	});
 
-	it("event with continuation token is correct", () => {
-		const event: DeleteAccountObjectsEvent = {
-			type: "DELETE_ACCOUNT_OBJECTS",
-			accountConfigId: "test-account-config-id-12345",
-			continuationToken: "abc123",
-		};
-
-		assert.equal(event.continuationToken, "abc123");
+	it("lists and deletes every object under the account prefix in one page", async () => {
+		s3Mock.on(ListObjectsV2Command).resolves({
+			Contents: [
+				{ Key: "accounts/cfg-1/a.eml" },
+				{ Key: "accounts/cfg-1/b.eml" },
+			],
+			IsTruncated: false,
+		});
+		s3Mock.on(DeleteObjectsCommand).resolves({});
+
+		await handleDeleteAccountObjects(event, noopLog);
+
+		const listCall = s3Mock.commandCalls(ListObjectsV2Command)[0];
+		assert.equal(listCall.args[0].input.Prefix, "accounts/cfg-1/");
+
+		const deleteCall = s3Mock.commandCalls(DeleteObjectsCommand)[0];
+		assert.deepEqual(deleteCall.args[0].input.Delete?.Objects, [
+			{ Key: "accounts/cfg-1/a.eml" },
+			{ Key: "accounts/cfg-1/b.eml" },
+		]);
+		assert.equal(sqsMock.commandCalls(SendMessageCommand).length, 0);
 	});
 
-	it("S3 prefix follows expected pattern", () => {
-		const accountConfigId = "test-account-config-id-12345";
-		const prefix = `accounts/${accountConfigId}/`;
-
-		assert.ok(prefix.startsWith("accounts/"));
-		assert.ok(prefix.endsWith("/"));
-		assert.ok(prefix.includes(accountConfigId));
+	it("follows the continuation token across truncated pages", async () => {
+		s3Mock
+			.on(ListObjectsV2Command)
+			.resolvesOnce({
+				Contents: [{ Key: "accounts/cfg-1/p1.eml" }],
+				IsTruncated: true,
+				NextContinuationToken: "tok-2",
+			})
+			.resolvesOnce({
+				Contents: [{ Key: "accounts/cfg-1/p2.eml" }],
+				IsTruncated: false,
+			});
+		s3Mock.on(DeleteObjectsCommand).resolves({});
+
+		await handleDeleteAccountObjects(event, noopLog);
+
+		const listCalls = s3Mock.commandCalls(ListObjectsV2Command);
+		assert.equal(listCalls.length, 2);
+		assert.equal(listCalls[1]?.args[0].input.ContinuationToken, "tok-2");
+		assert.equal(s3Mock.commandCalls(DeleteObjectsCommand).length, 2);
 	});
 
-	it("batch size calculation respects limit", () => {
-		const BATCH_SIZE = 1_000;
-		const keys = Array.from({ length: 2500 }, (_, i) => `key-${i}`);
+	it("skips the delete call when a page holds no keys", async () => {
+		s3Mock.on(ListObjectsV2Command).resolves({ IsTruncated: false });
 
-		// Simulate batching
-		const batches: string[][] = [];
-		for (let i = 0; i < keys.length; i += BATCH_SIZE) {
-			batches.push(keys.slice(i, i + BATCH_SIZE));
-		}
+		await handleDeleteAccountObjects(event, noopLog);
 
-		assert.equal(batches.length, 3);
-		assert.equal(batches[0].length, 1000);
-		assert.equal(batches[1].length, 1000);
-		assert.equal(batches[2].length, 500);
+		assert.equal(s3Mock.commandCalls(DeleteObjectsCommand).length, 0);
 	});
 
-	it("re-enqueue event preserves continuation token", () => {
-		const accountConfigId = "test-id";
-		const continuationToken = "next-page-token";
-
-		const reenqueueEvent: DeleteAccountObjectsEvent = {
-			type: "DELETE_ACCOUNT_OBJECTS",
-			accountConfigId,
-			continuationToken,
-		};
-
-		const body = JSON.stringify(reenqueueEvent);
-		const parsed = JSON.parse(body) as DeleteAccountObjectsEvent;
-
-		assert.equal(parsed.type, "DELETE_ACCOUNT_OBJECTS");
-		assert.equal(parsed.accountConfigId, accountConfigId);
-		assert.equal(parsed.continuationToken, continuationToken);
+	it("re-enqueues with the continuation token when time is nearly up", async () => {
+		s3Mock.on(ListObjectsV2Command).resolves({ IsTruncated: false });
+		sqsMock.on(SendMessageCommand).resolves({});
+
+		await handleDeleteAccountObjects(
+			{ ...event, continuationToken: "tok-mid" },
+			noopLog,
+			() => 5_000,
+		);
+
+		assert.equal(
+			s3Mock.commandCalls(ListObjectsV2Command).length,
+			0,
+			"bails before starting a new page",
+		);
+		const send = sqsMock.commandCalls(SendMessageCommand)[0];
+		const body = JSON.parse(
+			send?.args[0].input.MessageBody ?? "{}",
+		) as DeleteAccountObjectsEvent;
+		assert.equal(body.type, "DELETE_ACCOUNT_OBJECTS");
+		assert.equal(body.accountConfigId, "cfg-1");
+		assert.equal(body.continuationToken, "tok-mid");
 	});
 
-	it("timeout detection triggers re-enqueue", () => {
-		const MIN_REMAINING_MS = 30_000;
+	it("keeps paging while time remains", async () => {
+		s3Mock.on(ListObjectsV2Command).resolves({
+			Contents: [{ Key: "accounts/cfg-1/x.eml" }],
+			IsTruncated: false,
+		});
+		s3Mock.on(DeleteObjectsCommand).resolves({});
 
-		// Simulate near-timeout scenario
-		const getRemainingTimeMs = () => 25_000;
-		assert.ok(getRemainingTimeMs() < MIN_REMAINING_MS);
+		await handleDeleteAccountObjects(event, noopLog, () => 120_000);
 
-		// Simulate enough time
-		const getRemainingTimeMsOk = () => 60_000;
-		assert.ok(getRemainingTimeMsOk() >= MIN_REMAINING_MS);
+		assert.equal(s3Mock.commandCalls(ListObjectsV2Command).length, 1);
+		assert.equal(sqsMock.commandCalls(SendMessageCommand).length, 0);
 	});
 });

package/src/handlers/empty-trash.test.ts

@@ -0,0 +1,220 @@
+import assert from "node:assert/strict";
+import { beforeEach, describe, it } from "node:test";
+import type { Logger } from "@remit/logger-lambda";
+import type { EmptyTrashEvent } from "../events.js";
+import { type EmptyTrashDeps, handleEmptyTrash } from "./empty-trash.js";
+
+const noopLog = {
+	info: () => {},
+	warn: () => {},
+	error: () => {},
+	debug: () => {},
+	fatal: () => {},
+	trace: () => {},
+	child: () => noopLog,
+} as unknown as Logger;
+
+interface Call {
+	method: string;
+	args: unknown[];
+}
+
+interface Connection {
+	openBox: (
+		path: string,
+		readOnly?: boolean,
+	) => Promise<{ uidvalidity: number }>;
+	search: (criteria: string[]) => Promise<number[]>;
+	deleteMessages: (uids: number[]) => Promise<void>;
+}
+
+interface Harness {
+	calls: Call[];
+	account: {
+		accountId: string;
+		accountConfigId: string;
+		deletedAt?: number;
+	} | null;
+	mailbox: { mailboxId: string; uidValidity: number; cursorState?: string };
+	connection: Connection;
+	localMessages: { messageId: string }[];
+	threadMessage: { accountConfigId: string; threadMessageId: string } | null;
+	getConnectionCount: number;
+	disconnectCount: number;
+}
+
+let h: Harness;
+
+const record =
+	(method: string) =>
+	async (...args: unknown[]) => {
+		h.calls.push({ method, args });
+	};
+
+const buildConnection = (): Connection => ({
+	openBox: async () => ({ uidvalidity: 1 }),
+	search: async () => [10, 11],
+	deleteMessages: record(
+		"connection.deleteMessages",
+	) as Connection["deleteMessages"],
+});
+
+const fresh = (): Harness => ({
+	calls: [],
+	account: { accountId: "acc-1", accountConfigId: "cfg-1" },
+	mailbox: { mailboxId: "trash-mbx", uidValidity: 1, cursorState: undefined },
+	connection: buildConnection(),
+	localMessages: [{ messageId: "msg-1" }, { messageId: "msg-2" }],
+	threadMessage: { accountConfigId: "cfg-1", threadMessageId: "tm-1" },
+	getConnectionCount: 0,
+	disconnectCount: 0,
+});
+
+const deps = (): EmptyTrashDeps =>
+	({
+		getClient: async () => ({
+			account: {
+				get: async (accountId: string) => {
+					h.calls.push({ method: "account.get", args: [accountId] });
+					return h.account;
+				},
+			},
+			message: {
+				listAllByMailbox: async () => h.localMessages,
+				delete: record("message.delete"),
+			},
+			threadMessage: {
+				findByMessageId: async () => h.threadMessage,
+				delete: record("threadMessage.delete"),
+			},
+			mailbox: {
+				get: async () => h.mailbox,
+				update: record("mailbox.update"),
+			},
+			secrets: {},
+		}),
+		buildLifecycleDeps: () => ({}),
+		withOAuthLifecycle: async (
+			_deps: unknown,
+			_account: unknown,
+			_log: unknown,
+			cb: (credentials: unknown) => Promise<void>,
+		) => cb({}),
+		createConnectionScope: () => ({
+			getConnection: async () => {
+				h.getConnectionCount += 1;
+				return h.connection;
+			},
+			disconnect: async () => {
+				h.disconnectCount += 1;
+			},
+		}),
+	}) as unknown as EmptyTrashDeps;
+
+const event: EmptyTrashEvent = {
+	type: "EMPTY_TRASH",
+	accountId: "acc-1",
+	trashMailboxId: "trash-mbx",
+	trashMailboxPath: "Trash",
+} as EmptyTrashEvent;
+
+const called = (method: string): Call[] =>
+	h.calls.filter((c) => c.method === method);
+
+describe("handleEmptyTrash", () => {
+	beforeEach(() => {
+		h = fresh();
+	});
+
+	it("expunges every server uid and both local rows for each trashed message", async () => {
+		await handleEmptyTrash(event, noopLog, deps());
+
+		assert.deepEqual(called("connection.deleteMessages")[0]?.args, [[10, 11]]);
+		assert.deepEqual(
+			called("message.delete").map((c) => c.args[0]),
+			["msg-1", "msg-2"],
+		);
+		assert.equal(called("threadMessage.delete").length, 2);
+		assert.equal(h.disconnectCount, 1, "the scope is always disconnected");
+	});
+
+	it("skips the IMAP expunge when the trash is already empty on the server", async () => {
+		h.connection.search = async () => [];
+
+		await handleEmptyTrash(event, noopLog, deps());
+
+		assert.equal(called("connection.deleteMessages").length, 0);
+		assert.equal(
+			called("message.delete").length,
+			2,
+			"local rows are still cleaned up",
+		);
+	});
+
+	it("deletes the message even when it has no thread row", async () => {
+		h.threadMessage = null;
+
+		await handleEmptyTrash(event, noopLog, deps());
+
+		assert.equal(called("message.delete").length, 2);
+		assert.equal(called("threadMessage.delete").length, 0);
+	});
+
+	it("returns early without connecting when the account is soft-deleted", async () => {
+		h.account = {
+			accountId: "acc-1",
+			accountConfigId: "cfg-1",
+			deletedAt: Date.now(),
+		};
+
+		await handleEmptyTrash(event, noopLog, deps());
+
+		assert.equal(h.getConnectionCount, 0);
+	});
+
+	it("throws when the account no longer exists", async () => {
+		h.account = null;
+
+		await assert.rejects(handleEmptyTrash(event, noopLog, deps()), /not found/);
+	});
+
+	it("skips without opening a connection when the cursor is rebuilding", async () => {
+		h.mailbox = {
+			mailboxId: "trash-mbx",
+			uidValidity: 1,
+			cursorState: "rebuilding",
+		};
+
+		await handleEmptyTrash(event, noopLog, deps());
+
+		assert.equal(h.getConnectionCount, 0);
+		assert.equal(called("message.delete").length, 0);
+	});
+
+	it("pauses quietly when openBox trips a UIDVALIDITY mismatch", async () => {
+		h.connection.openBox = async () => ({ uidvalidity: 999 });
+
+		await handleEmptyTrash(event, noopLog, deps());
+
+		assert.equal(
+			(called("mailbox.update")[0]?.args[2] as { cursorState?: string })
+				?.cursorState,
+			"cursor_invalid",
+		);
+		assert.equal(called("message.delete").length, 0);
+		assert.equal(h.disconnectCount, 1);
+	});
+
+	it("rethrows an unclassified IMAP error so the event is retried", async () => {
+		h.connection.search = async () => {
+			throw new Error("server exploded");
+		};
+
+		await assert.rejects(
+			handleEmptyTrash(event, noopLog, deps()),
+			/server exploded/,
+		);
+
+		assert.equal(h.disconnectCount, 1);
+	});
+});