@remit/imap-worker 0.0.85 → 0.0.87

package/README.md

@@ -48,7 +48,9 @@ npm run cli -- -t SYNC_MESSAGE_BODY -a <accountId> -m <mailboxId> --messageIds i
 | Variable                      | Required | Description                                              |
 | ----------------------------- | -------- | -------------------------------------------------------- |
 | `DATA_BACKEND`                | Yes      | Relational backend to run against (`sqlite`)             |
-| `SQLITE_DB_PATH`              | Yes      | Path to the shared SQLite database file                  |
+| `SQLITE_APP_DB_PATH`          | Yes      | Path to the app SQLite database                          |
+| `SQLITE_MAIL_DB_PATH`         | Yes      | Path to the tenant mail SQLite database                  |
+| `SQLITE_SEARCH_DB_PATH`       | Yes      | Path to the tenant search SQLite database                |
 | `SQS_QUEUE_URL_MAILBOXES`     | Yes      | FIFO queue URL for `SYNC_MAILBOXES` events               |
 | `SQS_QUEUE_URL_MESSAGES`      | Yes      | FIFO queue URL for `SYNC_MESSAGES` events                |
 | `SQS_QUEUE_URL_BODY`          | Yes      | FIFO queue URL for `SYNC_MESSAGE_BODY` events            |

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@remit/imap-worker",
-  "version": "0.0.85",
+  "version": "0.0.87",
   "type": "module",
   "main": "dist/index.js",
   "exports": {
@@ -11,6 +11,14 @@
     "./scheduler": {
       "types": "./src/scheduler/handler.ts",
       "default": "./src/scheduler/handler.ts"
+    },
+    "./mailbox-management": {
+      "types": "./src/handlers/mailbox-management.ts",
+      "default": "./src/handlers/mailbox-management.ts"
+    },
+    "./test-helpers/test-world": {
+      "types": "./src/test-helpers/test-world.ts",
+      "default": "./src/test-helpers/test-world.ts"
     }
   },
   "bin": {
@@ -32,13 +40,13 @@
     "esbuild": "^0.28.1"
   },
   "dependencies": {
+    "@remit/logger": "*",
     "@aws-sdk/client-s3": "^3.1055.0",
     "@remit/config-transfer": "*",
     "@remit/data-ports": "*",
     "@remit/smtp-service": "*",
     "@remit/logger-lambda": "*",
     "@remit/mailbox-service": "*",
-    "@remit/search-index-worker": "*",
     "@remit/sqs-client": "*",
     "@remit/backend": "*",
     "@remit/calendar-service": "*",

package/src/account-check.ts

@@ -1,6 +1,6 @@
 import type { AccountItem } from "@remit/data-ports";
 import { AccountService } from "@remit/domain-enums";
-import type { Logger } from "@remit/logger-lambda";
+import type { Logger } from "@remit/logger";
 
 /**
  * RFC 2606 reserved placeholder namespaces that are guaranteed never to resolve

package/src/body-sync-gate.test.ts

@@ -5,7 +5,7 @@ import {
 	ParameterNotFound,
 	SSMClient,
 } from "@aws-sdk/client-ssm";
-import type { Logger } from "@remit/logger-lambda";
+import type { Logger } from "@remit/logger";
 import { mockClient } from "aws-sdk-client-mock";
 import { isBodySyncEnabled, resetBodySyncGateCache } from "./body-sync-gate.js";
 

package/src/body-sync-gate.ts

@@ -1,5 +1,5 @@
 import { GetParameterCommand, SSMClient } from "@aws-sdk/client-ssm";
-import type { Logger } from "@remit/logger-lambda";
+import type { Logger } from "@remit/logger";
 
 const CACHE_TTL_MS = 30_000;
 

package/src/e2e-processor-shim.ts

@@ -5,18 +5,15 @@ import {
 	DeleteMessageCommand,
 	ReceiveMessageCommand,
 } from "@aws-sdk/client-sqs";
-import { createLogger } from "@remit/logger-lambda";
-import { handler as searchIndexHandler } from "@remit/search-index-worker";
+import { createLogger } from "@remit/logger";
+import { installNodeLogger } from "@remit/logger-lambda/node";
 import { createQueueProducer } from "@remit/sqs-client/producer";
-import type {
-	Context,
-	SQSBatchResponse,
-	SQSEvent,
-	SQSHandler,
-} from "aws-lambda";
+import type { Context, SQSBatchResponse, SQSEvent } from "aws-lambda";
 import { env } from "expect-env";
 import { handler } from "./index.js";
 
+installNodeLogger();
+
 /**
  * E2E-only queue drainer.
  *
@@ -34,11 +31,6 @@ import { handler } from "./index.js";
  * fails loudly, which is the desired signal — there is no crash net.
  */
 
-// The search-index queue is drained by the production search-index-worker
-// handler instead of the imap handler. It is optional: when its URL is unset
-// (e.g. the e2e stack), the queue is simply not polled.
-const searchIndexQueueUrl = process.env.SQS_QUEUE_URL_SEARCH_INDEX;
-
 // Collect all unique queue URLs to poll. Every required queue URL crashes at
 // init via expect-env instead of silently dropping queues.
 const queueUrls = [
@@ -51,8 +43,6 @@ const queueUrls = [
 		env.SQS_QUEUE_URL_BODY,
 		env.SQS_QUEUE_URL_MAILBOX_MGMT,
 		env.SQS_QUEUE_URL_MESSAGE_MGMT,
-		// Standard queue for local search indexing (optional)
-		...(searchIndexQueueUrl ? [searchIndexQueueUrl] : []),
 	]),
 ];
 
@@ -109,11 +99,6 @@ if (cluster.isPrimary) {
 	const queueName = new URL(queueUrl).pathname.split("/").pop();
 	const log = createLogger().child({ queue: queueName });
 
-	// The search-index queue is owned by the search-index-worker handler; every
-	// other queue is an imap operation.
-	const activeHandler: SQSHandler =
-		queueUrl === searchIndexQueueUrl ? searchIndexHandler : handler;
-
 	const maxMessages = 10; // SQS API limit
 
 	const sqs = createQueueProducer({ queueUrl });
@@ -206,7 +191,7 @@ if (cluster.isPrimary) {
 				})),
 			};
 
-			const result = (await activeHandler(event, nextContext(), () => {})) as
+			const result = (await handler(event, nextContext(), () => {})) as
 				| SQSBatchResponse
 				| undefined;
 

package/src/emit.test.ts

@@ -1,8 +1,15 @@
 import assert from "node:assert/strict";
-import { before, beforeEach, describe, it } from "node:test";
+import { afterEach, before, beforeEach, describe, it } from "node:test";
 import { SendMessageCommand, SQSClient } from "@aws-sdk/client-sqs";
+import {
+	_resetForTest,
+	type RemitClient,
+	setClient,
+} from "@remit/backend/client";
+import { MailboxLockRepo } from "@remit/drizzle-service";
+import { createShippedSqliteDb } from "@remit/drizzle-service/test-sqlite";
 import { mockClient } from "aws-sdk-client-mock";
-import type { emitEvent as EmitEvent } from "./emit.js";
+import type * as Emit from "./emit.js";
 import type {
 	FlagPushEvent,
 	SyncMailboxesEvent,
@@ -20,7 +27,10 @@ const fifo = (name: string) =>
 
 const sqsMock = mockClient(SQSClient);
 
-let emitEvent: typeof EmitEvent;
+let emitEvent: typeof Emit.emitEvent;
+let SYNC_MESSAGES_PENDING: typeof Emit.SYNC_MESSAGES_PENDING;
+let locks: MailboxLockRepo;
+let closeStore: () => void;
 
 const sentCommands = (): SendMessageCommand["input"][] =>
 	sqsMock.commandCalls(SendMessageCommand).map((call) => call.args[0].input);
@@ -32,11 +42,21 @@ before(async () => {
 	process.env.SQS_QUEUE_URL_MAILBOXES = fifo("mailboxes");
 	process.env.SQS_QUEUE_URL_MESSAGES = fifo("messages");
 	process.env.SQS_QUEUE_URL_FLAGS = fifo("flags");
-	({ emitEvent } = await import("./emit.js"));
+	({ emitEvent, SYNC_MESSAGES_PENDING } = await import("./emit.js"));
 });
 
 beforeEach(() => {
 	sqsMock.reset();
+	sqsMock.on(SendMessageCommand).resolves({});
+	const store = createShippedSqliteDb();
+	closeStore = store.close;
+	locks = new MailboxLockRepo(store.db);
+	setClient({ mailboxLock: locks } as unknown as RemitClient);
+});
+
+afterEach(() => {
+	_resetForTest();
+	closeStore();
 });
 
 describe("emitEvent on a FIFO queue", () => {
@@ -68,6 +88,14 @@ describe("emitEvent on a FIFO queue", () => {
 		};
 
 		await emitEvent(event);
+		const [first] = sentCommands();
+		if (!first) throw new Error("expected a send");
+		await locks.releaseLock(
+			"acc-1",
+			"mbx-1",
+			SYNC_MESSAGES_PENDING,
+			String(bodyOf(first).eventId),
+		);
 		await emitEvent(event);
 
 		const sent = sentCommands();

package/src/emit.ts

@@ -1,5 +1,7 @@
 import { randomUUID } from "node:crypto";
 import { SendMessageCommand } from "@aws-sdk/client-sqs";
+import { getClient } from "@remit/backend/client";
+import { recordImapFailure } from "@remit/logger-lambda/metrics";
 import {
 	createQueueProducer,
 	isLocalEndpoint,
@@ -9,6 +11,10 @@ import type { ImapEvent } from "./events.js";
 
 type EventInput = Omit<ImapEvent, "eventId" | "timestamp">;
 
+export const SYNC_MESSAGES_PENDING = "SYNC_MESSAGES_PENDING";
+
+const SYNC_MESSAGES_PENDING_TTL_SECONDS = 300;
+
 const mailboxesQueueUrl = env.SQS_QUEUE_URL_MAILBOXES;
 const messagesQueueUrl = env.SQS_QUEUE_URL_MESSAGES;
 // SYNC_MESSAGE_BODY events route to the single standard body queue (#612). It is
@@ -77,29 +83,58 @@ export const emitEvent = async (
 	// ElasticMQ FIFO queues don't support per-message DelaySeconds
 	const useDelay = options?.delaySeconds && !isLocal;
 
-	await sqs.send(
-		new SendMessageCommand({
-			QueueUrl: queueUrl,
-			MessageBody: JSON.stringify(fullEvent),
-			// Delay delivery for retry backoff (skip for local ElasticMQ)
-			...(useDelay && { DelaySeconds: options.delaySeconds }),
-			// FIFO queue parameters — only set if the queue is FIFO. The
-			// deduplication id is the event's own id, so the queue suppresses a
-			// re-send of one event (the retry SQS's own idempotency guard is for)
-			// and nothing else. A shared id per account or per mailbox instead made
-			// the 5-minute window a rate limiter: the second sync of a mailbox
-			// within five minutes was discarded before any worker saw it, so mail
-			// that arrived after a sync could not be fetched until the window
-			// elapsed (issue #37).
-			//
-			// Dropping events is not how repeated work is bounded — that is the
-			// freshness gate in the sync-mailboxes fan-out, which decides whether
-			// a mailbox is worth enumerating at all. MessageGroupId only orders an
-			// account's events; it makes duplicates serial, not cheap.
-			...(useFifo && {
-				MessageGroupId: event.accountId,
-				MessageDeduplicationId: fullEvent.eventId,
+	const send = () =>
+		sqs.send(
+			new SendMessageCommand({
+				QueueUrl: queueUrl,
+				MessageBody: JSON.stringify(fullEvent),
+				// Delay delivery for retry backoff (skip for local ElasticMQ)
+				...(useDelay && { DelaySeconds: options.delaySeconds }),
+				// FIFO queue parameters — only set if the queue is FIFO. The
+				// deduplication id is the event's own id, so the queue suppresses a
+				// re-send of one event (the retry SQS's own idempotency guard is for)
+				// and nothing else. A shared id per account or per mailbox instead made
+				// the 5-minute window a rate limiter: the second sync of a mailbox
+				// within five minutes was discarded before any worker saw it, so mail
+				// that arrived after a sync could not be fetched until the window
+				// elapsed (issue #37).
+				...(useFifo && {
+					MessageGroupId: event.accountId,
+					MessageDeduplicationId: fullEvent.eventId,
+				}),
 			}),
-		}),
+		);
+	if (fullEvent.type !== "SYNC_MESSAGES") {
+		await send();
+		return;
+	}
+
+	const { mailboxLock } = await getClient();
+	const { accountId, mailboxId, eventId } = fullEvent;
+	const previous = await mailboxLock.get(
+		accountId,
+		mailboxId,
+		SYNC_MESSAGES_PENDING,
 	);
+	const acquired = await mailboxLock.tryAcquireLock(
+		mailboxId,
+		SYNC_MESSAGES_PENDING,
+		accountId,
+		eventId,
+		SYNC_MESSAGES_PENDING_TTL_SECONDS,
+	);
+	if (!acquired) return;
+	if (previous && previous.ttl < Math.floor(Date.now() / 1000)) {
+		recordImapFailure("SYNC_PENDING_EXPIRED", "other");
+	}
+
+	await send().catch(async (error: unknown) => {
+		await mailboxLock.releaseLock(
+			accountId,
+			mailboxId,
+			SYNC_MESSAGES_PENDING,
+			eventId,
+		);
+		throw error;
+	});
 };

package/src/failure-kind.ts

@@ -1,4 +1,4 @@
-import type { MailFailureKind } from "@remit/logger-lambda";
+import type { MailFailureKind } from "@remit/logger-lambda/metrics";
 import { RefreshTokenError } from "@remit/mail-oauth-service";
 import { MailConnectionError } from "@remit/mailbox-service";
 

package/src/handlers/append-sent-message.sqlite.test.ts

@@ -0,0 +1,176 @@
+import assert from "node:assert/strict";
+import { after, before, describe, it } from "node:test";
+import type { OutboxMessageItem } from "@remit/data-ports";
+import { MailboxSpecialUse, OutboxMessageStatus } from "@remit/domain-enums";
+import type { AppendSentMessageEvent } from "../events.js";
+import { openStore, type World } from "../test-helpers/test-world.js";
+import { handleAppendSentMessage } from "./append-sent-message.js";
+
+let store: Awaited<ReturnType<typeof openStore>>;
+
+before(async () => {
+	store = await openStore();
+});
+
+after(() => store.close());
+
+interface Arranged {
+	w: World;
+	outbox: OutboxMessageItem;
+	event: AppendSentMessageEvent;
+}
+
+/**
+ * A message composed and sent through the service the API calls, then marked
+ * sent the way the SMTP worker marks it once the relay accepted it.
+ */
+const arrange = async (
+	options: { sentFolder?: boolean; attachment?: Buffer } = {},
+): Promise<Arranged> => {
+	const w = await store.world();
+	await w.folder("INBOX");
+	if (options.sentFolder !== false) {
+		await w.folder("Sent", { specialUse: MailboxSpecialUse.Sent });
+	}
+	const { outbox: draft } = await w.draft(options.attachment);
+	const outbox = await w.client.outboxQueue.send(
+		w.account.accountConfigId,
+		draft.outboxMessageId,
+	);
+	await w.client.outboxMessage.update(
+		w.account.accountConfigId,
+		outbox.outboxMessageId,
+		{ status: OutboxMessageStatus.sent, sentAt: Date.now() },
+	);
+	return {
+		w,
+		outbox,
+		event: {
+			type: "APPEND_SENT_MESSAGE",
+			eventId: `evt-${outbox.outboxMessageId}`,
+			timestamp: Date.now(),
+			accountId: w.account.accountId,
+			outboxMessageId: outbox.outboxMessageId,
+		} as AppendSentMessageEvent,
+	};
+};
+
+const run = (a: Arranged, receiveCount = 1) =>
+	handleAppendSentMessage(a.event, a.w.log, receiveCount, a.w.scope);
+
+const outboxRow = (a: Arranged) =>
+	a.w.client.outboxMessage
+		.get(a.w.account.accountConfigId, a.outbox.outboxMessageId)
+		.catch(() => undefined);
+
+describe("filing a sent message", () => {
+	it("stores a seen copy under its own Message-ID in Sent and drops the outbox row", async () => {
+		const a = await arrange();
+
+		await run(a);
+
+		const [copy] = a.w.imap.messages("Sent");
+		assert.equal(copy?.messageIdHeader, `<${a.outbox.messageIdValue}>`);
+		assert.ok(copy?.flags.includes("\\Seen"));
+		assert.match(
+			copy?.raw.toString("utf8") ?? "",
+			/Subject: Quarterly numbers/,
+		);
+		assert.equal(await outboxRow(a), undefined);
+	});
+
+	// #679: the Sent copy carries the files the wire copy carried, and the row
+	// was the only reference to them, so they go with it.
+	it("files the copy with its attachment and removes the stored file", async () => {
+		const a = await arrange({ attachment: Buffer.from("q1,q2\n1,2\n") });
+
+		await run(a);
+
+		const raw = a.w.imap.messages("Sent")[0]?.raw.toString("utf8") ?? "";
+		assert.match(raw, /^Content-Type: multipart\/mixed;/m);
+		assert.match(raw, /filename=numbers\.csv/);
+		assert.deepEqual(
+			await a.w.client.storage.listOutboxAttachments(
+				a.w.account.accountConfigId,
+				a.w.account.accountId,
+				a.outbox.outboxMessageId,
+			),
+			[],
+		);
+	});
+
+	it("settles the row as unfiled when the account has no Sent folder", async () => {
+		const a = await arrange({ sentFolder: false });
+
+		await run(a);
+
+		assert.equal(a.w.imap.connections, 0);
+		const row = await outboxRow(a);
+		assert.equal(row?.status, OutboxMessageStatus.unfiled);
+		assert.match(
+			String(row?.lastError),
+			/no folder appointed to the Sent role/,
+		);
+	});
+
+	it("files nothing while the message has not been sent yet", async () => {
+		const a = await arrange();
+		await a.w.client.outboxMessage.update(
+			a.w.account.accountConfigId,
+			a.outbox.outboxMessageId,
+			{ status: OutboxMessageStatus.queued },
+		);
+
+		await run(a);
+
+		assert.deepEqual(a.w.imap.commands("APPEND"), []);
+		assert.equal((await outboxRow(a))?.status, OutboxMessageStatus.queued);
+	});
+
+	it("keeps the row and redelivers a refused APPEND within the budget", async () => {
+		const a = await arrange();
+		a.w.imap.failNext.APPEND = new Error("Connection reset");
+
+		await assert.rejects(run(a), /Connection reset/);
+
+		assert.equal((await outboxRow(a))?.status, OutboxMessageStatus.sent);
+	});
+
+	it("settles the row as unfiled on the last attempt instead of dead-lettering it", async () => {
+		const a = await arrange();
+		a.w.imap.failNext.APPEND = new Error("Mailbox is full");
+
+		await run(a, 3);
+
+		const row = await outboxRow(a);
+		assert.equal(row?.status, OutboxMessageStatus.unfiled);
+		assert.match(String(row?.lastError), /Mailbox is full/);
+	});
+
+	// #858: a redelivery after the APPEND landed must not file a second copy.
+	it("files nothing a second time when the copy was already recorded", async () => {
+		const a = await arrange();
+		await a.w.client.outboxMessage.update(
+			a.w.account.accountConfigId,
+			a.outbox.outboxMessageId,
+			{ appendedUid: 41 },
+		);
+
+		await run(a);
+
+		assert.deepEqual(a.w.imap.commands("APPEND"), []);
+		assert.equal(await outboxRow(a), undefined);
+	});
+
+	it("acks an event whose outbox row is already gone", async () => {
+		const a = await arrange();
+		await a.w.client.outboxMessage.delete(
+			a.w.account.accountConfigId,
+			a.outbox.outboxMessageId,
+		);
+
+		await run(a);
+
+		assert.equal(a.w.imap.connections, 0);
+	});
+});

package/src/handlers/append-sent-message.ts

@@ -6,7 +6,7 @@ import {
 } from "@remit/data-ports";
 import { isNotFoundError } from "@remit/data-ports/errors";
 import { OutboxMessageStatus } from "@remit/domain-enums";
-import type { Logger } from "@remit/logger-lambda";
+import type { Logger } from "@remit/logger";
 import { OutboxAttachmentUnavailableError } from "@remit/mailbox-service/outbox-attachment-content";
 import {
 	buildMailMessage,
@@ -39,20 +39,6 @@ export const getAppendSentMaxAttempts = (
 
 export const APPEND_SENT_MAX_ATTEMPTS = getAppendSentMaxAttempts();
 
-export interface AppendSentMessageDeps {
-	getClient: typeof getClient;
-	buildLifecycleDeps: typeof buildLifecycleDeps;
-	withOAuthLifecycle: typeof withOAuthLifecycle;
-	createConnectionScope: typeof createConnectionScopeWithCredentials;
-}
-
-const defaultDeps: AppendSentMessageDeps = {
-	getClient,
-	buildLifecycleDeps,
-	withOAuthLifecycle,
-	createConnectionScope: createConnectionScopeWithCredentials,
-};
-
 export const handleAppendSentMessage = async (
 	event: AppendSentMessageEvent,
 	log: Logger,
@@ -63,15 +49,8 @@ export const handleAppendSentMessage = async (
 	 * leaves the row at `sent`, which no view shows.
 	 */
 	receiveCount = 1,
-	deps: AppendSentMessageDeps = defaultDeps,
+	createConnectionScope = createConnectionScopeWithCredentials,
 ): Promise<void> => {
-	const {
-		getClient,
-		buildLifecycleDeps,
-		withOAuthLifecycle,
-		createConnectionScope: createConnectionScopeWithCredentials,
-	} = deps;
-
 	const {
 		account: accountService,
 		outboxMessage: outboxMessageService,
@@ -199,7 +178,7 @@ export const handleAppendSentMessage = async (
 				accountId,
 				outboxMessageId,
 			);
-			const scope = createConnectionScopeWithCredentials(account, credentials);
+			const scope = createConnectionScope(account, credentials);
 
 			await scope
 				.getConnection()

package/src/handlers/delete-account-objects.test.ts

@@ -6,7 +6,7 @@ import {
 	S3Client,
 } from "@aws-sdk/client-s3";
 import { SendMessageCommand, SQSClient } from "@aws-sdk/client-sqs";
-import { noopLogger } from "@remit/logger-lambda/noop-logger";
+import { noopLogger } from "@remit/logger";
 import { mockClient } from "aws-sdk-client-mock";
 import type { DeleteAccountObjectsEvent } from "../events.js";
 import { handleDeleteAccountObjects } from "./delete-account-objects.js";

package/src/handlers/delete-account-objects.ts

@@ -4,7 +4,7 @@ import {
 	S3Client,
 } from "@aws-sdk/client-s3";
 import { SendMessageCommand } from "@aws-sdk/client-sqs";
-import type { Logger } from "@remit/logger-lambda";
+import type { Logger } from "@remit/logger";
 import { createQueueProducer } from "@remit/sqs-client/producer";
 import { env } from "expect-env";
 import type { DeleteAccountObjectsEvent } from "../events.js";