@remit/imap-worker 0.0.89 → 0.0.91

package/src/mail-sync-selection.sqlite.test.ts

@@ -1,234 +0,0 @@
-import assert from "node:assert/strict";
-import { randomUUID } from "node:crypto";
-import { afterEach, beforeEach, describe, it } from "node:test";
-import {
-	_resetForTest,
-	type RemitClient,
-	setClient,
-} from "@remit/backend/client";
-import type { AccountItem } from "@remit/data-ports";
-import { recordingQueue } from "@remit/data-ports/recording-queue";
-import {
-	AccountAuthType,
-	AccountService,
-	ConnectionState,
-	MailboxCursorState,
-} from "@remit/domain-enums";
-import {
-	type AccountRepo,
-	createSqliteRepositories,
-	type DrizzleMessageRepository,
-	MailboxLockRepo,
-	type MailboxRepo,
-} from "@remit/drizzle-service";
-import { createShippedSqliteDb } from "@remit/drizzle-service/test-sqlite";
-import type { Logger } from "@remit/logger";
-import { syncMailboxes } from "./handlers/sync-mailboxes.js";
-import { syncMessages } from "./handlers/sync-messages.js";
-import { setWorkerQueues } from "./queues.js";
-import { runSchedulerTick } from "./scheduler/run-tick.js";
-
-const NOW = 1_700_000_000_000;
-
-const silentLogger: Logger = (() => {
-	const noop = () => {};
-	const stub = {
-		info: noop,
-		warn: noop,
-		error: noop,
-		debug: noop,
-		fatal: noop,
-		trace: noop,
-		child: () => stub,
-	};
-	return stub as unknown as Logger;
-})();
-
-let accounts: AccountRepo;
-let mailboxes: MailboxRepo;
-let messages: DrizzleMessageRepository;
-let locks: MailboxLockRepo;
-let client: RemitClient;
-let close: () => void;
-
-beforeEach(() => {
-	const store = createShippedSqliteDb();
-	close = store.close;
-	const repos = createSqliteRepositories({
-		app: store.db,
-		mail: store.db,
-		search: store.db,
-	});
-	accounts = repos.account;
-	mailboxes = repos.mailbox;
-	messages = repos.message;
-	locks = new MailboxLockRepo(store.db);
-	client = {
-		account: accounts,
-		mailbox: mailboxes,
-		message: messages,
-		mailboxLock: locks,
-	} as unknown as RemitClient;
-	setClient(client);
-	const queue = recordingQueue();
-	setWorkerQueues({
-		mailboxes: queue,
-		messages: queue,
-		flags: queue,
-		body: queue,
-		mailboxManagement: queue,
-		messageManagement: queue,
-	});
-});
-
-afterEach(() => {
-	_resetForTest();
-	close();
-});
-
-const seedSyncedAccount = async (): Promise<{
-	account: AccountItem;
-	mailboxId: string;
-	messageId: string;
-}> => {
-	const account = await accounts.create({
-		accountConfigId: "cfg-1181",
-		username: "person@example.com",
-		email: "person@example.com",
-		authType: AccountAuthType.Password,
-		passwordHash: '{"ciphertext":"x"}',
-		imapHost: "imap.example.com",
-		imapPort: 993,
-		imapTls: true,
-		imapStartTls: false,
-		isActive: true,
-		connectionState: ConnectionState.Authenticated,
-	});
-	const created = await mailboxes.create({
-		accountId: account.accountId,
-		namespacePrefix: "",
-		hierarchyDelimiter: "/",
-		fullPath: "INBOX",
-		uidValidity: 7,
-		uidNext: 4,
-		highestModseq: "42",
-		messageCount: 1,
-		unseenCount: 0,
-		deletedCount: 0,
-		totalSize: 10,
-		lastSyncUid: 3,
-		highWaterMarkUid: 3,
-		lastMessageSyncAt: NOW,
-		cursorState: MailboxCursorState.normal,
-	} as Parameters<MailboxRepo["create"]>[0]);
-	if (created.outcome !== "Created") throw new Error("INBOX path taken");
-	const mailbox = created.mailbox;
-	const messageId = randomUUID();
-	await messages.create({
-		messageId,
-		mailboxId: mailbox.mailboxId,
-		uid: 3,
-		sequenceNumber: 1,
-		rfc822Size: 10,
-		internalDate: NOW,
-		envelopeId: randomUUID(),
-		rootBodyPartId: randomUUID(),
-		status: "active",
-		syncStatus: "synced",
-	});
-	return { account, mailboxId: mailbox.mailboxId, messageId };
-};
-
-const tick = async (): Promise<{ enqueued: number; accountIds: string[] }> => {
-	const queue = recordingQueue();
-	const result = await runSchedulerTick({
-		redriveStrandedPlacements: async () => 0,
-		accountService: accounts,
-		queue,
-		log: silentLogger,
-		tickIntervalMs: 60_000,
-		offlineIntervalMs: 0,
-		now: NOW,
-	});
-	return {
-		enqueued: result.enqueued,
-		accountIds: queue.sent.map((message) => String(message.body.accountId)),
-	};
-};
-
-const setServices = (
-	accountId: string,
-	syncedServices: AccountItem["syncedServices"],
-) => accounts.update(accountId, { syncedServices });
-
-describe("turning mail sync off and on again", () => {
-	it("stops every mail sync entry point and keeps the stored mail", async () => {
-		const { account, mailboxId, messageId } = await seedSyncedAccount();
-		const storedMailboxes = await mailboxes.listAllByAccount(account.accountId);
-		const storedMessage = await messages.get(messageId);
-
-		await setServices(account.accountId, [AccountService.Calendar]);
-
-		assert.deepEqual(await tick(), { enqueued: 0, accountIds: [] });
-
-		await syncMailboxes(
-			{
-				type: "SYNC_MAILBOXES",
-				eventId: "evt-mailboxes",
-				timestamp: NOW,
-				accountId: account.accountId,
-			},
-			silentLogger,
-		);
-
-		let connections = 0;
-		await syncMessages(
-			{
-				type: "SYNC_MESSAGES",
-				eventId: "evt-messages",
-				timestamp: NOW,
-				accountId: account.accountId,
-				mailboxId,
-			},
-			silentLogger,
-			() => {
-				connections += 1;
-				throw new Error("a sync that is off must not connect");
-			},
-		);
-		assert.equal(connections, 0);
-
-		assert.deepEqual(
-			await mailboxes.listAllByAccount(account.accountId),
-			storedMailboxes,
-		);
-		assert.deepEqual(await messages.get(messageId), storedMessage);
-		assert.equal(
-			(await accounts.get(account.accountId)).connectionState,
-			ConnectionState.Authenticated,
-		);
-	});
-
-	it("resumes from the stored markers when mail is turned back on", async () => {
-		const { account, messageId } = await seedSyncedAccount();
-		const storedMailboxes = await mailboxes.listAllByAccount(account.accountId);
-		const storedMessage = await messages.get(messageId);
-
-		await setServices(account.accountId, [AccountService.Calendar]);
-		await tick();
-		await setServices(account.accountId, [
-			AccountService.Mail,
-			AccountService.Calendar,
-		]);
-
-		assert.deepEqual(await tick(), {
-			enqueued: 1,
-			accountIds: [account.accountId],
-		});
-		assert.deepEqual(
-			await mailboxes.listAllByAccount(account.accountId),
-			storedMailboxes,
-		);
-		assert.deepEqual(await messages.get(messageId), storedMessage);
-	});
-});

package/src/mailbox-row.ts

@@ -1,20 +0,0 @@
-import type { IMailboxRepository, MailboxItem } from "@remit/data-ports";
-import { isNotFoundError } from "@remit/data-ports/errors";
-
-export type MailboxRow =
-	| { kind: "present"; mailbox: MailboxItem }
-	| { kind: "gone"; mailboxId: string };
-
-export const findMailboxRow = (
-	mailboxService: Pick<IMailboxRepository, "get">,
-	accountId: string,
-	mailboxId: string,
-): Promise<MailboxRow> =>
-	mailboxService.get(accountId, mailboxId).then(
-		(mailbox): MailboxRow =>
-			mailbox ? { kind: "present", mailbox } : { kind: "gone", mailboxId },
-		(error: unknown): MailboxRow => {
-			if (isNotFoundError(error)) return { kind: "gone", mailboxId };
-			throw error;
-		},
-	);

package/src/pending-sync.sqlite.test.ts

@@ -1,166 +0,0 @@
-import assert from "node:assert/strict";
-import { afterEach, beforeEach, describe, it } from "node:test";
-import {
-	_resetForTest,
-	type RemitClient,
-	setClient,
-} from "@remit/backend/client";
-import {
-	type RecordingQueue,
-	recordingQueue,
-} from "@remit/data-ports/recording-queue";
-import { MailboxLockRepo } from "@remit/drizzle-service";
-import { createShippedSqliteDb } from "@remit/drizzle-service/test-sqlite";
-import { emitEvent, SYNC_MESSAGES_PENDING } from "./emit.js";
-import type { SyncMessagesEvent } from "./events.js";
-import { emitMailboxResync, emitMoveResync } from "./handlers/message-move.js";
-import { emitSyncMessagesEvents } from "./handlers/sync-mailboxes.js";
-import { setWorkerQueues } from "./queues.js";
-
-const ACCOUNT = "account-1366";
-const FOLDERS = [
-	{ mailboxId: "mbx-inbox", fullPath: "INBOX" },
-	{ mailboxId: "mbx-sent", fullPath: "Sent" },
-	{ mailboxId: "mbx-archive", fullPath: "Archive" },
-];
-
-let queue: RecordingQueue;
-let locks: MailboxLockRepo;
-let closeStore: () => void;
-
-type Queued = { mailboxId: string; eventId: string };
-
-const queued = (): Queued[] =>
-	queue.sent.map(({ body }) => ({
-		mailboxId: String(body.mailboxId),
-		eventId: String(body.eventId),
-	}));
-
-const inboxSync = (): Omit<SyncMessagesEvent, "eventId" | "timestamp"> => ({
-	type: "SYNC_MESSAGES",
-	accountId: ACCOUNT,
-	mailboxId: "mbx-inbox",
-});
-
-const fanOut = (): Promise<void> =>
-	emitSyncMessagesEvents(ACCOUNT, FOLDERS, emitEvent);
-
-beforeEach(() => {
-	queue = recordingQueue();
-	setWorkerQueues({
-		mailboxes: queue,
-		messages: queue,
-		flags: queue,
-		body: queue,
-		mailboxManagement: queue,
-		messageManagement: queue,
-	});
-	const store = createShippedSqliteDb();
-	closeStore = store.close;
-	locks = new MailboxLockRepo(store.db);
-	setClient({ mailboxLock: locks } as unknown as RemitClient);
-});
-
-afterEach(() => {
-	_resetForTest();
-	closeStore();
-});
-
-describe("SYNC_MESSAGES under explicit refreshes (#1366)", () => {
-	it("queues one event per folder however many rounds arrive while none has started", async () => {
-		for (let round = 0; round < 25; round += 1) await fanOut();
-
-		assert.deepEqual(
-			queued()
-				.map((event) => event.mailboxId)
-				.sort(),
-			["mbx-archive", "mbx-inbox", "mbx-sent"],
-		);
-	});
-
-	it("queues a folder again once its pending sync has started", async () => {
-		await fanOut();
-		const inbox = queued().find((event) => event.mailboxId === "mbx-inbox");
-		if (!inbox) throw new Error("INBOX was not queued");
-
-		await locks.releaseLock(
-			ACCOUNT,
-			inbox.mailboxId,
-			SYNC_MESSAGES_PENDING,
-			inbox.eventId,
-		);
-		await fanOut();
-
-		assert.equal(queued().length, FOLDERS.length + 1);
-		assert.equal(queued().at(-1)?.mailboxId, "mbx-inbox");
-	});
-
-	it("keeps a folder's pending marker when a stale event starts", async () => {
-		await fanOut();
-		await locks.releaseLock(
-			ACCOUNT,
-			"mbx-inbox",
-			SYNC_MESSAGES_PENDING,
-			"an-event-from-before-the-marker",
-		);
-		await fanOut();
-
-		assert.equal(queued().length, FOLDERS.length);
-	});
-
-	it("takes the same marker for a move's resync and a delete's resync", async () => {
-		await emitMoveResync(emitEvent, {
-			accountId: ACCOUNT,
-			sourceMailboxId: "mbx-inbox",
-			destinationMailboxId: "mbx-sent",
-		});
-		await emitMailboxResync(emitEvent, {
-			accountId: ACCOUNT,
-			mailboxId: "mbx-inbox",
-		});
-		await fanOut();
-
-		assert.equal(queued().length, FOLDERS.length);
-	});
-
-	it("takes over a marker whose event was lost once it has expired", async () => {
-		await locks.tryAcquireLock(
-			"mbx-inbox",
-			"SYNC_MESSAGES_PENDING",
-			ACCOUNT,
-			"lost-event",
-			-1,
-		);
-
-		await emitEvent(inboxSync());
-
-		assert.equal(queued().length, 1);
-	});
-
-	it("releases the marker when the send fails, so the next round is not swallowed", async () => {
-		let sends = 0;
-		queue.answer(async () => {
-			sends += 1;
-			if (sends === 1) throw new Error("queue unreachable");
-		});
-
-		await assert.rejects(emitEvent(inboxSync()), /queue unreachable/);
-		await emitEvent(inboxSync());
-
-		assert.equal(queued().length, 2);
-	});
-
-	it("fails the operation when the marker store fails", async () => {
-		setClient({
-			mailboxLock: {
-				get: async () => null,
-				tryAcquireLock: async () => {
-					throw new Error("marker store down");
-				},
-			},
-		} as unknown as RemitClient);
-
-		await assert.rejects(emitEvent(inboxSync()), /marker store down/);
-		assert.equal(queue.sent.length, 0);
-	});
-});

package/src/poller.ts

@@ -1,55 +0,0 @@
-import { createLogger } from "@remit/logger";
-import { startMetricsServer } from "@remit/logger-lambda/metrics";
-import { installNodeLogger } from "@remit/logger-lambda/node";
-import { runQueuePoller } from "@remit/sqs-client/poller";
-import { env } from "expect-env";
-import { handler } from "./index.js";
-
-installNodeLogger();
-
-/**
- * Production queue poller — the deployed form of `e2e-processor-shim.ts`.
- * Polls every imap-worker queue and invokes the production Lambda handler.
- */
-const log = createLogger();
-
-// /metrics and nothing else, on the compose network (standalone-observability
-// D2). No health route on it: worker liveness is a heartbeat file, which keeps
-// answering when this server does not.
-startMetricsServer();
-
-await runQueuePoller({
-	log,
-	targets: [
-		{
-			queueUrl: env.SQS_QUEUE_URL_MAILBOXES,
-			handler,
-			functionName: "imap-worker-mailboxes",
-		},
-		{
-			queueUrl: env.SQS_QUEUE_URL_MESSAGES,
-			handler,
-			functionName: "imap-worker-messages",
-		},
-		{
-			queueUrl: env.SQS_QUEUE_URL_FLAGS,
-			handler,
-			functionName: "imap-worker-flags",
-		},
-		{
-			queueUrl: env.SQS_QUEUE_URL_BODY,
-			handler,
-			functionName: "imap-worker-body",
-		},
-		{
-			queueUrl: env.SQS_QUEUE_URL_MAILBOX_MGMT,
-			handler,
-			functionName: "imap-worker-mailbox-mgmt",
-		},
-		{
-			queueUrl: env.SQS_QUEUE_URL_MESSAGE_MGMT,
-			handler,
-			functionName: "imap-worker-message-mgmt",
-		},
-	],
-});

package/src/processor.ts

@@ -1,60 +0,0 @@
-import type { Logger } from "@remit/logger";
-import type { WorkerEvent } from "./events.js";
-import { handleAppendSentMessage } from "./handlers/append-sent-message.js";
-import { handleDeleteAccountObjects } from "./handlers/delete-account-objects.js";
-import { handleEmptyTrash } from "./handlers/empty-trash.js";
-import { handleFlagPush } from "./handlers/flag-push.js";
-import { processMailboxManagement } from "./handlers/mailbox-management.js";
-import { handleMessageCopy } from "./handlers/message-copy.js";
-import { handleMessageDelete } from "./handlers/message-delete.js";
-import { handleMessageMove } from "./handlers/message-move.js";
-import { handlePlacementMovePush } from "./handlers/placement-move-push.js";
-import { syncMailboxes } from "./handlers/sync-mailboxes.js";
-import { syncMessageBody } from "./handlers/sync-message-body.js";
-import { syncMessages } from "./handlers/sync-messages.js";
-
-export const processEvent = async (
-	event: WorkerEvent,
-	log: Logger,
-	/**
-	 * SQS's own delivery count for the record carrying this event (1 on first
-	 * delivery). Read by SYNC_MESSAGE_BODY, PLACEMENT_MOVE_PUSH, FLAG_PUSH,
-	 * APPEND_SENT_MESSAGE, MESSAGE_MOVE, MESSAGE_DELETE and MESSAGE_COPY — each
-	 * knows from it when this is
-	 * the last attempt before the queue's own redrive policy would DLQ the
-	 * record, so it can resolve retry exhaustion into a terminal outcome
-	 * (issue #1270) instead of dead-lettering blindly.
-	 */
-	receiveCount = 1,
-): Promise<void> => {
-	switch (event.type) {
-		case "SYNC_MAILBOXES":
-			return syncMailboxes(event, log);
-		case "SYNC_MESSAGES":
-			return syncMessages(event, log);
-		case "SYNC_MESSAGE_BODY":
-			return syncMessageBody(event, log, receiveCount);
-		case "MAILBOX_CREATE":
-		case "MAILBOX_RENAME":
-		case "MAILBOX_DELETE":
-			return processMailboxManagement(event, log);
-		case "MESSAGE_DELETE":
-			return handleMessageDelete(event, log, receiveCount);
-		case "MESSAGE_MOVE":
-			return handleMessageMove(event, log, receiveCount);
-		case "PLACEMENT_MOVE_PUSH":
-			return handlePlacementMovePush(event, log, receiveCount);
-		case "FLAG_PUSH":
-			return handleFlagPush(event, log, receiveCount);
-		case "MESSAGE_COPY":
-			return handleMessageCopy(event, log, receiveCount);
-		case "EMPTY_TRASH":
-			return handleEmptyTrash(event, log);
-		case "APPEND_SENT_MESSAGE":
-			return handleAppendSentMessage(event, log, receiveCount);
-		case "DELETE_ACCOUNT_OBJECTS":
-			return handleDeleteAccountObjects(event, log);
-		default:
-			throw new Error(`Unknown event type: ${(event as WorkerEvent).type}`);
-	}
-};

package/src/queues.ts

@@ -1,49 +0,0 @@
-import type { FifoQueueProducer, QueueProducer } from "@remit/data-ports";
-import { createQueueProducer } from "@remit/sqs-client/producer";
-import { env } from "expect-env";
-import type { ImapEvent, WorkerEvent } from "./events.js";
-
-export interface WorkerQueues {
-	mailboxes: FifoQueueProducer<ImapEvent>;
-	messages: FifoQueueProducer<ImapEvent>;
-	flags: FifoQueueProducer<ImapEvent>;
-	body: QueueProducer<ImapEvent>;
-	mailboxManagement: QueueProducer<ImapEvent>;
-	messageManagement: QueueProducer<WorkerEvent>;
-}
-
-export type WorkerQueueUrls = Record<keyof WorkerQueues, string>;
-
-export const workerQueueUrlsFromEnv = (): WorkerQueueUrls => ({
-	mailboxes: env.SQS_QUEUE_URL_MAILBOXES,
-	messages: env.SQS_QUEUE_URL_MESSAGES,
-	flags: env.SQS_QUEUE_URL_FLAGS,
-	body: env.SQS_QUEUE_URL_BODY,
-	mailboxManagement: env.SQS_QUEUE_URL_MAILBOX_MGMT,
-	messageManagement: env.SQS_QUEUE_URL_MESSAGE_MGMT,
-});
-
-export const sqsWorkerQueues = (urls: WorkerQueueUrls): WorkerQueues => ({
-	mailboxes: createQueueProducer({ queueUrl: urls.mailboxes }),
-	messages: createQueueProducer({ queueUrl: urls.messages }),
-	flags: createQueueProducer({ queueUrl: urls.flags }),
-	body: createQueueProducer({ queueUrl: urls.body }),
-	mailboxManagement: createQueueProducer({ queueUrl: urls.mailboxManagement }),
-	messageManagement: createQueueProducer({ queueUrl: urls.messageManagement }),
-});
-
-let composed: WorkerQueues | null = null;
-
-/** Register the worker's queues from its entry, before the first event. */
-export const setWorkerQueues = (queues: WorkerQueues): void => {
-	composed = queues;
-};
-
-export const workerQueues = (): WorkerQueues => {
-	if (!composed) {
-		throw new Error(
-			"no worker queues registered — register them with setWorkerQueues() from the entry",
-		);
-	}
-	return composed;
-};

package/src/scheduler/attachment-sweep.sqlite.test.ts

@@ -1,75 +0,0 @@
-import assert from "node:assert/strict";
-import { randomUUID } from "node:crypto";
-import { after, before, describe, it } from "node:test";
-import { openStore, type World } from "../test-helpers/test-world.js";
-import { buildAttachmentSweep } from "./attachment-sweep.js";
-
-let store: Awaited<ReturnType<typeof openStore>>;
-
-before(async () => {
-	store = await openStore();
-});
-
-after(() => store.close());
-
-/**
- * A draft holding one uploaded file, and one object under the same draft that
- * no attachment row names — what a browser leaves behind when it uploads and
- * the reservation lapses before it is completed.
- */
-const arrange = async (w: World) => {
-	const { outbox, attachmentId } = await w.draft(Buffer.from("q1,q2\n1,2\n"));
-	await w.client.storage.storeOutboxAttachment({
-		accountConfigId: w.account.accountConfigId,
-		accountId: w.account.accountId,
-		outboxMessageId: outbox.outboxMessageId,
-		outboxAttachmentId: `orphan-${randomUUID()}`,
-		content: Buffer.from("left behind"),
-	});
-	return { outboxMessageId: outbox.outboxMessageId, attachmentId };
-};
-
-const storedIds = async (w: World, outboxMessageId: string) =>
-	(
-		await w.client.storage.listOutboxAttachments(
-			w.account.accountConfigId,
-			w.account.accountId,
-			outboxMessageId,
-		)
-	).map((item) => item.outboxAttachmentId);
-
-describe("the outbox attachment sweep", () => {
-	it("removes the objects no attachment row names and keeps the ones that are", async () => {
-		const w = await store.world();
-		const { outboxMessageId, attachmentId } = await arrange(w);
-
-		await buildAttachmentSweep(w.client, w.log)(w.account);
-
-		assert.deepEqual(await storedIds(w, outboxMessageId), [attachmentId]);
-	});
-
-	it("runs for every account on the hosted tick, due a sync or not", async () => {
-		const first = await store.world();
-		const firstDraft = await arrange(first);
-		await first.client.account.update(first.account.accountId, {
-			lastSyncAt: Date.now(),
-		});
-		const second = await store.world();
-		const secondDraft = await arrange(second);
-
-		const { handler } = await import("./handler.js");
-		store.bindWorkerQueues();
-		await handler(
-			{ time: new Date().toISOString() } as never,
-			{} as never,
-			() => {},
-		);
-
-		assert.deepEqual(await storedIds(first, firstDraft.outboxMessageId), [
-			firstDraft.attachmentId,
-		]);
-		assert.deepEqual(await storedIds(second, secondDraft.outboxMessageId), [
-			secondDraft.attachmentId,
-		]);
-	});
-});