@remit/mailbox-service 0.0.111 → 0.0.112

package/src/flag-push.test.ts

@@ -1,408 +1,112 @@
 import assert from "node:assert/strict";
-import { describe, it, mock } from "node:test";
-import type { IMessageFlagPushRepository } from "@remit/data-ports";
+import { afterEach, beforeEach, describe, it } from "node:test";
 import { MessageSystemFlag } from "@remit/domain-enums";
-import { type FlagPushConfig, FlagPushService } from "./flag-push.js";
-
-const accountId = "acc-1";
-const accountConfigId = "acc-cfg-1";
-const messageId = "msg-1";
-const mailboxId = "mbx-1";
-
-const createMockSqs = (impl?: () => Promise<unknown>) => {
-	const sent: unknown[] = [];
-	return {
-		send: mock.fn(async (cmd: { input: unknown }) => {
-			if (impl) await impl();
-			sent.push(cmd.input);
-			return { MessageId: "ok" };
-		}),
-		_sent: sent,
-	};
+import { FlagPushService } from "./flag-push.js";
+import {
+	captureQueue,
+	openStore,
+	type QueueCapture,
+	type Store,
+} from "./test-helpers/test-store.js";
+
+const target = {
+	accountId: "acc-1",
+	accountConfigId: "cfg-1",
+	messageId: "msg-1",
+	mailboxId: "mbx-1",
+	flagName: MessageSystemFlag.Seen,
+	operation: "add" as const,
 };
 
-interface Harness {
-	service: FlagPushService;
-	markerPuts: Array<Record<string, unknown>>;
-	markerStateUpdates: Array<{
-		messageId: string;
-		flagName: string;
-		state: string;
-	}>;
-	mockSqs: ReturnType<typeof createMockSqs>;
-	logs: { info: unknown[]; error: unknown[] };
-}
+let store: Store;
+let capture: QueueCapture;
+let errors: Array<Record<string, unknown>>;
 
-const createHarness = (opts?: {
-	sqsImpl?: () => Promise<unknown>;
-	queueUrl?: string;
-	updateStateImpl?: () => Promise<unknown>;
-	findImpl?: () => Promise<unknown>;
-}): Harness => {
-	const markerPuts: Array<Record<string, unknown>> = [];
-	const markerStateUpdates: Array<{
-		messageId: string;
-		flagName: string;
-		state: string;
-	}> = [];
-	const logs = { info: [] as unknown[], error: [] as unknown[] };
+beforeEach(async () => {
+	store = await openStore();
+	capture = captureQueue();
+	errors = [];
+});
 
-	const markerService: IMessageFlagPushRepository = {
-		put: mock.fn(async (input: Record<string, unknown>) => {
-			markerPuts.push(input);
-			return {
-				...input,
-				state: "pending",
-				createdAt: 1,
-				updatedAt: 1,
-			} as never;
-		}),
-		find: mock.fn(async () =>
-			opts?.findImpl ? ((await opts.findImpl()) as never) : null,
-		),
-		updateState: mock.fn(
-			async (id: string, flagName: string, state: string) => {
-				if (opts?.updateStateImpl) return opts.updateStateImpl() as never;
-				markerStateUpdates.push({ messageId: id, flagName, state });
-				return {} as never;
-			},
-		),
-		delete: mock.fn(async () => {}),
-		listByAccountId: mock.fn(async () => []),
-		listByMailboxId: mock.fn(async () => []),
-	};
+afterEach(async () => {
+	capture.restore();
+	await store.close();
+});
 
-	const mockSqs = createMockSqs(opts?.sqsImpl);
+const queueAnswers = (answer: () => Promise<unknown>) => capture.answer(answer);
 
-	const config: FlagPushConfig = {
-		markerService,
-		sqsQueueUrl: opts?.queueUrl ?? "http://localhost:4566/test-queue",
+const flips = (queueUrl = "http://localhost:9324/000/remit-flags") =>
+	new FlagPushService({
+		markerService: store.flagPush,
+		sqsQueueUrl: queueUrl,
 		logger: {
-			info: (fields: Record<string, unknown>) => {
-				logs.info.push(fields);
-			},
-			error: (fields: Record<string, unknown>) => {
-				logs.error.push(fields);
-			},
+			info: () => {},
+			error: (fields: Record<string, unknown>) => errors.push(fields),
 		},
-	};
-
-	const service = new FlagPushService(config);
-	// @ts-expect-error - inject mock SQS client for testing
-	service.sqs = mockSqs;
-
-	return { service, markerPuts, markerStateUpdates, mockSqs, logs };
-};
-
-describe("FlagPushService.flip", () => {
-	it("persists a pending marker before enqueueing the wake-up hint", async () => {
-		const { service, markerPuts } = createHarness();
-
-		await service.flip({
-			accountId,
-			accountConfigId,
-			messageId,
-			mailboxId,
-			flagName: MessageSystemFlag.Seen,
-			operation: "add",
-		});
-
-		assert.equal(markerPuts.length, 1);
-		assert.deepEqual(markerPuts[0], {
-			messageId,
-			flagName: MessageSystemFlag.Seen,
-			accountId,
-			accountConfigId,
-			mailboxId,
-			operation: "add",
-		});
 	});
 
-	it("advances the marker to queued once the SQS hint is confirmed sent", async () => {
-		const { service, markerStateUpdates } = createHarness();
-
-		await service.flip({
-			accountId,
-			accountConfigId,
-			messageId,
-			mailboxId,
-			flagName: MessageSystemFlag.Flagged,
-			operation: "remove",
-		});
+const marker = () => store.flagPush.find(target.messageId, target.flagName);
 
-		assert.deepEqual(markerStateUpdates, [
-			{ messageId, flagName: MessageSystemFlag.Flagged, state: "queued" },
-		]);
-	});
+describe("FlagPushService.flip", () => {
+	it("records the owed push and marks it queued once the hint is sent", async () => {
+		queueAnswers(async () => ({}));
 
-	it("does NOT reject flip when the wake-up hint enqueue fails (queue down)", async () => {
-		const { service } = createHarness({
-			sqsImpl: () => {
-				throw Object.assign(new Error(""), {
-					name: "AggregateError",
-					code: "ECONNREFUSED",
-				});
-			},
-		});
+		await flips().flip(target);
 
-		await assert.doesNotReject(
-			service.flip({
-				accountId,
-				accountConfigId,
-				messageId,
-				mailboxId,
-				flagName: MessageSystemFlag.Seen,
-				operation: "add",
-			}),
-		);
+		const recorded = await marker();
+		assert.equal(recorded?.operation, "add");
+		assert.equal(recorded?.mailboxId, target.mailboxId);
+		assert.equal(recorded?.state, "queued");
 	});
 
-	it("leaves the marker in `pending` (never advances to queued) when the hint enqueue fails", async () => {
-		const { service, markerStateUpdates } = createHarness({
-			sqsImpl: () => {
-				throw new Error("ECONNREFUSED");
-			},
-		});
+	it("names the message and flag in the hint, never the operation or a uid", async () => {
+		queueAnswers(async () => ({}));
 
-		await service.flip({
-			accountId,
-			accountConfigId,
-			messageId,
-			mailboxId,
-			flagName: MessageSystemFlag.Seen,
-			operation: "add",
-		});
+		await flips().flip(target);
 
-		assert.deepEqual(markerStateUpdates, []);
+		const event = JSON.parse(capture.inputs()[0].MessageBody ?? "{}");
+		assert.equal(event.type, "FLAG_PUSH");
+		assert.equal(event.messageId, target.messageId);
+		assert.equal(event.flagName, target.flagName);
+		assert.equal("operation" in event, false);
+		assert.equal("uid" in event, false);
 	});
 
-	it("logs the enqueue failure loudly with an alertable field (not silent)", async () => {
-		const { service, logs } = createHarness({
-			sqsImpl: () => {
-				throw Object.assign(new Error(""), {
-					name: "AggregateError",
-					code: "ECONNREFUSED",
-				});
-			},
-		});
+	it("groups a FIFO hint by account and leaves a standard queue ungrouped", async () => {
+		queueAnswers(async () => ({}));
 
-		await service.flip({
-			accountId,
-			accountConfigId,
-			messageId,
-			mailboxId,
-			flagName: MessageSystemFlag.Seen,
-			operation: "add",
-		});
+		await flips("http://localhost:9324/000/remit-flags.fifo").flip(target);
+		await flips("http://localhost:9324/000/remit-flags").flip(target);
 
-		const alerted = logs.error.find(
-			(entry) =>
-				(entry as { alert?: string }).alert === "flag_push_hint_enqueue_failed",
-		);
-		assert.ok(
-			alerted,
-			"expected an alertable flag_push_hint_enqueue_failed log",
-		);
-		assert.equal((alerted as { errorCode?: string }).errorCode, "ECONNREFUSED");
+		assert.equal(capture.inputs()[0].MessageGroupId, target.accountId);
+		assert.equal(capture.inputs()[1].MessageGroupId, undefined);
 	});
 
-	it("does NOT leak an unhandled rejection onto a concurrent caller when the enqueue fails", async () => {
-		const leaked: unknown[] = [];
-		const onUnhandled = (reason: unknown): void => {
-			leaked.push(reason);
-		};
-		process.on("unhandledRejection", onUnhandled);
-
-		const { service } = createHarness({
-			sqsImpl: () =>
-				new Promise((_resolve, reject) => {
-					setImmediate(() =>
-						reject(
-							Object.assign(new Error(""), {
-								name: "AggregateError",
-								code: "ECONNREFUSED",
-							}),
-						),
-					);
-				}),
-		});
-
-		try {
-			void service.flip({
-				accountId,
-				accountConfigId,
-				messageId,
-				mailboxId,
-				flagName: MessageSystemFlag.Seen,
-				operation: "add",
+	it("keeps the push owed and alarms, without rejecting, when the queue is down", async () => {
+		queueAnswers(async () => {
+			throw Object.assign(new Error("connect ECONNREFUSED"), {
+				code: "ECONNREFUSED",
 			});
-			await new Promise((resolve) => setImmediate(resolve));
-			await new Promise((resolve) => setImmediate(resolve));
-			await new Promise((resolve) => setImmediate(resolve));
-		} finally {
-			process.off("unhandledRejection", onUnhandled);
-		}
-
-		assert.equal(leaked.length, 0);
-	});
-
-	it("sets MessageGroupId=accountId for FIFO queue URLs", async () => {
-		const { service, mockSqs } = createHarness({
-			queueUrl:
-				"https://sqs.eu-west-1.amazonaws.com/123456789012/remit-dev-mailboxes.fifo",
-		});
-
-		await service.flip({
-			accountId,
-			accountConfigId,
-			messageId,
-			mailboxId,
-			flagName: MessageSystemFlag.Seen,
-			operation: "add",
-		});
-
-		assert.equal(mockSqs._sent.length, 1);
-		const sent = mockSqs._sent[0] as { MessageGroupId?: string };
-		assert.equal(sent.MessageGroupId, accountId);
-	});
-
-	it("omits FIFO params for standard queue URLs", async () => {
-		const { service, mockSqs } = createHarness({
-			queueUrl:
-				"https://sqs.eu-west-1.amazonaws.com/123456789012/remit-dev-mailboxes",
 		});
 
-		await service.flip({
-			accountId,
-			accountConfigId,
-			messageId,
-			mailboxId,
-			flagName: MessageSystemFlag.Seen,
-			operation: "add",
-		});
-
-		assert.equal(mockSqs._sent.length, 1);
-		const sent = mockSqs._sent[0] as { MessageGroupId?: string };
-		assert.equal(sent.MessageGroupId, undefined);
-	});
-
-	it("event body carries only messageId + flagName — never the operation or a UID", async () => {
-		const { service, mockSqs } = createHarness();
+		await flips().flip(target);
 
-		await service.flip({
-			accountId,
-			accountConfigId,
-			messageId,
-			mailboxId,
-			flagName: MessageSystemFlag.Flagged,
-			operation: "remove",
-		});
-
-		const sent = mockSqs._sent[0] as { MessageBody: string };
-		const event = JSON.parse(sent.MessageBody);
-		assert.equal(event.type, "FLAG_PUSH");
-		assert.equal(event.messageId, messageId);
-		assert.equal(event.flagName, MessageSystemFlag.Flagged);
-		assert.equal(event.accountId, accountId);
-		assert.equal(event.accountConfigId, accountConfigId);
-		assert.equal("operation" in event, false);
-		assert.equal("uid" in event, false);
+		assert.equal((await marker())?.state, "pending");
+		const alarm = errors.find(
+			(entry) => entry.alert === "flag_push_hint_enqueue_failed",
+		);
+		assert.equal(alarm?.errorCode, "ECONNREFUSED");
 	});
 
-	describe("fast-path race: the worker drains and deletes the marker before the queued-state transition runs (review finding on #1292)", () => {
-		it("does NOT log the flag_push_hint_enqueue_failed alarm when the marker is already gone", async () => {
-			const { service, logs } = createHarness({
-				updateStateImpl: () => {
-					throw new Error(
-						"Cannot update state on a MessageFlagPush that does not exist: msg-1/\\Seen",
-					);
-				},
-				findImpl: () => Promise.resolve(null),
-			});
-
-			await service.flip({
-				accountId,
-				accountConfigId,
-				messageId,
-				mailboxId,
-				flagName: MessageSystemFlag.Seen,
-				operation: "add",
-			});
-
-			assert.equal(
-				logs.error.length,
-				0,
-				"the enqueue succeeded; a fast-path race must never be logged as flag_push_hint_enqueue_failed",
-			);
-			assert.ok(
-				logs.info.length > 0,
-				"the race is still visible as routine info, not silent",
-			);
-		});
-
-		it("still logs the flag_push_hint_enqueue_failed alarm when updateState fails for a REAL reason (marker still present)", async () => {
-			const { service, logs } = createHarness({
-				updateStateImpl: () => {
-					throw new Error("ProvisionedThroughputExceededException");
-				},
-				findImpl: () =>
-					Promise.resolve({
-						messageId,
-						flagName: MessageSystemFlag.Seen,
-						state: "pending",
-					}),
-			});
-
-			await service.flip({
-				accountId,
-				accountConfigId,
-				messageId,
-				mailboxId,
-				flagName: MessageSystemFlag.Seen,
-				operation: "add",
-			});
-
-			const alerted = logs.error.find(
-				(entry) =>
-					(entry as { alert?: string }).alert ===
-					"flag_push_hint_enqueue_failed",
-			);
-			assert.ok(
-				alerted,
-				"a genuine failure behind the race-check must still alarm",
-			);
+	it("raises no alarm when the worker drained the push before it was marked queued", async () => {
+		queueAnswers(async () => {
+			await store.flagPush.delete(target.messageId, target.flagName);
+			return {};
 		});
 
-		it("still logs the flag_push_hint_enqueue_failed alarm when the re-check itself fails (backend genuinely down)", async () => {
-			const { service, logs } = createHarness({
-				updateStateImpl: () => {
-					throw new Error(
-						"Cannot update state on a MessageFlagPush that does not exist: msg-1/\\Seen",
-					);
-				},
-				findImpl: () => {
-					throw new Error("ECONNREFUSED");
-				},
-			});
-
-			await service.flip({
-				accountId,
-				accountConfigId,
-				messageId,
-				mailboxId,
-				flagName: MessageSystemFlag.Seen,
-				operation: "add",
-			});
+		await flips().flip(target);
 
-			const alerted = logs.error.find(
-				(entry) =>
-					(entry as { alert?: string }).alert ===
-					"flag_push_hint_enqueue_failed",
-			);
-			assert.ok(
-				alerted,
-				"a re-check failure is a real infra problem — must alarm",
-			);
-		});
+		assert.equal(await marker(), null);
+		assert.deepEqual(errors, []);
 	});
 });

package/src/flag-queue.test.ts

@@ -0,0 +1,197 @@
+import assert from "node:assert/strict";
+import { afterEach, beforeEach, describe, it } from "node:test";
+import type { AccountItem, MailboxItem } from "@remit/data-ports";
+import { MessageSystemFlag } from "@remit/domain-enums";
+import { FlagPushService } from "./flag-push.js";
+import { resolveExhaustedFlagPushFailure } from "./flag-push-terminal.js";
+import { FlagQueueService, type UpdateFlagsInput } from "./flag-queue.js";
+import {
+	eml,
+	type FakeImapConnection,
+	FakeImapServer,
+} from "./test-helpers/fake-imap.js";
+import {
+	captureQueue,
+	openStore,
+	type QueueCapture,
+	type Store,
+	seedAccount,
+	seedMailbox,
+	seedMessage,
+	silentLog,
+} from "./test-helpers/test-store.js";
+
+let store: Store;
+let capture: QueueCapture;
+let account: AccountItem;
+let inbox: MailboxItem;
+let messageId: string;
+
+beforeEach(async () => {
+	capture = captureQueue();
+	store = await openStore();
+	account = await seedAccount(store);
+	inbox = await seedMailbox(store, account, "INBOX");
+	messageId = (await seedMessage(store, account, inbox, 1)).message.messageId;
+});
+
+afterEach(async () => {
+	capture.restore();
+	await store.close();
+});
+
+const flags = () =>
+	new FlagQueueService({
+		messageFlagService: store.messageFlag,
+		messageService: store.message,
+		threadMessageService: store.threadMessage,
+		flagPushService: new FlagPushService({
+			markerService: store.flagPush,
+			sqsQueueUrl: "https://sqs.test/flags",
+		}),
+	});
+
+const update = (input: UpdateFlagsInput) =>
+	flags().updateFlags(
+		account.accountConfigId,
+		messageId,
+		account.accountId,
+		input,
+	);
+
+const listing = async () =>
+	(
+		await store.threadMessage.findAllByMessageId(
+			account.accountConfigId,
+			messageId,
+		)
+	)[0];
+
+const owedFlagged = async () =>
+	(await store.flagPush.find(messageId, MessageSystemFlag.Flagged))?.operation;
+
+describe("updateFlags keeps the star and its colour in step", () => {
+	for (const [input, hasStars, star, owed] of [
+		[{ isStarred: true }, true, "yellow", "add"],
+		[{ isStarred: true, starColor: "red" }, true, "red", "add"],
+		[{ starColor: "blue" }, true, "blue", "add"],
+		[{ isStarred: true, starColor: "none" }, true, "none", "add"],
+	] as const) {
+		it(`${JSON.stringify(input)} stars as ${star} and owes the server \\Flagged`, async () => {
+			await update(input);
+
+			const row = await listing();
+			assert.equal(row.hasStars, hasStars);
+			assert.equal(row.star, star);
+			assert.equal(await owedFlagged(), owed);
+		});
+	}
+
+	for (const input of [{ isStarred: false }, { starColor: "none" }] as const) {
+		it(`${JSON.stringify(input)} unstars a starred message`, async () => {
+			await update({ isStarred: true });
+
+			await update(input);
+
+			const row = await listing();
+			assert.equal(row.hasStars, false);
+			assert.equal(row.star, "none");
+			assert.equal(await owedFlagged(), "remove");
+		});
+	}
+
+	it("marks read once and treats a repeat as no change", async () => {
+		await update({ isRead: true });
+		const first = await store.flagPush.find(messageId, MessageSystemFlag.Seen);
+
+		const result = await update({ isRead: true });
+
+		assert.equal(result.isRead, true);
+		assert.equal((await listing()).isRead, true);
+		assert.deepEqual(
+			await store.flagPush.find(messageId, MessageSystemFlag.Seen),
+			first,
+		);
+	});
+});
+
+describe("a flag push that exhausted its retries", () => {
+	const resolve = async (
+		connect: () => FakeImapConnection,
+		afterRead: () => Promise<unknown> = async () => {},
+	) => {
+		await update({ isRead: true });
+		const read = await store.message.get(messageId);
+		await afterRead();
+		return resolveExhaustedFlagPushFailure(
+			{
+				messageService: store.message,
+				markerService: store.flagPush,
+				log: silentLog,
+			},
+			{
+				placement: {
+					status: read.status,
+					mailboxId: read.mailboxId,
+					uid: read.uid,
+				},
+				accountId: account.accountId,
+				accountConfigId: account.accountConfigId,
+				messageId,
+				flagName: MessageSystemFlag.Seen,
+				uid: 1,
+				mailboxPath: "INBOX",
+				getConnection: async () => connect(),
+			},
+		);
+	};
+
+	it("drops the push and the stale rows of a message the server no longer holds", async () => {
+		const imap = new FakeImapServer();
+		imap.addFolder("INBOX");
+
+		const { outcome } = await resolve(() => imap.connection());
+
+		assert.equal(outcome, "reconciled");
+		assert.equal(
+			await store.flagPush.find(messageId, MessageSystemFlag.Seen),
+			null,
+		);
+		await assert.rejects(store.message.get(messageId));
+	});
+
+	it("leaves a row the user moved after the push read it", async () => {
+		const imap = new FakeImapServer();
+		imap.addFolder("INBOX");
+		const archive = await seedMailbox(store, account, "Archive");
+
+		const { outcome } = await resolve(
+			() => imap.connection(),
+			() =>
+				store.message.transitionPlacement(
+					messageId,
+					{ mailboxId: inbox.mailboxId },
+					{ mailboxId: archive.mailboxId, uid: 7 },
+				),
+		);
+
+		assert.equal(outcome, "reconciled");
+		const moved = await store.message.get(messageId);
+		assert.equal(moved.mailboxId, archive.mailboxId);
+		assert.equal(moved.uid, 7);
+	});
+
+	it("keeps the push and the rows of a message still there, dropped FETCH row or not", async () => {
+		const imap = new FakeImapServer();
+		imap.addFolder("INBOX");
+		imap.deliver("INBOX", eml({}));
+		const connection = imap.connection();
+		connection.fetchMessages = async () => [];
+
+		const { outcome } = await resolve(() => connection);
+
+		assert.equal(outcome, "broken");
+		assert.ok(await store.flagPush.find(messageId, MessageSystemFlag.Seen));
+		assert.ok(await store.message.get(messageId));
+	});
+});