@remit/imap-worker 0.0.7 → 0.0.9

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@remit/imap-worker",
-  "version": "0.0.7",
+  "version": "0.0.9",
   "type": "module",
   "main": "dist/index.js",
   "exports": {

package/src/emit.test.ts

@@ -1,44 +1,134 @@
-import assert from "node:assert";
-import { describe, it } from "node:test";
-import { getDeduplicationId } from "./emit.js";
-import type { SyncMessagesEvent } from "./events.js";
-
-type SyncMessagesInput = Omit<SyncMessagesEvent, "eventId" | "timestamp">;
-
-const syncMessages = (over: Partial<SyncMessagesInput>): SyncMessagesInput => ({
-	type: "SYNC_MESSAGES",
-	accountId: "acc-1",
-	mailboxId: "mbx-1",
-	...over,
+import assert from "node:assert/strict";
+import { before, beforeEach, describe, it } from "node:test";
+import { SendMessageCommand, SQSClient } from "@aws-sdk/client-sqs";
+import { mockClient } from "aws-sdk-client-mock";
+import type { emitEvent as EmitEvent } from "./emit.js";
+import type {
+	FlagPushEvent,
+	SyncMailboxesEvent,
+	SyncMessageBodyEvent,
+	SyncMessagesEvent,
+} from "./events.js";
+
+type Emitted<T> = Omit<T, "eventId" | "timestamp">;
+
+// The queue urls are read once, at import. Point them at FIFO queues first —
+// this suite is about what emitEvent puts on a FIFO queue, which is what both
+// the self-host stack and the deployed stack run (deploy/vps/queues.json).
+const fifo = (name: string) =>
+	`https://sqs.eu-west-1.amazonaws.com/0/test-${name}.fifo`;
+
+const sqsMock = mockClient(SQSClient);
+
+let emitEvent: typeof EmitEvent;
+
+const sentCommands = (): SendMessageCommand["input"][] =>
+	sqsMock.commandCalls(SendMessageCommand).map((call) => call.args[0].input);
+
+const bodyOf = (input: SendMessageCommand["input"]): Record<string, unknown> =>
+	JSON.parse(input.MessageBody ?? "{}") as Record<string, unknown>;
+
+before(async () => {
+	process.env.SQS_QUEUE_URL_MAILBOXES = fifo("mailboxes");
+	process.env.SQS_QUEUE_URL_MESSAGES = fifo("messages");
+	process.env.SQS_QUEUE_URL_FLAGS = fifo("flags");
+	({ emitEvent } = await import("./emit.js"));
 });
 
-describe("getDeduplicationId for SYNC_MESSAGES continuation", () => {
-	it("gives an initial (cursor-less) sync a stable id so concurrent fresh syncs dedup", () => {
-		assert.strictEqual(
-			getDeduplicationId(syncMessages({})),
-			"SYNC_MESSAGES:mbx-1",
-		);
+beforeEach(() => {
+	sqsMock.reset();
+});
+
+describe("emitEvent on a FIFO queue", () => {
+	it("groups by account and deduplicates on the event's own id", async () => {
+		const event: Emitted<SyncMessagesEvent> = {
+			type: "SYNC_MESSAGES",
+			accountId: "acc-1",
+			mailboxId: "mbx-1",
+		};
+
+		await emitEvent(event);
+
+		const [sent] = sentCommands();
+		if (!sent) throw new Error("expected a send");
+		assert.equal(sent.MessageGroupId, "acc-1");
+		assert.equal(sent.MessageDeduplicationId, bodyOf(sent).eventId);
 	});
 
-	it("gives each continuation batch a distinct id so FIFO does not drop batches 2..N", () => {
-		// Regression guard: a constant dedup id per mailbox let SQS FIFO reject
-		// every continuation within the 5-minute window, capping a sync at one
-		// batch (~200 messages). Folding the batch's remaining count into the id
-		// keeps sequential batches distinct.
-		const batch2 = getDeduplicationId(syncMessages({ resumeCursor: 800 }));
-		const batch3 = getDeduplicationId(syncMessages({ resumeCursor: 600 }));
-
-		assert.strictEqual(batch2, "SYNC_MESSAGES:mbx-1:800");
-		assert.strictEqual(batch3, "SYNC_MESSAGES:mbx-1:600");
-		assert.notStrictEqual(batch2, batch3);
+	// Issue #37: every SYNC_MESSAGES for a mailbox carried one dedup id, so
+	// SQS FIFO's 5-minute window discarded the second sync of that mailbox
+	// before any worker saw it — mail appended after a sync could not be
+	// fetched until the window elapsed. Two syncs of one mailbox must both
+	// reach the queue.
+	it("lets a second sync of the same mailbox through", async () => {
+		const event: Emitted<SyncMessagesEvent> = {
+			type: "SYNC_MESSAGES",
+			accountId: "acc-1",
+			mailboxId: "mbx-1",
+		};
+
+		await emitEvent(event);
+		await emitEvent(event);
+
+		const sent = sentCommands();
+		assert.equal(sent.length, 2);
+		assert.notEqual(
+			sent[0]?.MessageDeduplicationId,
+			sent[1]?.MessageDeduplicationId,
+		);
 	});
 
-	it("separates a continuation from the initial event of the same mailbox", () => {
-		const initial = getDeduplicationId(syncMessages({}));
-		const continuation = getDeduplicationId(
-			syncMessages({ resumeCursor: 800 }),
+	it("lets a second sync of the same account's mailbox list through", async () => {
+		const event: Emitted<SyncMailboxesEvent> = {
+			type: "SYNC_MAILBOXES",
+			accountId: "acc-1",
+		};
+
+		await emitEvent(event);
+		await emitEvent(event);
+
+		const sent = sentCommands();
+		assert.equal(sent.length, 2);
+		assert.notEqual(
+			sent[0]?.MessageDeduplicationId,
+			sent[1]?.MessageDeduplicationId,
 		);
+	});
+
+	// A FIFO queue without content-based deduplication rejects a send carrying
+	// no MessageDeduplicationId, so every event routed onto one needs its own —
+	// not only the sync events.
+	it("gives a flag-push re-arm a deduplication id", async () => {
+		const event: Emitted<FlagPushEvent> = {
+			type: "FLAG_PUSH",
+			accountId: "acc-1",
+			accountConfigId: "cfg-1",
+			messageId: "msg-1",
+			flagName: "\\Seen",
+		};
+
+		await emitEvent(event);
+
+		const [sent] = sentCommands();
+		if (!sent) throw new Error("expected a send");
+		assert.equal(sent.MessageDeduplicationId, bodyOf(sent).eventId);
+	});
+});
+
+describe("emitEvent on a standard queue", () => {
+	it("carries no FIFO parameters", async () => {
+		const event: Emitted<SyncMessageBodyEvent> = {
+			type: "SYNC_MESSAGE_BODY",
+			accountId: "acc-1",
+			mailboxId: "mbx-1",
+			messageIds: ["msg-1"],
+		};
+
+		await emitEvent(event);
 
-		assert.notStrictEqual(initial, continuation);
+		const [sent] = sentCommands();
+		if (!sent) throw new Error("expected a send");
+		assert.equal(sent.MessageGroupId, undefined);
+		assert.equal(sent.MessageDeduplicationId, undefined);
 	});
 });

package/src/emit.ts

@@ -5,11 +5,7 @@ import {
 	isLocalEndpoint,
 } from "@remit/sqs-client/producer";
 import { env } from "expect-env";
-import type {
-	ImapEvent,
-	SyncMailboxesEvent,
-	SyncMessagesEvent,
-} from "./events.js";
+import type { ImapEvent } from "./events.js";
 
 type EventInput = Omit<ImapEvent, "eventId" | "timestamp">;
 
@@ -52,49 +48,6 @@ const queueUrlMap: Record<ImapEvent["type"], string> = {
 	FLAG_PUSH: flagsQueueUrl,
 };
 
-/**
- * FIFO queue event types that support deduplication.
- * Management events (MAILBOX_*, MESSAGE_*, EMPTY_TRASH) are not deduplicated
- * because each operation is unique.
- */
-const fifoEventTypes = new Set([
-	"SYNC_MAILBOXES",
-	"SYNC_MESSAGES",
-	// flagsQueue is FIFO and requires a MessageGroupId on every message.
-	// Content-based deduplication (queue-level setting) covers the dedup id —
-	// no per-event case is needed in getDeduplicationId below, same as the
-	// (standard-queue) management events that fall through its default.
-	"FLAG_PUSH",
-]);
-
-/**
- * Generate a deduplication ID for FIFO queue events.
- * SQS deduplication window is 5 minutes - duplicate messages within
- * this window are rejected.
- */
-export const getDeduplicationId = (event: EventInput): string | undefined => {
-	switch (event.type) {
-		case "SYNC_MAILBOXES": {
-			const e = event as Omit<SyncMailboxesEvent, "eventId" | "timestamp">;
-			return `SYNC_MAILBOXES:${e.accountId}`;
-		}
-
-		case "SYNC_MESSAGES": {
-			const e = event as Omit<SyncMessagesEvent, "eventId" | "timestamp">;
-			// A continuation carries a per-batch cursor so batches 2..N are not
-			// deduped against the initial event (which has none) or each other; the
-			// cursor-less initial id still dedups concurrent fresh syncs of a mailbox.
-			return e.resumeCursor === undefined
-				? `SYNC_MESSAGES:${e.mailboxId}`
-				: `SYNC_MESSAGES:${e.mailboxId}:${e.resumeCursor}`;
-		}
-
-		default:
-			// Management events are not deduplicated
-			return undefined;
-	}
-};
-
 /**
  * Check if queue URL is a FIFO queue (ends with .fifo)
  */
@@ -119,7 +72,7 @@ export const emitEvent = async (
 	} as ImapEvent;
 
 	const queueUrl = queueUrlMap[event.type];
-	const useFifo = isFifoQueue(queueUrl) && fifoEventTypes.has(event.type);
+	const useFifo = isFifoQueue(queueUrl);
 
 	// ElasticMQ FIFO queues don't support per-message DelaySeconds
 	const useDelay = options?.delaySeconds && !isLocal;
@@ -130,10 +83,22 @@ export const emitEvent = async (
 			MessageBody: JSON.stringify(fullEvent),
 			// Delay delivery for retry backoff (skip for local ElasticMQ)
 			...(useDelay && { DelaySeconds: options.delaySeconds }),
-			// FIFO queue parameters - only set if queue is FIFO
+			// FIFO queue parameters — only set if the queue is FIFO. The
+			// deduplication id is the event's own id, so the queue suppresses a
+			// re-send of one event (the retry SQS's own idempotency guard is for)
+			// and nothing else. A shared id per account or per mailbox instead made
+			// the 5-minute window a rate limiter: the second sync of a mailbox
+			// within five minutes was discarded before any worker saw it, so mail
+			// that arrived after a sync could not be fetched until the window
+			// elapsed (issue #37).
+			//
+			// Dropping events is not how repeated work is bounded — that is the
+			// freshness gate in the sync-mailboxes fan-out, which decides whether
+			// a mailbox is worth enumerating at all. MessageGroupId only orders an
+			// account's events; it makes duplicates serial, not cheap.
 			...(useFifo && {
 				MessageGroupId: event.accountId,
-				MessageDeduplicationId: getDeduplicationId(event),
+				MessageDeduplicationId: fullEvent.eventId,
 			}),
 		}),
 	);

package/src/events.ts

@@ -6,22 +6,19 @@ export interface BaseEvent {
 
 export interface SyncMailboxesEvent extends BaseEvent {
 	type: "SYNC_MAILBOXES";
+	/**
+	 * Set by POST /sync — the only trigger a person asks for by name. It makes
+	 * the fan-out sync every mailbox, skipping the freshness gate that keeps
+	 * incidental triggers (config load, OAuth connect, the scheduled tick) from
+	 * re-enumerating folders that were just enumerated.
+	 */
+	explicitRequest?: boolean;
 }
 
 export interface SyncMessagesEvent extends BaseEvent {
 	type: "SYNC_MESSAGES";
 	mailboxId: string;
 	fullSync?: boolean; // If true, ignore lastSyncUid
-	/**
-	 * Set on a continuation event (the "next batch" a batch emits when it drains
-	 * only part of a mailbox). Carries the batch's remaining-message count so the
-	 * FIFO deduplication id is distinct per batch — otherwise every continuation
-	 * for a mailbox shares one dedup id and the 5-minute window silently drops
-	 * batches 2..N, capping any mailbox over one batch. The value is not used by
-	 * the handler (the resume point comes from the persisted watermark); it only
-	 * makes the dedup id unique.
-	 */
-	resumeCursor?: number;
 }
 
 /**

package/src/handlers/sync-mailboxes.test.ts

@@ -0,0 +1,47 @@
+import assert from "node:assert/strict";
+import { describe, it } from "node:test";
+import { MAILBOX_FRESHNESS_MS, mailboxNeedsSync } from "./sync-mailboxes.js";
+
+const NOW = 1_700_000_000_000;
+
+const askedForByName = { explicitRequest: true } as const;
+const sideEffect = {} as const;
+
+describe("mailboxNeedsSync", () => {
+	// Issue #37: the gate this replaces applied to every trigger, so a refresh
+	// that landed just after a side-effect sync did nothing at all. A sync
+	// asked for by name (POST /sync) is never gated, however recently one ran.
+	it("syncs a mailbox synced a moment ago when the sync was asked for by name", () => {
+		const mailbox = { lastMessageSyncAt: NOW - 1_000 };
+
+		assert.equal(mailboxNeedsSync(mailbox, askedForByName, NOW), true);
+	});
+
+	// Without this, `GET /config` — which fires a trigger per account on every
+	// call — re-enumerates every folder an account owns on every page load.
+	it("skips a freshly-synced mailbox for a side-effect trigger", () => {
+		const mailbox = { lastMessageSyncAt: NOW - 1_000 };
+
+		assert.equal(mailboxNeedsSync(mailbox, sideEffect, NOW), false);
+	});
+
+	it("syncs a stale mailbox for a side-effect trigger", () => {
+		const mailbox = { lastMessageSyncAt: NOW - MAILBOX_FRESHNESS_MS - 1 };
+
+		assert.equal(mailboxNeedsSync(mailbox, sideEffect, NOW), true);
+	});
+
+	it("syncs exactly at the freshness threshold", () => {
+		const mailbox = { lastMessageSyncAt: NOW - MAILBOX_FRESHNESS_MS };
+
+		assert.equal(mailboxNeedsSync(mailbox, sideEffect, NOW), true);
+	});
+
+	it("always syncs a mailbox that has never synced", () => {
+		assert.equal(mailboxNeedsSync({}, sideEffect, NOW), true);
+		assert.equal(
+			mailboxNeedsSync({ lastMessageSyncAt: 0 }, sideEffect, NOW),
+			true,
+		);
+	});
+});

package/src/handlers/sync-mailboxes.ts

@@ -23,7 +23,42 @@ import { buildLifecycleDeps } from "../with-oauth-lifecycle-deps.js";
 import { orderMailboxesForSync } from "./mailbox-sync-order.js";
 
 const EVENT_EMIT_CONCURRENCY = 20;
-const SYNC_COOLDOWN_MS = 30_000; // 30 seconds
+
+/**
+ * How recently a mailbox must have been synced for a side-effect trigger to
+ * leave it alone. Short enough that a folder is never more than a minute
+ * staler than the trigger that arrived, long enough to collapse the burst a
+ * client produces when it loads (`GET /config` triggers a sync per account)
+ * into one round of IMAP work.
+ *
+ * The web client floors its automatic poll at this same window
+ * (`MIN_POLL_INTERVAL_MS` in useStaleAccountSync), so the one caller that
+ * skips this gate on a timer still cannot drive a fan-out faster than it.
+ */
+export const MAILBOX_FRESHNESS_MS = 60_000;
+
+/**
+ * Whether this fan-out should enqueue a mailbox.
+ *
+ * A sync asked for by name — POST /sync, which is the refresh control,
+ * pull-to-refresh, and the client's automatic poll — always syncs every
+ * mailbox, whatever ran a moment ago. A sync that happens as a side effect of
+ * something else (config load, OAuth connect, account create, the scheduled
+ * tick) skips mailboxes synced inside {@link MAILBOX_FRESHNESS_MS}, which is
+ * what stops those triggers from re-enumerating an account's folders on every
+ * page load.
+ *
+ * A mailbox that has never synced is always due.
+ */
+export const mailboxNeedsSync = (
+	mailbox: { lastMessageSyncAt?: number },
+	event: Pick<SyncMailboxesEvent, "explicitRequest">,
+	now: number,
+): boolean => {
+	if (event.explicitRequest) return true;
+	if (!mailbox.lastMessageSyncAt) return true;
+	return now - mailbox.lastMessageSyncAt >= MAILBOX_FRESHNESS_MS;
+};
 
 export const syncMailboxes = async (
 	event: SyncMailboxesEvent,
@@ -66,6 +101,7 @@ export const syncMailboxes = async (
 		async (credentials) => {
 			try {
 				await syncMailboxesForAccount(
+					event,
 					account,
 					credentials,
 					mailboxService,
@@ -97,6 +133,7 @@ export const syncMailboxes = async (
 };
 
 const syncMailboxesForAccount = async (
+	event: SyncMailboxesEvent,
 	account: AccountItem,
 	credentials: MailCredentials,
 	mailboxService: IMailboxRepository,
@@ -138,24 +175,42 @@ const syncMailboxesForAccount = async (
 
 	log.info({ result }, "Mailbox sync complete");
 
-	// Get all mailboxes and emit SYNC_MESSAGES for each
 	const allMailboxes = await collectAllMailboxes(accountId, mailboxService);
 
-	// Filter out mailboxes that were synced recently (cooldown)
-	// Always include mailboxes that were never synced (lastMessageSyncAt is 0, undefined, or null)
+	// This fan-out is where an account's IMAP work is decided: one SEARCH per
+	// mailbox per event, on a queue every account shares. `GET /config` fires a
+	// trigger per account on every call, so without a gate here an idle client
+	// re-enumerates every folder it owns on every page load.
+	//
+	// The gate is per mailbox and a sync asked for by name skips it outright.
+	// That distinction is the whole point: the previous cooldown gated
+	// everything, which is what made a refresh a no-op whenever a side-effect
+	// trigger had just run (issue #37).
 	const now = Date.now();
-	const mailboxes = allMailboxes.filter(
-		(m) => !m.lastMessageSyncAt || now - m.lastMessageSyncAt > SYNC_COOLDOWN_MS,
+	const mailboxes = allMailboxes.filter((mailbox) =>
+		mailboxNeedsSync(mailbox, event, now),
 	);
 
 	const skipped = allMailboxes.length - mailboxes.length;
 	if (skipped > 0) {
-		log.info({ accountId, skipped }, "Skipped mailboxes due to sync cooldown");
+		log.info(
+			{ accountId, skipped },
+			"Skipped recently-synced mailboxes for a side-effect trigger",
+		);
 	}
 
-	if (mailboxes.length === 0) {
+	if (allMailboxes.length === 0) {
 		log.info({ accountId }, "No mailboxes to sync messages for");
-		// If there are no mailboxes to sync, mark as complete
+		await accountService.update(accountId, {
+			syncPhase: SyncPhase.complete,
+			mailboxCountTotal: 0,
+			mailboxCountSynced: 0,
+		});
+		return;
+	}
+
+	if (mailboxes.length === 0) {
+		log.info({ accountId }, "Every mailbox is fresh; nothing to sync");
 		await accountService.update(accountId, {
 			syncPhase: SyncPhase.complete,
 			mailboxCountTotal: allMailboxes.length,
@@ -169,10 +224,16 @@ const syncMailboxesForAccount = async (
 		"Emitting SYNC_MESSAGES events",
 	);
 
-	// Phase transition. Completion events only fire for the enqueued
-	// (cooldown-filtered) set, so pre-credit the skipped mailboxes into
-	// mailboxCountSynced — otherwise synced can never reach total.
-	// If INBOX itself was skipped, go straight to syncing_others.
+	// Phase transition. Only the enqueued mailboxes emit a completion, so the
+	// gated ones are pre-credited — otherwise synced could never reach total.
+	//
+	// The counter is progress through THIS round, not a lifetime total: a new
+	// round restarts it, which is why it can read lower than a moment ago while
+	// a previous round is still draining. That is the same instant
+	// `account.lastSyncAt` is stamped, and the per-mailbox completion guard in
+	// sync-messages.ts keys off exactly that stamp — so a completion still in
+	// flight from the previous round counts once towards the new one, and each
+	// mailbox counts at most once per round.
 	const inboxEnqueued = mailboxes.some(
 		(m) => m.fullPath.toUpperCase() === "INBOX",
 	);

package/src/handlers/sync-messages.test.ts

@@ -147,6 +147,45 @@ describe("drainPendingFlagPushes — periodic per-mailbox re-arm (issue #1273)",
 		assert.equal(emitted.length, 2);
 	});
 
+	/**
+	 * Markers persisted before the system-flag wire-format fix carry the
+	 * unprefixed spelling (`Seen`, not `\Seen`) the enum emitter used to
+	 * produce. The drain re-arms from the stored `marker.flagName` and never
+	 * from the enum, and `handleFlagPush` threads that same value through
+	 * `find`/`updateState`/`delete` — so a marker written under the old
+	 * spelling still matches itself and drains to completion. No migration,
+	 * no orphans.
+	 */
+	it("re-arms a marker persisted under the pre-fix unprefixed spelling verbatim", async () => {
+		const markerService = {
+			listByMailboxId: async () => [
+				marker({ messageId: "legacy-msg", flagName: "Seen" }),
+				marker({ messageId: "legacy-star", flagName: "Flagged" }),
+			],
+		} as unknown as IMessageFlagPushRepository;
+
+		const emitted: Array<{ messageId: string; flagName: string }> = [];
+		const { log } = buildLogger();
+
+		await drainPendingFlagPushes(
+			markerService,
+			account,
+			"mbx-1",
+			log,
+			async (event) => {
+				emitted.push(event as unknown as (typeof emitted)[number]);
+			},
+		);
+
+		assert.deepEqual(
+			emitted.map((e) => [e.messageId, e.flagName]),
+			[
+				["legacy-msg", "Seen"],
+				["legacy-star", "Flagged"],
+			],
+		);
+	});
+
 	it("a re-arm (SQS) failure is caught per-marker and logged loudly — never thrown", async () => {
 		const markerService = {
 			listByMailboxId: async () => [marker()],

package/src/handlers/sync-messages.ts

@@ -369,7 +369,6 @@ const syncMailboxMessages = async (
 			type: "SYNC_MESSAGES",
 			accountId: event.accountId,
 			mailboxId,
-			resumeCursor: result.remainingCount,
 		};
 		await emitEvent(nextSyncEvent);
 		return;