@remit/imap-worker 0.0.34 → 0.0.36

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@remit/imap-worker",
-  "version": "0.0.34",
+  "version": "0.0.36",
   "type": "module",
   "main": "dist/index.js",
   "exports": {
@@ -28,7 +28,7 @@
     "@aws-sdk/core": "*",
     "@remit/storage-service": "*",
     "aws-sdk-client-mock": "*",
-    "esbuild": "^0.27.7"
+    "esbuild": "^0.28.1"
   },
   "dependencies": {
     "@aws-sdk/client-s3": "^3.1055.0",

package/src/handlers/flag-push.test.ts

@@ -4,7 +4,9 @@ import { getClient, type RemitClient, setClient } from "@remit/backend/client";
 import type { Logger } from "@remit/logger-lambda";
 import type { FlagPushEvent } from "../events.js";
 import {
+	FLAG_PUSH_DEFER_MAX_MS,
 	FLAG_PUSH_MAX_ATTEMPTS,
+	getFlagPushDeferMaxMs,
 	getFlagPushMaxAttempts,
 	handleFlagPush,
 } from "./flag-push.js";
@@ -45,6 +47,28 @@ describe("getFlagPushMaxAttempts — env-derived threshold (mirrors #1270's getB
 	});
 });
 
+describe("getFlagPushDeferMaxMs", () => {
+	it("parses an env override", () => {
+		assert.equal(
+			getFlagPushDeferMaxMs({ FLAG_PUSH_DEFER_MAX_MS: "1000" }),
+			1000,
+		);
+	});
+
+	it("defaults to 10 minutes when unset or invalid", () => {
+		assert.equal(getFlagPushDeferMaxMs({}), 10 * 60 * 1000);
+		assert.equal(
+			getFlagPushDeferMaxMs({ FLAG_PUSH_DEFER_MAX_MS: "nope" }),
+			10 * 60 * 1000,
+		);
+	});
+
+	it("the module-level constant reflects the actual process env at load time", () => {
+		assert.equal(typeof FLAG_PUSH_DEFER_MAX_MS, "number");
+		assert.ok(FLAG_PUSH_DEFER_MAX_MS > 0);
+	});
+});
+
 describe("handleFlagPush — deleted mailbox is terminal (#287/#289)", () => {
 	const accountId = "fp-acc-zzz";
 	const messageId = "fp-msg-zzz";
@@ -82,7 +106,7 @@ describe("handleFlagPush — deleted mailbox is terminal (#287/#289)", () => {
 			accountConfigId: "fp-cfg-zzz",
 		}));
 		mock.method(client.message, "get", async () => [
-			{ messageId, mailboxId: "gone-mbx", uid: 42 },
+			{ messageId, mailboxId: "gone-mbx", uid: 42, status: "active" },
 		]);
 		mock.method(client.mailbox, "get", async () => {
 			throw Object.assign(new Error("Mailbox not found: gone-mbx"), {
@@ -111,3 +135,191 @@ describe("handleFlagPush — deleted mailbox is terminal (#287/#289)", () => {
 		]);
 	});
 });
+
+describe("handleFlagPush — defers while a move is in flight, never on an ordinary pending sync", () => {
+	const accountId = "fp-acc-inflight";
+	const messageId = "fp-msg-inflight";
+	const flagName = "$Junk";
+
+	const event: FlagPushEvent = {
+		type: "FLAG_PUSH",
+		accountId,
+		accountConfigId: "fp-cfg-inflight",
+		messageId,
+		flagName,
+	} as FlagPushEvent;
+
+	before(() => {
+		setClient({
+			account: { get: async () => undefined },
+			message: { get: async () => undefined },
+			mailbox: { get: async () => undefined },
+			flagPush: {
+				find: async () => undefined,
+				updateState: async () => undefined,
+				delete: async () => undefined,
+			},
+		} as unknown as RemitClient);
+	});
+
+	afterEach(() => mock.restoreAll());
+
+	it("resets the marker to pending and never opens a connection when the message's move has not settled", async () => {
+		const client = await getClient();
+		mock.method(client.account, "get", async () => ({
+			accountId,
+			accountConfigId: "fp-cfg-inflight",
+			passwordHash: "not-actually-used-if-guard-works",
+		}));
+		mock.method(client.message, "get", async () => [
+			{
+				messageId,
+				mailboxId: "mbx-junk",
+				uid: 42,
+				// Exactly what MessageMoveService.moveMessage's local optimistic
+				// write leaves in place while the IMAP MOVE is still in flight.
+				status: "moving",
+				syncStatus: "pending",
+			},
+		]);
+		const mailboxGet = mock.method(client.mailbox, "get", async () => ({
+			mailboxId: "mbx-junk",
+			fullPath: "Junk",
+		}));
+		mock.method(client.flagPush, "find", async () => ({
+			operation: "add",
+			state: "queued",
+			createdAt: Date.now(),
+		}));
+		const updateState = mock.method(
+			client.flagPush,
+			"updateState",
+			async () => {},
+		);
+		const deleteMarker = mock.method(client.flagPush, "delete", async () => {});
+
+		await handleFlagPush(event, silentLogger, 1);
+
+		assert.equal(
+			mailboxGet.mock.calls.length,
+			0,
+			"never even resolves the mailbox — returns before touching IMAP",
+		);
+		assert.equal(
+			deleteMarker.mock.calls.length,
+			0,
+			"the marker is not cleared",
+		);
+		assert.equal(updateState.mock.calls.length, 1);
+		assert.deepEqual(updateState.mock.calls[0].arguments, [
+			messageId,
+			flagName,
+			"pending",
+		]);
+	});
+
+	it("drops the marker without ever deferring again once a move has been stuck past the defer window", async () => {
+		const client = await getClient();
+		mock.method(client.account, "get", async () => ({
+			accountId,
+			accountConfigId: "fp-cfg-inflight",
+			passwordHash: "not-actually-used-if-guard-works",
+		}));
+		mock.method(client.message, "get", async () => [
+			{ messageId, mailboxId: "mbx-junk", uid: 42, status: "moving" },
+		]);
+		const mailboxGet = mock.method(client.mailbox, "get", async () => ({
+			mailboxId: "mbx-junk",
+			fullPath: "Junk",
+		}));
+		mock.method(client.flagPush, "find", async () => ({
+			operation: "add",
+			state: "pending",
+			// Long past FLAG_PUSH_DEFER_MAX_MS — the periodic drain has already
+			// re-armed and re-deferred this marker many times over.
+			createdAt: Date.now() - (FLAG_PUSH_DEFER_MAX_MS + 60_000),
+		}));
+		const updateState = mock.method(
+			client.flagPush,
+			"updateState",
+			async () => {},
+		);
+		const deleteMarker = mock.method(client.flagPush, "delete", async () => {});
+
+		await handleFlagPush(event, silentLogger, 1);
+
+		assert.equal(
+			mailboxGet.mock.calls.length,
+			0,
+			"never even resolves the mailbox — returns before touching IMAP",
+		);
+		assert.equal(
+			updateState.mock.calls.length,
+			0,
+			"never re-armed to pending — this is the terminal outcome, not another defer",
+		);
+		assert.equal(deleteMarker.mock.calls.length, 1);
+		assert.deepEqual(deleteMarker.mock.calls[0].arguments, [
+			messageId,
+			flagName,
+		]);
+	});
+
+	it("does NOT defer an ordinary freshly-synced inbound message — syncStatus stays pending forever on the inbound path", async () => {
+		// This is the regression the guard must never reintroduce: every
+		// message message-sync.ts creates comes out `syncStatus: "pending"`
+		// (DrizzleMessageRepository.create's default) and nothing on the
+		// inbound path ever promotes it to `synced`. Only `status` names an
+		// actual move in flight.
+		const client = await getClient();
+		mock.method(client.account, "get", async () => ({
+			accountId,
+			accountConfigId: "fp-cfg-inflight",
+			passwordHash: "not-actually-used-if-guard-works",
+		}));
+		mock.method(client.message, "get", async () => [
+			{
+				messageId,
+				mailboxId: "mbx-junk",
+				uid: 42,
+				status: "active",
+				syncStatus: "pending",
+			},
+		]);
+		// Trips the (unrelated, already-covered) cursor-rebuild early return
+		// right after the mailbox lookup — proves the handler reached past the
+		// move-in-flight guard without opening a real IMAP connection.
+		const mailboxGet = mock.method(client.mailbox, "get", async () => ({
+			mailboxId: "mbx-junk",
+			fullPath: "Junk",
+			cursorState: "cursor_invalid",
+		}));
+		mock.method(client.flagPush, "find", async () => ({
+			operation: "add",
+			state: "queued",
+			createdAt: Date.now(),
+		}));
+		const updateState = mock.method(
+			client.flagPush,
+			"updateState",
+			async () => {},
+		);
+		const deleteMarker = mock.method(client.flagPush, "delete", async () => {});
+
+		await handleFlagPush(event, silentLogger, 1);
+
+		assert.equal(
+			mailboxGet.mock.calls.length,
+			1,
+			"the handler proceeded past the move-in-flight guard",
+		);
+		assert.equal(deleteMarker.mock.calls.length, 0);
+		// "processing" is the advance-to-attempt transition, never "pending" —
+		// a re-defer here would be exactly blocker 1 again.
+		assert.deepEqual(updateState.mock.calls[0]?.arguments, [
+			messageId,
+			flagName,
+			"processing",
+		]);
+	});
+});

package/src/handlers/flag-push.ts

@@ -1,4 +1,5 @@
 import { getClient } from "@remit/backend/client";
+import { MessageStatus } from "@remit/domain-enums";
 import type { Logger } from "@remit/logger-lambda";
 import { recordImapFailure } from "@remit/logger-lambda";
 import {
@@ -35,6 +36,29 @@ export const getFlagPushMaxAttempts = (
 
 export const FLAG_PUSH_MAX_ATTEMPTS = getFlagPushMaxAttempts();
 
+/**
+ * How long a marker may sit deferred behind a move before it is dropped
+ * outright. A move that settles takes seconds to low minutes; one stuck past
+ * this window has almost certainly already exhausted its own retries with no
+ * terminal resolver of its own for a regular move (unlike flag-push and
+ * placement-move), so deferring further would cycle one SQS round trip per
+ * sync tick forever instead of surfacing the stall.
+ */
+const DEFAULT_FLAG_PUSH_DEFER_MAX_MS = 10 * 60 * 1000;
+
+export const getFlagPushDeferMaxMs = (
+	processEnv: NodeJS.ProcessEnv = process.env,
+): number => {
+	const raw = processEnv.FLAG_PUSH_DEFER_MAX_MS;
+	if (!raw) return DEFAULT_FLAG_PUSH_DEFER_MAX_MS;
+	const parsed = Number.parseInt(raw, 10);
+	return Number.isFinite(parsed) && parsed > 0
+		? parsed
+		: DEFAULT_FLAG_PUSH_DEFER_MAX_MS;
+};
+
+export const FLAG_PUSH_DEFER_MAX_MS = getFlagPushDeferMaxMs();
+
 /**
  * Handle FLAG_PUSH events (issue #1273, epic #1281). Drains ONE pending
  * flag-push marker: resolves the message's UID and CURRENT mailbox fresh
@@ -108,6 +132,50 @@ export const handleFlagPush = async (
 		return;
 	}
 
+	// The message carries a move still in flight — its `mailboxId`/`uid` are
+	// the local optimistic write `MessageMoveService` made before enqueueing
+	// the IMAP MOVE, not yet confirmed by the server, so a STORE resolved
+	// against this row right now would land on the wrong UID or the wrong
+	// folder (or both). `syncStatus` is NOT the right signal here: an ordinary
+	// freshly-synced inbound row is `pending` too (nothing on the inbound path
+	// ever promotes it to `synced`), so keying off it would defer every
+	// outbound flag push in the product forever. `status === moving` is set
+	// only by an actual move/delete-to-trash and cleared only once it settles.
+	if (message.status === MessageStatus.moving) {
+		const deferredForMs = Date.now() - marker.createdAt;
+
+		if (deferredForMs > FLAG_PUSH_DEFER_MAX_MS) {
+			// The move this push was waiting on never settled within any
+			// reasonable window — deferring further would cycle one SQS round
+			// trip per sync tick forever. Drop the stale marker loudly rather
+			// than push against state nobody has confirmed.
+			await markerService.delete(messageId, flagName);
+			recordImapFailure("FLAG_PUSH_MOVE_NEVER_SETTLED", "other");
+			log.error(
+				{
+					alert: "flag_push_move_never_settled",
+					messageId,
+					flagName,
+					accountId,
+					deferredForMs,
+				},
+				"Message move never settled; dropping the flag-push marker that was waiting on it",
+			);
+			return;
+		}
+
+		// Reset to `pending` rather than advancing: `drainPendingFlagPushes`
+		// only re-arms markers in that state, scoped by the marker's own
+		// `mailboxId` (the push destination), so the next periodic sync tick
+		// of that mailbox picks this back up once the move has settled.
+		await markerService.updateState(messageId, flagName, "pending");
+		log.info(
+			{ messageId, flagName, accountId, deferredForMs },
+			"Message has a move in flight; pausing outbound flag push until it settles",
+		);
+		return;
+	}
+
 	// The worker has picked up the event and is about to actually attempt the
 	// IMAP STORE — advance the state engine (pending/queued -> processing).
 	// Idempotent to call again on a redelivered event (a prior attempt that

package/src/processor.ts

@@ -18,10 +18,11 @@ export const processEvent = async (
 	log: Logger,
 	/**
 	 * SQS's own delivery count for the record carrying this event (1 on first
-	 * delivery). Only SYNC_MESSAGE_BODY reads it — it's how the handler knows
-	 * this is the last attempt before the queue's own redrive policy would
-	 * DLQ the record, so it can resolve retry exhaustion into a terminal
-	 * outcome (issue #1270) instead of dead-lettering blindly.
+	 * delivery). Read by SYNC_MESSAGE_BODY, PLACEMENT_MOVE_PUSH and FLAG_PUSH
+	 * — each knows from it when this is the last attempt before the queue's
+	 * own redrive policy would DLQ the record, so it can resolve
+	 * retry exhaustion into a terminal outcome (issue #1270) instead of
+	 * dead-lettering blindly.
 	 */
 	receiveCount = 1,
 ): Promise<void> => {