@remit/imap-worker 0.0.33 → 0.0.35

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@remit/imap-worker",
-  "version": "0.0.33",
+  "version": "0.0.35",
   "type": "module",
   "main": "dist/index.js",
   "exports": {

package/src/handlers/flag-push.test.ts

@@ -4,7 +4,9 @@ import { getClient, type RemitClient, setClient } from "@remit/backend/client";
 import type { Logger } from "@remit/logger-lambda";
 import type { FlagPushEvent } from "../events.js";
 import {
+	FLAG_PUSH_DEFER_MAX_MS,
 	FLAG_PUSH_MAX_ATTEMPTS,
+	getFlagPushDeferMaxMs,
 	getFlagPushMaxAttempts,
 	handleFlagPush,
 } from "./flag-push.js";
@@ -45,6 +47,28 @@ describe("getFlagPushMaxAttempts — env-derived threshold (mirrors #1270's getB
 	});
 });
 
+describe("getFlagPushDeferMaxMs", () => {
+	it("parses an env override", () => {
+		assert.equal(
+			getFlagPushDeferMaxMs({ FLAG_PUSH_DEFER_MAX_MS: "1000" }),
+			1000,
+		);
+	});
+
+	it("defaults to 10 minutes when unset or invalid", () => {
+		assert.equal(getFlagPushDeferMaxMs({}), 10 * 60 * 1000);
+		assert.equal(
+			getFlagPushDeferMaxMs({ FLAG_PUSH_DEFER_MAX_MS: "nope" }),
+			10 * 60 * 1000,
+		);
+	});
+
+	it("the module-level constant reflects the actual process env at load time", () => {
+		assert.equal(typeof FLAG_PUSH_DEFER_MAX_MS, "number");
+		assert.ok(FLAG_PUSH_DEFER_MAX_MS > 0);
+	});
+});
+
 describe("handleFlagPush — deleted mailbox is terminal (#287/#289)", () => {
 	const accountId = "fp-acc-zzz";
 	const messageId = "fp-msg-zzz";
@@ -82,7 +106,7 @@ describe("handleFlagPush — deleted mailbox is terminal (#287/#289)", () => {
 			accountConfigId: "fp-cfg-zzz",
 		}));
 		mock.method(client.message, "get", async () => [
-			{ messageId, mailboxId: "gone-mbx", uid: 42 },
+			{ messageId, mailboxId: "gone-mbx", uid: 42, status: "active" },
 		]);
 		mock.method(client.mailbox, "get", async () => {
 			throw Object.assign(new Error("Mailbox not found: gone-mbx"), {
@@ -111,3 +135,191 @@ describe("handleFlagPush — deleted mailbox is terminal (#287/#289)", () => {
 		]);
 	});
 });
+
+describe("handleFlagPush — defers while a move is in flight, never on an ordinary pending sync", () => {
+	const accountId = "fp-acc-inflight";
+	const messageId = "fp-msg-inflight";
+	const flagName = "$Junk";
+
+	const event: FlagPushEvent = {
+		type: "FLAG_PUSH",
+		accountId,
+		accountConfigId: "fp-cfg-inflight",
+		messageId,
+		flagName,
+	} as FlagPushEvent;
+
+	before(() => {
+		setClient({
+			account: { get: async () => undefined },
+			message: { get: async () => undefined },
+			mailbox: { get: async () => undefined },
+			flagPush: {
+				find: async () => undefined,
+				updateState: async () => undefined,
+				delete: async () => undefined,
+			},
+		} as unknown as RemitClient);
+	});
+
+	afterEach(() => mock.restoreAll());
+
+	it("resets the marker to pending and never opens a connection when the message's move has not settled", async () => {
+		const client = await getClient();
+		mock.method(client.account, "get", async () => ({
+			accountId,
+			accountConfigId: "fp-cfg-inflight",
+			passwordHash: "not-actually-used-if-guard-works",
+		}));
+		mock.method(client.message, "get", async () => [
+			{
+				messageId,
+				mailboxId: "mbx-junk",
+				uid: 42,
+				// Exactly what MessageMoveService.moveMessage's local optimistic
+				// write leaves in place while the IMAP MOVE is still in flight.
+				status: "moving",
+				syncStatus: "pending",
+			},
+		]);
+		const mailboxGet = mock.method(client.mailbox, "get", async () => ({
+			mailboxId: "mbx-junk",
+			fullPath: "Junk",
+		}));
+		mock.method(client.flagPush, "find", async () => ({
+			operation: "add",
+			state: "queued",
+			createdAt: Date.now(),
+		}));
+		const updateState = mock.method(
+			client.flagPush,
+			"updateState",
+			async () => {},
+		);
+		const deleteMarker = mock.method(client.flagPush, "delete", async () => {});
+
+		await handleFlagPush(event, silentLogger, 1);
+
+		assert.equal(
+			mailboxGet.mock.calls.length,
+			0,
+			"never even resolves the mailbox — returns before touching IMAP",
+		);
+		assert.equal(
+			deleteMarker.mock.calls.length,
+			0,
+			"the marker is not cleared",
+		);
+		assert.equal(updateState.mock.calls.length, 1);
+		assert.deepEqual(updateState.mock.calls[0].arguments, [
+			messageId,
+			flagName,
+			"pending",
+		]);
+	});
+
+	it("drops the marker without ever deferring again once a move has been stuck past the defer window", async () => {
+		const client = await getClient();
+		mock.method(client.account, "get", async () => ({
+			accountId,
+			accountConfigId: "fp-cfg-inflight",
+			passwordHash: "not-actually-used-if-guard-works",
+		}));
+		mock.method(client.message, "get", async () => [
+			{ messageId, mailboxId: "mbx-junk", uid: 42, status: "moving" },
+		]);
+		const mailboxGet = mock.method(client.mailbox, "get", async () => ({
+			mailboxId: "mbx-junk",
+			fullPath: "Junk",
+		}));
+		mock.method(client.flagPush, "find", async () => ({
+			operation: "add",
+			state: "pending",
+			// Long past FLAG_PUSH_DEFER_MAX_MS — the periodic drain has already
+			// re-armed and re-deferred this marker many times over.
+			createdAt: Date.now() - (FLAG_PUSH_DEFER_MAX_MS + 60_000),
+		}));
+		const updateState = mock.method(
+			client.flagPush,
+			"updateState",
+			async () => {},
+		);
+		const deleteMarker = mock.method(client.flagPush, "delete", async () => {});
+
+		await handleFlagPush(event, silentLogger, 1);
+
+		assert.equal(
+			mailboxGet.mock.calls.length,
+			0,
+			"never even resolves the mailbox — returns before touching IMAP",
+		);
+		assert.equal(
+			updateState.mock.calls.length,
+			0,
+			"never re-armed to pending — this is the terminal outcome, not another defer",
+		);
+		assert.equal(deleteMarker.mock.calls.length, 1);
+		assert.deepEqual(deleteMarker.mock.calls[0].arguments, [
+			messageId,
+			flagName,
+		]);
+	});
+
+	it("does NOT defer an ordinary freshly-synced inbound message — syncStatus stays pending forever on the inbound path", async () => {
+		// This is the regression the guard must never reintroduce: every
+		// message message-sync.ts creates comes out `syncStatus: "pending"`
+		// (DrizzleMessageRepository.create's default) and nothing on the
+		// inbound path ever promotes it to `synced`. Only `status` names an
+		// actual move in flight.
+		const client = await getClient();
+		mock.method(client.account, "get", async () => ({
+			accountId,
+			accountConfigId: "fp-cfg-inflight",
+			passwordHash: "not-actually-used-if-guard-works",
+		}));
+		mock.method(client.message, "get", async () => [
+			{
+				messageId,
+				mailboxId: "mbx-junk",
+				uid: 42,
+				status: "active",
+				syncStatus: "pending",
+			},
+		]);
+		// Trips the (unrelated, already-covered) cursor-rebuild early return
+		// right after the mailbox lookup — proves the handler reached past the
+		// move-in-flight guard without opening a real IMAP connection.
+		const mailboxGet = mock.method(client.mailbox, "get", async () => ({
+			mailboxId: "mbx-junk",
+			fullPath: "Junk",
+			cursorState: "cursor_invalid",
+		}));
+		mock.method(client.flagPush, "find", async () => ({
+			operation: "add",
+			state: "queued",
+			createdAt: Date.now(),
+		}));
+		const updateState = mock.method(
+			client.flagPush,
+			"updateState",
+			async () => {},
+		);
+		const deleteMarker = mock.method(client.flagPush, "delete", async () => {});
+
+		await handleFlagPush(event, silentLogger, 1);
+
+		assert.equal(
+			mailboxGet.mock.calls.length,
+			1,
+			"the handler proceeded past the move-in-flight guard",
+		);
+		assert.equal(deleteMarker.mock.calls.length, 0);
+		// "processing" is the advance-to-attempt transition, never "pending" —
+		// a re-defer here would be exactly blocker 1 again.
+		assert.deepEqual(updateState.mock.calls[0]?.arguments, [
+			messageId,
+			flagName,
+			"processing",
+		]);
+	});
+});

package/src/handlers/flag-push.ts

@@ -1,4 +1,5 @@
 import { getClient } from "@remit/backend/client";
+import { MessageStatus } from "@remit/domain-enums";
 import type { Logger } from "@remit/logger-lambda";
 import { recordImapFailure } from "@remit/logger-lambda";
 import {
@@ -35,6 +36,29 @@ export const getFlagPushMaxAttempts = (
 
 export const FLAG_PUSH_MAX_ATTEMPTS = getFlagPushMaxAttempts();
 
+/**
+ * How long a marker may sit deferred behind a move before it is dropped
+ * outright. A move that settles takes seconds to low minutes; one stuck past
+ * this window has almost certainly already exhausted its own retries with no
+ * terminal resolver of its own for a regular move (unlike flag-push and
+ * placement-move), so deferring further would cycle one SQS round trip per
+ * sync tick forever instead of surfacing the stall.
+ */
+const DEFAULT_FLAG_PUSH_DEFER_MAX_MS = 10 * 60 * 1000;
+
+export const getFlagPushDeferMaxMs = (
+	processEnv: NodeJS.ProcessEnv = process.env,
+): number => {
+	const raw = processEnv.FLAG_PUSH_DEFER_MAX_MS;
+	if (!raw) return DEFAULT_FLAG_PUSH_DEFER_MAX_MS;
+	const parsed = Number.parseInt(raw, 10);
+	return Number.isFinite(parsed) && parsed > 0
+		? parsed
+		: DEFAULT_FLAG_PUSH_DEFER_MAX_MS;
+};
+
+export const FLAG_PUSH_DEFER_MAX_MS = getFlagPushDeferMaxMs();
+
 /**
  * Handle FLAG_PUSH events (issue #1273, epic #1281). Drains ONE pending
  * flag-push marker: resolves the message's UID and CURRENT mailbox fresh
@@ -108,6 +132,50 @@ export const handleFlagPush = async (
 		return;
 	}
 
+	// The message carries a move still in flight — its `mailboxId`/`uid` are
+	// the local optimistic write `MessageMoveService` made before enqueueing
+	// the IMAP MOVE, not yet confirmed by the server, so a STORE resolved
+	// against this row right now would land on the wrong UID or the wrong
+	// folder (or both). `syncStatus` is NOT the right signal here: an ordinary
+	// freshly-synced inbound row is `pending` too (nothing on the inbound path
+	// ever promotes it to `synced`), so keying off it would defer every
+	// outbound flag push in the product forever. `status === moving` is set
+	// only by an actual move/delete-to-trash and cleared only once it settles.
+	if (message.status === MessageStatus.moving) {
+		const deferredForMs = Date.now() - marker.createdAt;
+
+		if (deferredForMs > FLAG_PUSH_DEFER_MAX_MS) {
+			// The move this push was waiting on never settled within any
+			// reasonable window — deferring further would cycle one SQS round
+			// trip per sync tick forever. Drop the stale marker loudly rather
+			// than push against state nobody has confirmed.
+			await markerService.delete(messageId, flagName);
+			recordImapFailure("FLAG_PUSH_MOVE_NEVER_SETTLED", "other");
+			log.error(
+				{
+					alert: "flag_push_move_never_settled",
+					messageId,
+					flagName,
+					accountId,
+					deferredForMs,
+				},
+				"Message move never settled; dropping the flag-push marker that was waiting on it",
+			);
+			return;
+		}
+
+		// Reset to `pending` rather than advancing: `drainPendingFlagPushes`
+		// only re-arms markers in that state, scoped by the marker's own
+		// `mailboxId` (the push destination), so the next periodic sync tick
+		// of that mailbox picks this back up once the move has settled.
+		await markerService.updateState(messageId, flagName, "pending");
+		log.info(
+			{ messageId, flagName, accountId, deferredForMs },
+			"Message has a move in flight; pausing outbound flag push until it settles",
+		);
+		return;
+	}
+
 	// The worker has picked up the event and is about to actually attempt the
 	// IMAP STORE — advance the state engine (pending/queued -> processing).
 	// Idempotent to call again on a redelivered event (a prior attempt that

package/src/processor.ts

@@ -18,10 +18,11 @@ export const processEvent = async (
 	log: Logger,
 	/**
 	 * SQS's own delivery count for the record carrying this event (1 on first
-	 * delivery). Only SYNC_MESSAGE_BODY reads it — it's how the handler knows
-	 * this is the last attempt before the queue's own redrive policy would
-	 * DLQ the record, so it can resolve retry exhaustion into a terminal
-	 * outcome (issue #1270) instead of dead-lettering blindly.
+	 * delivery). Read by SYNC_MESSAGE_BODY, PLACEMENT_MOVE_PUSH and FLAG_PUSH
+	 * — each knows from it when this is the last attempt before the queue's
+	 * own redrive policy would DLQ the record, so it can resolve
+	 * retry exhaustion into a terminal outcome (issue #1270) instead of
+	 * dead-lettering blindly.
 	 */
 	receiveCount = 1,
 ): Promise<void> => {

package/src/scheduler/loop.test.ts

@@ -0,0 +1,133 @@
+import assert from "node:assert/strict";
+import { describe, it } from "node:test";
+import { runSchedulerLoop, type SchedulerLoopOptions } from "./loop.js";
+import type { RunSchedulerTickDeps, SchedulerTickResult } from "./run-tick.js";
+
+const RESULT: SchedulerTickResult = { scanned: 0, enqueued: 0, skipped: 0 };
+
+// Only the loop's own two calls matter here; the tick is a stub, so what it is
+// handed is never read.
+const TICK_DEPS = {} as RunSchedulerTickDeps;
+
+const buildLoop = (
+	overrides: Partial<SchedulerLoopOptions>,
+): SchedulerLoopOptions => ({
+	tick: () => Promise.resolve(RESULT),
+	tickDeps: TICK_DEPS,
+	heartbeat: () => Promise.resolve(),
+	onHeartbeatError: () => {},
+	tickIntervalMs: 1000,
+	wait: () => Promise.resolve(),
+	...overrides,
+});
+
+describe("runSchedulerLoop", () => {
+	it("writes no heartbeat for a tick that throws", async () => {
+		let beats = 0;
+		await assert.rejects(
+			runSchedulerLoop(
+				buildLoop({
+					tick: () => Promise.reject(new Error("tick failed")),
+					heartbeat: () => {
+						beats += 1;
+						return Promise.resolve();
+					},
+				}),
+			),
+			/tick failed/,
+		);
+		assert.equal(beats, 0);
+	});
+
+	it("beats once per completed round, and stops beating when a round stops completing", async () => {
+		let rounds = 0;
+		let beats = 0;
+		await assert.rejects(
+			runSchedulerLoop(
+				buildLoop({
+					tick: () => {
+						rounds += 1;
+						return rounds > 2
+							? Promise.reject(new Error("tick failed"))
+							: Promise.resolve(RESULT);
+					},
+					heartbeat: () => {
+						beats += 1;
+						return Promise.resolve();
+					},
+				}),
+			),
+			/tick failed/,
+		);
+		assert.equal(beats, 2);
+	});
+
+	it("keeps ticking when the heartbeat write fails", async () => {
+		const errors: unknown[] = [];
+		let rounds = 0;
+		await assert.rejects(
+			runSchedulerLoop(
+				buildLoop({
+					tick: () => {
+						rounds += 1;
+						return rounds > 3
+							? Promise.reject(new Error("done"))
+							: Promise.resolve(RESULT);
+					},
+					heartbeat: () => Promise.reject(new Error("ENOSPC")),
+					onHeartbeatError: (error) => errors.push(error),
+				}),
+			),
+			/done/,
+		);
+		assert.equal(rounds, 4);
+		assert.equal(errors.length, 3);
+	});
+
+	it("waits a tick interval between rounds", async () => {
+		const waited: number[] = [];
+		let rounds = 0;
+		await assert.rejects(
+			runSchedulerLoop(
+				buildLoop({
+					tickIntervalMs: 300_000,
+					tick: () => {
+						rounds += 1;
+						return rounds > 2
+							? Promise.reject(new Error("done"))
+							: Promise.resolve(RESULT);
+					},
+					wait: (ms) => {
+						waited.push(ms);
+						return Promise.resolve();
+					},
+				}),
+			),
+			/done/,
+		);
+		assert.deepEqual(waited, [300_000, 300_000]);
+	});
+
+	// The default is the only one production uses, so it is the one a wrong unit
+	// or a dropped argument would ship in.
+	it("sleeps the real timer when no wait is injected", async () => {
+		let rounds = 0;
+		const started = Date.now();
+		await assert.rejects(
+			runSchedulerLoop({
+				...buildLoop({
+					tickIntervalMs: 25,
+					tick: () => {
+						rounds += 1;
+						return rounds > 2
+							? Promise.reject(new Error("done"))
+							: Promise.resolve(RESULT);
+					},
+				}),
+				wait: undefined,
+			}),
+			/done/,
+		);
+		assert.ok(Date.now() - started >= 50);
+	});
+});

package/src/scheduler/loop.ts

@@ -0,0 +1,38 @@
+import { setTimeout as delay } from "node:timers/promises";
+import type { Heartbeat } from "@remit/sqs-client/heartbeat";
+import type { RunSchedulerTickDeps, SchedulerTickResult } from "./run-tick.js";
+
+/**
+ * The scheduled-sync loop, separated from the wiring in runner.ts for the same
+ * reason `runQueuePoller` is separate from the container that starts it: the
+ * order of the two calls below is a property worth holding, and a module that
+ * connects to a queue and a database at import time cannot be asked about it.
+ *
+ * The beat lands after the tick returns, never before. A tick that throws on
+ * every pass exits the process and is restarted every few seconds, so a beat
+ * written on the way in would report a scheduler that enqueues nothing as
+ * healthy for as long as it kept crashing.
+ */
+export interface SchedulerLoopOptions {
+	readonly tick: (deps: RunSchedulerTickDeps) => Promise<SchedulerTickResult>;
+	readonly tickDeps: RunSchedulerTickDeps;
+	readonly heartbeat: Heartbeat;
+	readonly onHeartbeatError: (error: unknown) => void;
+	readonly tickIntervalMs: number;
+	readonly wait?: (ms: number) => Promise<void>;
+}
+
+export const runSchedulerLoop = async ({
+	tick,
+	tickDeps,
+	heartbeat,
+	onHeartbeatError,
+	tickIntervalMs,
+	wait = (ms) => delay(ms),
+}: SchedulerLoopOptions): Promise<never> => {
+	for (;;) {
+		await tick(tickDeps);
+		await heartbeat().catch(onHeartbeatError);
+		await wait(tickIntervalMs);
+	}
+};

package/src/scheduler/runner.ts

@@ -2,9 +2,11 @@
 import { setTimeout as delay } from "node:timers/promises";
 import { getClient } from "@remit/backend/client";
 import { createLogger } from "@remit/logger-lambda";
+import { clearHeartbeats, createHeartbeat } from "@remit/sqs-client/heartbeat";
 import { createQueueProducer } from "@remit/sqs-client/producer";
 import { env } from "expect-env";
 import { getOfflineIntervalMs, getTickIntervalMs } from "./config.js";
+import { runSchedulerLoop } from "./loop.js";
 import { runSchedulerTick } from "./run-tick.js";
 
 /**
@@ -17,6 +19,14 @@ import { runSchedulerTick } from "./run-tick.js";
  *
  * A tick failure crashes the process loudly rather than being swallowed —
  * compose's `restart: unless-stopped` brings it back for the next tick.
+ *
+ * Liveness is a heartbeat file, the same mechanism as a worker's poll loop (D1
+ * of docs/design/standalone-observability.md); loop.ts carries where in the loop
+ * it is written and why. Clearing the previous generation's file here is what
+ * extends that answer across a restart. A completed round is still not the same
+ * as mail arriving: the accounts that came due are enqueued for workers to
+ * fetch, and whether that fetch lands is what
+ * `remit_account_sync_age_seconds` measures.
  */
 
 const log = createLogger();
@@ -41,18 +51,36 @@ log.info(
 );
 
 const runLoop = async (): Promise<void> => {
+	// Neither call on the monitoring file may take the scheduler down with it. An
+	// unreadable or full /data/heartbeat is a reason to keep enqueuing mail, not
+	// to stop. A clear that fails leaves the previous generation's file, which
+	// ages out at the same threshold; a write that fails is the missed beat that
+	// is itself the signal. Two messages because they are different problems: one
+	// is a volume this container could not read at boot, the other a write it
+	// could not make this round.
+	const onClearError = (error: unknown): void => {
+		log.error({ error }, "Scheduled-sync heartbeat clear failed");
+	};
+	const onBeatError = (error: unknown): void => {
+		log.error({ error }, "Scheduled-sync heartbeat write failed");
+	};
+
+	await clearHeartbeats().catch(onClearError);
 	const { account } = await getClient();
-	for (;;) {
-		await runSchedulerTick({
+	await runSchedulerLoop({
+		tick: runSchedulerTick,
+		tickDeps: {
 			accountService: account,
 			sqsClient,
 			queueUrl: mailboxesQueueUrl,
 			log,
 			tickIntervalMs,
 			offlineIntervalMs,
-		});
-		await delay(tickIntervalMs);
-	}
+		},
+		heartbeat: createHeartbeat("tick"),
+		onHeartbeatError: onBeatError,
+		tickIntervalMs,
+	});
 };
 
 runLoop()