@remit/imap-worker 0.0.33 → 0.0.34

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@remit/imap-worker",
-  "version": "0.0.33",
+  "version": "0.0.34",
   "type": "module",
   "main": "dist/index.js",
   "exports": {

package/src/scheduler/loop.test.ts

@@ -0,0 +1,133 @@
+import assert from "node:assert/strict";
+import { describe, it } from "node:test";
+import { runSchedulerLoop, type SchedulerLoopOptions } from "./loop.js";
+import type { RunSchedulerTickDeps, SchedulerTickResult } from "./run-tick.js";
+
+const RESULT: SchedulerTickResult = { scanned: 0, enqueued: 0, skipped: 0 };
+
+// Only the loop's own two calls matter here; the tick is a stub, so what it is
+// handed is never read.
+const TICK_DEPS = {} as RunSchedulerTickDeps;
+
+const buildLoop = (
+	overrides: Partial<SchedulerLoopOptions>,
+): SchedulerLoopOptions => ({
+	tick: () => Promise.resolve(RESULT),
+	tickDeps: TICK_DEPS,
+	heartbeat: () => Promise.resolve(),
+	onHeartbeatError: () => {},
+	tickIntervalMs: 1000,
+	wait: () => Promise.resolve(),
+	...overrides,
+});
+
+describe("runSchedulerLoop", () => {
+	it("writes no heartbeat for a tick that throws", async () => {
+		let beats = 0;
+		await assert.rejects(
+			runSchedulerLoop(
+				buildLoop({
+					tick: () => Promise.reject(new Error("tick failed")),
+					heartbeat: () => {
+						beats += 1;
+						return Promise.resolve();
+					},
+				}),
+			),
+			/tick failed/,
+		);
+		assert.equal(beats, 0);
+	});
+
+	it("beats once per completed round, and stops beating when a round stops completing", async () => {
+		let rounds = 0;
+		let beats = 0;
+		await assert.rejects(
+			runSchedulerLoop(
+				buildLoop({
+					tick: () => {
+						rounds += 1;
+						return rounds > 2
+							? Promise.reject(new Error("tick failed"))
+							: Promise.resolve(RESULT);
+					},
+					heartbeat: () => {
+						beats += 1;
+						return Promise.resolve();
+					},
+				}),
+			),
+			/tick failed/,
+		);
+		assert.equal(beats, 2);
+	});
+
+	it("keeps ticking when the heartbeat write fails", async () => {
+		const errors: unknown[] = [];
+		let rounds = 0;
+		await assert.rejects(
+			runSchedulerLoop(
+				buildLoop({
+					tick: () => {
+						rounds += 1;
+						return rounds > 3
+							? Promise.reject(new Error("done"))
+							: Promise.resolve(RESULT);
+					},
+					heartbeat: () => Promise.reject(new Error("ENOSPC")),
+					onHeartbeatError: (error) => errors.push(error),
+				}),
+			),
+			/done/,
+		);
+		assert.equal(rounds, 4);
+		assert.equal(errors.length, 3);
+	});
+
+	it("waits a tick interval between rounds", async () => {
+		const waited: number[] = [];
+		let rounds = 0;
+		await assert.rejects(
+			runSchedulerLoop(
+				buildLoop({
+					tickIntervalMs: 300_000,
+					tick: () => {
+						rounds += 1;
+						return rounds > 2
+							? Promise.reject(new Error("done"))
+							: Promise.resolve(RESULT);
+					},
+					wait: (ms) => {
+						waited.push(ms);
+						return Promise.resolve();
+					},
+				}),
+			),
+			/done/,
+		);
+		assert.deepEqual(waited, [300_000, 300_000]);
+	});
+
+	// The default is the only one production uses, so it is the one a wrong unit
+	// or a dropped argument would ship in.
+	it("sleeps the real timer when no wait is injected", async () => {
+		let rounds = 0;
+		const started = Date.now();
+		await assert.rejects(
+			runSchedulerLoop({
+				...buildLoop({
+					tickIntervalMs: 25,
+					tick: () => {
+						rounds += 1;
+						return rounds > 2
+							? Promise.reject(new Error("done"))
+							: Promise.resolve(RESULT);
+					},
+				}),
+				wait: undefined,
+			}),
+			/done/,
+		);
+		assert.ok(Date.now() - started >= 50);
+	});
+});

package/src/scheduler/loop.ts

@@ -0,0 +1,38 @@
+import { setTimeout as delay } from "node:timers/promises";
+import type { Heartbeat } from "@remit/sqs-client/heartbeat";
+import type { RunSchedulerTickDeps, SchedulerTickResult } from "./run-tick.js";
+
+/**
+ * The scheduled-sync loop, separated from the wiring in runner.ts for the same
+ * reason `runQueuePoller` is separate from the container that starts it: the
+ * order of the two calls below is a property worth holding, and a module that
+ * connects to a queue and a database at import time cannot be asked about it.
+ *
+ * The beat lands after the tick returns, never before. A tick that throws on
+ * every pass exits the process and is restarted every few seconds, so a beat
+ * written on the way in would report a scheduler that enqueues nothing as
+ * healthy for as long as it kept crashing.
+ */
+export interface SchedulerLoopOptions {
+	readonly tick: (deps: RunSchedulerTickDeps) => Promise<SchedulerTickResult>;
+	readonly tickDeps: RunSchedulerTickDeps;
+	readonly heartbeat: Heartbeat;
+	readonly onHeartbeatError: (error: unknown) => void;
+	readonly tickIntervalMs: number;
+	readonly wait?: (ms: number) => Promise<void>;
+}
+
+export const runSchedulerLoop = async ({
+	tick,
+	tickDeps,
+	heartbeat,
+	onHeartbeatError,
+	tickIntervalMs,
+	wait = (ms) => delay(ms),
+}: SchedulerLoopOptions): Promise<never> => {
+	for (;;) {
+		await tick(tickDeps);
+		await heartbeat().catch(onHeartbeatError);
+		await wait(tickIntervalMs);
+	}
+};

package/src/scheduler/runner.ts

@@ -2,9 +2,11 @@
 import { setTimeout as delay } from "node:timers/promises";
 import { getClient } from "@remit/backend/client";
 import { createLogger } from "@remit/logger-lambda";
+import { clearHeartbeats, createHeartbeat } from "@remit/sqs-client/heartbeat";
 import { createQueueProducer } from "@remit/sqs-client/producer";
 import { env } from "expect-env";
 import { getOfflineIntervalMs, getTickIntervalMs } from "./config.js";
+import { runSchedulerLoop } from "./loop.js";
 import { runSchedulerTick } from "./run-tick.js";
 
 /**
@@ -17,6 +19,14 @@ import { runSchedulerTick } from "./run-tick.js";
  *
  * A tick failure crashes the process loudly rather than being swallowed —
  * compose's `restart: unless-stopped` brings it back for the next tick.
+ *
+ * Liveness is a heartbeat file, the same mechanism as a worker's poll loop (D1
+ * of docs/design/standalone-observability.md); loop.ts carries where in the loop
+ * it is written and why. Clearing the previous generation's file here is what
+ * extends that answer across a restart. A completed round is still not the same
+ * as mail arriving: the accounts that came due are enqueued for workers to
+ * fetch, and whether that fetch lands is what
+ * `remit_account_sync_age_seconds` measures.
  */
 
 const log = createLogger();
@@ -41,18 +51,36 @@ log.info(
 );
 
 const runLoop = async (): Promise<void> => {
+	// Neither call on the monitoring file may take the scheduler down with it. An
+	// unreadable or full /data/heartbeat is a reason to keep enqueuing mail, not
+	// to stop. A clear that fails leaves the previous generation's file, which
+	// ages out at the same threshold; a write that fails is the missed beat that
+	// is itself the signal. Two messages because they are different problems: one
+	// is a volume this container could not read at boot, the other a write it
+	// could not make this round.
+	const onClearError = (error: unknown): void => {
+		log.error({ error }, "Scheduled-sync heartbeat clear failed");
+	};
+	const onBeatError = (error: unknown): void => {
+		log.error({ error }, "Scheduled-sync heartbeat write failed");
+	};
+
+	await clearHeartbeats().catch(onClearError);
 	const { account } = await getClient();
-	for (;;) {
-		await runSchedulerTick({
+	await runSchedulerLoop({
+		tick: runSchedulerTick,
+		tickDeps: {
 			accountService: account,
 			sqsClient,
 			queueUrl: mailboxesQueueUrl,
 			log,
 			tickIntervalMs,
 			offlineIntervalMs,
-		});
-		await delay(tickIntervalMs);
-	}
+		},
+		heartbeat: createHeartbeat("tick"),
+		onHeartbeatError: onBeatError,
+		tickIntervalMs,
+	});
 };
 
 runLoop()