@remit/imap-worker 0.0.32 → 0.0.34

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@remit/imap-worker",
-  "version": "0.0.32",
+  "version": "0.0.34",
   "type": "module",
   "main": "dist/index.js",
   "exports": {

package/src/scheduler/config.ts

@@ -6,15 +6,20 @@
  * in remit-account-worker/src/config.ts).
  *
  * `tickIntervalSeconds` drives how often the tick itself runs — the
- * EventBridge schedule rate in prod, the local-runner loop delay in dev — and
- * must stay well below `offlineIntervalSeconds` so a tick reliably observes
- * every account crossing the threshold. CDK and this runtime default must
- * agree — see infra/lib/config.ts's `mailboxSync` stage config.
+ * EventBridge schedule rate on a managed deployment, the runner's loop delay
+ * everywhere else — and must stay well below `offlineIntervalSeconds` so a tick
+ * reliably observes every account crossing the threshold. CDK and this runtime
+ * default must agree — see infra/lib/config.ts's `mailboxSync` stage config.
  *
  * `offlineIntervalSeconds` is the only due-ness threshold: an account is due
  * once its last successful sync is older than this interval. There is no
  * "online" tier — client-side polling (useStaleAccountSync) covers an
  * account while its mail is actively open in the web client.
+ *
+ * The defaults below are a floor for a deployment that sets neither. The
+ * standalone stack sets both, at a cadence sized for one box and a handful of
+ * accounts, and the checker's stall threshold is derived from those values —
+ * see deploy/vps/docker-compose.sqlite.yml's `scheduler` service.
  */
 
 const DEFAULT_TICK_INTERVAL_SECONDS = 60 * 60; // 1 hour

package/src/scheduler/handler.ts

@@ -16,8 +16,8 @@ const sqsClient = createQueueProducer({ queueUrl: mailboxesQueueUrl });
  * (#1247, restructured #1251). Ticks at `MAILBOX_SYNC_TICK_INTERVAL_SECONDS`
  * (rate schedule, wired in infra/stacks/dev/stacks/remit-worker-stack.ts) and
  * delegates the actual decision + enqueue to `runSchedulerTick` — the same
- * function the local dev-stack timer loop calls (see `local-runner.ts`), so
- * production and local dev run one code path.
+ * function the timer loop calls on a deployment with no EventBridge (see
+ * `runner.ts`), so every deployment runs one code path.
  *
  * Uses the EventBridge event's own `time` (the scheduled fire time) as the
  * tick's `now`, rather than `Date.now()` at processing time. This is what

package/src/scheduler/loop.test.ts

@@ -0,0 +1,133 @@
+import assert from "node:assert/strict";
+import { describe, it } from "node:test";
+import { runSchedulerLoop, type SchedulerLoopOptions } from "./loop.js";
+import type { RunSchedulerTickDeps, SchedulerTickResult } from "./run-tick.js";
+
+const RESULT: SchedulerTickResult = { scanned: 0, enqueued: 0, skipped: 0 };
+
+// Only the loop's own two calls matter here; the tick is a stub, so what it is
+// handed is never read.
+const TICK_DEPS = {} as RunSchedulerTickDeps;
+
+const buildLoop = (
+	overrides: Partial<SchedulerLoopOptions>,
+): SchedulerLoopOptions => ({
+	tick: () => Promise.resolve(RESULT),
+	tickDeps: TICK_DEPS,
+	heartbeat: () => Promise.resolve(),
+	onHeartbeatError: () => {},
+	tickIntervalMs: 1000,
+	wait: () => Promise.resolve(),
+	...overrides,
+});
+
+describe("runSchedulerLoop", () => {
+	it("writes no heartbeat for a tick that throws", async () => {
+		let beats = 0;
+		await assert.rejects(
+			runSchedulerLoop(
+				buildLoop({
+					tick: () => Promise.reject(new Error("tick failed")),
+					heartbeat: () => {
+						beats += 1;
+						return Promise.resolve();
+					},
+				}),
+			),
+			/tick failed/,
+		);
+		assert.equal(beats, 0);
+	});
+
+	it("beats once per completed round, and stops beating when a round stops completing", async () => {
+		let rounds = 0;
+		let beats = 0;
+		await assert.rejects(
+			runSchedulerLoop(
+				buildLoop({
+					tick: () => {
+						rounds += 1;
+						return rounds > 2
+							? Promise.reject(new Error("tick failed"))
+							: Promise.resolve(RESULT);
+					},
+					heartbeat: () => {
+						beats += 1;
+						return Promise.resolve();
+					},
+				}),
+			),
+			/tick failed/,
+		);
+		assert.equal(beats, 2);
+	});
+
+	it("keeps ticking when the heartbeat write fails", async () => {
+		const errors: unknown[] = [];
+		let rounds = 0;
+		await assert.rejects(
+			runSchedulerLoop(
+				buildLoop({
+					tick: () => {
+						rounds += 1;
+						return rounds > 3
+							? Promise.reject(new Error("done"))
+							: Promise.resolve(RESULT);
+					},
+					heartbeat: () => Promise.reject(new Error("ENOSPC")),
+					onHeartbeatError: (error) => errors.push(error),
+				}),
+			),
+			/done/,
+		);
+		assert.equal(rounds, 4);
+		assert.equal(errors.length, 3);
+	});
+
+	it("waits a tick interval between rounds", async () => {
+		const waited: number[] = [];
+		let rounds = 0;
+		await assert.rejects(
+			runSchedulerLoop(
+				buildLoop({
+					tickIntervalMs: 300_000,
+					tick: () => {
+						rounds += 1;
+						return rounds > 2
+							? Promise.reject(new Error("done"))
+							: Promise.resolve(RESULT);
+					},
+					wait: (ms) => {
+						waited.push(ms);
+						return Promise.resolve();
+					},
+				}),
+			),
+			/done/,
+		);
+		assert.deepEqual(waited, [300_000, 300_000]);
+	});
+
+	// The default is the only one production uses, so it is the one a wrong unit
+	// or a dropped argument would ship in.
+	it("sleeps the real timer when no wait is injected", async () => {
+		let rounds = 0;
+		const started = Date.now();
+		await assert.rejects(
+			runSchedulerLoop({
+				...buildLoop({
+					tickIntervalMs: 25,
+					tick: () => {
+						rounds += 1;
+						return rounds > 2
+							? Promise.reject(new Error("done"))
+							: Promise.resolve(RESULT);
+					},
+				}),
+				wait: undefined,
+			}),
+			/done/,
+		);
+		assert.ok(Date.now() - started >= 50);
+	});
+});

package/src/scheduler/loop.ts

@@ -0,0 +1,38 @@
+import { setTimeout as delay } from "node:timers/promises";
+import type { Heartbeat } from "@remit/sqs-client/heartbeat";
+import type { RunSchedulerTickDeps, SchedulerTickResult } from "./run-tick.js";
+
+/**
+ * The scheduled-sync loop, separated from the wiring in runner.ts for the same
+ * reason `runQueuePoller` is separate from the container that starts it: the
+ * order of the two calls below is a property worth holding, and a module that
+ * connects to a queue and a database at import time cannot be asked about it.
+ *
+ * The beat lands after the tick returns, never before. A tick that throws on
+ * every pass exits the process and is restarted every few seconds, so a beat
+ * written on the way in would report a scheduler that enqueues nothing as
+ * healthy for as long as it kept crashing.
+ */
+export interface SchedulerLoopOptions {
+	readonly tick: (deps: RunSchedulerTickDeps) => Promise<SchedulerTickResult>;
+	readonly tickDeps: RunSchedulerTickDeps;
+	readonly heartbeat: Heartbeat;
+	readonly onHeartbeatError: (error: unknown) => void;
+	readonly tickIntervalMs: number;
+	readonly wait?: (ms: number) => Promise<void>;
+}
+
+export const runSchedulerLoop = async ({
+	tick,
+	tickDeps,
+	heartbeat,
+	onHeartbeatError,
+	tickIntervalMs,
+	wait = (ms) => delay(ms),
+}: SchedulerLoopOptions): Promise<never> => {
+	for (;;) {
+		await tick(tickDeps);
+		await heartbeat().catch(onHeartbeatError);
+		await wait(tickIntervalMs);
+	}
+};

package/src/scheduler/runner.ts

@@ -0,0 +1,93 @@
+#!/usr/bin/env node
+import { setTimeout as delay } from "node:timers/promises";
+import { getClient } from "@remit/backend/client";
+import { createLogger } from "@remit/logger-lambda";
+import { clearHeartbeats, createHeartbeat } from "@remit/sqs-client/heartbeat";
+import { createQueueProducer } from "@remit/sqs-client/producer";
+import { env } from "expect-env";
+import { getOfflineIntervalMs, getTickIntervalMs } from "./config.js";
+import { runSchedulerLoop } from "./loop.js";
+import { runSchedulerTick } from "./run-tick.js";
+
+/**
+ * The scheduled-sync runner for every deployment that has no EventBridge: the
+ * self-host stack's `scheduler` service and the local dev compose stack. A
+ * managed deployment fires `runSchedulerTick` off an EventBridge schedule (see
+ * handler.ts); this process ticks the same function on a plain loop at the same
+ * `MAILBOX_SYNC_TICK_INTERVAL_SECONDS` cadence, so there is one scheduling
+ * implementation and one set of knobs.
+ *
+ * A tick failure crashes the process loudly rather than being swallowed —
+ * compose's `restart: unless-stopped` brings it back for the next tick.
+ *
+ * Liveness is a heartbeat file, the same mechanism as a worker's poll loop (D1
+ * of docs/design/standalone-observability.md); loop.ts carries where in the loop
+ * it is written and why. Clearing the previous generation's file here is what
+ * extends that answer across a restart. A completed round is still not the same
+ * as mail arriving: the accounts that came due are enqueued for workers to
+ * fetch, and whether that fetch lands is what
+ * `remit_account_sync_age_seconds` measures.
+ */
+
+const log = createLogger();
+
+const mailboxesQueueUrl = env.SQS_QUEUE_URL_MAILBOXES;
+const sqsClient = createQueueProducer({ queueUrl: mailboxesQueueUrl });
+
+const tickIntervalMs = getTickIntervalMs();
+const offlineIntervalMs = getOfflineIntervalMs();
+
+// A persistent failure (e.g. Postgres not up yet at container boot) throws
+// before the loop ever reaches its own `delay`, so `restart: unless-stopped`
+// would otherwise respawn the process immediately — a tight, log-flooding
+// crash loop (review #1250). This fixed pause before exiting is not retry
+// logic (there is nothing to retry here; the container restart IS the
+// retry) — it only paces how fast that restart can happen.
+const CRASH_BACKOFF_MS = 5_000;
+
+log.info(
+	{ tickIntervalMs, offlineIntervalMs },
+	"Scheduled-sync runner started",
+);
+
+const runLoop = async (): Promise<void> => {
+	// Neither call on the monitoring file may take the scheduler down with it. An
+	// unreadable or full /data/heartbeat is a reason to keep enqueuing mail, not
+	// to stop. A clear that fails leaves the previous generation's file, which
+	// ages out at the same threshold; a write that fails is the missed beat that
+	// is itself the signal. Two messages because they are different problems: one
+	// is a volume this container could not read at boot, the other a write it
+	// could not make this round.
+	const onClearError = (error: unknown): void => {
+		log.error({ error }, "Scheduled-sync heartbeat clear failed");
+	};
+	const onBeatError = (error: unknown): void => {
+		log.error({ error }, "Scheduled-sync heartbeat write failed");
+	};
+
+	await clearHeartbeats().catch(onClearError);
+	const { account } = await getClient();
+	await runSchedulerLoop({
+		tick: runSchedulerTick,
+		tickDeps: {
+			accountService: account,
+			sqsClient,
+			queueUrl: mailboxesQueueUrl,
+			log,
+			tickIntervalMs,
+			offlineIntervalMs,
+		},
+		heartbeat: createHeartbeat("tick"),
+		onHeartbeatError: onBeatError,
+		tickIntervalMs,
+	});
+};
+
+runLoop()
+	.catch(async (error) => {
+		log.error({ error }, "Scheduled-sync tick failed");
+		await delay(CRASH_BACKOFF_MS);
+	})
+	.finally(() => {
+		process.exit(1);
+	});

package/src/scheduler/local-runner.ts

@@ -1,68 +0,0 @@
-#!/usr/bin/env node
-import { setTimeout as delay } from "node:timers/promises";
-import { getClient } from "@remit/backend/client";
-import { createLogger } from "@remit/logger-lambda";
-import { createQueueProducer } from "@remit/sqs-client/producer";
-import { env } from "expect-env";
-import { getOfflineIntervalMs, getTickIntervalMs } from "./config.js";
-import { runSchedulerTick } from "./run-tick.js";
-
-/**
- * Standalone scheduled-sync runner for the local pg-dev docker-compose stack
- * (#1247, restructured #1251). Production ticks `runSchedulerTick` off an
- * EventBridge schedule (see handler.ts); ElasticMQ/the pg-dev stack has no
- * EventBridge, so this process ticks on a plain loop at the same
- * `MAILBOX_SYNC_TICK_INTERVAL_SECONDS` cadence instead — same function, same
- * config knobs, so local dev behaves like production rather than needing its
- * own scheduling logic.
- *
- * This is a dev-only harness, not production code: like
- * `e2e-processor-shim.ts`, a tick failure crashes the process loudly rather
- * than swallowing it — docker-compose's `restart: unless-stopped` brings it
- * back for the next tick.
- */
-
-const log = createLogger();
-
-const mailboxesQueueUrl = env.SQS_QUEUE_URL_MAILBOXES;
-const sqsClient = createQueueProducer({ queueUrl: mailboxesQueueUrl });
-
-const tickIntervalMs = getTickIntervalMs();
-const offlineIntervalMs = getOfflineIntervalMs();
-
-// A persistent failure (e.g. Postgres not up yet at container boot) throws
-// before the loop ever reaches its own `delay`, so `restart: unless-stopped`
-// would otherwise respawn the process immediately — a tight, log-flooding
-// crash loop (review #1250). This fixed pause before exiting is not retry
-// logic (there is nothing to retry here; the container restart IS the
-// retry) — it only paces how fast that restart can happen.
-const CRASH_BACKOFF_MS = 5_000;
-
-log.info(
-	{ tickIntervalMs, offlineIntervalMs },
-	"Local scheduled-sync runner started",
-);
-
-const runLoop = async (): Promise<void> => {
-	const { account } = await getClient();
-	for (;;) {
-		await runSchedulerTick({
-			accountService: account,
-			sqsClient,
-			queueUrl: mailboxesQueueUrl,
-			log,
-			tickIntervalMs,
-			offlineIntervalMs,
-		});
-		await delay(tickIntervalMs);
-	}
-};
-
-runLoop()
-	.catch(async (error) => {
-		log.error({ error }, "Scheduled-sync tick failed");
-		await delay(CRASH_BACKOFF_MS);
-	})
-	.finally(() => {
-		process.exit(1);
-	});