@remit/imap-worker 0.0.86 → 0.0.87

package/src/scheduler/attachment-sweep.test.ts

@@ -1,169 +0,0 @@
-/**
- * Both scheduler entry points must hand the tick a sweep.
- *
- * Only the hosted one strictly needs it — a browser PUTs straight to block
- * storage there, so nothing is in the path to refuse an upload against a draft
- * that is already gone — but a deployment that quietly stops collecting is
- * indistinguishable from one that has nothing to collect. Deleting either wiring
- * used to leave the suite green.
- */
-
-import assert from "node:assert/strict";
-import { readFileSync } from "node:fs";
-import { dirname, join } from "node:path";
-import { describe, it } from "node:test";
-import { fileURLToPath } from "node:url";
-import type { AccountItem } from "@remit/data-ports";
-import type { Logger } from "@remit/logger-lambda";
-import type { StorageService } from "@remit/storage-service";
-import { buildAttachmentSweep } from "./attachment-sweep.js";
-
-const here = dirname(fileURLToPath(import.meta.url));
-const TICK_TIME = 1_700_000_000_000;
-
-const silentLogger = (): Logger => {
-	const noop = () => {};
-	const log = {
-		info: noop,
-		warn: noop,
-		error: noop,
-		debug: noop,
-		fatal: noop,
-		trace: noop,
-		child: () => log,
-	} as unknown as Logger;
-	return log;
-};
-
-/** Enough of an account for the tick's eligibility checks to run. */
-const eligibleAccount = (accountId: string) =>
-	({
-		accountId,
-		accountConfigId: "cfg-1",
-		email: `${accountId}@example.com`,
-		username: accountId,
-		authType: "password",
-		syncedServices: ["Mail"],
-		imapHost: "imap.example.com",
-		imapPort: 993,
-		imapTls: true,
-		imapStartTls: false,
-		isActive: true,
-		connectionState: "authenticated",
-		// Freshly synced, so nothing is due and no SQS send is attempted. The
-		// sweep must still run: abandoned bytes have nothing to do with how
-		// recently mail was fetched.
-		lastSyncAt: TICK_TIME,
-		createdAt: 0,
-		updatedAt: 0,
-	}) as unknown as AccountItem;
-
-describe("the hosted tick sweeps", () => {
-	it("runs the sweep for every account, due a sync or not", async () => {
-		process.env.SQS_QUEUE_URL_MAILBOXES = "https://queue.test/mailboxes";
-
-		const swept: string[] = [];
-		const { setClient, _resetForTest } = await import("@remit/backend/client");
-		setClient({
-			account: {
-				listAllAccountsPage: async () => ({
-					items: [eligibleAccount("acc-1"), eligibleAccount("acc-2")],
-					cursor: null,
-				}),
-			},
-			storage: {
-				listOutboxDraftsWithAttachments: async (
-					_cfg: string,
-					accountId: string,
-				) => {
-					swept.push(accountId);
-					return [];
-				},
-			},
-			outboxAttachment: { reapAndListLive: async () => [] },
-		} as never);
-
-		try {
-			const { handler } = await import("./handler.js");
-			await handler(
-				{ time: new Date(TICK_TIME).toISOString() } as never,
-				{} as never,
-				() => {},
-			);
-			assert.deepEqual(swept.sort(), ["acc-1", "acc-2"]);
-		} finally {
-			_resetForTest();
-		}
-	});
-});
-
-describe("the self-host runner wires the sweep", () => {
-	it("hands runSchedulerTick a sweep", () => {
-		// runner.ts starts its loop on import, so it cannot be invoked from a test
-		// the way handler.ts can. This guards the wiring itself: without it,
-		// deleting the line leaves the suite green and self-host silently stops
-		// collecting.
-		const source = readFileSync(join(here, "runner.ts"), "utf8");
-		assert.match(source, /sweepAttachments:\s*buildAttachmentSweep\(/);
-	});
-});
-
-describe("buildAttachmentSweep", () => {
-	const account = {
-		accountId: "acc-1",
-		accountConfigId: "cfg-1",
-	} as unknown as AccountItem;
-
-	it("collects the objects the database does not name", async () => {
-		const deleted: string[] = [];
-		const storage = {
-			listOutboxDraftsWithAttachments: async () => ["draft-1"],
-			listOutboxAttachments: async () => [
-				{ outboxAttachmentId: "known", key: "k1", sizeBytes: 1 },
-				{ outboxAttachmentId: "orphan", key: "k2", sizeBytes: 1 },
-			],
-			deleteOutboxAttachment: async (
-				_cfg: string,
-				_acc: string,
-				_draft: string,
-				id: string,
-			) => {
-				deleted.push(id);
-			},
-		} as unknown as StorageService;
-
-		await buildAttachmentSweep(
-			{
-				storage,
-				outboxAttachment: {
-					reapAndListLive: async () => ["known"],
-				},
-			} as never,
-			silentLogger(),
-		)(account);
-
-		assert.deepEqual(deleted, ["orphan"]);
-	});
-
-	it("lets a storage failure out, so the tick can count it", async () => {
-		// Containment belongs to the tick, which keeps enqueuing mail regardless.
-		// Swallowing here would make a broken sweep look like a working one.
-		const storage = {
-			listOutboxDraftsWithAttachments: async () => {
-				throw new Error("storage root does not exist");
-			},
-		} as unknown as StorageService;
-
-		await assert.rejects(
-			() =>
-				buildAttachmentSweep(
-					{
-						storage,
-						outboxAttachment: { reapAndListLive: async () => [] },
-					} as never,
-					silentLogger(),
-				)(account),
-			/storage root does not exist/,
-		);
-	});
-});

package/src/scheduler/run-tick.test.ts

@@ -1,478 +0,0 @@
-import assert from "node:assert/strict";
-import { describe, it } from "node:test";
-import type { SendMessageCommand, SQSClient } from "@aws-sdk/client-sqs";
-import type { AccountItem, AccountSchedulerPage } from "@remit/data-ports";
-import { AccountService } from "@remit/domain-enums";
-import type { Logger } from "@remit/logger-lambda";
-import { runSchedulerTick } from "./run-tick.js";
-
-const TICK_INTERVAL_MS = 60 * 60 * 1000;
-const OFFLINE_INTERVAL_MS = 12 * 60 * 60 * 1000;
-const NOW = 1_700_000_000_000;
-
-const createNoopLogger = (): Logger => {
-	const noop = () => {};
-	const log = {
-		info: noop,
-		warn: noop,
-		error: noop,
-		debug: noop,
-		fatal: noop,
-		trace: noop,
-		child: () => log,
-	} as unknown as Logger;
-	return log;
-};
-
-const createCapturingLogger = (): {
-	log: Logger;
-	calls: { level: string; args: unknown[] }[];
-} => {
-	const calls: { level: string; args: unknown[] }[] = [];
-	const capture =
-		(level: string) =>
-		(...args: unknown[]) =>
-			calls.push({ level, args });
-	const log = {
-		info: capture("info"),
-		warn: capture("warn"),
-		error: capture("error"),
-		debug: capture("debug"),
-		fatal: capture("fatal"),
-		trace: capture("trace"),
-		child: () => log,
-	} as unknown as Logger;
-	return { log, calls };
-};
-
-const baseAccount = (overrides: Partial<AccountItem>): AccountItem =>
-	({
-		accountId: "acct_1",
-		accountConfigId: "acfg_1",
-		username: "user",
-		email: "user@example.com",
-		authType: "password",
-		passwordHash: '{"ciphertext":"x"}',
-		imapHost: "imap.example.com",
-		imapPort: 993,
-		imapTls: true,
-		imapStartTls: false,
-		smtpEnabled: false,
-		smtpHost: "",
-		smtpPort: 587,
-		smtpTls: false,
-		smtpStartTls: true,
-		smtpUsername: "",
-		syncedServices: [AccountService.Mail],
-		isActive: true,
-		connectionState: "authenticated",
-		createdAt: NOW - 1_000_000,
-		updatedAt: NOW - 1_000_000,
-		...overrides,
-	}) as AccountItem;
-
-const fakeAccountService = (pages: AccountSchedulerPage[]) => {
-	let call = 0;
-	return {
-		listAllAccountsPage: async (): Promise<AccountSchedulerPage> => {
-			const page = pages[call];
-			call++;
-			if (!page) throw new Error("no more pages configured");
-			return page;
-		},
-	};
-};
-
-const fakeSqsClient = (): {
-	sqsClient: SQSClient;
-	sent: SendMessageCommand[];
-} => {
-	const sent: SendMessageCommand[] = [];
-	const sqsClient = {
-		send: async (cmd: SendMessageCommand) => {
-			sent.push(cmd);
-			return {};
-		},
-	} as unknown as SQSClient;
-	return { sqsClient, sent };
-};
-
-describe("runSchedulerTick", () => {
-	it("enqueues only accounts that are due, and pages through every account", async () => {
-		const due = baseAccount({
-			accountId: "acct_due",
-			lastSyncAt: NOW - OFFLINE_INTERVAL_MS - 1,
-		});
-		const notDue = baseAccount({
-			accountId: "acct_not_due",
-			lastSyncAt: NOW - 60_000,
-		});
-
-		const accountService = fakeAccountService([
-			{ items: [due], cursor: "page2" },
-			{ items: [notDue], cursor: null },
-		]);
-		const { sqsClient, sent } = fakeSqsClient();
-
-		const result = await runSchedulerTick({
-			accountService,
-			sqsClient,
-			queueUrl:
-				"https://sqs.eu-west-1.amazonaws.com/123/remit-dev-mailboxes.fifo",
-			log: createNoopLogger(),
-			tickIntervalMs: TICK_INTERVAL_MS,
-			offlineIntervalMs: OFFLINE_INTERVAL_MS,
-			now: NOW,
-		});
-
-		assert.equal(result.scanned, 2);
-		assert.equal(result.enqueued, 1);
-		assert.equal(result.skipped, 1);
-		assert.equal(sent.length, 1);
-		const body = JSON.parse(sent[0]?.input.MessageBody ?? "{}");
-		assert.equal(body.accountId, "acct_due");
-	});
-
-	it("skips deleted, unsyncable-host, and reauth-required accounts", async () => {
-		const deleted = baseAccount({ accountId: "acct_deleted", deletedAt: NOW });
-		const unsyncable = baseAccount({
-			accountId: "acct_unsyncable",
-			imapHost: "mail.invalid",
-		});
-		const reauth = baseAccount({
-			accountId: "acct_reauth",
-			connectionState: "reauth_required",
-		});
-
-		const accountService = fakeAccountService([
-			{ items: [deleted, unsyncable, reauth], cursor: null },
-		]);
-		const { sqsClient, sent } = fakeSqsClient();
-
-		const result = await runSchedulerTick({
-			accountService,
-			sqsClient,
-			queueUrl:
-				"https://sqs.eu-west-1.amazonaws.com/123/remit-dev-mailboxes.fifo",
-			log: createNoopLogger(),
-			tickIntervalMs: TICK_INTERVAL_MS,
-			offlineIntervalMs: OFFLINE_INTERVAL_MS,
-			now: NOW,
-		});
-
-		assert.equal(result.enqueued, 0);
-		assert.equal(result.skipped, 3);
-		assert.equal(sent.length, 0);
-	});
-
-	it("skips an account storing no credential (issue #1120)", async () => {
-		const due = baseAccount({ accountId: "acct_due" });
-		const credentialless = baseAccount({
-			accountId: "acct_no_credential",
-			passwordHash: undefined,
-		});
-
-		const accountService = fakeAccountService([
-			{ items: [due, credentialless], cursor: null },
-		]);
-		const { sqsClient, sent } = fakeSqsClient();
-
-		const result = await runSchedulerTick({
-			accountService,
-			sqsClient,
-			queueUrl:
-				"https://sqs.eu-west-1.amazonaws.com/123/remit-dev-mailboxes.fifo",
-			log: createNoopLogger(),
-			tickIntervalMs: TICK_INTERVAL_MS,
-			offlineIntervalMs: OFFLINE_INTERVAL_MS,
-			now: NOW,
-		});
-
-		assert.equal(result.enqueued, 1);
-		assert.equal(result.skipped, 1);
-		assert.equal(sent.length, 1);
-		const body = JSON.parse(sent[0]?.input.MessageBody ?? "{}");
-		assert.equal(body.accountId, "acct_due");
-	});
-
-	it("never logs per-account for ineligible accounts — only the aggregate tick summary (review #1250)", async () => {
-		const deleted = baseAccount({ accountId: "acct_deleted", deletedAt: NOW });
-		const unsyncable = baseAccount({
-			accountId: "acct_unsyncable",
-			imapHost: "mail.invalid",
-		});
-		const reauth = baseAccount({
-			accountId: "acct_reauth",
-			connectionState: "reauth_required",
-		});
-
-		const accountService = fakeAccountService([
-			{ items: [deleted, unsyncable, reauth], cursor: null },
-		]);
-		const { sqsClient } = fakeSqsClient();
-		const { log, calls } = createCapturingLogger();
-
-		await runSchedulerTick({
-			accountService,
-			sqsClient,
-			queueUrl:
-				"https://sqs.eu-west-1.amazonaws.com/123/remit-dev-mailboxes.fifo",
-			log,
-			tickIntervalMs: TICK_INTERVAL_MS,
-			offlineIntervalMs: OFFLINE_INTERVAL_MS,
-			now: NOW,
-		});
-
-		// Sweeping 3 ineligible accounts must not produce 3 (or more) log
-		// lines through the tick's own logger — only the one aggregate
-		// "tick complete" summary. Per-account noise from the shared
-		// isAccountDeleted/isUnsyncableHost/isAccountReauthRequired helpers
-		// must be swallowed by a silent logger inside the sweep.
-		assert.equal(
-			calls.length,
-			1,
-			`expected exactly one aggregate log line, got ${calls.length}: ${JSON.stringify(calls)}`,
-		);
-	});
-
-	it("gives each enqueued account a scheduler-namespaced, time-bucketed dedup id", async () => {
-		const due = baseAccount({ accountId: "acct_due" });
-		const accountService = fakeAccountService([{ items: [due], cursor: null }]);
-		const { sqsClient, sent } = fakeSqsClient();
-
-		await runSchedulerTick({
-			accountService,
-			sqsClient,
-			queueUrl:
-				"https://sqs.eu-west-1.amazonaws.com/123/remit-dev-mailboxes.fifo",
-			log: createNoopLogger(),
-			tickIntervalMs: TICK_INTERVAL_MS,
-			offlineIntervalMs: OFFLINE_INTERVAL_MS,
-			now: NOW,
-		});
-
-		const dedupId = sent[0]?.input.MessageDeduplicationId;
-		assert.ok(dedupId?.startsWith("SYNC_MAILBOXES:scheduled:acct_due:"));
-		assert.notEqual(dedupId, "SYNC_MAILBOXES:acct_due");
-	});
-
-	it("terminates pagination and never revisits a page", async () => {
-		const accountService = fakeAccountService([
-			{ items: [], cursor: "page2" },
-			{ items: [], cursor: "page3" },
-			{ items: [], cursor: null },
-		]);
-		const { sqsClient } = fakeSqsClient();
-
-		const result = await runSchedulerTick({
-			accountService,
-			sqsClient,
-			queueUrl:
-				"https://sqs.eu-west-1.amazonaws.com/123/remit-dev-mailboxes.fifo",
-			log: createNoopLogger(),
-			tickIntervalMs: TICK_INTERVAL_MS,
-			offlineIntervalMs: OFFLINE_INTERVAL_MS,
-			now: NOW,
-		});
-
-		assert.equal(result.scanned, 0);
-		assert.equal(result.enqueued, 0);
-	});
-});
-
-describe("the attachment sweep inside a tick", () => {
-	const twoAccounts = () => [
-		{
-			items: [
-				baseAccount({ accountId: "acct_1", lastSyncAt: NOW - 1_000 }),
-				baseAccount({ accountId: "acct_2", lastSyncAt: 0 }),
-			],
-			cursor: null,
-		} as unknown as AccountSchedulerPage,
-	];
-
-	it("sweeps every account, not only the ones due a sync", async () => {
-		const { sqsClient } = fakeSqsClient();
-		const swept: string[] = [];
-
-		const result = await runSchedulerTick({
-			accountService: fakeAccountService(twoAccounts()),
-			sqsClient,
-			queueUrl: "https://queue.test/mailboxes",
-			log: createNoopLogger(),
-			tickIntervalMs: TICK_INTERVAL_MS,
-			offlineIntervalMs: OFFLINE_INTERVAL_MS,
-			now: NOW,
-			sweepAttachments: async (account) => {
-				swept.push(account.accountId);
-			},
-		});
-
-		assert.deepEqual(swept.sort(), ["acct_1", "acct_2"]);
-		assert.equal(result.swept, 2);
-		assert.equal(result.sweepFailed, 0);
-	});
-
-	it("keeps enqueuing mail when the sweep throws", async () => {
-		// The sweep is housekeeping; enqueuing mail is not. Uncontained, one
-		// storage error here reaches the loop, which exits, and the container
-		// restarts every few seconds without ever enqueuing an account again.
-		const { sqsClient, sent } = fakeSqsClient();
-		const { log, calls } = createCapturingLogger();
-
-		const result = await runSchedulerTick({
-			accountService: fakeAccountService(twoAccounts()),
-			sqsClient,
-			queueUrl: "https://queue.test/mailboxes",
-			log,
-			tickIntervalMs: TICK_INTERVAL_MS,
-			offlineIntervalMs: OFFLINE_INTERVAL_MS,
-			now: NOW,
-			sweepAttachments: async () => {
-				throw new Error("AccessDenied on ListObjectsV2");
-			},
-		});
-
-		assert.equal(result.sweepFailed, 2);
-		assert.equal(result.swept, 0);
-		// The tick still did its actual job.
-		assert.ok(sent.length > 0);
-		assert.equal(result.enqueued, sent.length);
-		assert.ok(
-			calls.some(
-				(entry) =>
-					entry.level === "error" &&
-					entry.args.some((arg) =>
-						String(arg).includes("Outbox attachment sweep failed"),
-					),
-			),
-		);
-	});
-
-	it("ticks without a sweep at all", async () => {
-		const { sqsClient } = fakeSqsClient();
-
-		const result = await runSchedulerTick({
-			accountService: fakeAccountService(twoAccounts()),
-			sqsClient,
-			queueUrl: "https://queue.test/mailboxes",
-			log: createNoopLogger(),
-			tickIntervalMs: TICK_INTERVAL_MS,
-			offlineIntervalMs: OFFLINE_INTERVAL_MS,
-			now: NOW,
-		});
-
-		assert.equal(result.swept, 0);
-		assert.equal(result.sweepFailed, 0);
-	});
-});
-
-describe("runSchedulerTick and the account's synced services", () => {
-	const tickWith = async (syncedServices: AccountItem["syncedServices"]) => {
-		const accountService = fakeAccountService([
-			{
-				items: [baseAccount({ accountId: "acct_1", syncedServices })],
-				cursor: null,
-			},
-		]);
-		const { sqsClient, sent } = fakeSqsClient();
-		const result = await runSchedulerTick({
-			accountService,
-			sqsClient,
-			queueUrl: "https://queue.test/mailboxes",
-			log: createNoopLogger(),
-			tickIntervalMs: TICK_INTERVAL_MS,
-			offlineIntervalMs: OFFLINE_INTERVAL_MS,
-			now: NOW,
-		});
-		return { result, sent };
-	};
-
-	it("enqueues mail sync for a mail-only account", async () => {
-		const { result, sent } = await tickWith([AccountService.Mail]);
-
-		assert.equal(result.enqueued, 1);
-		assert.equal(sent.length, 1);
-	});
-
-	it("enqueues mail sync for an account syncing mail and calendar", async () => {
-		const { result, sent } = await tickWith([
-			AccountService.Mail,
-			AccountService.Calendar,
-		]);
-
-		assert.equal(result.enqueued, 1);
-		assert.equal(sent.length, 1);
-	});
-
-	it("skips a calendar-only account", async () => {
-		const { result, sent } = await tickWith([AccountService.Calendar]);
-
-		assert.equal(result.enqueued, 0);
-		assert.equal(result.skipped, 1);
-		assert.equal(sent.length, 0);
-	});
-});
-
-describe("the calendar subscription refresh inside a tick", () => {
-	const oneAccount = () => [
-		{
-			items: [baseAccount({ accountId: "acct_1", lastSyncAt: 0 })],
-			cursor: null,
-		} as unknown as AccountSchedulerPage,
-	];
-
-	it("runs once per tick, against the offline interval", async () => {
-		const { sqsClient } = fakeSqsClient();
-		const calls: [number, number][] = [];
-
-		const result = await runSchedulerTick({
-			accountService: fakeAccountService(oneAccount()),
-			sqsClient,
-			queueUrl: "https://queue.test/mailboxes",
-			log: createNoopLogger(),
-			tickIntervalMs: TICK_INTERVAL_MS,
-			offlineIntervalMs: OFFLINE_INTERVAL_MS,
-			now: NOW,
-			refreshCalendarSubscriptions: async (now, intervalMs) => {
-				calls.push([now, intervalMs]);
-				return { refreshed: 2, failed: 1, notDue: 4 };
-			},
-		});
-
-		assert.deepEqual(calls, [[NOW, OFFLINE_INTERVAL_MS]]);
-		assert.equal(result.subscriptionsRefreshed, 2);
-		assert.equal(result.subscriptionsFailed, 1);
-		assert.equal(result.enqueued, 1);
-	});
-
-	it("keeps the tick alive when the refresh throws", async () => {
-		const { sqsClient, sent } = fakeSqsClient();
-		const { log, calls } = createCapturingLogger();
-
-		const result = await runSchedulerTick({
-			accountService: fakeAccountService(oneAccount()),
-			sqsClient,
-			queueUrl: "https://queue.test/mailboxes",
-			log,
-			tickIntervalMs: TICK_INTERVAL_MS,
-			offlineIntervalMs: OFFLINE_INTERVAL_MS,
-			now: NOW,
-			refreshCalendarSubscriptions: async () => {
-				throw new Error("database is locked");
-			},
-		});
-
-		assert.equal(sent.length, 1);
-		assert.equal(result.subscriptionsRefreshed, 0);
-		assert.ok(
-			calls.some(
-				(entry) =>
-					entry.level === "error" &&
-					entry.args.some((arg) =>
-						String(arg).includes("Calendar subscription refresh failed"),
-					),
-			),
-		);
-	});
-});