@remit/imap-worker 0.0.86 → 0.0.87

package/src/scheduler/{calendar-subscriptions.test.ts → calendar-subscriptions.sqlite.test.ts}

@@ -1,10 +1,10 @@
 import assert from "node:assert/strict";
 import { describe, it } from "node:test";
 import type { CalendarFeedFetcher } from "@remit/calendar-service";
-import { MemoryCalendarStore } from "@remit/calendar-service/memory-store";
 import type { CalendarCollectionItem } from "@remit/data-ports";
 import { CalendarSource } from "@remit/domain-enums";
-import type { Logger } from "@remit/logger-lambda";
+import type { Logger } from "@remit/logger";
+import { openStore } from "../test-helpers/test-world.js";
 import { refreshDueCalendarSubscriptions } from "./calendar-subscriptions.js";
 
 const INTERVAL_MS = 15 * 60 * 1000;
@@ -41,27 +41,25 @@ const logged = (): { log: Logger; lines: string[] } => {
 	return { log, lines };
 };
 
+type Client = Awaited<ReturnType<typeof openStore>>["client"];
+
 const subscription = async (
-	store: MemoryCalendarStore,
+	client: Client,
 	urlSegment: string,
 	checkedAt: number,
 ): Promise<CalendarCollectionItem> => {
-	const created = await store.transaction((repos) =>
-		repos.calendarCollection.create({
-			accountConfigId: "account-config-1",
-			urlSegment,
-			displayName: urlSegment,
-			source: CalendarSource.Subscribed,
-			subscriptionUrl: `https://calendar.example/${urlSegment}/private-feedfacecafe.ics`,
-			subscriptionEnabled: true,
-		}),
-	);
-	return store.transaction((repos) =>
-		repos.calendarCollection.update(
-			created.accountConfigId,
-			created.calendarId,
-			{ subscriptionCheckedAt: checkedAt },
-		),
+	const created = await client.calendarCollection.create({
+		accountConfigId: "account-config-1",
+		urlSegment,
+		displayName: urlSegment,
+		source: CalendarSource.Subscribed,
+		subscriptionUrl: `https://calendar.example/${urlSegment}/private-feedfacecafe.ics`,
+		subscriptionEnabled: true,
+	});
+	return client.calendarCollection.update(
+		created.accountConfigId,
+		created.calendarId,
+		{ subscriptionCheckedAt: checkedAt },
 	);
 };
 
@@ -73,29 +71,23 @@ const recording =
 	};
 
 describe("refreshDueCalendarSubscriptions", () => {
-	it("fetches the subscriptions that are due and skips the rest", async () => {
-		const store = new MemoryCalendarStore();
-		const due = await subscription(store, "due", NOW - INTERVAL_MS);
-		await subscription(store, "fresh", NOW - 60_000);
-		const paused = await subscription(store, "paused", 0);
-		await store.transaction((repos) =>
-			repos.calendarCollection.update(
-				paused.accountConfigId,
-				paused.calendarId,
-				{ subscriptionEnabled: false },
-			),
+	it("fetches the subscriptions that are due and skips the rest", async (t) => {
+		const { client, close } = await openStore();
+		t.after(close);
+		const due = await subscription(client, "due", NOW - INTERVAL_MS);
+		await subscription(client, "fresh", NOW - 60_000);
+		const paused = await subscription(client, "paused", 0);
+		await client.calendarCollection.update(
+			paused.accountConfigId,
+			paused.calendarId,
+			{ subscriptionEnabled: false },
 		);
 		const asked: string[] = [];
 
 		const tally = await refreshDueCalendarSubscriptions(
 			{
-				calendarCollection: {
-					listEnabledSubscriptions: () =>
-						store.transaction((repos) =>
-							repos.calendarCollection.listEnabledSubscriptions(),
-						),
-				},
-				calendarUnitOfWork: store,
+				calendarCollection: client.calendarCollection,
+				calendarStore: client.calendarStore,
 				log: logged().log,
 				fetcher: recording(asked, () => new Response(FEED, { status: 200 })),
 			},
@@ -105,30 +97,24 @@ describe("refreshDueCalendarSubscriptions", () => {
 
 		assert.deepEqual(asked, [due.subscriptionUrl]);
 		assert.deepEqual(tally, { refreshed: 1, failed: 0, notDue: 1 });
-		const objects = await store.transaction((repos) =>
-			repos.calendarObject.listByCalendar(due.calendarId),
-		);
+		const objects = await client.calendarObject.listByCalendar(due.calendarId);
 		assert.deepEqual(
 			objects.map((object) => object.summary),
 			["Ferry crossing"],
 		);
 	});
 
-	it("counts a feed that fails, keeps refreshing the others, and never logs an address", async () => {
-		const store = new MemoryCalendarStore();
-		const broken = await subscription(store, "broken", 0);
-		const working = await subscription(store, "working", 0);
+	it("counts a feed that fails, keeps refreshing the others, and never logs an address", async (t) => {
+		const { client, close } = await openStore();
+		t.after(close);
+		const broken = await subscription(client, "broken", 0);
+		const working = await subscription(client, "working", 0);
 		const { log, lines } = logged();
 
 		const tally = await refreshDueCalendarSubscriptions(
 			{
-				calendarCollection: {
-					listEnabledSubscriptions: () =>
-						store.transaction((repos) =>
-							repos.calendarCollection.listEnabledSubscriptions(),
-						),
-				},
-				calendarUnitOfWork: store,
+				calendarCollection: client.calendarCollection,
+				calendarStore: client.calendarStore,
 				log,
 				fetcher: recording([], (url) =>
 					url === broken.subscriptionUrl
@@ -141,13 +127,15 @@ describe("refreshDueCalendarSubscriptions", () => {
 		);
 
 		assert.deepEqual(tally, { refreshed: 1, failed: 1, notDue: 0 });
-		const stored = await store.transaction((repos) =>
-			repos.calendarCollection.get(broken.accountConfigId, broken.calendarId),
+		const stored = await client.calendarCollection.get(
+			broken.accountConfigId,
+			broken.calendarId,
 		);
 		assert.equal(stored.subscriptionError, "the feed answered HTTP 410");
 		assert.equal(stored.subscriptionCheckedAt, NOW);
-		const refreshed = await store.transaction((repos) =>
-			repos.calendarCollection.get(working.accountConfigId, working.calendarId),
+		const refreshed = await client.calendarCollection.get(
+			working.accountConfigId,
+			working.calendarId,
 		);
 		assert.equal(refreshed.subscriptionFetchedAt, NOW);
 		assert.ok(lines.some((line) => line.includes(broken.calendarId)));

package/src/scheduler/calendar-subscriptions.ts

@@ -6,9 +6,9 @@ import {
 } from "@remit/calendar-service";
 import type {
 	ICalendarCollectionRepository,
-	ICalendarUnitOfWork,
+	ICalendarStore,
 } from "@remit/data-ports";
-import type { Logger } from "@remit/logger-lambda";
+import type { Logger } from "@remit/logger";
 import pMap from "p-map";
 
 const REFRESH_CONCURRENCY = 4;
@@ -24,7 +24,7 @@ export interface CalendarSubscriptionRefreshDeps {
 		ICalendarCollectionRepository,
 		"listEnabledSubscriptions"
 	>;
-	calendarUnitOfWork: ICalendarUnitOfWork;
+	calendarStore: ICalendarStore;
 	log: Logger;
 	fetcher?: CalendarFeedFetcher;
 }
@@ -52,7 +52,7 @@ export const refreshDueCalendarSubscriptions = async (
 		due,
 		(collection) =>
 			refreshCalendarSubscription(
-				deps.calendarUnitOfWork,
+				deps.calendarStore,
 				collection,
 				now,
 				deps.fetcher,
@@ -85,7 +85,7 @@ export const refreshDueCalendarSubscriptions = async (
 
 export const buildCalendarSubscriptionRefresh =
 	(
-		client: Pick<RemitClient, "calendarCollection" | "calendarUnitOfWork">,
+		client: Pick<RemitClient, "calendarCollection" | "calendarStore">,
 		log: Logger,
 	): ((
 		now: number,
@@ -95,7 +95,7 @@ export const buildCalendarSubscriptionRefresh =
 		refreshDueCalendarSubscriptions(
 			{
 				calendarCollection: client.calendarCollection,
-				calendarUnitOfWork: client.calendarUnitOfWork,
+				calendarStore: client.calendarStore,
 				log,
 			},
 			now,

package/src/scheduler/half-deleted-accounts.sqlite.test.ts

@@ -0,0 +1,203 @@
+import assert from "node:assert/strict";
+import { randomBytes } from "node:crypto";
+import { after, before, beforeEach, describe, it } from "node:test";
+import type { SendMessageCommand, SQSClient } from "@aws-sdk/client-sqs";
+import { finishHalfDeletedAccounts } from "@remit/backend/account-deletion";
+import type { RemitClient } from "@remit/backend/client";
+import { importConfig } from "@remit/config-transfer";
+import type { AccountItem } from "@remit/data-ports";
+import { AccountAuthType, ConnectionState } from "@remit/domain-enums";
+import { noopLogger } from "@remit/logger";
+import { openStore } from "../test-helpers/test-world.js";
+
+const FANOUT_URL = "http://queue:9324/000000000000/remit-account-fanout";
+
+let store: Awaited<ReturnType<typeof openStore>>;
+let client: RemitClient;
+
+before(async () => {
+	store = await openStore();
+	client = store.client;
+});
+
+after(() => store.close());
+
+const storedId = (): string => randomBytes(13).toString("hex").slice(0, 25);
+
+const configDocument = (accountId: string, email: string) => ({
+	kind: "reader.config",
+	schemaVersion: 1,
+	generator: {
+		app: "reader",
+		version: "v0.1.0",
+		exportedAt: "2026-08-27T09:15:00+02:00",
+	},
+	provenance: { accountConfigId: storedId(), instance: "reader.example.com" },
+	accountConfig: { name: "Imported" },
+	accounts: [
+		{
+			accountId,
+			email,
+			username: email,
+			authType: "password",
+			credentials: { required: "password" },
+			isActive: true,
+			imap: { host: "imap.example.com", port: 993, tls: true, startTls: false },
+			smtp: {
+				enabled: false,
+				host: "",
+				port: 587,
+				tls: false,
+				startTls: false,
+				username: "",
+			},
+			displayName: "",
+			muted: null,
+			composeLanguages: [],
+			signature: { plainText: "", html: "" },
+			folderRoles: [],
+			folderOverrides: [],
+			pinnedFolders: [],
+		},
+	],
+	labels: [],
+	filters: [],
+	addressFlags: [],
+});
+
+const seedAccount = (
+	accountConfigId: string,
+	name: string,
+	extra: { deletedAt?: number } = {},
+): Promise<AccountItem> =>
+	client.account.create({
+		accountConfigId,
+		username: `${name}@example.com`,
+		email: `${name}@example.com`,
+		authType: AccountAuthType.Password,
+		passwordHash: '{"ciphertext":"x"}',
+		imapHost: "imap.example.com",
+		imapPort: 993,
+		imapTls: true,
+		imapStartTls: false,
+		isActive: true,
+		connectionState: ConnectionState.Authenticated,
+		...extra,
+	});
+
+const importAccount = async (email: string): Promise<AccountItem> => {
+	const accountConfigId = storedId();
+	const accountId = storedId();
+	const outcome = await importConfig(
+		{
+			repositories: client,
+			appointFolderRole: async () => {},
+		},
+		{
+			accountConfigId,
+			userId: accountConfigId,
+			document: configDocument(accountId, email),
+			mode: "apply",
+			onExisting: "abort",
+		},
+	);
+	assert.equal(outcome.outcome, "report");
+	return client.account.get(accountId);
+};
+
+const recordingSqs = (sent: SendMessageCommand[], fail = false): SQSClient =>
+	({
+		send: async (command: SendMessageCommand) => {
+			if (fail) throw new Error("queue unreachable");
+			sent.push(command);
+			return {};
+		},
+	}) as unknown as SQSClient;
+
+const heal = (sqsClient: SQSClient) =>
+	finishHalfDeletedAccounts({
+		account: client.account,
+		configImport: client.configImport,
+		sqsClient,
+		accountFanoutQueueUrl: FANOUT_URL,
+		log: noopLogger,
+	});
+
+describe("finishing an account deleted before #1384 on scheduler start", () => {
+	let halfDeleted: AccountItem;
+	let live: AccountItem;
+	let deleted: AccountItem;
+	let imported: AccountItem;
+
+	beforeEach(async () => {
+		const accountConfigId = storedId();
+		live = await seedAccount(accountConfigId, `live-${storedId()}`);
+		halfDeleted = await seedAccount(accountConfigId, `gone-${storedId()}`);
+		await client.account.update(halfDeleted.accountId, { isActive: false });
+		deleted = await seedAccount(accountConfigId, `deleted-${storedId()}`, {
+			deletedAt: 1_700_000_000_000,
+		});
+		await client.account.update(deleted.accountId, { isActive: false });
+		imported = await importAccount(`imported-${storedId()}@example.com`);
+	});
+
+	const allAccounts = () =>
+		client.account.get([
+			halfDeleted.accountId,
+			live.accountId,
+			deleted.accountId,
+			imported.accountId,
+		]);
+
+	it("tombstones it and asks for the same purge a delete asks for", async () => {
+		const sent: SendMessageCommand[] = [];
+
+		await heal(recordingSqs(sent));
+
+		const purges = sent
+			.filter((command) => command.input.QueueUrl === FANOUT_URL)
+			.map((command) => JSON.parse(command.input.MessageBody ?? "{}"));
+		assert.ok(
+			purges.some(
+				(body) =>
+					body.type === "AccountDataPurge" &&
+					body.accountId === halfDeleted.accountId &&
+					body.accountConfigId === halfDeleted.accountConfigId,
+			),
+		);
+		const byId = new Map(
+			(await allAccounts()).map((account) => [account.accountId, account]),
+		);
+		assert.ok(byId.get(halfDeleted.accountId)?.deletedAt);
+		assert.equal(byId.get(live.accountId)?.deletedAt, undefined);
+		assert.equal(byId.get(imported.accountId)?.deletedAt, undefined);
+		assert.equal(byId.get(deleted.accountId)?.deletedAt, 1_700_000_000_000);
+		assert.ok(
+			!purges.some((body) =>
+				[live.accountId, imported.accountId, deleted.accountId].includes(
+					body.accountId,
+				),
+			),
+		);
+	});
+
+	it("does nothing for it on the next start", async () => {
+		await heal(recordingSqs([]));
+		const sent: SendMessageCommand[] = [];
+
+		await heal(recordingSqs(sent));
+
+		assert.ok(
+			!sent.some((command) =>
+				(command.input.MessageBody ?? "").includes(halfDeleted.accountId),
+			),
+		);
+	});
+
+	it("fails the start and leaves it to be found again when the purge cannot be asked for", async () => {
+		await assert.rejects(heal(recordingSqs([], true)), /queue unreachable/);
+
+		const after = await client.account.get(halfDeleted.accountId);
+		assert.equal(after.deletedAt, undefined);
+	});
+});

package/src/scheduler/handler.ts

@@ -1,12 +1,17 @@
 import { getClient } from "@remit/backend/client";
-import { createLogger, withTelemetry } from "@remit/logger-lambda";
+import { createLogger } from "@remit/logger";
+import { installNodeLogger, withTelemetry } from "@remit/logger-lambda/node";
 import { createQueueProducer } from "@remit/sqs-client/producer";
 import type { ScheduledHandler } from "aws-lambda";
 import { env } from "expect-env";
+import { emitEvent } from "../emit.js";
 import { buildAttachmentSweep } from "./attachment-sweep.js";
 import { buildCalendarSubscriptionRefresh } from "./calendar-subscriptions.js";
 import { getOfflineIntervalMs, getTickIntervalMs } from "./config.js";
 import { runSchedulerTick } from "./run-tick.js";
+import { redriveStrandedPlacements } from "./stranded-placement-redrive.js";
+
+installNodeLogger();
 
 const log = createLogger();
 
@@ -47,5 +52,11 @@ export const handler: ScheduledHandler = withTelemetry(async (event) => {
 		// already gone and this sweep is the only thing that collects it.
 		sweepAttachments: buildAttachmentSweep(client, log),
 		refreshCalendarSubscriptions: buildCalendarSubscriptionRefresh(client, log),
+		redriveStrandedPlacements: (account, tickNow) =>
+			redriveStrandedPlacements(
+				{ client, emit: emitEvent, log },
+				account,
+				tickNow,
+			),
 	});
 });

package/src/scheduler/loop.test.ts

@@ -11,6 +11,8 @@ const RESULT: SchedulerTickResult = {
 	sweepFailed: 0,
 	subscriptionsRefreshed: 0,
 	subscriptionsFailed: 0,
+	redriven: 0,
+	redriveFailed: 0,
 };
 
 // Only the loop's own two calls matter here; the tick is a stub, so what it is

package/src/scheduler/run-tick.sqlite.test.ts

@@ -0,0 +1,192 @@
+import assert from "node:assert/strict";
+import { afterEach, beforeEach, describe, it } from "node:test";
+import type { SendMessageCommand, SQSClient } from "@aws-sdk/client-sqs";
+import type { AccountItem } from "@remit/data-ports";
+import {
+	AccountAuthType,
+	AccountService,
+	ConnectionState,
+} from "@remit/domain-enums";
+import { AccountRepo } from "@remit/drizzle-service";
+import { createShippedSqliteDb } from "@remit/drizzle-service/test-sqlite";
+import { capturingLogger, type LogLine } from "../test-helpers/test-world.js";
+import { runSchedulerTick } from "./run-tick.js";
+
+const TICK_INTERVAL_MS = 60 * 60 * 1000;
+const OFFLINE_INTERVAL_MS = 12 * 60 * 60 * 1000;
+const NOW = 1_700_000_000_000;
+
+let accounts: AccountRepo;
+let close: () => void;
+
+beforeEach(() => {
+	const store = createShippedSqliteDb();
+	close = store.close;
+	accounts = new AccountRepo(store.db, store.db);
+});
+
+afterEach(() => close());
+
+const account = async (
+	name: string,
+	overrides: Partial<AccountItem> = {},
+	stored: { credential: boolean } = { credential: true },
+): Promise<AccountItem> => {
+	const created = await accounts.create({
+		accountConfigId: `cfg-${name}`,
+		username: `${name}@example.com`,
+		email: `${name}@example.com`,
+		authType: AccountAuthType.Password,
+		passwordHash: stored.credential ? '{"ciphertext":"x"}' : undefined,
+		imapHost: "imap.example.com",
+		imapPort: 993,
+		imapTls: true,
+		imapStartTls: false,
+		isActive: true,
+		connectionState: ConnectionState.Authenticated,
+	} as Parameters<AccountRepo["create"]>[0]);
+	if (Object.keys(overrides).length === 0) return created;
+	await accounts.update(created.accountId, overrides);
+	return accounts.get(created.accountId);
+};
+
+const tick = async (
+	options: Partial<Parameters<typeof runSchedulerTick>[0]> = {},
+) => {
+	const sent: SendMessageCommand[] = [];
+	const logs: LogLine[] = [];
+	const result = await runSchedulerTick({
+		accountService: accounts,
+		sqsClient: {
+			send: async (command: SendMessageCommand) => {
+				sent.push(command);
+				return {};
+			},
+		} as unknown as SQSClient,
+		queueUrl:
+			"https://sqs.eu-west-1.amazonaws.com/123/remit-dev-mailboxes.fifo",
+		log: capturingLogger(logs),
+		tickIntervalMs: TICK_INTERVAL_MS,
+		offlineIntervalMs: OFFLINE_INTERVAL_MS,
+		now: NOW,
+		redriveStrandedPlacements: async () => 0,
+		...options,
+	});
+	const enqueued = sent.map(
+		(command) => JSON.parse(command.input.MessageBody ?? "{}").accountId,
+	);
+	return { result, sent, enqueued, logs };
+};
+
+describe("a scheduler tick", () => {
+	it("enqueues the accounts that are due and only those", async () => {
+		const due = await account("due", {
+			lastSyncAt: NOW - OFFLINE_INTERVAL_MS - 1,
+		});
+		const never = await account("never");
+		await account("fresh", { lastSyncAt: NOW - 60_000 });
+
+		const { enqueued } = await tick();
+
+		assert.deepEqual(enqueued.sort(), [due.accountId, never.accountId].sort());
+	});
+
+	it("skips an account it cannot or should not sync, and says so once in aggregate", async () => {
+		const live = await account("live");
+		await account("unsyncable", { imapHost: "mail.invalid" });
+		await account("reauth", {
+			connectionState: ConnectionState.ReauthRequired,
+		});
+		await account("calendar", { syncedServices: [AccountService.Calendar] });
+
+		const { enqueued, result, logs } = await tick();
+
+		assert.deepEqual(enqueued, [live.accountId]);
+		assert.equal(result.skipped, 3);
+		assert.equal(logs.length, 1, "one aggregate line, no per-account noise");
+	});
+
+	// Issue #1120: a sync for an account with no stored credential fenced the
+	// account's FIFO group for its whole visibility timeout.
+	it("skips an account that stores no credential", async () => {
+		const live = await account("live");
+		await account("missing", {}, { credential: false });
+
+		const { enqueued } = await tick();
+
+		assert.deepEqual(enqueued, [live.accountId]);
+	});
+
+	it("gives each enqueued account a scheduler-namespaced, time-bucketed dedup id", async () => {
+		const live = await account("live");
+
+		const { sent } = await tick();
+
+		const dedupId = sent[0]?.input.MessageDeduplicationId;
+		assert.ok(
+			dedupId?.startsWith(`SYNC_MAILBOXES:scheduled:${live.accountId}:`),
+		);
+	});
+
+	it("sweeps every account's attachments, due a sync or not", async () => {
+		const due = await account("due");
+		const fresh = await account("fresh", { lastSyncAt: NOW - 60_000 });
+		const swept: string[] = [];
+
+		const { result } = await tick({
+			sweepAttachments: async (item) => {
+				swept.push(item.accountId);
+			},
+		});
+
+		assert.deepEqual(swept.sort(), [due.accountId, fresh.accountId].sort());
+		assert.equal(result.swept, 2);
+	});
+
+	// Uncontained, one storage error reaches the loop, which exits, and the
+	// container restarts without ever enqueuing an account again.
+	it("keeps enqueuing mail when the attachment sweep throws", async () => {
+		const live = await account("live");
+
+		const { enqueued, result } = await tick({
+			sweepAttachments: async () => {
+				throw new Error("AccessDenied on ListObjectsV2");
+			},
+		});
+
+		assert.deepEqual(enqueued, [live.accountId]);
+		assert.equal(result.sweepFailed, 1);
+	});
+
+	it("keeps the tick alive when the calendar subscription refresh throws", async () => {
+		const live = await account("live");
+
+		const { enqueued, logs } = await tick({
+			refreshCalendarSubscriptions: async () => {
+				throw new Error("database is locked");
+			},
+		});
+
+		assert.deepEqual(enqueued, [live.accountId]);
+		assert.ok(logs.some((line) => line.level === "error"));
+	});
+
+	it("counts what each account re-drove and keeps enqueuing past one that fails", async () => {
+		const live = await account("live");
+		const busy = await account("busy");
+
+		const { enqueued, result, logs } = await tick({
+			redriveStrandedPlacements: async (redriven) => {
+				if (redriven.accountId === busy.accountId) {
+					throw new Error("SQLITE_BUSY");
+				}
+				return 2;
+			},
+		});
+
+		assert.deepEqual(enqueued.sort(), [live.accountId, busy.accountId].sort());
+		assert.equal(result.redriven, 2);
+		assert.equal(result.redriveFailed, 1);
+		assert.ok(logs.some((line) => line.level === "error"));
+	});
+});