@remit/imap-worker 0.0.38 → 0.0.39

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@remit/imap-worker",
-  "version": "0.0.38",
+  "version": "0.0.39",
   "type": "module",
   "main": "dist/index.js",
   "exports": {

package/src/handlers/append-sent-message.test.ts

@@ -89,6 +89,9 @@ const deps = (): AppendSentMessageDeps =>
 				get: async () => h.outbox,
 				delete: record("outboxMessage.delete"),
 			},
+			outboxAttachment: {
+				discardAll: record("outboxAttachment.discardAll"),
+			},
 			mailboxSpecialUse: {
 				findBySpecialUse: async () => h.specialUseSent,
 			},
@@ -142,6 +145,19 @@ describe("handleAppendSentMessage", () => {
 		assert.equal(h.disconnectCount, 1);
 	});
 
+	it("takes the draft's stored attachments with the row (#679)", async () => {
+		await handleAppendSentMessage(event, noopLog, deps());
+
+		// The row is the only reference to those objects. Sweeping has to happen
+		// here too, not only on a discard, or a sent message leaves its files
+		// behind with nothing able to reach them.
+		assert.deepEqual(called("outboxAttachment.discardAll")[0]?.args, [
+			"cfg-1",
+			"acc-1",
+			"out-1",
+		]);
+	});
+
 	it("builds the message from the outbox row's own headers", async () => {
 		await handleAppendSentMessage(event, noopLog, deps());
 

package/src/handlers/append-sent-message.ts

@@ -77,6 +77,7 @@ export const handleAppendSentMessage = async (
 	const {
 		account: accountService,
 		outboxMessage: outboxMessageService,
+		outboxAttachment: outboxAttachmentService,
 		mailboxSpecialUse: mailboxSpecialUseService,
 		mailbox: mailboxService,
 		secrets,
@@ -145,10 +146,23 @@ export const handleAppendSentMessage = async (
 
 			// The message now lives in the IMAP Sent folder. Drop the outbox row so
 			// the user does not see it twice in the UI (Outbox + Sent). Issue #178.
+			//
+			// The row goes first, and nothing that can fail may come before it. The
+			// APPEND above has already happened; anything between it and this delete
+			// that throws leaves the row for the job to retry, and the retry appends
+			// a second copy to the user's Sent folder. A storage error is not worth
+			// a duplicate message — the attachment objects that outlive their row
+			// are exactly what the sweep collects, so losing this delete costs a
+			// sweep and nothing else.
 			await outboxMessageService.delete(
 				account.accountConfigId,
 				outboxMessageId,
 			);
+			await outboxAttachmentService.discardAll(
+				account.accountConfigId,
+				accountId,
+				outboxMessageId,
+			);
 			log.info(
 				{ outboxMessageId },
 				"Deleted outbox row after successful APPEND to Sent",

package/src/scheduler/attachment-sweep.test.ts

@@ -0,0 +1,168 @@
+/**
+ * Both scheduler entry points must hand the tick a sweep.
+ *
+ * Only the hosted one strictly needs it — a browser PUTs straight to block
+ * storage there, so nothing is in the path to refuse an upload against a draft
+ * that is already gone — but a deployment that quietly stops collecting is
+ * indistinguishable from one that has nothing to collect. Deleting either wiring
+ * used to leave the suite green.
+ */
+
+import assert from "node:assert/strict";
+import { readFileSync } from "node:fs";
+import { dirname, join } from "node:path";
+import { describe, it } from "node:test";
+import { fileURLToPath } from "node:url";
+import type { AccountItem } from "@remit/data-ports";
+import type { Logger } from "@remit/logger-lambda";
+import type { StorageService } from "@remit/storage-service";
+import { buildAttachmentSweep } from "./attachment-sweep.js";
+
+const here = dirname(fileURLToPath(import.meta.url));
+const TICK_TIME = 1_700_000_000_000;
+
+const silentLogger = (): Logger => {
+	const noop = () => {};
+	const log = {
+		info: noop,
+		warn: noop,
+		error: noop,
+		debug: noop,
+		fatal: noop,
+		trace: noop,
+		child: () => log,
+	} as unknown as Logger;
+	return log;
+};
+
+/** Enough of an account for the tick's eligibility checks to run. */
+const eligibleAccount = (accountId: string) =>
+	({
+		accountId,
+		accountConfigId: "cfg-1",
+		email: `${accountId}@example.com`,
+		username: accountId,
+		authType: "password",
+		imapHost: "imap.example.com",
+		imapPort: 993,
+		imapTls: true,
+		imapStartTls: false,
+		isActive: true,
+		connectionState: "authenticated",
+		// Freshly synced, so nothing is due and no SQS send is attempted. The
+		// sweep must still run: abandoned bytes have nothing to do with how
+		// recently mail was fetched.
+		lastSyncAt: TICK_TIME,
+		createdAt: 0,
+		updatedAt: 0,
+	}) as unknown as AccountItem;
+
+describe("the hosted tick sweeps", () => {
+	it("runs the sweep for every account, due a sync or not", async () => {
+		process.env.SQS_QUEUE_URL_MAILBOXES = "https://queue.test/mailboxes";
+
+		const swept: string[] = [];
+		const { setClient, _resetForTest } = await import("@remit/backend/client");
+		setClient({
+			account: {
+				listAllAccountsPage: async () => ({
+					items: [eligibleAccount("acc-1"), eligibleAccount("acc-2")],
+					cursor: null,
+				}),
+			},
+			storage: {
+				listOutboxDraftsWithAttachments: async (
+					_cfg: string,
+					accountId: string,
+				) => {
+					swept.push(accountId);
+					return [];
+				},
+			},
+			outboxAttachment: { reapAndListLive: async () => [] },
+		} as never);
+
+		try {
+			const { handler } = await import("./handler.js");
+			await handler(
+				{ time: new Date(TICK_TIME).toISOString() } as never,
+				{} as never,
+				() => {},
+			);
+			assert.deepEqual(swept.sort(), ["acc-1", "acc-2"]);
+		} finally {
+			_resetForTest();
+		}
+	});
+});
+
+describe("the self-host runner wires the sweep", () => {
+	it("hands runSchedulerTick a sweep", () => {
+		// runner.ts starts its loop on import, so it cannot be invoked from a test
+		// the way handler.ts can. This guards the wiring itself: without it,
+		// deleting the line leaves the suite green and self-host silently stops
+		// collecting.
+		const source = readFileSync(join(here, "runner.ts"), "utf8");
+		assert.match(source, /sweepAttachments:\s*buildAttachmentSweep\(/);
+	});
+});
+
+describe("buildAttachmentSweep", () => {
+	const account = {
+		accountId: "acc-1",
+		accountConfigId: "cfg-1",
+	} as unknown as AccountItem;
+
+	it("collects the objects the database does not name", async () => {
+		const deleted: string[] = [];
+		const storage = {
+			listOutboxDraftsWithAttachments: async () => ["draft-1"],
+			listOutboxAttachments: async () => [
+				{ outboxAttachmentId: "known", key: "k1", sizeBytes: 1 },
+				{ outboxAttachmentId: "orphan", key: "k2", sizeBytes: 1 },
+			],
+			deleteOutboxAttachment: async (
+				_cfg: string,
+				_acc: string,
+				_draft: string,
+				id: string,
+			) => {
+				deleted.push(id);
+			},
+		} as unknown as StorageService;
+
+		await buildAttachmentSweep(
+			{
+				storage,
+				outboxAttachment: {
+					reapAndListLive: async () => ["known"],
+				},
+			} as never,
+			silentLogger(),
+		)(account);
+
+		assert.deepEqual(deleted, ["orphan"]);
+	});
+
+	it("lets a storage failure out, so the tick can count it", async () => {
+		// Containment belongs to the tick, which keeps enqueuing mail regardless.
+		// Swallowing here would make a broken sweep look like a working one.
+		const storage = {
+			listOutboxDraftsWithAttachments: async () => {
+				throw new Error("storage root does not exist");
+			},
+		} as unknown as StorageService;
+
+		await assert.rejects(
+			() =>
+				buildAttachmentSweep(
+					{
+						storage,
+						outboxAttachment: { reapAndListLive: async () => [] },
+					} as never,
+					silentLogger(),
+				)(account),
+			/storage root does not exist/,
+		);
+	});
+});

package/src/scheduler/attachment-sweep.ts

@@ -0,0 +1,46 @@
+import type { RemitClient } from "@remit/backend/client";
+import type { AccountItem } from "@remit/data-ports";
+import type { Logger } from "@remit/logger-lambda";
+import { sweepAbandonedOutboxAttachments } from "@remit/storage-service";
+/**
+ * Collect attachment bytes the database does not know about. Both scheduler
+ * entry points build it the same way, so the hosted tick and the self-host loop
+ * collect identically — this is the only defence on the hosted side, where the
+ * browser PUTs straight to block storage and nothing consults the database on
+ * the way in.
+ */
+export const buildAttachmentSweep =
+	(
+		client: Pick<RemitClient, "storage" | "outboxAttachment">,
+		log: Logger,
+	): ((account: AccountItem) => Promise<void>) =>
+	async (account) => {
+		const { deleted, skipped } = await sweepAbandonedOutboxAttachments(
+			{
+				storage: client.storage,
+				// Reaps the draft's lapsed reservations first, so a row that has
+				// stopped holding room stops vouching for its bytes too and the
+				// object is collected on this same pass.
+				listKnownAttachmentIds: (accountConfigId, outboxMessageId) =>
+					client.outboxAttachment.reapAndListLive(
+						accountConfigId,
+						outboxMessageId,
+					),
+				onSkipped: (outboxMessageId, error) => {
+					log.error(
+						{ outboxMessageId, error },
+						"Left a draft's attachment objects alone: its rows could not be read",
+					);
+				},
+			},
+			account.accountConfigId,
+			account.accountId,
+		);
+
+		if (deleted > 0 || skipped > 0) {
+			log.warn(
+				{ accountId: account.accountId, deleted, skipped },
+				"Outbox attachment sweep",
+			);
+		}
+	};

package/src/scheduler/handler.ts

@@ -3,6 +3,7 @@ import { createLogger, withTelemetry } from "@remit/logger-lambda";
 import { createQueueProducer } from "@remit/sqs-client/producer";
 import type { ScheduledHandler } from "aws-lambda";
 import { env } from "expect-env";
+import { buildAttachmentSweep } from "./attachment-sweep.js";
 import { getOfflineIntervalMs, getTickIntervalMs } from "./config.js";
 import { runSchedulerTick } from "./run-tick.js";
 
@@ -29,16 +30,20 @@ const sqsClient = createQueueProducer({ queueUrl: mailboxesQueueUrl });
  * correctly dedupes.
  */
 export const handler: ScheduledHandler = withTelemetry(async (event) => {
-	const { account } = await getClient();
+	const client = await getClient();
 	const now = Date.parse(event.time);
 
 	await runSchedulerTick({
-		accountService: account,
+		accountService: client.account,
 		sqsClient,
 		queueUrl: mailboxesQueueUrl,
 		log,
 		tickIntervalMs: getTickIntervalMs(),
 		offlineIntervalMs: getOfflineIntervalMs(),
 		now,
+		// The hosted side is the one that needs this: a browser PUTs straight
+		// to block storage, so nothing refuses an upload against a draft that is
+		// already gone and this sweep is the only thing that collects it.
+		sweepAttachments: buildAttachmentSweep(client, log),
 	});
 });

package/src/scheduler/loop.test.ts

@@ -3,7 +3,13 @@ import { describe, it } from "node:test";
 import { runSchedulerLoop, type SchedulerLoopOptions } from "./loop.js";
 import type { RunSchedulerTickDeps, SchedulerTickResult } from "./run-tick.js";
 
-const RESULT: SchedulerTickResult = { scanned: 0, enqueued: 0, skipped: 0 };
+const RESULT: SchedulerTickResult = {
+	scanned: 0,
+	enqueued: 0,
+	skipped: 0,
+	swept: 0,
+	sweepFailed: 0,
+};
 
 // Only the loop's own two calls matter here; the tick is a stub, so what it is
 // handed is never read.

package/src/scheduler/run-tick.test.ts

@@ -246,3 +246,90 @@ describe("runSchedulerTick", () => {
 		assert.equal(result.enqueued, 0);
 	});
 });
+
+describe("the attachment sweep inside a tick", () => {
+	const twoAccounts = () => [
+		{
+			items: [
+				baseAccount({ accountId: "acct_1", lastSyncAt: NOW - 1_000 }),
+				baseAccount({ accountId: "acct_2", lastSyncAt: 0 }),
+			],
+			cursor: null,
+		} as unknown as AccountSchedulerPage,
+	];
+
+	it("sweeps every account, not only the ones due a sync", async () => {
+		const { sqsClient } = fakeSqsClient();
+		const swept: string[] = [];
+
+		const result = await runSchedulerTick({
+			accountService: fakeAccountService(twoAccounts()),
+			sqsClient,
+			queueUrl: "https://queue.test/mailboxes",
+			log: createNoopLogger(),
+			tickIntervalMs: TICK_INTERVAL_MS,
+			offlineIntervalMs: OFFLINE_INTERVAL_MS,
+			now: NOW,
+			sweepAttachments: async (account) => {
+				swept.push(account.accountId);
+			},
+		});
+
+		assert.deepEqual(swept.sort(), ["acct_1", "acct_2"]);
+		assert.equal(result.swept, 2);
+		assert.equal(result.sweepFailed, 0);
+	});
+
+	it("keeps enqueuing mail when the sweep throws", async () => {
+		// The sweep is housekeeping; enqueuing mail is not. Uncontained, one
+		// storage error here reaches the loop, which exits, and the container
+		// restarts every few seconds without ever enqueuing an account again.
+		const { sqsClient, sent } = fakeSqsClient();
+		const { log, calls } = createCapturingLogger();
+
+		const result = await runSchedulerTick({
+			accountService: fakeAccountService(twoAccounts()),
+			sqsClient,
+			queueUrl: "https://queue.test/mailboxes",
+			log,
+			tickIntervalMs: TICK_INTERVAL_MS,
+			offlineIntervalMs: OFFLINE_INTERVAL_MS,
+			now: NOW,
+			sweepAttachments: async () => {
+				throw new Error("AccessDenied on ListObjectsV2");
+			},
+		});
+
+		assert.equal(result.sweepFailed, 2);
+		assert.equal(result.swept, 0);
+		// The tick still did its actual job.
+		assert.ok(sent.length > 0);
+		assert.equal(result.enqueued, sent.length);
+		assert.ok(
+			calls.some(
+				(entry) =>
+					entry.level === "error" &&
+					entry.args.some((arg) =>
+						String(arg).includes("Outbox attachment sweep failed"),
+					),
+			),
+		);
+	});
+
+	it("ticks without a sweep at all", async () => {
+		const { sqsClient } = fakeSqsClient();
+
+		const result = await runSchedulerTick({
+			accountService: fakeAccountService(twoAccounts()),
+			sqsClient,
+			queueUrl: "https://queue.test/mailboxes",
+			log: createNoopLogger(),
+			tickIntervalMs: TICK_INTERVAL_MS,
+			offlineIntervalMs: OFFLINE_INTERVAL_MS,
+			now: NOW,
+		});
+
+		assert.equal(result.swept, 0);
+		assert.equal(result.sweepFailed, 0);
+	});
+});

package/src/scheduler/run-tick.ts

@@ -30,12 +30,25 @@ export interface RunSchedulerTickDeps {
 	tickIntervalMs: number;
 	/** Injectable for tests; defaults to `Date.now()`. */
 	now?: number;
+	/**
+	 * Collect attachment bytes the database does not know about, for one account.
+	 *
+	 * The tick already walks every account on a cadence, which is the only thing
+	 * this needs. It exists because an upload URL minted for a draft stays valid
+	 * after that draft is discarded, and where the browser writes straight to
+	 * block storage nothing is in the path to refuse it — see
+	 * `sweepAbandonedOutboxAttachments`. Optional only so a test can leave it out;
+	 * both real entry points pass it.
+	 */
+	sweepAttachments?: (account: AccountItem) => Promise<void>;
 }
 
 export interface SchedulerTickResult {
 	scanned: number;
 	enqueued: number;
 	skipped: number;
+	swept: number;
+	sweepFailed: number;
 }
 
 /**
@@ -101,6 +114,8 @@ export const runSchedulerTick = async (
 	let scanned = 0;
 	let enqueued = 0;
 	let skipped = 0;
+	let swept = 0;
+	let sweepFailed = 0;
 
 	do {
 		const page = await accountService.listAllAccountsPage({
@@ -132,10 +147,49 @@ export const runSchedulerTick = async (
 		);
 		enqueued += due.length;
 
+		if (deps.sweepAttachments) {
+			// After the enqueue, never before it. Enqueuing mail is the tick's job
+			// and collecting bytes is housekeeping; on a hosted Lambda a slow sweep
+			// ahead of the enqueue eats the invocation budget, and a timeout is not
+			// catchable, so the remaining pages would never be enqueued at all.
+			//
+			// Every account, not only the ones due a sync: abandoned bytes have
+			// nothing to do with how recently mail was fetched. Bounded to the same
+			// concurrency as the enqueues.
+			//
+			// Contained, because this is housekeeping and enqueuing mail is not. An
+			// unreadable input or a storage permission error thrown from here would
+			// otherwise reach the loop, which logs and exits, and compose would
+			// restart the process every five seconds — no account would ever be
+			// enqueued again. A failed sweep costs a tick's collection and nothing
+			// else; the count is what makes that visible.
+			const sweep = deps.sweepAttachments;
+			const outcomes = await pMap(
+				page.items,
+				(account) =>
+					sweep(account).then(
+						() => true,
+						(error: unknown) => {
+							log.error(
+								{ accountId: account.accountId, error },
+								"Outbox attachment sweep failed for account",
+							);
+							return false;
+						},
+					),
+				{ concurrency: SCHEDULER_ENQUEUE_CONCURRENCY },
+			);
+			swept += outcomes.filter(Boolean).length;
+			sweepFailed += outcomes.filter((ok) => !ok).length;
+		}
+
 		cursor = page.cursor ?? undefined;
 	} while (cursor);
 
-	log.info({ scanned, enqueued, skipped }, "Scheduled-sync tick complete");
+	log.info(
+		{ scanned, enqueued, skipped, swept, sweepFailed },
+		"Scheduled-sync tick complete",
+	);
 
-	return { scanned, enqueued, skipped };
+	return { scanned, enqueued, skipped, swept, sweepFailed };
 };

package/src/scheduler/runner.ts

@@ -4,7 +4,9 @@ import { getClient } from "@remit/backend/client";
 import { createLogger } from "@remit/logger-lambda";
 import { clearHeartbeats, createHeartbeat } from "@remit/sqs-client/heartbeat";
 import { createQueueProducer } from "@remit/sqs-client/producer";
+
 import { env } from "expect-env";
+import { buildAttachmentSweep } from "./attachment-sweep.js";
 import { getOfflineIntervalMs, getTickIntervalMs } from "./config.js";
 import { runSchedulerLoop } from "./loop.js";
 import { runSchedulerTick } from "./run-tick.js";
@@ -66,7 +68,8 @@ const runLoop = async (): Promise<void> => {
 	};
 
 	await clearHeartbeats().catch(onClearError);
-	const { account } = await getClient();
+	const client = await getClient();
+	const { account } = client;
 	await runSchedulerLoop({
 		tick: runSchedulerTick,
 		tickDeps: {
@@ -76,6 +79,7 @@ const runLoop = async (): Promise<void> => {
 			log,
 			tickIntervalMs,
 			offlineIntervalMs,
+			sweepAttachments: buildAttachmentSweep(client, log),
 		},
 		heartbeat: createHeartbeat("tick"),
 		onHeartbeatError: onBeatError,