@remit/account-worker 0.0.33 → 0.0.35

package/src/handlers/account-export.sqlite.test.ts

@@ -0,0 +1,305 @@
+import assert from "node:assert/strict";
+import { mkdtemp, stat } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { finished } from "node:stream/promises";
+import { after, before, describe, it } from "node:test";
+import { MailboxCursorState } from "@remit/domain-enums";
+import {
+	accountTable,
+	BETTER_SQLITE3_LIMITS,
+	composeSqliteRepositories,
+	mailboxTable,
+	messageObjectsInMemory,
+	messageTable,
+	openSqliteDatabases,
+	syncStores,
+} from "@remit/drizzle-service";
+import { createShippedSqliteFiles } from "@remit/drizzle-service/test-sqlite";
+import { noopLogger } from "@remit/logger";
+import {
+	buildExportArchiveKey,
+	type StorageService,
+} from "@remit/storage-service";
+import { createFilesystemStorageService } from "@remit/storage-service/filesystem";
+import { cascadeServicesOf } from "../cascade.js";
+import { processAccountExport } from "./account-export.js";
+
+const NOW = 1700000000000;
+const MESSAGE_COUNT = 3;
+
+const exists = (path: string): Promise<boolean> =>
+	stat(path).then(
+		() => true,
+		() => false,
+	);
+
+interface Tracked {
+	readonly storage: StorageService;
+	readonly maxOpen: () => number;
+}
+
+const trackingOpenBodies = (
+	storage: StorageService,
+	onFirstRead: () => Promise<void>,
+): Tracked => {
+	let open = 0;
+	let max = 0;
+	let reads = 0;
+	return {
+		maxOpen: () => max,
+		storage: {
+			...storage,
+			retrieveMessageBodyStream: async (
+				accountConfigId,
+				accountId,
+				messageId,
+			) => {
+				reads += 1;
+				if (reads === 1) await onFirstRead();
+				const body = await storage.retrieveMessageBodyStream(
+					accountConfigId,
+					accountId,
+					messageId,
+				);
+				if (body === null) return null;
+				open += 1;
+				max = Math.max(max, open);
+				body.once("close", () => {
+					open -= 1;
+				});
+				return body;
+			},
+		},
+	};
+};
+
+describe("processAccountExport over SQLite repositories and filesystem storage", () => {
+	let directory: string;
+	let close: () => Promise<void>;
+	let repositories: ReturnType<typeof composeSqliteRepositories>;
+	let databases: Awaited<ReturnType<typeof openSqliteDatabases>>;
+	let storage: StorageService;
+
+	const seedTenant = async (): Promise<{
+		accountConfigId: string;
+		accountExportRequestId: string;
+	}> => {
+		const accountConfig = await repositories.accountConfig.create({
+			userId: "user000000000000000000001",
+		});
+		const { accountConfigId } = accountConfig;
+		const accountId = `${accountConfigId}-acc`;
+		const mailboxId = `${accountConfigId}-mbx`;
+		await databases.app.insert(accountTable).values({
+			accountId,
+			accountConfigId,
+			username: "u",
+			email: "u@example.com",
+			authType: "password",
+			syncedServices: ["Mail"],
+			imapHost: "imap.example.com",
+			imapPort: 993,
+			imapTls: true,
+			imapStartTls: false,
+			smtpPort: 587,
+			isActive: true,
+			connectionState: "authenticated",
+			createdAt: NOW,
+			updatedAt: NOW,
+		});
+		await databases.mail.insert(mailboxTable).values({
+			mailboxId,
+			accountId,
+			namespaceType: "personal",
+			namespacePrefix: "",
+			hierarchyDelimiter: "/",
+			fullPath: "INBOX",
+			uidValidity: 1,
+			uidNext: 1,
+			highestModseq: "0",
+			messageCount: 0,
+			unseenCount: 0,
+			deletedCount: 0,
+			totalSize: 0,
+			lastSyncUid: 0,
+			highWaterMarkUid: 0,
+			lastMessageSyncAt: NOW,
+			cursorState: MailboxCursorState.normal,
+			createdAt: NOW,
+			updatedAt: NOW,
+		});
+		for (let uid = 1; uid <= MESSAGE_COUNT; uid += 1) {
+			const messageId = `${accountConfigId}-msg-${uid}`;
+			await databases.mail.insert(messageTable).values({
+				messageId,
+				mailboxId,
+				uid,
+				sequenceNumber: uid,
+				rfc822Size: 10,
+				internalDate: NOW,
+				envelopeId: `${messageId}-env`,
+				rootBodyPartId: `${messageId}-root`,
+				status: "active",
+				syncStatus: "synced",
+				category: "uncategorized",
+				createdAt: NOW,
+				updatedAt: NOW,
+			});
+			await storage.storeMessageBody({
+				accountConfigId,
+				accountId,
+				messageId,
+				content: Buffer.from(`Subject: message ${uid}\r\n\r\nbody`),
+			});
+		}
+		const request = await repositories.accountExportRequest.create({
+			accountConfigId,
+			userId: accountConfig.userId,
+			state: "Pending",
+		});
+		return {
+			accountConfigId,
+			accountExportRequestId: request.accountExportRequestId,
+		};
+	};
+
+	before(async () => {
+		directory = await mkdtemp(join(tmpdir(), "remit-account-export-"));
+		databases = await openSqliteDatabases(createShippedSqliteFiles(directory));
+		close = databases.close;
+		repositories = composeSqliteRepositories(
+			syncStores(databases, BETTER_SQLITE3_LIMITS),
+			messageObjectsInMemory(),
+		);
+		storage = createFilesystemStorageService(join(directory, "storage"));
+	});
+
+	after(async () => {
+		await close();
+	});
+
+	it("archives every body while holding one open at a time", async () => {
+		const event = { type: "AccountExport" as const, ...(await seedTenant()) };
+		const tracked = trackingOpenBodies(storage, async () => {});
+
+		await processAccountExport(event, noopLogger, {
+			services: cascadeServicesOf(repositories),
+			storageService: tracked.storage,
+		});
+
+		const request = await repositories.accountExportRequest.get(
+			event.accountExportRequestId,
+		);
+		assert.equal(request.state, "Ready");
+		assert.equal(tracked.maxOpen(), 1);
+		assert.equal(
+			await exists(
+				join(
+					directory,
+					"storage",
+					buildExportArchiveKey(
+						event.accountConfigId,
+						event.accountExportRequestId,
+					),
+				),
+			),
+			true,
+		);
+	});
+
+	it("leaves no archive when the account config is deleted while it runs", async () => {
+		const event = { type: "AccountExport" as const, ...(await seedTenant()) };
+		const tracked = trackingOpenBodies(storage, async () => {
+			await repositories.accountConfig.update(event.accountConfigId, {
+				state: "deleting",
+				deletedAt: Date.now(),
+			});
+		});
+
+		await processAccountExport(event, noopLogger, {
+			services: cascadeServicesOf(repositories),
+			storageService: tracked.storage,
+		});
+
+		const request = await repositories.accountExportRequest.get(
+			event.accountExportRequestId,
+		);
+		assert.notEqual(request.state, "Ready");
+		assert.equal(
+			await exists(
+				join(
+					directory,
+					"storage",
+					buildExportArchiveKey(
+						event.accountConfigId,
+						event.accountExportRequestId,
+					),
+				),
+			),
+			false,
+		);
+	});
+
+	it("skips a message with no stored body and still completes", async () => {
+		const event = { type: "AccountExport" as const, ...(await seedTenant()) };
+		const consulted: string[] = [];
+		const skipped = `${event.accountConfigId}-msg-2`;
+
+		await processAccountExport(event, noopLogger, {
+			services: cascadeServicesOf(repositories),
+			storageService: {
+				...storage,
+				retrieveMessageBodyStream: async (
+					accountConfigId,
+					accountId,
+					messageId,
+				) => {
+					consulted.push(messageId);
+					return messageId === skipped
+						? null
+						: storage.retrieveMessageBodyStream(
+								accountConfigId,
+								accountId,
+								messageId,
+							);
+				},
+			},
+		});
+
+		assert.equal(consulted.length, MESSAGE_COUNT);
+		const request = await repositories.accountExportRequest.get(
+			event.accountExportRequestId,
+		);
+		assert.equal(request.state, "Ready");
+	});
+
+	it("marks the request Failed and rethrows when the upload fails", async () => {
+		const event = { type: "AccountExport" as const, ...(await seedTenant()) };
+
+		await assert.rejects(
+			processAccountExport(event, noopLogger, {
+				services: cascadeServicesOf(repositories),
+				storageService: {
+					...storage,
+					storeExportArchiveStream: async (
+						_accountConfigId,
+						_exportRequestId,
+						body,
+					) => {
+						body.resume();
+						await finished(body);
+						throw new Error("storage refused the archive");
+					},
+				},
+			}),
+			/storage refused the archive/,
+		);
+
+		const request = await repositories.accountExportRequest.get(
+			event.accountExportRequestId,
+		);
+		assert.equal(request.state, "Failed");
+		assert.equal(request.errorMessage, "storage refused the archive");
+	});
+});

package/src/handlers/account-export.ts

@@ -1,9 +1,9 @@
+import { once } from "node:events";
 import { PassThrough } from "node:stream";
 import type { Logger } from "@remit/logger";
 import type { StorageService } from "@remit/storage-service";
 import { ZipArchive } from "archiver";
 import type { CascadeServices } from "../cascade.js";
-import { getCascadeServices, getStorageService } from "../config.js";
 import type { AccountExportEvent } from "../events.js";
 
 const THIRTY_DAYS_MILLIS = 30 * 24 * 60 * 60 * 1000;
@@ -13,11 +13,6 @@ export interface ProcessAccountExportDeps {
 	storageService: StorageService;
 }
 
-const defaultDeps = async (): Promise<ProcessAccountExportDeps> => ({
-	services: await getCascadeServices(),
-	storageService: getStorageService(),
-});
-
 const appendMessageBodies = async (
 	archive: ZipArchive,
 	accountConfigId: string,
@@ -52,9 +47,13 @@ const appendMessageBodies = async (
 					continue;
 				}
 
+				// One body in flight at a time, so an export holds one message in
+				// memory and one open storage read, not the whole account.
+				const written = once(archive, "entry");
 				archive.append(body, {
 					name: `accounts/${account.accountId}/${mailbox.fullPath ?? mailbox.mailboxId}/${message.messageId}.eml`,
 				});
+				await written;
 				appended += 1;
 			}
 		}
@@ -63,15 +62,31 @@ const appendMessageBodies = async (
 	return appended;
 };
 
+const accountConfigLive = async (
+	services: CascadeServices,
+	accountConfigId: string,
+): Promise<boolean> => {
+	const [accountConfig] = await services.accountConfigService.get([
+		accountConfigId,
+	]);
+	return accountConfig !== undefined && accountConfig.deletedAt === undefined;
+};
+
 export const processAccountExport = async (
 	event: AccountExportEvent,
 	log: Logger,
-	deps?: ProcessAccountExportDeps,
+	{ services, storageService }: ProcessAccountExportDeps,
 ): Promise<void> => {
 	const { accountConfigId, accountExportRequestId } = event;
-	const { services, storageService } = deps ?? (await defaultDeps());
 	const { accountExportRequestService } = services;
 
+	if (!(await accountConfigLive(services, accountConfigId))) {
+		log.info(
+			{ accountConfigId, accountExportRequestId },
+			"Account config is being deleted; export skipped",
+		);
+		return;
+	}
 	await accountExportRequestService.get(accountExportRequestId);
 	await accountExportRequestService.update(accountExportRequestId, {
 		state: "Processing",
@@ -110,6 +125,20 @@ export const processAccountExport = async (
 		await archiveSettled;
 		const objectKey = await uploadDone;
 
+		// Delete-me sets deletedAt before it queues the erase, so a config that
+		// is gone or deleting now means this archive would outlive the account.
+		if (!(await accountConfigLive(services, accountConfigId))) {
+			await storageService.deleteExportArchive(
+				accountConfigId,
+				accountExportRequestId,
+			);
+			log.info(
+				{ accountConfigId, accountExportRequestId },
+				"Account config deleted during export; archive removed",
+			);
+			return;
+		}
+
 		const expiresAt = Date.now() + THIRTY_DAYS_MILLIS;
 
 		await accountExportRequestService.update(accountExportRequestId, {

package/src/handlers/account-fanout.test.ts

@@ -8,12 +8,13 @@ import { processAccountFanout } from "./account-fanout.js";
 
 const buildServices = (): CascadeServices =>
 	({
-		accountConfigService: { get: async () => ({ userId: "user-1" }) },
+		accountConfigService: { get: async () => [{ userId: "user-1" }] },
 	}) as unknown as CascadeServices;
 
 const deleteEvent: AccountDeleteEvent = {
 	type: "AccountDelete",
 	accountConfigId: "cfg-1",
+	userId: "user-1",
 };
 
 describe("processAccountFanout on account delete", () => {

package/src/handlers/account-fanout.ts

@@ -10,18 +10,16 @@ import type { CascadeServices } from "../cascade.js";
 import {
 	type FanoutQueues,
 	getCascadeServices,
+	getStorageService,
 	sqsFanoutQueues,
 } from "../config.js";
 import {
 	type DeletionCapabilities,
 	getDeletionCapabilities,
 } from "../deletion-capabilities.js";
-import type {
-	AccountDeleteEvent,
-	AccountFanoutEvent,
-	AccountFinalizeEvent,
-} from "../events.js";
+import type { AccountFanoutEvent } from "../events.js";
 import { processAccountFanoutEvent } from "../process.js";
+import { processAccountDelete } from "./account-delete.js";
 import { processAccountExport } from "./account-export.js";
 import { processAccountDataPurge } from "./account-purge.js";
 
@@ -49,7 +47,10 @@ export const processAccountFanout = async (
 	}
 
 	if (event.type === "AccountExport") {
-		await processAccountExport(event, log);
+		await processAccountExport(event, log, {
+			services,
+			storageService: getStorageService(),
+		});
 		return;
 	}
 
@@ -61,31 +62,12 @@ export const processAccountFanout = async (
 		return;
 	}
 
-	await processAccountDelete(event, log, services, deps);
-};
-
-const processAccountDelete = async (
-	event: AccountDeleteEvent,
-	log: Logger,
-	services: CascadeServices,
-	deps: ProcessAccountFanoutDeps,
-): Promise<void> => {
-	const { accountConfigId } = event;
-	const signOut = deps.signOut ?? (await getDeletionCapabilities()).signOut;
-	const { accountFinalize } = deps.queues;
-
-	const accountConfig =
-		await services.accountConfigService.get(accountConfigId);
-	const userId = accountConfig.userId;
-
-	await signOut(userId, log);
-
-	const finalizeEvent: AccountFinalizeEvent = {
-		type: "FinalizeAccountDelete",
-		accountConfigId,
-	};
-	await accountFinalize.send({ body: finalizeEvent });
-	log.info({ accountConfigId }, "Enqueued finalize event");
+	await processAccountDelete(event, log, {
+		services,
+		signOut: deps.signOut ?? (await getDeletionCapabilities()).signOut,
+		enqueueFinalize: (finalizeEvent) =>
+			deps.queues.accountFinalize.send({ body: finalizeEvent }),
+	});
 };
 
 const log = createLogger();

package/src/handlers/account-finalize.ts

@@ -6,11 +6,7 @@ import {
 } from "@remit/logger-lambda/metrics";
 import { installNodeLogger, withTelemetry } from "@remit/logger-lambda/node";
 import type { SQSEvent, SQSHandler } from "aws-lambda";
-import {
-	type CascadeEntity,
-	type CascadeServices,
-	enumerateCascadeEntities,
-} from "../cascade.js";
+import type { CascadeServices } from "../cascade.js";
 import { getCascadeServices } from "../config.js";
 import {
 	type DeletionCapabilities,
@@ -20,6 +16,7 @@ import type {
 	AccountDeleteFinalizeEvent,
 	AccountFinalizeEvent,
 } from "../events.js";
+import { finalizeAccountDelete } from "./account-delete.js";
 import { processAccountDataPurgeFinalize } from "./account-purge-finalize.js";
 
 installNodeLogger();
@@ -39,89 +36,11 @@ const resolveDeps = async (
 	services: deps.cascadeServices ?? (await getCascadeServices()),
 });
 
-/**
- * GDPR hard-delete: every row tied to the deleted AccountConfig is removed and
- * every stored object under `accounts/{accountConfigId}/` is purged. The
- * AccountConfig row itself is the LAST delete so that a mid-cascade replay still
- * sees the cascade-in-progress flag (`deletedAt` set, `isActive=false`, written
- * API-side) and re-runs cleanly. After a successful run nothing tied to the
- * AccountConfig persists.
- *
- * Step order (non-negotiable per #320):
- *   1. Content invalidation — runs FIRST so cached body parts cannot leak after
- *      the underlying objects are gone (a no-op where no CDN fronts content).
- *   2. Relational/DDB cascade delete in dependency order (children → parents).
- *   3. Storage prefix cleanup `accounts/{accountConfigId}/`.
- *   4. AccountConfig delete.
- *
- * Idempotency: every step is replay-safe. The cascade on a missing key is a
- * no-op, storage prefix delete on missing keys is a no-op, and invalidation is
- * always idempotent. No explicit "already deleted" pre-checks — they make
- * replays racier, not safer. Errors propagate so SQS retries the message;
- * eventually the DLQ alarm fires and an operator inspects the failure.
- */
 export const processAccountFinalize = async (
 	event: AccountDeleteFinalizeEvent,
 	log: Logger,
 	deps: ProcessFinalizeDeps = {},
-): Promise<void> => {
-	const { accountConfigId } = event;
-	const { capabilities, services } = await resolveDeps(deps);
-
-	// Validate every deploy-time precondition BEFORE the first destructive step,
-	// so a missing var fails loud instead of deleting rows in the cascade and
-	// only then erroring at the storage step.
-	await capabilities.assertReady(log);
-
-	// Step 1: content invalidation — first, so cached body parts cannot leak
-	// after the underlying objects are gone.
-	log.info(
-		{ accountConfigId },
-		"Invalidating cached content for erased account",
-	);
-	await capabilities.invalidateContent(accountConfigId, log);
-
-	// Step 2: cascade delete (children → parents). The AccountConfig row is
-	// excluded from the cascade plan and removed last, after storage.
-	//
-	// SQS is at-least-once: a successful cascade can be redelivered. After a
-	// clean run the AccountConfig row is gone, so `describe()` throws
-	// `NotFoundError` — that's the success signal, not a failure. Treat it as
-	// "cascade already complete" and return cleanly. Any other error propagates
-	// so SQS retries → DLQ.
-	let entities: CascadeEntity[];
-	try {
-		const enumeration = await enumerateCascadeEntities(
-			accountConfigId,
-			services,
-			log,
-		);
-		entities = enumeration.entities;
-	} catch (error) {
-		if ((error as { name?: string })?.name === "NotFoundError") {
-			log.info(
-				{ accountConfigId },
-				"Cascade already complete (AccountConfig not found on replay) — no-op",
-			);
-			return;
-		}
-		throw error;
-	}
-	const cascadeEntities = entities.filter(
-		(e) => e.entityType !== "AccountConfig",
-	);
-	await capabilities.cascadeDelete(cascadeEntities, log);
-
-	// Step 3: storage prefix cleanup. Runs AFTER the cascade so a mid-cascade
-	// replay always re-runs the (idempotent) storage step.
-	await capabilities.deleteStoragePrefix(`accounts/${accountConfigId}/`, log);
-
-	// Step 4: AccountConfig delete — the last write. The presence of an
-	// AccountConfig row with `deletedAt` set is the cascade-in-progress flag;
-	// removing the row is the only signal the cascade fully finished.
-	await services.accountConfigService.delete(accountConfigId);
-	log.info({ accountConfigId }, "AccountConfig deleted; cascade complete");
-};
+): Promise<void> => finalizeAccountDelete(event, log, await resolveDeps(deps));
 
 // ---------- SQS handler ----------
 

package/src/handlers/account-purge-finalize.test.ts

@@ -71,6 +71,9 @@ const recordingCapabilities = (): Recorded => {
 			cascadeDelete: async (entities) => {
 				deleted.push(...entities);
 			},
+			eraseAccountConfig: async () => {
+				throw new Error("a per-account purge never erases the account config");
+			},
 		},
 	};
 };

package/src/handlers/organize-job.test.ts

@@ -13,6 +13,7 @@ import {
 	BETTER_SQLITE3_LIMITS,
 	createSqliteRepositories,
 	type DrizzleThreadMessageRepository,
+	messageObjectsInMemory,
 } from "@remit/drizzle-service";
 import { createShippedSqliteDb } from "@remit/drizzle-service/test-sqlite";
 import { noopLogger } from "@remit/logger";
@@ -193,6 +194,7 @@ const world = async (filters: FilterItem[]): Promise<World> => {
 			search: store.db,
 		},
 		BETTER_SQLITE3_LIMITS,
+		messageObjectsInMemory(),
 	).threadMessage;
 	const { inbox, archive } = await seedThreads(threadMessage);
 	const updates: Update[] = [];