@remit/account-worker 0.0.1

package/src/config.ts

@@ -0,0 +1,70 @@
+import { SQSClient } from "@aws-sdk/client-sqs";
+import { getClient } from "@remit/backend/client";
+import { resolveSqsCredentials } from "@remit/sqs-client";
+import type { StorageService } from "@remit/storage-service";
+import { createStorageService } from "@remit/storage-service/s3";
+import { env } from "expect-env";
+import type { CascadeServices } from "./cascade.js";
+
+const remitClient = await getClient();
+
+export const sqsClient = new SQSClient({
+	credentials: resolveSqsCredentials(),
+});
+
+// SQS env vars are lazy-evaluated. The fanout worker needs them at handler time;
+// the finalize worker imports `cascadeServices` from this module but talks to no
+// queue, so its Lambda doesn't carry the env vars. Eager evaluation here would
+// crash finalize at module load — getters defer the read to the fanout-only call
+// sites.
+export const getSearchIndexQueueUrl = (): string =>
+	env.SQS_QUEUE_URL_SEARCH_INDEX;
+export const getImapWorkerQueueUrl = (): string =>
+	env.SQS_QUEUE_URL_IMAP_WORKER;
+export const getAccountFinalizeQueueUrl = (): string =>
+	env.SQS_QUEUE_URL_ACCOUNT_FINALIZE;
+export const getAccountPurgeDeleteQueueUrl = (): string =>
+	env.SQS_QUEUE_URL_ACCOUNT_PURGE_DELETE;
+
+const graceSecondsRaw = process.env.ACCOUNT_DELETION_GRACE_SECONDS;
+export const graceSeconds = graceSecondsRaw
+	? Number.parseInt(graceSecondsRaw, 10)
+	: 60;
+
+// Every cascade service is a `RemitClient` repository, so the whole enumeration
+// runs on whatever backend `getClient()` selected — DynamoDB (ElectroDB),
+// Postgres, or SQLite (Drizzle). Filter/FilterAnchor/Label/MessageLabel (Smart
+// Organize, RFC 034) are present on all three backends via the client, so they
+// need no backend-specific wiring here.
+export const cascadeServices: CascadeServices = {
+	accountConfigService: remitClient.accountConfig,
+	accountService: remitClient.account,
+	addressService: remitClient.address,
+	mailboxService: remitClient.mailbox,
+	messageService: remitClient.message,
+	messageFlagService: remitClient.messageFlag,
+	envelopeService: remitClient.envelope,
+	outboxMessageService: remitClient.outboxMessage,
+	threadMessageService: remitClient.threadMessage,
+	mailboxLockService: remitClient.mailboxLock,
+	messagePlacementMoveService: remitClient.placementMove,
+	messageFlagPushService: remitClient.flagPush,
+	accountExportRequestService: remitClient.accountExportRequest,
+	accountSettingService: remitClient.accountSetting,
+	filterService: remitClient.filter,
+	filterAnchorService: remitClient.filterAnchor,
+	labelService: remitClient.label,
+	messageLabelService: remitClient.messageLabel,
+};
+
+export const accountConfigService = cascadeServices.accountConfigService;
+
+let storageService: StorageService | null = null;
+export const getStorageService = (): StorageService => {
+	if (!storageService) {
+		storageService = createStorageService();
+	}
+	return storageService;
+};
+
+export const dataBackend = process.env.DATA_BACKEND;

package/src/deletion-capabilities.ts

@@ -0,0 +1,94 @@
+import type { Logger } from "@remit/logger-lambda";
+import type { CascadeEntity } from "./cascade.js";
+
+/**
+ * The four steps of the deletion cascade that vary by deployment. Everything
+ * else — enumerating the AccountConfig's rows, the FIFO purge orchestration, the
+ * queue sends — is backend-neutral and lives in the handlers unchanged.
+ *
+ * Two composition roots build this, selected by `DATA_BACKEND` exactly like
+ * `@remit/backend`'s `compose-{dynamodb,postgres,sqlite}`:
+ *
+ * - `compose-dynamodb.ts` (AWS custody stack): Cognito global sign-out,
+ *   CloudFront invalidation, raw-S3 prefix delete, DynamoDB `BatchWriteItem`
+ *   cascade. Byte-for-byte the pre-split behavior. Stripped from the open-core
+ *   export and loaded through a lazy `import()` so the closed AWS SDK/ElectroDB
+ *   graph never reaches the relational tree.
+ * - `compose-relational.ts` (self-host stack, RFC 035/036): no-op sign-out
+ *   (deleting the AccountConfig severs the session's data resolution; there is
+ *   no CDN to sign out of), no-op invalidation (Caddy proxies `/content/*`
+ *   straight to the backend with no cache), filesystem prefix delete, and the
+ *   Drizzle cascade over Postgres or SQLite.
+ */
+export interface DeletionCapabilities {
+	/**
+	 * Validate every precondition the destructive steps depend on, BEFORE any row
+	 * or object is deleted. AWS checks both `CONTENT_DISTRIBUTION_ID` and
+	 * `S3_STORAGE_BUCKET_NAME` up front so a missing var fails loud instead of
+	 * destroying rows in the cascade and only then erroring at the storage step.
+	 * Relational is a no-op — its steps carry no such deploy-time configuration.
+	 */
+	assertReady(log: Logger): Promise<void>;
+	/**
+	 * Invalidate the deleted user's active auth sessions. AWS: Cognito
+	 * `AdminUserGlobalSignOut`. Relational: no-op.
+	 */
+	signOut(userId: string, log: Logger): Promise<void>;
+	/**
+	 * Purge any CDN/edge cache under a tenant/account content prefix
+	 * (`accountConfigId` or `accountConfigId/accountId`). AWS: CloudFront
+	 * invalidation of `/content/accounts/{prefix}/*`. Relational: no-op.
+	 */
+	invalidateContent(contentPrefix: string, log: Logger): Promise<void>;
+	/**
+	 * Delete every stored object under a storage key prefix
+	 * (`accounts/{accountConfigId}/…`). AWS: raw-S3 list+delete. Relational:
+	 * recursive filesystem removal under the local storage root.
+	 */
+	deleteStoragePrefix(keyPrefix: string, log: Logger): Promise<void>;
+	/**
+	 * Remove the enumerated rows in dependency order (children → parents). AWS:
+	 * DynamoDB `BatchWriteItem`. Relational: Drizzle transaction (Postgres or
+	 * SQLite). Excludes the AccountConfig row, which the caller deletes last
+	 * through its repository as the cascade-in-progress marker.
+	 */
+	cascadeDelete(entities: CascadeEntity[], log: Logger): Promise<void>;
+}
+
+const isRelationalBackend = (): boolean => {
+	const backend = process.env.DATA_BACKEND;
+	return backend === "postgres" || backend === "sqlite";
+};
+
+const buildDeletionCapabilitiesFromEnv =
+	async (): Promise<DeletionCapabilities> => {
+		if (isRelationalBackend()) {
+			const { buildRelationalDeletionCapabilities } = await import(
+				"./compose-relational.js"
+			);
+			return buildRelationalDeletionCapabilities();
+		}
+		// `as string` stops tsgo resolving the module in the open-core tree, which
+		// strips it; the named contract keeps the call site typed rather than `any`.
+		type ComposeDynamoDBModule = {
+			buildDynamoDBDeletionCapabilities: () => DeletionCapabilities;
+		};
+		const { buildDynamoDBDeletionCapabilities } = (await import(
+			"./compose-dynamodb.js" as string
+		)) as ComposeDynamoDBModule;
+		return buildDynamoDBDeletionCapabilities();
+	};
+
+let capabilitiesPromise: Promise<DeletionCapabilities> | undefined;
+
+/**
+ * The process-wide deletion capabilities, built once per backend and reused
+ * across invocations. Both handlers await this in their default path; tests
+ * inject a `DeletionCapabilities` directly.
+ */
+export const getDeletionCapabilities = (): Promise<DeletionCapabilities> => {
+	if (!capabilitiesPromise) {
+		capabilitiesPromise = buildDeletionCapabilitiesFromEnv();
+	}
+	return capabilitiesPromise;
+};

package/src/events.ts

@@ -0,0 +1,85 @@
+export type AccountDeleteEvent = {
+	type: "AccountDelete";
+	accountConfigId: string;
+};
+
+/**
+ * Per-account hard-delete. Purges ONE mail account's data (mailboxes,
+ * messages, body parts, thread messages, S3 objects, search vectors) while
+ * keeping the AccountConfig/tenant and any sibling accounts intact. Emitted
+ * by the API on `deleteAccount`; consumed by the fanout worker.
+ */
+export type AccountDataPurgeEvent = {
+	type: "AccountDataPurge";
+	accountId: string;
+	accountConfigId: string;
+};
+
+export type AccountExportEvent = {
+	type: "AccountExport";
+	accountConfigId: string;
+	accountExportRequestId: string;
+};
+
+/**
+ * A "all like these" back-apply job (RFC 034, #1278). Rides the same
+ * account-fanout queue as export: the API writes a Pending OrganizeJobRequest
+ * row and enqueues this; the fanout worker matches the corpus and applies the
+ * action, driving the row to Complete/Failed. Never creates a Filter row.
+ */
+export type OrganizeJobEvent = {
+	type: "OrganizeJob";
+	accountConfigId: string;
+	organizeJobId: string;
+};
+
+export type AccountFanoutEvent =
+	| AccountDeleteEvent
+	| AccountDataPurgeEvent
+	| AccountExportEvent
+	| OrganizeJobEvent;
+
+export type AccountDeleteFinalizeEvent = {
+	type: "FinalizeAccountDelete";
+	accountConfigId: string;
+};
+
+/** One message subtree to delete: its manifest row plus the Message it indexes. */
+export type AccountDataPurgeSubtreeItem = {
+	threadMessageId: string;
+	messageId: string;
+};
+
+/**
+ * Destructive phase of the per-account purge. The fanout worker reads the
+ * account's ThreadMessage manifest and emits a stream of these onto a FIFO
+ * queue, single message group per account: a series of `subtrees` batches
+ * followed by exactly one `container` leftover. FIFO ordering guarantees the
+ * container runs only after every subtree delete — no fan-in counter and no
+ * self-re-enqueue. The finalize worker consumes them and already holds the
+ * S3-delete, CloudFront, and DDB batch-write grants.
+ *
+ * - `subtrees`: delete each `{ threadMessageId, messageId }` subtree (the
+ *   Message, its 9 child entities, and the manifest row). Idempotent.
+ * - `container`: the last message — delete the account-keyed container rows
+ *   (mailboxes, outbox, locks), the S3 prefix, and the CloudFront cache. The
+ *   tenant-shared Address is never deleted.
+ */
+export type AccountDataPurgeFinalizeEvent =
+	| {
+			type: "FinalizeAccountDataPurge";
+			kind: "subtrees";
+			accountId: string;
+			accountConfigId: string;
+			items: AccountDataPurgeSubtreeItem[];
+	  }
+	| {
+			type: "FinalizeAccountDataPurge";
+			kind: "container";
+			accountId: string;
+			accountConfigId: string;
+	  };
+
+export type AccountFinalizeEvent =
+	| AccountDeleteFinalizeEvent
+	| AccountDataPurgeFinalizeEvent;

package/src/handlers/account-export.ts

@@ -0,0 +1,132 @@
+import { PassThrough } from "node:stream";
+import type { Logger } from "@remit/logger-lambda";
+import type { StorageService } from "@remit/storage-service";
+import { ZipArchive } from "archiver";
+import type { CascadeServices } from "../cascade.js";
+import { cascadeServices, getStorageService } from "../config.js";
+import type { AccountExportEvent } from "../events.js";
+
+const THIRTY_DAYS_MILLIS = 30 * 24 * 60 * 60 * 1000;
+
+export interface ProcessAccountExportDeps {
+	services: CascadeServices;
+	storageService: StorageService;
+}
+
+const defaultDeps = (): ProcessAccountExportDeps => ({
+	services: cascadeServices,
+	storageService: getStorageService(),
+});
+
+const appendMessageBodies = async (
+	archive: ZipArchive,
+	accountConfigId: string,
+	services: CascadeServices,
+	storageService: StorageService,
+	log: Logger,
+): Promise<number> => {
+	const { accountConfigService, accountService, messageService } = services;
+	let appended = 0;
+
+	const accountConfigDescription =
+		await accountConfigService.describe(accountConfigId);
+
+	for (const account of accountConfigDescription.account) {
+		const accountDescription = await accountService.describe(account.accountId);
+
+		for (const mailbox of accountDescription.mailbox) {
+			const messages = await messageService.listAllByMailbox(mailbox.mailboxId);
+
+			for (const message of messages) {
+				const body = await storageService.retrieveMessageBodyStream(
+					accountConfigId,
+					account.accountId,
+					message.messageId,
+				);
+
+				if (!body) {
+					log.info(
+						{ accountConfigId, messageId: message.messageId },
+						"No raw body for message, skipping",
+					);
+					continue;
+				}
+
+				archive.append(body, {
+					name: `accounts/${account.accountId}/${mailbox.fullPath ?? mailbox.mailboxId}/${message.messageId}.eml`,
+				});
+				appended += 1;
+			}
+		}
+	}
+
+	return appended;
+};
+
+export const processAccountExport = async (
+	event: AccountExportEvent,
+	log: Logger,
+	deps: ProcessAccountExportDeps = defaultDeps(),
+): Promise<void> => {
+	const { accountConfigId, accountExportRequestId } = event;
+	const { services, storageService } = deps;
+	const { accountExportRequestService } = services;
+
+	await accountExportRequestService.get(accountExportRequestId);
+	await accountExportRequestService.update(accountExportRequestId, {
+		state: "Processing",
+	});
+	log.info(
+		{ accountConfigId, accountExportRequestId },
+		"Export processing started",
+	);
+
+	try {
+		const archive = new ZipArchive({ zlib: { level: 6 } });
+		const passthrough = new PassThrough();
+		archive.pipe(passthrough);
+
+		const uploadDone = storageService.storeExportArchiveStream(
+			accountConfigId,
+			accountExportRequestId,
+			passthrough,
+		);
+
+		const archiveSettled = new Promise<void>((resolve, reject) => {
+			archive.on("warning", reject);
+			archive.on("error", reject);
+			archive.on("end", resolve);
+		});
+
+		const entryCount = await appendMessageBodies(
+			archive,
+			accountConfigId,
+			services,
+			storageService,
+			log,
+		);
+
+		await archive.finalize();
+		await archiveSettled;
+		const objectKey = await uploadDone;
+
+		const expiresAt = Date.now() + THIRTY_DAYS_MILLIS;
+
+		await accountExportRequestService.update(accountExportRequestId, {
+			state: "Ready",
+			objectKey,
+			expiresAt,
+		});
+
+		log.info(
+			{ accountConfigId, accountExportRequestId, entryCount },
+			"Export ready",
+		);
+	} catch (error) {
+		await accountExportRequestService.update(accountExportRequestId, {
+			state: "Failed",
+			errorMessage: error instanceof Error ? error.message : String(error),
+		});
+		throw error;
+	}
+};

package/src/handlers/account-fanout.ts

@@ -0,0 +1,165 @@
+import { SendMessageCommand, type SQSClient } from "@aws-sdk/client-sqs";
+import { createLogger, type Logger, withTelemetry } from "@remit/logger-lambda";
+import type { SQSBatchResponse, SQSEvent, SQSHandler } from "aws-lambda";
+import type { CascadeServices } from "../cascade.js";
+import { enumerateCascadeEntities } from "../cascade.js";
+import {
+	cascadeServices,
+	getAccountFinalizeQueueUrl,
+	getAccountPurgeDeleteQueueUrl,
+	getImapWorkerQueueUrl,
+	sqsClient,
+} from "../config.js";
+import {
+	type DeletionCapabilities,
+	getDeletionCapabilities,
+} from "../deletion-capabilities.js";
+import type {
+	AccountDeleteEvent,
+	AccountFanoutEvent,
+	AccountFinalizeEvent,
+} from "../events.js";
+import { processAccountExport } from "./account-export.js";
+import { processAccountDataPurge } from "./account-purge.js";
+import { processOrganizeJob } from "./organize-job.js";
+
+export interface ProcessAccountFanoutDeps {
+	services?: CascadeServices;
+	sqs?: SQSClient;
+	signOut?: DeletionCapabilities["signOut"];
+	imapWorkerQueueUrl?: string;
+	accountFinalizeQueueUrl?: string;
+	accountPurgeDeleteQueueUrl?: string;
+}
+
+/**
+ * Fanout step of the account-deletion cascade. Read-only: enumerates the
+ * AccountConfig's data and dispatches downstream work — search-index
+ * deletes, per-account imap-worker stops, a sign-out, and the finalize
+ * enqueue. Throws on any non-swallowable error so SQS partial batch-failure
+ * retries the whole record.
+ */
+export const processAccountFanout = async (
+	event: AccountFanoutEvent,
+	log: Logger,
+	deps: ProcessAccountFanoutDeps = {},
+): Promise<void> => {
+	const services = deps.services ?? cascadeServices;
+	const sqs = deps.sqs ?? sqsClient;
+
+	if (event.type === "AccountDataPurge") {
+		await processAccountDataPurge(event, log, {
+			services,
+			sqs,
+			accountPurgeDeleteQueueUrl:
+				deps.accountPurgeDeleteQueueUrl ?? getAccountPurgeDeleteQueueUrl(),
+		});
+		return;
+	}
+
+	if (event.type === "AccountExport") {
+		await processAccountExport(event, log);
+		return;
+	}
+
+	if (event.type === "OrganizeJob") {
+		await processOrganizeJob(event, log);
+		return;
+	}
+
+	await processAccountDelete(event, log, services, sqs, deps);
+};
+
+const processAccountDelete = async (
+	event: AccountDeleteEvent,
+	log: Logger,
+	services: CascadeServices,
+	sqs: SQSClient,
+	deps: ProcessAccountFanoutDeps,
+): Promise<void> => {
+	const { accountConfigId } = event;
+	const signOut = deps.signOut ?? (await getDeletionCapabilities()).signOut;
+	const imapWorkerQueueUrl = deps.imapWorkerQueueUrl ?? getImapWorkerQueueUrl();
+	const accountFinalizeQueueUrl =
+		deps.accountFinalizeQueueUrl ?? getAccountFinalizeQueueUrl();
+
+	const accountConfig =
+		await services.accountConfigService.get(accountConfigId);
+	const userId = accountConfig.userId;
+
+	const { entities, messageIds: _messageIds } = await enumerateCascadeEntities(
+		accountConfigId,
+		services,
+		log,
+	);
+
+	const accountIds = entities
+		.filter((e) => e.entityType === "Account")
+		.map((e) => e.key.accountId);
+
+	for (const accountId of accountIds) {
+		await sqs.send(
+			new SendMessageCommand({
+				QueueUrl: imapWorkerQueueUrl,
+				MessageBody: JSON.stringify({
+					type: "IMAP_WORKER_STOP",
+					accountConfigId,
+					accountId,
+				}),
+			}),
+		);
+	}
+	log.info(
+		{ accountConfigId, count: accountIds.length },
+		"Enqueued imap-worker stop signals",
+	);
+
+	await signOut(userId, log);
+
+	const finalizeEvent: AccountFinalizeEvent = {
+		type: "FinalizeAccountDelete",
+		accountConfigId,
+	};
+	await sqs.send(
+		new SendMessageCommand({
+			QueueUrl: accountFinalizeQueueUrl,
+			MessageBody: JSON.stringify(finalizeEvent),
+		}),
+	);
+	log.info({ accountConfigId }, "Enqueued finalize event");
+};
+
+const log = createLogger();
+
+export const handler: SQSHandler = withTelemetry(
+	async (event: SQSEvent): Promise<SQSBatchResponse> => {
+		const batchItemFailures: { itemIdentifier: string }[] = [];
+
+		for (const record of event.Records) {
+			const fanoutEvent: AccountFanoutEvent = JSON.parse(record.body);
+			log.info(
+				{
+					eventType: fanoutEvent.type,
+					accountConfigId: fanoutEvent.accountConfigId,
+				},
+				"Processing account fanout event",
+			);
+
+			const failed = await processAccountFanout(fanoutEvent, log)
+				.then(() => false)
+				.catch((error) => {
+					log.error(
+						{ error, messageId: record.messageId },
+						"Account fanout event processing failed",
+					);
+					return true;
+				});
+
+			if (failed) {
+				batchItemFailures.push({ itemIdentifier: record.messageId });
+			}
+		}
+
+		return { batchItemFailures };
+	},
+);