@remit/account-worker 0.0.9 → 0.0.11

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@remit/account-worker",
-  "version": "0.0.9",
+  "version": "0.0.11",
   "type": "module",
   "main": "dist/index.js",
   "exports": {
@@ -28,7 +28,7 @@
   "scripts": {
     "bundle": "esbuild src/index.ts --sourcemap --bundle --platform=node --format=esm --outfile=dist/index.js",
     "test:typecheck": "tsgo --noEmit",
-    "test:run": "node --env-file=../../localhost-test-unit.env --experimental-test-coverage --test-coverage-include='src/**' --test-coverage-exclude='src/**/*.test.ts' --test-coverage-exclude='src/**/*.test.tsx' --test-coverage-lines=50 --test 'src/**/*.test.ts'",
+    "test:run": "node --env-file=../../localhost-test-unit.env --experimental-test-coverage --test-coverage-include='src/**' --test-coverage-exclude='src/**/*.test.ts' --test-coverage-exclude='src/**/*.test.tsx' --test-coverage-lines=75 --test 'src/**/*.test.ts'",
     "test": "npm run test:typecheck && npm run test:run"
   },
   "dependencies": {

package/src/cascade-enumerate.test.ts

@@ -0,0 +1,216 @@
+import assert from "node:assert/strict";
+import { describe, it } from "node:test";
+import type { MessageDescription } from "@remit/data-ports";
+import type { Logger } from "@remit/logger-lambda";
+import {
+	type CascadeEntity,
+	type CascadeServices,
+	collectMessageChildEntities,
+	enumerateCascadeEntities,
+} from "./cascade.js";
+
+const noopLog = {
+	info: () => {},
+	warn: () => {},
+	error: () => {},
+	debug: () => {},
+	fatal: () => {},
+	trace: () => {},
+	child: () => noopLog,
+} as unknown as Logger;
+
+const messageWithChildren = {
+	messageFlag: [{ messageFlagId: "flag-1" }],
+	envelope: [{ envelopeId: "env-1" }],
+	messageReference: [{ messageReferenceId: "ref-1" }],
+	envelopeAddress: [{ envelopeAddressId: "ea-1" }],
+	bodyPart: [{ bodyPartId: "bp-1" }],
+	bodyPartParameter: [{ bodyPartParameterId: "bpp-1" }],
+	rawMessageStorage: [{ rawStorageId: "rms-1" }],
+	bodyPartStorage: [{ bodyPartStorageId: "bps-1" }],
+	bodyPartContent: [{ bodyPartContentId: "bpc-1" }],
+} as unknown as MessageDescription;
+
+describe("collectMessageChildEntities", () => {
+	it("expands the nine child entities of one message", () => {
+		const entities: CascadeEntity[] = [];
+		collectMessageChildEntities(entities, messageWithChildren);
+
+		const byType = entities.map((e) => e.entityType);
+		assert.deepEqual(byType, [
+			"MessageFlag",
+			"Envelope",
+			"MessageReference",
+			"EnvelopeAddress",
+			"BodyPart",
+			"BodyPartParameter",
+			"RawMessageStorage",
+			"BodyPartStorage",
+			"BodyPartContent",
+		]);
+		assert.equal(
+			entities.find((e) => e.entityType === "BodyPartContent")?.key
+				.bodyPartContentId,
+			"bpc-1",
+		);
+	});
+
+	it("appends nothing when every child list is empty", () => {
+		const entities: CascadeEntity[] = [];
+		collectMessageChildEntities(entities, {
+			messageFlag: [],
+			envelope: [],
+			messageReference: [],
+			envelopeAddress: [],
+			bodyPart: [],
+			bodyPartParameter: [],
+			rawMessageStorage: [],
+			bodyPartStorage: [],
+			bodyPartContent: [],
+		} as unknown as MessageDescription);
+		assert.equal(entities.length, 0);
+	});
+});
+
+const fullServices = (): CascadeServices =>
+	({
+		accountConfigService: {
+			describe: async () => ({
+				account: [{ accountId: "acc-1" }],
+				address: [{ addressId: "addr-1" }],
+			}),
+		},
+		accountService: {
+			describe: async () => ({ mailbox: [{ mailboxId: "mbx-1" }] }),
+		},
+		messageService: {
+			listAllByMailbox: async () => [{ messageId: "msg-1" }],
+			describe: async () => messageWithChildren,
+		},
+		outboxMessageService: {
+			listByAccount: async () => ({ items: [{ outboxMessageId: "out-1" }] }),
+		},
+		mailboxLockService: {
+			listByAccount: async () => [{ mailboxId: "mbx-1", eventName: "expunge" }],
+		},
+		messagePlacementMoveService: {
+			listByAccountId: async () => [{ messageId: "msg-1" }],
+		},
+		messageFlagPushService: {
+			listByAccountId: async () => [{ messageId: "msg-1", flagName: "\\Seen" }],
+		},
+		threadMessageService: {
+			listAllByAccount: async () => [{ threadMessageId: "tm-1" }],
+		},
+		accountSettingService: {
+			listByAccountConfig: async () => [{ accountSettingId: "set-1" }],
+		},
+		filterService: {
+			listByAccountConfig: async () => [
+				{ filterId: "flt-1", hasAnchor: true },
+				{ filterId: "flt-2", hasAnchor: false },
+			],
+		},
+		filterAnchorService: {
+			get: async () => ({ filterId: "flt-1" }),
+		},
+		labelService: {
+			listByAccountConfig: async () => [{ labelId: "lbl-1" }],
+		},
+		messageLabelService: {
+			listByLabelId: async () => [{ messageLabelId: "ml-1" }],
+		},
+	}) as unknown as CascadeServices;
+
+describe("enumerateCascadeEntities", () => {
+	it("walks the whole account tree into a flat cascade plan", async () => {
+		const { entities, messageIds } = await enumerateCascadeEntities(
+			"cfg-1",
+			fullServices(),
+			noopLog,
+		);
+
+		assert.deepEqual(messageIds, ["msg-1"]);
+
+		const types = entities.map((e) => e.entityType);
+		for (const expected of [
+			"Account",
+			"Mailbox",
+			"Message",
+			"MessageFlag",
+			"BodyPartContent",
+			"OutboxMessage",
+			"MailboxLock",
+			"MessagePlacementMove",
+			"MessageFlagPush",
+			"ThreadMessage",
+			"AccountSetting",
+			"Filter",
+			"FilterAnchor",
+			"Label",
+			"MessageLabel",
+			"Address",
+			"AccountConfig",
+		]) {
+			assert.ok(types.includes(expected), `missing ${expected}`);
+		}
+
+		assert.equal(types[types.length - 1], "AccountConfig");
+	});
+
+	it("emits a Filter without a FilterAnchor when the filter has no anchor", async () => {
+		const services = fullServices();
+		services.filterService.listByAccountConfig = async () =>
+			[{ filterId: "flt-2", hasAnchor: false }] as never;
+
+		const { entities } = await enumerateCascadeEntities(
+			"cfg-1",
+			services,
+			noopLog,
+		);
+
+		assert.equal(entities.filter((e) => e.entityType === "Filter").length, 1);
+		assert.equal(
+			entities.filter((e) => e.entityType === "FilterAnchor").length,
+			0,
+		);
+	});
+
+	it("skips the FilterAnchor when the anchor lookup returns null", async () => {
+		const services = fullServices();
+		services.filterAnchorService.get = async () => null as never;
+
+		const { entities } = await enumerateCascadeEntities(
+			"cfg-1",
+			services,
+			noopLog,
+		);
+
+		assert.equal(
+			entities.filter((e) => e.entityType === "FilterAnchor").length,
+			0,
+		);
+	});
+
+	it("yields only the AccountConfig row for an empty tenant", async () => {
+		const services = fullServices();
+		services.accountConfigService.describe = async () =>
+			({ account: [], address: [] }) as never;
+		services.threadMessageService.listAllByAccount = async () => [];
+		services.accountSettingService.listByAccountConfig = async () => [];
+		services.filterService.listByAccountConfig = async () => [];
+		services.labelService.listByAccountConfig = async () => [];
+
+		const { entities, messageIds } = await enumerateCascadeEntities(
+			"cfg-empty",
+			services,
+			noopLog,
+		);
+
+		assert.equal(messageIds.length, 0);
+		assert.deepEqual(
+			entities.map((e) => e.entityType),
+			["AccountConfig"],
+		);
+	});
+});

package/src/handlers/account-export.test.ts

@@ -0,0 +1,150 @@
+import assert from "node:assert/strict";
+import { Readable } from "node:stream";
+import { describe, it } from "node:test";
+import type { Logger } from "@remit/logger-lambda";
+import type { StorageService } from "@remit/storage-service";
+import type { CascadeServices } from "../cascade.js";
+import type { AccountExportEvent } from "../events.js";
+import {
+	type ProcessAccountExportDeps,
+	processAccountExport,
+} from "./account-export.js";
+
+const noopLog = {
+	info: () => {},
+	warn: () => {},
+	error: () => {},
+	debug: () => {},
+	fatal: () => {},
+	trace: () => {},
+	child: () => noopLog,
+} as unknown as Logger;
+
+interface Update {
+	state: string;
+	objectKey?: string;
+	expiresAt?: number;
+	errorMessage?: string;
+}
+
+const bodyOf = (text: string): Readable => Readable.from([Buffer.from(text)]);
+
+const buildDeps = (
+	updates: Update[],
+	overrides: {
+		retrieveMessageBodyStream?: StorageService["retrieveMessageBodyStream"];
+		storeExportArchiveStream?: StorageService["storeExportArchiveStream"];
+	} = {},
+): ProcessAccountExportDeps => ({
+	services: {
+		accountExportRequestService: {
+			get: async () => ({ accountExportRequestId: "exp-1" }),
+			update: async (_id: string, patch: Update) => {
+				updates.push(patch);
+			},
+		},
+		accountConfigService: {
+			describe: async () => ({ account: [{ accountId: "acc-1" }] }),
+		},
+		accountService: {
+			describe: async () => ({
+				mailbox: [{ mailboxId: "mbx-1", fullPath: "INBOX" }],
+			}),
+		},
+		messageService: {
+			listAllByMailbox: async () => [
+				{ messageId: "msg-1" },
+				{ messageId: "msg-2" },
+			],
+		},
+	} as unknown as CascadeServices,
+	storageService: {
+		retrieveMessageBodyStream:
+			overrides.retrieveMessageBodyStream ??
+			(async (_cfg: string, _acc: string, messageId: string) =>
+				messageId === "msg-2" ? null : bodyOf("raw-eml")),
+		storeExportArchiveStream:
+			overrides.storeExportArchiveStream ??
+			(async (_cfg: string, _req: string, stream: NodeJS.ReadableStream) => {
+				await new Promise<void>((resolve) => {
+					stream.on("data", () => {});
+					stream.on("end", resolve);
+				});
+				return "exports/exp-1.zip";
+			}),
+	} as unknown as StorageService,
+});
+
+const event: AccountExportEvent = {
+	type: "AccountExport",
+	accountConfigId: "cfg-1",
+	accountExportRequestId: "exp-1",
+};
+
+describe("processAccountExport", () => {
+	it("drives the request Processing then Ready and stores the archive key", async () => {
+		const updates: Update[] = [];
+		await processAccountExport(event, noopLog, buildDeps(updates));
+
+		assert.deepEqual(
+			updates.map((u) => u.state),
+			["Processing", "Ready"],
+		);
+		const ready = updates[1];
+		assert.equal(ready?.objectKey, "exports/exp-1.zip");
+		assert.ok(
+			ready?.expiresAt && ready.expiresAt > Date.now(),
+			"expiresAt is set in the future",
+		);
+	});
+
+	it("skips messages that have no raw body without failing the export", async () => {
+		const updates: Update[] = [];
+		let retrieved = 0;
+		await processAccountExport(
+			event,
+			noopLog,
+			buildDeps(updates, {
+				retrieveMessageBodyStream: async (
+					_cfg: string,
+					_acc: string,
+					messageId: string,
+				) => {
+					retrieved += 1;
+					return messageId === "msg-2" ? null : bodyOf("raw");
+				},
+			}),
+		);
+
+		assert.equal(retrieved, 2, "both messages were consulted");
+		assert.equal(updates.at(-1)?.state, "Ready");
+	});
+
+	it("marks the request Failed and rethrows when storage upload rejects", async () => {
+		const updates: Update[] = [];
+		await assert.rejects(
+			processAccountExport(
+				event,
+				noopLog,
+				buildDeps(updates, {
+					storeExportArchiveStream: async (
+						_cfg: string,
+						_req: string,
+						stream: NodeJS.ReadableStream,
+					) => {
+						await new Promise<void>((resolve) => {
+							stream.on("data", () => {});
+							stream.on("end", resolve);
+						});
+						throw new Error("s3 down");
+					},
+				}),
+			),
+			/s3 down/,
+		);
+
+		const failed = updates.at(-1);
+		assert.equal(failed?.state, "Failed");
+		assert.equal(failed?.errorMessage, "s3 down");
+	});
+});

package/src/handlers/account-purge.test.ts

@@ -0,0 +1,191 @@
+import assert from "node:assert/strict";
+import { describe, it } from "node:test";
+import type { SQSClient } from "@aws-sdk/client-sqs";
+import type { Logger } from "@remit/logger-lambda";
+import type { CascadeServices } from "../cascade.js";
+import type { AccountDataPurgeEvent } from "../events.js";
+import {
+	type ProcessPurgeFanoutDeps,
+	processAccountDataPurge,
+} from "./account-purge.js";
+
+const noopLog = {
+	info: () => {},
+	warn: () => {},
+	error: () => {},
+	debug: () => {},
+	fatal: () => {},
+	trace: () => {},
+	child: () => noopLog,
+} as unknown as Logger;
+
+interface Sent {
+	queueUrl: string | undefined;
+	body: Record<string, unknown>;
+	entries?: Array<{ Id: string; MessageBody: string }>;
+}
+
+const recordingSqs = (sent: Sent[], failBatch = false): SQSClient =>
+	({
+		send: async (command: {
+			input: {
+				QueueUrl?: string;
+				MessageBody?: string;
+				Entries?: Array<{ Id: string; MessageBody: string }>;
+			};
+		}) => {
+			sent.push({
+				queueUrl: command.input.QueueUrl,
+				body: command.input.MessageBody
+					? JSON.parse(command.input.MessageBody)
+					: {},
+				entries: command.input.Entries,
+			});
+			if (command.input.Entries && failBatch) {
+				return { Failed: [{ Id: "0" }] };
+			}
+			return {};
+		},
+	}) as unknown as SQSClient;
+
+const buildServices = (
+	manifest: Array<{
+		threadMessageId: string;
+		messageId: string;
+		mailboxId: string;
+	}>,
+	mailboxIds: string[] = ["mbx-1"],
+): CascadeServices =>
+	({
+		accountService: {
+			describe: async () => ({
+				mailbox: mailboxIds.map((mailboxId) => ({ mailboxId })),
+			}),
+		},
+		threadMessageService: {
+			listAllByAccount: async () => manifest,
+		},
+	}) as unknown as CascadeServices;
+
+const event: AccountDataPurgeEvent = {
+	type: "AccountDataPurge",
+	accountId: "acc-1",
+	accountConfigId: "cfg-1",
+};
+
+const baseDeps = (
+	sent: Sent[],
+	overrides: Partial<ProcessPurgeFanoutDeps> = {},
+): ProcessPurgeFanoutDeps => ({
+	services: buildServices([
+		{ threadMessageId: "tm-1", messageId: "msg-1", mailboxId: "mbx-1" },
+		{ threadMessageId: "tm-2", messageId: "msg-2", mailboxId: "mbx-1" },
+	]),
+	sqs: recordingSqs(sent),
+	accountPurgeDeleteQueueUrl: "http://queue/purge",
+	searchIndexQueueUrl: "http://queue/search",
+	dataBackend: "dynamodb",
+	...overrides,
+});
+
+describe("processAccountDataPurge", () => {
+	it("enqueues vector deletes then a subtrees batch and one container leftover", async () => {
+		const sent: Sent[] = [];
+		await processAccountDataPurge(event, noopLog, baseDeps(sent));
+
+		const vectorBatch = sent.find((m) => m.queueUrl === "http://queue/search");
+		assert.equal(vectorBatch?.entries?.length, 2, "one entry per message");
+
+		const finalize = sent.filter((m) => m.queueUrl === "http://queue/purge");
+		const kinds = finalize.map((m) => m.body.kind);
+		assert.deepEqual(kinds, ["subtrees", "container"]);
+		assert.equal(
+			(finalize[0]?.body.items as unknown[])?.length,
+			2,
+			"both subtrees ride the one batch",
+		);
+	});
+
+	it("skips vector deletes on the postgres backend, still enqueuing finalize", async () => {
+		const sent: Sent[] = [];
+		await processAccountDataPurge(
+			event,
+			noopLog,
+			baseDeps(sent, { dataBackend: "postgres" }),
+		);
+
+		assert.equal(
+			sent.some((m) => m.queueUrl === "http://queue/search"),
+			false,
+			"no search-index enqueue on postgres",
+		);
+		assert.deepEqual(
+			sent
+				.filter((m) => m.queueUrl === "http://queue/purge")
+				.map((m) => m.body.kind),
+			["subtrees", "container"],
+		);
+	});
+
+	it("drops manifest rows outside the account's mailbox set", async () => {
+		const sent: Sent[] = [];
+		await processAccountDataPurge(event, noopLog, {
+			...baseDeps(sent),
+			services: buildServices([
+				{ threadMessageId: "tm-1", messageId: "msg-1", mailboxId: "mbx-1" },
+				{ threadMessageId: "tm-9", messageId: "msg-9", mailboxId: "other" },
+			]),
+		});
+
+		const subtrees = sent.find((m) => m.body.kind === "subtrees");
+		assert.equal((subtrees?.body.items as unknown[])?.length, 1);
+	});
+
+	it("no-ops when the account is already gone", async () => {
+		const sent: Sent[] = [];
+		await processAccountDataPurge(event, noopLog, {
+			...baseDeps(sent),
+			services: {
+				accountService: {
+					describe: async () => {
+						throw Object.assign(new Error("gone"), {
+							name: "NotFoundError",
+						});
+					},
+				},
+				threadMessageService: { listAllByAccount: async () => [] },
+			} as unknown as CascadeServices,
+		});
+
+		assert.equal(sent.length, 0, "nothing enqueued for a missing account");
+	});
+
+	it("rethrows a non-NotFound describe error", async () => {
+		const sent: Sent[] = [];
+		await assert.rejects(
+			processAccountDataPurge(event, noopLog, {
+				...baseDeps(sent),
+				services: {
+					accountService: {
+						describe: async () => {
+							throw new Error("dynamo throttled");
+						},
+					},
+					threadMessageService: { listAllByAccount: async () => [] },
+				} as unknown as CascadeServices,
+			}),
+			/dynamo throttled/,
+		);
+	});
+
+	it("throws when a vector-delete batch reports failed entries", async () => {
+		const sent: Sent[] = [];
+		await assert.rejects(
+			processAccountDataPurge(event, noopLog, {
+				...baseDeps(sent),
+				sqs: recordingSqs(sent, true),
+			}),
+			/failed entries/,
+		);
+	});
+});