@remit/mailbox-service 0.0.110 → 0.0.112

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@remit/mailbox-service",
-  "version": "0.0.110",
+  "version": "0.0.112",
   "type": "module",
   "main": "src/index.ts",
   "types": "src/index.ts",
@@ -32,25 +32,40 @@
     "./types": {
       "types": "./src/types.ts",
       "default": "./src/types.ts"
+    },
+    "./test-helpers/fake-imap": {
+      "types": "./src/test-helpers/fake-imap.ts",
+      "default": "./src/test-helpers/fake-imap.ts"
+    },
+    "./test-helpers/test-queue": {
+      "types": "./src/test-helpers/test-queue.ts",
+      "default": "./src/test-helpers/test-queue.ts"
+    },
+    "./test-helpers/test-store": {
+      "types": "./src/test-helpers/test-store.ts",
+      "default": "./src/test-helpers/test-store.ts"
     }
   },
   "scripts": {
     "test:typecheck": "tsgo --noEmit",
     "test:run": "node $NODE_TEST_FLAGS --env-file=../../localhost-test-unit.env --import tsx --experimental-test-coverage --test-coverage-include='src/**' --test-coverage-exclude='src/**/*.test.ts' --test-coverage-exclude='src/**/*.test.tsx' --test-coverage-lines=86 --test 'src/**/*.test.ts'",
-    "test:integ": "RUN_INTEG_TESTS=1 node --env-file=../../localhost-test-unit.env --import tsx --test 'src/**/*.integ.test.ts'",
-    "test:e2e": "RUN_E2E_TESTS=1 node --env-file=../../localhost-test-e2e.env --import tsx --test --test-concurrency=1 'src/**/*.e2e.test.ts'",
     "test": "npm run test:typecheck && npm run test:run"
   },
+  "devDependencies": {
+    "@remit/drizzle-service": "*",
+    "@types/better-sqlite3": "^7.6.13",
+    "aws-sdk-client-mock": "*",
+    "better-sqlite3": "^12.11.1"
+  },
   "dependencies": {
     "imapflow": "^1.3.3",
-    "mailparser": "^3.9.14",
-    "nodemailer": "^9.0.3",
+    "mailparser": "3.12.2",
+    "nodemailer": "^10.0.6",
     "p-map": "^7.0.4",
     "tldts": "^7.4.9",
     "@remit/data-ports": "*",
     "@remit/domain-enums": "*",
     "@remit/mail-oauth-service": "*",
-    "@remit/search-service": "*",
     "@remit/secrets-service": "*",
     "@remit/sqs-client": "*",
     "@remit/storage-service": "*",

package/src/body-part-mapper.ts

@@ -46,7 +46,12 @@
 
 import { Buffer } from "node:buffer";
 import type { BodyPartItem } from "@remit/data-ports";
-import type { Attachment, ParsedMail } from "mailparser";
+import {
+	encapsulatedParts,
+	outermostMessages,
+	ownParts,
+} from "@remit/data-ports/message-parts";
+import { type Attachment, type ParsedMail, simpleParser } from "mailparser";
 
 export interface BodyPartContentPair {
 	partPath: string;
@@ -358,3 +363,48 @@ export const mapBodyPartsToContent = (
 
 	return pairs;
 };
+
+/**
+ * The parse every part's bytes are paired against. A message/rfc822 part stays
+ * one leaf holding the raw forwarded message whatever its disposition, so the
+ * forwarded message's own parts come only from parsing it in turn.
+ */
+export const parsePartSource = (source: Buffer): Promise<ParsedMail> =>
+	simpleParser(source, { ignoreEmbedded: true });
+
+/**
+ * `mapBodyPartsToContent`, extended to forwarded messages. mailparser hands a
+ * message/rfc822 part over as one attachment holding the raw forwarded
+ * message, so the parts inside it have no bytes in `parsed`. Each forwarded
+ * message is parsed on its own and its parts are paired against that parse,
+ * with their paths rebased so `<messagePath>.0` is the forwarded message's
+ * root — the shape `walkMimeStructure` stores.
+ */
+export const mapMessagePartsToContent = async (
+	bodyParts: readonly MapperInput[],
+	parsed: ParsedMail,
+	context: MapperContext = {},
+): Promise<BodyPartContentPair[]> => {
+	const messages = outermostMessages(bodyParts);
+	const pairs = mapBodyPartsToContent(ownParts(bodyParts), parsed, context);
+
+	for (const message of messages) {
+		const raw = pairs.find((pair) => pair.partPath === message.partPath);
+		const inner = encapsulatedParts(bodyParts, message.partPath);
+		if (!raw || inner.length === 0) continue;
+		const prefix = `${message.partPath}.`;
+		const nested = await mapMessagePartsToContent(
+			inner.map((part) => ({
+				...part,
+				partPath: part.partPath.slice(prefix.length),
+			})),
+			await parsePartSource(raw.content),
+			context,
+		);
+		for (const pair of nested) {
+			pairs.push({ ...pair, partPath: `${prefix}${pair.partPath}` });
+		}
+	}
+
+	return pairs;
+};

package/src/body-sync-queue.test.ts

@@ -1,123 +1,63 @@
 import assert from "node:assert";
-import { describe, it, mock } from "node:test";
-import {
-	type BodySyncQueueLogger,
-	BodySyncQueueService,
-} from "./body-sync-queue.js";
+import { afterEach, beforeEach, describe, it } from "node:test";
+import { BodySyncQueueService } from "./body-sync-queue.js";
+import { captureQueue, type QueueCapture } from "./test-helpers/test-store.js";
 
-interface CapturedInput {
-	QueueUrl: string;
-	MessageBody: string;
-	MessageGroupId?: string;
-	MessageDeduplicationId?: string;
-}
+let capture: QueueCapture;
+let errors: Array<[Record<string, unknown>, string]>;
 
-const buildService = (
-	overrides: {
-		sqsQueueUrl?: string;
-		send?: (command: { input: CapturedInput }) => Promise<unknown>;
-		logger?: BodySyncQueueLogger;
-	} = {},
-) => {
-	const sent: CapturedInput[] = [];
-	const send = mock.fn(
-		overrides.send ??
-			(async (command: { input: CapturedInput }) => {
-				sent.push(command.input);
-				return { MessageId: "id" };
-			}),
-	);
-	const service = new BodySyncQueueService({
-		sqsQueueUrl:
-			overrides.sqsQueueUrl ?? "http://localhost:9324/000/remit-body",
-		logger: overrides.logger,
+beforeEach(() => {
+	capture = captureQueue();
+	errors = [];
+});
+
+afterEach(() => capture.restore());
+
+const request = (uid?: number) =>
+	new BodySyncQueueService({
+		sqsQueueUrl: "http://localhost:9324/000/remit-body",
+		logger: {
+			info: () => {},
+			error: (obj, msg) => {
+				errors.push([obj, msg]);
+			},
+		},
+	}).requestBodySync({
+		accountId: "acc-1",
+		mailboxId: "mbx-1",
+		messageId: "msg-1",
+		...(uid === undefined ? {} : { uid }),
 	});
-	// @ts-expect-error - inject mock SQS client for the test
-	service.sqs = { send };
-	return { service, send, sent };
-};
 
 describe("BodySyncQueueService.requestBodySync", () => {
-	it("emits a SYNC_MESSAGE_BODY event with the message id and uid pair", async () => {
-		const { service, send, sent } = buildService();
+	it("asks for a forced re-sync of the message and its uid on a standard queue", async () => {
+		await request(42);
 
-		await service.requestBodySync({
-			accountId: "acc-1",
-			mailboxId: "mbx-1",
-			messageId: "msg-1",
-			uid: 42,
-		});
-
-		assert.equal(send.mock.calls.length, 1);
-		const event = JSON.parse(sent[0].MessageBody);
+		const [input] = capture.inputs();
+		const event = JSON.parse(input.MessageBody ?? "{}");
 		assert.equal(event.type, "SYNC_MESSAGE_BODY");
-		assert.equal(event.accountId, "acc-1");
-		assert.equal(event.mailboxId, "mbx-1");
 		assert.deepEqual(event.messageIds, ["msg-1"]);
 		assert.deepEqual(event.messages, [{ messageId: "msg-1", uid: 42 }]);
-		// The worker's skip guard must be bypassed for this message — a read
-		// miss already proved bodyStorageKey is stale.
 		assert.equal(event.force, true);
-		// A standard queue must not carry FIFO parameters.
-		assert.equal(sent[0].MessageGroupId, undefined);
+		assert.equal(input.MessageGroupId, undefined);
 	});
 
 	it("omits the uid pair when no uid is known", async () => {
-		const { service, sent } = buildService();
+		await request();
 
-		await service.requestBodySync({
-			accountId: "acc-1",
-			mailboxId: "mbx-1",
-			messageId: "msg-1",
-		});
-
-		const event = JSON.parse(sent[0].MessageBody);
-		assert.deepEqual(event.messageIds, ["msg-1"]);
-		assert.equal(event.messages, undefined);
+		assert.equal(capture.sent[0].body.messages, undefined);
 	});
 
-	it("never rejects when the queue send fails — the read path already returned 202", async () => {
-		const { service } = buildService({
-			send: async () => {
-				throw new Error("queue down");
-			},
-		});
-
-		await assert.doesNotReject(() =>
-			service.requestBodySync({
-				accountId: "acc-1",
-				mailboxId: "mbx-1",
-				messageId: "msg-1",
-			}),
-		);
-	});
-
-	it("logs an alertable body_sync_cue_enqueue_failed error instead of rejecting — the swallow is not silent operationally, only silent to the caller (RFC 033 D5)", async () => {
-		const errorCalls: Array<[Record<string, unknown>, string]> = [];
-		const logger: BodySyncQueueLogger = {
-			info: () => {},
-			error: (obj, msg) => {
-				errorCalls.push([obj, msg]);
-			},
-		};
-		const { service } = buildService({
-			send: async () => {
-				throw new Error("queue down");
-			},
-			logger,
+	it("never rejects when the queue is down, and raises an alertable error", async () => {
+		capture.answer(async () => {
+			throw new Error("queue down");
 		});
 
-		await service.requestBodySync({
-			accountId: "acc-1",
-			mailboxId: "mbx-1",
-			messageId: "msg-1",
-		});
+		await assert.doesNotReject(request());
 
-		assert.equal(errorCalls.length, 1);
-		const [fields, message] = errorCalls[0];
+		assert.equal(errors.length, 1);
+		const [fields, message] = errors[0];
 		assert.equal(fields.alert, "body_sync_cue_enqueue_failed");
-		assert.equal(fields.accountId, "acc-1");
-		assert.equal(fields.mailboxId, "mbx-1");
 		assert.equal(fields.messageId, "msg-1");
 		assert.match(message, /read path returned a retryable 202/);
 	});