@remit/mailbox-service 0.0.103 → 0.0.104

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@remit/mailbox-service",
-  "version": "0.0.103",
+  "version": "0.0.104",
   "type": "module",
   "main": "src/index.ts",
   "types": "src/index.ts",
@@ -25,6 +25,10 @@
       "types": "./src/account-credentials.ts",
       "default": "./src/account-credentials.ts"
     },
+    "./outbox-attachment-content": {
+      "types": "./src/outbox-attachment-content.ts",
+      "default": "./src/outbox-attachment-content.ts"
+    },
     "./types": {
       "types": "./src/types.ts",
       "default": "./src/types.ts"

package/src/outbox-attachment-content.test.ts

@@ -0,0 +1,145 @@
+import assert from "node:assert/strict";
+import { describe, it } from "node:test";
+import type { OutboxAttachmentItem } from "@remit/data-ports";
+import { createMockStorageService } from "@remit/storage-service";
+import {
+	loadOutboxAttachmentContents,
+	OutboxAttachmentUnavailableError,
+} from "./outbox-attachment-content.js";
+
+const OWNER = {
+	accountConfigId: "cfg-1",
+	accountId: "acc-1",
+	outboxMessageId: "ob-1",
+};
+const NOW = 1_000;
+
+const row = (
+	outboxAttachmentId: string,
+	overrides: Partial<OutboxAttachmentItem> = {},
+): OutboxAttachmentItem => ({
+	outboxAttachmentId,
+	outboxMessageId: OWNER.outboxMessageId,
+	accountId: OWNER.accountId,
+	accountConfigId: OWNER.accountConfigId,
+	filename: `${outboxAttachmentId}.txt`,
+	contentType: "text/plain",
+	sizeBytes: 5,
+	state: "Stored",
+	storageKey: `key/${outboxAttachmentId}`,
+	reservationExpiresAt: 0,
+	createdAt: 0,
+	updatedAt: 0,
+	...overrides,
+});
+
+const setup = async (rows: OutboxAttachmentItem[], stored: string[]) => {
+	const storage = createMockStorageService();
+	for (const outboxAttachmentId of stored) {
+		await storage.storeOutboxAttachment({
+			...OWNER,
+			outboxAttachmentId,
+			content: Buffer.from(`bytes of ${outboxAttachmentId}`),
+		});
+	}
+	return {
+		attachments: { listByOutboxMessage: async () => rows },
+		storage,
+	};
+};
+
+describe("loadOutboxAttachmentContents", () => {
+	it("returns every stored file with its bytes, in the order it was attached", async () => {
+		const deps = await setup(
+			[row("second", { createdAt: 20 }), row("first", { createdAt: 10 })],
+			["first", "second"],
+		);
+
+		const contents = await loadOutboxAttachmentContents(deps, OWNER, NOW);
+
+		assert.deepEqual(
+			contents.map((item) => [
+				item.filename,
+				item.contentType,
+				item.content.toString(),
+			]),
+			[
+				["first.txt", "text/plain", "bytes of first"],
+				["second.txt", "text/plain", "bytes of second"],
+			],
+		);
+	});
+
+	it("leaves out a reservation that lapsed without its bytes", async () => {
+		const deps = await setup(
+			[
+				row("kept"),
+				row("lapsed", { state: "Pending", reservationExpiresAt: NOW - 1 }),
+			],
+			["kept"],
+		);
+
+		const contents = await loadOutboxAttachmentContents(deps, OWNER, NOW);
+
+		assert.deepEqual(
+			contents.map((item) => item.filename),
+			["kept.txt"],
+		);
+	});
+
+	it("refuses to build a message while an upload is still open", async () => {
+		const deps = await setup(
+			[row("open", { state: "Pending", reservationExpiresAt: NOW + 60 })],
+			[],
+		);
+
+		await assert.rejects(
+			() => loadOutboxAttachmentContents(deps, OWNER, NOW),
+			(error: unknown) => {
+				assert.ok(error instanceof OutboxAttachmentUnavailableError);
+				assert.equal(error.filename, "open.txt");
+				assert.match(error.message, /never finished uploading/);
+				return true;
+			},
+		);
+	});
+
+	it("refuses to build a message whose stored file has no bytes behind it", async () => {
+		const deps = await setup([row("gone")], []);
+
+		await assert.rejects(
+			() => loadOutboxAttachmentContents(deps, OWNER, NOW),
+			(error: unknown) => {
+				assert.ok(error instanceof OutboxAttachmentUnavailableError);
+				assert.equal(
+					error.message,
+					'"gone.txt" could not be read: nothing is stored at key/gone',
+				);
+				return true;
+			},
+		);
+	});
+
+	it("names the file when storage itself fails", async () => {
+		const deps = {
+			attachments: { listByOutboxMessage: async () => [row("payroll")] },
+			storage: {
+				retrieveOutboxAttachment: async (): Promise<Buffer | null> => {
+					throw new Error("S3 is down");
+				},
+			},
+		};
+
+		await assert.rejects(
+			() => loadOutboxAttachmentContents(deps, OWNER, NOW),
+			(error: unknown) => {
+				assert.ok(error instanceof OutboxAttachmentUnavailableError);
+				assert.equal(
+					error.message,
+					'"payroll.txt" could not be read: storage failed (S3 is down)',
+				);
+				return true;
+			},
+		);
+	});
+});

package/src/outbox-attachment-content.ts

@@ -0,0 +1,99 @@
+import type { IOutboxAttachmentRepository } from "@remit/data-ports";
+import { holdsRoom } from "@remit/data-ports";
+import type { StorageService } from "@remit/storage-service";
+
+export interface OutboxAttachmentContent {
+	filename: string;
+	contentType: string;
+	content: Buffer;
+}
+
+/**
+ * A file on a message that cannot be put into it. Carries the file's name so
+ * whoever settles the message can tell the sender which file to fix.
+ */
+export class OutboxAttachmentUnavailableError extends Error {
+	constructor(
+		readonly filename: string,
+		readonly detail: string,
+		options?: { cause: unknown },
+	) {
+		super(`"${filename}" could not be read: ${detail}`, options);
+		this.name = "OutboxAttachmentUnavailableError";
+	}
+}
+
+const describeCause = (error: unknown): string =>
+	error instanceof Error ? error.message : String(error);
+
+export interface OutboxAttachmentContentDeps {
+	attachments: Pick<IOutboxAttachmentRepository, "listByOutboxMessage">;
+	storage: Pick<StorageService, "retrieveOutboxAttachment">;
+}
+
+export interface OutboxAttachmentOwner {
+	accountConfigId: string;
+	accountId: string;
+	outboxMessageId: string;
+}
+
+/**
+ * Every file a message goes out with, bytes included, in the order they were
+ * attached. The send op refuses a draft with an upload still open, so a live
+ * `Pending` row here, or a `Stored` row with nothing behind it, is a message
+ * that cannot be built as the sender saw it — and sending it without the file
+ * would be worse than not sending it.
+ */
+export const loadOutboxAttachmentContents = async (
+	deps: OutboxAttachmentContentDeps,
+	owner: OutboxAttachmentOwner,
+	nowSeconds: number,
+): Promise<OutboxAttachmentContent[]> => {
+	const held = await deps.attachments.listByOutboxMessage(
+		owner.accountConfigId,
+		owner.outboxMessageId,
+	);
+	const live = held
+		.filter((item) => holdsRoom(item, nowSeconds))
+		.sort((a, b) => a.createdAt - b.createdAt);
+
+	const unfinished = live.find((item) => item.state !== "Stored");
+	if (unfinished) {
+		throw new OutboxAttachmentUnavailableError(
+			unfinished.filename,
+			`attachment ${unfinished.outboxAttachmentId} never finished uploading`,
+		);
+	}
+
+	return Promise.all(
+		live.map(async (item) => {
+			const content = await Promise.resolve()
+				.then(() =>
+					deps.storage.retrieveOutboxAttachment(
+						owner.accountConfigId,
+						owner.accountId,
+						owner.outboxMessageId,
+						item.outboxAttachmentId,
+					),
+				)
+				.catch((error: unknown) => {
+					throw new OutboxAttachmentUnavailableError(
+						item.filename,
+						`storage failed (${describeCause(error)})`,
+						{ cause: error },
+					);
+				});
+			if (content === null) {
+				throw new OutboxAttachmentUnavailableError(
+					item.filename,
+					`nothing is stored at ${item.storageKey}`,
+				);
+			}
+			return {
+				filename: item.filename,
+				contentType: item.contentType,
+				content,
+			};
+		}),
+	);
+};

package/src/outbox-attachment.ts

@@ -13,6 +13,10 @@ import {
 	buildOutboxAttachmentKey,
 	UPLOAD_URL_TTL_SECONDS,
 } from "@remit/storage-service";
+import {
+	loadOutboxAttachmentContents,
+	type OutboxAttachmentContent,
+} from "./outbox-attachment-content.js";
 import {
 	normalizeAttachmentContentType,
 	sanitizeAttachmentFilename,
@@ -407,6 +411,34 @@ export class OutboxAttachmentService {
 		return held.filter((item) => holdsRoom(item, nowSeconds));
 	};
 
+	/**
+	 * The files a message goes out with, bytes included. The Sent copy is built
+	 * from the same rows the wire copy was, so the two carry the same parts.
+	 */
+	contentsFor = (
+		accountConfigId: string,
+		accountId: string,
+		outboxMessageId: string,
+	): Promise<OutboxAttachmentContent[]> =>
+		loadOutboxAttachmentContents(
+			{ attachments: this.attachments, storage: this.storage },
+			{ accountConfigId, accountId, outboxMessageId },
+			this.now(),
+		);
+
+	/**
+	 * The file that stops a send, if any: one still uploading, or one whose
+	 * upload never finished. A message must not leave without a file its sender
+	 * can see in the composer.
+	 */
+	unfinishedUpload = async (
+		accountConfigId: string,
+		outboxMessageId: string,
+	): Promise<OutboxAttachmentItem | undefined> => {
+		const live = await this.listFor(accountConfigId, outboxMessageId);
+		return live.find((item) => item.state !== "Stored");
+	};
+
 	/**
 	 * Drop the draft's lapsed reservations and answer with the ids that are still
 	 * good. The sweep runs this before it decides what to collect: a lapsed row

package/src/outbox-failed-recovery.test.ts

@@ -122,7 +122,9 @@ const createHarness = (
 
 	harness.service = new OutboxQueueService({
 		outboxMessageService,
-		outboxAttachmentService: {} as unknown as OutboxAttachmentService,
+		outboxAttachmentService: {
+			unfinishedUpload: async () => undefined,
+		} as unknown as OutboxAttachmentService,
 		accountService: {} as unknown as IAccountRepository,
 		sqsSmtpQueueUrl: "http://localhost/queue",
 		sqsClient: {

package/src/outbox-queue-settled-row.test.ts

@@ -87,6 +87,7 @@ const createHarness = (stored: OutboxMessageItem): Harness => {
 			discardAll: async (_configId: string, _accountId: string, id: string) => {
 				harness.discarded.push(id);
 			},
+			unfinishedUpload: async () => undefined,
 		} as unknown as OutboxAttachmentService,
 		accountService: {} as unknown as IAccountRepository,
 		sqsSmtpQueueUrl: "http://localhost/queue",

package/src/outbox-queue.test.ts

@@ -15,6 +15,7 @@ import { describe, it } from "node:test";
 import type {
 	IAccountRepository,
 	IOutboxMessageRepository,
+	OutboxAttachmentItem,
 	OutboxMessageItem,
 } from "@remit/data-ports";
 import { BadRequestError } from "@remit/data-ports/errors";
@@ -50,7 +51,12 @@ interface Harness {
 	statusWrites: string[];
 }
 
-const createHarness = (stored: OutboxMessageItem): Harness => {
+const createHarness = (
+	stored: OutboxMessageItem,
+	attachments: OutboxAttachmentItem[] = [],
+	lateAttachments: OutboxAttachmentItem[] = [],
+): Harness => {
+	let unfinishedChecks = 0;
 	const harness: Harness = {
 		service: undefined as unknown as OutboxQueueService,
 		enqueued: [],
@@ -85,7 +91,13 @@ const createHarness = (stored: OutboxMessageItem): Harness => {
 
 	harness.service = new OutboxQueueService({
 		outboxMessageService,
-		outboxAttachmentService: {} as unknown as OutboxAttachmentService,
+		outboxAttachmentService: {
+			unfinishedUpload: async () => {
+				unfinishedChecks += 1;
+				const held = unfinishedChecks === 1 ? attachments : lateAttachments;
+				return held.find((item) => item.state !== "Stored");
+			},
+		} as unknown as OutboxAttachmentService,
 		accountService: {} as unknown as IAccountRepository,
 		sqsSmtpQueueUrl: "http://localhost/queue",
 		sqsClient: {
@@ -99,6 +111,21 @@ const createHarness = (stored: OutboxMessageItem): Harness => {
 	return harness;
 };
 
+const pendingFile: OutboxAttachmentItem = {
+	outboxAttachmentId: "att-late",
+	outboxMessageId: OUTBOX_MESSAGE_ID,
+	accountId: ACCOUNT_ID,
+	accountConfigId: ACCOUNT_CONFIG_ID,
+	filename: "report.pdf",
+	contentType: "application/pdf",
+	sizeBytes: 10,
+	state: "Pending",
+	storageKey: "k",
+	reservationExpiresAt: Number.MAX_SAFE_INTEGER,
+	createdAt: 0,
+	updatedAt: 0,
+};
+
 const sendInput = (overrides: Record<string, unknown>) => ({
 	accountId: ACCOUNT_ID,
 	accountConfigId: ACCOUNT_CONFIG_ID,
@@ -126,6 +153,56 @@ describe("OutboxQueueService and a message with nowhere to go", () => {
 		assert.deepEqual(harness.enqueued, [], "nothing reached the SMTP queue");
 	});
 
+	it("puts the draft back when a reservation lands between the check and the queue", async () => {
+		const harness = createHarness(draft({}), [], [pendingFile]);
+
+		await assert.rejects(
+			() => harness.service.send(ACCOUNT_CONFIG_ID, OUTBOX_MESSAGE_ID),
+			(error: unknown) => {
+				assert.ok(error instanceof BadRequestError);
+				assert.match(error.message, /"report\.pdf" has not finished uploading/);
+				return true;
+			},
+		);
+
+		assert.deepEqual(harness.statusWrites, [
+			OutboxMessageStatus.queued,
+			OutboxMessageStatus.draft,
+		]);
+		assert.deepEqual(harness.enqueued, [], "nothing reached the SMTP queue");
+	});
+
+	it("refuses to queue a draft with a file still uploading, and names the file", async () => {
+		const harness = createHarness(draft({}), [
+			{
+				outboxAttachmentId: "att-1",
+				outboxMessageId: OUTBOX_MESSAGE_ID,
+				accountId: ACCOUNT_ID,
+				accountConfigId: ACCOUNT_CONFIG_ID,
+				filename: "report.pdf",
+				contentType: "application/pdf",
+				sizeBytes: 10,
+				state: "Pending",
+				storageKey: "k",
+				reservationExpiresAt: Number.MAX_SAFE_INTEGER,
+				createdAt: 0,
+				updatedAt: 0,
+			},
+		]);
+
+		await assert.rejects(
+			() => harness.service.send(ACCOUNT_CONFIG_ID, OUTBOX_MESSAGE_ID),
+			(error: unknown) => {
+				assert.ok(error instanceof BadRequestError);
+				assert.match(error.message, /"report\.pdf" has not finished uploading/);
+				return true;
+			},
+		);
+
+		assert.deepEqual(harness.statusWrites, [], "it stayed a draft");
+		assert.deepEqual(harness.enqueued, [], "nothing reached the SMTP queue");
+	});
+
 	it("queues a draft addressed only in Bcc — that envelope is real", async () => {
 		const harness = createHarness(
 			draft({ toAddresses: [], bccAddresses: ["them@example.com"] }),

package/src/outbox-queue.ts

@@ -269,6 +269,16 @@ export class OutboxQueueService {
 			throw new BadRequestError(NO_RECIPIENT_MESSAGE);
 		}
 
+		const unfinished = await this.outboxAttachmentService.unfinishedUpload(
+			accountConfigId,
+			outboxMessageId,
+		);
+		if (unfinished) {
+			throw new BadRequestError(
+				`"${unfinished.filename}" has not finished uploading. Wait for it, or remove it, and send again.`,
+			);
+		}
+
 		// Conditional on the status this send was decided against, so two presses —
 		// or a press racing the worker — produce one queued row and one conflict
 		// rather than two events for the same message.
@@ -280,6 +290,25 @@ export class OutboxQueueService {
 		);
 		if (!updated) throw new ConflictError(MOVED_WHILE_SENDING_MESSAGE);
 
+		// A reservation made from another tab between the check above and the
+		// move to `queued` got past it; one made after the move is refused, so a
+		// second look now is the last word.
+		const lateUpload = await this.outboxAttachmentService.unfinishedUpload(
+			accountConfigId,
+			outboxMessageId,
+		);
+		if (lateUpload) {
+			await this.outboxMessageService.updateIfStatus(
+				accountConfigId,
+				outboxMessageId,
+				OutboxMessageStatus.queued,
+				{ status: existing.status },
+			);
+			throw new BadRequestError(
+				`"${lateUpload.filename}" has not finished uploading. Wait for it, or remove it, and send again.`,
+			);
+		}
+
 		// `queued` is a dead end for a row the queue never accepted: `send` takes
 		// draft, failed and blocked, `deleteDraft` those three plus unfiled, so a
 		// row parked at `queued` by a failed enqueue is neither sendable nor