@remit/smtp-worker 0.0.26 → 0.0.28

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@remit/smtp-worker",
-  "version": "0.0.26",
+  "version": "0.0.28",
   "type": "module",
   "main": "dist/index.js",
   "exports": {
@@ -16,17 +16,19 @@
   "scripts": {
     "bundle": "node build.mjs",
     "test:typecheck": "tsgo --noEmit",
-    "test:run": "node $NODE_TEST_FLAGS --env-file=../../localhost-test-unit.env --experimental-test-module-mocks --experimental-test-coverage --test-coverage-include='src/**' --test-coverage-exclude='src/**/*.test.ts' --test-coverage-exclude='src/**/*.test.tsx' --test-coverage-lines=83 --test 'src/**/*.test.ts'",
+    "test:run": "node $NODE_TEST_FLAGS --env-file=../../localhost-test-unit.env --experimental-test-module-mocks --experimental-test-coverage --test-coverage-include='src/**' --test-coverage-exclude='src/**/*.test.ts' --test-coverage-exclude='src/**/*.test.tsx' --test-coverage-exclude='src/handlers/send-message.ts' --test-coverage-lines=83 --test 'src/**/*.test.ts'",
     "test": "npm run test:typecheck && npm run test:run",
     "dev": "node --import tsx src/e2e-processor-shim.ts"
   },
   "dependencies": {
+    "@remit/logger": "*",
     "@remit/data-ports": "*",
     "@remit/drizzle-service": "*",
     "@remit/logger-lambda": "*",
     "@remit/sqs-client": "*",
     "@remit/secrets-service": "*",
     "@remit/smtp-service": "*",
+    "@remit/storage-service": "*",
     "@remit/domain-enums": "*",
     "@remit/mail-oauth-service": "*",
     "@remit/mailbox-service": "*",

package/src/data-ports.ts

@@ -3,6 +3,7 @@ import type {
 	IAddressRepository,
 	IEnvelopeRepository,
 	IMessageRepository,
+	IOutboxAttachmentRepository,
 	IOutboxMessageRepository,
 } from "@remit/data-ports";
 
@@ -13,6 +14,7 @@ import type {
 export interface SmtpDataPorts {
 	account: IAccountRepository;
 	outboxMessage: IOutboxMessageRepository;
+	outboxAttachment: IOutboxAttachmentRepository;
 	address: IAddressRepository;
 	message: IMessageRepository;
 	envelope: IEnvelopeRepository;
@@ -24,31 +26,25 @@ export interface SmtpDataPorts {
 // marked `external` for the Lambda esbuild build (LAMBDA_ESBUILD_OPTIONS), so
 // esbuild leaves this `import()` unresolved in the bundle. It resolves only
 // where the relational composition actually runs, which has the package
-// installed. Mirrors the search-index-worker data-ports seam.
+// installed.
 const buildSqliteDataPorts = async (): Promise<SmtpDataPorts> => {
-	const sqliteDbPath = process.env.SQLITE_DB_PATH;
-	if (!sqliteDbPath) throw new Error("SQLITE_DB_PATH is required");
-
 	const {
-		AccountRepo,
-		AddressRepo,
-		createSqliteDatabase,
-		DrizzleEnvelopeRepository,
-		DrizzleMessageRepository,
-		messageDataSchema,
-		OutboxMessageRepo,
+		createSqliteRepositories,
+		openSqliteDatabases,
+		sqliteDatabasePathsFromEnv,
 	} = await import("@remit/drizzle-service");
 
-	const { db } = await createSqliteDatabase(messageDataSchema, {
-		filename: sqliteDbPath,
-	});
+	const repositories = createSqliteRepositories(
+		await openSqliteDatabases(sqliteDatabasePathsFromEnv()),
+	);
 
 	return {
-		account: new AccountRepo(db),
-		outboxMessage: new OutboxMessageRepo(db),
-		address: new AddressRepo(db),
-		message: new DrizzleMessageRepository(db),
-		envelope: new DrizzleEnvelopeRepository(db),
+		account: repositories.account,
+		outboxMessage: repositories.outboxMessage,
+		outboxAttachment: repositories.outboxAttachment,
+		address: repositories.address,
+		message: repositories.message,
+		envelope: repositories.envelope,
 	};
 };
 
@@ -67,12 +63,12 @@ export const setSmtpDataPorts = (ports: SmtpDataPorts): void => {
 /**
  * The SMTP data ports for this process: the ones a composition root registered,
  * or — for a process that registered none — the SQLite composition this build
- * contains. `SQLITE_DB_PATH` is that composition's precondition, not a backend
+ * contains. `SQLITE_APP_DB_PATH` is that composition's precondition, not a backend
  * selection.
  */
 export const buildDataPortsFromEnv = async (): Promise<SmtpDataPorts> => {
 	if (injectedDataPorts) return injectedDataPorts;
-	if (process.env.SQLITE_DB_PATH) return buildSqliteDataPorts();
+	if (process.env.SQLITE_APP_DB_PATH) return buildSqliteDataPorts();
 	throw new Error(
 		"no SMTP data ports registered — register them with setSmtpDataPorts() from your composition root",
 	);

package/src/e2e-processor-shim.ts

@@ -4,12 +4,15 @@ import {
 	DeleteMessageCommand,
 	ReceiveMessageCommand,
 } from "@aws-sdk/client-sqs";
-import { createLogger } from "@remit/logger-lambda";
+import { createLogger } from "@remit/logger";
+import { installNodeLogger } from "@remit/logger-lambda/node";
 import { createQueueProducer } from "@remit/sqs-client/producer";
 import type { Context, SQSBatchResponse, SQSEvent } from "aws-lambda";
 import { env } from "expect-env";
 import { handler } from "./index.js";
 
+installNodeLogger();
+
 /**
  * E2E-only queue drainer.
  *

package/src/failure-kind.ts

@@ -1,4 +1,4 @@
-import type { MailFailureKind } from "@remit/logger-lambda";
+import type { MailFailureKind } from "@remit/logger-lambda/metrics";
 import { RefreshTokenError } from "@remit/mail-oauth-service";
 import { SmtpConnectionError } from "@remit/smtp-service";
 

package/src/handlers/attachment-storage.test.ts

@@ -0,0 +1,106 @@
+import assert from "node:assert/strict";
+import { describe, it } from "node:test";
+import type { OutboxAttachmentItem } from "@remit/data-ports";
+import { createMockStorageService } from "@remit/storage-service";
+import {
+	createAttachmentLoader,
+	createAttachmentReader,
+} from "./attachment-storage.js";
+
+describe("createAttachmentReader", () => {
+	it("refuses to read on a Lambda with no bucket, naming the missing setting", () => {
+		let created = 0;
+		const reader = createAttachmentReader(
+			{ AWS_LAMBDA_FUNCTION_NAME: "smtp-worker" },
+			() => {
+				created += 1;
+				return createMockStorageService();
+			},
+		);
+
+		assert.throws(
+			() => reader.retrieveOutboxAttachment("cfg", "acc", "ob", "att"),
+			/S3_BUCKET_NAME is not set on this worker/,
+		);
+		assert.equal(created, 0, "no filesystem fallback was built");
+	});
+
+	it("reads through storage built once, on first use", async () => {
+		const storage = createMockStorageService();
+		await storage.storeOutboxAttachment({
+			accountConfigId: "cfg",
+			accountId: "acc",
+			outboxMessageId: "ob",
+			outboxAttachmentId: "att",
+			content: Buffer.from("bytes"),
+		});
+		let created = 0;
+		const reader = createAttachmentReader(
+			{ AWS_LAMBDA_FUNCTION_NAME: "smtp-worker", S3_BUCKET_NAME: "bucket" },
+			() => {
+				created += 1;
+				return storage;
+			},
+		);
+
+		assert.equal(
+			String(await reader.retrieveOutboxAttachment("cfg", "acc", "ob", "att")),
+			"bytes",
+		);
+		await reader.retrieveOutboxAttachment("cfg", "acc", "ob", "att");
+		assert.equal(created, 1);
+	});
+});
+
+describe("createAttachmentLoader", () => {
+	it("reads the draft's stored files for the tenant it is given", async () => {
+		const storage = createMockStorageService();
+		await storage.storeOutboxAttachment({
+			accountConfigId: "cfg",
+			accountId: "acc",
+			outboxMessageId: "ob",
+			outboxAttachmentId: "att",
+			content: Buffer.from("%PDF"),
+		});
+		const row: OutboxAttachmentItem = {
+			outboxAttachmentId: "att",
+			outboxMessageId: "ob",
+			accountId: "acc",
+			accountConfigId: "cfg",
+			filename: "report.pdf",
+			contentType: "application/pdf",
+			sizeBytes: 4,
+			state: "Stored",
+			storageKey: "k",
+			reservationExpiresAt: 0,
+			createdAt: 0,
+			updatedAt: 0,
+		};
+		const asked: string[][] = [];
+		const load = createAttachmentLoader(
+			storage,
+			async () => ({
+				listByOutboxMessage: async (accountConfigId, outboxMessageId) => {
+					asked.push([accountConfigId, outboxMessageId]);
+					return [row];
+				},
+			}),
+			() => 1_000_000,
+		);
+
+		const files = await load(
+			{ accountConfigId: "cfg", accountId: "acc" },
+			"ob",
+		);
+
+		assert.deepEqual(asked, [["cfg", "ob"]]);
+		assert.deepEqual(
+			files.map((file) => [
+				file.filename,
+				file.contentType,
+				String(file.content),
+			]),
+			[["report.pdf", "application/pdf", "%PDF"]],
+		);
+	});
+});

package/src/handlers/attachment-storage.ts

@@ -0,0 +1,52 @@
+import type { IOutboxAttachmentRepository } from "@remit/data-ports";
+import { loadOutboxAttachmentContents } from "@remit/mailbox-service/outbox-attachment-content";
+import type { MailAttachment } from "@remit/smtp-service";
+import type { StorageService } from "@remit/storage-service";
+import type { SendTenant } from "./send-message-core.js";
+
+export type AttachmentReader = Pick<StorageService, "retrieveOutboxAttachment">;
+
+/**
+ * Where this worker reads a message's files from, built on the first read.
+ *
+ * Without a bucket the storage factory falls back to the local filesystem,
+ * which a Lambda does not have: every read would come back empty and blame the
+ * file. The check runs on the first read rather than at start, so a deployment
+ * not yet given the bucket still sends every message that carries no files.
+ */
+export const createAttachmentReader = (
+	processEnv: NodeJS.ProcessEnv,
+	createStorage: () => StorageService,
+): AttachmentReader => {
+	let storage: StorageService | null = null;
+	return {
+		retrieveOutboxAttachment: (...args) => {
+			if (processEnv.AWS_LAMBDA_FUNCTION_NAME && !processEnv.S3_BUCKET_NAME) {
+				throw new Error(
+					"S3_BUCKET_NAME is not set on this worker, so it has no attachment storage to read from",
+				);
+			}
+			if (!storage) storage = createStorage();
+			return storage.retrieveOutboxAttachment(...args);
+		},
+	};
+};
+
+/** The files a message goes out with, read from this worker's ports and storage. */
+export const createAttachmentLoader =
+	(
+		reader: AttachmentReader,
+		getRepository: () => Promise<
+			Pick<IOutboxAttachmentRepository, "listByOutboxMessage">
+		>,
+		now: () => number = Date.now,
+	) =>
+	async (
+		tenant: SendTenant,
+		outboxMessageId: string,
+	): Promise<MailAttachment[]> =>
+		loadOutboxAttachmentContents(
+			{ attachments: await getRepository(), storage: reader },
+			{ ...tenant, outboxMessageId },
+			Math.floor(now() / 1000),
+		);

package/src/handlers/engagement-counters.ts

@@ -1,5 +1,6 @@
 import type { OutboxMessageItem } from "@remit/data-ports";
-import type { Logger } from "@remit/logger-lambda";
+import { bareMessageId, wireMessageId } from "@remit/data-ports/id";
+import type { Logger } from "@remit/logger";
 import type { EngagementCounterDeps, SendTenant } from "./send-message-core.js";
 
 /**
@@ -119,8 +120,7 @@ const collectReferenceHeaders = (outbox: OutboxMessageItem): string[] => {
 	return result;
 };
 
-const headerVariants = (header: string): string[] => {
-	const stripped = header.replace(/^<|>$/g, "");
-	if (stripped === header) return [`<${header}>`, header];
-	return [header, stripped];
-};
+const headerVariants = (header: string): string[] => [
+	wireMessageId(header),
+	bareMessageId(header),
+];

package/src/handlers/send-message-core.ts

@@ -5,12 +5,15 @@ import type {
 	UpdateOutboxMessageInput,
 } from "@remit/data-ports";
 import { AccountAuthType, OutboxMessageStatus } from "@remit/domain-enums";
-import { type Logger, recordSmtpFailure } from "@remit/logger-lambda";
+import type { Logger } from "@remit/logger";
+import { recordSmtpFailure } from "@remit/logger-lambda/metrics";
 import { RefreshTokenError } from "@remit/mail-oauth-service";
 import type { CredentialResolution } from "@remit/mailbox-service/account-credentials";
+import { OutboxAttachmentUnavailableError } from "@remit/mailbox-service/outbox-attachment-content";
 import type { SecretsService } from "@remit/secrets-service";
 import {
 	buildMailMessage,
+	type MailAttachment,
 	type SendResult,
 	SmtpConnectionError,
 	type sendMail,
@@ -100,6 +103,10 @@ export interface SendMessageDeps {
 		lastError?: string,
 	) => Promise<void>;
 	send: typeof sendMail;
+	loadAttachments: (
+		tenant: SendTenant,
+		outboxMessageId: string,
+	) => Promise<MailAttachment[]>;
 	emitAppendSentMessage: (
 		accountId: string,
 		outboxMessageId: string,
@@ -156,6 +163,16 @@ const SENDABLE_STATUSES: ReadonlySet<OutboxMessageItem["status"]> = new Set([
 	OutboxMessageStatus.sending,
 ]);
 
+/**
+ * Why a message stopped at its files, in words the Outbox can show. A `failed`
+ * row can be edited, sent again and discarded (#933), so the sentence names
+ * the file and what to do about it.
+ */
+const attachmentFailureReason = (error: unknown): string =>
+	error instanceof OutboxAttachmentUnavailableError
+		? `${error.message}, so this message was not sent. Remove "${error.filename}", attach it again, and send.`
+		: `The files on this message could not be read (${error instanceof Error ? error.message : String(error)}), so it was not sent. Send it again.`;
+
 export const getSendMessageMaxAttempts = (
 	processEnv: NodeJS.ProcessEnv = process.env,
 ): number => attemptBudget("SEND_MESSAGE_MAX_ATTEMPTS", 3, processEnv);
@@ -301,9 +318,38 @@ export const sendMessage = async (
 	}
 	const smtpConfig = resolved.config;
 
+	// Below the budget a storage hiccup is retried like any other; at it the
+	// record would dead-letter with the row at `queued`, which can be neither
+	// sent again nor discarded (#845.8), so the row settles `failed` instead.
+	const attachments = await deps
+		.loadAttachments(
+			{ accountConfigId, accountId: account.accountId },
+			outboxMessageId,
+		)
+		.catch(async (error: unknown) => {
+			if (receiveCount < SEND_MESSAGE_MAX_ATTEMPTS) throw error;
+			log.error(
+				{ outboxMessageId, receiveCount, error: String(error) },
+				"Attachments could not be loaded; settling the row as failed",
+			);
+			if (outbox.status !== OutboxMessageStatus.draft) {
+				await deps.updateOutboxIfStatus(
+					accountConfigId,
+					outboxMessageId,
+					outbox.status,
+					{
+						status: OutboxMessageStatus.failed,
+						lastError: attachmentFailureReason(error),
+					},
+				);
+			}
+			return null;
+		});
+	if (attachments === null) return;
+
 	await deps.updateOutboxStatus(accountConfigId, outboxMessageId, "sending");
 
-	const message = buildMailMessage(outbox);
+	const message = buildMailMessage(outbox, attachments);
 
 	log.info(
 		{ outboxMessageId, to: outbox.toAddresses, subject: outbox.subject },

package/src/handlers/send-message.test.ts

@@ -7,11 +7,17 @@ import type {
 } from "@remit/data-ports";
 import { AccountAuthType } from "@remit/domain-enums";
 import { RefreshTokenError } from "@remit/mail-oauth-service";
+import { OutboxAttachmentUnavailableError } from "@remit/mailbox-service/outbox-attachment-content";
 import {
 	type EncryptedPayload,
 	serializeEncryptedPayload,
 } from "@remit/secrets-service";
-import { type SendResult, SmtpConnectionError } from "@remit/smtp-service";
+import {
+	type MailAttachment,
+	type MailMessage,
+	type SendResult,
+	SmtpConnectionError,
+} from "@remit/smtp-service";
 import type { SendMessageEvent } from "../events.js";
 import {
 	getSendMessageMaxAttempts,
@@ -104,6 +110,8 @@ const buildDeps = (
 		appendThrows?: Error;
 		resolveCredentials?: SendMessageDeps["resolveCredentials"];
 		send?: SendMessageDeps["send"];
+		attachments?: MailAttachment[];
+		attachmentsFail?: Error;
 	} = {},
 ): { deps: SendMessageDeps; recorded: Recorded } => {
 	const recorded: Recorded = {
@@ -194,6 +202,13 @@ const buildDeps = (
 					}
 				);
 			}),
+		loadAttachments: async (tenant, outboxMessageId) => {
+			assert.equal(tenant.accountConfigId, account.accountConfigId);
+			assert.equal(tenant.accountId, account.accountId);
+			assert.equal(outboxMessageId, outbox.outboxMessageId);
+			if (options.attachmentsFail) throw options.attachmentsFail;
+			return options.attachments ?? [];
+		},
 		emitAppendSentMessage: async (accountId, outboxMessageId) => {
 			recorded.appendCalls.push({ accountId, outboxMessageId });
 			if (options.appendThrows) throw options.appendThrows;
@@ -225,6 +240,65 @@ const event: SendMessageEvent = {
 };
 
 describe("sendMessage handler", () => {
+	it("retries a message whose files could not be read, below the budget", async () => {
+		const { deps, recorded } = buildDeps({
+			outbox: buildOutbox({ status: "queued" }),
+			attachmentsFail: new OutboxAttachmentUnavailableError(
+				"payroll.xlsx",
+				"nothing is stored at key/payroll",
+			),
+		});
+
+		await assert.rejects(() => sendMessage(event, silentLogger, deps, 1));
+
+		assert.equal(recorded.sendCalls, 0);
+		assert.deepEqual(recorded.updates, []);
+		assert.deepEqual(recorded.statuses, [], "never moved to sending");
+	});
+
+	it("settles a message whose files cannot be read as failed, naming the file, at the budget", async () => {
+		const { deps, recorded } = buildDeps({
+			outbox: buildOutbox({ status: "queued" }),
+			attachmentsFail: new OutboxAttachmentUnavailableError(
+				"payroll.xlsx",
+				"nothing is stored at key/payroll",
+			),
+		});
+
+		await sendMessage(event, silentLogger, deps, SEND_MESSAGE_MAX_ATTEMPTS);
+
+		assert.equal(recorded.sendCalls, 0);
+		assert.equal(recorded.updates.length, 1);
+		assert.equal(recorded.updates[0].patch.status, "failed");
+		assert.equal(
+			recorded.updates[0].patch.lastError,
+			'"payroll.xlsx" could not be read: nothing is stored at key/payroll, so this message was not sent. Remove "payroll.xlsx", attach it again, and send.',
+		);
+	});
+
+	it("sends every file the draft carries as a part of the message", async () => {
+		const attachments: MailAttachment[] = [
+			{
+				filename: "report.pdf",
+				contentType: "application/pdf",
+				content: Buffer.from("%PDF-1.4"),
+			},
+		];
+		const sent: MailMessage[] = [];
+		const { deps } = buildDeps({
+			attachments,
+			send: async (_config, message) => {
+				sent.push(message);
+				return { success: true, messageId: "smtp-mid-1", isTransient: false };
+			},
+		});
+
+		await sendMessage(event, silentLogger, deps);
+
+		assert.equal(sent.length, 1);
+		assert.deepEqual(sent[0].attachments, attachments);
+	});
+
 	it("marks status `blocked` when SMTP is disabled — never `sent`", async () => {
 		const { deps, recorded } = buildDeps({
 			account: buildAccount({ smtpEnabled: false }),

package/src/handlers/send-message.ts

@@ -2,7 +2,7 @@ import { randomUUID } from "node:crypto";
 import { SendMessageCommand } from "@aws-sdk/client-sqs";
 import { deriveAddressId, deriveMessageId } from "@remit/data-ports/id";
 import type { ConnectionState } from "@remit/domain-enums";
-import type { Logger } from "@remit/logger-lambda";
+import type { Logger } from "@remit/logger";
 import {
 	createMailOAuthService,
 	microsoftProviderConfig,
@@ -21,9 +21,14 @@ import {
 } from "@remit/secrets-service";
 import { sendMail } from "@remit/smtp-service";
 import { createQueueProducer } from "@remit/sqs-client/producer";
+import { createStorageService } from "@remit/storage-service/s3";
 import { env } from "expect-env";
 import { buildDataPortsFromEnv, type SmtpDataPorts } from "../data-ports.js";
 import type { SendMessageEvent } from "../events.js";
+import {
+	createAttachmentLoader,
+	createAttachmentReader,
+} from "./attachment-storage.js";
 import { sendMessage } from "./send-message-core.js";
 
 // The data ports are resolved lazily and cached, not at module load: the
@@ -35,6 +40,11 @@ const getPorts = (): Promise<SmtpDataPorts> => {
 	return portsPromise;
 };
 
+const loadAttachments = createAttachmentLoader(
+	createAttachmentReader(process.env, createStorageService),
+	async () => (await getPorts()).outboxAttachment,
+);
+
 const dataKeyProvider = createKmsDataKeyProvider(env.KMS_KEY_ID);
 const secrets = createSecretsService(dataKeyProvider);
 
@@ -177,6 +187,7 @@ export const handleSendMessage = (
 				);
 			},
 			send: sendMail,
+			loadAttachments,
 			emitAppendSentMessage,
 			engagement: {
 				resolveAddressId: deriveAddressId,

package/src/index.ts

@@ -1,16 +1,18 @@
 import { inspect } from "node:util";
+import { createLogger } from "@remit/logger";
 import {
-	createLogger,
 	queueNameFromEventSource,
 	recordQueueEvent,
 	recordSmtpFailure,
-	withTelemetry,
-} from "@remit/logger-lambda";
+} from "@remit/logger-lambda/metrics";
+import { installNodeLogger, withTelemetry } from "@remit/logger-lambda/node";
 import type { SQSBatchResponse, SQSEvent } from "aws-lambda";
 import type { SmtpEvent } from "./events.js";
 import { smtpFailureKind } from "./failure-kind.js";
 import { processEvent } from "./processor.js";
 
+installNodeLogger();
+
 const log = createLogger();
 
 /**

package/src/poller.ts

@@ -1,8 +1,12 @@
-import { createLogger, startMetricsServer } from "@remit/logger-lambda";
+import { createLogger } from "@remit/logger";
+import { startMetricsServer } from "@remit/logger-lambda/metrics";
+import { installNodeLogger } from "@remit/logger-lambda/node";
 import { runQueuePoller } from "@remit/sqs-client/poller";
 import { env } from "expect-env";
 import { handler } from "./index.js";
 
+installNodeLogger();
+
 /** Production queue poller — the deployed form of `e2e-processor-shim.ts`. */
 const log = createLogger();
 

package/src/processor.ts

@@ -1,4 +1,4 @@
-import type { Logger } from "@remit/logger-lambda";
+import type { Logger } from "@remit/logger";
 import type { SmtpEvent } from "./events.js";
 import { handleSendMessage } from "./handlers/send-message.js";