@remit/smtp-worker 0.0.1

package/build.mjs

@@ -0,0 +1,17 @@
+import * as esbuild from "esbuild";
+
+await esbuild.build({
+	entryPoints: ["src/index.ts"],
+	bundle: true,
+	platform: "node",
+	target: "node20",
+	format: "esm",
+	outdir: "dist",
+	sourcemap: true,
+	external: [
+		"@aws-sdk/*", // Use Lambda-provided SDK
+	],
+	banner: {
+		js: "import { createRequire } from 'module'; const require = createRequire(import.meta.url);",
+	},
+});

package/package.json

@@ -0,0 +1,40 @@
+{
+	"name": "@remit/smtp-worker",
+	"version": "0.0.1",
+	"type": "module",
+	"main": "dist/index.js",
+	"scripts": {
+		"bundle": "node build.mjs",
+		"test:typecheck": "tsgo --noEmit",
+		"test:run": "node --env-file=../../localhost-test-unit.env --test 'src/**/*.test.ts'",
+		"test": "npm run test:typecheck && npm run test:run"
+	},
+	"devDependencies": {
+		"@aws-sdk/client-sqs": "*",
+		"@aws-sdk/core": "*",
+		"@remit/data-ports": "*",
+		"@remit/drizzle-service": "*",
+		"@remit/domain-enums": "*",
+		"@remit/logger-lambda": "*",
+		"@remit/mail-oauth-service": "*",
+		"@remit/mailbox-service": "*",
+		"@remit/secrets-service": "*",
+		"@remit/smtp-service": "*",
+		"@remit/sqs-client": "*",
+		"@remit/storage-service": "*",
+		"@types/aws-lambda": "*",
+		"@types/pg": "^8.0.0",
+		"esbuild": "^0.27.7",
+		"expect-env": "*",
+		"p-map": "*"
+	},
+	"license": "MIT",
+	"publishConfig": {
+		"access": "public"
+	},
+	"repository": {
+		"type": "git",
+		"url": "git+https://github.com/remit-mail/remit.git",
+		"directory": "packages/smtp-worker"
+	}
+}

package/src/data-ports.ts

@@ -0,0 +1,126 @@
+import type {
+	IAccountRepository,
+	IAddressRepository,
+	IEnvelopeRepository,
+	IMessageRepository,
+	IOutboxMessageRepository,
+} from "@remit/data-ports";
+// Type-only: erased at build, so it carries no runtime dependency on
+// drizzle-orm. The value import lives inside the relational builders as a
+// dynamic `import()` — see the comment there for why.
+import type { NodePgDatabase } from "drizzle-orm/node-postgres";
+
+/**
+ * The repositories the SMTP send path reads and writes. One neutral seam over
+ * the three backends (DynamoDB / Postgres / SQLite), selected by
+ * `buildDataPortsFromEnv` so the same handler serves every stack.
+ */
+export interface SmtpDataPorts {
+	account: IAccountRepository;
+	outboxMessage: IOutboxMessageRepository;
+	address: IAddressRepository;
+	message: IMessageRepository;
+	envelope: IEnvelopeRepository;
+}
+
+// `@remit/drizzle-service` and `drizzle-orm/node-postgres` are loaded
+// lazily, inside these functions, instead of as static top-level imports. The
+// relational branches only ever run when `DATA_BACKEND` selects them — never on
+// the deployed Lambda — but a static import is bundled (and evaluated at module
+// load) regardless of whether its branch runs. Both packages are marked
+// `external` for the Lambda esbuild build (LAMBDA_ESBUILD_OPTIONS), so esbuild
+// leaves this `import()` unresolved in the bundle; it resolves only on the
+// relational path, which runs via `tsx`/the container bundle with both packages
+// installed. Mirrors the search-index-worker data-ports seam.
+const buildPostgresDataPorts = async (): Promise<SmtpDataPorts> => {
+	const pgConnectionUrl = process.env.PG_CONNECTION_URL;
+	if (!pgConnectionUrl) throw new Error("PG_CONNECTION_URL is required");
+
+	const {
+		AccountRepo,
+		AddressRepo,
+		DrizzleEnvelopeRepository,
+		DrizzleMessageRepository,
+		messageDataSchema,
+		OutboxMessageRepo,
+	} = await import("@remit/drizzle-service");
+	const { drizzle } = await import("drizzle-orm/node-postgres");
+
+	const db = drizzle(pgConnectionUrl, { schema: messageDataSchema });
+	const genericDb = db as unknown as NodePgDatabase<Record<string, unknown>>;
+	const messageDataDb = db as unknown as NodePgDatabase<
+		typeof messageDataSchema
+	>;
+
+	return {
+		account: new AccountRepo(genericDb),
+		outboxMessage: new OutboxMessageRepo(genericDb),
+		address: new AddressRepo(genericDb),
+		message: new DrizzleMessageRepository(messageDataDb),
+		envelope: new DrizzleEnvelopeRepository(messageDataDb),
+	};
+};
+
+// The SQLite twin of `buildPostgresDataPorts` (RFC 036): the same Drizzle repos
+// over the one shared SQLite file. `createSqliteDatabase` (and better-sqlite3
+// behind it) is kept external from the DynamoDB Lambda bundle by the same
+// dynamic-import treatment.
+const buildSqliteDataPorts = async (): Promise<SmtpDataPorts> => {
+	const sqliteDbPath = process.env.SQLITE_DB_PATH;
+	if (!sqliteDbPath) throw new Error("SQLITE_DB_PATH is required");
+
+	const {
+		AccountRepo,
+		AddressRepo,
+		createSqliteDatabase,
+		DrizzleEnvelopeRepository,
+		DrizzleMessageRepository,
+		messageDataSchema,
+		OutboxMessageRepo,
+	} = await import("@remit/drizzle-service");
+
+	const { db } = await createSqliteDatabase(messageDataSchema, {
+		filename: sqliteDbPath,
+	});
+	const genericDb = db as unknown as NodePgDatabase<Record<string, unknown>>;
+	const messageDataDb = db as unknown as NodePgDatabase<
+		typeof messageDataSchema
+	>;
+
+	return {
+		account: new AccountRepo(genericDb),
+		outboxMessage: new OutboxMessageRepo(genericDb),
+		address: new AddressRepo(genericDb),
+		message: new DrizzleMessageRepository(messageDataDb),
+		envelope: new DrizzleEnvelopeRepository(messageDataDb),
+	};
+};
+
+/**
+ * Select the SMTP data ports from the environment, mirroring the other workers'
+ * `DATA_BACKEND` selection:
+ *
+ * - `DATA_BACKEND=postgres` → Drizzle repos over `PG_CONNECTION_URL`.
+ * - `DATA_BACKEND=sqlite` → Drizzle repos over the shared `SQLITE_DB_PATH` file.
+ * - otherwise → ElectroDB services over `DYNAMODB_TABLE_NAME` (the AWS path,
+ *   unchanged from the pre-convergence worker).
+ *
+ * The DynamoDB branch loads its composition through a dynamic `import()`, so the
+ * sole importer of the closed `@remit/remit-electrodb-service`
+ * (`./compose-dynamodb.js`) is never loaded on the relational (open-core) tree,
+ * where that module is not shipped. esbuild still bundles it into the Lambda
+ * because the specifier is a relative import, not an `external`.
+ */
+export const buildDataPortsFromEnv = async (): Promise<SmtpDataPorts> => {
+	if (process.env.DATA_BACKEND === "postgres") return buildPostgresDataPorts();
+	if (process.env.DATA_BACKEND === "sqlite") return buildSqliteDataPorts();
+	// `as string` stops tsgo resolving the module in the open-core tree, which
+	// strips it; the named contract keeps the call site typed rather than `any`.
+	type ComposeDynamoDBModule = {
+		buildDynamoDBDataPorts: () => SmtpDataPorts;
+	};
+	const { buildDynamoDBDataPorts } = (await import(
+		"./compose-dynamodb.js" as string
+	)) as ComposeDynamoDBModule;
+	return buildDynamoDBDataPorts();
+};

package/src/e2e-processor-shim.ts

@@ -0,0 +1,162 @@
+#!/usr/bin/env node
+import {
+	DeleteMessageCommand,
+	ReceiveMessageCommand,
+	SQSClient,
+} from "@aws-sdk/client-sqs";
+import { AwsQueryProtocol } from "@aws-sdk/core/protocols";
+import { createLogger } from "@remit/logger-lambda";
+import { resolveSqsCredentials } from "@remit/sqs-client";
+import type { Context, SQSBatchResponse, SQSEvent } from "aws-lambda";
+import { env } from "expect-env";
+import { handler } from "./index.js";
+
+/**
+ * E2E-only queue drainer.
+ *
+ * Production binds the Lambda `handler` (`src/index.ts`) to the SMTP queue via
+ * an SQS event-source mapping. The e2e/CI stack runs on ElasticMQ, which has no
+ * event-source mapping, so this process supplies that missing piece: it
+ * long-polls the queue, wraps every received batch in an `SQSEvent`, and invokes
+ * the exact production `handler`. It then honours the returned
+ * `batchItemFailures` the way the SQS service would — deleting the messages
+ * that succeeded and leaving the failures un-deleted so their visibility
+ * timeout lapses and SQS redelivers them. No processing or failure logic lives
+ * here; the prod handler owns all of it.
+ *
+ * This is a test harness, not production code. If it crashes the e2e suite
+ * fails loudly, which is the desired signal — there is no crash net.
+ */
+
+const queueUrl = env.SQS_QUEUE_URL_SMTP;
+const queueName = new URL(queueUrl).pathname.split("/").pop();
+const log = createLogger().child({ queue: queueName });
+
+const isLocal = queueUrl.startsWith("http://localhost");
+const sqs = new SQSClient({
+	endpoint: isLocal ? new URL(queueUrl).origin : undefined,
+	...(isLocal && { protocol: AwsQueryProtocol }),
+	credentials: resolveSqsCredentials(),
+});
+
+const maxMessages = 10;
+
+let isShuttingDown = false;
+
+process.on("SIGINT", () => {
+	log.info("Worker received SIGINT, shutting down...");
+	isShuttingDown = true;
+});
+
+process.on("SIGTERM", () => {
+	log.info("Worker received SIGTERM, shutting down...");
+	isShuttingDown = true;
+});
+
+// Minimal Lambda Context: `withTelemetry` only reads `functionName` and adds it
+// to the logger; the prod handler never touches the rest.
+const lambdaContext = {
+	functionName: `e2e-smtp-worker-${queueName}`,
+} as Context;
+
+const pollQueue = async (): Promise<void> => {
+	log.info("Worker started, polling...", { maxMessages });
+
+	let consecutiveEmptyPolls = 0;
+
+	while (!isShuttingDown) {
+		const waitTime = consecutiveEmptyPolls > 0 ? 20 : 0;
+
+		const response = await sqs.send(
+			new ReceiveMessageCommand({
+				QueueUrl: queueUrl,
+				MaxNumberOfMessages: maxMessages,
+				WaitTimeSeconds: waitTime,
+				VisibilityTimeout: 300,
+				MessageSystemAttributeNames: ["ApproximateReceiveCount"],
+			}),
+		);
+
+		if (!response.Messages || response.Messages.length === 0) {
+			consecutiveEmptyPolls++;
+			continue;
+		}
+
+		consecutiveEmptyPolls = 0;
+
+		const messages = response.Messages.flatMap((m) =>
+			m.Body && m.ReceiptHandle && m.MessageId
+				? [
+						{
+							messageId: m.MessageId,
+							receiptHandle: m.ReceiptHandle,
+							body: m.Body,
+							receiveCount: m.Attributes?.ApproximateReceiveCount ?? "1",
+						},
+					]
+				: [],
+		);
+
+		if (messages.length === 0) {
+			continue;
+		}
+
+		log.info("Invoking handler for batch", { count: messages.length });
+
+		const event: SQSEvent = {
+			Records: messages.map((m) => ({
+				messageId: m.messageId,
+				receiptHandle: m.receiptHandle,
+				body: m.body,
+				attributes: {
+					ApproximateReceiveCount: m.receiveCount,
+					SentTimestamp: "0",
+					SenderId: "e2e",
+					ApproximateFirstReceiveTimestamp: "0",
+				},
+				messageAttributes: {},
+				md5OfBody: "",
+				eventSource: "aws:sqs",
+				eventSourceARN: queueUrl,
+				awsRegion: "local",
+			})),
+		};
+
+		const result = (await handler(event, lambdaContext)) as
+			| SQSBatchResponse
+			| undefined;
+
+		const failedIds = new Set(
+			(result?.batchItemFailures ?? []).map((f) => f.itemIdentifier),
+		);
+
+		// Mirror SQS partial-batch-failure semantics: delete the messages the
+		// handler reported as succeeded; leave failures un-deleted so their
+		// visibility timeout lapses and SQS redelivers them (and eventually
+		// dead-letters once maxReceiveCount is hit).
+		const succeeded = messages.filter((m) => !failedIds.has(m.messageId));
+
+		for (const message of succeeded) {
+			await sqs.send(
+				new DeleteMessageCommand({
+					QueueUrl: queueUrl,
+					ReceiptHandle: message.receiptHandle,
+				}),
+			);
+		}
+
+		log.info("Batch processed", {
+			deleted: succeeded.length,
+			leftForRedelivery: failedIds.size,
+		});
+	}
+
+	log.info("Worker polling stopped");
+};
+
+pollQueue()
+	.then(() => process.exit(0))
+	.catch((error) => {
+		log.error("Worker error", { error });
+		process.exit(1);
+	});

package/src/events.ts

@@ -0,0 +1,17 @@
+export interface BaseEvent {
+	accountId: string;
+	eventId: string; // Idempotency key
+	timestamp: number;
+}
+
+export interface SendMessageEvent extends BaseEvent {
+	type: "SEND_MESSAGE";
+	outboxMessageId: string;
+}
+
+export interface ProcessOutboxEvent extends BaseEvent {
+	type: "PROCESS_OUTBOX";
+	// Processes all queued messages for the account
+}
+
+export type SmtpEvent = SendMessageEvent | ProcessOutboxEvent;

package/src/handlers/engagement-counters.ts

@@ -0,0 +1,126 @@
+import type { OutboxMessageItem } from "@remit/data-ports";
+import type { Logger } from "@remit/logger-lambda";
+import type { EngagementCounterDeps, SendTenant } from "./send-message-core.js";
+
+/**
+ * Engagement counters for an outbound send.
+ *
+ * - Per recipient (To/Cc/Bcc): increment `outboundCount`, set `lastOutboundAt`
+ * - Per recipient that also appears as the original sender of a stored
+ *   message referenced by `In-Reply-To` or `References`: also increment
+ *   `replyCount`, set `lastReplyAt`
+ *
+ * Counters use atomic `ADD`. Drift under at-least-once SQS is accepted
+ * residual per EDD #232.
+ */
+export const writeEngagementCounters = async (
+	outbox: OutboxMessageItem,
+	tenant: SendTenant,
+	deps: EngagementCounterDeps,
+	log: Logger,
+): Promise<void> => {
+	const now = Date.now();
+	const recipients = collectRecipients(outbox);
+	if (recipients.length === 0) return;
+
+	const recipientAddressIds = new Map<string, string>();
+	for (const email of recipients) {
+		const addressId = deps.resolveAddressId(tenant.accountConfigId, email);
+		recipientAddressIds.set(email, addressId);
+	}
+
+	for (const addressId of recipientAddressIds.values()) {
+		await deps.incrementOutboundCount(tenant.accountConfigId, addressId, now);
+	}
+
+	const replyTargets = await resolveReplyTargets(outbox, tenant, deps, log);
+	for (const email of replyTargets) {
+		const addressId = recipientAddressIds.get(email);
+		if (!addressId) continue;
+		await deps.incrementReplyCount(tenant.accountConfigId, addressId, now);
+	}
+};
+
+const collectRecipients = (outbox: OutboxMessageItem): string[] => {
+	const all = [
+		...(outbox.toAddresses ?? []),
+		...(outbox.ccAddresses ?? []),
+		...(outbox.bccAddresses ?? []),
+	];
+	const seen = new Set<string>();
+	const result: string[] = [];
+	for (const raw of all) {
+		const email = normalizeEmail(raw);
+		if (!email) continue;
+		if (seen.has(email)) continue;
+		seen.add(email);
+		result.push(email);
+	}
+	return result;
+};
+
+const normalizeEmail = (raw: string): string | null => {
+	if (!raw) return null;
+	const trimmed = raw.trim();
+	if (trimmed === "") return null;
+	const angle = trimmed.match(/<([^>]+)>/);
+	const candidate = angle ? angle[1] : trimmed;
+	if (!candidate.includes("@")) return null;
+	return candidate.toLowerCase();
+};
+
+const resolveReplyTargets = async (
+	outbox: OutboxMessageItem,
+	tenant: SendTenant,
+	deps: EngagementCounterDeps,
+	log: Logger,
+): Promise<Set<string>> => {
+	const headerCandidates = collectReferenceHeaders(outbox);
+	if (headerCandidates.length === 0) return new Set();
+
+	const recipientSet = new Set(collectRecipients(outbox));
+
+	const matched = new Set<string>();
+	for (const header of headerCandidates) {
+		for (const variant of headerVariants(header)) {
+			const message = await deps.findMessageByHeader(tenant.accountId, variant);
+			if (!message) continue;
+			const fromEmail = await deps.getEnvelopeFromEmail(message.messageId);
+			if (!fromEmail) continue;
+			const normalized = fromEmail.toLowerCase();
+			if (recipientSet.has(normalized)) {
+				matched.add(normalized);
+			}
+			break;
+		}
+	}
+
+	if (matched.size === 0) {
+		log.debug?.(
+			{ outboxMessageId: outbox.outboxMessageId },
+			"No reply targets resolved from In-Reply-To/References",
+		);
+	}
+	return matched;
+};
+
+const collectReferenceHeaders = (outbox: OutboxMessageItem): string[] => {
+	const result: string[] = [];
+	const seen = new Set<string>();
+	const push = (raw: string | undefined): void => {
+		if (!raw) return;
+		const trimmed = raw.trim();
+		if (trimmed === "" || seen.has(trimmed)) return;
+		seen.add(trimmed);
+		result.push(trimmed);
+	};
+	push(outbox.inReplyTo);
+	for (const ref of outbox.references ?? []) push(ref);
+	return result;
+};
+
+const headerVariants = (header: string): string[] => {
+	const stripped = header.replace(/^<|>$/g, "");
+	if (stripped === header) return [`<${header}>`, header];
+	return [header, stripped];
+};