@remit/backend 0.0.58 → 0.0.60

package/dev-server/server.ts

@@ -7,7 +7,10 @@ import {
 	renderMetrics,
 	setAccountSyncAges,
 } from "@remit/logger-lambda/metrics";
-import { isStorageNotFoundError } from "@remit/storage-service";
+import {
+	isStorageNotFoundError,
+	UPLOAD_ROUTE_PREFIX,
+} from "@remit/storage-service";
 import type { APIGatewayProxyResult } from "aws-lambda";
 import { env } from "expect-env";
 import express, {
@@ -25,6 +28,7 @@ import { parseAllowedOrigins, resolveAllowOrigin } from "./cors.js";
 import { createLambdaContext, createLambdaEvent } from "./lambda-helpers.js";
 import { checkRelationalStore } from "./relational-health.js";
 import { collectAccountSyncAges } from "./sync-age.js";
+import { receiveUpload } from "./upload-handler.js";
 
 const app = express();
 
@@ -100,9 +104,9 @@ if (isSelfHostBackend) {
 	app.all(/^\/api\/auth\//, toNodeHandler(auth));
 }
 
-app.use(express.json({ limit: "10mb" }));
-app.use(express.urlencoded({ extended: true }));
-
+// Ahead of the body parsers, not after them: a body refused for its size never
+// reaches a route, and without these headers the browser reads that refusal as
+// a network failure instead of the reason it carries.
 app.use((req: Request, res: Response, next: NextFunction) => {
 	const allowOrigin = resolveAllowOrigin(
 		req.headers.origin,
@@ -128,6 +132,42 @@ app.use((req: Request, res: Response, next: NextFunction) => {
 	}
 });
 
+// The self-hosted upload receiver, the write-side twin of /content below. The
+// URL that reaches here was minted by the storage backend and carries its own
+// authority: an HMAC over the storage key, an expiry and the exact byte count.
+// No bearer token, for the same reason /content has none — the grant has to
+// travel in the URL itself. A hosted deployment presigns against block storage
+// instead and nothing arrives here at all.
+app.put(
+	new RegExp(`^${UPLOAD_ROUTE_PREFIX}.+$`),
+	async (req: Request, res: Response) => {
+		const storageKey = req.path.slice(UPLOAD_ROUTE_PREFIX.length);
+		const client = await getClient();
+
+		const result = await receiveUpload(client.storage, {
+			storageKey,
+			exp: typeof req.query.exp === "string" ? req.query.exp : undefined,
+			max: typeof req.query.max === "string" ? req.query.max : undefined,
+			sig: typeof req.query.sig === "string" ? req.query.sig : undefined,
+			body: req,
+			nowSeconds: Math.floor(Date.now() / 1000),
+			secret: process.env.BETTER_AUTH_SECRET,
+			findLiveReservation: (accountConfigId, outboxAttachmentId) =>
+				client.outboxAttachment.hasLiveReservation(
+					accountConfigId,
+					outboxAttachmentId,
+				),
+		});
+
+		res.setHeader("x-remit-upload-reason", result.reason);
+		res.status(result.status);
+		res.send(result.status === 204 ? undefined : result.reason);
+	},
+);
+
+app.use(express.json({ limit: "10mb" }));
+app.use(express.urlencoded({ extended: true }));
+
 app.get("/.well-known/appspecific/com.chrome.devtools.json", () => ({}));
 
 app.get("/health", async (_req: Request, res: Response) => {
@@ -319,7 +359,9 @@ const port = env.SERVER_PORT;
 //
 // This is also the everyday `npm run dev` server, so the addresses stay whole
 // and clickable — `url`, not a port a developer has to assemble one themselves.
-app.listen(Number(port), "0.0.0.0", () => {
+// Exported so a test that drives this exact app can shut the listener down
+// afterwards; nothing else has any business holding it.
+export const listener = app.listen(Number(port), "0.0.0.0", () => {
 	// biome-ignore lint/plugin/no-logger-info: the configuration a container came up on is an audit-grade signal
 	logger.info(
 		{

package/dev-server/upload-handler.ts

@@ -0,0 +1,132 @@
+import type { Readable } from "node:stream";
+import {
+	authorizeUploadRequest,
+	parseOutboxAttachmentKey,
+	type StorageService,
+} from "@remit/storage-service";
+
+/**
+ * The self-hosted receiver for an attachment upload.
+ *
+ * A hosted deployment mints a presigned PUT and none of this runs: the browser
+ * writes to block storage and the size is bound by the storage service itself.
+ * Where there is no storage service to presign against, this route is the
+ * equivalent, and it has to bind the same things — one attachment, one size,
+ * for a while — on its own.
+ */
+
+export type CollectedBody =
+	| { readonly outcome: "Collected"; readonly content: Buffer }
+	| { readonly outcome: "TooLarge" };
+
+/**
+ * Read a request body, refusing it the moment it passes what was reserved.
+ *
+ * The count is checked per chunk and the stream destroyed on the one that
+ * crosses the line, so a client that reserved a megabyte and started sending a
+ * hundred is cut off after the first megabyte rather than after all of it.
+ */
+export const collectLimitedBody = async (
+	stream: Readable,
+	maxBytes: number,
+): Promise<CollectedBody> => {
+	const chunks: Buffer[] = [];
+	let received = 0;
+
+	for await (const chunk of stream) {
+		const buffer = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk);
+		received += buffer.length;
+		if (received > maxBytes) {
+			// Stop reading, but leave the socket alone: destroying it here races the
+			// 413 the caller is about to write, and the client sees a connection
+			// reset instead of the reason it was refused.
+			stream.pause();
+			return { outcome: "TooLarge" };
+		}
+		chunks.push(buffer);
+	}
+
+	return { outcome: "Collected", content: Buffer.concat(chunks) };
+};
+
+export interface ReceiveUploadInput {
+	/** Decoded storage key, the `/outbox-upload/` prefix already stripped. */
+	storageKey: string;
+	exp: string | undefined;
+	max: string | undefined;
+	sig: string | undefined;
+	body: Readable;
+	nowSeconds: number;
+	secret: string | undefined;
+	/**
+	 * Whether the database still holds a live reservation for this attachment.
+	 * Injected rather than reached for, so the receiver stays a function of its
+	 * request and the storage it writes to.
+	 */
+	findLiveReservation: (
+		accountConfigId: string,
+		outboxAttachmentId: string,
+		nowSeconds: number,
+	) => Promise<boolean>;
+}
+
+export interface UploadResult {
+	status: number;
+	reason: string;
+}
+
+export const receiveUpload = async (
+	storage: StorageService,
+	input: ReceiveUploadInput,
+): Promise<UploadResult> => {
+	const target = parseOutboxAttachmentKey(input.storageKey);
+	if (!target) return { status: 404, reason: "not-an-attachment-key" };
+
+	const auth = authorizeUploadRequest({
+		secret: input.secret,
+		relativePath: input.storageKey,
+		exp: input.exp,
+		max: input.max,
+		sig: input.sig,
+		nowSeconds: input.nowSeconds,
+	});
+	if (!auth.authorized) {
+		return { status: auth.status, reason: auth.reason };
+	}
+
+	// A signed URL outlives the draft it was minted for, so possession of one is
+	// not enough: the database has to still hold a live reservation for this
+	// attachment. Without this a URL minted before a discard writes bytes back
+	// under a draft that no longer exists. The hosted shape cannot make this
+	// check — block storage receives the PUT directly — which is what the sweep
+	// exists for.
+	const reserved = await input.findLiveReservation(
+		target.accountConfigId,
+		target.outboxAttachmentId,
+		input.nowSeconds,
+	);
+	if (!reserved) return { status: 409, reason: "no-live-reservation" };
+
+	const collected = await collectLimitedBody(input.body, auth.maxBytes);
+	if (collected.outcome === "TooLarge") {
+		return { status: 413, reason: "over-reserved-size" };
+	}
+
+	// Exactly the reserved size, not merely under it — the same thing S3 binds
+	// when it recomputes a presigned PUT's signature over the content-length it
+	// was sent. A short body is a failed upload, and the completion would refuse
+	// it anyway; refusing here means no partial object is left to refuse.
+	if (collected.content.length !== auth.maxBytes) {
+		return { status: 400, reason: "size-mismatch" };
+	}
+
+	await storage.storeOutboxAttachment({
+		accountConfigId: target.accountConfigId,
+		accountId: target.accountId,
+		outboxMessageId: target.outboxMessageId,
+		outboxAttachmentId: target.outboxAttachmentId,
+		content: collected.content,
+	});
+
+	return { status: 204, reason: "stored" };
+};

package/dev-server/upload-route.test.ts

@@ -0,0 +1,196 @@
+/**
+ * Issue #679: the upload route through the real dev-server app.
+ *
+ * Not a copy of its middleware order — the app object itself, imported from
+ * `server.ts`. A copy is what let a JSON-typed upload silently read zero bytes:
+ * `express.json` drains the stream before any route below it sees the body, and
+ * a test that rebuilds the stack stays green when the real one is reordered.
+ */
+
+import assert from "node:assert/strict";
+import { mkdtemp, rm } from "node:fs/promises";
+import type { Server } from "node:http";
+import { join } from "node:path";
+import { after, before, describe, it } from "node:test";
+import type {
+	IOutboxAttachmentRepository,
+	OutboxAttachmentItem,
+} from "@remit/data-ports";
+import { NotFoundError } from "@remit/data-ports/errors";
+import { OutboxAttachmentService } from "@remit/mailbox-service";
+import {
+	type StorageService,
+	UPLOAD_ROUTE_PREFIX,
+} from "@remit/storage-service";
+import { createFilesystemStorageService } from "@remit/storage-service/filesystem";
+import {
+	_resetForTest,
+	type RemitClient,
+	setClient,
+} from "../src/service/data-client.js";
+
+const SECRET = "a-signing-secret-of-at-least-32-characters";
+const CFG = "cfg-upload";
+const ACC = "acc-upload";
+const DRAFT = "draft-upload";
+
+let storage: StorageService;
+let basePath: string;
+let rows: Map<string, OutboxAttachmentItem>;
+let server: Server;
+let port: number;
+
+const repository = (): IOutboxAttachmentRepository =>
+	({
+		get: async (accountConfigId: string, outboxAttachmentId: string) => {
+			const row = rows.get(outboxAttachmentId);
+			if (!row || row.accountConfigId !== accountConfigId) {
+				throw new NotFoundError("gone");
+			}
+			return row;
+		},
+		listByOutboxMessage: async () => [...rows.values()],
+	}) as unknown as IOutboxAttachmentRepository;
+
+const reserve = async (
+	outboxAttachmentId: string,
+	sizeBytes: number,
+): Promise<string> => {
+	const expiresAt = Math.floor(Date.now() / 1000) + 900;
+	rows.set(outboxAttachmentId, {
+		outboxAttachmentId,
+		outboxMessageId: DRAFT,
+		accountId: ACC,
+		accountConfigId: CFG,
+		filename: `${outboxAttachmentId}.bin`,
+		contentType: "application/octet-stream",
+		sizeBytes,
+		state: "Pending",
+		storageKey: "",
+		reservationExpiresAt: expiresAt,
+		createdAt: 0,
+		updatedAt: 0,
+	});
+	const { uploadUrl } = await storage.createOutboxAttachmentUploadUrl({
+		accountConfigId: CFG,
+		accountId: ACC,
+		outboxMessageId: DRAFT,
+		outboxAttachmentId,
+		sizeBytes,
+		expiresAt,
+	});
+	const url = new URL(uploadUrl);
+	return `http://127.0.0.1:${port}${url.pathname}${url.search}`;
+};
+
+before(async () => {
+	// Repo-local scratch, not the machine's shared temp directory.
+	basePath = await mkdtemp(join(process.cwd(), ".tmp-upload-route-"));
+	storage = createFilesystemStorageService(basePath, {
+		origin: "http://127.0.0.1",
+		signingSecret: SECRET,
+	});
+	rows = new Map();
+
+	setClient({
+		storage,
+		outboxAttachment: new OutboxAttachmentService({
+			outboxMessageService: {} as never,
+			outboxAttachmentService: repository(),
+			storage,
+		}),
+	} as unknown as RemitClient);
+
+	process.env.BETTER_AUTH_SECRET = SECRET;
+	// Port 0: the app binds whatever is free and tells us which.
+	process.env.SERVER_PORT = "0";
+	process.env.CORS_ALLOWED_ORIGINS = "*";
+
+	const imported = await import("./server.js");
+	server = imported.listener;
+	if (!server.listening) {
+		await new Promise<void>((resolve) => server.once("listening", resolve));
+	}
+	const address = server.address();
+	port = typeof address === "object" && address ? address.port : 0;
+});
+
+after(async () => {
+	await new Promise<void>((resolve) => server.close(() => resolve()));
+	await rm(basePath, { recursive: true, force: true });
+	_resetForTest();
+});
+
+describe("PUT /outbox-upload through the dev-server's own app", () => {
+	it("stores every byte of a file the browser labelled application/json", async () => {
+		const content = Buffer.from(JSON.stringify({ note: "x".repeat(100) }));
+		const url = await reserve("attjson", content.length);
+
+		const response = await fetch(url, {
+			method: "PUT",
+			body: content,
+			// What `fetch(url, { body: file })` sends for a .json file: File.type.
+			headers: { "content-type": "application/json" },
+		});
+
+		assert.equal(response.status, 204);
+		assert.equal(
+			(await storage.statOutboxAttachment(CFG, ACC, DRAFT, "attjson"))
+				?.sizeBytes,
+			content.length,
+		);
+	});
+
+	it("stores every byte of a form-urlencoded-looking upload too", async () => {
+		const content = Buffer.from(`a=1&b=2&c=${"z".repeat(200)}`);
+		const url = await reserve("attform", content.length);
+
+		const response = await fetch(url, {
+			method: "PUT",
+			body: content,
+			headers: { "content-type": "application/x-www-form-urlencoded" },
+		});
+
+		assert.equal(response.status, 204);
+		assert.equal(
+			(await storage.statOutboxAttachment(CFG, ACC, DRAFT, "attform"))
+				?.sizeBytes,
+			content.length,
+		);
+	});
+
+	it("stores binary unchanged", async () => {
+		const content = Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x00, 0xff, 0x10]);
+		const url = await reserve("attpng", content.length);
+
+		const response = await fetch(url, {
+			method: "PUT",
+			body: content,
+			headers: { "content-type": "image/png" },
+		});
+
+		assert.equal(response.status, 204);
+		assert.equal(
+			(await storage.statOutboxAttachment(CFG, ACC, DRAFT, "attpng"))
+				?.sizeBytes,
+			content.length,
+		);
+	});
+
+	it("refuses a PUT once the draft no longer has a live reservation", async () => {
+		const url = await reserve("attgone", 8);
+		rows.delete("attgone");
+
+		const response = await fetch(url, {
+			method: "PUT",
+			body: Buffer.alloc(8),
+			headers: { "content-type": "application/octet-stream" },
+		});
+
+		assert.equal(response.status, 409);
+		assert.equal(
+			await storage.statOutboxAttachment(CFG, ACC, DRAFT, "attgone"),
+			null,
+		);
+	});
+});

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@remit/backend",
-  "version": "0.0.58",
+  "version": "0.0.60",
   "description": "Remit Mail Inspector API backend",
   "license": "MIT",
   "author": "",

package/src/derive/contentSignature.test.ts

@@ -1,6 +1,7 @@
 import assert from "node:assert/strict";
 import { afterEach, describe, it } from "node:test";
 import {
+	CONTENT_URL_TTL_SECONDS,
 	createContentSigner,
 	getContentSigner,
 	verifyContentSignature,
@@ -133,3 +134,28 @@ describe("getContentSigner", () => {
 		assert.throws(() => getContentSigner(), /BETTER_AUTH_SECRET/);
 	});
 });
+
+describe("the wire format is pinned, not merely self-consistent", () => {
+	/**
+	 * Every other test here round-trips through this module, so a change to the
+	 * canonical message or the key derivation passes them all while breaking every
+	 * `/content` URL already in a browser. This vector was computed once, by hand,
+	 * from the scheme as shipped: HMAC-SHA256 with a subkey derived from the
+	 * master secret under `remit-content-url-signing-v1`, over `path\nexp`,
+	 * base64url. If it fails, the format moved and in-flight URLs moved with it.
+	 */
+	it("still produces the signature it produced when the scheme shipped", () => {
+		const signer = createContentSigner(
+			"a-signing-secret-of-at-least-32-characters",
+		);
+		const originalNow = Date.now;
+		Date.now = () => 1_800_000_000_000 - CONTENT_URL_TTL_SECONDS * 1000;
+		try {
+			const { exp, sig } = signer("accounts/cfg1/acc1/messages/msg1/parts/1.2");
+			assert.equal(exp, 1_800_000_000);
+			assert.equal(sig, "eFKwOYf42a6DWwaZahIm7aVtWuqMOfhOAPoyxtpFybo");
+		} finally {
+			Date.now = originalNow;
+		}
+	});
+});

package/src/derive/contentSignature.ts

@@ -1,4 +1,8 @@
-import { createHmac, timingSafeEqual } from "node:crypto";
+import {
+	type SignedPathFailure,
+	signStoragePath,
+	verifyStoragePath,
+} from "@remit/storage-service/signed-path";
 import { usesBetterAuthJwt } from "../data-backend.js";
 
 /**
@@ -20,6 +24,10 @@ import { usesBetterAuthJwt } from "../data-backend.js";
  * valid signature for another account. This is the presigned-URL capability
  * model the export flow already uses: possession of the URL grants access until
  * it expires, bounded by a short TTL.
+ *
+ * The HMAC itself lives in `@remit/storage-service/signed-path`, shared with the
+ * write side (an attachment upload URL) under a different derivation label. This
+ * module is the read side's label, TTL and public shape.
  */
 
 const KEY_DERIVATION_LABEL = "remit-content-url-signing-v1";
@@ -32,28 +40,6 @@ const KEY_DERIVATION_LABEL = "remit-content-url-signing-v1";
  */
 export const CONTENT_URL_TTL_SECONDS = 3600;
 
-/**
- * Derive a purpose-specific signing subkey from the master secret. Domain
- * separation via a fixed label keeps the content-signing key distinct from the
- * raw better-auth secret, so a compromise of one signing space does not reveal
- * the other. Reusing `BETTER_AUTH_SECRET` as the master means no new secret has
- * to be provisioned on the self-host stack — it is already required there.
- */
-const deriveSigningKey = (masterSecret: string): Buffer =>
-	createHmac("sha256", masterSecret).update(KEY_DERIVATION_LABEL).digest();
-
-const canonicalMessage = (relativePath: string, exp: number): string =>
-	`${relativePath}\n${exp}`;
-
-const computeSignature = (
-	key: Buffer,
-	relativePath: string,
-	exp: number,
-): string =>
-	createHmac("sha256", key)
-		.update(canonicalMessage(relativePath, exp))
-		.digest("base64url");
-
 export interface ContentSignature {
 	exp: number;
 	sig: string;
@@ -69,10 +55,14 @@ export const createContentSigner = (
 	masterSecret: string,
 	ttlSeconds: number = CONTENT_URL_TTL_SECONDS,
 ): ContentSigner => {
-	const key = deriveSigningKey(masterSecret);
 	return (relativePath) => {
 		const exp = Math.floor(Date.now() / 1000) + ttlSeconds;
-		return { exp, sig: computeSignature(key, relativePath, exp) };
+		return {
+			exp,
+			sig: signStoragePath(masterSecret, KEY_DERIVATION_LABEL, relativePath, [
+				exp,
+			]),
+		};
 	};
 };
 
@@ -96,11 +86,7 @@ export const getContentSigner = (): ContentSigner | undefined => {
 	return createContentSigner(secret);
 };
 
-export type ContentSignatureFailure =
-	| "missing"
-	| "malformed"
-	| "expired"
-	| "bad-signature";
+export type ContentSignatureFailure = SignedPathFailure;
 
 export type ContentSignatureResult =
 	| { valid: true }
@@ -124,16 +110,13 @@ export const verifyContentSignature = (
 	if (!Number.isInteger(exp) || exp <= 0) {
 		return { valid: false, reason: "malformed" };
 	}
-	if (exp < nowSeconds) return { valid: false, reason: "expired" };
 
-	const key = deriveSigningKey(masterSecret);
-	const expected = Buffer.from(computeSignature(key, relativePath, exp));
-	const presented = Buffer.from(sig);
-	if (expected.length !== presented.length) {
-		return { valid: false, reason: "bad-signature" };
-	}
-	if (!timingSafeEqual(expected, presented)) {
-		return { valid: false, reason: "bad-signature" };
-	}
-	return { valid: true };
+	return verifyStoragePath(
+		masterSecret,
+		KEY_DERIVATION_LABEL,
+		relativePath,
+		[exp],
+		sig,
+		nowSeconds,
+	);
 };

package/src/handlers/account-guards.ts

@@ -122,6 +122,7 @@ export const toAccountResponse = (
 	smtpUsername: account.smtpUsername ?? "",
 	signaturePlainText: signature.plainText,
 	signatureHtml: signature.html,
+	composeLanguages: overrides.composeLanguages,
 	isActive: account.isActive,
 	connectionState: account.connectionState,
 	lastConnectedAt: account.lastConnectedAt,

package/src/handlers/account-overrides.test.ts

@@ -33,6 +33,25 @@ describe("groupAccountOverrides", () => {
 		assert.equal(grouped.get("acc-2")?.displayName, "Bob");
 	});
 
+	it("groups the compose-language list by accountId", () => {
+		const settings = [
+			setting("AccountComposeLanguages#acc-1", {
+				kind: "StringList",
+				value: ["nl", "en"],
+			}),
+			stringSetting("AccountDisplayName#acc-1", "Alice"),
+		];
+		const grouped = groupAccountOverrides(settings);
+		assert.deepEqual(grouped.get("acc-1")?.composeLanguages, ["nl", "en"]);
+		assert.equal(grouped.get("acc-1")?.displayName, "Alice");
+	});
+
+	it("ignores a compose-language row stored under the wrong value kind", () => {
+		const settings = [stringSetting("AccountComposeLanguages#acc-1", "nl")];
+		const grouped = groupAccountOverrides(settings);
+		assert.equal(grouped.get("acc-1"), undefined);
+	});
+
 	it("ignores a leftover MailboxRole#<mailboxId> row instead of throwing", () => {
 		// Leftover rows from the #963/#964 backfill (superseded by
 		// FolderRoleAppointment, RFC 032 exclusive-folder-appointment #976) must