@remit/storage-service 0.0.8 → 0.0.10

package/src/outbox-upload-url.test.ts

@@ -0,0 +1,290 @@
+import assert from "node:assert";
+import { mkdtemp, rm } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { describe, test } from "node:test";
+import { S3Client } from "@aws-sdk/client-s3";
+import { createFilesystemStorageService } from "./backends/filesystem.js";
+import { createS3StorageService } from "./backends/s3.js";
+import {
+	authorizeUploadRequest,
+	UPLOAD_ROUTE_PREFIX,
+	UPLOAD_URL_SIGNING_LABEL,
+} from "./outbox-upload-url.js";
+import { signStoragePath } from "./signed-path.js";
+import {
+	buildOutboxAttachmentKey,
+	parseOutboxAttachmentKey,
+} from "./storage.js";
+
+const SECRET = "a-signing-secret-of-at-least-32-characters";
+const KEY = buildOutboxAttachmentKey("cfg1", "acc1", "draft1", "att1");
+
+const sign = (path: string, exp: number, max: number): string =>
+	signStoragePath(SECRET, UPLOAD_URL_SIGNING_LABEL, path, [exp, max]);
+
+describe("authorizeUploadRequest", () => {
+	const now = 1_000_000;
+	const exp = now + 900;
+
+	test("admits a request presenting the signature that was minted for it", () => {
+		const result = authorizeUploadRequest({
+			secret: SECRET,
+			relativePath: KEY,
+			exp: String(exp),
+			max: "2048",
+			sig: sign(KEY, exp, 2048),
+			nowSeconds: now,
+		});
+		assert.deepStrictEqual(result, { authorized: true, maxBytes: 2048 });
+	});
+
+	test("refuses a signature minted for another attachment's key", () => {
+		const otherKey = buildOutboxAttachmentKey("cfg1", "acc1", "draft1", "att2");
+		const result = authorizeUploadRequest({
+			secret: SECRET,
+			relativePath: otherKey,
+			exp: String(exp),
+			max: "2048",
+			sig: sign(KEY, exp, 2048),
+			nowSeconds: now,
+		});
+		assert.strictEqual(result.authorized, false);
+	});
+
+	test("refuses a signature minted for another account's key", () => {
+		const otherTenant = buildOutboxAttachmentKey(
+			"cfg2",
+			"acc1",
+			"draft1",
+			"att1",
+		);
+		const result = authorizeUploadRequest({
+			secret: SECRET,
+			relativePath: otherTenant,
+			exp: String(exp),
+			max: "2048",
+			sig: sign(KEY, exp, 2048),
+			nowSeconds: now,
+		});
+		assert.strictEqual(result.authorized, false);
+	});
+
+	test("refuses a raised byte count — the size is part of what was signed", () => {
+		const result = authorizeUploadRequest({
+			secret: SECRET,
+			relativePath: KEY,
+			exp: String(exp),
+			max: "999999",
+			sig: sign(KEY, exp, 2048),
+			nowSeconds: now,
+		});
+		assert.strictEqual(result.authorized, false);
+	});
+
+	test("refuses an extended expiry", () => {
+		const result = authorizeUploadRequest({
+			secret: SECRET,
+			relativePath: KEY,
+			exp: String(exp + 10_000),
+			max: "2048",
+			sig: sign(KEY, exp, 2048),
+			nowSeconds: now,
+		});
+		assert.strictEqual(result.authorized, false);
+	});
+
+	test("refuses a lapsed URL", () => {
+		const result = authorizeUploadRequest({
+			secret: SECRET,
+			relativePath: KEY,
+			exp: String(exp),
+			max: "2048",
+			sig: sign(KEY, exp, 2048),
+			nowSeconds: exp + 1,
+		});
+		assert.strictEqual(result.authorized, false);
+		assert.strictEqual(result.authorized === false && result.reason, "expired");
+	});
+
+	test("refuses a request presenting no signature at all", () => {
+		const result = authorizeUploadRequest({
+			secret: SECRET,
+			relativePath: KEY,
+			exp: String(exp),
+			max: "2048",
+			sig: undefined,
+			nowSeconds: now,
+		});
+		assert.strictEqual(result.authorized === false && result.status, 401);
+	});
+
+	test("fails closed when no secret is configured", () => {
+		const result = authorizeUploadRequest({
+			secret: undefined,
+			relativePath: KEY,
+			exp: String(exp),
+			max: "2048",
+			sig: sign(KEY, exp, 2048),
+			nowSeconds: now,
+		});
+		assert.strictEqual(result.authorized === false && result.status, 500);
+	});
+
+	test("a read grant for the same path is not a write grant", () => {
+		// The content signer covers the path and an expiry under its own label.
+		const readSignature = signStoragePath(
+			SECRET,
+			"remit-content-url-signing-v1",
+			KEY,
+			[exp],
+		);
+		const result = authorizeUploadRequest({
+			secret: SECRET,
+			relativePath: KEY,
+			exp: String(exp),
+			max: "2048",
+			sig: readSignature,
+			nowSeconds: now,
+		});
+		assert.strictEqual(result.authorized, false);
+	});
+});
+
+describe("attachment keys", () => {
+	test("parseOutboxAttachmentKey recovers the ids the upload route needs", () => {
+		assert.deepStrictEqual(parseOutboxAttachmentKey(KEY), {
+			accountConfigId: "cfg1",
+			accountId: "acc1",
+			outboxMessageId: "draft1",
+			outboxAttachmentId: "att1",
+		});
+	});
+
+	test("parseOutboxAttachmentKey refuses a message content key", () => {
+		assert.strictEqual(
+			parseOutboxAttachmentKey("accounts/cfg1/acc1/messages/msg1/parts/1.2"),
+			null,
+		);
+	});
+
+	test("parseOutboxAttachmentKey refuses a traversal, signature or no signature", () => {
+		// `..` matches a naive [^/]+ and the write side joins this onto a
+		// filesystem root, so the shape has to refuse it before the HMAC is the
+		// only thing left standing.
+		assert.strictEqual(
+			parseOutboxAttachmentKey(
+				"accounts/cfg1/acc1/outbox/draft1/attachments/..",
+			),
+			null,
+		);
+		assert.strictEqual(
+			parseOutboxAttachmentKey("accounts/../../etc/outbox/x/attachments/y"),
+			null,
+		);
+		assert.strictEqual(
+			parseOutboxAttachmentKey(
+				"accounts/cfg1/acc1/outbox/draft1/attachments/a.b",
+			),
+			null,
+		);
+	});
+});
+
+describe("minting an upload URL", () => {
+	test("the filesystem backend mints a URL its own route will admit", async () => {
+		const basePath = await mkdtemp(join(tmpdir(), "remit-upload-url-"));
+		try {
+			const storage = createFilesystemStorageService(basePath, {
+				origin: "https://mail.example.test/",
+				signingSecret: SECRET,
+			});
+			const expiresAt = Math.floor(Date.now() / 1000) + 900;
+
+			const { uploadUrl } = await storage.createOutboxAttachmentUploadUrl({
+				accountConfigId: "cfg1",
+				accountId: "acc1",
+				outboxMessageId: "draft1",
+				outboxAttachmentId: "att1",
+				sizeBytes: 2048,
+				expiresAt,
+			});
+
+			const url = new URL(uploadUrl);
+			assert.strictEqual(url.origin, "https://mail.example.test");
+			assert.strictEqual(url.pathname, `${UPLOAD_ROUTE_PREFIX}${KEY}`);
+
+			const admitted = authorizeUploadRequest({
+				secret: SECRET,
+				relativePath: url.pathname.slice(UPLOAD_ROUTE_PREFIX.length),
+				exp: url.searchParams.get("exp") ?? undefined,
+				max: url.searchParams.get("max") ?? undefined,
+				sig: url.searchParams.get("sig") ?? undefined,
+				nowSeconds: Math.floor(Date.now() / 1000),
+			});
+			assert.deepStrictEqual(admitted, { authorized: true, maxBytes: 2048 });
+		} finally {
+			await rm(basePath, { recursive: true, force: true });
+		}
+	});
+
+	test("the filesystem backend refuses to mint without an origin and a secret", async () => {
+		const basePath = await mkdtemp(join(tmpdir(), "remit-upload-url-"));
+		try {
+			const storage = createFilesystemStorageService(basePath);
+			await assert.rejects(
+				() =>
+					storage.createOutboxAttachmentUploadUrl({
+						accountConfigId: "cfg1",
+						accountId: "acc1",
+						outboxMessageId: "draft1",
+						outboxAttachmentId: "att1",
+						sizeBytes: 1,
+						expiresAt: Math.floor(Date.now() / 1000) + 60,
+					}),
+				/cannot mint an upload URL/,
+			);
+		} finally {
+			await rm(basePath, { recursive: true, force: true });
+		}
+	});
+
+	test("the S3 backend mints a presigned PUT with the size inside the signature", async () => {
+		const client = new S3Client({
+			region: "eu-west-1",
+			credentials: {
+				accessKeyId: "AKIATESTTESTTEST",
+				secretAccessKey: "test-secret",
+			},
+		});
+		const storage = createS3StorageService(client, "a-bucket");
+		const params = {
+			accountConfigId: "cfg1",
+			accountId: "acc1",
+			outboxMessageId: "draft1",
+			outboxAttachmentId: "att1",
+			sizeBytes: 2048,
+			expiresAt: Math.floor(Date.now() / 1000) + 900,
+		};
+
+		const { uploadUrl } = await storage.createOutboxAttachmentUploadUrl(params);
+		const url = new URL(uploadUrl);
+
+		assert.ok(url.pathname.endsWith(KEY));
+		// content-length inside SignedHeaders is what makes the declared size
+		// unforgeable: S3 recomputes the signature over the length it is sent.
+		assert.match(
+			url.searchParams.get("X-Amz-SignedHeaders") ?? "",
+			/content-length/,
+		);
+
+		const larger = await storage.createOutboxAttachmentUploadUrl({
+			...params,
+			sizeBytes: 4096,
+		});
+		assert.notStrictEqual(
+			url.searchParams.get("X-Amz-Signature"),
+			new URL(larger.uploadUrl).searchParams.get("X-Amz-Signature"),
+		);
+	});
+});

package/src/outbox-upload-url.ts

@@ -0,0 +1,78 @@
+import { verifyStoragePath } from "./signed-path.js";
+
+/**
+ * The self-hosted half of the upload seam: what a minted URL looks like when
+ * there is no block storage to presign against, and how the route that receives
+ * the bytes decides to accept them.
+ *
+ * A hosted deployment mints a presigned PUT instead and none of this runs — the
+ * bytes never reach the deployment at all. Both halves bind the same two
+ * things, an expiry and an exact byte count, so a client sees one contract.
+ */
+
+/** Domain separation: a read grant must never be presentable as a write grant. */
+export const UPLOAD_URL_SIGNING_LABEL = "remit-outbox-upload-url-signing-v1";
+
+export const UPLOAD_ROUTE_PREFIX = "/outbox-upload/";
+
+/**
+ * How long a mint's URL, and the reservation behind it, stay good. Long enough
+ * for a slow phone to push 25 MB, short enough that a draft abandoned mid-upload
+ * stops holding room within the same session.
+ */
+export const UPLOAD_URL_TTL_SECONDS = 900;
+
+export interface AuthorizeUploadInput {
+	secret: string | undefined;
+	/** Decoded storage key the signature was minted over. */
+	relativePath: string;
+	exp: string | undefined;
+	max: string | undefined;
+	sig: string | undefined;
+	nowSeconds: number;
+}
+
+export type AuthorizeUploadResult =
+	| { authorized: true; maxBytes: number }
+	| { authorized: false; status: number; reason: string };
+
+/**
+ * Decide whether an upload request may write to the key it names.
+ *
+ * Pure, so the decision is testable without a live server. Mirrors
+ * `authorizeContentRequest` on the read side, with one addition: the byte count
+ * is part of the signed message, so the caller learns from the signature how
+ * much this URL is allowed to carry and can cut the stream off there.
+ */
+export const authorizeUploadRequest = (
+	input: AuthorizeUploadInput,
+): AuthorizeUploadResult => {
+	if (!input.secret || input.secret.length === 0) {
+		// No secret means no upload URL could have been signed, so nothing
+		// presented here can be trusted. Fail closed.
+		return {
+			authorized: false,
+			status: 500,
+			reason: "upload-signing-not-configured",
+		};
+	}
+
+	const exp = Number(input.exp);
+	const maxBytes = Number(input.max);
+	if (!Number.isInteger(exp) || !Number.isInteger(maxBytes)) {
+		return { authorized: false, status: 403, reason: "malformed" };
+	}
+
+	const result = verifyStoragePath(
+		input.secret,
+		UPLOAD_URL_SIGNING_LABEL,
+		input.relativePath,
+		[exp, maxBytes],
+		input.sig,
+		input.nowSeconds,
+	);
+	if (result.valid) return { authorized: true, maxBytes };
+
+	const status = result.reason === "missing" ? 401 : 403;
+	return { authorized: false, status, reason: result.reason };
+};

package/src/s3.ts

@@ -11,10 +11,26 @@ export const createStorageService = (): StorageService => {
 	if (bucketName) {
 		const client = new S3Client({
 			endpoint: process.env.S3_ENDPOINT,
+			// The SDK default (WHEN_SUPPORTED) hangs a CRC32 of an empty body off
+			// every presigned PUT as signed query params. Real S3 tolerates it;
+			// MinIO and LocalStack reject the request, and S3_ENDPOINT exists to
+			// point at exactly those.
+			requestChecksumCalculation: "WHEN_REQUIRED",
 		});
 		return createS3StorageService(client, bucketName);
 	}
 
 	const basePath = process.env.STORAGE_LOCAL_PATH ?? ".remit/storage";
-	return createFilesystemStorageService(basePath);
+	// BETTER_AUTH_URL is the deployment's own public base — the value the token
+	// issuer and the JWKS discovery document already agree on — so a minted
+	// upload URL resolves from wherever the browser reached the API. Both it and
+	// the signing secret are required on the self-host stack; without them a
+	// mint fails loud inside the backend rather than handing out a URL that
+	// cannot be verified.
+	const origin = process.env.BETTER_AUTH_URL;
+	const signingSecret = process.env.BETTER_AUTH_SECRET;
+	return createFilesystemStorageService(
+		basePath,
+		origin && signingSecret ? { origin, signingSecret } : undefined,
+	);
 };

package/src/signed-path.ts

@@ -0,0 +1,86 @@
+import { createHmac, timingSafeEqual } from "node:crypto";
+
+/**
+ * The HMAC scheme behind every signed storage URL this stack mints.
+ *
+ * It started as the read side — `/content/*` is served against a signature
+ * because a bearer token cannot ride on an `<img src>` inside email HTML — and
+ * the write side needs exactly the same capability: a URL that grants access to
+ * one path, for a while, and is unforgeable without the key. Rather than a
+ * second scheme, both use this one with a different derivation label, so a
+ * signature minted to read content can never be replayed as permission to
+ * write, and vice versa.
+ *
+ * The signed message is the storage path followed by every claim in a fixed
+ * order. Because the path is part of the message, a signature minted for one
+ * account's object cannot be presented for another's.
+ */
+
+const deriveSigningKey = (masterSecret: string, label: string): Buffer =>
+	createHmac("sha256", masterSecret).update(label).digest();
+
+const canonicalMessage = (
+	relativePath: string,
+	claims: readonly number[],
+): string => [relativePath, ...claims].join("\n");
+
+const computeSignature = (
+	key: Buffer,
+	relativePath: string,
+	claims: readonly number[],
+): string =>
+	createHmac("sha256", key)
+		.update(canonicalMessage(relativePath, claims))
+		.digest("base64url");
+
+export const signStoragePath = (
+	masterSecret: string,
+	label: string,
+	relativePath: string,
+	claims: readonly number[],
+): string =>
+	computeSignature(deriveSigningKey(masterSecret, label), relativePath, claims);
+
+export type SignedPathFailure =
+	| "missing"
+	| "malformed"
+	| "expired"
+	| "bad-signature";
+
+export type SignedPathResult =
+	| { valid: true }
+	| { valid: false; reason: SignedPathFailure };
+
+/**
+ * Verify a presented signature against the path and claims it must cover. The
+ * first claim is always the expiry, in unix seconds. Pure, so the decision can
+ * be unit-tested without a live server.
+ */
+export const verifyStoragePath = (
+	masterSecret: string,
+	label: string,
+	relativePath: string,
+	claims: readonly number[],
+	sig: string | undefined,
+	nowSeconds: number,
+): SignedPathResult => {
+	if (!sig || claims.length === 0) return { valid: false, reason: "missing" };
+
+	const [exp] = claims;
+	if (!claims.every((claim) => Number.isInteger(claim) && claim > 0)) {
+		return { valid: false, reason: "malformed" };
+	}
+	if (exp < nowSeconds) return { valid: false, reason: "expired" };
+
+	const expected = Buffer.from(
+		signStoragePath(masterSecret, label, relativePath, claims),
+	);
+	const presented = Buffer.from(sig);
+	if (expected.length !== presented.length) {
+		return { valid: false, reason: "bad-signature" };
+	}
+	if (!timingSafeEqual(expected, presented)) {
+		return { valid: false, reason: "bad-signature" };
+	}
+	return { valid: true };
+};

package/src/storage.test.ts

@@ -14,6 +14,8 @@ import {
 	buildExtractedSkippedKey,
 	buildExtractedTextKey,
 	buildMessageBodyKey,
+	buildOutboxAttachmentKey,
+	buildOutboxAttachmentPrefix,
 	buildParsedBodyKey,
 	computeChecksum,
 	createMockStorageService,
@@ -85,6 +87,26 @@ describe("path builders", () => {
 		);
 	});
 
+	test("buildOutboxAttachmentKey hangs the file off its own draft, under the same tenant prefix", () => {
+		const key = buildOutboxAttachmentKey("cfg1", "acc123", "draft9", "att7");
+		assert.strictEqual(
+			key,
+			"accounts/cfg1/acc123/outbox/draft9/attachments/att7",
+		);
+		assert.ok(
+			key.startsWith(buildOutboxAttachmentPrefix("cfg1", "acc123", "draft9")),
+		);
+	});
+
+	test("parseContentStorageKey does not recognise an outbox attachment key", () => {
+		assert.strictEqual(
+			parseContentStorageKey(
+				buildOutboxAttachmentKey("cfg1", "acc123", "draft9", "att7"),
+			),
+			null,
+		);
+	});
+
 	test("computeChecksum returns SHA-256 hex", () => {
 		const content = Buffer.from("test content");
 		const expected = createHash("sha256").update(content).digest("hex");
@@ -744,6 +766,120 @@ describe("createFilesystemStorageService", () => {
 		assert.deepStrictEqual(items, []);
 	});
 
+	test("storeOutboxAttachment writes the bytes uncompressed so the size on disk is the size the cap counts", async () => {
+		const storage = createFilesystemStorageService(testBasePath);
+		// Highly compressible: gzip would make the stored size disagree with the
+		// decoded size the per-message cap is expressed in.
+		const content = Buffer.alloc(4096, 0x61);
+
+		const ref = await storage.storeOutboxAttachment({
+			accountConfigId: "cfg1",
+			accountId: "acc123",
+			outboxMessageId: "draft1",
+			outboxAttachmentId: "att1",
+			content,
+		});
+
+		assert.strictEqual(ref.contentEncoding, ContentEncoding.None);
+		assert.strictEqual(ref.sizeBytes, content.length);
+		assert.strictEqual(ref.checksumSha256, computeChecksum(content));
+		assert.deepStrictEqual(await storage.retrieve(ref.uri), content);
+
+		const [item] = await storage.listOutboxAttachments(
+			"cfg1",
+			"acc123",
+			"draft1",
+		);
+		assert.strictEqual(item.sizeBytes, content.length);
+	});
+
+	test("listOutboxAttachments returns only the named draft, and every object in it", async () => {
+		const storage = createFilesystemStorageService(testBasePath);
+		for (const id of ["a", "b", "c"]) {
+			await storage.storeOutboxAttachment({
+				accountConfigId: "cfg1",
+				accountId: "acc123",
+				outboxMessageId: "draft-many",
+				outboxAttachmentId: id,
+				content: Buffer.from(`file-${id}`),
+			});
+		}
+		await storage.storeOutboxAttachment({
+			accountConfigId: "cfg1",
+			accountId: "acc123",
+			outboxMessageId: "draft-other",
+			outboxAttachmentId: "d",
+			content: Buffer.from("elsewhere"),
+		});
+
+		const all = await storage.listOutboxAttachments(
+			"cfg1",
+			"acc123",
+			"draft-many",
+		);
+		assert.deepStrictEqual(all.map((item) => item.outboxAttachmentId).sort(), [
+			"a",
+			"b",
+			"c",
+		]);
+	});
+
+	test("listOutboxAttachments returns an empty array for a draft that has none", async () => {
+		const storage = createFilesystemStorageService(testBasePath);
+		assert.deepStrictEqual(
+			await storage.listOutboxAttachments("cfg1", "acc123", "draft-empty"),
+			[],
+		);
+	});
+
+	test("deleting a draft's attachments empties the prefix and the ledger with it", async () => {
+		const storage = createFilesystemStorageService(testBasePath);
+		await storage.storeOutboxAttachment({
+			accountConfigId: "cfg1",
+			accountId: "acc123",
+			outboxMessageId: "draft-gone",
+			outboxAttachmentId: "att1",
+			content: Buffer.alloc(4),
+		});
+		await storage.deleteOutboxAttachments("cfg1", "acc123", "draft-gone");
+
+		assert.deepStrictEqual(
+			await storage.listOutboxAttachments("cfg1", "acc123", "draft-gone"),
+			[],
+		);
+	});
+
+	test("listOutboxDraftsWithAttachments finds prefixes whose row may be long gone", async () => {
+		const storage = createFilesystemStorageService(testBasePath);
+		for (const draft of ["draft-x", "draft-y"]) {
+			await storage.storeOutboxAttachment({
+				accountConfigId: "cfg-sweep",
+				accountId: "acc-sweep",
+				outboxMessageId: draft,
+				outboxAttachmentId: "att1",
+				content: Buffer.alloc(4),
+			});
+		}
+
+		assert.deepStrictEqual(
+			(
+				await storage.listOutboxDraftsWithAttachments("cfg-sweep", "acc-sweep")
+			).sort(),
+			["draft-x", "draft-y"],
+		);
+	});
+
+	test("deleting attachments on a draft that never had any is quiet", async () => {
+		const storage = createFilesystemStorageService(testBasePath);
+		await storage.deleteOutboxAttachments("cfg1", "acc123", "draft-never");
+		await storage.deleteOutboxAttachment(
+			"cfg1",
+			"acc123",
+			"draft-never",
+			"att1",
+		);
+	});
+
 	test("cleanup test directory", async () => {
 		await rm(testBasePath, { recursive: true, force: true });
 	});