@remit/storage-service 0.0.12 → 0.0.14

package/src/outbox-attachment-retrieve.test.ts

@@ -1,56 +0,0 @@
-import assert from "node:assert/strict";
-import { mkdtemp, rm } from "node:fs/promises";
-import { tmpdir } from "node:os";
-import { join } from "node:path";
-import { describe, it } from "node:test";
-import { createFilesystemStorageService } from "./backends/filesystem.js";
-import { createMockStorageService, type StorageService } from "./storage.js";
-
-const OWNER = {
-	accountConfigId: "cfg-1",
-	accountId: "acc-1",
-	outboxMessageId: "ob-1",
-};
-
-const readsBackWhatWasStored = async (storage: StorageService) => {
-	const content = Buffer.from([0x25, 0x50, 0x44, 0x46, 0x00, 0xff]);
-	await storage.storeOutboxAttachment({
-		...OWNER,
-		outboxAttachmentId: "att-1",
-		content,
-	});
-
-	assert.deepEqual(
-		await storage.retrieveOutboxAttachment(
-			OWNER.accountConfigId,
-			OWNER.accountId,
-			OWNER.outboxMessageId,
-			"att-1",
-		),
-		content,
-	);
-	assert.equal(
-		await storage.retrieveOutboxAttachment(
-			OWNER.accountConfigId,
-			OWNER.accountId,
-			OWNER.outboxMessageId,
-			"att-missing",
-		),
-		null,
-	);
-};
-
-describe("retrieveOutboxAttachment", () => {
-	it("reads back the exact bytes on the filesystem backend, and null for none", async () => {
-		const basePath = await mkdtemp(join(tmpdir(), "remit-outbox-retrieve-"));
-		try {
-			await readsBackWhatWasStored(createFilesystemStorageService(basePath));
-		} finally {
-			await rm(basePath, { recursive: true, force: true });
-		}
-	});
-
-	it("reads back the exact bytes on the mock backend, and null for none", async () => {
-		await readsBackWhatWasStored(createMockStorageService());
-	});
-});

package/src/outbox-sweep.test.ts

@@ -1,196 +0,0 @@
-import assert from "node:assert";
-import { describe, test } from "node:test";
-import { sweepAbandonedOutboxAttachments } from "./outbox-sweep.js";
-import { createMockStorageService, type StorageService } from "./storage.js";
-
-const CFG = "cfg1";
-const ACC = "acc1";
-
-const withObjects = async (
-	drafts: Record<string, string[]>,
-): Promise<StorageService> => {
-	const storage = createMockStorageService();
-	for (const [outboxMessageId, ids] of Object.entries(drafts)) {
-		for (const outboxAttachmentId of ids) {
-			await storage.storeOutboxAttachment({
-				accountConfigId: CFG,
-				accountId: ACC,
-				outboxMessageId,
-				outboxAttachmentId,
-				content: Buffer.alloc(4),
-			});
-		}
-	}
-	return storage;
-};
-
-const remaining = (storage: StorageService, draft: string) =>
-	storage
-		.listOutboxAttachments(CFG, ACC, draft)
-		.then((items) => items.map((item) => item.outboxAttachmentId).sort());
-
-describe("sweepAbandonedOutboxAttachments", () => {
-	test("deletes objects the database does not know about", async () => {
-		const storage = await withObjects({ draft1: ["known", "orphan"] });
-
-		const result = await sweepAbandonedOutboxAttachments(
-			{
-				storage,
-				listKnownAttachmentIds: async () => ["known"],
-				onSkipped: () => assert.fail("nothing should be skipped"),
-			},
-			CFG,
-			ACC,
-		);
-
-		assert.deepStrictEqual(await remaining(storage, "draft1"), ["known"]);
-		assert.strictEqual(result.deleted, 1);
-		assert.strictEqual(result.skipped, 0);
-	});
-
-	test("empties the prefix of a draft the database has forgotten", async () => {
-		// A discarded draft has no rows at all, and a presigned PUT can still have
-		// landed under it. This is the case the sweep exists for.
-		const storage = await withObjects({ discarded: ["a", "b"] });
-
-		const result = await sweepAbandonedOutboxAttachments(
-			{
-				storage,
-				listKnownAttachmentIds: async () => [],
-				onSkipped: () => assert.fail("nothing should be skipped"),
-			},
-			CFG,
-			ACC,
-		);
-
-		assert.deepStrictEqual(await remaining(storage, "discarded"), []);
-		assert.strictEqual(result.deleted, 2);
-	});
-
-	test("leaves a draft alone when its rows cannot be read", async () => {
-		// "I could not find out what is live" must never be acted on as "nothing
-		// is live" — that would delete a whole draft's files on a transient error.
-		const storage = await withObjects({ draft1: ["a", "b"] });
-		const skips: string[] = [];
-
-		const result = await sweepAbandonedOutboxAttachments(
-			{
-				storage,
-				listKnownAttachmentIds: async () => {
-					throw new Error("database unavailable");
-				},
-				onSkipped: (outboxMessageId) => skips.push(outboxMessageId),
-			},
-			CFG,
-			ACC,
-		);
-
-		assert.deepStrictEqual(await remaining(storage, "draft1"), ["a", "b"]);
-		assert.strictEqual(result.deleted, 0);
-		assert.strictEqual(result.skipped, 1);
-		assert.deepStrictEqual(skips, ["draft1"]);
-	});
-
-	test("one unreadable draft does not stop the others being collected", async () => {
-		const storage = await withObjects({ bad: ["x"], good: ["orphan"] });
-
-		const result = await sweepAbandonedOutboxAttachments(
-			{
-				storage,
-				listKnownAttachmentIds: async (_cfg, outboxMessageId) => {
-					if (outboxMessageId === "bad") throw new Error("unavailable");
-					return [];
-				},
-				onSkipped: () => {},
-			},
-			CFG,
-			ACC,
-		);
-
-		assert.deepStrictEqual(await remaining(storage, "bad"), ["x"]);
-		assert.deepStrictEqual(await remaining(storage, "good"), []);
-		assert.strictEqual(result.skipped, 1);
-		assert.strictEqual(result.deleted, 1);
-	});
-
-	test("an attachment that lands between the two reads is kept", async () => {
-		// Objects are listed before the rows are read. A mint that lands in that
-		// window is seen as a row and not as an object, so it survives. Read the
-		// rows first and the same mint is an object with no row behind it — this
-		// fails if those two lines ever swap.
-		const storage = await withObjects({ draft1: ["already-there"] });
-		let listedObjects = false;
-
-		const inner = storage.listOutboxAttachments.bind(storage);
-		const observing = {
-			...storage,
-			listOutboxAttachments: async (
-				accountConfigId: string,
-				accountId: string,
-				outboxMessageId: string,
-			) => {
-				listedObjects = true;
-				return inner(accountConfigId, accountId, outboxMessageId);
-			},
-		} as StorageService;
-
-		await sweepAbandonedOutboxAttachments(
-			{
-				storage: observing,
-				listKnownAttachmentIds: async () => {
-					assert.strictEqual(
-						listedObjects,
-						true,
-						"rows must be read after the objects were listed, never before",
-					);
-					// The mint that landed in the window.
-					await storage.storeOutboxAttachment({
-						accountConfigId: CFG,
-						accountId: ACC,
-						outboxMessageId: "draft1",
-						outboxAttachmentId: "just-reserved",
-						content: Buffer.alloc(4),
-					});
-					return ["already-there", "just-reserved"];
-				},
-				onSkipped: () => {},
-			},
-			CFG,
-			ACC,
-		);
-
-		assert.deepStrictEqual(await remaining(storage, "draft1"), [
-			"already-there",
-			"just-reserved",
-		]);
-	});
-
-	test("never reaches another tenant's objects", async () => {
-		const storage = createMockStorageService();
-		await storage.storeOutboxAttachment({
-			accountConfigId: "cfg-other",
-			accountId: ACC,
-			outboxMessageId: "draft1",
-			outboxAttachmentId: "theirs",
-			content: Buffer.alloc(4),
-		});
-
-		const result = await sweepAbandonedOutboxAttachments(
-			{
-				storage,
-				listKnownAttachmentIds: async () => [],
-				onSkipped: () => {},
-			},
-			CFG,
-			ACC,
-		);
-
-		assert.strictEqual(result.deleted, 0);
-		assert.deepStrictEqual(
-			(await storage.listOutboxAttachments("cfg-other", ACC, "draft1")).map(
-				(item) => item.outboxAttachmentId,
-			),
-			["theirs"],
-		);
-	});
-});

package/src/outbox-sweep.ts

@@ -1,103 +0,0 @@
-import type { StorageService } from "./storage.js";
-
-/**
- * Collect attachment bytes the database does not know about.
- *
- * A presigned upload URL stays valid for its whole window, including after the
- * draft it was minted for is discarded, and on a deployment where the browser
- * PUTs straight to block storage there is nothing in that path to consult. So
- * an object can exist that no row names. That is the only case this exists for,
- * and the rule is the whole of it: an object survives if the database still has
- * an attachment row for it.
- *
- * Two orderings matter and both are deliberate. Objects are listed *before* the
- * rows are read, so an attachment reserved between the two reads is seen as a
- * row and not as an object, and is kept. And a draft whose row lookup fails is
- * skipped entirely — "I could not find out what is live" must never be acted on
- * as "nothing is live".
- */
-export interface OutboxSweepResult {
-	inspected: number;
-	deleted: number;
-	/** Drafts skipped because their attachment rows could not be read. */
-	skipped: number;
-}
-
-export interface OutboxSweepDeps {
-	storage: StorageService;
-	/**
-	 * The attachment ids the database holds for a draft. Throwing is the right
-	 * way to say "unknown" — the sweep then leaves that draft alone.
-	 */
-	listKnownAttachmentIds: (
-		accountConfigId: string,
-		outboxMessageId: string,
-	) => Promise<string[]>;
-	onSkipped: (outboxMessageId: string, error: unknown) => void;
-}
-
-export const sweepAbandonedOutboxAttachments = async (
-	deps: OutboxSweepDeps,
-	accountConfigId: string,
-	accountId: string,
-): Promise<OutboxSweepResult> => {
-	let inspected = 0;
-	let deleted = 0;
-	let skipped = 0;
-
-	const outboxMessageIds = await deps.storage.listOutboxDraftsWithAttachments(
-		accountConfigId,
-		accountId,
-	);
-
-	for (const outboxMessageId of outboxMessageIds) {
-		const objects = await deps.storage.listOutboxAttachments(
-			accountConfigId,
-			accountId,
-			outboxMessageId,
-		);
-		if (objects.length === 0) continue;
-
-		const known = await deps
-			.listKnownAttachmentIds(accountConfigId, outboxMessageId)
-			.then((ids) => new Set(ids))
-			.catch((error: unknown) => {
-				deps.onSkipped(outboxMessageId, error);
-				return null;
-			});
-		if (known === null) {
-			skipped += 1;
-			continue;
-		}
-
-		inspected += objects.length;
-		const abandoned = objects.filter(
-			(object) => !known.has(object.outboxAttachmentId),
-		);
-
-		// A draft whose every object is abandoned is the common case — it was
-		// discarded — and taking the prefix in one call beats one round trip per
-		// object.
-		if (abandoned.length === objects.length) {
-			await deps.storage.deleteOutboxAttachments(
-				accountConfigId,
-				accountId,
-				outboxMessageId,
-			);
-			deleted += abandoned.length;
-			continue;
-		}
-
-		for (const object of abandoned) {
-			await deps.storage.deleteOutboxAttachment(
-				accountConfigId,
-				accountId,
-				outboxMessageId,
-				object.outboxAttachmentId,
-			);
-			deleted += 1;
-		}
-	}
-
-	return { inspected, deleted, skipped };
-};

package/src/outbox-upload-url.test.ts

@@ -1,290 +0,0 @@
-import assert from "node:assert";
-import { mkdtemp, rm } from "node:fs/promises";
-import { tmpdir } from "node:os";
-import { join } from "node:path";
-import { describe, test } from "node:test";
-import { S3Client } from "@aws-sdk/client-s3";
-import { createFilesystemStorageService } from "./backends/filesystem.js";
-import { createS3StorageService } from "./backends/s3.js";
-import {
-	authorizeUploadRequest,
-	UPLOAD_ROUTE_PREFIX,
-	UPLOAD_URL_SIGNING_LABEL,
-} from "./outbox-upload-url.js";
-import { signStoragePath } from "./signed-path.js";
-import {
-	buildOutboxAttachmentKey,
-	parseOutboxAttachmentKey,
-} from "./storage.js";
-
-const SECRET = "a-signing-secret-of-at-least-32-characters";
-const KEY = buildOutboxAttachmentKey("cfg1", "acc1", "draft1", "att1");
-
-const sign = (path: string, exp: number, max: number): string =>
-	signStoragePath(SECRET, UPLOAD_URL_SIGNING_LABEL, path, [exp, max]);
-
-describe("authorizeUploadRequest", () => {
-	const now = 1_000_000;
-	const exp = now + 900;
-
-	test("admits a request presenting the signature that was minted for it", () => {
-		const result = authorizeUploadRequest({
-			secret: SECRET,
-			relativePath: KEY,
-			exp: String(exp),
-			max: "2048",
-			sig: sign(KEY, exp, 2048),
-			nowSeconds: now,
-		});
-		assert.deepStrictEqual(result, { authorized: true, maxBytes: 2048 });
-	});
-
-	test("refuses a signature minted for another attachment's key", () => {
-		const otherKey = buildOutboxAttachmentKey("cfg1", "acc1", "draft1", "att2");
-		const result = authorizeUploadRequest({
-			secret: SECRET,
-			relativePath: otherKey,
-			exp: String(exp),
-			max: "2048",
-			sig: sign(KEY, exp, 2048),
-			nowSeconds: now,
-		});
-		assert.strictEqual(result.authorized, false);
-	});
-
-	test("refuses a signature minted for another account's key", () => {
-		const otherTenant = buildOutboxAttachmentKey(
-			"cfg2",
-			"acc1",
-			"draft1",
-			"att1",
-		);
-		const result = authorizeUploadRequest({
-			secret: SECRET,
-			relativePath: otherTenant,
-			exp: String(exp),
-			max: "2048",
-			sig: sign(KEY, exp, 2048),
-			nowSeconds: now,
-		});
-		assert.strictEqual(result.authorized, false);
-	});
-
-	test("refuses a raised byte count — the size is part of what was signed", () => {
-		const result = authorizeUploadRequest({
-			secret: SECRET,
-			relativePath: KEY,
-			exp: String(exp),
-			max: "999999",
-			sig: sign(KEY, exp, 2048),
-			nowSeconds: now,
-		});
-		assert.strictEqual(result.authorized, false);
-	});
-
-	test("refuses an extended expiry", () => {
-		const result = authorizeUploadRequest({
-			secret: SECRET,
-			relativePath: KEY,
-			exp: String(exp + 10_000),
-			max: "2048",
-			sig: sign(KEY, exp, 2048),
-			nowSeconds: now,
-		});
-		assert.strictEqual(result.authorized, false);
-	});
-
-	test("refuses a lapsed URL", () => {
-		const result = authorizeUploadRequest({
-			secret: SECRET,
-			relativePath: KEY,
-			exp: String(exp),
-			max: "2048",
-			sig: sign(KEY, exp, 2048),
-			nowSeconds: exp + 1,
-		});
-		assert.strictEqual(result.authorized, false);
-		assert.strictEqual(result.authorized === false && result.reason, "expired");
-	});
-
-	test("refuses a request presenting no signature at all", () => {
-		const result = authorizeUploadRequest({
-			secret: SECRET,
-			relativePath: KEY,
-			exp: String(exp),
-			max: "2048",
-			sig: undefined,
-			nowSeconds: now,
-		});
-		assert.strictEqual(result.authorized === false && result.status, 401);
-	});
-
-	test("fails closed when no secret is configured", () => {
-		const result = authorizeUploadRequest({
-			secret: undefined,
-			relativePath: KEY,
-			exp: String(exp),
-			max: "2048",
-			sig: sign(KEY, exp, 2048),
-			nowSeconds: now,
-		});
-		assert.strictEqual(result.authorized === false && result.status, 500);
-	});
-
-	test("a read grant for the same path is not a write grant", () => {
-		// The content signer covers the path and an expiry under its own label.
-		const readSignature = signStoragePath(
-			SECRET,
-			"remit-content-url-signing-v1",
-			KEY,
-			[exp],
-		);
-		const result = authorizeUploadRequest({
-			secret: SECRET,
-			relativePath: KEY,
-			exp: String(exp),
-			max: "2048",
-			sig: readSignature,
-			nowSeconds: now,
-		});
-		assert.strictEqual(result.authorized, false);
-	});
-});
-
-describe("attachment keys", () => {
-	test("parseOutboxAttachmentKey recovers the ids the upload route needs", () => {
-		assert.deepStrictEqual(parseOutboxAttachmentKey(KEY), {
-			accountConfigId: "cfg1",
-			accountId: "acc1",
-			outboxMessageId: "draft1",
-			outboxAttachmentId: "att1",
-		});
-	});
-
-	test("parseOutboxAttachmentKey refuses a message content key", () => {
-		assert.strictEqual(
-			parseOutboxAttachmentKey("accounts/cfg1/acc1/messages/msg1/parts/1.2"),
-			null,
-		);
-	});
-
-	test("parseOutboxAttachmentKey refuses a traversal, signature or no signature", () => {
-		// `..` matches a naive [^/]+ and the write side joins this onto a
-		// filesystem root, so the shape has to refuse it before the HMAC is the
-		// only thing left standing.
-		assert.strictEqual(
-			parseOutboxAttachmentKey(
-				"accounts/cfg1/acc1/outbox/draft1/attachments/..",
-			),
-			null,
-		);
-		assert.strictEqual(
-			parseOutboxAttachmentKey("accounts/../../etc/outbox/x/attachments/y"),
-			null,
-		);
-		assert.strictEqual(
-			parseOutboxAttachmentKey(
-				"accounts/cfg1/acc1/outbox/draft1/attachments/a.b",
-			),
-			null,
-		);
-	});
-});
-
-describe("minting an upload URL", () => {
-	test("the filesystem backend mints a URL its own route will admit", async () => {
-		const basePath = await mkdtemp(join(tmpdir(), "remit-upload-url-"));
-		try {
-			const storage = createFilesystemStorageService(basePath, {
-				origin: "https://mail.example.test/",
-				signingSecret: SECRET,
-			});
-			const expiresAt = Math.floor(Date.now() / 1000) + 900;
-
-			const { uploadUrl } = await storage.createOutboxAttachmentUploadUrl({
-				accountConfigId: "cfg1",
-				accountId: "acc1",
-				outboxMessageId: "draft1",
-				outboxAttachmentId: "att1",
-				sizeBytes: 2048,
-				expiresAt,
-			});
-
-			const url = new URL(uploadUrl);
-			assert.strictEqual(url.origin, "https://mail.example.test");
-			assert.strictEqual(url.pathname, `${UPLOAD_ROUTE_PREFIX}${KEY}`);
-
-			const admitted = authorizeUploadRequest({
-				secret: SECRET,
-				relativePath: url.pathname.slice(UPLOAD_ROUTE_PREFIX.length),
-				exp: url.searchParams.get("exp") ?? undefined,
-				max: url.searchParams.get("max") ?? undefined,
-				sig: url.searchParams.get("sig") ?? undefined,
-				nowSeconds: Math.floor(Date.now() / 1000),
-			});
-			assert.deepStrictEqual(admitted, { authorized: true, maxBytes: 2048 });
-		} finally {
-			await rm(basePath, { recursive: true, force: true });
-		}
-	});
-
-	test("the filesystem backend refuses to mint without an origin and a secret", async () => {
-		const basePath = await mkdtemp(join(tmpdir(), "remit-upload-url-"));
-		try {
-			const storage = createFilesystemStorageService(basePath);
-			await assert.rejects(
-				() =>
-					storage.createOutboxAttachmentUploadUrl({
-						accountConfigId: "cfg1",
-						accountId: "acc1",
-						outboxMessageId: "draft1",
-						outboxAttachmentId: "att1",
-						sizeBytes: 1,
-						expiresAt: Math.floor(Date.now() / 1000) + 60,
-					}),
-				/cannot mint an upload URL/,
-			);
-		} finally {
-			await rm(basePath, { recursive: true, force: true });
-		}
-	});
-
-	test("the S3 backend mints a presigned PUT with the size inside the signature", async () => {
-		const client = new S3Client({
-			region: "eu-west-1",
-			credentials: {
-				accessKeyId: "AKIATESTTESTTEST",
-				secretAccessKey: "test-secret",
-			},
-		});
-		const storage = createS3StorageService(client, "a-bucket");
-		const params = {
-			accountConfigId: "cfg1",
-			accountId: "acc1",
-			outboxMessageId: "draft1",
-			outboxAttachmentId: "att1",
-			sizeBytes: 2048,
-			expiresAt: Math.floor(Date.now() / 1000) + 900,
-		};
-
-		const { uploadUrl } = await storage.createOutboxAttachmentUploadUrl(params);
-		const url = new URL(uploadUrl);
-
-		assert.ok(url.pathname.endsWith(KEY));
-		// content-length inside SignedHeaders is what makes the declared size
-		// unforgeable: S3 recomputes the signature over the length it is sent.
-		assert.match(
-			url.searchParams.get("X-Amz-SignedHeaders") ?? "",
-			/content-length/,
-		);
-
-		const larger = await storage.createOutboxAttachmentUploadUrl({
-			...params,
-			sizeBytes: 4096,
-		});
-		assert.notStrictEqual(
-			url.searchParams.get("X-Amz-Signature"),
-			new URL(larger.uploadUrl).searchParams.get("X-Amz-Signature"),
-		);
-	});
-});

package/src/s3.ts

@@ -1,36 +0,0 @@
-import { S3Client } from "@aws-sdk/client-s3";
-import { createFilesystemStorageService } from "./backends/filesystem.js";
-import { createS3StorageService } from "./backends/s3.js";
-import type { StorageService } from "./storage.js";
-
-export { createS3StorageService } from "./backends/s3.js";
-
-export const createStorageService = (): StorageService => {
-	const bucketName = process.env.S3_BUCKET_NAME;
-
-	if (bucketName) {
-		const client = new S3Client({
-			endpoint: process.env.S3_ENDPOINT,
-			// The SDK default (WHEN_SUPPORTED) hangs a CRC32 of an empty body off
-			// every presigned PUT as signed query params. Real S3 tolerates it;
-			// MinIO and LocalStack reject the request, and S3_ENDPOINT exists to
-			// point at exactly those.
-			requestChecksumCalculation: "WHEN_REQUIRED",
-		});
-		return createS3StorageService(client, bucketName);
-	}
-
-	const basePath = process.env.STORAGE_LOCAL_PATH ?? ".remit/storage";
-	// BETTER_AUTH_URL is the deployment's own public base — the value the token
-	// issuer and the JWKS discovery document already agree on — so a minted
-	// upload URL resolves from wherever the browser reached the API. Both it and
-	// the signing secret are required on the self-host stack; without them a
-	// mint fails loud inside the backend rather than handing out a URL that
-	// cannot be verified.
-	const origin = process.env.BETTER_AUTH_URL;
-	const signingSecret = process.env.BETTER_AUTH_SECRET;
-	return createFilesystemStorageService(
-		basePath,
-		origin && signingSecret ? { origin, signingSecret } : undefined,
-	);
-};