@remit/storage-service 0.0.12 → 0.0.13

package/src/storage.test.ts

@@ -1,646 +0,0 @@
-import assert from "node:assert";
-import { createHash } from "node:crypto";
-import { rm } from "node:fs/promises";
-import { tmpdir } from "node:os";
-import { join } from "node:path";
-import { Readable } from "node:stream";
-import { describe, test } from "node:test";
-import { ContentEncoding, StorageType } from "@remit/domain-enums";
-import { createFilesystemStorageService } from "./backends/filesystem.js";
-import {
-	buildBodyPartKey,
-	buildDeduplicatedKey,
-	buildMessageBodyKey,
-	buildOutboxAttachmentKey,
-	buildOutboxAttachmentPrefix,
-	buildParsedBodyKey,
-	computeChecksum,
-	createMockStorageService,
-	isStorageNotFoundError,
-	parseContentStorageKey,
-} from "./storage.js";
-import { buildStorageUri, parseStorageUri } from "./uri.js";
-
-describe("parseStorageUri", () => {
-	test("parses S3 URIs", () => {
-		const result = parseStorageUri("s3://my-bucket/path/to/object");
-		assert.strictEqual(result.storageType, StorageType.S3);
-		assert.strictEqual(result.storageLocation, "my-bucket");
-		assert.strictEqual(result.storageKey, "path/to/object");
-	});
-
-	test("parses file URIs", () => {
-		const result = parseStorageUri("file:///var/data/remit/path/to/file");
-		assert.strictEqual(result.storageType, StorageType.Filesystem);
-		assert.strictEqual(result.storageKey, "/var/data/remit/path/to/file");
-	});
-
-	test("throws for unsupported schemes", () => {
-		assert.throws(
-			() => parseStorageUri("http://example.com/file"),
-			/Unsupported storage URI scheme/,
-		);
-	});
-});
-
-describe("buildStorageUri", () => {
-	test("builds S3 URIs", () => {
-		const uri = buildStorageUri(StorageType.S3, "my-bucket", "path/to/object");
-		assert.strictEqual(uri, "s3://my-bucket/path/to/object");
-	});
-
-	test("builds file URIs", () => {
-		const uri = buildStorageUri(
-			StorageType.Filesystem,
-			"/var/data",
-			"path/to/file",
-		);
-		assert.strictEqual(uri, "file:///var/data/path/to/file");
-	});
-});
-
-describe("path builders", () => {
-	test("buildMessageBodyKey nests accountId under accountConfigId for tenant-scoped CloudFront caching", () => {
-		const key = buildMessageBodyKey("cfg1", "acc123", "msg456");
-		assert.strictEqual(key, "accounts/cfg1/acc123/messages/msg456/body.eml");
-	});
-
-	test("buildBodyPartKey nests accountId under accountConfigId", () => {
-		const key = buildBodyPartKey("cfg1", "acc123", "msg456", "1.2");
-		assert.strictEqual(key, "accounts/cfg1/acc123/messages/msg456/parts/1.2");
-	});
-
-	test("buildDeduplicatedKey nests accountId under accountConfigId", () => {
-		const hash = "abcdef1234567890";
-		const key = buildDeduplicatedKey("cfg1", "acc123", hash);
-		assert.strictEqual(key, "accounts/cfg1/acc123/dedup/ab/abcdef1234567890");
-	});
-
-	test("buildParsedBodyKey nests accountId under accountConfigId", () => {
-		const key = buildParsedBodyKey("cfg1", "acc123", "msg456");
-		assert.strictEqual(
-			key,
-			"accounts/cfg1/acc123/messages/msg456/parsed.json.gz",
-		);
-	});
-
-	test("buildOutboxAttachmentKey hangs the file off its own draft, under the same tenant prefix", () => {
-		const key = buildOutboxAttachmentKey("cfg1", "acc123", "draft9", "att7");
-		assert.strictEqual(
-			key,
-			"accounts/cfg1/acc123/outbox/draft9/attachments/att7",
-		);
-		assert.ok(
-			key.startsWith(buildOutboxAttachmentPrefix("cfg1", "acc123", "draft9")),
-		);
-	});
-
-	test("parseContentStorageKey does not recognise an outbox attachment key", () => {
-		assert.strictEqual(
-			parseContentStorageKey(
-				buildOutboxAttachmentKey("cfg1", "acc123", "draft9", "att7"),
-			),
-			null,
-		);
-	});
-
-	test("computeChecksum returns SHA-256 hex", () => {
-		const content = Buffer.from("test content");
-		const expected = createHash("sha256").update(content).digest("hex");
-		assert.strictEqual(computeChecksum(content), expected);
-	});
-
-	test("path order is accountConfigId before accountId so /content/accounts/{configId}/* matches at the edge", () => {
-		const key = buildMessageBodyKey("CFG", "ACC", "MSG");
-		const segments = key.split("/");
-		assert.strictEqual(segments[0], "accounts");
-		assert.strictEqual(segments[1], "CFG");
-		assert.strictEqual(segments[2], "ACC");
-	});
-});
-
-describe("parseContentStorageKey", () => {
-	test("parses accountConfigId/accountId/messageId out of a body key", () => {
-		const key = buildMessageBodyKey("cfg1", "acc123", "msg456");
-		assert.deepEqual(parseContentStorageKey(key), {
-			accountConfigId: "cfg1",
-			accountId: "acc123",
-			messageId: "msg456",
-		});
-	});
-
-	test("parses a part key the same way as a body key", () => {
-		const key = buildBodyPartKey("cfg1", "acc123", "msg456", "1.2");
-		assert.deepEqual(parseContentStorageKey(key), {
-			accountConfigId: "cfg1",
-			accountId: "acc123",
-			messageId: "msg456",
-		});
-	});
-
-	test("returns null for a key that doesn't match the accounts/.../messages/... layout", () => {
-		assert.strictEqual(parseContentStorageKey("nope/whatever"), null);
-	});
-});
-
-describe("isStorageNotFoundError", () => {
-	test("matches S3 NoSuchKey via name", () => {
-		assert.equal(
-			isStorageNotFoundError(
-				Object.assign(new Error("missing"), { name: "NoSuchKey" }),
-			),
-			true,
-		);
-	});
-
-	test("matches S3 NoSuchKey via Code", () => {
-		assert.equal(
-			isStorageNotFoundError({ Code: "NoSuchKey", message: "missing" }),
-			true,
-		);
-	});
-
-	test("matches filesystem ENOENT", () => {
-		assert.equal(
-			isStorageNotFoundError(
-				Object.assign(new Error("nope"), { code: "ENOENT" }),
-			),
-			true,
-		);
-	});
-
-	test("does not match generic errors", () => {
-		assert.equal(isStorageNotFoundError(new Error("boom")), false);
-		assert.equal(isStorageNotFoundError(null), false);
-		assert.equal(isStorageNotFoundError("oops"), false);
-	});
-});
-
-describe("createMockStorageService", () => {
-	test("storeMessageBody stores and retrieves content with nested accountConfigId/accountId path", async () => {
-		const storage = createMockStorageService();
-		const content = Buffer.from("Hello, world!");
-
-		const ref = await storage.storeMessageBody({
-			accountConfigId: "cfg1",
-			accountId: "acc123",
-			messageId: "msg456",
-			content,
-		});
-
-		assert.strictEqual(
-			ref.storageKey,
-			"accounts/cfg1/acc123/messages/msg456/body.eml",
-		);
-		assert.strictEqual(ref.sizeBytes, content.length);
-		assert.strictEqual(ref.contentEncoding, ContentEncoding.None);
-
-		const retrieved = await storage.retrieve(ref.uri);
-		assert.deepStrictEqual(retrieved, content);
-	});
-
-	test("storeBodyPart stores with part path", async () => {
-		const storage = createMockStorageService();
-		const content = Buffer.from("attachment content");
-
-		const ref = await storage.storeBodyPart({
-			accountConfigId: "cfg1",
-			accountId: "acc123",
-			messageId: "msg456",
-			partPath: "1.2",
-			content,
-		});
-
-		assert.strictEqual(
-			ref.storageKey,
-			"accounts/cfg1/acc123/messages/msg456/parts/1.2",
-		);
-
-		const retrieved = await storage.retrieve(ref.uri);
-		assert.deepStrictEqual(retrieved, content);
-	});
-
-	test("storeDeduplicated uses content hash", async () => {
-		const storage = createMockStorageService();
-		const content = Buffer.from("deduplicate me");
-		const hash = computeChecksum(content);
-
-		const ref = await storage.storeDeduplicated({
-			accountConfigId: "cfg1",
-			accountId: "acc123",
-			content,
-		});
-
-		assert.strictEqual(
-			ref.storageKey,
-			`accounts/cfg1/acc123/dedup/${hash.slice(0, 2)}/${hash}`,
-		);
-		assert.strictEqual(ref.checksumSha256, hash);
-	});
-
-	test("checks existence", async () => {
-		const storage = createMockStorageService();
-		const content = Buffer.from("test");
-
-		const ref = await storage.storeMessageBody({
-			accountConfigId: "cfg1",
-			accountId: "acc123",
-			messageId: "msg456",
-			content,
-		});
-
-		assert.strictEqual(await storage.exists(ref.uri), true);
-		assert.strictEqual(await storage.exists("mock://does-not-exist"), false);
-	});
-
-	test("deletes content", async () => {
-		const storage = createMockStorageService();
-		const content = Buffer.from("to delete");
-
-		const ref = await storage.storeMessageBody({
-			accountConfigId: "cfg1",
-			accountId: "acc123",
-			messageId: "msg456",
-			content,
-		});
-		assert.strictEqual(await storage.exists(ref.uri), true);
-
-		await storage.delete(ref.uri);
-		assert.strictEqual(await storage.exists(ref.uri), false);
-	});
-
-	test("storeParsedBody / retrieveParsedBody round-trip", async () => {
-		const storage = createMockStorageService();
-		const parsed = {
-			text: "hello",
-			html: "<p>hello</p>",
-			attachments: [
-				{
-					filename: "a.pdf",
-					contentType: "application/pdf",
-					contentDisposition: "attachment",
-					contentId: null,
-					size: 1234,
-				},
-			],
-		};
-
-		const ref = await storage.storeParsedBody({
-			accountConfigId: "cfg1",
-			accountId: "acc123",
-			messageId: "msg456",
-			parsed,
-		});
-
-		assert.strictEqual(
-			ref.storageKey,
-			"accounts/cfg1/acc123/messages/msg456/parsed.json.gz",
-		);
-
-		const retrieved = await storage.retrieveParsedBody(
-			"cfg1",
-			"acc123",
-			"msg456",
-		);
-		assert.deepStrictEqual(retrieved, parsed);
-	});
-
-	test("retrieveParsedBody returns null on miss", async () => {
-		const storage = createMockStorageService();
-		const result = await storage.retrieveParsedBody("cfg1", "acc123", "nope");
-		assert.strictEqual(result, null);
-	});
-
-	test("retrieveBodyPart round-trips stored content and returns null on miss", async () => {
-		const storage = createMockStorageService();
-		const content = Buffer.from("attachment bytes");
-
-		await storage.storeBodyPart({
-			accountConfigId: "cfg1",
-			accountId: "acc123",
-			messageId: "msg456",
-			partPath: "1.2",
-			content,
-		});
-
-		const retrieved = await storage.retrieveBodyPart(
-			"cfg1",
-			"acc123",
-			"msg456",
-			"1.2",
-		);
-		assert.deepStrictEqual(retrieved, content);
-
-		const miss = await storage.retrieveBodyPart(
-			"cfg1",
-			"acc123",
-			"msg456",
-			"9",
-		);
-		assert.strictEqual(miss, null);
-	});
-});
-
-describe("createFilesystemStorageService", () => {
-	const testBasePath = join(tmpdir(), `remit-storage-test-${Date.now()}`);
-
-	test("storeMessageBody stores and retrieves content", async () => {
-		const storage = createFilesystemStorageService(testBasePath);
-		const content = Buffer.from("Filesystem test content");
-
-		const ref = await storage.storeMessageBody({
-			accountConfigId: "cfg1",
-			accountId: "acc123",
-			messageId: "msg456",
-			content,
-		});
-
-		assert.strictEqual(ref.storageType, StorageType.Filesystem);
-		assert.strictEqual(ref.storageLocation, testBasePath);
-		assert.strictEqual(
-			ref.storageKey,
-			"accounts/cfg1/acc123/messages/msg456/body.eml",
-		);
-		assert.ok(ref.uri.startsWith("file://"));
-
-		const retrieved = await storage.retrieve(ref.uri);
-		assert.deepStrictEqual(retrieved, content);
-	});
-
-	test("storeMessageBodyStream streams content to disk and round-trips via retrieve", async () => {
-		const storage = createFilesystemStorageService(testBasePath);
-		const content = Buffer.from("streamed filesystem body");
-
-		const ref = await storage.storeMessageBodyStream({
-			accountConfigId: "cfg1",
-			accountId: "acc123",
-			messageId: "msg-stream",
-			content: Readable.from(content),
-		});
-
-		assert.strictEqual(ref.storageType, StorageType.Filesystem);
-		assert.strictEqual(ref.contentEncoding, ContentEncoding.Gzip);
-		assert.strictEqual(ref.checksumSha256, computeChecksum(content));
-
-		const retrieved = await storage.retrieve(ref.uri);
-		assert.deepStrictEqual(retrieved, content);
-	});
-
-	test("storeBodyPart stores with part path", async () => {
-		const storage = createFilesystemStorageService(testBasePath);
-		const content = Buffer.from("attachment content");
-
-		const ref = await storage.storeBodyPart({
-			accountConfigId: "cfg1",
-			accountId: "acc123",
-			messageId: "msg456",
-			partPath: "1.2",
-			content,
-		});
-
-		assert.strictEqual(
-			ref.storageKey,
-			"accounts/cfg1/acc123/messages/msg456/parts/1.2",
-		);
-
-		const retrieved = await storage.retrieve(ref.uri);
-		assert.deepStrictEqual(retrieved, content);
-	});
-
-	test("storeDeduplicated uses content hash", async () => {
-		const storage = createFilesystemStorageService(testBasePath);
-		const content = Buffer.from("dedupe fs content");
-		const hash = computeChecksum(content);
-
-		const ref = await storage.storeDeduplicated({
-			accountConfigId: "cfg1",
-			accountId: "acc123",
-			content,
-		});
-
-		assert.strictEqual(
-			ref.storageKey,
-			`accounts/cfg1/acc123/dedup/${hash.slice(0, 2)}/${hash}`,
-		);
-		assert.strictEqual(ref.checksumSha256, hash);
-
-		const retrieved = await storage.retrieve(ref.uri);
-		assert.deepStrictEqual(retrieved, content);
-	});
-
-	test("checks existence", async () => {
-		const storage = createFilesystemStorageService(testBasePath);
-		const content = Buffer.from("exists test");
-
-		const ref = await storage.storeMessageBody({
-			accountConfigId: "cfg1",
-			accountId: "acc123",
-			messageId: "msg789",
-			content,
-		});
-
-		assert.strictEqual(await storage.exists(ref.uri), true);
-		assert.strictEqual(await storage.exists("file:///nonexistent/path"), false);
-	});
-
-	test("deletes content", async () => {
-		const storage = createFilesystemStorageService(testBasePath);
-		const content = Buffer.from("to be deleted");
-
-		const ref = await storage.storeMessageBody({
-			accountConfigId: "cfg1",
-			accountId: "acc123",
-			messageId: "msg-delete",
-			content,
-		});
-		assert.strictEqual(await storage.exists(ref.uri), true);
-
-		await storage.delete(ref.uri);
-		assert.strictEqual(await storage.exists(ref.uri), false);
-	});
-
-	test("storeParsedBody / retrieveParsedBody round-trip with gzip", async () => {
-		const storage = createFilesystemStorageService(testBasePath);
-		const parsed = {
-			text: "hello fs",
-			html: "<p>hello fs</p>",
-			attachments: [],
-		};
-
-		const ref = await storage.storeParsedBody({
-			accountConfigId: "cfg1",
-			accountId: "acc123",
-			messageId: "msg-parsed",
-			parsed,
-		});
-
-		assert.strictEqual(
-			ref.storageKey,
-			"accounts/cfg1/acc123/messages/msg-parsed/parsed.json.gz",
-		);
-
-		const retrieved = await storage.retrieveParsedBody(
-			"cfg1",
-			"acc123",
-			"msg-parsed",
-		);
-		assert.deepStrictEqual(retrieved, parsed);
-	});
-
-	test("retrieveParsedBody returns null on miss (ENOENT)", async () => {
-		const storage = createFilesystemStorageService(testBasePath);
-		const result = await storage.retrieveParsedBody(
-			"cfg1",
-			"acc123",
-			"missing",
-		);
-		assert.strictEqual(result, null);
-	});
-
-	test("retrieveBodyPart round-trips gzipped content and returns null on miss (ENOENT)", async () => {
-		const storage = createFilesystemStorageService(testBasePath);
-		const content = Buffer.from("fs attachment bytes");
-
-		await storage.storeBodyPart({
-			accountConfigId: "cfg1",
-			accountId: "acc123",
-			messageId: "msg-part-fs",
-			partPath: "1.2",
-			content,
-		});
-
-		const retrieved = await storage.retrieveBodyPart(
-			"cfg1",
-			"acc123",
-			"msg-part-fs",
-			"1.2",
-		);
-		assert.deepStrictEqual(retrieved, content);
-
-		const miss = await storage.retrieveBodyPart(
-			"cfg1",
-			"acc123",
-			"msg-part-fs",
-			"nope",
-		);
-		assert.strictEqual(miss, null);
-	});
-
-	test("storeOutboxAttachment writes the bytes uncompressed so the size on disk is the size the cap counts", async () => {
-		const storage = createFilesystemStorageService(testBasePath);
-		// Highly compressible: gzip would make the stored size disagree with the
-		// decoded size the per-message cap is expressed in.
-		const content = Buffer.alloc(4096, 0x61);
-
-		const ref = await storage.storeOutboxAttachment({
-			accountConfigId: "cfg1",
-			accountId: "acc123",
-			outboxMessageId: "draft1",
-			outboxAttachmentId: "att1",
-			content,
-		});
-
-		assert.strictEqual(ref.contentEncoding, ContentEncoding.None);
-		assert.strictEqual(ref.sizeBytes, content.length);
-		assert.strictEqual(ref.checksumSha256, computeChecksum(content));
-		assert.deepStrictEqual(await storage.retrieve(ref.uri), content);
-
-		const [item] = await storage.listOutboxAttachments(
-			"cfg1",
-			"acc123",
-			"draft1",
-		);
-		assert.strictEqual(item.sizeBytes, content.length);
-	});
-
-	test("listOutboxAttachments returns only the named draft, and every object in it", async () => {
-		const storage = createFilesystemStorageService(testBasePath);
-		for (const id of ["a", "b", "c"]) {
-			await storage.storeOutboxAttachment({
-				accountConfigId: "cfg1",
-				accountId: "acc123",
-				outboxMessageId: "draft-many",
-				outboxAttachmentId: id,
-				content: Buffer.from(`file-${id}`),
-			});
-		}
-		await storage.storeOutboxAttachment({
-			accountConfigId: "cfg1",
-			accountId: "acc123",
-			outboxMessageId: "draft-other",
-			outboxAttachmentId: "d",
-			content: Buffer.from("elsewhere"),
-		});
-
-		const all = await storage.listOutboxAttachments(
-			"cfg1",
-			"acc123",
-			"draft-many",
-		);
-		assert.deepStrictEqual(all.map((item) => item.outboxAttachmentId).sort(), [
-			"a",
-			"b",
-			"c",
-		]);
-	});
-
-	test("listOutboxAttachments returns an empty array for a draft that has none", async () => {
-		const storage = createFilesystemStorageService(testBasePath);
-		assert.deepStrictEqual(
-			await storage.listOutboxAttachments("cfg1", "acc123", "draft-empty"),
-			[],
-		);
-	});
-
-	test("deleting a draft's attachments empties the prefix and the ledger with it", async () => {
-		const storage = createFilesystemStorageService(testBasePath);
-		await storage.storeOutboxAttachment({
-			accountConfigId: "cfg1",
-			accountId: "acc123",
-			outboxMessageId: "draft-gone",
-			outboxAttachmentId: "att1",
-			content: Buffer.alloc(4),
-		});
-		await storage.deleteOutboxAttachments("cfg1", "acc123", "draft-gone");
-
-		assert.deepStrictEqual(
-			await storage.listOutboxAttachments("cfg1", "acc123", "draft-gone"),
-			[],
-		);
-	});
-
-	test("listOutboxDraftsWithAttachments finds prefixes whose row may be long gone", async () => {
-		const storage = createFilesystemStorageService(testBasePath);
-		for (const draft of ["draft-x", "draft-y"]) {
-			await storage.storeOutboxAttachment({
-				accountConfigId: "cfg-sweep",
-				accountId: "acc-sweep",
-				outboxMessageId: draft,
-				outboxAttachmentId: "att1",
-				content: Buffer.alloc(4),
-			});
-		}
-
-		assert.deepStrictEqual(
-			(
-				await storage.listOutboxDraftsWithAttachments("cfg-sweep", "acc-sweep")
-			).sort(),
-			["draft-x", "draft-y"],
-		);
-	});
-
-	test("deleting attachments on a draft that never had any is quiet", async () => {
-		const storage = createFilesystemStorageService(testBasePath);
-		await storage.deleteOutboxAttachments("cfg1", "acc123", "draft-never");
-		await storage.deleteOutboxAttachment(
-			"cfg1",
-			"acc123",
-			"draft-never",
-			"att1",
-		);
-	});
-
-	test("cleanup test directory", async () => {
-		await rm(testBasePath, { recursive: true, force: true });
-	});
-});