@remit/storage-service 0.0.12 → 0.0.13

package/package.json

@@ -1,48 +1,53 @@
 {
-  "name": "@remit/storage-service",
-  "version": "0.0.12",
-  "type": "module",
-  "main": "src/index.ts",
-  "types": "src/index.ts",
-  "exports": {
-    ".": {
-      "types": "./src/index.ts",
-      "default": "./src/index.ts"
-    },
-    "./filesystem": {
-      "types": "./src/filesystem.ts",
-      "default": "./src/filesystem.ts"
-    },
-    "./s3": {
-      "types": "./src/s3.ts",
-      "default": "./src/s3.ts"
-    },
-    "./signed-path": {
-      "types": "./src/signed-path.ts",
-      "default": "./src/signed-path.ts"
-    }
-  },
-  "scripts": {
-    "test:typecheck": "tsgo --noEmit",
-    "test:run": "node $NODE_TEST_FLAGS --env-file=../../localhost-test-unit.env --import tsx --experimental-test-coverage --test-coverage-include='src/**' --test-coverage-exclude='src/**/*.test.ts' --test-coverage-exclude='src/**/*.test.tsx' --test-coverage-lines=87 --test 'src/**/*.test.ts'",
-    "test": "npm run test:typecheck && npm run test:run"
-  },
-  "dependencies": {
-    "@aws-sdk/client-s3": "*",
-    "@aws-sdk/lib-storage": "*",
-    "@aws-sdk/s3-request-presigner": "*",
-    "@remit/domain-enums": "*"
-  },
-  "devDependencies": {
-    "aws-sdk-client-mock": "*"
-  },
-  "license": "MIT",
-  "publishConfig": {
-    "access": "public"
-  },
-  "repository": {
-    "type": "git",
-    "url": "git+https://github.com/remit-mail/reader.git",
-    "directory": "packages/storage-service"
-  }
+	"name": "@remit/storage-service",
+	"version": "0.0.13",
+	"type": "module",
+	"main": "dist/index.js",
+	"types": "dist/index.d.ts",
+	"exports": {
+		".": {
+			"types": "./dist/index.d.ts",
+			"default": "./dist/index.js"
+		},
+		"./filesystem": {
+			"types": "./dist/filesystem.d.ts",
+			"default": "./dist/filesystem.js"
+		},
+		"./s3": {
+			"types": "./dist/s3.d.ts",
+			"default": "./dist/s3.js"
+		},
+		"./signed-path": {
+			"types": "./dist/signed-path.d.ts",
+			"default": "./dist/signed-path.js"
+		}
+	},
+	"scripts": {
+		"prepack": "node ../../npm-scripts/pack-dist.mjs prepack",
+		"postpack": "node ../../npm-scripts/pack-dist.mjs postpack",
+		"test:typecheck": "tsgo --noEmit",
+		"test:run": "node $NODE_TEST_FLAGS --env-file=../../localhost-test-unit.env --import tsx --experimental-test-coverage --test-coverage-include='src/**' --test-coverage-exclude='src/**/*.test.ts' --test-coverage-exclude='src/**/*.test.tsx' --test-coverage-lines=87 --test 'src/**/*.test.ts'",
+		"test": "npm run test:typecheck && npm run test:run"
+	},
+	"dependencies": {
+		"@aws-sdk/client-s3": "^3.1090.0",
+		"@aws-sdk/lib-storage": "^3.1090.0",
+		"@aws-sdk/s3-request-presigner": "^3.1090.0",
+		"@remit/domain-enums": "*"
+	},
+	"devDependencies": {
+		"aws-sdk-client-mock": "*"
+	},
+	"license": "MIT",
+	"publishConfig": {
+		"access": "public"
+	},
+	"repository": {
+		"type": "git",
+		"url": "git+https://github.com/remit-mail/reader.git",
+		"directory": "packages/storage-service"
+	},
+	"files": [
+		"dist"
+	]
 }

package/src/backends/filesystem.ts

@@ -1,517 +0,0 @@
-import { createHash } from "node:crypto";
-import { createWriteStream } from "node:fs";
-import {
-	mkdir,
-	readdir,
-	readFile,
-	rm,
-	stat,
-	unlink,
-	writeFile,
-} from "node:fs/promises";
-import { dirname, join, resolve } from "node:path";
-import { PassThrough, Readable } from "node:stream";
-import { pipeline } from "node:stream/promises";
-import { createGunzip, createGzip, gunzipSync, gzipSync } from "node:zlib";
-import { ContentEncoding, StorageType } from "@remit/domain-enums";
-import {
-	UPLOAD_ROUTE_PREFIX,
-	UPLOAD_URL_SIGNING_LABEL,
-} from "../outbox-upload-url.js";
-import { signStoragePath } from "../signed-path.js";
-import type {
-	OutboxAttachmentListItem,
-	ParsedBody,
-	StorageReference,
-	StorageService,
-} from "../storage.js";
-import {
-	buildBodyPartKey,
-	buildDeduplicatedKey,
-	buildExportArchiveKey,
-	buildMessageBodyKey,
-	buildOutboxAttachmentKey,
-	buildOutboxAttachmentPrefix,
-	buildParsedBodyKey,
-	computeChecksum,
-	isStorageNotFoundError,
-} from "../storage.js";
-
-interface StoreParams {
-	key: string;
-	content: Buffer;
-	compress?: boolean;
-}
-
-/**
- * Where a minted upload URL points, and what signs it. Absent on a process that
- * only reads storage; a mint then fails loud rather than handing out a URL
- * nothing can verify.
- */
-export interface FilesystemUploadUrlConfig {
-	/** Public origin of the deployment serving the upload route. */
-	origin: string;
-	signingSecret: string;
-}
-
-export const createFilesystemStorageService = (
-	basePath: string,
-	uploadUrls?: FilesystemUploadUrlConfig,
-): StorageService => {
-	const storeInternal = async (
-		params: StoreParams,
-	): Promise<StorageReference> => {
-		const { key, content, compress = true } = params;
-		const checksumSha256 = computeChecksum(content);
-		const contentEncoding = compress
-			? ContentEncoding.Gzip
-			: ContentEncoding.None;
-		const body = compress ? gzipSync(content) : content;
-
-		const fullPath = join(basePath, key);
-		await mkdir(dirname(fullPath), { recursive: true });
-		await writeFile(fullPath, body);
-
-		return {
-			uri: `file://${fullPath}`,
-			storageType: StorageType.Filesystem,
-			storageLocation: basePath,
-			storageKey: key,
-			sizeBytes: body.length,
-			checksumSha256,
-			contentEncoding,
-		};
-	};
-
-	const storeMessageBody: StorageService["storeMessageBody"] = (params) => {
-		const { accountConfigId, accountId, messageId, content } = params;
-		return storeInternal({
-			key: buildMessageBodyKey(accountConfigId, accountId, messageId),
-			content,
-		});
-	};
-
-	const storeMessageBodyStream: StorageService["storeMessageBodyStream"] =
-		async (params) => {
-			const { accountConfigId, accountId, messageId, content } = params;
-			const key = buildMessageBodyKey(accountConfigId, accountId, messageId);
-			const fullPath = join(basePath, key);
-			await mkdir(dirname(fullPath), { recursive: true });
-
-			const hash = createHash("sha256");
-			const hashTap = new PassThrough();
-			hashTap.on("data", (chunk: Buffer) => hash.update(chunk));
-
-			await pipeline(
-				content,
-				hashTap,
-				createGzip(),
-				createWriteStream(fullPath),
-			);
-
-			const { size } = await stat(fullPath);
-
-			return {
-				uri: `file://${fullPath}`,
-				storageType: StorageType.Filesystem,
-				storageLocation: basePath,
-				storageKey: key,
-				sizeBytes: size,
-				checksumSha256: hash.digest("hex"),
-				contentEncoding: ContentEncoding.Gzip,
-			};
-		};
-
-	const storeBodyPart: StorageService["storeBodyPart"] = (params) => {
-		const { accountConfigId, accountId, messageId, partPath, content } = params;
-		return storeInternal({
-			key: buildBodyPartKey(accountConfigId, accountId, messageId, partPath),
-			content,
-		});
-	};
-
-	const bodyPartExists: StorageService["bodyPartExists"] = (
-		accountConfigId,
-		accountId,
-		messageId,
-		partPath,
-	) => {
-		const key = buildBodyPartKey(
-			accountConfigId,
-			accountId,
-			messageId,
-			partPath,
-		);
-		const fullPath = join(basePath, key);
-		return stat(fullPath)
-			.then(() => true)
-			.catch(() => false);
-	};
-
-	const retrieveBodyPart: StorageService["retrieveBodyPart"] = async (
-		accountConfigId,
-		accountId,
-		messageId,
-		partPath,
-	) => {
-		const key = buildBodyPartKey(
-			accountConfigId,
-			accountId,
-			messageId,
-			partPath,
-		);
-		const fullPath = join(basePath, key);
-		const buffer = await readFile(fullPath).catch((error: unknown) => {
-			if (isStorageNotFoundError(error)) return null;
-			throw error;
-		});
-		if (!buffer) return null;
-		return buffer[0] === 0x1f && buffer[1] === 0x8b
-			? gunzipSync(buffer)
-			: buffer;
-	};
-
-	const storeOutboxAttachment: StorageService["storeOutboxAttachment"] = (
-		params,
-	) => {
-		const {
-			accountConfigId,
-			accountId,
-			outboxMessageId,
-			outboxAttachmentId,
-			content,
-		} = params;
-		return storeInternal({
-			key: buildOutboxAttachmentKey(
-				accountConfigId,
-				accountId,
-				outboxMessageId,
-				outboxAttachmentId,
-			),
-			content,
-			compress: false,
-		});
-	};
-
-	const listOutboxAttachments: StorageService["listOutboxAttachments"] = async (
-		accountConfigId,
-		accountId,
-		outboxMessageId,
-	) => {
-		const prefix = buildOutboxAttachmentPrefix(
-			accountConfigId,
-			accountId,
-			outboxMessageId,
-		);
-		const entries = await readdir(join(basePath, prefix)).catch(
-			(error: unknown) => {
-				if (isStorageNotFoundError(error)) return [];
-				throw error;
-			},
-		);
-
-		const items: OutboxAttachmentListItem[] = [];
-		for (const name of entries) {
-			const { size } = await stat(join(basePath, prefix, name));
-			items.push({
-				outboxAttachmentId: name,
-				key: `${prefix}${name}`,
-				sizeBytes: size,
-			});
-		}
-		return items;
-	};
-
-	const listOutboxDraftsWithAttachments: StorageService["listOutboxDraftsWithAttachments"] =
-		async (accountConfigId, accountId) => {
-			// An account that has never held an attachment has no directory, and
-			// that is genuinely nothing to do. A storage root that is not there at
-			// all is a different thing entirely — a process that cannot see the
-			// volume it is meant to be collecting from — and it must not read as an
-			// empty answer. Unmounted, the sweep would report every account swept
-			// and collect nothing, forever.
-			const rootExists = await stat(basePath)
-				.then(() => true)
-				.catch((error: unknown) => {
-					if (isStorageNotFoundError(error)) return false;
-					throw error;
-				});
-			if (!rootExists) {
-				throw new Error(
-					`storage root ${basePath} does not exist; this process cannot see the storage it was asked to read`,
-				);
-			}
-
-			const root = join(
-				basePath,
-				`accounts/${accountConfigId}/${accountId}/outbox`,
-			);
-			const entries = await readdir(root, { withFileTypes: true }).catch(
-				(error: unknown) => {
-					if (isStorageNotFoundError(error)) return [];
-					throw error;
-				},
-			);
-			return entries
-				.filter((entry) => entry.isDirectory())
-				.map((entry) => entry.name);
-		};
-
-	const deleteOutboxAttachment: StorageService["deleteOutboxAttachment"] =
-		async (accountConfigId, accountId, outboxMessageId, outboxAttachmentId) => {
-			const prefix = buildOutboxAttachmentPrefix(
-				accountConfigId,
-				accountId,
-				outboxMessageId,
-			);
-			const entries = await readdir(join(basePath, prefix)).catch(
-				(error: unknown) => {
-					if (isStorageNotFoundError(error)) return [];
-					throw error;
-				},
-			);
-			for (const name of entries) {
-				if (name !== outboxAttachmentId) continue;
-				await rm(join(basePath, prefix, name), { force: true });
-			}
-		};
-
-	const statOutboxAttachment: StorageService["statOutboxAttachment"] = async (
-		accountConfigId,
-		accountId,
-		outboxMessageId,
-		outboxAttachmentId,
-	) => {
-		const fullPath = join(
-			basePath,
-			buildOutboxAttachmentKey(
-				accountConfigId,
-				accountId,
-				outboxMessageId,
-				outboxAttachmentId,
-			),
-		);
-		const stats = await stat(fullPath).catch((error: unknown) => {
-			if (isStorageNotFoundError(error)) return null;
-			throw error;
-		});
-		return stats ? { sizeBytes: stats.size } : null;
-	};
-
-	const retrieveOutboxAttachment: StorageService["retrieveOutboxAttachment"] = (
-		accountConfigId,
-		accountId,
-		outboxMessageId,
-		outboxAttachmentId,
-	) =>
-		readFile(
-			join(
-				basePath,
-				buildOutboxAttachmentKey(
-					accountConfigId,
-					accountId,
-					outboxMessageId,
-					outboxAttachmentId,
-				),
-			),
-		).catch((error: unknown) => {
-			if (isStorageNotFoundError(error)) return null;
-			throw error;
-		});
-
-	// This backend has no presigned anything, so the URL addresses the
-	// deployment's own upload route. Its authority is the same HMAC the read side
-	// uses on /content, under the write label, covering the storage key, the
-	// expiry and the exact byte count — a URL for one attachment, for a while,
-	// for one size.
-	const createOutboxAttachmentUploadUrl: StorageService["createOutboxAttachmentUploadUrl"] =
-		async (params) => {
-			if (!uploadUrls) {
-				throw new Error(
-					"the filesystem storage backend cannot mint an upload URL without an origin and a signing secret",
-				);
-			}
-			const key = buildOutboxAttachmentKey(
-				params.accountConfigId,
-				params.accountId,
-				params.outboxMessageId,
-				params.outboxAttachmentId,
-			);
-			const sig = signStoragePath(
-				uploadUrls.signingSecret,
-				UPLOAD_URL_SIGNING_LABEL,
-				key,
-				[params.expiresAt, params.sizeBytes],
-			);
-			const origin = uploadUrls.origin.replace(/\/+$/, "");
-			const query = new URLSearchParams({
-				exp: String(params.expiresAt),
-				max: String(params.sizeBytes),
-				sig,
-			});
-			return {
-				uploadUrl: `${origin}${UPLOAD_ROUTE_PREFIX}${key}?${query.toString()}`,
-			};
-		};
-
-	const deleteOutboxAttachments: StorageService["deleteOutboxAttachments"] =
-		async (accountConfigId, accountId, outboxMessageId) => {
-			const dirPath = join(
-				basePath,
-				buildOutboxAttachmentPrefix(
-					accountConfigId,
-					accountId,
-					outboxMessageId,
-				),
-			);
-			await rm(dirPath, { recursive: true, force: true });
-		};
-
-	const storeDeduplicated: StorageService["storeDeduplicated"] = (params) => {
-		const { accountConfigId, accountId, content } = params;
-		const checksumSha256 = computeChecksum(content);
-		return storeInternal({
-			key: buildDeduplicatedKey(accountConfigId, accountId, checksumSha256),
-			content,
-		});
-	};
-
-	const storeParsedBody: StorageService["storeParsedBody"] = (params) => {
-		const { accountConfigId, accountId, messageId, parsed } = params;
-		const content = Buffer.from(JSON.stringify(parsed), "utf8");
-		return storeInternal({
-			key: buildParsedBodyKey(accountConfigId, accountId, messageId),
-			content,
-		});
-	};
-
-	const retrieveParsedBody: StorageService["retrieveParsedBody"] = async (
-		accountConfigId,
-		accountId,
-		messageId,
-	) => {
-		const key = buildParsedBodyKey(accountConfigId, accountId, messageId);
-		const fullPath = join(basePath, key);
-
-		const buffer = await readFile(fullPath).catch((error: unknown) => {
-			if (isStorageNotFoundError(error)) return null;
-			throw error;
-		});
-
-		if (!buffer) return null;
-
-		const decoded =
-			buffer[0] === 0x1f && buffer[1] === 0x8b ? gunzipSync(buffer) : buffer;
-		return JSON.parse(decoded.toString("utf8")) as ParsedBody;
-	};
-
-	// Resolve path from URI, handling both absolute and relative paths
-	// Note: URIs like file://.remit/storage/... parse incorrectly - '.remit' becomes hostname
-	// We reconstruct the original path by combining hostname + pathname
-	const resolvePathFromUri = (uri: string): string => {
-		const url = new URL(uri);
-		const hostname = url.hostname;
-		const pathname = url.pathname;
-
-		// If hostname exists, it was part of a relative path (e.g., file://.remit/... -> hostname='.remit')
-		if (hostname) {
-			const relativePath = hostname + pathname;
-			return resolve(relativePath);
-		}
-
-		// Absolute path (file:///absolute/path)
-		return pathname;
-	};
-
-	const retrieve = async (uri: string): Promise<Buffer> => {
-		const fullPath = resolvePathFromUri(uri);
-
-		const buffer = await readFile(fullPath);
-
-		if (buffer[0] === 0x1f && buffer[1] === 0x8b) {
-			return gunzipSync(buffer);
-		}
-
-		return buffer;
-	};
-
-	const exists = async (uri: string): Promise<boolean> => {
-		const fullPath = resolvePathFromUri(uri);
-		return stat(fullPath)
-			.then(() => true)
-			.catch(() => false);
-	};
-
-	const del = async (uri: string): Promise<void> => {
-		const fullPath = resolvePathFromUri(uri);
-		await unlink(fullPath);
-	};
-
-	return {
-		storeMessageBody,
-		storeMessageBodyStream,
-		storeBodyPart,
-		bodyPartExists,
-		retrieveBodyPart,
-		storeOutboxAttachment,
-		listOutboxAttachments,
-		listOutboxDraftsWithAttachments,
-		deleteOutboxAttachments,
-		deleteOutboxAttachment,
-		statOutboxAttachment,
-		retrieveOutboxAttachment,
-		createOutboxAttachmentUploadUrl,
-		storeDeduplicated,
-		storeParsedBody,
-		retrieveParsedBody,
-		retrieveMessageBody: async (accountConfigId, accountId, messageId) => {
-			const key = buildMessageBodyKey(accountConfigId, accountId, messageId);
-			const fullPath = join(basePath, key);
-			const buffer = await readFile(fullPath).catch((error: unknown) => {
-				if (isStorageNotFoundError(error)) return null;
-				throw error;
-			});
-			if (!buffer) return null;
-			if (buffer[0] === 0x1f && buffer[1] === 0x8b) {
-				return gunzipSync(buffer);
-			}
-			return buffer;
-		},
-		retrieveMessageBodyStream: async (
-			accountConfigId,
-			accountId,
-			messageId,
-		) => {
-			const key = buildMessageBodyKey(accountConfigId, accountId, messageId);
-			const fullPath = join(basePath, key);
-			const buffer = await readFile(fullPath).catch((error: unknown) => {
-				if (isStorageNotFoundError(error)) return null;
-				throw error;
-			});
-			if (!buffer) return null;
-			if (buffer[0] === 0x1f && buffer[1] === 0x8b) {
-				return Readable.from(buffer).pipe(createGunzip());
-			}
-			return Readable.from(buffer);
-		},
-		retrieve,
-		exists,
-		delete: del,
-		storeExportArchiveStream: async (
-			accountConfigId,
-			exportRequestId,
-			body,
-		) => {
-			const key = buildExportArchiveKey(accountConfigId, exportRequestId);
-			const fullPath = join(basePath, key);
-			await mkdir(dirname(fullPath), { recursive: true });
-			await pipeline(body, createWriteStream(fullPath));
-			return key;
-		},
-		getPresignedDownloadUrl: (_key: string, _expiresInSeconds: number) => {
-			throw new Error(
-				"getPresignedDownloadUrl is not supported on the filesystem backend",
-			);
-		},
-	};
-};