@remit/storage-service 0.0.1

package/README.md

@@ -0,0 +1,267 @@
+# @remit/storage-service
+
+Abstract storage service for email body parts and raw message content. Provides a unified interface for storing and retrieving binary content with configurable backends (S3 or local filesystem).
+
+## Installation
+
+```bash
+npm install -w packages/storage-service
+```
+
+## Quick Start
+
+```typescript
+import { createStorageService } from "@remit/storage-service";
+import { ContentEncoding } from "@remit/domain-enums";
+
+const storage = createStorageService();
+
+// Store content
+const ref = await storage.store(Buffer.from("Hello, world!"), {
+  key: "accounts/abc123/messages/def456/body",
+  contentEncoding: ContentEncoding.Gzip,
+  contentType: "text/plain; charset=utf-8",
+});
+
+// Retrieve content
+const content = await storage.retrieve(ref.uri);
+
+// Check existence
+if (await storage.exists(ref.uri)) {
+  await storage.delete(ref.uri);
+}
+```
+
+## Environment Variables
+
+| Variable             | Required | Description                                  |
+| -------------------- | -------- | -------------------------------------------- |
+| `S3_BUCKET_NAME`     | No       | S3 bucket name. If set, S3 backend is used   |
+| `S3_ENDPOINT`        | No       | Custom S3 endpoint (for LocalStack/MinIO)    |
+| `STORAGE_LOCAL_PATH` | No       | Local filesystem path. Default: `.remit/storage` |
+
+### Backend Selection
+
+The factory function automatically selects the backend:
+
+```
+if (S3_BUCKET_NAME is defined) → S3 backend
+else → Filesystem backend
+```
+
+## API
+
+### StorageService
+
+```typescript
+interface StorageService {
+  store(content: Buffer, options: StoreOptions): Promise<StorageReference>;
+  retrieve(uri: string): Promise<Buffer>;
+  exists(uri: string): Promise<boolean>;
+  delete(uri: string): Promise<void>;
+  contentAddressableKey(content: Buffer, prefix?: string): string;
+}
+```
+
+### StoreOptions
+
+```typescript
+interface StoreOptions {
+  key: string; // Storage path
+  contentEncoding?: ContentEncoding; // Compression (default: none)
+  contentType?: string; // MIME type for S3 metadata
+  contentAddressable?: boolean; // Use content hash as key
+}
+```
+
+### StorageReference
+
+Returned by `store()`, contains all information needed to retrieve or reference the stored content:
+
+```typescript
+interface StorageReference {
+  uri: string; // Full URI: s3://bucket/key or file:///path
+  storageType: StorageTypeValue; // "s3" | "filesystem"
+  storageLocation: string; // Bucket name or base path
+  storageKey: string; // Object key or relative path
+  sizeBytes: number; // Size after compression
+  checksumSha256: string; // Hex-encoded SHA-256 of original content
+  contentEncoding: ContentEncoding;
+}
+```
+
+## URI Format
+
+Storage references use URIs that encode all information needed to retrieve content:
+
+```
+s3://bucket-name/path/to/object
+file:///var/data/remit/path/to/file
+```
+
+### URI Utilities
+
+```typescript
+import { parseStorageUri, buildStorageUri } from "@remit/storage-service";
+import { StorageType } from "@remit/domain-enums";
+
+// Parse URI into components
+const parsed = parseStorageUri("s3://my-bucket/path/to/object");
+// { storageType: "s3", storageLocation: "my-bucket", storageKey: "path/to/object" }
+
+// Build URI from components
+const uri = buildStorageUri(StorageType.S3, "my-bucket", "path/to/object");
+// "s3://my-bucket/path/to/object"
+```
+
+## Content-Addressable Storage
+
+For deduplication of attachments that may appear in multiple messages:
+
+```typescript
+const storage = createStorageService();
+const attachmentBuffer = Buffer.from(/* ... */);
+
+// Generate content-addressable key
+const key = storage.contentAddressableKey(attachmentBuffer);
+// "dedup/a3/a3f2b8c9d0e1f2..."
+
+// Check if already stored
+const dedupUri = `s3://${bucketName}/${key}`;
+if (await storage.exists(dedupUri)) {
+  // Content already stored, just reference it
+  return { uri: dedupUri, isDeduped: true };
+}
+
+// Store with content-addressable key
+const ref = await storage.store(attachmentBuffer, {
+  key: "ignored-when-content-addressable",
+  contentAddressable: true,
+});
+```
+
+## Compression
+
+Gzip compression is automatically handled:
+
+```typescript
+// Store with compression
+const ref = await storage.store(content, {
+  key: "messages/123/body",
+  contentEncoding: ContentEncoding.Gzip,
+});
+
+// Retrieve automatically decompresses
+const original = await storage.retrieve(ref.uri);
+```
+
+## Using Specific Backends
+
+For cases where you need direct access to a specific backend:
+
+```typescript
+import {
+  createS3StorageService,
+  createFilesystemStorageService,
+} from "@remit/storage-service";
+import { S3Client } from "@aws-sdk/client-s3";
+
+// S3 backend
+const s3Client = new S3Client({ endpoint: "http://localhost:4566" });
+const s3Storage = createS3StorageService(s3Client, "my-bucket");
+
+// Filesystem backend
+const fsStorage = createFilesystemStorageService("/var/data/remit");
+```
+
+## Testing
+
+### Mock Service
+
+For unit tests, use the mock implementation:
+
+```typescript
+import { createMockStorageService } from "@remit/storage-service";
+
+const storage = createMockStorageService();
+
+// Works like the real service but stores in memory
+const ref = await storage.store(Buffer.from("test"), { key: "test.txt" });
+const content = await storage.retrieve(ref.uri);
+```
+
+### Running Tests
+
+```bash
+# Run package tests
+npm test -w packages/storage-service
+
+# With LocalStack for S3 tests
+S3_BUCKET_NAME=test-bucket S3_ENDPOINT=http://localhost:4566 npm test -w packages/storage-service
+```
+
+## Key Path Conventions
+
+Consistent key paths enable efficient listing and cleanup:
+
+```
+accounts/{accountId}/
+  messages/{messageId}/
+    raw                          # Raw RFC822 message
+    parts/
+      {partPath}                 # Body part content (1, 1.1, 2.1.3, etc.)
+
+dedup/
+  {hash[0:2]}/
+    {hash}                       # Content-addressable storage for attachments
+```
+
+## Integration with ElectroDB Entities
+
+```typescript
+import { BodyPartStorageService } from "@remit/remit-electrodb-service";
+import { createStorageService } from "@remit/storage-service";
+
+const storage = createStorageService();
+const bodyPartStorageService = new BodyPartStorageService(config);
+
+// Store content and create entity reference
+const ref = await storage.store(content, {
+  key: `messages/${messageId}/parts/${partPath}`,
+  contentEncoding: ContentEncoding.Gzip,
+});
+
+await bodyPartStorageService.create({
+  messageId,
+  bodyPartId,
+  storageType: ref.storageType,
+  storageLocation: ref.storageLocation,
+  storageKey: ref.storageKey,
+  decodedSizeBytes: content.length,
+  checksumSha256: ref.checksumSha256,
+  contentEncoding: ref.contentEncoding,
+  isDeduped: false,
+  storedAt: Date.now(),
+});
+```
+
+## Exports
+
+```typescript
+// Factory functions
+export { createStorageService, createMockStorageService } from "./storage.js";
+export { createFilesystemStorageService } from "./backends/filesystem.js";
+export { createS3StorageService } from "./backends/s3.js";
+
+// URI utilities
+export { parseStorageUri, buildStorageUri } from "./uri.js";
+
+// Types
+export type {
+  StorageService,
+  StorageReference,
+  StoreOptions,
+} from "./storage.js";
+export type { StorageTypeValue, ContentEncodingValue } from "./storage.js";
+export type { ParsedStorageUri } from "./uri.js";
+```

package/package.json

@@ -0,0 +1,44 @@
+{
+	"name": "@remit/storage-service",
+	"version": "0.0.1",
+	"type": "module",
+	"main": "src/index.ts",
+	"types": "src/index.ts",
+	"exports": {
+		".": {
+			"types": "./src/index.ts",
+			"default": "./src/index.ts"
+		},
+		"./filesystem": {
+			"types": "./src/filesystem.ts",
+			"default": "./src/filesystem.ts"
+		},
+		"./s3": {
+			"types": "./src/s3.ts",
+			"default": "./src/s3.ts"
+		}
+	},
+	"scripts": {
+		"test:typecheck": "tsgo --noEmit",
+		"test:run": "node --env-file=../../localhost-test-unit.env --import tsx --test 'src/**/*.test.ts'",
+		"test": "npm run test:typecheck && npm run test:run"
+	},
+	"dependencies": {
+		"@aws-sdk/client-s3": "*",
+		"@aws-sdk/lib-storage": "*",
+		"@aws-sdk/s3-request-presigner": "*"
+	},
+	"devDependencies": {
+		"@remit/domain-enums": "*",
+		"aws-sdk-client-mock": "*"
+	},
+	"license": "MIT",
+	"publishConfig": {
+		"access": "public"
+	},
+	"repository": {
+		"type": "git",
+		"url": "git+https://github.com/remit-mail/remit.git",
+		"directory": "packages/storage-service"
+	}
+}

package/src/backends/filesystem.ts

@@ -0,0 +1,399 @@
+import { createHash } from "node:crypto";
+import { createWriteStream } from "node:fs";
+import {
+	mkdir,
+	readdir,
+	readFile,
+	stat,
+	unlink,
+	writeFile,
+} from "node:fs/promises";
+import { dirname, join, resolve } from "node:path";
+import { PassThrough, Readable } from "node:stream";
+import { pipeline } from "node:stream/promises";
+import { createGunzip, createGzip, gunzipSync, gzipSync } from "node:zlib";
+import { ContentEncoding, StorageType } from "@remit/domain-enums";
+import type {
+	ParsedBody,
+	StorageReference,
+	StorageService,
+} from "../storage.js";
+import {
+	buildBodyPartKey,
+	buildDeduplicatedKey,
+	buildExportArchiveKey,
+	buildExtractedPrefix,
+	buildExtractedSkippedKey,
+	buildExtractedTextKey,
+	buildMessageBodyKey,
+	buildParsedBodyKey,
+	computeChecksum,
+	isStorageNotFoundError,
+} from "../storage.js";
+
+interface StoreParams {
+	key: string;
+	content: Buffer;
+	compress?: boolean;
+}
+
+export const createFilesystemStorageService = (
+	basePath: string,
+): StorageService => {
+	const storeInternal = async (
+		params: StoreParams,
+	): Promise<StorageReference> => {
+		const { key, content, compress = true } = params;
+		const checksumSha256 = computeChecksum(content);
+		const contentEncoding = compress
+			? ContentEncoding.Gzip
+			: ContentEncoding.None;
+		const body = compress ? gzipSync(content) : content;
+
+		const fullPath = join(basePath, key);
+		await mkdir(dirname(fullPath), { recursive: true });
+		await writeFile(fullPath, body);
+
+		return {
+			uri: `file://${fullPath}`,
+			storageType: StorageType.Filesystem,
+			storageLocation: basePath,
+			storageKey: key,
+			sizeBytes: body.length,
+			checksumSha256,
+			contentEncoding,
+		};
+	};
+
+	const storeMessageBody: StorageService["storeMessageBody"] = (params) => {
+		const { accountConfigId, accountId, messageId, content } = params;
+		return storeInternal({
+			key: buildMessageBodyKey(accountConfigId, accountId, messageId),
+			content,
+		});
+	};
+
+	const storeMessageBodyStream: StorageService["storeMessageBodyStream"] =
+		async (params) => {
+			const { accountConfigId, accountId, messageId, content } = params;
+			const key = buildMessageBodyKey(accountConfigId, accountId, messageId);
+			const fullPath = join(basePath, key);
+			await mkdir(dirname(fullPath), { recursive: true });
+
+			const hash = createHash("sha256");
+			const hashTap = new PassThrough();
+			hashTap.on("data", (chunk: Buffer) => hash.update(chunk));
+
+			await pipeline(
+				content,
+				hashTap,
+				createGzip(),
+				createWriteStream(fullPath),
+			);
+
+			const { size } = await stat(fullPath);
+
+			return {
+				uri: `file://${fullPath}`,
+				storageType: StorageType.Filesystem,
+				storageLocation: basePath,
+				storageKey: key,
+				sizeBytes: size,
+				checksumSha256: hash.digest("hex"),
+				contentEncoding: ContentEncoding.Gzip,
+			};
+		};
+
+	const storeBodyPart: StorageService["storeBodyPart"] = (params) => {
+		const { accountConfigId, accountId, messageId, partPath, content } = params;
+		return storeInternal({
+			key: buildBodyPartKey(accountConfigId, accountId, messageId, partPath),
+			content,
+		});
+	};
+
+	const bodyPartExists: StorageService["bodyPartExists"] = (
+		accountConfigId,
+		accountId,
+		messageId,
+		partPath,
+	) => {
+		const key = buildBodyPartKey(
+			accountConfigId,
+			accountId,
+			messageId,
+			partPath,
+		);
+		const fullPath = join(basePath, key);
+		return stat(fullPath)
+			.then(() => true)
+			.catch(() => false);
+	};
+
+	const retrieveBodyPart: StorageService["retrieveBodyPart"] = async (
+		accountConfigId,
+		accountId,
+		messageId,
+		partPath,
+	) => {
+		const key = buildBodyPartKey(
+			accountConfigId,
+			accountId,
+			messageId,
+			partPath,
+		);
+		const fullPath = join(basePath, key);
+		const buffer = await readFile(fullPath).catch((error: unknown) => {
+			if (isStorageNotFoundError(error)) return null;
+			throw error;
+		});
+		if (!buffer) return null;
+		return buffer[0] === 0x1f && buffer[1] === 0x8b
+			? gunzipSync(buffer)
+			: buffer;
+	};
+
+	const storeExtractedText: StorageService["storeExtractedText"] = (params) => {
+		const { accountConfigId, accountId, messageId, partPath, text } = params;
+		return storeInternal({
+			key: buildExtractedTextKey(
+				accountConfigId,
+				accountId,
+				messageId,
+				partPath,
+			),
+			content: Buffer.from(text, "utf8"),
+		});
+	};
+
+	const storeExtractedSkipped: StorageService["storeExtractedSkipped"] = (
+		params,
+	) => {
+		const { accountConfigId, accountId, messageId, partPath, marker } = params;
+		return storeInternal({
+			key: buildExtractedSkippedKey(
+				accountConfigId,
+				accountId,
+				messageId,
+				partPath,
+			),
+			content: Buffer.from(JSON.stringify(marker), "utf8"),
+			compress: false,
+		});
+	};
+
+	const extractedResultExists: StorageService["extractedResultExists"] = async (
+		accountConfigId,
+		accountId,
+		messageId,
+		partPath,
+	) => {
+		const textPath = join(
+			basePath,
+			buildExtractedTextKey(accountConfigId, accountId, messageId, partPath),
+		);
+		const skippedPath = join(
+			basePath,
+			buildExtractedSkippedKey(accountConfigId, accountId, messageId, partPath),
+		);
+		const exists = (fullPath: string): Promise<boolean> =>
+			stat(fullPath)
+				.then(() => true)
+				.catch(() => false);
+
+		const [textExists, skippedExists] = await Promise.all([
+			exists(textPath),
+			exists(skippedPath),
+		]);
+		return textExists || skippedExists;
+	};
+
+	const retrieveExtractedText: StorageService["retrieveExtractedText"] = async (
+		accountConfigId,
+		accountId,
+		messageId,
+		partPath,
+	) => {
+		const fullPath = join(
+			basePath,
+			buildExtractedTextKey(accountConfigId, accountId, messageId, partPath),
+		);
+		const buffer = await readFile(fullPath).catch((error: unknown) => {
+			if (isStorageNotFoundError(error)) return null;
+			throw error;
+		});
+		if (!buffer) return null;
+		const decoded =
+			buffer[0] === 0x1f && buffer[1] === 0x8b ? gunzipSync(buffer) : buffer;
+		return decoded.toString("utf8");
+	};
+
+	const listExtractedTexts: StorageService["listExtractedTexts"] = async (
+		accountConfigId,
+		accountId,
+		messageId,
+	) => {
+		const prefix = buildExtractedPrefix(accountConfigId, accountId, messageId);
+		const suffix = ".txt.gz";
+		const dirPath = join(basePath, prefix);
+
+		const entries = await readdir(dirPath, { recursive: true }).catch(
+			(error: unknown) => {
+				if (isStorageNotFoundError(error)) return [];
+				throw error;
+			},
+		);
+
+		return entries
+			.filter((entry) => entry.endsWith(suffix))
+			.map((entry) => ({
+				partPath: entry.slice(0, -suffix.length),
+				key: `${prefix}${entry}`,
+			}));
+	};
+
+	const storeDeduplicated: StorageService["storeDeduplicated"] = (params) => {
+		const { accountConfigId, accountId, content } = params;
+		const checksumSha256 = computeChecksum(content);
+		return storeInternal({
+			key: buildDeduplicatedKey(accountConfigId, accountId, checksumSha256),
+			content,
+		});
+	};
+
+	const storeParsedBody: StorageService["storeParsedBody"] = (params) => {
+		const { accountConfigId, accountId, messageId, parsed } = params;
+		const content = Buffer.from(JSON.stringify(parsed), "utf8");
+		return storeInternal({
+			key: buildParsedBodyKey(accountConfigId, accountId, messageId),
+			content,
+		});
+	};
+
+	const retrieveParsedBody: StorageService["retrieveParsedBody"] = async (
+		accountConfigId,
+		accountId,
+		messageId,
+	) => {
+		const key = buildParsedBodyKey(accountConfigId, accountId, messageId);
+		const fullPath = join(basePath, key);
+
+		const buffer = await readFile(fullPath).catch((error: unknown) => {
+			if (isStorageNotFoundError(error)) return null;
+			throw error;
+		});
+
+		if (!buffer) return null;
+
+		const decoded =
+			buffer[0] === 0x1f && buffer[1] === 0x8b ? gunzipSync(buffer) : buffer;
+		return JSON.parse(decoded.toString("utf8")) as ParsedBody;
+	};
+
+	// Resolve path from URI, handling both absolute and relative paths
+	// Note: URIs like file://.remit/storage/... parse incorrectly - '.remit' becomes hostname
+	// We reconstruct the original path by combining hostname + pathname
+	const resolvePathFromUri = (uri: string): string => {
+		const url = new URL(uri);
+		const hostname = url.hostname;
+		const pathname = url.pathname;
+
+		// If hostname exists, it was part of a relative path (e.g., file://.remit/... -> hostname='.remit')
+		if (hostname) {
+			const relativePath = hostname + pathname;
+			return resolve(relativePath);
+		}
+
+		// Absolute path (file:///absolute/path)
+		return pathname;
+	};
+
+	const retrieve = async (uri: string): Promise<Buffer> => {
+		const fullPath = resolvePathFromUri(uri);
+
+		const buffer = await readFile(fullPath);
+
+		if (buffer[0] === 0x1f && buffer[1] === 0x8b) {
+			return gunzipSync(buffer);
+		}
+
+		return buffer;
+	};
+
+	const exists = async (uri: string): Promise<boolean> => {
+		const fullPath = resolvePathFromUri(uri);
+		return stat(fullPath)
+			.then(() => true)
+			.catch(() => false);
+	};
+
+	const del = async (uri: string): Promise<void> => {
+		const fullPath = resolvePathFromUri(uri);
+		await unlink(fullPath);
+	};
+
+	return {
+		storeMessageBody,
+		storeMessageBodyStream,
+		storeBodyPart,
+		bodyPartExists,
+		retrieveBodyPart,
+		storeDeduplicated,
+		storeParsedBody,
+		retrieveParsedBody,
+		storeExtractedText,
+		storeExtractedSkipped,
+		extractedResultExists,
+		retrieveExtractedText,
+		listExtractedTexts,
+		retrieveMessageBody: async (accountConfigId, accountId, messageId) => {
+			const key = buildMessageBodyKey(accountConfigId, accountId, messageId);
+			const fullPath = join(basePath, key);
+			const buffer = await readFile(fullPath).catch((error: unknown) => {
+				if (isStorageNotFoundError(error)) return null;
+				throw error;
+			});
+			if (!buffer) return null;
+			if (buffer[0] === 0x1f && buffer[1] === 0x8b) {
+				return gunzipSync(buffer);
+			}
+			return buffer;
+		},
+		retrieveMessageBodyStream: async (
+			accountConfigId,
+			accountId,
+			messageId,
+		) => {
+			const key = buildMessageBodyKey(accountConfigId, accountId, messageId);
+			const fullPath = join(basePath, key);
+			const buffer = await readFile(fullPath).catch((error: unknown) => {
+				if (isStorageNotFoundError(error)) return null;
+				throw error;
+			});
+			if (!buffer) return null;
+			if (buffer[0] === 0x1f && buffer[1] === 0x8b) {
+				return Readable.from(buffer).pipe(createGunzip());
+			}
+			return Readable.from(buffer);
+		},
+		retrieve,
+		exists,
+		delete: del,
+		storeExportArchiveStream: async (
+			accountConfigId,
+			exportRequestId,
+			body,
+		) => {
+			const key = buildExportArchiveKey(accountConfigId, exportRequestId);
+			const fullPath = join(basePath, key);
+			await mkdir(dirname(fullPath), { recursive: true });
+			await pipeline(body, createWriteStream(fullPath));
+			return key;
+		},
+		getPresignedDownloadUrl: (_key: string, _expiresInSeconds: number) => {
+			throw new Error(
+				"getPresignedDownloadUrl is not supported on the filesystem backend",
+			);
+		},
+	};
+};