@remit/storage-service 0.0.7 → 0.0.9

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@remit/storage-service",
-  "version": "0.0.7",
+  "version": "0.0.9",
   "type": "module",
   "main": "src/index.ts",
   "types": "src/index.ts",
@@ -16,6 +16,10 @@
     "./s3": {
       "types": "./src/s3.ts",
       "default": "./src/s3.ts"
+    },
+    "./signed-path": {
+      "types": "./src/signed-path.ts",
+      "default": "./src/signed-path.ts"
     }
   },
   "scripts": {

package/src/backends/filesystem.ts

@@ -4,6 +4,7 @@ import {
 	mkdir,
 	readdir,
 	readFile,
+	rm,
 	stat,
 	unlink,
 	writeFile,
@@ -13,7 +14,13 @@ import { PassThrough, Readable } from "node:stream";
 import { pipeline } from "node:stream/promises";
 import { createGunzip, createGzip, gunzipSync, gzipSync } from "node:zlib";
 import { ContentEncoding, StorageType } from "@remit/domain-enums";
+import {
+	UPLOAD_ROUTE_PREFIX,
+	UPLOAD_URL_SIGNING_LABEL,
+} from "../outbox-upload-url.js";
+import { signStoragePath } from "../signed-path.js";
 import type {
+	OutboxAttachmentListItem,
 	ParsedBody,
 	StorageReference,
 	StorageService,
@@ -26,6 +33,8 @@ import {
 	buildExtractedSkippedKey,
 	buildExtractedTextKey,
 	buildMessageBodyKey,
+	buildOutboxAttachmentKey,
+	buildOutboxAttachmentPrefix,
 	buildParsedBodyKey,
 	computeChecksum,
 	isStorageNotFoundError,
@@ -37,8 +46,20 @@ interface StoreParams {
 	compress?: boolean;
 }
 
+/**
+ * Where a minted upload URL points, and what signs it. Absent on a process that
+ * only reads storage; a mint then fails loud rather than handing out a URL
+ * nothing can verify.
+ */
+export interface FilesystemUploadUrlConfig {
+	/** Public origin of the deployment serving the upload route. */
+	origin: string;
+	signingSecret: string;
+}
+
 export const createFilesystemStorageService = (
 	basePath: string,
+	uploadUrls?: FilesystemUploadUrlConfig,
 ): StorageService => {
 	const storeInternal = async (
 		params: StoreParams,
@@ -252,6 +273,181 @@ export const createFilesystemStorageService = (
 			}));
 	};
 
+	const storeOutboxAttachment: StorageService["storeOutboxAttachment"] = (
+		params,
+	) => {
+		const {
+			accountConfigId,
+			accountId,
+			outboxMessageId,
+			outboxAttachmentId,
+			content,
+		} = params;
+		return storeInternal({
+			key: buildOutboxAttachmentKey(
+				accountConfigId,
+				accountId,
+				outboxMessageId,
+				outboxAttachmentId,
+			),
+			content,
+			compress: false,
+		});
+	};
+
+	const listOutboxAttachments: StorageService["listOutboxAttachments"] = async (
+		accountConfigId,
+		accountId,
+		outboxMessageId,
+	) => {
+		const prefix = buildOutboxAttachmentPrefix(
+			accountConfigId,
+			accountId,
+			outboxMessageId,
+		);
+		const entries = await readdir(join(basePath, prefix)).catch(
+			(error: unknown) => {
+				if (isStorageNotFoundError(error)) return [];
+				throw error;
+			},
+		);
+
+		const items: OutboxAttachmentListItem[] = [];
+		for (const name of entries) {
+			const { size } = await stat(join(basePath, prefix, name));
+			items.push({
+				outboxAttachmentId: name,
+				key: `${prefix}${name}`,
+				sizeBytes: size,
+			});
+		}
+		return items;
+	};
+
+	const listOutboxDraftsWithAttachments: StorageService["listOutboxDraftsWithAttachments"] =
+		async (accountConfigId, accountId) => {
+			// An account that has never held an attachment has no directory, and
+			// that is genuinely nothing to do. A storage root that is not there at
+			// all is a different thing entirely — a process that cannot see the
+			// volume it is meant to be collecting from — and it must not read as an
+			// empty answer. Unmounted, the sweep would report every account swept
+			// and collect nothing, forever.
+			const rootExists = await stat(basePath)
+				.then(() => true)
+				.catch((error: unknown) => {
+					if (isStorageNotFoundError(error)) return false;
+					throw error;
+				});
+			if (!rootExists) {
+				throw new Error(
+					`storage root ${basePath} does not exist; this process cannot see the storage it was asked to read`,
+				);
+			}
+
+			const root = join(
+				basePath,
+				`accounts/${accountConfigId}/${accountId}/outbox`,
+			);
+			const entries = await readdir(root, { withFileTypes: true }).catch(
+				(error: unknown) => {
+					if (isStorageNotFoundError(error)) return [];
+					throw error;
+				},
+			);
+			return entries
+				.filter((entry) => entry.isDirectory())
+				.map((entry) => entry.name);
+		};
+
+	const deleteOutboxAttachment: StorageService["deleteOutboxAttachment"] =
+		async (accountConfigId, accountId, outboxMessageId, outboxAttachmentId) => {
+			const prefix = buildOutboxAttachmentPrefix(
+				accountConfigId,
+				accountId,
+				outboxMessageId,
+			);
+			const entries = await readdir(join(basePath, prefix)).catch(
+				(error: unknown) => {
+					if (isStorageNotFoundError(error)) return [];
+					throw error;
+				},
+			);
+			for (const name of entries) {
+				if (name !== outboxAttachmentId) continue;
+				await rm(join(basePath, prefix, name), { force: true });
+			}
+		};
+
+	const statOutboxAttachment: StorageService["statOutboxAttachment"] = async (
+		accountConfigId,
+		accountId,
+		outboxMessageId,
+		outboxAttachmentId,
+	) => {
+		const fullPath = join(
+			basePath,
+			buildOutboxAttachmentKey(
+				accountConfigId,
+				accountId,
+				outboxMessageId,
+				outboxAttachmentId,
+			),
+		);
+		const stats = await stat(fullPath).catch((error: unknown) => {
+			if (isStorageNotFoundError(error)) return null;
+			throw error;
+		});
+		return stats ? { sizeBytes: stats.size } : null;
+	};
+
+	// This backend has no presigned anything, so the URL addresses the
+	// deployment's own upload route. Its authority is the same HMAC the read side
+	// uses on /content, under the write label, covering the storage key, the
+	// expiry and the exact byte count — a URL for one attachment, for a while,
+	// for one size.
+	const createOutboxAttachmentUploadUrl: StorageService["createOutboxAttachmentUploadUrl"] =
+		async (params) => {
+			if (!uploadUrls) {
+				throw new Error(
+					"the filesystem storage backend cannot mint an upload URL without an origin and a signing secret",
+				);
+			}
+			const key = buildOutboxAttachmentKey(
+				params.accountConfigId,
+				params.accountId,
+				params.outboxMessageId,
+				params.outboxAttachmentId,
+			);
+			const sig = signStoragePath(
+				uploadUrls.signingSecret,
+				UPLOAD_URL_SIGNING_LABEL,
+				key,
+				[params.expiresAt, params.sizeBytes],
+			);
+			const origin = uploadUrls.origin.replace(/\/+$/, "");
+			const query = new URLSearchParams({
+				exp: String(params.expiresAt),
+				max: String(params.sizeBytes),
+				sig,
+			});
+			return {
+				uploadUrl: `${origin}${UPLOAD_ROUTE_PREFIX}${key}?${query.toString()}`,
+			};
+		};
+
+	const deleteOutboxAttachments: StorageService["deleteOutboxAttachments"] =
+		async (accountConfigId, accountId, outboxMessageId) => {
+			const dirPath = join(
+				basePath,
+				buildOutboxAttachmentPrefix(
+					accountConfigId,
+					accountId,
+					outboxMessageId,
+				),
+			);
+			await rm(dirPath, { recursive: true, force: true });
+		};
+
 	const storeDeduplicated: StorageService["storeDeduplicated"] = (params) => {
 		const { accountConfigId, accountId, content } = params;
 		const checksumSha256 = computeChecksum(content);
@@ -338,6 +534,13 @@ export const createFilesystemStorageService = (
 		storeBodyPart,
 		bodyPartExists,
 		retrieveBodyPart,
+		storeOutboxAttachment,
+		listOutboxAttachments,
+		listOutboxDraftsWithAttachments,
+		deleteOutboxAttachments,
+		deleteOutboxAttachment,
+		statOutboxAttachment,
+		createOutboxAttachmentUploadUrl,
 		storeDeduplicated,
 		storeParsedBody,
 		retrieveParsedBody,

package/src/backends/s3.test.ts

@@ -3,6 +3,8 @@ import { Readable } from "node:stream";
 import { describe, test } from "node:test";
 import { gunzipSync } from "node:zlib";
 import {
+	DeleteObjectCommand,
+	DeleteObjectsCommand,
 	GetObjectCommand,
 	HeadObjectCommand,
 	ListObjectsV2Command,
@@ -22,16 +24,34 @@ interface CapturedCommand {
 const createFakeS3Client = (): {
 	client: S3Client;
 	commands: CapturedCommand[];
-	stored: Map<string, { body: Buffer; contentEncoding?: string }>;
+	stored: Map<string, { body: Buffer; etag: string; contentEncoding?: string }>;
 } => {
 	const commands: CapturedCommand[] = [];
-	const stored = new Map<string, { body: Buffer; contentEncoding?: string }>();
+	const stored = new Map<
+		string,
+		{ body: Buffer; etag: string; contentEncoding?: string }
+	>();
+	let etagCounter = 0;
 
 	const send = async (command: unknown): Promise<unknown> => {
 		if (command instanceof PutObjectCommand) {
 			const input = command.input as unknown as Record<string, unknown>;
 			commands.push({ name: "PutObjectCommand", input });
 
+			// S3's conditional write, which is what the ledger's compare-and-set
+			// rides on: If-None-Match refuses an overwrite, If-Match refuses a
+			// stale one. Both answer 412.
+			const existing = stored.get(String(input.Key));
+			const precondition =
+				(input.IfNoneMatch === "*" && existing !== undefined) ||
+				(typeof input.IfMatch === "string" && input.IfMatch !== existing?.etag);
+			if (precondition) {
+				throw Object.assign(new Error("PreconditionFailed"), {
+					name: "PreconditionFailed",
+					$metadata: { httpStatusCode: 412 },
+				});
+			}
+
 			if (input.ChecksumSHA256 !== undefined) {
 				throw Object.assign(
 					new Error(
@@ -46,8 +66,10 @@ const createFakeS3Client = (): {
 			if (!Buffer.isBuffer(body)) {
 				throw new Error("expected Body to be a Buffer");
 			}
+			etagCounter += 1;
 			stored.set(key, {
 				body,
+				etag: `"etag-${etagCounter}"`,
 				contentEncoding:
 					typeof input.ContentEncoding === "string"
 						? input.ContentEncoding
@@ -70,26 +92,65 @@ const createFakeS3Client = (): {
 					transformToByteArray: async () => new Uint8Array(entry.body),
 				},
 				ContentEncoding: entry.contentEncoding,
+				ETag: entry.etag,
 			};
 		}
 
 		if (command instanceof HeadObjectCommand) {
 			const input = command.input as unknown as Record<string, unknown>;
 			const key = String(input.Key);
-			if (!stored.has(key)) {
+			const entry = stored.get(key);
+			if (!entry) {
 				throw Object.assign(new Error(`not found: ${key}`), {
 					name: "NotFound",
 				});
 			}
+			return { ContentLength: entry.body.length };
+		}
+
+		if (command instanceof DeleteObjectCommand) {
+			const input = command.input as unknown as Record<string, unknown>;
+			commands.push({ name: "DeleteObjectCommand", input });
+			stored.delete(String(input.Key));
+			return {};
+		}
+
+		if (command instanceof DeleteObjectsCommand) {
+			const input = command.input as unknown as Record<string, unknown>;
+			commands.push({ name: "DeleteObjectsCommand", input });
+			const del = input.Delete as { Objects: { Key: string }[] };
+			for (const { Key } of del.Objects) stored.delete(Key);
 			return {};
 		}
 
 		if (command instanceof ListObjectsV2Command) {
 			const input = command.input as unknown as Record<string, unknown>;
 			const prefix = String(input.Prefix ?? "");
-			const Contents = [...stored.keys()]
-				.filter((key) => key.startsWith(prefix))
-				.map((Key) => ({ Key }));
+			const maxKeys = Number(input.MaxKeys ?? Number.POSITIVE_INFINITY);
+			const matching = [...stored.entries()].filter(([key]) =>
+				key.startsWith(prefix),
+			);
+
+			// A delimited listing answers with directory-like prefixes instead of
+			// the objects beneath them, the way listOutboxDraftsWithAttachments
+			// asks for them.
+			if (typeof input.Delimiter === "string") {
+				const delimiter = input.Delimiter;
+				const prefixes = new Set<string>();
+				for (const [key] of matching) {
+					const rest = key.slice(prefix.length);
+					const at = rest.indexOf(delimiter);
+					if (at >= 0) prefixes.add(prefix + rest.slice(0, at + 1));
+				}
+				return {
+					CommonPrefixes: [...prefixes].map((Prefix) => ({ Prefix })),
+					IsTruncated: false,
+				};
+			}
+
+			const Contents = matching
+				.slice(0, maxKeys)
+				.map(([Key, entry]) => ({ Key, Size: entry.body.length }));
 			return { Contents, IsTruncated: false };
 		}
 
@@ -461,3 +522,80 @@ describe("createS3StorageService", () => {
 		}
 	});
 });
+
+describe("createS3StorageService: outbox attachments", () => {
+	test("an uploaded object lists at the size the bucket holds", async () => {
+		const { client } = createFakeS3Client();
+		const storage = createS3StorageService(client, "bucket");
+
+		await storage.storeOutboxAttachment({
+			accountConfigId: "cfg1",
+			accountId: "acc1",
+			outboxMessageId: "draft1",
+			outboxAttachmentId: "att1",
+			content: Buffer.alloc(777),
+		});
+
+		const [entry] = await storage.listOutboxAttachments(
+			"cfg1",
+			"acc1",
+			"draft1",
+		);
+		assert.strictEqual(entry.outboxAttachmentId, "att1");
+		// Uncompressed, so the size in the bucket is the size the cap counts.
+		assert.strictEqual(entry.sizeBytes, 777);
+		assert.strictEqual(
+			await storage
+				.statOutboxAttachment("cfg1", "acc1", "draft1", "att1")
+				.then((found) => found?.sizeBytes),
+			777,
+		);
+	});
+
+	test("statOutboxAttachment answers null for an object that was never uploaded", async () => {
+		const { client } = createFakeS3Client();
+		const storage = createS3StorageService(client, "bucket");
+
+		assert.strictEqual(
+			await storage.statOutboxAttachment("cfg1", "acc1", "draft1", "nothing"),
+			null,
+		);
+	});
+
+	test("deleting a draft's attachments empties the prefix", async () => {
+		const { client } = createFakeS3Client();
+		const storage = createS3StorageService(client, "bucket");
+		await storage.storeOutboxAttachment({
+			accountConfigId: "cfg1",
+			accountId: "acc1",
+			outboxMessageId: "draft1",
+			outboxAttachmentId: "att2",
+			content: Buffer.alloc(8),
+		});
+		await storage.deleteOutboxAttachments("cfg1", "acc1", "draft1");
+
+		assert.deepStrictEqual(
+			await storage.listOutboxAttachments("cfg1", "acc1", "draft1"),
+			[],
+		);
+	});
+
+	test("listOutboxDraftsWithAttachments finds prefixes whose row may be long gone", async () => {
+		const { client } = createFakeS3Client();
+		const storage = createS3StorageService(client, "bucket");
+		for (const draft of ["draft1", "draft2"]) {
+			await storage.storeOutboxAttachment({
+				accountConfigId: "cfg1",
+				accountId: "acc1",
+				outboxMessageId: draft,
+				outboxAttachmentId: "att1",
+				content: Buffer.alloc(4),
+			});
+		}
+
+		assert.deepStrictEqual(
+			(await storage.listOutboxDraftsWithAttachments("cfg1", "acc1")).sort(),
+			["draft1", "draft2"],
+		);
+	});
+});