@remit/storage-service 0.0.7 → 0.0.9

package/src/backends/s3.ts

@@ -4,6 +4,7 @@ import { pipeline } from "node:stream/promises";
 import { createGunzip, createGzip, gunzipSync, gzipSync } from "node:zlib";
 import {
 	DeleteObjectCommand,
+	DeleteObjectsCommand,
 	GetObjectCommand,
 	HeadObjectCommand,
 	ListObjectsV2Command,
@@ -15,6 +16,7 @@ import { getSignedUrl } from "@aws-sdk/s3-request-presigner";
 import { ContentEncoding, StorageType } from "@remit/domain-enums";
 import type {
 	ExtractedTextListItem,
+	OutboxAttachmentListItem,
 	ParsedBody,
 	StorageReference,
 	StorageService,
@@ -27,6 +29,8 @@ import {
 	buildExtractedSkippedKey,
 	buildExtractedTextKey,
 	buildMessageBodyKey,
+	buildOutboxAttachmentKey,
+	buildOutboxAttachmentPrefix,
 	buildParsedBodyKey,
 	computeChecksum,
 	isStorageNotFoundError,
@@ -325,6 +329,209 @@ export const createS3StorageService = (
 		return items;
 	};
 
+	const storeOutboxAttachment: StorageService["storeOutboxAttachment"] = (
+		params,
+	) => {
+		const {
+			accountConfigId,
+			accountId,
+			outboxMessageId,
+			outboxAttachmentId,
+			content,
+		} = params;
+		// `application/octet-stream` rather than the uploader's declared type:
+		// the object is only ever read back to build a MIME part, and a stored
+		// `text/html` would be a type any future signed-URL read path could be
+		// tricked into serving from the content origin.
+		return storeInternal({
+			key: buildOutboxAttachmentKey(
+				accountConfigId,
+				accountId,
+				outboxMessageId,
+				outboxAttachmentId,
+			),
+			content,
+			contentType: "application/octet-stream",
+			compress: false,
+		});
+	};
+
+	const listOutboxAttachments: StorageService["listOutboxAttachments"] = async (
+		accountConfigId,
+		accountId,
+		outboxMessageId,
+	) => {
+		const prefix = buildOutboxAttachmentPrefix(
+			accountConfigId,
+			accountId,
+			outboxMessageId,
+		);
+		const items: OutboxAttachmentListItem[] = [];
+		let continuationToken: string | undefined;
+
+		do {
+			const response = await client.send(
+				new ListObjectsV2Command({
+					Bucket: bucketName,
+					Prefix: prefix,
+					ContinuationToken: continuationToken,
+				}),
+			);
+			for (const object of response.Contents ?? []) {
+				if (!object.Key) continue;
+				items.push({
+					outboxAttachmentId: object.Key.slice(prefix.length),
+					key: object.Key,
+					sizeBytes: object.Size ?? 0,
+				});
+			}
+			continuationToken = response.IsTruncated
+				? response.NextContinuationToken
+				: undefined;
+		} while (continuationToken);
+
+		return items;
+	};
+
+	const listOutboxDraftsWithAttachments: StorageService["listOutboxDraftsWithAttachments"] =
+		async (accountConfigId, accountId) => {
+			const root = `accounts/${accountConfigId}/${accountId}/outbox/`;
+			const drafts = new Set<string>();
+			let continuationToken: string | undefined;
+
+			// Delimited listing: S3 answers with the directory-like prefixes rather
+			// than every object under them, so this stays one call per page of
+			// drafts instead of one per object.
+			do {
+				const response = await client.send(
+					new ListObjectsV2Command({
+						Bucket: bucketName,
+						Prefix: root,
+						Delimiter: "/",
+						ContinuationToken: continuationToken,
+					}),
+				);
+				for (const entry of response.CommonPrefixes ?? []) {
+					if (!entry.Prefix) continue;
+					drafts.add(entry.Prefix.slice(root.length).replace(/\/$/, ""));
+				}
+				continuationToken = response.IsTruncated
+					? response.NextContinuationToken
+					: undefined;
+			} while (continuationToken);
+
+			return [...drafts];
+		};
+
+	const deleteOutboxAttachment: StorageService["deleteOutboxAttachment"] =
+		async (accountConfigId, accountId, outboxMessageId, outboxAttachmentId) => {
+			const prefix = buildOutboxAttachmentPrefix(
+				accountConfigId,
+				accountId,
+				outboxMessageId,
+			);
+			await client.send(
+				new DeleteObjectCommand({
+					Bucket: bucketName,
+					Key: `${prefix}${outboxAttachmentId}`,
+				}),
+			);
+		};
+
+	const statOutboxAttachment: StorageService["statOutboxAttachment"] = async (
+		accountConfigId,
+		accountId,
+		outboxMessageId,
+		outboxAttachmentId,
+	) => {
+		const head = await client
+			.send(
+				new HeadObjectCommand({
+					Bucket: bucketName,
+					Key: buildOutboxAttachmentKey(
+						accountConfigId,
+						accountId,
+						outboxMessageId,
+						outboxAttachmentId,
+					),
+				}),
+			)
+			.catch((error: unknown) => {
+				if (isStorageNotFoundError(error)) return null;
+				// A HEAD on a missing key answers 404 with no NoSuchKey code, so the
+				// SDK's NotFound is the same absence.
+				if ((error as { name?: string })?.name === "NotFound") return null;
+				throw error;
+			});
+		if (!head) return null;
+		return { sizeBytes: head.ContentLength ?? 0 };
+	};
+
+	// The browser PUTs straight to the bucket and these bytes never touch the
+	// API. `ContentLength` on the command hoists `content-length` into
+	// X-Amz-SignedHeaders, so S3 recomputes the signature over the length it was
+	// sent: a request declaring any other size fails the signature check, and a
+	// PUT is read for exactly that many bytes. The size is bound by S3 itself,
+	// not by anything downstream trusting the client.
+	const createOutboxAttachmentUploadUrl: StorageService["createOutboxAttachmentUploadUrl"] =
+		async (params) => {
+			const expiresIn = Math.max(
+				1,
+				params.expiresAt - Math.floor(Date.now() / 1000),
+			);
+			const uploadUrl = await getSignedUrl(
+				client,
+				new PutObjectCommand({
+					Bucket: bucketName,
+					Key: buildOutboxAttachmentKey(
+						params.accountConfigId,
+						params.accountId,
+						params.outboxMessageId,
+						params.outboxAttachmentId,
+					),
+					ContentLength: params.sizeBytes,
+				}),
+				{ expiresIn },
+			);
+			return { uploadUrl };
+		};
+
+	const deleteOutboxAttachments: StorageService["deleteOutboxAttachments"] =
+		async (accountConfigId, accountId, outboxMessageId) => {
+			const prefix = buildOutboxAttachmentPrefix(
+				accountConfigId,
+				accountId,
+				outboxMessageId,
+			);
+			let continuationToken: string | undefined;
+
+			do {
+				const response = await client.send(
+					new ListObjectsV2Command({
+						Bucket: bucketName,
+						Prefix: prefix,
+						ContinuationToken: continuationToken,
+					}),
+				);
+				const keys = (response.Contents ?? [])
+					.map((object) => object.Key)
+					.filter((key): key is string => typeof key === "string");
+
+				if (keys.length > 0) {
+					await client.send(
+						new DeleteObjectsCommand({
+							Bucket: bucketName,
+							Delete: { Objects: keys.map((Key) => ({ Key })) },
+						}),
+					);
+				}
+
+				continuationToken = response.IsTruncated
+					? response.NextContinuationToken
+					: undefined;
+			} while (continuationToken);
+		};
+
 	const storeDeduplicated: StorageService["storeDeduplicated"] = (params) => {
 		const { accountConfigId, accountId, content, contentType } = params;
 		const checksumSha256 = computeChecksum(content);
@@ -505,6 +712,13 @@ export const createS3StorageService = (
 		storeBodyPart,
 		bodyPartExists,
 		retrieveBodyPart,
+		storeOutboxAttachment,
+		listOutboxAttachments,
+		listOutboxDraftsWithAttachments,
+		deleteOutboxAttachments,
+		deleteOutboxAttachment,
+		statOutboxAttachment,
+		createOutboxAttachmentUploadUrl,
 		storeDeduplicated,
 		storeParsedBody,
 		retrieveParsedBody,

package/src/index.ts

@@ -1,3 +1,21 @@
+export {
+	type OutboxSweepResult,
+	sweepAbandonedOutboxAttachments,
+} from "./outbox-sweep.js";
+export {
+	type AuthorizeUploadInput,
+	type AuthorizeUploadResult,
+	authorizeUploadRequest,
+	UPLOAD_ROUTE_PREFIX,
+	UPLOAD_URL_SIGNING_LABEL,
+	UPLOAD_URL_TTL_SECONDS,
+} from "./outbox-upload-url.js";
+export {
+	type SignedPathFailure,
+	type SignedPathResult,
+	signStoragePath,
+	verifyStoragePath,
+} from "./signed-path.js";
 export {
 	buildBodyPartKey,
 	buildDeduplicatedKey,
@@ -6,17 +24,24 @@ export {
 	buildExtractedSkippedKey,
 	buildExtractedTextKey,
 	buildMessageBodyKey,
+	buildOutboxAttachmentKey,
+	buildOutboxAttachmentPrefix,
 	buildParsedBodyKey,
 	type ContentEncodingValue,
+	type CreateOutboxAttachmentUploadUrlParams,
 	computeChecksum,
 	createMockStorageService,
 	type ExtractedTextListItem,
 	type ExtractedTextMarker,
 	isStorageNotFoundError,
+	type OutboxAttachmentListItem,
+	type OutboxAttachmentUploadTarget,
 	type ParsedAttachmentMeta,
 	type ParsedBody,
 	type ParsedContentStorageKey,
+	type ParsedOutboxAttachmentKey,
 	parseContentStorageKey,
+	parseOutboxAttachmentKey,
 	type StorageReference,
 	type StorageService,
 	type StorageTypeValue,
@@ -25,6 +50,7 @@ export {
 	type StoreExtractedSkippedParams,
 	type StoreExtractedTextParams,
 	type StoreMessageBodyParams,
+	type StoreOutboxAttachmentParams,
 	type StoreParsedBodyParams,
 } from "./storage.js";
 export {

package/src/outbox-sweep.test.ts

@@ -0,0 +1,196 @@
+import assert from "node:assert";
+import { describe, test } from "node:test";
+import { sweepAbandonedOutboxAttachments } from "./outbox-sweep.js";
+import { createMockStorageService, type StorageService } from "./storage.js";
+
+const CFG = "cfg1";
+const ACC = "acc1";
+
+const withObjects = async (
+	drafts: Record<string, string[]>,
+): Promise<StorageService> => {
+	const storage = createMockStorageService();
+	for (const [outboxMessageId, ids] of Object.entries(drafts)) {
+		for (const outboxAttachmentId of ids) {
+			await storage.storeOutboxAttachment({
+				accountConfigId: CFG,
+				accountId: ACC,
+				outboxMessageId,
+				outboxAttachmentId,
+				content: Buffer.alloc(4),
+			});
+		}
+	}
+	return storage;
+};
+
+const remaining = (storage: StorageService, draft: string) =>
+	storage
+		.listOutboxAttachments(CFG, ACC, draft)
+		.then((items) => items.map((item) => item.outboxAttachmentId).sort());
+
+describe("sweepAbandonedOutboxAttachments", () => {
+	test("deletes objects the database does not know about", async () => {
+		const storage = await withObjects({ draft1: ["known", "orphan"] });
+
+		const result = await sweepAbandonedOutboxAttachments(
+			{
+				storage,
+				listKnownAttachmentIds: async () => ["known"],
+				onSkipped: () => assert.fail("nothing should be skipped"),
+			},
+			CFG,
+			ACC,
+		);
+
+		assert.deepStrictEqual(await remaining(storage, "draft1"), ["known"]);
+		assert.strictEqual(result.deleted, 1);
+		assert.strictEqual(result.skipped, 0);
+	});
+
+	test("empties the prefix of a draft the database has forgotten", async () => {
+		// A discarded draft has no rows at all, and a presigned PUT can still have
+		// landed under it. This is the case the sweep exists for.
+		const storage = await withObjects({ discarded: ["a", "b"] });
+
+		const result = await sweepAbandonedOutboxAttachments(
+			{
+				storage,
+				listKnownAttachmentIds: async () => [],
+				onSkipped: () => assert.fail("nothing should be skipped"),
+			},
+			CFG,
+			ACC,
+		);
+
+		assert.deepStrictEqual(await remaining(storage, "discarded"), []);
+		assert.strictEqual(result.deleted, 2);
+	});
+
+	test("leaves a draft alone when its rows cannot be read", async () => {
+		// "I could not find out what is live" must never be acted on as "nothing
+		// is live" — that would delete a whole draft's files on a transient error.
+		const storage = await withObjects({ draft1: ["a", "b"] });
+		const skips: string[] = [];
+
+		const result = await sweepAbandonedOutboxAttachments(
+			{
+				storage,
+				listKnownAttachmentIds: async () => {
+					throw new Error("database unavailable");
+				},
+				onSkipped: (outboxMessageId) => skips.push(outboxMessageId),
+			},
+			CFG,
+			ACC,
+		);
+
+		assert.deepStrictEqual(await remaining(storage, "draft1"), ["a", "b"]);
+		assert.strictEqual(result.deleted, 0);
+		assert.strictEqual(result.skipped, 1);
+		assert.deepStrictEqual(skips, ["draft1"]);
+	});
+
+	test("one unreadable draft does not stop the others being collected", async () => {
+		const storage = await withObjects({ bad: ["x"], good: ["orphan"] });
+
+		const result = await sweepAbandonedOutboxAttachments(
+			{
+				storage,
+				listKnownAttachmentIds: async (_cfg, outboxMessageId) => {
+					if (outboxMessageId === "bad") throw new Error("unavailable");
+					return [];
+				},
+				onSkipped: () => {},
+			},
+			CFG,
+			ACC,
+		);
+
+		assert.deepStrictEqual(await remaining(storage, "bad"), ["x"]);
+		assert.deepStrictEqual(await remaining(storage, "good"), []);
+		assert.strictEqual(result.skipped, 1);
+		assert.strictEqual(result.deleted, 1);
+	});
+
+	test("an attachment that lands between the two reads is kept", async () => {
+		// Objects are listed before the rows are read. A mint that lands in that
+		// window is seen as a row and not as an object, so it survives. Read the
+		// rows first and the same mint is an object with no row behind it — this
+		// fails if those two lines ever swap.
+		const storage = await withObjects({ draft1: ["already-there"] });
+		let listedObjects = false;
+
+		const inner = storage.listOutboxAttachments.bind(storage);
+		const observing = {
+			...storage,
+			listOutboxAttachments: async (
+				accountConfigId: string,
+				accountId: string,
+				outboxMessageId: string,
+			) => {
+				listedObjects = true;
+				return inner(accountConfigId, accountId, outboxMessageId);
+			},
+		} as StorageService;
+
+		await sweepAbandonedOutboxAttachments(
+			{
+				storage: observing,
+				listKnownAttachmentIds: async () => {
+					assert.strictEqual(
+						listedObjects,
+						true,
+						"rows must be read after the objects were listed, never before",
+					);
+					// The mint that landed in the window.
+					await storage.storeOutboxAttachment({
+						accountConfigId: CFG,
+						accountId: ACC,
+						outboxMessageId: "draft1",
+						outboxAttachmentId: "just-reserved",
+						content: Buffer.alloc(4),
+					});
+					return ["already-there", "just-reserved"];
+				},
+				onSkipped: () => {},
+			},
+			CFG,
+			ACC,
+		);
+
+		assert.deepStrictEqual(await remaining(storage, "draft1"), [
+			"already-there",
+			"just-reserved",
+		]);
+	});
+
+	test("never reaches another tenant's objects", async () => {
+		const storage = createMockStorageService();
+		await storage.storeOutboxAttachment({
+			accountConfigId: "cfg-other",
+			accountId: ACC,
+			outboxMessageId: "draft1",
+			outboxAttachmentId: "theirs",
+			content: Buffer.alloc(4),
+		});
+
+		const result = await sweepAbandonedOutboxAttachments(
+			{
+				storage,
+				listKnownAttachmentIds: async () => [],
+				onSkipped: () => {},
+			},
+			CFG,
+			ACC,
+		);
+
+		assert.strictEqual(result.deleted, 0);
+		assert.deepStrictEqual(
+			(await storage.listOutboxAttachments("cfg-other", ACC, "draft1")).map(
+				(item) => item.outboxAttachmentId,
+			),
+			["theirs"],
+		);
+	});
+});

package/src/outbox-sweep.ts

@@ -0,0 +1,103 @@
+import type { StorageService } from "./storage.js";
+
+/**
+ * Collect attachment bytes the database does not know about.
+ *
+ * A presigned upload URL stays valid for its whole window, including after the
+ * draft it was minted for is discarded, and on a deployment where the browser
+ * PUTs straight to block storage there is nothing in that path to consult. So
+ * an object can exist that no row names. That is the only case this exists for,
+ * and the rule is the whole of it: an object survives if the database still has
+ * an attachment row for it.
+ *
+ * Two orderings matter and both are deliberate. Objects are listed *before* the
+ * rows are read, so an attachment reserved between the two reads is seen as a
+ * row and not as an object, and is kept. And a draft whose row lookup fails is
+ * skipped entirely — "I could not find out what is live" must never be acted on
+ * as "nothing is live".
+ */
+export interface OutboxSweepResult {
+	inspected: number;
+	deleted: number;
+	/** Drafts skipped because their attachment rows could not be read. */
+	skipped: number;
+}
+
+export interface OutboxSweepDeps {
+	storage: StorageService;
+	/**
+	 * The attachment ids the database holds for a draft. Throwing is the right
+	 * way to say "unknown" — the sweep then leaves that draft alone.
+	 */
+	listKnownAttachmentIds: (
+		accountConfigId: string,
+		outboxMessageId: string,
+	) => Promise<string[]>;
+	onSkipped: (outboxMessageId: string, error: unknown) => void;
+}
+
+export const sweepAbandonedOutboxAttachments = async (
+	deps: OutboxSweepDeps,
+	accountConfigId: string,
+	accountId: string,
+): Promise<OutboxSweepResult> => {
+	let inspected = 0;
+	let deleted = 0;
+	let skipped = 0;
+
+	const outboxMessageIds = await deps.storage.listOutboxDraftsWithAttachments(
+		accountConfigId,
+		accountId,
+	);
+
+	for (const outboxMessageId of outboxMessageIds) {
+		const objects = await deps.storage.listOutboxAttachments(
+			accountConfigId,
+			accountId,
+			outboxMessageId,
+		);
+		if (objects.length === 0) continue;
+
+		const known = await deps
+			.listKnownAttachmentIds(accountConfigId, outboxMessageId)
+			.then((ids) => new Set(ids))
+			.catch((error: unknown) => {
+				deps.onSkipped(outboxMessageId, error);
+				return null;
+			});
+		if (known === null) {
+			skipped += 1;
+			continue;
+		}
+
+		inspected += objects.length;
+		const abandoned = objects.filter(
+			(object) => !known.has(object.outboxAttachmentId),
+		);
+
+		// A draft whose every object is abandoned is the common case — it was
+		// discarded — and taking the prefix in one call beats one round trip per
+		// object.
+		if (abandoned.length === objects.length) {
+			await deps.storage.deleteOutboxAttachments(
+				accountConfigId,
+				accountId,
+				outboxMessageId,
+			);
+			deleted += abandoned.length;
+			continue;
+		}
+
+		for (const object of abandoned) {
+			await deps.storage.deleteOutboxAttachment(
+				accountConfigId,
+				accountId,
+				outboxMessageId,
+				object.outboxAttachmentId,
+			);
+			deleted += 1;
+		}
+	}
+
+	return { inspected, deleted, skipped };
+};