@remit/storage-service 0.0.11 → 0.0.13

package/dist/backends/filesystem.d.ts

@@ -0,0 +1,12 @@
+import type { StorageService } from "../storage.js";
+/**
+ * Where a minted upload URL points, and what signs it. Absent on a process that
+ * only reads storage; a mint then fails loud rather than handing out a URL
+ * nothing can verify.
+ */
+export interface FilesystemUploadUrlConfig {
+    /** Public origin of the deployment serving the upload route. */
+    origin: string;
+    signingSecret: string;
+}
+export declare const createFilesystemStorageService: (basePath: string, uploadUrls?: FilesystemUploadUrlConfig) => StorageService;

package/dist/backends/filesystem.js

@@ -0,0 +1,316 @@
+import { createHash } from "node:crypto";
+import { createWriteStream } from "node:fs";
+import { mkdir, readdir, readFile, rm, stat, unlink, writeFile, } from "node:fs/promises";
+import { dirname, join, resolve } from "node:path";
+import { PassThrough, Readable } from "node:stream";
+import { pipeline } from "node:stream/promises";
+import { createGunzip, createGzip, gunzipSync, gzipSync } from "node:zlib";
+import { ContentEncoding, StorageType } from "@remit/domain-enums";
+import { UPLOAD_ROUTE_PREFIX, UPLOAD_URL_SIGNING_LABEL, } from "../outbox-upload-url.js";
+import { signStoragePath } from "../signed-path.js";
+import { buildBodyPartKey, buildDeduplicatedKey, buildExportArchiveKey, buildMessageBodyKey, buildOutboxAttachmentKey, buildOutboxAttachmentPrefix, buildParsedBodyKey, computeChecksum, isStorageNotFoundError, } from "../storage.js";
+export const createFilesystemStorageService = (basePath, uploadUrls) => {
+    const storeInternal = async (params) => {
+        const { key, content, compress = true } = params;
+        const checksumSha256 = computeChecksum(content);
+        const contentEncoding = compress
+            ? ContentEncoding.Gzip
+            : ContentEncoding.None;
+        const body = compress ? gzipSync(content) : content;
+        const fullPath = join(basePath, key);
+        await mkdir(dirname(fullPath), { recursive: true });
+        await writeFile(fullPath, body);
+        return {
+            uri: `file://${fullPath}`,
+            storageType: StorageType.Filesystem,
+            storageLocation: basePath,
+            storageKey: key,
+            sizeBytes: body.length,
+            checksumSha256,
+            contentEncoding,
+        };
+    };
+    const storeMessageBody = (params) => {
+        const { accountConfigId, accountId, messageId, content } = params;
+        return storeInternal({
+            key: buildMessageBodyKey(accountConfigId, accountId, messageId),
+            content,
+        });
+    };
+    const storeMessageBodyStream = async (params) => {
+        const { accountConfigId, accountId, messageId, content } = params;
+        const key = buildMessageBodyKey(accountConfigId, accountId, messageId);
+        const fullPath = join(basePath, key);
+        await mkdir(dirname(fullPath), { recursive: true });
+        const hash = createHash("sha256");
+        const hashTap = new PassThrough();
+        hashTap.on("data", (chunk) => hash.update(chunk));
+        await pipeline(content, hashTap, createGzip(), createWriteStream(fullPath));
+        const { size } = await stat(fullPath);
+        return {
+            uri: `file://${fullPath}`,
+            storageType: StorageType.Filesystem,
+            storageLocation: basePath,
+            storageKey: key,
+            sizeBytes: size,
+            checksumSha256: hash.digest("hex"),
+            contentEncoding: ContentEncoding.Gzip,
+        };
+    };
+    const storeBodyPart = (params) => {
+        const { accountConfigId, accountId, messageId, partPath, content } = params;
+        return storeInternal({
+            key: buildBodyPartKey(accountConfigId, accountId, messageId, partPath),
+            content,
+        });
+    };
+    const bodyPartExists = (accountConfigId, accountId, messageId, partPath) => {
+        const key = buildBodyPartKey(accountConfigId, accountId, messageId, partPath);
+        const fullPath = join(basePath, key);
+        return stat(fullPath)
+            .then(() => true)
+            .catch(() => false);
+    };
+    const retrieveBodyPart = async (accountConfigId, accountId, messageId, partPath) => {
+        const key = buildBodyPartKey(accountConfigId, accountId, messageId, partPath);
+        const fullPath = join(basePath, key);
+        const buffer = await readFile(fullPath).catch((error) => {
+            if (isStorageNotFoundError(error))
+                return null;
+            throw error;
+        });
+        if (!buffer)
+            return null;
+        return buffer[0] === 0x1f && buffer[1] === 0x8b
+            ? gunzipSync(buffer)
+            : buffer;
+    };
+    const storeOutboxAttachment = (params) => {
+        const { accountConfigId, accountId, outboxMessageId, outboxAttachmentId, content, } = params;
+        return storeInternal({
+            key: buildOutboxAttachmentKey(accountConfigId, accountId, outboxMessageId, outboxAttachmentId),
+            content,
+            compress: false,
+        });
+    };
+    const listOutboxAttachments = async (accountConfigId, accountId, outboxMessageId) => {
+        const prefix = buildOutboxAttachmentPrefix(accountConfigId, accountId, outboxMessageId);
+        const entries = await readdir(join(basePath, prefix)).catch((error) => {
+            if (isStorageNotFoundError(error))
+                return [];
+            throw error;
+        });
+        const items = [];
+        for (const name of entries) {
+            const { size } = await stat(join(basePath, prefix, name));
+            items.push({
+                outboxAttachmentId: name,
+                key: `${prefix}${name}`,
+                sizeBytes: size,
+            });
+        }
+        return items;
+    };
+    const listOutboxDraftsWithAttachments = async (accountConfigId, accountId) => {
+        // An account that has never held an attachment has no directory, and
+        // that is genuinely nothing to do. A storage root that is not there at
+        // all is a different thing entirely — a process that cannot see the
+        // volume it is meant to be collecting from — and it must not read as an
+        // empty answer. Unmounted, the sweep would report every account swept
+        // and collect nothing, forever.
+        const rootExists = await stat(basePath)
+            .then(() => true)
+            .catch((error) => {
+            if (isStorageNotFoundError(error))
+                return false;
+            throw error;
+        });
+        if (!rootExists) {
+            throw new Error(`storage root ${basePath} does not exist; this process cannot see the storage it was asked to read`);
+        }
+        const root = join(basePath, `accounts/${accountConfigId}/${accountId}/outbox`);
+        const entries = await readdir(root, { withFileTypes: true }).catch((error) => {
+            if (isStorageNotFoundError(error))
+                return [];
+            throw error;
+        });
+        return entries
+            .filter((entry) => entry.isDirectory())
+            .map((entry) => entry.name);
+    };
+    const deleteOutboxAttachment = async (accountConfigId, accountId, outboxMessageId, outboxAttachmentId) => {
+        const prefix = buildOutboxAttachmentPrefix(accountConfigId, accountId, outboxMessageId);
+        const entries = await readdir(join(basePath, prefix)).catch((error) => {
+            if (isStorageNotFoundError(error))
+                return [];
+            throw error;
+        });
+        for (const name of entries) {
+            if (name !== outboxAttachmentId)
+                continue;
+            await rm(join(basePath, prefix, name), { force: true });
+        }
+    };
+    const statOutboxAttachment = async (accountConfigId, accountId, outboxMessageId, outboxAttachmentId) => {
+        const fullPath = join(basePath, buildOutboxAttachmentKey(accountConfigId, accountId, outboxMessageId, outboxAttachmentId));
+        const stats = await stat(fullPath).catch((error) => {
+            if (isStorageNotFoundError(error))
+                return null;
+            throw error;
+        });
+        return stats ? { sizeBytes: stats.size } : null;
+    };
+    const retrieveOutboxAttachment = (accountConfigId, accountId, outboxMessageId, outboxAttachmentId) => readFile(join(basePath, buildOutboxAttachmentKey(accountConfigId, accountId, outboxMessageId, outboxAttachmentId))).catch((error) => {
+        if (isStorageNotFoundError(error))
+            return null;
+        throw error;
+    });
+    // This backend has no presigned anything, so the URL addresses the
+    // deployment's own upload route. Its authority is the same HMAC the read side
+    // uses on /content, under the write label, covering the storage key, the
+    // expiry and the exact byte count — a URL for one attachment, for a while,
+    // for one size.
+    const createOutboxAttachmentUploadUrl = async (params) => {
+        if (!uploadUrls) {
+            throw new Error("the filesystem storage backend cannot mint an upload URL without an origin and a signing secret");
+        }
+        const key = buildOutboxAttachmentKey(params.accountConfigId, params.accountId, params.outboxMessageId, params.outboxAttachmentId);
+        const sig = signStoragePath(uploadUrls.signingSecret, UPLOAD_URL_SIGNING_LABEL, key, [params.expiresAt, params.sizeBytes]);
+        const origin = uploadUrls.origin.replace(/\/+$/, "");
+        const query = new URLSearchParams({
+            exp: String(params.expiresAt),
+            max: String(params.sizeBytes),
+            sig,
+        });
+        return {
+            uploadUrl: `${origin}${UPLOAD_ROUTE_PREFIX}${key}?${query.toString()}`,
+        };
+    };
+    const deleteOutboxAttachments = async (accountConfigId, accountId, outboxMessageId) => {
+        const dirPath = join(basePath, buildOutboxAttachmentPrefix(accountConfigId, accountId, outboxMessageId));
+        await rm(dirPath, { recursive: true, force: true });
+    };
+    const storeDeduplicated = (params) => {
+        const { accountConfigId, accountId, content } = params;
+        const checksumSha256 = computeChecksum(content);
+        return storeInternal({
+            key: buildDeduplicatedKey(accountConfigId, accountId, checksumSha256),
+            content,
+        });
+    };
+    const storeParsedBody = (params) => {
+        const { accountConfigId, accountId, messageId, parsed } = params;
+        const content = Buffer.from(JSON.stringify(parsed), "utf8");
+        return storeInternal({
+            key: buildParsedBodyKey(accountConfigId, accountId, messageId),
+            content,
+        });
+    };
+    const retrieveParsedBody = async (accountConfigId, accountId, messageId) => {
+        const key = buildParsedBodyKey(accountConfigId, accountId, messageId);
+        const fullPath = join(basePath, key);
+        const buffer = await readFile(fullPath).catch((error) => {
+            if (isStorageNotFoundError(error))
+                return null;
+            throw error;
+        });
+        if (!buffer)
+            return null;
+        const decoded = buffer[0] === 0x1f && buffer[1] === 0x8b ? gunzipSync(buffer) : buffer;
+        return JSON.parse(decoded.toString("utf8"));
+    };
+    // Resolve path from URI, handling both absolute and relative paths
+    // Note: URIs like file://.remit/storage/... parse incorrectly - '.remit' becomes hostname
+    // We reconstruct the original path by combining hostname + pathname
+    const resolvePathFromUri = (uri) => {
+        const url = new URL(uri);
+        const hostname = url.hostname;
+        const pathname = url.pathname;
+        // If hostname exists, it was part of a relative path (e.g., file://.remit/... -> hostname='.remit')
+        if (hostname) {
+            const relativePath = hostname + pathname;
+            return resolve(relativePath);
+        }
+        // Absolute path (file:///absolute/path)
+        return pathname;
+    };
+    const retrieve = async (uri) => {
+        const fullPath = resolvePathFromUri(uri);
+        const buffer = await readFile(fullPath);
+        if (buffer[0] === 0x1f && buffer[1] === 0x8b) {
+            return gunzipSync(buffer);
+        }
+        return buffer;
+    };
+    const exists = async (uri) => {
+        const fullPath = resolvePathFromUri(uri);
+        return stat(fullPath)
+            .then(() => true)
+            .catch(() => false);
+    };
+    const del = async (uri) => {
+        const fullPath = resolvePathFromUri(uri);
+        await unlink(fullPath);
+    };
+    return {
+        storeMessageBody,
+        storeMessageBodyStream,
+        storeBodyPart,
+        bodyPartExists,
+        retrieveBodyPart,
+        storeOutboxAttachment,
+        listOutboxAttachments,
+        listOutboxDraftsWithAttachments,
+        deleteOutboxAttachments,
+        deleteOutboxAttachment,
+        statOutboxAttachment,
+        retrieveOutboxAttachment,
+        createOutboxAttachmentUploadUrl,
+        storeDeduplicated,
+        storeParsedBody,
+        retrieveParsedBody,
+        retrieveMessageBody: async (accountConfigId, accountId, messageId) => {
+            const key = buildMessageBodyKey(accountConfigId, accountId, messageId);
+            const fullPath = join(basePath, key);
+            const buffer = await readFile(fullPath).catch((error) => {
+                if (isStorageNotFoundError(error))
+                    return null;
+                throw error;
+            });
+            if (!buffer)
+                return null;
+            if (buffer[0] === 0x1f && buffer[1] === 0x8b) {
+                return gunzipSync(buffer);
+            }
+            return buffer;
+        },
+        retrieveMessageBodyStream: async (accountConfigId, accountId, messageId) => {
+            const key = buildMessageBodyKey(accountConfigId, accountId, messageId);
+            const fullPath = join(basePath, key);
+            const buffer = await readFile(fullPath).catch((error) => {
+                if (isStorageNotFoundError(error))
+                    return null;
+                throw error;
+            });
+            if (!buffer)
+                return null;
+            if (buffer[0] === 0x1f && buffer[1] === 0x8b) {
+                return Readable.from(buffer).pipe(createGunzip());
+            }
+            return Readable.from(buffer);
+        },
+        retrieve,
+        exists,
+        delete: del,
+        storeExportArchiveStream: async (accountConfigId, exportRequestId, body) => {
+            const key = buildExportArchiveKey(accountConfigId, exportRequestId);
+            const fullPath = join(basePath, key);
+            await mkdir(dirname(fullPath), { recursive: true });
+            await pipeline(body, createWriteStream(fullPath));
+            return key;
+        },
+        getPresignedDownloadUrl: (_key, _expiresInSeconds) => {
+            throw new Error("getPresignedDownloadUrl is not supported on the filesystem backend");
+        },
+    };
+};

package/dist/backends/s3.d.ts

@@ -0,0 +1,3 @@
+import { type S3Client } from "@aws-sdk/client-s3";
+import type { StorageService } from "../storage.js";
+export declare const createS3StorageService: (client: S3Client, bucketName: string) => StorageService;