@remit/storage-service 0.0.13 → 0.0.15

package/dist/backends/filesystem.js

@@ -1,14 +1,15 @@
-import { createHash } from "node:crypto";
 import { createWriteStream } from "node:fs";
 import { mkdir, readdir, readFile, rm, stat, unlink, writeFile, } from "node:fs/promises";
 import { dirname, join, resolve } from "node:path";
-import { PassThrough, Readable } from "node:stream";
+import { Readable } from "node:stream";
 import { pipeline } from "node:stream/promises";
-import { createGunzip, createGzip, gunzipSync, gzipSync } from "node:zlib";
+import { promisify } from "node:util";
+import { createGunzip, gunzipSync, gzip } from "node:zlib";
 import { ContentEncoding, StorageType } from "@remit/domain-enums";
 import { UPLOAD_ROUTE_PREFIX, UPLOAD_URL_SIGNING_LABEL, } from "../outbox-upload-url.js";
 import { signStoragePath } from "../signed-path.js";
 import { buildBodyPartKey, buildDeduplicatedKey, buildExportArchiveKey, buildMessageBodyKey, buildOutboxAttachmentKey, buildOutboxAttachmentPrefix, buildParsedBodyKey, computeChecksum, isStorageNotFoundError, } from "../storage.js";
+const gzipAsync = promisify(gzip);
 export const createFilesystemStorageService = (basePath, uploadUrls) => {
     const storeInternal = async (params) => {
         const { key, content, compress = true } = params;
@@ -16,7 +17,7 @@ export const createFilesystemStorageService = (basePath, uploadUrls) => {
         const contentEncoding = compress
             ? ContentEncoding.Gzip
             : ContentEncoding.None;
-        const body = compress ? gzipSync(content) : content;
+        const body = compress ? await gzipAsync(content) : content;
         const fullPath = join(basePath, key);
         await mkdir(dirname(fullPath), { recursive: true });
         await writeFile(fullPath, body);
@@ -37,26 +38,6 @@ export const createFilesystemStorageService = (basePath, uploadUrls) => {
             content,
         });
     };
-    const storeMessageBodyStream = async (params) => {
-        const { accountConfigId, accountId, messageId, content } = params;
-        const key = buildMessageBodyKey(accountConfigId, accountId, messageId);
-        const fullPath = join(basePath, key);
-        await mkdir(dirname(fullPath), { recursive: true });
-        const hash = createHash("sha256");
-        const hashTap = new PassThrough();
-        hashTap.on("data", (chunk) => hash.update(chunk));
-        await pipeline(content, hashTap, createGzip(), createWriteStream(fullPath));
-        const { size } = await stat(fullPath);
-        return {
-            uri: `file://${fullPath}`,
-            storageType: StorageType.Filesystem,
-            storageLocation: basePath,
-            storageKey: key,
-            sizeBytes: size,
-            checksumSha256: hash.digest("hex"),
-            contentEncoding: ContentEncoding.Gzip,
-        };
-    };
     const storeBodyPart = (params) => {
         const { accountConfigId, accountId, messageId, partPath, content } = params;
         return storeInternal({
@@ -254,7 +235,6 @@ export const createFilesystemStorageService = (basePath, uploadUrls) => {
     };
     return {
         storeMessageBody,
-        storeMessageBodyStream,
         storeBodyPart,
         bodyPartExists,
         retrieveBodyPart,
@@ -309,6 +289,9 @@ export const createFilesystemStorageService = (basePath, uploadUrls) => {
             await pipeline(body, createWriteStream(fullPath));
             return key;
         },
+        deleteExportArchive: (accountConfigId, exportRequestId) => rm(join(basePath, buildExportArchiveKey(accountConfigId, exportRequestId)), {
+            force: true,
+        }),
         getPresignedDownloadUrl: (_key, _expiresInSeconds) => {
             throw new Error("getPresignedDownloadUrl is not supported on the filesystem backend");
         },

package/dist/backends/s3.js

@@ -1,13 +1,12 @@
-import { createHash } from "node:crypto";
-import { PassThrough } from "node:stream";
-import { pipeline } from "node:stream/promises";
-import { createGunzip, createGzip, gunzipSync, gzipSync } from "node:zlib";
+import { promisify } from "node:util";
+import { createGunzip, gunzipSync, gzip } from "node:zlib";
 import { DeleteObjectCommand, DeleteObjectsCommand, GetObjectCommand, HeadObjectCommand, ListObjectsV2Command, PutObjectCommand, } from "@aws-sdk/client-s3";
 import { Upload } from "@aws-sdk/lib-storage";
 import { getSignedUrl } from "@aws-sdk/s3-request-presigner";
 import { ContentEncoding, StorageType } from "@remit/domain-enums";
 import { buildBodyPartKey, buildDeduplicatedKey, buildExportArchiveKey, buildMessageBodyKey, buildOutboxAttachmentKey, buildOutboxAttachmentPrefix, buildParsedBodyKey, computeChecksum, isStorageNotFoundError, } from "../storage.js";
 import { parseStorageUri } from "../uri.js";
+const gzipAsync = promisify(gzip);
 export const createS3StorageService = (client, bucketName) => {
     const storeInternal = async (params) => {
         const { key, content, contentType, compress = true } = params;
@@ -15,7 +14,7 @@ export const createS3StorageService = (client, bucketName) => {
         const contentEncoding = compress
             ? ContentEncoding.Gzip
             : ContentEncoding.None;
-        const body = compress ? gzipSync(content) : content;
+        const body = compress ? await gzipAsync(content) : content;
         await client.send(new PutObjectCommand({
             Bucket: bucketName,
             Key: key,
@@ -41,47 +40,6 @@ export const createS3StorageService = (client, bucketName) => {
             contentType: "message/rfc822",
         });
     };
-    const storeMessageBodyStream = async (params) => {
-        const { accountConfigId, accountId, messageId, content } = params;
-        const key = buildMessageBodyKey(accountConfigId, accountId, messageId);
-        // Hash the logical (pre-gzip) bytes as they flow, gzip them, and feed
-        // the gzipped stream to a multipart Upload — the whole body is never
-        // held in memory. `Upload` buffers only one part at a time.
-        const hash = createHash("sha256");
-        const hashTap = new PassThrough();
-        hashTap.on("data", (chunk) => hash.update(chunk));
-        const gzip = createGzip();
-        const uploadBody = new PassThrough();
-        let storedBytes = 0;
-        uploadBody.on("data", (chunk) => {
-            storedBytes += chunk.length;
-        });
-        const upload = new Upload({
-            client,
-            params: {
-                Bucket: bucketName,
-                Key: key,
-                Body: uploadBody,
-                ContentType: "message/rfc822",
-                ContentEncoding: "gzip",
-            },
-        });
-        const pumped = pipeline(content, hashTap, gzip, uploadBody);
-        await Promise.all([upload.done(), pumped]).catch(async (error) => {
-            // Abort the multipart upload so a failed pump doesn't leak parts.
-            await upload.abort().catch(() => { });
-            throw error;
-        });
-        return {
-            uri: `s3://${bucketName}/${key}`,
-            storageType: StorageType.S3,
-            storageLocation: bucketName,
-            storageKey: key,
-            sizeBytes: storedBytes,
-            checksumSha256: hash.digest("hex"),
-            contentEncoding: ContentEncoding.Gzip,
-        };
-    };
     const storeBodyPart = (params) => {
         const { accountConfigId, accountId, messageId, partPath, content, contentType, } = params;
         return storeInternal({
@@ -384,10 +342,15 @@ export const createS3StorageService = (client, bucketName) => {
         });
         return key;
     };
+    const deleteExportArchive = async (accountConfigId, exportRequestId) => {
+        await client.send(new DeleteObjectCommand({
+            Bucket: bucketName,
+            Key: buildExportArchiveKey(accountConfigId, exportRequestId),
+        }));
+    };
     const getPresignedDownloadUrl = (key, expiresInSeconds) => getSignedUrl(client, new GetObjectCommand({ Bucket: bucketName, Key: key }), { expiresIn: expiresInSeconds });
     return {
         storeMessageBody,
-        storeMessageBodyStream,
         storeBodyPart,
         bodyPartExists,
         retrieveBodyPart,
@@ -408,6 +371,7 @@ export const createS3StorageService = (client, bucketName) => {
         exists,
         delete: del,
         storeExportArchiveStream,
+        deleteExportArchive,
         getPresignedDownloadUrl,
     };
 };

package/dist/storage.d.ts

@@ -25,14 +25,6 @@ export interface StoreMessageBodyParams {
     messageId: string;
     content: Buffer;
 }
-/** Parameters for streaming a message body (raw RFC822 content) to storage */
-export interface StoreMessageBodyStreamParams {
-    accountConfigId: string;
-    accountId: string;
-    messageId: string;
-    /** Readable stream of the raw message bytes; streamed to storage, never buffered whole. */
-    content: Readable;
-}
 /** Parameters for storing a body part (attachment, inline content) */
 export interface StoreBodyPartParams {
     accountConfigId: string;
@@ -115,12 +107,6 @@ export interface StoreParsedBodyParams {
 export interface StorageService {
     /** Store a message body (raw RFC822 content) */
     storeMessageBody(params: StoreMessageBodyParams): Promise<StorageReference>;
-    /**
-     * Store a message body from a readable stream, never buffering the whole
-     * body. Use this on the sync hot path so a ranged FETCH streams straight to
-     * storage.
-     */
-    storeMessageBodyStream(params: StoreMessageBodyStreamParams): Promise<StorageReference>;
     /** Store a body part (attachment, inline content) */
     storeBodyPart(params: StoreBodyPartParams): Promise<StorageReference>;
     /**
@@ -211,6 +197,11 @@ export interface StorageService {
      * storage key, which `getPresignedDownloadUrl` can then sign.
      */
     storeExportArchiveStream(accountConfigId: string, exportRequestId: string, body: Readable): Promise<string>;
+    /**
+     * Delete the export archive of the given config + request. Deleting a
+     * missing archive is a no-op.
+     */
+    deleteExportArchive(accountConfigId: string, exportRequestId: string): Promise<void>;
     /**
      * Generate a presigned URL that grants temporary GET access to a raw S3 key.
      * Use for export archives — the key must already exist in the bucket.

package/dist/storage.js

@@ -85,14 +85,6 @@ export const createMockStorageService = () => {
         const { accountConfigId, accountId, messageId, content } = params;
         return storeInternal(buildMessageBodyKey(accountConfigId, accountId, messageId), content);
     };
-    const storeMessageBodyStream = async (params) => {
-        const { accountConfigId, accountId, messageId, content } = params;
-        const chunks = [];
-        for await (const chunk of content) {
-            chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk));
-        }
-        return storeInternal(buildMessageBodyKey(accountConfigId, accountId, messageId), Buffer.concat(chunks));
-    };
     const storeBodyPart = async (params) => {
         const { accountConfigId, accountId, messageId, partPath, content } = params;
         return storeInternal(buildBodyPartKey(accountConfigId, accountId, messageId, partPath), content);
@@ -184,7 +176,6 @@ export const createMockStorageService = () => {
     };
     return {
         storeMessageBody,
-        storeMessageBodyStream,
         storeBodyPart,
         bodyPartExists,
         retrieveBodyPart,
@@ -223,6 +214,9 @@ export const createMockStorageService = () => {
             storage.set(`mock://${key}`, Buffer.concat(chunks));
             return key;
         },
+        deleteExportArchive: async (accountConfigId, exportRequestId) => {
+            storage.delete(`mock://${buildExportArchiveKey(accountConfigId, exportRequestId)}`);
+        },
         getPresignedDownloadUrl: async (key, _expiresInSeconds) => `mock://presigned/${key}`,
     };
 };

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@remit/storage-service",
-	"version": "0.0.13",
+	"version": "0.0.15",
 	"type": "module",
 	"main": "dist/index.js",
 	"types": "dist/index.d.ts",