@remit/storage-service 0.0.16 → 0.0.17

package/dist/backends/filesystem.js

@@ -1,5 +1,5 @@
 import { createWriteStream } from "node:fs";
-import { mkdir, readdir, readFile, rm, stat, unlink, writeFile, } from "node:fs/promises";
+import { mkdir, readdir, readFile, rm, rmdir, stat, unlink, writeFile, } from "node:fs/promises";
 import { dirname, join, resolve } from "node:path";
 import { Readable } from "node:stream";
 import { pipeline } from "node:stream/promises";
@@ -10,6 +10,17 @@ import { UPLOAD_ROUTE_PREFIX, UPLOAD_URL_SIGNING_LABEL, } from "../outbox-upload
 import { signStoragePath } from "../signed-path.js";
 import { buildBodyPartKey, buildDeduplicatedKey, buildExportArchiveKey, buildMessageBodyKey, buildOutboxAttachmentKey, buildOutboxAttachmentPrefix, buildParsedBodyKey, computeChecksum, isStorageNotFoundError, } from "../storage.js";
 const gzipAsync = promisify(gzip);
+const DIRECTORY_IN_USE = new Set(["ENOENT", "ENOTEMPTY", "EEXIST"]);
+const keepDirectoryInUse = (error) => {
+    if (typeof error === "object" &&
+        error !== null &&
+        "code" in error &&
+        typeof error.code === "string" &&
+        DIRECTORY_IN_USE.has(error.code)) {
+        return;
+    }
+    throw error;
+};
 export const createFilesystemStorageService = (basePath, uploadUrls) => {
     const storeInternal = async (params) => {
         const { key, content, compress = true } = params;
@@ -173,6 +184,14 @@ export const createFilesystemStorageService = (basePath, uploadUrls) => {
     const deletePrefix = async (prefix) => {
         await rm(join(basePath, prefix), { recursive: true, force: true });
     };
+    const deleteKeys = async (keys) => {
+        const paths = keys.map((key) => join(basePath, key));
+        await Promise.all(paths.map((path) => rm(path, { force: true })));
+        const directories = [...new Set(paths.map((path) => dirname(path)))].sort((a, b) => b.length - a.length);
+        for (const directory of directories) {
+            await rmdir(directory).catch(keepDirectoryInUse);
+        }
+    };
     const storeDeduplicated = (params) => {
         const { accountConfigId, accountId, content } = params;
         const checksumSha256 = computeChecksum(content);
@@ -245,6 +264,7 @@ export const createFilesystemStorageService = (basePath, uploadUrls) => {
         listOutboxDraftsWithAttachments,
         deleteOutboxAttachments,
         deletePrefix,
+        deleteKeys,
         deleteOutboxAttachment,
         statOutboxAttachment,
         retrieveOutboxAttachment,

package/dist/backends/s3.js

@@ -7,6 +7,7 @@ import { ContentEncoding, StorageType } from "@remit/domain-enums";
 import { buildBodyPartKey, buildDeduplicatedKey, buildExportArchiveKey, buildMessageBodyKey, buildOutboxAttachmentKey, buildOutboxAttachmentPrefix, buildParsedBodyKey, computeChecksum, isStorageNotFoundError, } from "../storage.js";
 import { parseStorageUri } from "../uri.js";
 const gzipAsync = promisify(gzip);
+const STORAGE_DELETE_BATCH_SIZE = 1000;
 export const createS3StorageService = (client, bucketName) => {
     const storeInternal = async (params) => {
         const { key, content, contentType, compress = true } = params;
@@ -216,6 +217,19 @@ export const createS3StorageService = (client, bucketName) => {
                 : undefined;
         } while (continuationToken);
     };
+    const deleteKeys = async (keys) => {
+        for (let start = 0; start < keys.length; start += STORAGE_DELETE_BATCH_SIZE) {
+            const batch = keys.slice(start, start + STORAGE_DELETE_BATCH_SIZE);
+            const response = await client.send(new DeleteObjectsCommand({
+                Bucket: bucketName,
+                Delete: { Objects: batch.map((Key) => ({ Key })), Quiet: true },
+            }));
+            const failed = response.Errors ?? [];
+            if (failed.length > 0) {
+                throw new Error(`S3 refused to delete ${failed.length} of ${batch.length} objects, first ${failed[0].Key}: ${failed[0].Code} ${failed[0].Message}`);
+            }
+        }
+    };
     const deleteOutboxAttachments = (accountConfigId, accountId, outboxMessageId) => deletePrefix(buildOutboxAttachmentPrefix(accountConfigId, accountId, outboxMessageId));
     const storeDeduplicated = (params) => {
         const { accountConfigId, accountId, content, contentType } = params;
@@ -359,6 +373,7 @@ export const createS3StorageService = (client, bucketName) => {
         listOutboxDraftsWithAttachments,
         deleteOutboxAttachments,
         deletePrefix,
+        deleteKeys,
         deleteOutboxAttachment,
         statOutboxAttachment,
         retrieveOutboxAttachment,

package/dist/index.d.ts

@@ -1,5 +1,5 @@
 export { type OutboxSweepResult, sweepAbandonedOutboxAttachments, } from "./outbox-sweep.js";
 export { type AuthorizeUploadInput, type AuthorizeUploadResult, authorizeUploadRequest, UPLOAD_ROUTE_PREFIX, UPLOAD_URL_SIGNING_LABEL, UPLOAD_URL_TTL_SECONDS, } from "./outbox-upload-url.js";
 export { type SignedPathFailure, type SignedPathResult, signStoragePath, verifyStoragePath, } from "./signed-path.js";
-export { buildBodyPartKey, buildDeduplicatedKey, buildExportArchiveKey, buildMessageBodyKey, buildOutboxAttachmentKey, buildOutboxAttachmentPrefix, buildParsedBodyKey, type ContentEncodingValue, type CreateOutboxAttachmentUploadUrlParams, computeChecksum, createMockStorageService, isStorageNotFoundError, messageObjectPrefixOf, type OutboxAttachmentListItem, type OutboxAttachmentUploadTarget, type ParsedAttachmentMeta, type ParsedBody, type ParsedContentStorageKey, type ParsedOutboxAttachmentKey, parseContentStorageKey, parseOutboxAttachmentKey, type StorageReference, type StorageService, type StorageTypeValue, type StoreBodyPartParams, type StoreDeduplicatedParams, type StoreMessageBodyParams, type StoreOutboxAttachmentParams, type StoreParsedBodyParams, } from "./storage.js";
+export { buildBodyPartKey, buildDeduplicatedKey, buildExportArchiveKey, buildMessageBodyKey, buildOutboxAttachmentKey, buildOutboxAttachmentPrefix, buildParsedBodyKey, type ContentEncodingValue, type CreateOutboxAttachmentUploadUrlParams, computeChecksum, createMockStorageService, isStorageNotFoundError, MATERIALIZED_SENTINEL_PART_PATH, MATERIALIZED_SENTINEL_V2_PART_PATH, type MessageObjectKeys, messageObjectKeysOf, type OutboxAttachmentListItem, type OutboxAttachmentUploadTarget, type ParsedAttachmentMeta, type ParsedBody, type ParsedContentStorageKey, type ParsedOutboxAttachmentKey, parseContentStorageKey, parseOutboxAttachmentKey, type StorageReference, type StorageService, type StorageTypeValue, type StoreBodyPartParams, type StoreDeduplicatedParams, type StoreMessageBodyParams, type StoreOutboxAttachmentParams, type StoreParsedBodyParams, } from "./storage.js";
 export { buildStorageUri, type ParsedStorageUri, parseStorageUri, } from "./uri.js";

package/dist/index.js

@@ -1,5 +1,5 @@
 export { sweepAbandonedOutboxAttachments, } from "./outbox-sweep.js";
 export { authorizeUploadRequest, UPLOAD_ROUTE_PREFIX, UPLOAD_URL_SIGNING_LABEL, UPLOAD_URL_TTL_SECONDS, } from "./outbox-upload-url.js";
 export { signStoragePath, verifyStoragePath, } from "./signed-path.js";
-export { buildBodyPartKey, buildDeduplicatedKey, buildExportArchiveKey, buildMessageBodyKey, buildOutboxAttachmentKey, buildOutboxAttachmentPrefix, buildParsedBodyKey, computeChecksum, createMockStorageService, isStorageNotFoundError, messageObjectPrefixOf, parseContentStorageKey, parseOutboxAttachmentKey, } from "./storage.js";
+export { buildBodyPartKey, buildDeduplicatedKey, buildExportArchiveKey, buildMessageBodyKey, buildOutboxAttachmentKey, buildOutboxAttachmentPrefix, buildParsedBodyKey, computeChecksum, createMockStorageService, isStorageNotFoundError, MATERIALIZED_SENTINEL_PART_PATH, MATERIALIZED_SENTINEL_V2_PART_PATH, messageObjectKeysOf, parseContentStorageKey, parseOutboxAttachmentKey, } from "./storage.js";
 export { buildStorageUri, parseStorageUri, } from "./uri.js";

package/dist/storage.d.ts

@@ -195,6 +195,11 @@ export interface StorageService {
      * in `/`. Deleting a prefix that holds nothing is a no-op.
      */
     deletePrefix(keyPrefix: string): Promise<void>;
+    /**
+     * Delete the objects at `keys`, issuing one backend call per 1000 keys and
+     * no listing. A key that holds nothing is skipped.
+     */
+    deleteKeys(keys: readonly string[]): Promise<void>;
     /**
      * Stream a finished export archive (already-compressed ZIP) to the export
      * key for the given config + request, uncompressed at rest. The body is
@@ -245,7 +250,35 @@ export interface ParsedContentStorageKey {
  * above). Returns null when the shape doesn't match.
  */
 export declare const parseContentStorageKey: (storageKey: string) => ParsedContentStorageKey | null;
-export declare const messageObjectPrefixOf: (bodyStorageUri: string) => string;
+/**
+ * Sentinel object written under a message's parts prefix once lazy
+ * materialization has stored EVERY leaf. The read path checks it first with a
+ * single HEAD and early-returns, skipping the body.eml GET, the MIME re-parse,
+ * and the per-leaf HEADs on every warm re-open.
+ *
+ * Deliberately an S3-only marker (no Message-row flag). A leading dot keeps it
+ * out of any real IMAP section path (those are dot-separated digits).
+ */
+export declare const MATERIALIZED_SENTINEL_PART_PATH = ".materialized";
+/**
+ * The sentinel for a message carrying a forwarded message. Those parts were
+ * once stored under a layout in which the forwarded message's root overwrote
+ * its own row, so a `.materialized` written then vouches for objects at the
+ * wrong paths. Its own name makes every such message materialize once more,
+ * overwriting what is there; no other message is touched.
+ */
+export declare const MATERIALIZED_SENTINEL_V2_PART_PATH = ".materialized.2";
+export interface MessageObjectKeys {
+    own: string[];
+    shared: string[];
+}
+/**
+ * The storage keys a message's objects live at. `own` sit under the message's
+ * own id and no other message reads them. `shared` is the raw body
+ * `bodyStorageKey` names, which a copy shares with its source and every other
+ * copy, so it goes only once no row names it.
+ */
+export declare const messageObjectKeysOf: (bodyStorageKey: string, messageId: string, partPaths: Iterable<string>) => MessageObjectKeys;
 export declare const computeChecksum: (content: Buffer) => string;
 export declare const buildExportArchiveKey: (accountConfigId: string, exportRequestId: string) => string;
 export declare const isStorageNotFoundError: (error: unknown) => boolean;

package/dist/storage.js

@@ -7,9 +7,12 @@ import { parseStorageUri } from "./uri.js";
 // the edge can verify cross-tenant isolation at the URL prefix without
 // loading any database record. The inner `accountId` distinguishes between
 // multiple IMAP-account records owned by the same tenant.
-export const buildMessageBodyKey = (accountConfigId, accountId, messageId) => `accounts/${accountConfigId}/${accountId}/messages/${messageId}/body.eml`;
-export const buildParsedBodyKey = (accountConfigId, accountId, messageId) => `accounts/${accountConfigId}/${accountId}/messages/${messageId}/parsed.json.gz`;
-export const buildBodyPartKey = (accountConfigId, accountId, messageId, partPath) => `accounts/${accountConfigId}/${accountId}/messages/${messageId}/parts/${partPath}`;
+const MESSAGE_BODY_OBJECT = "body.eml";
+const PARSED_BODY_OBJECT = "parsed.json.gz";
+const BODY_PARTS_PREFIX = "parts/";
+export const buildMessageBodyKey = (accountConfigId, accountId, messageId) => `accounts/${accountConfigId}/${accountId}/messages/${messageId}/${MESSAGE_BODY_OBJECT}`;
+export const buildParsedBodyKey = (accountConfigId, accountId, messageId) => `accounts/${accountConfigId}/${accountId}/messages/${messageId}/${PARSED_BODY_OBJECT}`;
+export const buildBodyPartKey = (accountConfigId, accountId, messageId, partPath) => `accounts/${accountConfigId}/${accountId}/messages/${messageId}/${BODY_PARTS_PREFIX}${partPath}`;
 export const buildOutboxAttachmentPrefix = (accountConfigId, accountId, outboxMessageId) => `accounts/${accountConfigId}/${accountId}/outbox/${outboxMessageId}/attachments/`;
 export const buildOutboxAttachmentKey = (accountConfigId, accountId, outboxMessageId, outboxAttachmentId) => `${buildOutboxAttachmentPrefix(accountConfigId, accountId, outboxMessageId)}${outboxAttachmentId}`;
 // Every segment is a generated id, so the shape is closed rather than
@@ -52,13 +55,49 @@ export const parseContentStorageKey = (storageKey) => {
         messageId: match[3],
     };
 };
-const MESSAGE_OBJECT_PREFIX = /(?:^|\/)(accounts\/[^/]+\/(?:[^/]+\/)?messages\/[^/]+\/)[^/]+$/;
-export const messageObjectPrefixOf = (bodyStorageUri) => {
-    const match = parseStorageUri(bodyStorageUri).storageKey.match(MESSAGE_OBJECT_PREFIX);
+/**
+ * Sentinel object written under a message's parts prefix once lazy
+ * materialization has stored EVERY leaf. The read path checks it first with a
+ * single HEAD and early-returns, skipping the body.eml GET, the MIME re-parse,
+ * and the per-leaf HEADs on every warm re-open.
+ *
+ * Deliberately an S3-only marker (no Message-row flag). A leading dot keeps it
+ * out of any real IMAP section path (those are dot-separated digits).
+ */
+export const MATERIALIZED_SENTINEL_PART_PATH = ".materialized";
+/**
+ * The sentinel for a message carrying a forwarded message. Those parts were
+ * once stored under a layout in which the forwarded message's root overwrote
+ * its own row, so a `.materialized` written then vouches for objects at the
+ * wrong paths. Its own name makes every such message materialize once more,
+ * overwriting what is there; no other message is touched.
+ */
+export const MATERIALIZED_SENTINEL_V2_PART_PATH = ".materialized.2";
+const MESSAGE_BODY_KEY = /(?:^|\/)(accounts\/[^/]+\/(?:[^/]+\/)?messages\/)([^/]+)\/([^/]+)$/;
+/**
+ * The storage keys a message's objects live at. `own` sit under the message's
+ * own id and no other message reads them. `shared` is the raw body
+ * `bodyStorageKey` names, which a copy shares with its source and every other
+ * copy, so it goes only once no row names it.
+ */
+export const messageObjectKeysOf = (bodyStorageKey, messageId, partPaths) => {
+    const match = parseStorageUri(bodyStorageKey).storageKey.match(MESSAGE_BODY_KEY);
     if (!match) {
-        throw new Error(`${bodyStorageUri} is not a message body key; its objects cannot be found to delete`);
+        throw new Error(`${bodyStorageKey} is not a message body key; its objects cannot be found to delete`);
     }
-    return match[1];
+    const [, messages, ownerId, body] = match;
+    const own = `${messages}${messageId}/`;
+    return {
+        own: [
+            `${own}${PARSED_BODY_OBJECT}`,
+            ...[
+                MATERIALIZED_SENTINEL_PART_PATH,
+                MATERIALIZED_SENTINEL_V2_PART_PATH,
+                ...partPaths,
+            ].map((partPath) => `${own}${BODY_PARTS_PREFIX}${partPath}`),
+        ],
+        shared: [`${messages}${ownerId}/${body}`],
+    };
 };
 export const computeChecksum = (content) => createHash("sha256").update(content).digest("hex");
 export const buildExportArchiveKey = (accountConfigId, exportRequestId) => `exports/${accountConfigId}/${exportRequestId}/export.zip`;
@@ -147,6 +186,10 @@ export const createMockStorageService = () => {
                 storage.delete(uri);
         }
     };
+    const deleteKeys = async (keys) => {
+        for (const key of keys)
+            storage.delete(`${MOCK_ROOT}${key}`);
+    };
     const deleteOutboxAttachments = (accountConfigId, accountId, outboxMessageId) => deletePrefix(buildOutboxAttachmentPrefix(accountConfigId, accountId, outboxMessageId));
     const storeDeduplicated = async (params) => {
         const { accountConfigId, accountId, content } = params;
@@ -194,6 +237,7 @@ export const createMockStorageService = () => {
         listOutboxDraftsWithAttachments,
         deleteOutboxAttachments,
         deletePrefix,
+        deleteKeys,
         deleteOutboxAttachment,
         statOutboxAttachment,
         retrieveOutboxAttachment,

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@remit/storage-service",
-	"version": "0.0.16",
+	"version": "0.0.17",
 	"type": "module",
 	"main": "dist/index.js",
 	"types": "dist/index.d.ts",