@visulima/storage 1.0.0-alpha.1 → 1.0.0-alpha.3

package/dist/packem_shared/s3-base-storage-C1DIHTYY.js

@@ -0,0 +1,510 @@
+import { Readable } from 'node:stream';
+import { p as partMatch, d as detectFileTypeFromStream, g as getFileStatus } from './part-match-BMNqHDYD.js';
+import { throwErrorCode, ERRORS } from './ERRORS-D0apMqnc.js';
+import { p as parseDuration, B as BaseStorage, b as parseBytes, t as toMilliseconds } from './storage-C6NSXb8h.js';
+import { createRetryWrapper } from './isRetryableError-Dycp7127.js';
+import { L as LocalMetaStorage } from './local-meta-storage-By-3SNBI.js';
+import { u as updateSize } from './update-size-CCGm6i1J.js';
+import { h as hasContent } from './has-content-CY66ehMK.js';
+import { i as isExpired } from './is-expired-CTThU1q5.js';
+
+const mapValues = (object, function_) => {
+  const result = {};
+  Object.keys(object).forEach((key) => {
+    result[key] = function_(object[key]);
+  });
+  return result;
+};
+
+const toSeconds = (value) => {
+  if (value === Number(value)) {
+    return value;
+  }
+  return (parseDuration(value) ?? 0) / 1e3;
+};
+
+const MIN_PART_SIZE = 5 * 1024 * 1024;
+const PART_SIZE = 16 * 1024 * 1024;
+class S3BaseStorage extends BaseStorage {
+  checksumTypes = ["md5", "crc32", "crc32c", "sha1", "sha256"];
+  bucket;
+  meta;
+  /**
+   * S3 multipart upload does not allow more than 10000 parts.
+   */
+  MAX_PARTS = 1e4;
+  partSize;
+  retry;
+  constructor(config) {
+    super(config);
+    this.bucket = config.bucket;
+    this.partSize = typeof config.partSize === "string" ? parseBytes(config.partSize) : config.partSize || PART_SIZE;
+    if (this.partSize < MIN_PART_SIZE) {
+      throw new Error("Minimum allowed partSize value is 5MB");
+    }
+    const retryConfig = {
+      backoffMultiplier: 2,
+      initialDelay: 1e3,
+      maxDelay: 3e4,
+      maxRetries: 3,
+      retryableStatusCodes: [408, 429, 500, 502, 503, 504],
+      shouldRetry: (error) => {
+        const errorWithMetadata = error;
+        if (errorWithMetadata.statusCode && [408, 429, 500, 502, 503, 504].includes(errorWithMetadata.statusCode)) {
+          return true;
+        }
+        return errorWithMetadata.retryable ?? false;
+      },
+      ...config.retryConfig
+    };
+    this.retry = createRetryWrapper(retryConfig);
+    const { metaStorage, metaStorageConfig } = config;
+    if (metaStorage) {
+      this.meta = metaStorage;
+    } else {
+      const metaConfig = { ...config, ...metaStorageConfig, logger: this.logger };
+      const localMeta = "directory" in metaConfig;
+      if (localMeta) {
+        this.logger?.debug("Using local meta storage");
+        this.meta = new LocalMetaStorage(metaConfig);
+      } else {
+        this.meta = new LocalMetaStorage(metaConfig);
+      }
+    }
+    this.isReady = false;
+    this.accessCheck().then(() => {
+      this.isReady = true;
+      return void 0;
+    }).catch((error) => {
+      this.logger?.error("Storage access check failed: %O", error);
+    });
+  }
+  /**
+   * Creates a new S3 multipart upload.
+   */
+  async create(config) {
+    return this.instrumentOperation("create", async () => {
+      const processedConfig = { ...config };
+      if (config.ttl) {
+        const ttlMs = typeof config.ttl === "string" ? toMilliseconds(config.ttl) : config.ttl;
+        if (ttlMs !== void 0) {
+          processedConfig.expiredAt = Date.now() + ttlMs;
+        }
+      }
+      const file = new (this.getFileClass())(processedConfig);
+      file.name = this.namingFunction(file);
+      await this.validate(file);
+      try {
+        const existing = await this.getMeta(file.id);
+        if (existing.bytesWritten >= 0) {
+          return existing;
+        }
+      } catch {
+      }
+      const s3Api = this.getS3Api();
+      let result;
+      try {
+        result = await this.retry(
+          () => s3Api.createMultipartUpload({
+            ACL: this.getAcl(),
+            Bucket: this.bucket,
+            ContentType: file.contentType,
+            Key: file.name,
+            Metadata: mapValues({ originalName: file.originalName, ...file.metadata }, (value) => encodeURI(String(value)))
+          })
+        );
+      } catch {
+        return throwErrorCode(ERRORS.FILE_ERROR, "s3 create upload error");
+      }
+      const { UploadId } = result || {};
+      if (!UploadId) {
+        return throwErrorCode(ERRORS.FILE_ERROR, "s3 create upload error");
+      }
+      file.UploadId = UploadId;
+      file.bytesWritten = 0;
+      if (this.config.clientDirectUpload) {
+        file.partSize ??= this.partSize;
+      }
+      await this.saveMeta(file);
+      file.status = "created";
+      await this.onCreate(file);
+      if (this.config.clientDirectUpload) {
+        return this.buildPresigned(file);
+      }
+      return file;
+    });
+  }
+  /**
+   * Writes data to an S3 multipart upload.
+   */
+  async write(part) {
+    return this.instrumentOperation("write", async () => {
+      let file;
+      if ("contentType" in part && "metadata" in part && !("body" in part) && !("start" in part)) {
+        file = part;
+      } else {
+        file = await this.getMeta(part.id);
+        await this.checkIfExpired(file);
+      }
+      if (file.status === "completed") {
+        return file;
+      }
+      if (typeof part.size === "number" && part.size > 0) {
+        updateSize(file, part.size);
+      }
+      if (!partMatch(part, file)) {
+        return throwErrorCode(ERRORS.FILE_CONFLICT);
+      }
+      if (this.config.clientDirectUpload) {
+        return this.buildPresigned(file);
+      }
+      file.Parts ??= await this.getParts(file);
+      file.bytesWritten = file.Parts.map((item) => item.Size || 0).reduce((p, c) => p + c, 0);
+      await this.lock(part.id);
+      try {
+        if (hasContent(part)) {
+          if (this.isUnsupportedChecksum(part.checksumAlgorithm)) {
+            return throwErrorCode(ERRORS.UNSUPPORTED_CHECKSUM_ALGORITHM);
+          }
+          if (file.Parts.length === 0 && (!file.contentType || file.contentType === "application/octet-stream")) {
+            try {
+              const readable = part.body instanceof Readable ? part.body : Readable.fromWeb(part.body);
+              const { fileType, stream: detectedStream } = await detectFileTypeFromStream(readable);
+              if (fileType?.mime) {
+                file.contentType = fileType.mime;
+              }
+              part.body = detectedStream;
+            } catch {
+            }
+          }
+          if (file.Parts.length > this.MAX_PARTS) {
+            throw new Error(`Exceeded ${this.MAX_PARTS} as part of the upload to ${this.bucket}.`);
+          }
+          const partNumber = file.Parts.length + 1;
+          const s3Api = this.getS3Api();
+          const uploadId = file.UploadId;
+          if (!uploadId) {
+            throw new Error("UploadId is required");
+          }
+          const { ETag } = await this.retry(
+            () => s3Api.uploadPart({
+              Body: part.body,
+              Bucket: this.bucket,
+              ContentLength: part.contentLength || 0,
+              Key: file.name,
+              PartNumber: partNumber,
+              UploadId: uploadId,
+              ...part.checksumAlgorithm === "md5" ? { ContentMD5: part.checksum } : {}
+            })
+          );
+          const uploadPart = { ETag, PartNumber: partNumber, Size: part.contentLength };
+          file.Parts = [...file.Parts, uploadPart];
+          file.bytesWritten += part.contentLength || 0;
+        }
+        this.cache.set(file.id, file);
+        file.status = getFileStatus(file);
+        if (file.status === "completed") {
+          const [completed] = await this.internalOnComplete(file);
+          delete file.Parts;
+          file.uri = completed.Location;
+          file.ETag = completed.ETag;
+        }
+      } finally {
+        await this.unlock(part.id);
+      }
+      return file;
+    });
+  }
+  /**
+   * Deletes an upload and its metadata.
+   */
+  async delete({ id }) {
+    return this.instrumentOperation("delete", async () => {
+      const file = await this.getMeta(id);
+      file.status = "deleted";
+      await Promise.all([this.deleteMeta(file.id), this.retry(() => this.abortMultipartUpload(file))]);
+      const deletedFile = { ...file };
+      await this.onDelete(deletedFile);
+      return deletedFile;
+    });
+  }
+  /**
+   * Copies an upload file to a new location.
+   */
+  async copy(name, destination, options) {
+    return this.instrumentOperation("copy", async () => {
+      const sourceFile = await this.getMeta(name);
+      const CopySource = `${this.bucket}/${name}`;
+      let Bucket = this.bucket;
+      let Key = destination;
+      if (destination.startsWith("/")) {
+        const [, bucketName, ...pathSegments] = destination.split("/");
+        Bucket = bucketName || this.bucket;
+        Key = pathSegments.join("/");
+      }
+      const s3Api = this.getS3Api();
+      await this.retry(
+        () => s3Api.copyObject({
+          Bucket,
+          CopySource,
+          Key,
+          ...options?.storageClass && { StorageClass: options.storageClass }
+        })
+      );
+      return { ...sourceFile, id: Key, name: Key };
+    });
+  }
+  /**
+   * Moves an upload file to a new location.
+   */
+  async move(name, destination) {
+    return this.instrumentOperation("move", async () => {
+      await this.copy(name, destination);
+      const s3Api = this.getS3Api();
+      await this.retry(() => s3Api.deleteObject({ Bucket: this.bucket, Key: name }));
+      return await this.getMeta(destination);
+    });
+  }
+  /**
+   * Lists files in the bucket.
+   */
+  async list(limit = 1e3) {
+    return this.instrumentOperation(
+      "list",
+      // eslint-disable-next-line sonarjs/cognitive-complexity
+      async () => {
+        const s3Api = this.getS3Api();
+        let parameters = {
+          Bucket: this.bucket,
+          MaxKeys: limit
+        };
+        const items = [];
+        let truncated = true;
+        while (truncated) {
+          try {
+            const response = await this.retry(() => s3Api.listObjectsV2(parameters));
+            for (const { Key, LastModified } of response?.Contents || []) {
+              if (Key !== void 0) {
+                const { Expires } = await this.retry(() => s3Api.headObject({ Bucket: this.bucket, Key }));
+                if (Expires && isExpired({ expiredAt: Expires })) {
+                  await this.delete({ id: Key });
+                } else {
+                  items.push({
+                    id: Key,
+                    ...LastModified && { createdAt: LastModified }
+                  });
+                }
+              }
+            }
+            truncated = response.IsTruncated || false;
+            if (truncated && response.NextContinuationToken) {
+              parameters = { ...parameters, ContinuationToken: response.NextContinuationToken };
+            }
+          } catch (error) {
+            const httpError = this.normalizeError(error instanceof Error ? error : new Error(String(error)));
+            await this.onError(httpError);
+            throw error;
+          }
+        }
+        return items;
+      },
+      { limit }
+    );
+  }
+  /**
+   * Checks if a file exists by verifying both metadata and the actual S3 object.
+   * Returns true only if both the metadata and the S3 object exist.
+   * @param query File query containing the file ID to check.
+   * @returns Promise resolving to true if both metadata and S3 object exist, false otherwise.
+   */
+  async exists({ id }) {
+    return this.instrumentOperation("exists", async () => {
+      try {
+        await this.getMeta(id);
+        const s3Api = this.getS3Api();
+        await this.retry(
+          () => s3Api.headObject({
+            Bucket: this.bucket,
+            Key: id
+          })
+        );
+        return true;
+      } catch (error) {
+        const errorWithMetadata = error;
+        if (errorWithMetadata.$metadata?.httpStatusCode === 404) {
+          return false;
+        }
+        return false;
+      }
+    });
+  }
+  /**
+   * Gets an uploaded file by ID.
+   */
+  async get({ id }) {
+    return this.instrumentOperation("get", async () => {
+      const s3Api = this.getS3Api();
+      const { Body, ContentLength, ContentType, ETag, Expires, LastModified, Metadata } = await this.retry(
+        () => s3Api.getObject({
+          Bucket: this.bucket,
+          Key: id
+        })
+      );
+      await this.checkIfExpired({ expiredAt: Expires });
+      const chunks = [];
+      if (Body) {
+        if (Body instanceof Readable) {
+          for await (const chunk of Body) {
+            chunks.push(chunk instanceof Uint8Array ? chunk : new Uint8Array(chunk));
+          }
+        } else {
+          const reader = Body.getReader();
+          while (true) {
+            const { done, value } = await reader.read();
+            if (done) {
+              break;
+            }
+            chunks.push(value);
+          }
+        }
+      }
+      const { originalName, ...meta } = Metadata || {};
+      return {
+        content: Buffer.concat(chunks),
+        contentType: ContentType,
+        ETag,
+        expiredAt: Expires,
+        id,
+        metadata: meta,
+        modifiedAt: LastModified,
+        name: id,
+        originalName: originalName || id,
+        size: Number(ContentLength)
+      };
+    });
+  }
+  /**
+   * Builds presigned URLs for client uploads.
+   */
+  async buildPresigned(file) {
+    const fileWithParts = file;
+    if (!fileWithParts.Parts?.length) {
+      fileWithParts.Parts = await this.getParts(file);
+    }
+    fileWithParts.bytesWritten = fileWithParts.Parts.map((item) => item.Size || 0).reduce((p, c) => p + c, 0);
+    file.status = getFileStatus(fileWithParts);
+    if (!fileWithParts.partsUrls?.length) {
+      fileWithParts.partsUrls = await this.getPartsPresignedUrls(file);
+    }
+    if (file.status === "completed") {
+      const [completed] = await this.internalOnComplete(file);
+      delete fileWithParts.Parts;
+      fileWithParts.uri = completed.Location;
+      return file;
+    }
+    return file;
+  }
+  /**
+   * Gets presigned URLs for all parts.
+   */
+  async getPartsPresignedUrls(file) {
+    file.partSize ??= this.partSize;
+    const uploadId = file.UploadId;
+    if (!uploadId) {
+      throw new Error("UploadId is required for getting presigned URLs");
+    }
+    const partsNumber = Math.trunc(file.size / this.partSize) + 1;
+    const promises = [];
+    const expiresIn = Math.trunc(toSeconds(this.config.expiration?.maxAge || "6hrs"));
+    const s3Api = this.getS3Api();
+    for (let index = 0; index < partsNumber; index++) {
+      promises.push(
+        s3Api.getPresignedUrl({
+          Bucket: this.bucket,
+          expiresIn,
+          Key: file.name,
+          PartNumber: index + 1,
+          UploadId: uploadId
+        })
+      );
+    }
+    return Promise.all(promises);
+  }
+  /**
+   * Gets parts for a multipart upload.
+   */
+  async getParts(file) {
+    const s3Api = this.getS3Api();
+    const uploadId = file.UploadId;
+    if (!uploadId) {
+      throw new Error("UploadId is required");
+    }
+    const { Parts = [] } = await this.retry(
+      () => s3Api.listParts({
+        Bucket: this.bucket,
+        Key: file.name,
+        UploadId: uploadId
+      })
+    );
+    return Parts;
+  }
+  /**
+   * Completes a multipart upload.
+   */
+  completeMultipartUpload(file) {
+    const s3Api = this.getS3Api();
+    const uploadId = file.UploadId;
+    if (!uploadId) {
+      throw new Error("UploadId is required");
+    }
+    const parts = file.Parts?.map(({ ETag, PartNumber }) => {
+      if (!ETag || !PartNumber) {
+        throw new Error("ETag and PartNumber are required");
+      }
+      return { ETag, PartNumber };
+    }) || [];
+    return this.retry(
+      () => s3Api.completeMultipartUpload({
+        Bucket: this.bucket,
+        Key: file.name,
+        Parts: parts,
+        UploadId: uploadId
+      })
+    );
+  }
+  /**
+   * Aborts a multipart upload.
+   */
+  async abortMultipartUpload(file) {
+    if (file.status === "completed") {
+      return;
+    }
+    try {
+      const s3Api = this.getS3Api();
+      const uploadId = file.UploadId;
+      if (!uploadId) {
+        return;
+      }
+      await this.retry(
+        () => s3Api.abortMultipartUpload({
+          Bucket: this.bucket,
+          Key: file.name,
+          UploadId: uploadId
+        })
+      );
+    } catch (error) {
+      this.logger?.error("abortMultipartUploadError: ", error);
+      const httpError = this.normalizeError(error instanceof Error ? error : new Error(String(error)));
+      await this.onError(httpError);
+    }
+  }
+  /**
+   * Internal onComplete handler.
+   */
+  internalOnComplete = async (file) => {
+    const [completed] = await Promise.all([this.completeMultipartUpload(file), this.deleteMeta(file.id)]);
+    return [completed, file];
+  };
+}
+
+export { S3BaseStorage as S };