importer-storage 1.0.16 → 1.0.18

package/dist/index.cjs

@@ -62,6 +62,7 @@ var import_axios = __toESM(require("axios"), 1);
 var import_jimp = require("jimp");
 var import_https_proxy_agent = require("https-proxy-agent");
 var import_perf_hooks = require("perf_hooks");
+var import_p_limit = __toESM(require("p-limit"), 1);
 (0, import_dotenv.config)();
 if (!process.env.AWS_EXECUTION_ENV && !process.env.AWS_ACCESS_KEY_ID) {
   throw new Error("'AWS_ACCESS_KEY_ID' not set");
@@ -92,110 +93,147 @@ if (!BUCKET_NAME) {
 function isHttp(url) {
   return url.startsWith("http://") || url.startsWith("https://");
 }
+var processRecord = (_0, _1, _2) => __async(null, [_0, _1, _2], function* ({
+  filePath,
+  key
+}, options, proxyAgent) {
+  const start = import_perf_hooks.performance.now();
+  if (options.logging) {
+    console.info(`
+[Processing Start] key=${key}, filePath=${filePath}`);
+  }
+  try {
+    const s3Key = `${options.savePath}/${key}`;
+    if (options.logging) {
+      console.info(`[S3 Key] ${s3Key}`);
+    }
+    let stream = void 0;
+    let buffer = void 0;
+    if (isHttp(filePath)) {
+      if (options.logging) {
+        console.info(`[Download Start] ${filePath}`);
+      }
+      const axiosResponse = yield import_axios.default.get(filePath, {
+        responseType: "arraybuffer",
+        headers: options.headers,
+        httpsAgent: proxyAgent
+      });
+      if (options.logging) {
+        console.info(`[Download Success] ${filePath}`);
+      }
+      buffer = Buffer.from(axiosResponse.data);
+    } else {
+      if (options.logging) {
+        console.info(`[Read from FS] ${filePath}`);
+      }
+      stream = (0, import_fs.createReadStream)(filePath);
+    }
+    const tagString = options.tags.map(
+      (tag) => `${encodeURIComponent(tag.split(":")[0])}=${encodeURIComponent(tag.split(":")[1])}`
+    ).join("&");
+    if (!stream && !buffer) {
+      if (options.logging) {
+        console.warn(`[Error] Neither stream nor buffer is set for ${key}`);
+      }
+      throw new Error("buffer or stream not set");
+    }
+    if (options.logging) {
+      console.info(`[Image Processing Start] ${key}`);
+    }
+    const image = buffer ? yield import_jimp.Jimp.fromBuffer(buffer) : yield import_jimp.Jimp.read(filePath);
+    if (options.logging) {
+      console.info(
+        `[Image Loaded] key=${key}, width=${image.width}, height=${image.height}`
+      );
+    }
+    const command = new import_client_s3.PutObjectCommand({
+      Body: buffer || stream,
+      Bucket: BUCKET_NAME,
+      Key: s3Key,
+      Tagging: tagString || void 0,
+      ContentType: `image/${extractFileInfo(key).extension}`,
+      ContentDisposition: "inline",
+      Metadata: {
+        "img-width": image.width.toString(),
+        "img-height": image.height.toString()
+      }
+    });
+    if (options.logging) {
+      console.info(`[Uploading] ${key} to S3...`);
+    }
+    yield s3.send(command);
+    if (options.logging) {
+      console.info(`[Upload Success] ${key}`);
+    }
+    return {
+      error: false,
+      key,
+      filePath,
+      uploadUrl: `https://${BUCKET_NAME}.s3.${process.env.AWS_REGION}.amazonaws.com/${s3Key}`
+    };
+  } catch (e) {
+    const error = e;
+    if (options.logging) {
+      console.error(`[Upload Error] key=${key}, error=${error.message}`);
+    }
+    return {
+      error: true,
+      key,
+      filePath,
+      message: error.message
+    };
+  } finally {
+    const end = import_perf_hooks.performance.now();
+    if (options.logging) {
+      console.info(
+        `[Processing End] key=${key}, duration=${(end - start).toFixed(0)}ms`
+      );
+    }
+  }
+});
 var S3Storage = class {
   saveFiles(keys, options) {
     return __async(this, null, function* () {
+      const parallelLimit = options.parallelLimit || 2;
+      const limit = (0, import_p_limit.default)(parallelLimit);
       const result = [];
       const errored = [];
       const proxyAgent = (options == null ? void 0 : options.proxyUrl) && new import_https_proxy_agent.HttpsProxyAgent(options == null ? void 0 : options.proxyUrl);
       if (options.logging) {
         console.info(`[SaveFiles] Start saving ${keys.length} file(s)`);
-        console.time(`[SaveFiles] Total time`);
+        console.time(`[SaveFiles] Total time ${parallelLimit}`);
       }
-      for (const { key, filePath } of keys) {
-        const start = import_perf_hooks.performance.now();
-        if (options.logging) {
-          console.info(`
-[Processing Start] key=${key}, filePath=${filePath}`);
-        }
-        try {
-          const s3Key = `${options.savePath}/${key}`;
-          if (options.logging) {
-            console.info(`[S3 Key] ${s3Key}`);
-          }
-          let stream = void 0;
-          let buffer = void 0;
-          if (isHttp(filePath)) {
+      yield Promise.all(
+        keys.map(
+          (record, index) => limit(() => __async(null, null, function* () {
             if (options.logging) {
-              console.info(`[Download Start] ${filePath}`);
+              console.info(
+                `Running p-limit task ${index}, limit: ${parallelLimit}`
+              );
             }
-            const axiosResponse = yield import_axios.default.get(filePath, {
-              responseType: "arraybuffer",
-              headers: options.headers,
-              httpsAgent: proxyAgent
-            });
-            if (options.logging) {
-              console.info(`[Download Success] ${filePath}`);
-            }
-            buffer = Buffer.from(axiosResponse.data);
-          } else {
-            if (options.logging) {
-              console.info(`[Read from FS] ${filePath}`);
-            }
-            stream = (0, import_fs.createReadStream)(filePath);
-          }
-          const tagString = options.tags.map(
-            (tag) => `${encodeURIComponent(tag.split(":")[0])}=${encodeURIComponent(tag.split(":")[1])}`
-          ).join("&");
-          if (!stream && !buffer) {
-            if (options.logging) {
-              console.warn(`[Error] Neither stream nor buffer is set for ${key}`);
-            }
-            throw new Error("buffer or stream not set");
-          }
-          if (options.logging) {
-            console.info(`[Image Processing Start] ${key}`);
-          }
-          const image = buffer ? yield import_jimp.Jimp.fromBuffer(buffer) : yield import_jimp.Jimp.read(filePath);
-          if (options.logging) {
-            console.info(
-              `[Image Loaded] key=${key}, width=${image.width}, height=${image.height}`
+            const processResult = yield processRecord(
+              record,
+              options,
+              proxyAgent || void 0
             );
-          }
-          const command = new import_client_s3.PutObjectCommand({
-            Body: buffer || stream,
-            Bucket: BUCKET_NAME,
-            Key: s3Key,
-            Tagging: tagString || void 0,
-            ContentType: `image/${extractFileInfo(key).extension}`,
-            ContentDisposition: "inline",
-            Metadata: {
-              "img-width": image.width.toString(),
-              "img-height": image.height.toString()
+            if (processResult.message) {
+              errored.push({
+                key: processResult.key,
+                filePath: processResult.filePath,
+                message: processResult.message
+              });
+            } else if (processResult.uploadUrl) {
+              result.push({
+                key: processResult.key,
+                filePath: processResult.filePath,
+                uploadUrl: processResult.uploadUrl
+              });
             }
-          });
-          result.push({
-            key,
-            filePath,
-            uploadUrl: `https://${BUCKET_NAME}.s3.${process.env.AWS_REGION}.amazonaws.com/${s3Key}`
-          });
-          if (options.logging) {
-            console.info(`[Uploading] ${key} to S3...`);
-          }
-          yield s3.send(command);
-          if (options.logging) {
-            console.info(`[Upload Success] ${key}`);
-          }
-        } catch (e) {
-          const error = e;
-          if (options.logging) {
-            console.error(`[Upload Error] key=${key}, error=${error.message}`);
-          }
-          errored.push({
-            key,
-            filePath,
-            message: error.message
-          });
-        }
-        const end = import_perf_hooks.performance.now();
-        if (options.logging) {
-          console.info(
-            `[Processing End] key=${key}, duration=${(end - start).toFixed(0)}ms`
-          );
-        }
-      }
+          }))
+        )
+      );
       if (options.logging) {
-        console.timeEnd(`[SaveFiles] Total time`);
+        console.timeEnd(`[SaveFiles] Total time ${parallelLimit}`);
         console.info(
           `[SaveFiles] Completed. Success: ${result.length}, Failed: ${errored.length}`
         );

package/dist/index.d.cts

@@ -25,17 +25,19 @@ interface IStorage {
     removeTag(savePath: string, tags: Array<StorageTag>): Promise<void>;
 }
 
+type Options = {
+    tags: Array<StorageTag>;
+    savePath: string;
+    headers?: Record<string, string>;
+    logging?: boolean;
+    proxyUrl?: string;
+    parallelLimit?: number;
+};
 declare class S3Storage implements IStorage {
     saveFiles(keys: Array<{
         key: string;
         filePath: string;
-    }>, options: {
-        tags: Array<StorageTag>;
-        savePath: string;
-        headers?: Record<string, string>;
-        logging?: boolean;
-        proxyUrl?: string;
-    }): Promise<{
+    }>, options: Options): Promise<{
         result: {
             key: string;
             filePath: string;

package/dist/index.d.ts

@@ -25,17 +25,19 @@ interface IStorage {
     removeTag(savePath: string, tags: Array<StorageTag>): Promise<void>;
 }
 
+type Options = {
+    tags: Array<StorageTag>;
+    savePath: string;
+    headers?: Record<string, string>;
+    logging?: boolean;
+    proxyUrl?: string;
+    parallelLimit?: number;
+};
 declare class S3Storage implements IStorage {
     saveFiles(keys: Array<{
         key: string;
         filePath: string;
-    }>, options: {
-        tags: Array<StorageTag>;
-        savePath: string;
-        headers?: Record<string, string>;
-        logging?: boolean;
-        proxyUrl?: string;
-    }): Promise<{
+    }>, options: Options): Promise<{
         result: {
             key: string;
             filePath: string;

package/dist/index.global.js

@@ -74,6 +74,18 @@ var CalculatorModule = (() => {
   ));
   var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
   var __publicField = (obj, key, value) => __defNormalProp(obj, typeof key !== "symbol" ? key + "" : key, value);
+  var __accessCheck = (obj, member, msg) => member.has(obj) || __typeError("Cannot " + msg);
+  var __privateGet = (obj, member, getter) => (__accessCheck(obj, member, "read from private field"), getter ? getter.call(obj) : member.get(obj));
+  var __privateAdd = (obj, member, value) => member.has(obj) ? __typeError("Cannot add the same private member more than once") : member instanceof WeakSet ? member.add(obj) : member.set(obj, value);
+  var __privateSet = (obj, member, value, setter) => (__accessCheck(obj, member, "write to private field"), setter ? setter.call(obj, value) : member.set(obj, value), value);
+  var __privateWrapper = (obj, member, setter, getter) => ({
+    set _(value) {
+      __privateSet(obj, member, value, setter);
+    },
+    get _() {
+      return __privateGet(obj, member, getter);
+    }
+  });
   var __async = (__this, __arguments, generator) => {
     return new Promise((resolve2, reject2) => {
       var fulfilled = (value) => {
@@ -76711,6 +76723,156 @@ For more information please go to https://github.com/aws/aws-sdk-js-v3#functiona
   // src/logic/storage.ts
   var import_https_proxy_agent = __toESM(require_dist2(), 1);
   var import_perf_hooks = __require("perf_hooks");
+
+  // node_modules/yocto-queue/index.js
+  var Node = class {
+    constructor(value) {
+      __publicField(this, "value");
+      __publicField(this, "next");
+      this.value = value;
+    }
+  };
+  var _head, _tail, _size;
+  var Queue = class {
+    constructor() {
+      __privateAdd(this, _head);
+      __privateAdd(this, _tail);
+      __privateAdd(this, _size);
+      this.clear();
+    }
+    enqueue(value) {
+      const node = new Node(value);
+      if (__privateGet(this, _head)) {
+        __privateGet(this, _tail).next = node;
+        __privateSet(this, _tail, node);
+      } else {
+        __privateSet(this, _head, node);
+        __privateSet(this, _tail, node);
+      }
+      __privateWrapper(this, _size)._++;
+    }
+    dequeue() {
+      const current = __privateGet(this, _head);
+      if (!current) {
+        return;
+      }
+      __privateSet(this, _head, __privateGet(this, _head).next);
+      __privateWrapper(this, _size)._--;
+      if (!__privateGet(this, _head)) {
+        __privateSet(this, _tail, void 0);
+      }
+      return current.value;
+    }
+    peek() {
+      if (!__privateGet(this, _head)) {
+        return;
+      }
+      return __privateGet(this, _head).value;
+    }
+    clear() {
+      __privateSet(this, _head, void 0);
+      __privateSet(this, _tail, void 0);
+      __privateSet(this, _size, 0);
+    }
+    get size() {
+      return __privateGet(this, _size);
+    }
+    *[Symbol.iterator]() {
+      let current = __privateGet(this, _head);
+      while (current) {
+        yield current.value;
+        current = current.next;
+      }
+    }
+    *drain() {
+      while (__privateGet(this, _head)) {
+        yield this.dequeue();
+      }
+    }
+  };
+  _head = new WeakMap();
+  _tail = new WeakMap();
+  _size = new WeakMap();
+
+  // node_modules/p-limit/index.js
+  function pLimit(concurrency) {
+    validateConcurrency(concurrency);
+    const queue = new Queue();
+    let activeCount = 0;
+    const resumeNext = () => {
+      if (activeCount < concurrency && queue.size > 0) {
+        activeCount++;
+        queue.dequeue()();
+      }
+    };
+    const next = () => {
+      activeCount--;
+      resumeNext();
+    };
+    const run = (function_, resolve2, arguments_) => __async(null, null, function* () {
+      const result = (() => __async(null, null, function* () {
+        return function_(...arguments_);
+      }))();
+      resolve2(result);
+      try {
+        yield result;
+      } catch (e5) {
+      }
+      next();
+    });
+    const enqueue = (function_, resolve2, arguments_) => {
+      new Promise((internalResolve) => {
+        queue.enqueue(internalResolve);
+      }).then(run.bind(void 0, function_, resolve2, arguments_));
+      if (activeCount < concurrency) {
+        resumeNext();
+      }
+    };
+    const generator = (function_, ...arguments_) => new Promise((resolve2) => {
+      enqueue(function_, resolve2, arguments_);
+    });
+    Object.defineProperties(generator, {
+      activeCount: {
+        get: () => activeCount
+      },
+      pendingCount: {
+        get: () => queue.size
+      },
+      clearQueue: {
+        value() {
+          queue.clear();
+        }
+      },
+      concurrency: {
+        get: () => concurrency,
+        set(newConcurrency) {
+          validateConcurrency(newConcurrency);
+          concurrency = newConcurrency;
+          queueMicrotask(() => {
+            while (activeCount < concurrency && queue.size > 0) {
+              resumeNext();
+            }
+          });
+        }
+      },
+      map: {
+        value(iterable, function_) {
+          return __async(this, null, function* () {
+            const promises = Array.from(iterable, (value, index) => this(function_, value, index));
+            return Promise.all(promises);
+          });
+        }
+      }
+    });
+    return generator;
+  }
+  function validateConcurrency(concurrency) {
+    if (!((Number.isInteger(concurrency) || concurrency === Number.POSITIVE_INFINITY) && concurrency > 0)) {
+      throw new TypeError("Expected `concurrency` to be a number from 1 and up");
+    }
+  }
+
+  // src/logic/storage.ts
   (0, import_dotenv.config)();
   if (!process.env.AWS_EXECUTION_ENV && !process.env.AWS_ACCESS_KEY_ID) {
     throw new Error("'AWS_ACCESS_KEY_ID' not set");
@@ -76741,110 +76903,147 @@ For more information please go to https://github.com/aws/aws-sdk-js-v3#functiona
   function isHttp(url2) {
     return url2.startsWith("http://") || url2.startsWith("https://");
   }
+  var processRecord = (_0, _1, _2) => __async(null, [_0, _1, _2], function* ({
+    filePath,
+    key
+  }, options, proxyAgent) {
+    const start = import_perf_hooks.performance.now();
+    if (options.logging) {
+      console.info(`
+[Processing Start] key=${key}, filePath=${filePath}`);
+    }
+    try {
+      const s3Key = `${options.savePath}/${key}`;
+      if (options.logging) {
+        console.info(`[S3 Key] ${s3Key}`);
+      }
+      let stream5 = void 0;
+      let buffer = void 0;
+      if (isHttp(filePath)) {
+        if (options.logging) {
+          console.info(`[Download Start] ${filePath}`);
+        }
+        const axiosResponse = yield axios_default.get(filePath, {
+          responseType: "arraybuffer",
+          headers: options.headers,
+          httpsAgent: proxyAgent
+        });
+        if (options.logging) {
+          console.info(`[Download Success] ${filePath}`);
+        }
+        buffer = Buffer.from(axiosResponse.data);
+      } else {
+        if (options.logging) {
+          console.info(`[Read from FS] ${filePath}`);
+        }
+        stream5 = (0, import_fs8.createReadStream)(filePath);
+      }
+      const tagString = options.tags.map(
+        (tag) => `${encodeURIComponent(tag.split(":")[0])}=${encodeURIComponent(tag.split(":")[1])}`
+      ).join("&");
+      if (!stream5 && !buffer) {
+        if (options.logging) {
+          console.warn(`[Error] Neither stream nor buffer is set for ${key}`);
+        }
+        throw new Error("buffer or stream not set");
+      }
+      if (options.logging) {
+        console.info(`[Image Processing Start] ${key}`);
+      }
+      const image2 = buffer ? yield Jimp.fromBuffer(buffer) : yield Jimp.read(filePath);
+      if (options.logging) {
+        console.info(
+          `[Image Loaded] key=${key}, width=${image2.width}, height=${image2.height}`
+        );
+      }
+      const command = new PutObjectCommand({
+        Body: buffer || stream5,
+        Bucket: BUCKET_NAME,
+        Key: s3Key,
+        Tagging: tagString || void 0,
+        ContentType: `image/${extractFileInfo(key).extension}`,
+        ContentDisposition: "inline",
+        Metadata: {
+          "img-width": image2.width.toString(),
+          "img-height": image2.height.toString()
+        }
+      });
+      if (options.logging) {
+        console.info(`[Uploading] ${key} to S3...`);
+      }
+      yield s32.send(command);
+      if (options.logging) {
+        console.info(`[Upload Success] ${key}`);
+      }
+      return {
+        error: false,
+        key,
+        filePath,
+        uploadUrl: `https://${BUCKET_NAME}.s3.${process.env.AWS_REGION}.amazonaws.com/${s3Key}`
+      };
+    } catch (e5) {
+      const error = e5;
+      if (options.logging) {
+        console.error(`[Upload Error] key=${key}, error=${error.message}`);
+      }
+      return {
+        error: true,
+        key,
+        filePath,
+        message: error.message
+      };
+    } finally {
+      const end = import_perf_hooks.performance.now();
+      if (options.logging) {
+        console.info(
+          `[Processing End] key=${key}, duration=${(end - start).toFixed(0)}ms`
+        );
+      }
+    }
+  });
   var S3Storage = class {
     saveFiles(keys, options) {
       return __async(this, null, function* () {
+        const parallelLimit = options.parallelLimit || 2;
+        const limit = pLimit(parallelLimit);
         const result = [];
         const errored = [];
         const proxyAgent = (options == null ? void 0 : options.proxyUrl) && new import_https_proxy_agent.HttpsProxyAgent(options == null ? void 0 : options.proxyUrl);
         if (options.logging) {
           console.info(`[SaveFiles] Start saving ${keys.length} file(s)`);
-          console.time(`[SaveFiles] Total time`);
+          console.time(`[SaveFiles] Total time ${parallelLimit}`);
         }
-        for (const { key, filePath } of keys) {
-          const start = import_perf_hooks.performance.now();
-          if (options.logging) {
-            console.info(`
-[Processing Start] key=${key}, filePath=${filePath}`);
-          }
-          try {
-            const s3Key = `${options.savePath}/${key}`;
-            if (options.logging) {
-              console.info(`[S3 Key] ${s3Key}`);
-            }
-            let stream5 = void 0;
-            let buffer = void 0;
-            if (isHttp(filePath)) {
-              if (options.logging) {
-                console.info(`[Download Start] ${filePath}`);
-              }
-              const axiosResponse = yield axios_default.get(filePath, {
-                responseType: "arraybuffer",
-                headers: options.headers,
-                httpsAgent: proxyAgent
-              });
-              if (options.logging) {
-                console.info(`[Download Success] ${filePath}`);
-              }
-              buffer = Buffer.from(axiosResponse.data);
-            } else {
-              if (options.logging) {
-                console.info(`[Read from FS] ${filePath}`);
-              }
-              stream5 = (0, import_fs8.createReadStream)(filePath);
-            }
-            const tagString = options.tags.map(
-              (tag) => `${encodeURIComponent(tag.split(":")[0])}=${encodeURIComponent(tag.split(":")[1])}`
-            ).join("&");
-            if (!stream5 && !buffer) {
+        yield Promise.all(
+          keys.map(
+            (record, index) => limit(() => __async(null, null, function* () {
               if (options.logging) {
-                console.warn(`[Error] Neither stream nor buffer is set for ${key}`);
+                console.info(
+                  `Running p-limit task ${index}, limit: ${parallelLimit}`
+                );
               }
-              throw new Error("buffer or stream not set");
-            }
-            if (options.logging) {
-              console.info(`[Image Processing Start] ${key}`);
-            }
-            const image2 = buffer ? yield Jimp.fromBuffer(buffer) : yield Jimp.read(filePath);
-            if (options.logging) {
-              console.info(
-                `[Image Loaded] key=${key}, width=${image2.width}, height=${image2.height}`
+              const processResult = yield processRecord(
+                record,
+                options,
+                proxyAgent || void 0
               );
-            }
-            const command = new PutObjectCommand({
-              Body: buffer || stream5,
-              Bucket: BUCKET_NAME,
-              Key: s3Key,
-              Tagging: tagString || void 0,
-              ContentType: `image/${extractFileInfo(key).extension}`,
-              ContentDisposition: "inline",
-              Metadata: {
-                "img-width": image2.width.toString(),
-                "img-height": image2.height.toString()
+              if (processResult.message) {
+                errored.push({
+                  key: processResult.key,
+                  filePath: processResult.filePath,
+                  message: processResult.message
+                });
+              } else if (processResult.uploadUrl) {
+                result.push({
+                  key: processResult.key,
+                  filePath: processResult.filePath,
+                  uploadUrl: processResult.uploadUrl
+                });
               }
-            });
-            result.push({
-              key,
-              filePath,
-              uploadUrl: `https://${BUCKET_NAME}.s3.${process.env.AWS_REGION}.amazonaws.com/${s3Key}`
-            });
-            if (options.logging) {
-              console.info(`[Uploading] ${key} to S3...`);
-            }
-            yield s32.send(command);
-            if (options.logging) {
-              console.info(`[Upload Success] ${key}`);
-            }
-          } catch (e5) {
-            const error = e5;
-            if (options.logging) {
-              console.error(`[Upload Error] key=${key}, error=${error.message}`);
-            }
-            errored.push({
-              key,
-              filePath,
-              message: error.message
-            });
-          }
-          const end = import_perf_hooks.performance.now();
-          if (options.logging) {
-            console.info(
-              `[Processing End] key=${key}, duration=${(end - start).toFixed(0)}ms`
-            );
-          }
-        }
+            }))
+          )
+        );
         if (options.logging) {
-          console.timeEnd(`[SaveFiles] Total time`);
+          console.timeEnd(`[SaveFiles] Total time ${parallelLimit}`);
           console.info(
             `[SaveFiles] Completed. Success: ${result.length}, Failed: ${errored.length}`
           );

package/dist/index.js

@@ -33,6 +33,7 @@ import axios from "axios";
 import { Jimp } from "jimp";
 import { HttpsProxyAgent } from "https-proxy-agent";
 import { performance } from "perf_hooks";
+import pLimit from "p-limit";
 config();
 if (!process.env.AWS_EXECUTION_ENV && !process.env.AWS_ACCESS_KEY_ID) {
   throw new Error("'AWS_ACCESS_KEY_ID' not set");
@@ -63,110 +64,147 @@ if (!BUCKET_NAME) {
 function isHttp(url) {
   return url.startsWith("http://") || url.startsWith("https://");
 }
+var processRecord = (_0, _1, _2) => __async(null, [_0, _1, _2], function* ({
+  filePath,
+  key
+}, options, proxyAgent) {
+  const start = performance.now();
+  if (options.logging) {
+    console.info(`
+[Processing Start] key=${key}, filePath=${filePath}`);
+  }
+  try {
+    const s3Key = `${options.savePath}/${key}`;
+    if (options.logging) {
+      console.info(`[S3 Key] ${s3Key}`);
+    }
+    let stream = void 0;
+    let buffer = void 0;
+    if (isHttp(filePath)) {
+      if (options.logging) {
+        console.info(`[Download Start] ${filePath}`);
+      }
+      const axiosResponse = yield axios.get(filePath, {
+        responseType: "arraybuffer",
+        headers: options.headers,
+        httpsAgent: proxyAgent
+      });
+      if (options.logging) {
+        console.info(`[Download Success] ${filePath}`);
+      }
+      buffer = Buffer.from(axiosResponse.data);
+    } else {
+      if (options.logging) {
+        console.info(`[Read from FS] ${filePath}`);
+      }
+      stream = createReadStream(filePath);
+    }
+    const tagString = options.tags.map(
+      (tag) => `${encodeURIComponent(tag.split(":")[0])}=${encodeURIComponent(tag.split(":")[1])}`
+    ).join("&");
+    if (!stream && !buffer) {
+      if (options.logging) {
+        console.warn(`[Error] Neither stream nor buffer is set for ${key}`);
+      }
+      throw new Error("buffer or stream not set");
+    }
+    if (options.logging) {
+      console.info(`[Image Processing Start] ${key}`);
+    }
+    const image = buffer ? yield Jimp.fromBuffer(buffer) : yield Jimp.read(filePath);
+    if (options.logging) {
+      console.info(
+        `[Image Loaded] key=${key}, width=${image.width}, height=${image.height}`
+      );
+    }
+    const command = new PutObjectCommand({
+      Body: buffer || stream,
+      Bucket: BUCKET_NAME,
+      Key: s3Key,
+      Tagging: tagString || void 0,
+      ContentType: `image/${extractFileInfo(key).extension}`,
+      ContentDisposition: "inline",
+      Metadata: {
+        "img-width": image.width.toString(),
+        "img-height": image.height.toString()
+      }
+    });
+    if (options.logging) {
+      console.info(`[Uploading] ${key} to S3...`);
+    }
+    yield s3.send(command);
+    if (options.logging) {
+      console.info(`[Upload Success] ${key}`);
+    }
+    return {
+      error: false,
+      key,
+      filePath,
+      uploadUrl: `https://${BUCKET_NAME}.s3.${process.env.AWS_REGION}.amazonaws.com/${s3Key}`
+    };
+  } catch (e) {
+    const error = e;
+    if (options.logging) {
+      console.error(`[Upload Error] key=${key}, error=${error.message}`);
+    }
+    return {
+      error: true,
+      key,
+      filePath,
+      message: error.message
+    };
+  } finally {
+    const end = performance.now();
+    if (options.logging) {
+      console.info(
+        `[Processing End] key=${key}, duration=${(end - start).toFixed(0)}ms`
+      );
+    }
+  }
+});
 var S3Storage = class {
   saveFiles(keys, options) {
     return __async(this, null, function* () {
+      const parallelLimit = options.parallelLimit || 2;
+      const limit = pLimit(parallelLimit);
       const result = [];
       const errored = [];
       const proxyAgent = (options == null ? void 0 : options.proxyUrl) && new HttpsProxyAgent(options == null ? void 0 : options.proxyUrl);
       if (options.logging) {
         console.info(`[SaveFiles] Start saving ${keys.length} file(s)`);
-        console.time(`[SaveFiles] Total time`);
+        console.time(`[SaveFiles] Total time ${parallelLimit}`);
       }
-      for (const { key, filePath } of keys) {
-        const start = performance.now();
-        if (options.logging) {
-          console.info(`
-[Processing Start] key=${key}, filePath=${filePath}`);
-        }
-        try {
-          const s3Key = `${options.savePath}/${key}`;
-          if (options.logging) {
-            console.info(`[S3 Key] ${s3Key}`);
-          }
-          let stream = void 0;
-          let buffer = void 0;
-          if (isHttp(filePath)) {
+      yield Promise.all(
+        keys.map(
+          (record, index) => limit(() => __async(null, null, function* () {
             if (options.logging) {
-              console.info(`[Download Start] ${filePath}`);
+              console.info(
+                `Running p-limit task ${index}, limit: ${parallelLimit}`
+              );
             }
-            const axiosResponse = yield axios.get(filePath, {
-              responseType: "arraybuffer",
-              headers: options.headers,
-              httpsAgent: proxyAgent
-            });
-            if (options.logging) {
-              console.info(`[Download Success] ${filePath}`);
-            }
-            buffer = Buffer.from(axiosResponse.data);
-          } else {
-            if (options.logging) {
-              console.info(`[Read from FS] ${filePath}`);
-            }
-            stream = createReadStream(filePath);
-          }
-          const tagString = options.tags.map(
-            (tag) => `${encodeURIComponent(tag.split(":")[0])}=${encodeURIComponent(tag.split(":")[1])}`
-          ).join("&");
-          if (!stream && !buffer) {
-            if (options.logging) {
-              console.warn(`[Error] Neither stream nor buffer is set for ${key}`);
-            }
-            throw new Error("buffer or stream not set");
-          }
-          if (options.logging) {
-            console.info(`[Image Processing Start] ${key}`);
-          }
-          const image = buffer ? yield Jimp.fromBuffer(buffer) : yield Jimp.read(filePath);
-          if (options.logging) {
-            console.info(
-              `[Image Loaded] key=${key}, width=${image.width}, height=${image.height}`
+            const processResult = yield processRecord(
+              record,
+              options,
+              proxyAgent || void 0
             );
-          }
-          const command = new PutObjectCommand({
-            Body: buffer || stream,
-            Bucket: BUCKET_NAME,
-            Key: s3Key,
-            Tagging: tagString || void 0,
-            ContentType: `image/${extractFileInfo(key).extension}`,
-            ContentDisposition: "inline",
-            Metadata: {
-              "img-width": image.width.toString(),
-              "img-height": image.height.toString()
+            if (processResult.message) {
+              errored.push({
+                key: processResult.key,
+                filePath: processResult.filePath,
+                message: processResult.message
+              });
+            } else if (processResult.uploadUrl) {
+              result.push({
+                key: processResult.key,
+                filePath: processResult.filePath,
+                uploadUrl: processResult.uploadUrl
+              });
             }
-          });
-          result.push({
-            key,
-            filePath,
-            uploadUrl: `https://${BUCKET_NAME}.s3.${process.env.AWS_REGION}.amazonaws.com/${s3Key}`
-          });
-          if (options.logging) {
-            console.info(`[Uploading] ${key} to S3...`);
-          }
-          yield s3.send(command);
-          if (options.logging) {
-            console.info(`[Upload Success] ${key}`);
-          }
-        } catch (e) {
-          const error = e;
-          if (options.logging) {
-            console.error(`[Upload Error] key=${key}, error=${error.message}`);
-          }
-          errored.push({
-            key,
-            filePath,
-            message: error.message
-          });
-        }
-        const end = performance.now();
-        if (options.logging) {
-          console.info(
-            `[Processing End] key=${key}, duration=${(end - start).toFixed(0)}ms`
-          );
-        }
-      }
+          }))
+        )
+      );
       if (options.logging) {
-        console.timeEnd(`[SaveFiles] Total time`);
+        console.timeEnd(`[SaveFiles] Total time ${parallelLimit}`);
         console.info(
           `[SaveFiles] Completed. Success: ${result.length}, Failed: ${errored.length}`
         );

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "importer-storage",
   "private": false,
-  "version": "1.0.16",
+  "version": "1.0.18",
   "main": "dist/index.js",
   "files": [
     "dist",
@@ -23,7 +23,8 @@
     "dotenv": "^16.5.0",
     "globals": "^16.2.0",
     "https-proxy-agent": "^7.0.6",
-    "jimp": "^1.6.0"
+    "jimp": "^1.6.0",
+    "p-limit": "^7.2.0"
   },
   "devDependencies": {
     "@eslint/js": "^9.19.0",