@xen-orchestra/fs 0.19.3 → 0.20.0

package/dist/s3.js

@@ -13,19 +13,46 @@ var _http = _interopRequireDefault(require("http"));
 
 var _https = _interopRequireDefault(require("https"));
 
+var _retry = _interopRequireDefault(require("promise-toolbox/retry"));
+
+var _log = require("@xen-orchestra/log");
+
+var _decorateWith = require("@vates/decorate-with");
+
 var _xoRemoteParser = require("xo-remote-parser");
 
 var _abstract = _interopRequireDefault(require("./abstract"));
 
+var _asyncEach = require("@vates/async-each");
+
+var _dec, _class;
+
 function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; }
 
+function _applyDecoratedDescriptor(target, property, decorators, descriptor, context) { var desc = {}; Object.keys(descriptor).forEach(function (key) { desc[key] = descriptor[key]; }); desc.enumerable = !!desc.enumerable; desc.configurable = !!desc.configurable; if ('value' in desc || desc.initializer) { desc.writable = true; } desc = decorators.slice().reverse().reduce(function (desc, decorator) { return decorator(target, property, desc) || desc; }, desc); if (context && desc.initializer !== void 0) { desc.value = desc.initializer ? desc.initializer.call(context) : void 0; desc.initializer = undefined; } if (desc.initializer === void 0) { Object.defineProperty(target, property, desc); desc = null; } return desc; }
+
 const MIN_PART_SIZE = 1024 * 1024 * 5;
 const MAX_PART_SIZE = 1024 * 1024 * 1024 * 5;
 const MAX_PARTS_COUNT = 10000;
 const MAX_OBJECT_SIZE = 1024 * 1024 * 1024 * 1024 * 5;
 const IDEAL_FRAGMENT_SIZE = Math.ceil(MAX_OBJECT_SIZE / MAX_PARTS_COUNT);
+const {
+  warn
+} = (0, _log.createLogger)('xo:fs:s3');
+let S3Handler = (_dec = (0, _decorateWith.decorateWith)(_retry.default.wrap, {
+  delays: [100, 200, 500, 1000, 2000],
+  when: e => e.code === 'InternalError',
+
+  onRetry(error) {
+    warn('retrying writing file', {
+      attemptNumber: this.attemptNumber,
+      delay: this.delay,
+      error,
+      file: this.arguments[0]
+    });
+  }
 
-class S3Handler extends _abstract.default {
+}), (_class = class S3Handler extends _abstract.default {
   constructor(remote, _opts) {
     super(remote);
     const {
@@ -82,12 +109,13 @@ class S3Handler extends _abstract.default {
     };
   }
 
-  async _copy(oldPath, newPath) {
+  async _multipartCopy(oldPath, newPath) {
     const size = await this._getSize(oldPath);
+    const CopySource = `/${this._bucket}/${this._dir}${oldPath}`;
     const multipartParams = await this._s3.createMultipartUpload({ ...this._createParams(newPath)
     });
     const param2 = { ...multipartParams,
-      CopySource: `/${this._bucket}/${this._dir}${oldPath}`
+      CopySource
     };
 
     try {
@@ -119,6 +147,22 @@ class S3Handler extends _abstract.default {
     }
   }
 
+  async _copy(oldPath, newPath) {
+    const CopySource = `/${this._bucket}/${this._dir}${oldPath}`;
+
+    try {
+      await this._s3.copyObject({ ...this._createParams(newPath),
+        CopySource
+      });
+    } catch (e) {
+      if (e.code === 'EntityTooLarge') {
+        return this._multipartCopy(oldPath, newPath);
+      }
+
+      throw e;
+    }
+  }
+
   async _isNotEmptyDir(path) {
     const result = await this._s3.listObjectsV2({
       Bucket: this._bucket,
@@ -202,7 +246,7 @@ class S3Handler extends _abstract.default {
       Delimiter: '/'
     });
 
-    if (result.isTruncated) {
+    if (result.IsTruncated) {
       const error = new Error('more than 1000 objects, unsupported in this implementation');
       error.dir = dir;
       throw error;
@@ -294,16 +338,17 @@ class S3Handler extends _abstract.default {
         Prefix: this._dir + path + '/',
         ContinuationToken: NextContinuationToken
       });
-      NextContinuationToken = result.isTruncated ? result.NextContinuationToken : undefined;
-
-      for (const {
+      NextContinuationToken = result.IsTruncated ? result.NextContinuationToken : undefined;
+      await (0, _asyncEach.asyncEach)(result.Contents, async ({
         Key
-      } of result.Contents) {
+      }) => {
         await this._s3.deleteObject({
           Bucket: this._bucket,
           Key
         });
-      }
+      }, {
+        concurrency: 16
+      });
     } while (NextContinuationToken !== undefined);
   }
 
@@ -455,7 +500,6 @@ class S3Handler extends _abstract.default {
 
   async _closeFile(fd) {}
 
-}
-
+}, (_applyDecoratedDescriptor(_class.prototype, "_writeFile", [_dec], Object.getOwnPropertyDescriptor(_class.prototype, "_writeFile"), _class.prototype)), _class));
 exports.default = S3Handler;
 //# sourceMappingURL=s3.js.map

package/dist/s3.js.map

@@ -1 +1 @@
-{"version":3,"sources":["../src/s3.js"],"names":["MIN_PART_SIZE","MAX_PART_SIZE","MAX_PARTS_COUNT","MAX_OBJECT_SIZE","IDEAL_FRAGMENT_SIZE","Math","ceil","S3Handler","RemoteHandlerAbstract","constructor","remote","_opts","allowUnauthorized","host","path","username","password","protocol","region","url","params","accessKeyId","apiVersion","endpoint","s3ForcePathStyle","secretAccessKey","signatureVersion","httpOptions","timeout","agent","http","Agent","keepAlive","sslEnabled","https","rejectUnauthorized","undefined","_s3","s3","splitPath","split","filter","s","length","_bucket","shift","_dir","join","type","_createParams","file","Bucket","Key","_copy","oldPath","newPath","size","_getSize","multipartParams","createMultipartUpload","param2","CopySource","parts","start","range","min","partParams","PartNumber","CopySourceRange","upload","uploadPartCopy","push","ETag","CopyPartResult","completeMultipartUpload","MultipartUpload","Parts","e","abortMultipartUpload","_isNotEmptyDir","result","listObjectsV2","MaxKeys","Prefix","Contents","_isFile","headObject","error","code","_outputStream","input","validator","Body","partSize","queueSize","call","unlink","_writeFile","data","options","putObject","_createReadStream","Error","getObject","raw","createReadStream","_unlink","deleteObject","_list","dir","d","prefix","splitPrefix","Delimiter","isTruncated","uniq","entry","CommonPrefixes","line","_mkdir","_rename","copy","fd","ContentLength","_read","buffer","position","Range","bytesRead","_rmdir","_rmtree","NextContinuationToken","ContinuationToken","_write","uploadParams","fileSize","resultBuffer","Buffer","alloc","max","fileContent","bytesWritten","copyMultipartParams","prefixSize","suffixOffset","suffixSize","hasSuffix","editBuffer","editBufferOffset","partNumber","prefixPosition","fragmentsCount","floor","prefixFragmentSize","prefixLastFragmentSize","i","fragmentEnd","assert","strictEqual","copyPrefixParams","part","downloadParams","prefixBuffer","concat","complementSize","complementOffset","prefixRange","complementBuffer","editParams","editPart","uploadPart","suffixFragments","suffixFragmentOffset","suffixRange","copySuffixParams","suffixPart","_openFile","flags","_closeFile"],"mappings":";;;;;;;AAAA;;AACA;;AACA;;AACA;;AACA;;AAEA;;;;AAKA,MAAMA,aAAa,GAAG,OAAO,IAAP,GAAc,CAApC;AACA,MAAMC,aAAa,GAAG,OAAO,IAAP,GAAc,IAAd,GAAqB,CAA3C;AACA,MAAMC,eAAe,GAAG,KAAxB;AACA,MAAMC,eAAe,GAAG,OAAO,IAAP,GAAc,IAAd,GAAqB,IAArB,GAA4B,CAApD;AACA,MAAMC,mBAAmB,GAAGC,IAAI,CAACC,IAAL,CAAUH,eAAe,GAAGD,eAA5B,CAA5B;;AACe,MAAMK,SAAN,SAAwBC,iBAAxB,CAA8C;AAC3DC,EAAAA,WAAW,CAACC,MAAD,EAASC,KAAT,EAAgB;AACzB,UAAMD,MAAN;AACA,UAAM;AAAEE,MAAAA,iBAAF;AAAqBC,MAAAA,IAArB;AAA2BC,MAAAA,IAA3B;AAAiCC,MAAAA,QAAjC;AAA2CC,MAAAA,QAA3C;AAAqDC,MAAAA,QAArD;AAA+DC,MAAAA;AAA/D,QAA0E,2BAAMR,MAAM,CAACS,GAAb,CAAhF;AACA,UAAMC,MAAM,GAAG;AACbC,MAAAA,WAAW,EAAEN,QADA;AAEbO,MAAAA,UAAU,EAAE,YAFC;AAGbC,MAAAA,QAAQ,EAAEV,IAHG;AAIbW,MAAAA,gBAAgB,EAAE,IAJL;AAKbC,MAAAA,eAAe,EAAET,QALJ;AAMbU,MAAAA,gBAAgB,EAAE,IANL;AAObC,MAAAA,WAAW,EAAE;AACXC,QAAAA,OAAO,EAAE;AADE;AAPA,KAAf;;AAWA,QAAIX,QAAQ,KAAK,MAAjB,EAAyB;AACvBG,MAAAA,MAAM,CAACO,WAAP,CAAmBE,KAAnB,GAA2B,IAAIC,cAAKC,KAAT,CAAe;AAAEC,QAAAA,SAAS,EAAE;AAAb,OAAf,CAA3B;AACAZ,MAAAA,MAAM,CAACa,UAAP,GAAoB,KAApB;AACD,KAHD,MAGO,IAAIhB,QAAQ,KAAK,OAAjB,EAA0B;AAC/BG,MAAAA,MAAM,CAACO,WAAP,CAAmBE,KAAnB,GAA2B,IAAIK,eAAMH,KAAV,CAAgB;AACzCI,QAAAA,kBAAkB,EAAE,CAACvB,iBADoB;AAEzCoB,QAAAA,SAAS,EAAE;AAF8B,OAAhB,CAA3B;AAID;;AACD,QAAId,MAAM,KAAKkB,SAAf,EAA0B;AACxBhB,MAAAA,MAAM,CAACF,MAAP,GAAgBA,MAAhB;AACD;;AAED,SAAKmB,GAAL,GAAW,qBAAIjB,MAAJ,EAAYkB,EAAvB;AAEA,UAAMC,SAAS,GAAGzB,IAAI,CAAC0B,KAAL,CAAW,GAAX,EAAgBC,MAAhB,CAAuBC,CAAC,IAAIA,CAAC,CAACC,MAA9B,CAAlB;AACA,SAAKC,OAAL,GAAeL,SAAS,CAACM,KAAV,EAAf;AACA,SAAKC,IAAL,GAAYP,SAAS,CAACQ,IAAV,CAAe,GAAf,CAAZ;AACD;;AAEO,MAAJC,IAAI,GAAG;AACT,WAAO,IAAP;AACD;;AAEDC,EAAAA,aAAa,CAACC,IAAD,EAAO;AAClB,WAAO;AAAEC,MAAAA,MAAM,EAAE,KAAKP,OAAf;AAAwBQ,MAAAA,GAAG,EAAE,KAAKN,IAAL,GAAYI;AAAzC,KAAP;AACD;;AAEU,QAALG,KAAK,CAACC,OAAD,EAAUC,OAAV,EAAmB;AAC5B,UAAMC,IAAI,GAAG,MAAM,KAAKC,QAAL,CAAcH,OAAd,CAAnB;AACA,UAAMI,eAAe,GAAG,MAAM,KAAKrB,GAAL,CAASsB,qBAAT,CAA+B,EAAE,GAAG,KAAKV,aAAL,CAAmBM,OAAnB;AAAL,KAA/B,CAA9B;AACA,UAAMK,MAAM,GAAG,EAAE,GAAGF,eAAL;AAAsBG,MAAAA,UAAU,EAAG,IAAG,KAAKjB,OAAQ,IAAG,KAAKE,IAAK,GAAEQ,OAAQ;AAA1E,KAAf;;AACA,QAAI;AACF,YAAMQ,KAAK,GAAG,EAAd;AACA,UAAIC,KAAK,GAAG,CAAZ;;AACA,aAAOA,KAAK,GAAGP,IAAf,EAAqB;AACnB,cAAMQ,KAAK,GAAI,SAAQD,KAAM,IAAG1D,IAAI,CAAC4D,GAAL,CAASF,KAAK,GAAG9D,aAAjB,EAAgCuD,IAAhC,IAAwC,CAAE,EAA1E;AACA,cAAMU,UAAU,GAAG,EAAE,GAAGN,MAAL;AAAaO,UAAAA,UAAU,EAAEL,KAAK,CAACnB,MAAN,GAAe,CAAxC;AAA2CyB,UAAAA,eAAe,EAAEJ;AAA5D,SAAnB;AACA,cAAMK,MAAM,GAAG,MAAM,KAAKhC,GAAL,CAASiC,cAAT,CAAwBJ,UAAxB,CAArB;AACAJ,QAAAA,KAAK,CAACS,IAAN,CAAW;AAAEC,UAAAA,IAAI,EAAEH,MAAM,CAACI,cAAP,CAAsBD,IAA9B;AAAoCL,UAAAA,UAAU,EAAED,UAAU,CAACC;AAA3D,SAAX;AACAJ,QAAAA,KAAK,IAAI9D,aAAT;AACD;;AACD,YAAM,KAAKoC,GAAL,CAASqC,uBAAT,CAAiC,EAAE,GAAGhB,eAAL;AAAsBiB,QAAAA,eAAe,EAAE;AAAEC,UAAAA,KAAK,EAAEd;AAAT;AAAvC,OAAjC,CAAN;AACD,KAXD,CAWE,OAAOe,CAAP,EAAU;AACV,YAAM,KAAKxC,GAAL,CAASyC,oBAAT,CAA8BpB,eAA9B,CAAN;AACA,YAAMmB,CAAN;AACD;AACF;;AAEmB,QAAdE,cAAc,CAACjE,IAAD,EAAO;AACzB,UAAMkE,MAAM,GAAG,MAAM,KAAK3C,GAAL,CAAS4C,aAAT,CAAuB;AAC1C9B,MAAAA,MAAM,EAAE,KAAKP,OAD6B;AAE1CsC,MAAAA,OAAO,EAAE,CAFiC;AAG1CC,MAAAA,MAAM,EAAE,KAAKrC,IAAL,GAAYhC,IAAZ,GAAmB;AAHe,KAAvB,CAArB;AAKA,WAAOkE,MAAM,CAACI,QAAP,CAAgBzC,MAAhB,KAA2B,CAAlC;AACD;;AAEY,QAAP0C,OAAO,CAACvE,IAAD,EAAO;AAClB,QAAI;AACF,YAAM,KAAKuB,GAAL,CAASiD,UAAT,CAAoB,KAAKrC,aAAL,CAAmBnC,IAAnB,CAApB,CAAN;AACA,aAAO,IAAP;AACD,KAHD,CAGE,OAAOyE,KAAP,EAAc;AACd,UAAIA,KAAK,CAACC,IAAN,KAAe,UAAnB,EAA+B;AAC7B,eAAO,KAAP;AACD;;AACD,YAAMD,KAAN;AACD;AACF;;AAEkB,QAAbE,aAAa,CAAC3E,IAAD,EAAO4E,KAAP,EAAc;AAAEC,IAAAA;AAAF,GAAd,EAA6B;AAC9C,UAAM,KAAKtD,GAAL,CAASgC,MAAT,CACJ,EACE,GAAG,KAAKpB,aAAL,CAAmBnC,IAAnB,CADL;AAEE8E,MAAAA,IAAI,EAAEF;AAFR,KADI,EAKJ;AAAEG,MAAAA,QAAQ,EAAEzF,mBAAZ;AAAiC0F,MAAAA,SAAS,EAAE;AAA5C,KALI,CAAN;;AAOA,QAAIH,SAAS,KAAKvD,SAAlB,EAA6B;AAC3B,UAAI;AACF,cAAMuD,SAAS,CAACI,IAAV,CAAe,IAAf,EAAqBjF,IAArB,CAAN;AACD,OAFD,CAEE,OAAOyE,KAAP,EAAc;AACd,cAAM,KAAKS,MAAL,CAAYlF,IAAZ,CAAN;AACA,cAAMyE,KAAN;AACD;AACF;AACF;;AAEe,QAAVU,UAAU,CAAC/C,IAAD,EAAOgD,IAAP,EAAaC,OAAb,EAAsB;AACpC,WAAO,KAAK9D,GAAL,CAAS+D,SAAT,CAAmB,EAAE,GAAG,KAAKnD,aAAL,CAAmBC,IAAnB,CAAL;AAA+B0C,MAAAA,IAAI,EAAEM;AAArC,KAAnB,CAAP;AACD;;AAEsB,QAAjBG,iBAAiB,CAACvF,IAAD,EAAOqF,OAAP,EAAgB;AACrC,QAAI,EAAE,MAAM,KAAKd,OAAL,CAAavE,IAAb,CAAR,CAAJ,EAAiC;AAC/B,YAAMyE,KAAK,GAAG,IAAIe,KAAJ,CAAW,yBAAwBxF,IAAK,GAAxC,CAAd;AACAyE,MAAAA,KAAK,CAACC,IAAN,GAAa,QAAb;AACAD,MAAAA,KAAK,CAACzE,IAAN,GAAaA,IAAb;AACA,YAAMyE,KAAN;AACD;;AAGD,WAAO,KAAKlD,GAAL,CAASkE,SAAT,CAAmBC,GAAnB,CAAuB,KAAKvD,aAAL,CAAmBnC,IAAnB,CAAvB,EAAiD2F,gBAAjD,EAAP;AACD;;AAEY,QAAPC,OAAO,CAAC5F,IAAD,EAAO;AAClB,UAAM,KAAKuB,GAAL,CAASsE,YAAT,CAAsB,KAAK1D,aAAL,CAAmBnC,IAAnB,CAAtB,CAAN;;AACA,QAAI,MAAM,KAAKiE,cAAL,CAAoBjE,IAApB,CAAV,EAAqC;AACnC,YAAMyE,KAAK,GAAG,IAAIe,KAAJ,CAAW,qDAAoDxF,IAAK,GAApE,CAAd;AACAyE,MAAAA,KAAK,CAACC,IAAN,GAAa,QAAb;AACAD,MAAAA,KAAK,CAACzE,IAAN,GAAaA,IAAb;AACA,YAAMyE,KAAN;AACD;AACF;;AAEU,QAALqB,KAAK,CAACC,GAAD,EAAM;AACf,aAAStE,SAAT,CAAmBzB,IAAnB,EAAyB;AACvB,aAAOA,IAAI,CAAC0B,KAAL,CAAW,GAAX,EAAgBC,MAAhB,CAAuBqE,CAAC,IAAIA,CAAC,CAACnE,MAA9B,CAAP;AACD;;AAED,UAAMoE,MAAM,GAAG,CAAC,KAAKjE,IAAN,EAAY+D,GAAZ,EAAiB9D,IAAjB,CAAsB,GAAtB,CAAf;AACA,UAAMiE,WAAW,GAAGzE,SAAS,CAACwE,MAAD,CAA7B;AACA,UAAM/B,MAAM,GAAG,MAAM,KAAK3C,GAAL,CAAS4C,aAAT,CAAuB;AAC1C9B,MAAAA,MAAM,EAAE,KAAKP,OAD6B;AAE1CuC,MAAAA,MAAM,EAAE6B,WAAW,CAACjE,IAAZ,CAAiB,GAAjB,IAAwB,GAFU;AAG1CkE,MAAAA,SAAS,EAAE;AAH+B,KAAvB,CAArB;;AAMA,QAAIjC,MAAM,CAACkC,WAAX,EAAwB;AACtB,YAAM3B,KAAK,GAAG,IAAIe,KAAJ,CAAU,4DAAV,CAAd;AACAf,MAAAA,KAAK,CAACsB,GAAN,GAAYA,GAAZ;AACA,YAAMtB,KAAN;AACD;;AAED,UAAM4B,IAAI,GAAG,EAAb;;AAGA,SAAK,MAAMC,KAAX,IAAoBpC,MAAM,CAACqC,cAA3B,EAA2C;AACzC,YAAMC,IAAI,GAAG/E,SAAS,CAAC6E,KAAK,CAACjC,MAAP,CAAtB;AACAgC,MAAAA,IAAI,CAAC5C,IAAL,CAAU+C,IAAI,CAACA,IAAI,CAAC3E,MAAL,GAAc,CAAf,CAAd;AACD;;AAED,SAAK,MAAMyE,KAAX,IAAoBpC,MAAM,CAACI,QAA3B,EAAqC;AACnC,YAAMkC,IAAI,GAAG/E,SAAS,CAAC6E,KAAK,CAAChE,GAAP,CAAtB;AACA+D,MAAAA,IAAI,CAAC5C,IAAL,CAAU+C,IAAI,CAACA,IAAI,CAAC3E,MAAL,GAAc,CAAf,CAAd;AACD;;AAED,WAAOwE,IAAP;AACD;;AAEW,QAANI,MAAM,CAACzG,IAAD,EAAO;AACjB,QAAI,MAAM,KAAKuE,OAAL,CAAavE,IAAb,CAAV,EAA8B;AAC5B,YAAMyE,KAAK,GAAG,IAAIe,KAAJ,CAAW,wCAAuCxF,IAAK,GAAvD,CAAd;AACAyE,MAAAA,KAAK,CAACC,IAAN,GAAa,SAAb;AACAD,MAAAA,KAAK,CAACzE,IAAN,GAAaA,IAAb;AACA,YAAMyE,KAAN;AACD;AAEF;;AAGY,QAAPiC,OAAO,CAAClE,OAAD,EAAUC,OAAV,EAAmB;AAC9B,UAAM,KAAKkE,IAAL,CAAUnE,OAAV,EAAmBC,OAAnB,CAAN;AACA,UAAM,KAAKlB,GAAL,CAASsE,YAAT,CAAsB,KAAK1D,aAAL,CAAmBK,OAAnB,CAAtB,CAAN;AACD;;AAEa,QAARG,QAAQ,CAACP,IAAD,EAAO;AACnB,QAAI,OAAOA,IAAP,KAAgB,QAApB,EAA8B;AAC5BA,MAAAA,IAAI,GAAGA,IAAI,CAACwE,EAAZ;AACD;;AACD,UAAM1C,MAAM,GAAG,MAAM,KAAK3C,GAAL,CAASiD,UAAT,CAAoB,KAAKrC,aAAL,CAAmBC,IAAnB,CAApB,CAArB;AACA,WAAO,CAAC8B,MAAM,CAAC2C,aAAf;AACD;;AAEU,QAALC,KAAK,CAAC1E,IAAD,EAAO2E,MAAP,EAAeC,QAAQ,GAAG,CAA1B,EAA6B;AACtC,QAAI,OAAO5E,IAAP,KAAgB,QAApB,EAA8B;AAC5BA,MAAAA,IAAI,GAAGA,IAAI,CAACwE,EAAZ;AACD;;AACD,UAAMtG,MAAM,GAAG,KAAK6B,aAAL,CAAmBC,IAAnB,CAAf;;AACA9B,IAAAA,MAAM,CAAC2G,KAAP,GAAgB,SAAQD,QAAS,IAAGA,QAAQ,GAAGD,MAAM,CAAClF,MAAlB,GAA2B,CAAE,EAAjE;;AACA,QAAI;AACF,YAAMqC,MAAM,GAAG,MAAM,KAAK3C,GAAL,CAASkE,SAAT,CAAmBnF,MAAnB,CAArB;AACA4D,MAAAA,MAAM,CAACY,IAAP,CAAY6B,IAAZ,CAAiBI,MAAjB;AACA,aAAO;AAAEG,QAAAA,SAAS,EAAEhD,MAAM,CAACY,IAAP,CAAYjD,MAAzB;AAAiCkF,QAAAA;AAAjC,OAAP;AACD,KAJD,CAIE,OAAOhD,CAAP,EAAU;AACV,UAAIA,CAAC,CAACW,IAAF,KAAW,WAAf,EAA4B;AAC1B,YAAI,MAAM,KAAKT,cAAL,CAAoB7B,IAApB,CAAV,EAAqC;AACnC,gBAAMqC,KAAK,GAAG,IAAIe,KAAJ,CAAW,GAAEpD,IAAK,iBAAlB,CAAd;AACAqC,UAAAA,KAAK,CAACC,IAAN,GAAa,QAAb;AACAD,UAAAA,KAAK,CAACzE,IAAN,GAAaoC,IAAb;AACA,gBAAMqC,KAAN;AACD;AACF;;AACD,YAAMV,CAAN;AACD;AACF;;AAEW,QAANoD,MAAM,CAACnH,IAAD,EAAO;AACjB,QAAI,MAAM,KAAKiE,cAAL,CAAoBjE,IAApB,CAAV,EAAqC;AACnC,YAAMyE,KAAK,GAAG,IAAIe,KAAJ,CAAW,0CAAyCxF,IAAK,EAAzD,CAAd;AACAyE,MAAAA,KAAK,CAACC,IAAN,GAAa,WAAb;AACAD,MAAAA,KAAK,CAACzE,IAAN,GAAaA,IAAb;AACA,YAAMyE,KAAN;AACD;AAGF;;AAIY,QAAP2C,OAAO,CAACpH,IAAD,EAAO;AAClB,QAAIqH,qBAAJ;;AACA,OAAG;AACD,YAAMnD,MAAM,GAAG,MAAM,KAAK3C,GAAL,CAAS4C,aAAT,CAAuB;AAC1C9B,QAAAA,MAAM,EAAE,KAAKP,OAD6B;AAE1CuC,QAAAA,MAAM,EAAE,KAAKrC,IAAL,GAAYhC,IAAZ,GAAmB,GAFe;AAG1CsH,QAAAA,iBAAiB,EAAED;AAHuB,OAAvB,CAArB;AAKAA,MAAAA,qBAAqB,GAAGnD,MAAM,CAACkC,WAAP,GAAqBlC,MAAM,CAACmD,qBAA5B,GAAoD/F,SAA5E;;AACA,WAAK,MAAM;AAAEgB,QAAAA;AAAF,OAAX,IAAsB4B,MAAM,CAACI,QAA7B,EAAuC;AAGrC,cAAM,KAAK/C,GAAL,CAASsE,YAAT,CAAsB;AAC1BxD,UAAAA,MAAM,EAAE,KAAKP,OADa;AAE1BQ,UAAAA;AAF0B,SAAtB,CAAN;AAID;AACF,KAfD,QAeS+E,qBAAqB,KAAK/F,SAfnC;AAgBD;;AAEW,QAANiG,MAAM,CAACnF,IAAD,EAAO2E,MAAP,EAAeC,QAAf,EAAyB;AACnC,QAAI,OAAO5E,IAAP,KAAgB,QAApB,EAA8B;AAC5BA,MAAAA,IAAI,GAAGA,IAAI,CAACwE,EAAZ;AACD;;AACD,UAAMY,YAAY,GAAG,KAAKrF,aAAL,CAAmBC,IAAnB,CAArB;;AACA,QAAIqF,QAAJ;;AACA,QAAI;AACFA,MAAAA,QAAQ,GAAG,CAAC,CAAC,MAAM,KAAKlG,GAAL,CAASiD,UAAT,CAAoBgD,YAApB,CAAP,EAA0CX,aAAtD;AACD,KAFD,CAEE,OAAO9C,CAAP,EAAU;AACV,UAAIA,CAAC,CAACW,IAAF,KAAW,UAAf,EAA2B;AACzB+C,QAAAA,QAAQ,GAAG,CAAX;AACD,OAFD,MAEO;AACL,cAAM1D,CAAN;AACD;AACF;;AACD,QAAI0D,QAAQ,GAAGvI,aAAf,EAA8B;AAC5B,YAAMwI,YAAY,GAAGC,MAAM,CAACC,KAAP,CAAarI,IAAI,CAACsI,GAAL,CAASJ,QAAT,EAAmBT,QAAQ,GAAGD,MAAM,CAAClF,MAArC,CAAb,CAArB;AACA,YAAMiG,WAAW,GAAGL,QAAQ,KAAK,CAAb,GAAiB,CAAC,MAAM,KAAKlG,GAAL,CAASkE,SAAT,CAAmB+B,YAAnB,CAAP,EAAyC1C,IAA1D,GAAiE6C,MAAM,CAACC,KAAP,CAAa,CAAb,CAArF;AACAE,MAAAA,WAAW,CAACnB,IAAZ,CAAiBe,YAAjB;AACAX,MAAAA,MAAM,CAACJ,IAAP,CAAYe,YAAZ,EAA0BV,QAA1B;AACA,YAAM,KAAKzF,GAAL,CAAS+D,SAAT,CAAmB,EAAE,GAAGkC,YAAL;AAAmB1C,QAAAA,IAAI,EAAE4C;AAAzB,OAAnB,CAAN;AACA,aAAO;AAAEX,QAAAA,MAAF;AAAUgB,QAAAA,YAAY,EAAEhB,MAAM,CAAClF;AAA/B,OAAP;AACD,KAPD,MAOO;AASL,YAAMe,eAAe,GAAG,MAAM,KAAKrB,GAAL,CAASsB,qBAAT,CAA+B2E,YAA/B,CAA9B;AACA,YAAMQ,mBAAmB,GAAG,EAC1B,GAAGpF,eADuB;AAE1BG,QAAAA,UAAU,EAAG,IAAG,KAAKjB,OAAQ,IAAG,KAAKE,IAAL,GAAYI,IAAK;AAFvB,OAA5B;;AAIA,UAAI;AACF,cAAMY,KAAK,GAAG,EAAd;AACA,cAAMiF,UAAU,GAAGjB,QAAnB;AACA,YAAIkB,YAAY,GAAGD,UAAU,GAAGlB,MAAM,CAAClF,MAAvC;AACA,YAAIsG,UAAU,GAAG5I,IAAI,CAACsI,GAAL,CAAS,CAAT,EAAYJ,QAAQ,GAAGS,YAAvB,CAAjB;AACA,YAAIE,SAAS,GAAGD,UAAU,GAAG,CAA7B;AACA,YAAIE,UAAU,GAAGtB,MAAjB;AACA,YAAIuB,gBAAgB,GAAGtB,QAAvB;AACA,YAAIuB,UAAU,GAAG,CAAjB;AACA,YAAIC,cAAc,GAAG,CAArB;AAEA,YAAIC,cAAc,GAAGlJ,IAAI,CAACmJ,KAAL,CAAWT,UAAU,GAAG9I,aAAxB,CAArB;AACA,cAAMwJ,kBAAkB,GAAGxJ,aAA3B;AACA,YAAIyJ,sBAAsB,GAAGX,UAAU,GAAGU,kBAAkB,GAAGF,cAA/D;;AACA,YAAIG,sBAAsB,IAAI1J,aAA9B,EAA6C;AAG3CuJ,UAAAA,cAAc;AACdG,UAAAA,sBAAsB,GAAG,CAAzB;AACD;;AACD,aAAK,IAAIC,CAAC,GAAG,CAAb,EAAgBA,CAAC,GAAGJ,cAApB,EAAoCI,CAAC,EAArC,EAAyC;AACvC,gBAAMC,WAAW,GAAGvJ,IAAI,CAAC4D,GAAL,CAASqF,cAAc,GAAGG,kBAA1B,EAA8CV,UAA9C,CAApB;;AACAc,0BAAOC,WAAP,CAAmBF,WAAW,GAAGN,cAAd,IAAgCrJ,aAAnD,EAAkE,IAAlE;;AACA,gBAAM+D,KAAK,GAAI,SAAQsF,cAAe,IAAGM,WAAW,GAAG,CAAE,EAAzD;AACA,gBAAMG,gBAAgB,GAAG,EAAE,GAAGjB,mBAAL;AAA0B3E,YAAAA,UAAU,EAAEkF,UAAU,EAAhD;AAAoDjF,YAAAA,eAAe,EAAEJ;AAArE,WAAzB;AACA,gBAAMgG,IAAI,GAAG,MAAM,KAAK3H,GAAL,CAASiC,cAAT,CAAwByF,gBAAxB,CAAnB;AACAjG,UAAAA,KAAK,CAACS,IAAN,CAAW;AAAEC,YAAAA,IAAI,EAAEwF,IAAI,CAACvF,cAAL,CAAoBD,IAA5B;AAAkCL,YAAAA,UAAU,EAAE4F,gBAAgB,CAAC5F;AAA/D,WAAX;AACAmF,UAAAA,cAAc,IAAIG,kBAAlB;AACD;;AACD,YAAIC,sBAAJ,EAA4B;AAE1B,gBAAMO,cAAc,GAAG,EAAE,GAAG3B,YAAL;AAAmBP,YAAAA,KAAK,EAAG,SAAQuB,cAAe,IAAGP,UAAU,GAAG,CAAE;AAApE,WAAvB;AACA,gBAAMmB,YAAY,GAAGnB,UAAU,GAAG,CAAb,GAAiB,CAAC,MAAM,KAAK1G,GAAL,CAASkE,SAAT,CAAmB0D,cAAnB,CAAP,EAA2CrE,IAA5D,GAAmE6C,MAAM,CAACC,KAAP,CAAa,CAAb,CAAxF;AACAS,UAAAA,UAAU,GAAGV,MAAM,CAAC0B,MAAP,CAAc,CAACD,YAAD,EAAerC,MAAf,CAAd,CAAb;AACAuB,UAAAA,gBAAgB,IAAIM,sBAApB;AACD;;AACD,YAAIR,SAAS,IAAIC,UAAU,CAACxG,MAAX,GAAoB3C,aAArC,EAAoD;AAIlD,gBAAMoK,cAAc,GAAG/J,IAAI,CAAC4D,GAAL,CAASjE,aAAa,GAAGmJ,UAAU,CAACxG,MAApC,EAA4CsG,UAA5C,CAAvB;AACA,gBAAMoB,gBAAgB,GAAGjB,gBAAgB,GAAGD,UAAU,CAACxG,MAAvD;AACAqG,UAAAA,YAAY,IAAIoB,cAAhB;AACAnB,UAAAA,UAAU,IAAImB,cAAd;AACAlB,UAAAA,SAAS,GAAGD,UAAU,GAAG,CAAzB;AACA,gBAAMqB,WAAW,GAAI,SAAQD,gBAAiB,IAAGA,gBAAgB,GAAGD,cAAnB,GAAoC,CAAE,EAAvF;AACA,gBAAMH,cAAc,GAAG,EAAE,GAAG3B,YAAL;AAAmBP,YAAAA,KAAK,EAAEuC;AAA1B,WAAvB;AACA,gBAAMC,gBAAgB,GAAG,CAAC,MAAM,KAAKlI,GAAL,CAASkE,SAAT,CAAmB0D,cAAnB,CAAP,EAA2CrE,IAApE;AACAuD,UAAAA,UAAU,GAAGV,MAAM,CAAC0B,MAAP,CAAc,CAAChB,UAAD,EAAaoB,gBAAb,CAAd,CAAb;AACD;;AACD,cAAMC,UAAU,GAAG,EAAE,GAAG9G,eAAL;AAAsBkC,UAAAA,IAAI,EAAEuD,UAA5B;AAAwChF,UAAAA,UAAU,EAAEkF,UAAU;AAA9D,SAAnB;AACA,cAAMoB,QAAQ,GAAG,MAAM,KAAKpI,GAAL,CAASqI,UAAT,CAAoBF,UAApB,CAAvB;AACA1G,QAAAA,KAAK,CAACS,IAAN,CAAW;AAAEC,UAAAA,IAAI,EAAEiG,QAAQ,CAACjG,IAAjB;AAAuBL,UAAAA,UAAU,EAAEqG,UAAU,CAACrG;AAA9C,SAAX;;AACA,YAAI+E,SAAJ,EAAe;AAEb,gBAAMyB,eAAe,GAAGtK,IAAI,CAACC,IAAL,CAAU2I,UAAU,GAAGhJ,aAAvB,CAAxB;AACA,cAAI2K,oBAAoB,GAAG5B,YAA3B;;AACA,eAAK,IAAIW,CAAC,GAAG,CAAb,EAAgBA,CAAC,GAAGgB,eAApB,EAAqChB,CAAC,EAAtC,EAA0C;AACxC,kBAAMC,WAAW,GAAGgB,oBAAoB,GAAG3K,aAA3C;;AACA4J,4BAAOC,WAAP,CAAmBzJ,IAAI,CAAC4D,GAAL,CAASsE,QAAT,EAAmBqB,WAAnB,IAAkCgB,oBAAlC,IAA0D3K,aAA7E,EAA4F,IAA5F;;AACA,kBAAM4K,WAAW,GAAI,SAAQD,oBAAqB,IAAGvK,IAAI,CAAC4D,GAAL,CAASsE,QAAT,EAAmBqB,WAAnB,IAAkC,CAAE,EAAzF;AACA,kBAAMkB,gBAAgB,GAAG,EAAE,GAAGhC,mBAAL;AAA0B3E,cAAAA,UAAU,EAAEkF,UAAU,EAAhD;AAAoDjF,cAAAA,eAAe,EAAEyG;AAArE,aAAzB;AACA,kBAAME,UAAU,GAAG,CAAC,MAAM,KAAK1I,GAAL,CAASiC,cAAT,CAAwBwG,gBAAxB,CAAP,EAAkDrG,cAArE;AACAX,YAAAA,KAAK,CAACS,IAAN,CAAW;AAAEC,cAAAA,IAAI,EAAEuG,UAAU,CAACvG,IAAnB;AAAyBL,cAAAA,UAAU,EAAE2G,gBAAgB,CAAC3G;AAAtD,aAAX;AACAyG,YAAAA,oBAAoB,GAAGhB,WAAvB;AACD;AACF;;AACD,cAAM,KAAKvH,GAAL,CAASqC,uBAAT,CAAiC,EACrC,GAAGhB,eADkC;AAErCiB,UAAAA,eAAe,EAAE;AAAEC,YAAAA,KAAK,EAAEd;AAAT;AAFoB,SAAjC,CAAN;AAID,OAvED,CAuEE,OAAOe,CAAP,EAAU;AACV,cAAM,KAAKxC,GAAL,CAASyC,oBAAT,CAA8BpB,eAA9B,CAAN;AACA,cAAMmB,CAAN;AACD;AACF;AACF;;AAEc,QAATmG,SAAS,CAAClK,IAAD,EAAOmK,KAAP,EAAc;AAC3B,WAAOnK,IAAP;AACD;;AAEe,QAAVoK,UAAU,CAACxD,EAAD,EAAK,CAAE;;AA1WoC","sourcesContent":["import aws from '@sullux/aws-sdk'\nimport assert from 'assert'\nimport http from 'http'\nimport https from 'https'\nimport { parse } from 'xo-remote-parser'\n\nimport RemoteHandlerAbstract from './abstract'\n\n// endpoints https://docs.aws.amazon.com/general/latest/gr/s3.html\n\n// limits: https://docs.aws.amazon.com/AmazonS3/latest/dev/qfacts.html\nconst MIN_PART_SIZE = 1024 * 1024 * 5 // 5MB\nconst MAX_PART_SIZE = 1024 * 1024 * 1024 * 5 // 5GB\nconst MAX_PARTS_COUNT = 10000\nconst MAX_OBJECT_SIZE = 1024 * 1024 * 1024 * 1024 * 5 // 5TB\nconst IDEAL_FRAGMENT_SIZE = Math.ceil(MAX_OBJECT_SIZE / MAX_PARTS_COUNT) // the smallest fragment size that still allows a 5TB upload in 10000 fragments, about 524MB\nexport default class S3Handler extends RemoteHandlerAbstract {\n  constructor(remote, _opts) {\n    super(remote)\n    const { allowUnauthorized, host, path, username, password, protocol, region } = parse(remote.url)\n    const params = {\n      accessKeyId: username,\n      apiVersion: '2006-03-01',\n      endpoint: host,\n      s3ForcePathStyle: true,\n      secretAccessKey: password,\n      signatureVersion: 'v4',\n      httpOptions: {\n        timeout: 600000,\n      },\n    }\n    if (protocol === 'http') {\n      params.httpOptions.agent = new http.Agent({ keepAlive: true })\n      params.sslEnabled = false\n    } else if (protocol === 'https') {\n      params.httpOptions.agent = new https.Agent({\n        rejectUnauthorized: !allowUnauthorized,\n        keepAlive: true,\n      })\n    }\n    if (region !== undefined) {\n      params.region = region\n    }\n\n    this._s3 = aws(params).s3\n\n    const splitPath = path.split('/').filter(s => s.length)\n    this._bucket = splitPath.shift()\n    this._dir = splitPath.join('/')\n  }\n\n  get type() {\n    return 's3'\n  }\n\n  _createParams(file) {\n    return { Bucket: this._bucket, Key: this._dir + file }\n  }\n\n  async _copy(oldPath, newPath) {\n    const size = await this._getSize(oldPath)\n    const multipartParams = await this._s3.createMultipartUpload({ ...this._createParams(newPath) })\n    const param2 = { ...multipartParams, CopySource: `/${this._bucket}/${this._dir}${oldPath}` }\n    try {\n      const parts = []\n      let start = 0\n      while (start < size) {\n        const range = `bytes=${start}-${Math.min(start + MAX_PART_SIZE, size) - 1}`\n        const partParams = { ...param2, PartNumber: parts.length + 1, CopySourceRange: range }\n        const upload = await this._s3.uploadPartCopy(partParams)\n        parts.push({ ETag: upload.CopyPartResult.ETag, PartNumber: partParams.PartNumber })\n        start += MAX_PART_SIZE\n      }\n      await this._s3.completeMultipartUpload({ ...multipartParams, MultipartUpload: { Parts: parts } })\n    } catch (e) {\n      await this._s3.abortMultipartUpload(multipartParams)\n      throw e\n    }\n  }\n\n  async _isNotEmptyDir(path) {\n    const result = await this._s3.listObjectsV2({\n      Bucket: this._bucket,\n      MaxKeys: 1,\n      Prefix: this._dir + path + '/',\n    })\n    return result.Contents.length !== 0\n  }\n\n  async _isFile(path) {\n    try {\n      await this._s3.headObject(this._createParams(path))\n      return true\n    } catch (error) {\n      if (error.code === 'NotFound') {\n        return false\n      }\n      throw error\n    }\n  }\n\n  async _outputStream(path, input, { validator }) {\n    await this._s3.upload(\n      {\n        ...this._createParams(path),\n        Body: input,\n      },\n      { partSize: IDEAL_FRAGMENT_SIZE, queueSize: 1 }\n    )\n    if (validator !== undefined) {\n      try {\n        await validator.call(this, path)\n      } catch (error) {\n        await this.unlink(path)\n        throw error\n      }\n    }\n  }\n\n  async _writeFile(file, data, options) {\n    return this._s3.putObject({ ...this._createParams(file), Body: data })\n  }\n\n  async _createReadStream(path, options) {\n    if (!(await this._isFile(path))) {\n      const error = new Error(`ENOENT: no such file '${path}'`)\n      error.code = 'ENOENT'\n      error.path = path\n      throw error\n    }\n\n    // https://github.com/Sullux/aws-sdk/issues/11\n    return this._s3.getObject.raw(this._createParams(path)).createReadStream()\n  }\n\n  async _unlink(path) {\n    await this._s3.deleteObject(this._createParams(path))\n    if (await this._isNotEmptyDir(path)) {\n      const error = new Error(`EISDIR: illegal operation on a directory, unlink '${path}'`)\n      error.code = 'EISDIR'\n      error.path = path\n      throw error\n    }\n  }\n\n  async _list(dir) {\n    function splitPath(path) {\n      return path.split('/').filter(d => d.length)\n    }\n\n    const prefix = [this._dir, dir].join('/')\n    const splitPrefix = splitPath(prefix)\n    const result = await this._s3.listObjectsV2({\n      Bucket: this._bucket,\n      Prefix: splitPrefix.join('/') + '/', // need slash at the end with the use of delimiters\n      Delimiter: '/', // will only return path until delimiters\n    })\n\n    if (result.isTruncated) {\n      const error = new Error('more than 1000 objects, unsupported in this implementation')\n      error.dir = dir\n      throw error\n    }\n\n    const uniq = []\n\n    // sub directories\n    for (const entry of result.CommonPrefixes) {\n      const line = splitPath(entry.Prefix)\n      uniq.push(line[line.length - 1])\n    }\n    // files\n    for (const entry of result.Contents) {\n      const line = splitPath(entry.Key)\n      uniq.push(line[line.length - 1])\n    }\n\n    return uniq\n  }\n\n  async _mkdir(path) {\n    if (await this._isFile(path)) {\n      const error = new Error(`ENOTDIR: file already exists, mkdir '${path}'`)\n      error.code = 'ENOTDIR'\n      error.path = path\n      throw error\n    }\n    // nothing to do, directories do not exist, they are part of the files' path\n  }\n\n  // s3 doesn't have a rename operation, so copy + delete source\n  async _rename(oldPath, newPath) {\n    await this.copy(oldPath, newPath)\n    await this._s3.deleteObject(this._createParams(oldPath))\n  }\n\n  async _getSize(file) {\n    if (typeof file !== 'string') {\n      file = file.fd\n    }\n    const result = await this._s3.headObject(this._createParams(file))\n    return +result.ContentLength\n  }\n\n  async _read(file, buffer, position = 0) {\n    if (typeof file !== 'string') {\n      file = file.fd\n    }\n    const params = this._createParams(file)\n    params.Range = `bytes=${position}-${position + buffer.length - 1}`\n    try {\n      const result = await this._s3.getObject(params)\n      result.Body.copy(buffer)\n      return { bytesRead: result.Body.length, buffer }\n    } catch (e) {\n      if (e.code === 'NoSuchKey') {\n        if (await this._isNotEmptyDir(file)) {\n          const error = new Error(`${file} is a directory`)\n          error.code = 'EISDIR'\n          error.path = file\n          throw error\n        }\n      }\n      throw e\n    }\n  }\n\n  async _rmdir(path) {\n    if (await this._isNotEmptyDir(path)) {\n      const error = new Error(`ENOTEMPTY: directory not empty, rmdir '${path}`)\n      error.code = 'ENOTEMPTY'\n      error.path = path\n      throw error\n    }\n\n    // nothing to do, directories do not exist, they are part of the files' path\n  }\n\n  // reimplement _rmtree to handle efficiantly path with more than 1000 entries in trees\n  // @todo : use parallel processing for unlink\n  async _rmtree(path) {\n    let NextContinuationToken\n    do {\n      const result = await this._s3.listObjectsV2({\n        Bucket: this._bucket,\n        Prefix: this._dir + path + '/',\n        ContinuationToken: NextContinuationToken,\n      })\n      NextContinuationToken = result.isTruncated ? result.NextContinuationToken : undefined\n      for (const { Key } of result.Contents) {\n        // _unlink will add the prefix, but Key contains everything\n        // also we don't need to check if we delete a directory, since the list only return files\n        await this._s3.deleteObject({\n          Bucket: this._bucket,\n          Key,\n        })\n      }\n    } while (NextContinuationToken !== undefined)\n  }\n\n  async _write(file, buffer, position) {\n    if (typeof file !== 'string') {\n      file = file.fd\n    }\n    const uploadParams = this._createParams(file)\n    let fileSize\n    try {\n      fileSize = +(await this._s3.headObject(uploadParams)).ContentLength\n    } catch (e) {\n      if (e.code === 'NotFound') {\n        fileSize = 0\n      } else {\n        throw e\n      }\n    }\n    if (fileSize < MIN_PART_SIZE) {\n      const resultBuffer = Buffer.alloc(Math.max(fileSize, position + buffer.length))\n      const fileContent = fileSize !== 0 ? (await this._s3.getObject(uploadParams)).Body : Buffer.alloc(0)\n      fileContent.copy(resultBuffer)\n      buffer.copy(resultBuffer, position)\n      await this._s3.putObject({ ...uploadParams, Body: resultBuffer })\n      return { buffer, bytesWritten: buffer.length }\n    } else {\n      // using this trick: https://stackoverflow.com/a/38089437/72637\n      // multipart fragments have a minimum size of 5Mo and a max of 5Go unless they are last\n      // splitting the file in 3 parts: [prefix, edit, suffix]\n      // if `prefix` is bigger than 5Mo, it will be sourced from uploadPartCopy()\n      // otherwise otherwise it will be downloaded, concatenated to `edit`\n      // `edit` will always be an upload part\n      // `suffix` will always be sourced from uploadPartCopy()\n      // Then everything will be sliced in 5Gb parts before getting uploaded\n      const multipartParams = await this._s3.createMultipartUpload(uploadParams)\n      const copyMultipartParams = {\n        ...multipartParams,\n        CopySource: `/${this._bucket}/${this._dir + file}`,\n      }\n      try {\n        const parts = []\n        const prefixSize = position\n        let suffixOffset = prefixSize + buffer.length\n        let suffixSize = Math.max(0, fileSize - suffixOffset)\n        let hasSuffix = suffixSize > 0\n        let editBuffer = buffer\n        let editBufferOffset = position\n        let partNumber = 1\n        let prefixPosition = 0\n        // use floor() so that last fragment is handled in the if bellow\n        let fragmentsCount = Math.floor(prefixSize / MAX_PART_SIZE)\n        const prefixFragmentSize = MAX_PART_SIZE\n        let prefixLastFragmentSize = prefixSize - prefixFragmentSize * fragmentsCount\n        if (prefixLastFragmentSize >= MIN_PART_SIZE) {\n          // the last fragment of the prefix is smaller than MAX_PART_SIZE, but bigger than the minimum\n          // so we can copy it too\n          fragmentsCount++\n          prefixLastFragmentSize = 0\n        }\n        for (let i = 0; i < fragmentsCount; i++) {\n          const fragmentEnd = Math.min(prefixPosition + prefixFragmentSize, prefixSize)\n          assert.strictEqual(fragmentEnd - prefixPosition <= MAX_PART_SIZE, true)\n          const range = `bytes=${prefixPosition}-${fragmentEnd - 1}`\n          const copyPrefixParams = { ...copyMultipartParams, PartNumber: partNumber++, CopySourceRange: range }\n          const part = await this._s3.uploadPartCopy(copyPrefixParams)\n          parts.push({ ETag: part.CopyPartResult.ETag, PartNumber: copyPrefixParams.PartNumber })\n          prefixPosition += prefixFragmentSize\n        }\n        if (prefixLastFragmentSize) {\n          // grab everything from the prefix that was too small to be copied, download and merge to the edit buffer.\n          const downloadParams = { ...uploadParams, Range: `bytes=${prefixPosition}-${prefixSize - 1}` }\n          const prefixBuffer = prefixSize > 0 ? (await this._s3.getObject(downloadParams)).Body : Buffer.alloc(0)\n          editBuffer = Buffer.concat([prefixBuffer, buffer])\n          editBufferOffset -= prefixLastFragmentSize\n        }\n        if (hasSuffix && editBuffer.length < MIN_PART_SIZE) {\n          // the edit fragment is too short and is not the last fragment\n          // let's steal from the suffix fragment to reach the minimum size\n          // the suffix might be too short and itself entirely absorbed in the edit fragment, making it the last one.\n          const complementSize = Math.min(MIN_PART_SIZE - editBuffer.length, suffixSize)\n          const complementOffset = editBufferOffset + editBuffer.length\n          suffixOffset += complementSize\n          suffixSize -= complementSize\n          hasSuffix = suffixSize > 0\n          const prefixRange = `bytes=${complementOffset}-${complementOffset + complementSize - 1}`\n          const downloadParams = { ...uploadParams, Range: prefixRange }\n          const complementBuffer = (await this._s3.getObject(downloadParams)).Body\n          editBuffer = Buffer.concat([editBuffer, complementBuffer])\n        }\n        const editParams = { ...multipartParams, Body: editBuffer, PartNumber: partNumber++ }\n        const editPart = await this._s3.uploadPart(editParams)\n        parts.push({ ETag: editPart.ETag, PartNumber: editParams.PartNumber })\n        if (hasSuffix) {\n          // use ceil because the last fragment can be arbitrarily small.\n          const suffixFragments = Math.ceil(suffixSize / MAX_PART_SIZE)\n          let suffixFragmentOffset = suffixOffset\n          for (let i = 0; i < suffixFragments; i++) {\n            const fragmentEnd = suffixFragmentOffset + MAX_PART_SIZE\n            assert.strictEqual(Math.min(fileSize, fragmentEnd) - suffixFragmentOffset <= MAX_PART_SIZE, true)\n            const suffixRange = `bytes=${suffixFragmentOffset}-${Math.min(fileSize, fragmentEnd) - 1}`\n            const copySuffixParams = { ...copyMultipartParams, PartNumber: partNumber++, CopySourceRange: suffixRange }\n            const suffixPart = (await this._s3.uploadPartCopy(copySuffixParams)).CopyPartResult\n            parts.push({ ETag: suffixPart.ETag, PartNumber: copySuffixParams.PartNumber })\n            suffixFragmentOffset = fragmentEnd\n          }\n        }\n        await this._s3.completeMultipartUpload({\n          ...multipartParams,\n          MultipartUpload: { Parts: parts },\n        })\n      } catch (e) {\n        await this._s3.abortMultipartUpload(multipartParams)\n        throw e\n      }\n    }\n  }\n\n  async _openFile(path, flags) {\n    return path\n  }\n\n  async _closeFile(fd) {}\n}\n"],"file":"s3.js"}
+{"version":3,"sources":["../src/s3.js"],"names":["MIN_PART_SIZE","MAX_PART_SIZE","MAX_PARTS_COUNT","MAX_OBJECT_SIZE","IDEAL_FRAGMENT_SIZE","Math","ceil","warn","S3Handler","pRetry","wrap","delays","when","e","code","onRetry","error","attemptNumber","delay","file","arguments","RemoteHandlerAbstract","constructor","remote","_opts","allowUnauthorized","host","path","username","password","protocol","region","url","params","accessKeyId","apiVersion","endpoint","s3ForcePathStyle","secretAccessKey","signatureVersion","httpOptions","timeout","agent","http","Agent","keepAlive","sslEnabled","https","rejectUnauthorized","undefined","_s3","s3","splitPath","split","filter","s","length","_bucket","shift","_dir","join","type","_createParams","Bucket","Key","_multipartCopy","oldPath","newPath","size","_getSize","CopySource","multipartParams","createMultipartUpload","param2","parts","start","range","min","partParams","PartNumber","CopySourceRange","upload","uploadPartCopy","push","ETag","CopyPartResult","completeMultipartUpload","MultipartUpload","Parts","abortMultipartUpload","_copy","copyObject","_isNotEmptyDir","result","listObjectsV2","MaxKeys","Prefix","Contents","_isFile","headObject","_outputStream","input","validator","Body","partSize","queueSize","call","unlink","_writeFile","data","options","putObject","_createReadStream","Error","getObject","raw","createReadStream","_unlink","deleteObject","_list","dir","d","prefix","splitPrefix","Delimiter","IsTruncated","uniq","entry","CommonPrefixes","line","_mkdir","_rename","copy","fd","ContentLength","_read","buffer","position","Range","bytesRead","_rmdir","_rmtree","NextContinuationToken","ContinuationToken","concurrency","_write","uploadParams","fileSize","resultBuffer","Buffer","alloc","max","fileContent","bytesWritten","copyMultipartParams","prefixSize","suffixOffset","suffixSize","hasSuffix","editBuffer","editBufferOffset","partNumber","prefixPosition","fragmentsCount","floor","prefixFragmentSize","prefixLastFragmentSize","i","fragmentEnd","assert","strictEqual","copyPrefixParams","part","downloadParams","prefixBuffer","concat","complementSize","complementOffset","prefixRange","complementBuffer","editParams","editPart","uploadPart","suffixFragments","suffixFragmentOffset","suffixRange","copySuffixParams","suffixPart","_openFile","flags","_closeFile"],"mappings":";;;;;;;AAAA;;AACA;;AACA;;AACA;;AACA;;AACA;;AACA;;AACA;;AAEA;;AACA;;;;;;;;AAKA,MAAMA,aAAa,GAAG,OAAO,IAAP,GAAc,CAApC;AACA,MAAMC,aAAa,GAAG,OAAO,IAAP,GAAc,IAAd,GAAqB,CAA3C;AACA,MAAMC,eAAe,GAAG,KAAxB;AACA,MAAMC,eAAe,GAAG,OAAO,IAAP,GAAc,IAAd,GAAqB,IAArB,GAA4B,CAApD;AACA,MAAMC,mBAAmB,GAAGC,IAAI,CAACC,IAAL,CAAUH,eAAe,GAAGD,eAA5B,CAA5B;AAEA,MAAM;AAAEK,EAAAA;AAAF,IAAW,uBAAa,UAAb,CAAjB;IAEqBC,S,WA2HlB,gCAAaC,eAAOC,IAApB,EAA0B;AACzBC,EAAAA,MAAM,EAAE,CAAC,GAAD,EAAM,GAAN,EAAW,GAAX,EAAgB,IAAhB,EAAsB,IAAtB,CADiB;AAEzBC,EAAAA,IAAI,EAAEC,CAAC,IAAIA,CAAC,CAACC,IAAF,KAAW,eAFG;;AAGzBC,EAAAA,OAAO,CAACC,KAAD,EAAQ;AACbT,IAAAA,IAAI,CAAC,uBAAD,EAA0B;AAC5BU,MAAAA,aAAa,EAAE,KAAKA,aADQ;AAE5BC,MAAAA,KAAK,EAAE,KAAKA,KAFgB;AAG5BF,MAAAA,KAH4B;AAI5BG,MAAAA,IAAI,EAAE,KAAKC,SAAL,CAAe,CAAf;AAJsB,KAA1B,CAAJ;AAMD;;AAVwB,CAA1B,C,YA3HY,MAAMZ,SAAN,SAAwBa,iBAAxB,CAA8C;AAC3DC,EAAAA,WAAW,CAACC,MAAD,EAASC,KAAT,EAAgB;AACzB,UAAMD,MAAN;AACA,UAAM;AAAEE,MAAAA,iBAAF;AAAqBC,MAAAA,IAArB;AAA2BC,MAAAA,IAA3B;AAAiCC,MAAAA,QAAjC;AAA2CC,MAAAA,QAA3C;AAAqDC,MAAAA,QAArD;AAA+DC,MAAAA;AAA/D,QAA0E,2BAAMR,MAAM,CAACS,GAAb,CAAhF;AACA,UAAMC,MAAM,GAAG;AACbC,MAAAA,WAAW,EAAEN,QADA;AAEbO,MAAAA,UAAU,EAAE,YAFC;AAGbC,MAAAA,QAAQ,EAAEV,IAHG;AAIbW,MAAAA,gBAAgB,EAAE,IAJL;AAKbC,MAAAA,eAAe,EAAET,QALJ;AAMbU,MAAAA,gBAAgB,EAAE,IANL;AAObC,MAAAA,WAAW,EAAE;AACXC,QAAAA,OAAO,EAAE;AADE;AAPA,KAAf;;AAWA,QAAIX,QAAQ,KAAK,MAAjB,EAAyB;AACvBG,MAAAA,MAAM,CAACO,WAAP,CAAmBE,KAAnB,GAA2B,IAAIC,cAAKC,KAAT,CAAe;AAAEC,QAAAA,SAAS,EAAE;AAAb,OAAf,CAA3B;AACAZ,MAAAA,MAAM,CAACa,UAAP,GAAoB,KAApB;AACD,KAHD,MAGO,IAAIhB,QAAQ,KAAK,OAAjB,EAA0B;AAC/BG,MAAAA,MAAM,CAACO,WAAP,CAAmBE,KAAnB,GAA2B,IAAIK,eAAMH,KAAV,CAAgB;AACzCI,QAAAA,kBAAkB,EAAE,CAACvB,iBADoB;AAEzCoB,QAAAA,SAAS,EAAE;AAF8B,OAAhB,CAA3B;AAID;;AACD,QAAId,MAAM,KAAKkB,SAAf,EAA0B;AACxBhB,MAAAA,MAAM,CAACF,MAAP,GAAgBA,MAAhB;AACD;;AAED,SAAKmB,GAAL,GAAW,qBAAIjB,MAAJ,EAAYkB,EAAvB;AAEA,UAAMC,SAAS,GAAGzB,IAAI,CAAC0B,KAAL,CAAW,GAAX,EAAgBC,MAAhB,CAAuBC,CAAC,IAAIA,CAAC,CAACC,MAA9B,CAAlB;AACA,SAAKC,OAAL,GAAeL,SAAS,CAACM,KAAV,EAAf;AACA,SAAKC,IAAL,GAAYP,SAAS,CAACQ,IAAV,CAAe,GAAf,CAAZ;AACD;;AAEO,MAAJC,IAAI,GAAG;AACT,WAAO,IAAP;AACD;;AAEDC,EAAAA,aAAa,CAAC3C,IAAD,EAAO;AAClB,WAAO;AAAE4C,MAAAA,MAAM,EAAE,KAAKN,OAAf;AAAwBO,MAAAA,GAAG,EAAE,KAAKL,IAAL,GAAYxC;AAAzC,KAAP;AACD;;AAEmB,QAAd8C,cAAc,CAACC,OAAD,EAAUC,OAAV,EAAmB;AACrC,UAAMC,IAAI,GAAG,MAAM,KAAKC,QAAL,CAAcH,OAAd,CAAnB;AACA,UAAMI,UAAU,GAAI,IAAG,KAAKb,OAAQ,IAAG,KAAKE,IAAK,GAAEO,OAAQ,EAA3D;AACA,UAAMK,eAAe,GAAG,MAAM,KAAKrB,GAAL,CAASsB,qBAAT,CAA+B,EAAE,GAAG,KAAKV,aAAL,CAAmBK,OAAnB;AAAL,KAA/B,CAA9B;AACA,UAAMM,MAAM,GAAG,EAAE,GAAGF,eAAL;AAAsBD,MAAAA;AAAtB,KAAf;;AACA,QAAI;AACF,YAAMI,KAAK,GAAG,EAAd;AACA,UAAIC,KAAK,GAAG,CAAZ;;AACA,aAAOA,KAAK,GAAGP,IAAf,EAAqB;AACnB,cAAMQ,KAAK,GAAI,SAAQD,KAAM,IAAGtE,IAAI,CAACwE,GAAL,CAASF,KAAK,GAAG1E,aAAjB,EAAgCmE,IAAhC,IAAwC,CAAE,EAA1E;AACA,cAAMU,UAAU,GAAG,EAAE,GAAGL,MAAL;AAAaM,UAAAA,UAAU,EAAEL,KAAK,CAAClB,MAAN,GAAe,CAAxC;AAA2CwB,UAAAA,eAAe,EAAEJ;AAA5D,SAAnB;AACA,cAAMK,MAAM,GAAG,MAAM,KAAK/B,GAAL,CAASgC,cAAT,CAAwBJ,UAAxB,CAArB;AACAJ,QAAAA,KAAK,CAACS,IAAN,CAAW;AAAEC,UAAAA,IAAI,EAAEH,MAAM,CAACI,cAAP,CAAsBD,IAA9B;AAAoCL,UAAAA,UAAU,EAAED,UAAU,CAACC;AAA3D,SAAX;AACAJ,QAAAA,KAAK,IAAI1E,aAAT;AACD;;AACD,YAAM,KAAKiD,GAAL,CAASoC,uBAAT,CAAiC,EAAE,GAAGf,eAAL;AAAsBgB,QAAAA,eAAe,EAAE;AAAEC,UAAAA,KAAK,EAAEd;AAAT;AAAvC,OAAjC,CAAN;AACD,KAXD,CAWE,OAAO7D,CAAP,EAAU;AACV,YAAM,KAAKqC,GAAL,CAASuC,oBAAT,CAA8BlB,eAA9B,CAAN;AACA,YAAM1D,CAAN;AACD;AACF;;AAEU,QAAL6E,KAAK,CAACxB,OAAD,EAAUC,OAAV,EAAmB;AAC5B,UAAMG,UAAU,GAAI,IAAG,KAAKb,OAAQ,IAAG,KAAKE,IAAK,GAAEO,OAAQ,EAA3D;;AACA,QAAI;AACF,YAAM,KAAKhB,GAAL,CAASyC,UAAT,CAAoB,EACxB,GAAG,KAAK7B,aAAL,CAAmBK,OAAnB,CADqB;AAExBG,QAAAA;AAFwB,OAApB,CAAN;AAID,KALD,CAKE,OAAOzD,CAAP,EAAU;AAEV,UAAIA,CAAC,CAACC,IAAF,KAAW,gBAAf,EAAiC;AAC/B,eAAO,KAAKmD,cAAL,CAAoBC,OAApB,EAA6BC,OAA7B,CAAP;AACD;;AACD,YAAMtD,CAAN;AACD;AACF;;AAEmB,QAAd+E,cAAc,CAACjE,IAAD,EAAO;AACzB,UAAMkE,MAAM,GAAG,MAAM,KAAK3C,GAAL,CAAS4C,aAAT,CAAuB;AAC1C/B,MAAAA,MAAM,EAAE,KAAKN,OAD6B;AAE1CsC,MAAAA,OAAO,EAAE,CAFiC;AAG1CC,MAAAA,MAAM,EAAE,KAAKrC,IAAL,GAAYhC,IAAZ,GAAmB;AAHe,KAAvB,CAArB;AAKA,WAAOkE,MAAM,CAACI,QAAP,CAAgBzC,MAAhB,KAA2B,CAAlC;AACD;;AAEY,QAAP0C,OAAO,CAACvE,IAAD,EAAO;AAClB,QAAI;AACF,YAAM,KAAKuB,GAAL,CAASiD,UAAT,CAAoB,KAAKrC,aAAL,CAAmBnC,IAAnB,CAApB,CAAN;AACA,aAAO,IAAP;AACD,KAHD,CAGE,OAAOX,KAAP,EAAc;AACd,UAAIA,KAAK,CAACF,IAAN,KAAe,UAAnB,EAA+B;AAC7B,eAAO,KAAP;AACD;;AACD,YAAME,KAAN;AACD;AACF;;AAEkB,QAAboF,aAAa,CAACzE,IAAD,EAAO0E,KAAP,EAAc;AAAEC,IAAAA;AAAF,GAAd,EAA6B;AAC9C,UAAM,KAAKpD,GAAL,CAAS+B,MAAT,CACJ,EACE,GAAG,KAAKnB,aAAL,CAAmBnC,IAAnB,CADL;AAEE4E,MAAAA,IAAI,EAAEF;AAFR,KADI,EAKJ;AAAEG,MAAAA,QAAQ,EAAEpG,mBAAZ;AAAiCqG,MAAAA,SAAS,EAAE;AAA5C,KALI,CAAN;;AAOA,QAAIH,SAAS,KAAKrD,SAAlB,EAA6B;AAC3B,UAAI;AACF,cAAMqD,SAAS,CAACI,IAAV,CAAe,IAAf,EAAqB/E,IAArB,CAAN;AACD,OAFD,CAEE,OAAOX,KAAP,EAAc;AACd,cAAM,KAAK2F,MAAL,CAAYhF,IAAZ,CAAN;AACA,cAAMX,KAAN;AACD;AACF;AACF;;AAiBe,QAAV4F,UAAU,CAACzF,IAAD,EAAO0F,IAAP,EAAaC,OAAb,EAAsB;AACpC,WAAO,KAAK5D,GAAL,CAAS6D,SAAT,CAAmB,EAAE,GAAG,KAAKjD,aAAL,CAAmB3C,IAAnB,CAAL;AAA+BoF,MAAAA,IAAI,EAAEM;AAArC,KAAnB,CAAP;AACD;;AAEsB,QAAjBG,iBAAiB,CAACrF,IAAD,EAAOmF,OAAP,EAAgB;AACrC,QAAI,EAAE,MAAM,KAAKZ,OAAL,CAAavE,IAAb,CAAR,CAAJ,EAAiC;AAC/B,YAAMX,KAAK,GAAG,IAAIiG,KAAJ,CAAW,yBAAwBtF,IAAK,GAAxC,CAAd;AACAX,MAAAA,KAAK,CAACF,IAAN,GAAa,QAAb;AACAE,MAAAA,KAAK,CAACW,IAAN,GAAaA,IAAb;AACA,YAAMX,KAAN;AACD;;AAGD,WAAO,KAAKkC,GAAL,CAASgE,SAAT,CAAmBC,GAAnB,CAAuB,KAAKrD,aAAL,CAAmBnC,IAAnB,CAAvB,EAAiDyF,gBAAjD,EAAP;AACD;;AAEY,QAAPC,OAAO,CAAC1F,IAAD,EAAO;AAClB,UAAM,KAAKuB,GAAL,CAASoE,YAAT,CAAsB,KAAKxD,aAAL,CAAmBnC,IAAnB,CAAtB,CAAN;;AACA,QAAI,MAAM,KAAKiE,cAAL,CAAoBjE,IAApB,CAAV,EAAqC;AACnC,YAAMX,KAAK,GAAG,IAAIiG,KAAJ,CAAW,qDAAoDtF,IAAK,GAApE,CAAd;AACAX,MAAAA,KAAK,CAACF,IAAN,GAAa,QAAb;AACAE,MAAAA,KAAK,CAACW,IAAN,GAAaA,IAAb;AACA,YAAMX,KAAN;AACD;AACF;;AAEU,QAALuG,KAAK,CAACC,GAAD,EAAM;AACf,aAASpE,SAAT,CAAmBzB,IAAnB,EAAyB;AACvB,aAAOA,IAAI,CAAC0B,KAAL,CAAW,GAAX,EAAgBC,MAAhB,CAAuBmE,CAAC,IAAIA,CAAC,CAACjE,MAA9B,CAAP;AACD;;AAED,UAAMkE,MAAM,GAAG,CAAC,KAAK/D,IAAN,EAAY6D,GAAZ,EAAiB5D,IAAjB,CAAsB,GAAtB,CAAf;AACA,UAAM+D,WAAW,GAAGvE,SAAS,CAACsE,MAAD,CAA7B;AACA,UAAM7B,MAAM,GAAG,MAAM,KAAK3C,GAAL,CAAS4C,aAAT,CAAuB;AAC1C/B,MAAAA,MAAM,EAAE,KAAKN,OAD6B;AAE1CuC,MAAAA,MAAM,EAAE2B,WAAW,CAAC/D,IAAZ,CAAiB,GAAjB,IAAwB,GAFU;AAG1CgE,MAAAA,SAAS,EAAE;AAH+B,KAAvB,CAArB;;AAMA,QAAI/B,MAAM,CAACgC,WAAX,EAAwB;AACtB,YAAM7G,KAAK,GAAG,IAAIiG,KAAJ,CAAU,4DAAV,CAAd;AACAjG,MAAAA,KAAK,CAACwG,GAAN,GAAYA,GAAZ;AACA,YAAMxG,KAAN;AACD;;AAED,UAAM8G,IAAI,GAAG,EAAb;;AAGA,SAAK,MAAMC,KAAX,IAAoBlC,MAAM,CAACmC,cAA3B,EAA2C;AACzC,YAAMC,IAAI,GAAG7E,SAAS,CAAC2E,KAAK,CAAC/B,MAAP,CAAtB;AACA8B,MAAAA,IAAI,CAAC3C,IAAL,CAAU8C,IAAI,CAACA,IAAI,CAACzE,MAAL,GAAc,CAAf,CAAd;AACD;;AAED,SAAK,MAAMuE,KAAX,IAAoBlC,MAAM,CAACI,QAA3B,EAAqC;AACnC,YAAMgC,IAAI,GAAG7E,SAAS,CAAC2E,KAAK,CAAC/D,GAAP,CAAtB;AACA8D,MAAAA,IAAI,CAAC3C,IAAL,CAAU8C,IAAI,CAACA,IAAI,CAACzE,MAAL,GAAc,CAAf,CAAd;AACD;;AAED,WAAOsE,IAAP;AACD;;AAEW,QAANI,MAAM,CAACvG,IAAD,EAAO;AACjB,QAAI,MAAM,KAAKuE,OAAL,CAAavE,IAAb,CAAV,EAA8B;AAC5B,YAAMX,KAAK,GAAG,IAAIiG,KAAJ,CAAW,wCAAuCtF,IAAK,GAAvD,CAAd;AACAX,MAAAA,KAAK,CAACF,IAAN,GAAa,SAAb;AACAE,MAAAA,KAAK,CAACW,IAAN,GAAaA,IAAb;AACA,YAAMX,KAAN;AACD;AAEF;;AAGY,QAAPmH,OAAO,CAACjE,OAAD,EAAUC,OAAV,EAAmB;AAC9B,UAAM,KAAKiE,IAAL,CAAUlE,OAAV,EAAmBC,OAAnB,CAAN;AACA,UAAM,KAAKjB,GAAL,CAASoE,YAAT,CAAsB,KAAKxD,aAAL,CAAmBI,OAAnB,CAAtB,CAAN;AACD;;AAEa,QAARG,QAAQ,CAAClD,IAAD,EAAO;AACnB,QAAI,OAAOA,IAAP,KAAgB,QAApB,EAA8B;AAC5BA,MAAAA,IAAI,GAAGA,IAAI,CAACkH,EAAZ;AACD;;AACD,UAAMxC,MAAM,GAAG,MAAM,KAAK3C,GAAL,CAASiD,UAAT,CAAoB,KAAKrC,aAAL,CAAmB3C,IAAnB,CAApB,CAArB;AACA,WAAO,CAAC0E,MAAM,CAACyC,aAAf;AACD;;AAEU,QAALC,KAAK,CAACpH,IAAD,EAAOqH,MAAP,EAAeC,QAAQ,GAAG,CAA1B,EAA6B;AACtC,QAAI,OAAOtH,IAAP,KAAgB,QAApB,EAA8B;AAC5BA,MAAAA,IAAI,GAAGA,IAAI,CAACkH,EAAZ;AACD;;AACD,UAAMpG,MAAM,GAAG,KAAK6B,aAAL,CAAmB3C,IAAnB,CAAf;;AACAc,IAAAA,MAAM,CAACyG,KAAP,GAAgB,SAAQD,QAAS,IAAGA,QAAQ,GAAGD,MAAM,CAAChF,MAAlB,GAA2B,CAAE,EAAjE;;AACA,QAAI;AACF,YAAMqC,MAAM,GAAG,MAAM,KAAK3C,GAAL,CAASgE,SAAT,CAAmBjF,MAAnB,CAArB;AACA4D,MAAAA,MAAM,CAACU,IAAP,CAAY6B,IAAZ,CAAiBI,MAAjB;AACA,aAAO;AAAEG,QAAAA,SAAS,EAAE9C,MAAM,CAACU,IAAP,CAAY/C,MAAzB;AAAiCgF,QAAAA;AAAjC,OAAP;AACD,KAJD,CAIE,OAAO3H,CAAP,EAAU;AACV,UAAIA,CAAC,CAACC,IAAF,KAAW,WAAf,EAA4B;AAC1B,YAAI,MAAM,KAAK8E,cAAL,CAAoBzE,IAApB,CAAV,EAAqC;AACnC,gBAAMH,KAAK,GAAG,IAAIiG,KAAJ,CAAW,GAAE9F,IAAK,iBAAlB,CAAd;AACAH,UAAAA,KAAK,CAACF,IAAN,GAAa,QAAb;AACAE,UAAAA,KAAK,CAACW,IAAN,GAAaR,IAAb;AACA,gBAAMH,KAAN;AACD;AACF;;AACD,YAAMH,CAAN;AACD;AACF;;AAEW,QAAN+H,MAAM,CAACjH,IAAD,EAAO;AACjB,QAAI,MAAM,KAAKiE,cAAL,CAAoBjE,IAApB,CAAV,EAAqC;AACnC,YAAMX,KAAK,GAAG,IAAIiG,KAAJ,CAAW,0CAAyCtF,IAAK,EAAzD,CAAd;AACAX,MAAAA,KAAK,CAACF,IAAN,GAAa,WAAb;AACAE,MAAAA,KAAK,CAACW,IAAN,GAAaA,IAAb;AACA,YAAMX,KAAN;AACD;AAGF;;AAIY,QAAP6H,OAAO,CAAClH,IAAD,EAAO;AAClB,QAAImH,qBAAJ;;AACA,OAAG;AACD,YAAMjD,MAAM,GAAG,MAAM,KAAK3C,GAAL,CAAS4C,aAAT,CAAuB;AAC1C/B,QAAAA,MAAM,EAAE,KAAKN,OAD6B;AAE1CuC,QAAAA,MAAM,EAAE,KAAKrC,IAAL,GAAYhC,IAAZ,GAAmB,GAFe;AAG1CoH,QAAAA,iBAAiB,EAAED;AAHuB,OAAvB,CAArB;AAKAA,MAAAA,qBAAqB,GAAGjD,MAAM,CAACgC,WAAP,GAAqBhC,MAAM,CAACiD,qBAA5B,GAAoD7F,SAA5E;AACA,YAAM,0BACJ4C,MAAM,CAACI,QADH,EAEJ,OAAO;AAAEjC,QAAAA;AAAF,OAAP,KAAmB;AAGjB,cAAM,KAAKd,GAAL,CAASoE,YAAT,CAAsB;AAC1BvD,UAAAA,MAAM,EAAE,KAAKN,OADa;AAE1BO,UAAAA;AAF0B,SAAtB,CAAN;AAID,OATG,EAUJ;AACEgF,QAAAA,WAAW,EAAE;AADf,OAVI,CAAN;AAcD,KArBD,QAqBSF,qBAAqB,KAAK7F,SArBnC;AAsBD;;AAEW,QAANgG,MAAM,CAAC9H,IAAD,EAAOqH,MAAP,EAAeC,QAAf,EAAyB;AACnC,QAAI,OAAOtH,IAAP,KAAgB,QAApB,EAA8B;AAC5BA,MAAAA,IAAI,GAAGA,IAAI,CAACkH,EAAZ;AACD;;AACD,UAAMa,YAAY,GAAG,KAAKpF,aAAL,CAAmB3C,IAAnB,CAArB;;AACA,QAAIgI,QAAJ;;AACA,QAAI;AACFA,MAAAA,QAAQ,GAAG,CAAC,CAAC,MAAM,KAAKjG,GAAL,CAASiD,UAAT,CAAoB+C,YAApB,CAAP,EAA0CZ,aAAtD;AACD,KAFD,CAEE,OAAOzH,CAAP,EAAU;AACV,UAAIA,CAAC,CAACC,IAAF,KAAW,UAAf,EAA2B;AACzBqI,QAAAA,QAAQ,GAAG,CAAX;AACD,OAFD,MAEO;AACL,cAAMtI,CAAN;AACD;AACF;;AACD,QAAIsI,QAAQ,GAAGnJ,aAAf,EAA8B;AAC5B,YAAMoJ,YAAY,GAAGC,MAAM,CAACC,KAAP,CAAajJ,IAAI,CAACkJ,GAAL,CAASJ,QAAT,EAAmBV,QAAQ,GAAGD,MAAM,CAAChF,MAArC,CAAb,CAArB;AACA,YAAMgG,WAAW,GAAGL,QAAQ,KAAK,CAAb,GAAiB,CAAC,MAAM,KAAKjG,GAAL,CAASgE,SAAT,CAAmBgC,YAAnB,CAAP,EAAyC3C,IAA1D,GAAiE8C,MAAM,CAACC,KAAP,CAAa,CAAb,CAArF;AACAE,MAAAA,WAAW,CAACpB,IAAZ,CAAiBgB,YAAjB;AACAZ,MAAAA,MAAM,CAACJ,IAAP,CAAYgB,YAAZ,EAA0BX,QAA1B;AACA,YAAM,KAAKvF,GAAL,CAAS6D,SAAT,CAAmB,EAAE,GAAGmC,YAAL;AAAmB3C,QAAAA,IAAI,EAAE6C;AAAzB,OAAnB,CAAN;AACA,aAAO;AAAEZ,QAAAA,MAAF;AAAUiB,QAAAA,YAAY,EAAEjB,MAAM,CAAChF;AAA/B,OAAP;AACD,KAPD,MAOO;AASL,YAAMe,eAAe,GAAG,MAAM,KAAKrB,GAAL,CAASsB,qBAAT,CAA+B0E,YAA/B,CAA9B;AACA,YAAMQ,mBAAmB,GAAG,EAC1B,GAAGnF,eADuB;AAE1BD,QAAAA,UAAU,EAAG,IAAG,KAAKb,OAAQ,IAAG,KAAKE,IAAL,GAAYxC,IAAK;AAFvB,OAA5B;;AAIA,UAAI;AACF,cAAMuD,KAAK,GAAG,EAAd;AACA,cAAMiF,UAAU,GAAGlB,QAAnB;AACA,YAAImB,YAAY,GAAGD,UAAU,GAAGnB,MAAM,CAAChF,MAAvC;AACA,YAAIqG,UAAU,GAAGxJ,IAAI,CAACkJ,GAAL,CAAS,CAAT,EAAYJ,QAAQ,GAAGS,YAAvB,CAAjB;AACA,YAAIE,SAAS,GAAGD,UAAU,GAAG,CAA7B;AACA,YAAIE,UAAU,GAAGvB,MAAjB;AACA,YAAIwB,gBAAgB,GAAGvB,QAAvB;AACA,YAAIwB,UAAU,GAAG,CAAjB;AACA,YAAIC,cAAc,GAAG,CAArB;AAEA,YAAIC,cAAc,GAAG9J,IAAI,CAAC+J,KAAL,CAAWT,UAAU,GAAG1J,aAAxB,CAArB;AACA,cAAMoK,kBAAkB,GAAGpK,aAA3B;AACA,YAAIqK,sBAAsB,GAAGX,UAAU,GAAGU,kBAAkB,GAAGF,cAA/D;;AACA,YAAIG,sBAAsB,IAAItK,aAA9B,EAA6C;AAG3CmK,UAAAA,cAAc;AACdG,UAAAA,sBAAsB,GAAG,CAAzB;AACD;;AACD,aAAK,IAAIC,CAAC,GAAG,CAAb,EAAgBA,CAAC,GAAGJ,cAApB,EAAoCI,CAAC,EAArC,EAAyC;AACvC,gBAAMC,WAAW,GAAGnK,IAAI,CAACwE,GAAL,CAASqF,cAAc,GAAGG,kBAA1B,EAA8CV,UAA9C,CAApB;;AACAc,0BAAOC,WAAP,CAAmBF,WAAW,GAAGN,cAAd,IAAgCjK,aAAnD,EAAkE,IAAlE;;AACA,gBAAM2E,KAAK,GAAI,SAAQsF,cAAe,IAAGM,WAAW,GAAG,CAAE,EAAzD;AACA,gBAAMG,gBAAgB,GAAG,EAAE,GAAGjB,mBAAL;AAA0B3E,YAAAA,UAAU,EAAEkF,UAAU,EAAhD;AAAoDjF,YAAAA,eAAe,EAAEJ;AAArE,WAAzB;AACA,gBAAMgG,IAAI,GAAG,MAAM,KAAK1H,GAAL,CAASgC,cAAT,CAAwByF,gBAAxB,CAAnB;AACAjG,UAAAA,KAAK,CAACS,IAAN,CAAW;AAAEC,YAAAA,IAAI,EAAEwF,IAAI,CAACvF,cAAL,CAAoBD,IAA5B;AAAkCL,YAAAA,UAAU,EAAE4F,gBAAgB,CAAC5F;AAA/D,WAAX;AACAmF,UAAAA,cAAc,IAAIG,kBAAlB;AACD;;AACD,YAAIC,sBAAJ,EAA4B;AAE1B,gBAAMO,cAAc,GAAG,EAAE,GAAG3B,YAAL;AAAmBR,YAAAA,KAAK,EAAG,SAAQwB,cAAe,IAAGP,UAAU,GAAG,CAAE;AAApE,WAAvB;AACA,gBAAMmB,YAAY,GAAGnB,UAAU,GAAG,CAAb,GAAiB,CAAC,MAAM,KAAKzG,GAAL,CAASgE,SAAT,CAAmB2D,cAAnB,CAAP,EAA2CtE,IAA5D,GAAmE8C,MAAM,CAACC,KAAP,CAAa,CAAb,CAAxF;AACAS,UAAAA,UAAU,GAAGV,MAAM,CAAC0B,MAAP,CAAc,CAACD,YAAD,EAAetC,MAAf,CAAd,CAAb;AACAwB,UAAAA,gBAAgB,IAAIM,sBAApB;AACD;;AACD,YAAIR,SAAS,IAAIC,UAAU,CAACvG,MAAX,GAAoBxD,aAArC,EAAoD;AAIlD,gBAAMgL,cAAc,GAAG3K,IAAI,CAACwE,GAAL,CAAS7E,aAAa,GAAG+J,UAAU,CAACvG,MAApC,EAA4CqG,UAA5C,CAAvB;AACA,gBAAMoB,gBAAgB,GAAGjB,gBAAgB,GAAGD,UAAU,CAACvG,MAAvD;AACAoG,UAAAA,YAAY,IAAIoB,cAAhB;AACAnB,UAAAA,UAAU,IAAImB,cAAd;AACAlB,UAAAA,SAAS,GAAGD,UAAU,GAAG,CAAzB;AACA,gBAAMqB,WAAW,GAAI,SAAQD,gBAAiB,IAAGA,gBAAgB,GAAGD,cAAnB,GAAoC,CAAE,EAAvF;AACA,gBAAMH,cAAc,GAAG,EAAE,GAAG3B,YAAL;AAAmBR,YAAAA,KAAK,EAAEwC;AAA1B,WAAvB;AACA,gBAAMC,gBAAgB,GAAG,CAAC,MAAM,KAAKjI,GAAL,CAASgE,SAAT,CAAmB2D,cAAnB,CAAP,EAA2CtE,IAApE;AACAwD,UAAAA,UAAU,GAAGV,MAAM,CAAC0B,MAAP,CAAc,CAAChB,UAAD,EAAaoB,gBAAb,CAAd,CAAb;AACD;;AACD,cAAMC,UAAU,GAAG,EAAE,GAAG7G,eAAL;AAAsBgC,UAAAA,IAAI,EAAEwD,UAA5B;AAAwChF,UAAAA,UAAU,EAAEkF,UAAU;AAA9D,SAAnB;AACA,cAAMoB,QAAQ,GAAG,MAAM,KAAKnI,GAAL,CAASoI,UAAT,CAAoBF,UAApB,CAAvB;AACA1G,QAAAA,KAAK,CAACS,IAAN,CAAW;AAAEC,UAAAA,IAAI,EAAEiG,QAAQ,CAACjG,IAAjB;AAAuBL,UAAAA,UAAU,EAAEqG,UAAU,CAACrG;AAA9C,SAAX;;AACA,YAAI+E,SAAJ,EAAe;AAEb,gBAAMyB,eAAe,GAAGlL,IAAI,CAACC,IAAL,CAAUuJ,UAAU,GAAG5J,aAAvB,CAAxB;AACA,cAAIuL,oBAAoB,GAAG5B,YAA3B;;AACA,eAAK,IAAIW,CAAC,GAAG,CAAb,EAAgBA,CAAC,GAAGgB,eAApB,EAAqChB,CAAC,EAAtC,EAA0C;AACxC,kBAAMC,WAAW,GAAGgB,oBAAoB,GAAGvL,aAA3C;;AACAwK,4BAAOC,WAAP,CAAmBrK,IAAI,CAACwE,GAAL,CAASsE,QAAT,EAAmBqB,WAAnB,IAAkCgB,oBAAlC,IAA0DvL,aAA7E,EAA4F,IAA5F;;AACA,kBAAMwL,WAAW,GAAI,SAAQD,oBAAqB,IAAGnL,IAAI,CAACwE,GAAL,CAASsE,QAAT,EAAmBqB,WAAnB,IAAkC,CAAE,EAAzF;AACA,kBAAMkB,gBAAgB,GAAG,EAAE,GAAGhC,mBAAL;AAA0B3E,cAAAA,UAAU,EAAEkF,UAAU,EAAhD;AAAoDjF,cAAAA,eAAe,EAAEyG;AAArE,aAAzB;AACA,kBAAME,UAAU,GAAG,CAAC,MAAM,KAAKzI,GAAL,CAASgC,cAAT,CAAwBwG,gBAAxB,CAAP,EAAkDrG,cAArE;AACAX,YAAAA,KAAK,CAACS,IAAN,CAAW;AAAEC,cAAAA,IAAI,EAAEuG,UAAU,CAACvG,IAAnB;AAAyBL,cAAAA,UAAU,EAAE2G,gBAAgB,CAAC3G;AAAtD,aAAX;AACAyG,YAAAA,oBAAoB,GAAGhB,WAAvB;AACD;AACF;;AACD,cAAM,KAAKtH,GAAL,CAASoC,uBAAT,CAAiC,EACrC,GAAGf,eADkC;AAErCgB,UAAAA,eAAe,EAAE;AAAEC,YAAAA,KAAK,EAAEd;AAAT;AAFoB,SAAjC,CAAN;AAID,OAvED,CAuEE,OAAO7D,CAAP,EAAU;AACV,cAAM,KAAKqC,GAAL,CAASuC,oBAAT,CAA8BlB,eAA9B,CAAN;AACA,cAAM1D,CAAN;AACD;AACF;AACF;;AAEc,QAAT+K,SAAS,CAACjK,IAAD,EAAOkK,KAAP,EAAc;AAC3B,WAAOlK,IAAP;AACD;;AAEe,QAAVmK,UAAU,CAACzD,EAAD,EAAK,CAAE;;AAhZoC,C","sourcesContent":["import aws from '@sullux/aws-sdk'\nimport assert from 'assert'\nimport http from 'http'\nimport https from 'https'\nimport pRetry from 'promise-toolbox/retry'\nimport { createLogger } from '@xen-orchestra/log'\nimport { decorateWith } from '@vates/decorate-with'\nimport { parse } from 'xo-remote-parser'\n\nimport RemoteHandlerAbstract from './abstract'\nimport { asyncEach } from '@vates/async-each'\n\n// endpoints https://docs.aws.amazon.com/general/latest/gr/s3.html\n\n// limits: https://docs.aws.amazon.com/AmazonS3/latest/dev/qfacts.html\nconst MIN_PART_SIZE = 1024 * 1024 * 5 // 5MB\nconst MAX_PART_SIZE = 1024 * 1024 * 1024 * 5 // 5GB\nconst MAX_PARTS_COUNT = 10000\nconst MAX_OBJECT_SIZE = 1024 * 1024 * 1024 * 1024 * 5 // 5TB\nconst IDEAL_FRAGMENT_SIZE = Math.ceil(MAX_OBJECT_SIZE / MAX_PARTS_COUNT) // the smallest fragment size that still allows a 5TB upload in 10000 fragments, about 524MB\n\nconst { warn } = createLogger('xo:fs:s3')\n\nexport default class S3Handler extends RemoteHandlerAbstract {\n  constructor(remote, _opts) {\n    super(remote)\n    const { allowUnauthorized, host, path, username, password, protocol, region } = parse(remote.url)\n    const params = {\n      accessKeyId: username,\n      apiVersion: '2006-03-01',\n      endpoint: host,\n      s3ForcePathStyle: true,\n      secretAccessKey: password,\n      signatureVersion: 'v4',\n      httpOptions: {\n        timeout: 600000,\n      },\n    }\n    if (protocol === 'http') {\n      params.httpOptions.agent = new http.Agent({ keepAlive: true })\n      params.sslEnabled = false\n    } else if (protocol === 'https') {\n      params.httpOptions.agent = new https.Agent({\n        rejectUnauthorized: !allowUnauthorized,\n        keepAlive: true,\n      })\n    }\n    if (region !== undefined) {\n      params.region = region\n    }\n\n    this._s3 = aws(params).s3\n\n    const splitPath = path.split('/').filter(s => s.length)\n    this._bucket = splitPath.shift()\n    this._dir = splitPath.join('/')\n  }\n\n  get type() {\n    return 's3'\n  }\n\n  _createParams(file) {\n    return { Bucket: this._bucket, Key: this._dir + file }\n  }\n\n  async _multipartCopy(oldPath, newPath) {\n    const size = await this._getSize(oldPath)\n    const CopySource = `/${this._bucket}/${this._dir}${oldPath}`\n    const multipartParams = await this._s3.createMultipartUpload({ ...this._createParams(newPath) })\n    const param2 = { ...multipartParams, CopySource }\n    try {\n      const parts = []\n      let start = 0\n      while (start < size) {\n        const range = `bytes=${start}-${Math.min(start + MAX_PART_SIZE, size) - 1}`\n        const partParams = { ...param2, PartNumber: parts.length + 1, CopySourceRange: range }\n        const upload = await this._s3.uploadPartCopy(partParams)\n        parts.push({ ETag: upload.CopyPartResult.ETag, PartNumber: partParams.PartNumber })\n        start += MAX_PART_SIZE\n      }\n      await this._s3.completeMultipartUpload({ ...multipartParams, MultipartUpload: { Parts: parts } })\n    } catch (e) {\n      await this._s3.abortMultipartUpload(multipartParams)\n      throw e\n    }\n  }\n\n  async _copy(oldPath, newPath) {\n    const CopySource = `/${this._bucket}/${this._dir}${oldPath}`\n    try {\n      await this._s3.copyObject({\n        ...this._createParams(newPath),\n        CopySource,\n      })\n    } catch (e) {\n      // object > 5GB must be copied part by part\n      if (e.code === 'EntityTooLarge') {\n        return this._multipartCopy(oldPath, newPath)\n      }\n      throw e\n    }\n  }\n\n  async _isNotEmptyDir(path) {\n    const result = await this._s3.listObjectsV2({\n      Bucket: this._bucket,\n      MaxKeys: 1,\n      Prefix: this._dir + path + '/',\n    })\n    return result.Contents.length !== 0\n  }\n\n  async _isFile(path) {\n    try {\n      await this._s3.headObject(this._createParams(path))\n      return true\n    } catch (error) {\n      if (error.code === 'NotFound') {\n        return false\n      }\n      throw error\n    }\n  }\n\n  async _outputStream(path, input, { validator }) {\n    await this._s3.upload(\n      {\n        ...this._createParams(path),\n        Body: input,\n      },\n      { partSize: IDEAL_FRAGMENT_SIZE, queueSize: 1 }\n    )\n    if (validator !== undefined) {\n      try {\n        await validator.call(this, path)\n      } catch (error) {\n        await this.unlink(path)\n        throw error\n      }\n    }\n  }\n\n  // some objectstorage provider like backblaze, can answer a 500/503 routinely\n  // in this case we should retry,  and let their load balancing do its magic\n  // https://www.backblaze.com/b2/docs/calling.html#error_handling\n  @decorateWith(pRetry.wrap, {\n    delays: [100, 200, 500, 1000, 2000],\n    when: e => e.code === 'InternalError',\n    onRetry(error) {\n      warn('retrying writing file', {\n        attemptNumber: this.attemptNumber,\n        delay: this.delay,\n        error,\n        file: this.arguments[0],\n      })\n    },\n  })\n  async _writeFile(file, data, options) {\n    return this._s3.putObject({ ...this._createParams(file), Body: data })\n  }\n\n  async _createReadStream(path, options) {\n    if (!(await this._isFile(path))) {\n      const error = new Error(`ENOENT: no such file '${path}'`)\n      error.code = 'ENOENT'\n      error.path = path\n      throw error\n    }\n\n    // https://github.com/Sullux/aws-sdk/issues/11\n    return this._s3.getObject.raw(this._createParams(path)).createReadStream()\n  }\n\n  async _unlink(path) {\n    await this._s3.deleteObject(this._createParams(path))\n    if (await this._isNotEmptyDir(path)) {\n      const error = new Error(`EISDIR: illegal operation on a directory, unlink '${path}'`)\n      error.code = 'EISDIR'\n      error.path = path\n      throw error\n    }\n  }\n\n  async _list(dir) {\n    function splitPath(path) {\n      return path.split('/').filter(d => d.length)\n    }\n\n    const prefix = [this._dir, dir].join('/')\n    const splitPrefix = splitPath(prefix)\n    const result = await this._s3.listObjectsV2({\n      Bucket: this._bucket,\n      Prefix: splitPrefix.join('/') + '/', // need slash at the end with the use of delimiters\n      Delimiter: '/', // will only return path until delimiters\n    })\n\n    if (result.IsTruncated) {\n      const error = new Error('more than 1000 objects, unsupported in this implementation')\n      error.dir = dir\n      throw error\n    }\n\n    const uniq = []\n\n    // sub directories\n    for (const entry of result.CommonPrefixes) {\n      const line = splitPath(entry.Prefix)\n      uniq.push(line[line.length - 1])\n    }\n    // files\n    for (const entry of result.Contents) {\n      const line = splitPath(entry.Key)\n      uniq.push(line[line.length - 1])\n    }\n\n    return uniq\n  }\n\n  async _mkdir(path) {\n    if (await this._isFile(path)) {\n      const error = new Error(`ENOTDIR: file already exists, mkdir '${path}'`)\n      error.code = 'ENOTDIR'\n      error.path = path\n      throw error\n    }\n    // nothing to do, directories do not exist, they are part of the files' path\n  }\n\n  // s3 doesn't have a rename operation, so copy + delete source\n  async _rename(oldPath, newPath) {\n    await this.copy(oldPath, newPath)\n    await this._s3.deleteObject(this._createParams(oldPath))\n  }\n\n  async _getSize(file) {\n    if (typeof file !== 'string') {\n      file = file.fd\n    }\n    const result = await this._s3.headObject(this._createParams(file))\n    return +result.ContentLength\n  }\n\n  async _read(file, buffer, position = 0) {\n    if (typeof file !== 'string') {\n      file = file.fd\n    }\n    const params = this._createParams(file)\n    params.Range = `bytes=${position}-${position + buffer.length - 1}`\n    try {\n      const result = await this._s3.getObject(params)\n      result.Body.copy(buffer)\n      return { bytesRead: result.Body.length, buffer }\n    } catch (e) {\n      if (e.code === 'NoSuchKey') {\n        if (await this._isNotEmptyDir(file)) {\n          const error = new Error(`${file} is a directory`)\n          error.code = 'EISDIR'\n          error.path = file\n          throw error\n        }\n      }\n      throw e\n    }\n  }\n\n  async _rmdir(path) {\n    if (await this._isNotEmptyDir(path)) {\n      const error = new Error(`ENOTEMPTY: directory not empty, rmdir '${path}`)\n      error.code = 'ENOTEMPTY'\n      error.path = path\n      throw error\n    }\n\n    // nothing to do, directories do not exist, they are part of the files' path\n  }\n\n  // reimplement _rmtree to handle efficiantly path with more than 1000 entries in trees\n  // @todo : use parallel processing for unlink\n  async _rmtree(path) {\n    let NextContinuationToken\n    do {\n      const result = await this._s3.listObjectsV2({\n        Bucket: this._bucket,\n        Prefix: this._dir + path + '/',\n        ContinuationToken: NextContinuationToken,\n      })\n      NextContinuationToken = result.IsTruncated ? result.NextContinuationToken : undefined\n      await asyncEach(\n        result.Contents,\n        async ({ Key }) => {\n          // _unlink will add the prefix, but Key contains everything\n          // also we don't need to check if we delete a directory, since the list only return files\n          await this._s3.deleteObject({\n            Bucket: this._bucket,\n            Key,\n          })\n        },\n        {\n          concurrency: 16,\n        }\n      )\n    } while (NextContinuationToken !== undefined)\n  }\n\n  async _write(file, buffer, position) {\n    if (typeof file !== 'string') {\n      file = file.fd\n    }\n    const uploadParams = this._createParams(file)\n    let fileSize\n    try {\n      fileSize = +(await this._s3.headObject(uploadParams)).ContentLength\n    } catch (e) {\n      if (e.code === 'NotFound') {\n        fileSize = 0\n      } else {\n        throw e\n      }\n    }\n    if (fileSize < MIN_PART_SIZE) {\n      const resultBuffer = Buffer.alloc(Math.max(fileSize, position + buffer.length))\n      const fileContent = fileSize !== 0 ? (await this._s3.getObject(uploadParams)).Body : Buffer.alloc(0)\n      fileContent.copy(resultBuffer)\n      buffer.copy(resultBuffer, position)\n      await this._s3.putObject({ ...uploadParams, Body: resultBuffer })\n      return { buffer, bytesWritten: buffer.length }\n    } else {\n      // using this trick: https://stackoverflow.com/a/38089437/72637\n      // multipart fragments have a minimum size of 5Mo and a max of 5Go unless they are last\n      // splitting the file in 3 parts: [prefix, edit, suffix]\n      // if `prefix` is bigger than 5Mo, it will be sourced from uploadPartCopy()\n      // otherwise otherwise it will be downloaded, concatenated to `edit`\n      // `edit` will always be an upload part\n      // `suffix` will always be sourced from uploadPartCopy()\n      // Then everything will be sliced in 5Gb parts before getting uploaded\n      const multipartParams = await this._s3.createMultipartUpload(uploadParams)\n      const copyMultipartParams = {\n        ...multipartParams,\n        CopySource: `/${this._bucket}/${this._dir + file}`,\n      }\n      try {\n        const parts = []\n        const prefixSize = position\n        let suffixOffset = prefixSize + buffer.length\n        let suffixSize = Math.max(0, fileSize - suffixOffset)\n        let hasSuffix = suffixSize > 0\n        let editBuffer = buffer\n        let editBufferOffset = position\n        let partNumber = 1\n        let prefixPosition = 0\n        // use floor() so that last fragment is handled in the if bellow\n        let fragmentsCount = Math.floor(prefixSize / MAX_PART_SIZE)\n        const prefixFragmentSize = MAX_PART_SIZE\n        let prefixLastFragmentSize = prefixSize - prefixFragmentSize * fragmentsCount\n        if (prefixLastFragmentSize >= MIN_PART_SIZE) {\n          // the last fragment of the prefix is smaller than MAX_PART_SIZE, but bigger than the minimum\n          // so we can copy it too\n          fragmentsCount++\n          prefixLastFragmentSize = 0\n        }\n        for (let i = 0; i < fragmentsCount; i++) {\n          const fragmentEnd = Math.min(prefixPosition + prefixFragmentSize, prefixSize)\n          assert.strictEqual(fragmentEnd - prefixPosition <= MAX_PART_SIZE, true)\n          const range = `bytes=${prefixPosition}-${fragmentEnd - 1}`\n          const copyPrefixParams = { ...copyMultipartParams, PartNumber: partNumber++, CopySourceRange: range }\n          const part = await this._s3.uploadPartCopy(copyPrefixParams)\n          parts.push({ ETag: part.CopyPartResult.ETag, PartNumber: copyPrefixParams.PartNumber })\n          prefixPosition += prefixFragmentSize\n        }\n        if (prefixLastFragmentSize) {\n          // grab everything from the prefix that was too small to be copied, download and merge to the edit buffer.\n          const downloadParams = { ...uploadParams, Range: `bytes=${prefixPosition}-${prefixSize - 1}` }\n          const prefixBuffer = prefixSize > 0 ? (await this._s3.getObject(downloadParams)).Body : Buffer.alloc(0)\n          editBuffer = Buffer.concat([prefixBuffer, buffer])\n          editBufferOffset -= prefixLastFragmentSize\n        }\n        if (hasSuffix && editBuffer.length < MIN_PART_SIZE) {\n          // the edit fragment is too short and is not the last fragment\n          // let's steal from the suffix fragment to reach the minimum size\n          // the suffix might be too short and itself entirely absorbed in the edit fragment, making it the last one.\n          const complementSize = Math.min(MIN_PART_SIZE - editBuffer.length, suffixSize)\n          const complementOffset = editBufferOffset + editBuffer.length\n          suffixOffset += complementSize\n          suffixSize -= complementSize\n          hasSuffix = suffixSize > 0\n          const prefixRange = `bytes=${complementOffset}-${complementOffset + complementSize - 1}`\n          const downloadParams = { ...uploadParams, Range: prefixRange }\n          const complementBuffer = (await this._s3.getObject(downloadParams)).Body\n          editBuffer = Buffer.concat([editBuffer, complementBuffer])\n        }\n        const editParams = { ...multipartParams, Body: editBuffer, PartNumber: partNumber++ }\n        const editPart = await this._s3.uploadPart(editParams)\n        parts.push({ ETag: editPart.ETag, PartNumber: editParams.PartNumber })\n        if (hasSuffix) {\n          // use ceil because the last fragment can be arbitrarily small.\n          const suffixFragments = Math.ceil(suffixSize / MAX_PART_SIZE)\n          let suffixFragmentOffset = suffixOffset\n          for (let i = 0; i < suffixFragments; i++) {\n            const fragmentEnd = suffixFragmentOffset + MAX_PART_SIZE\n            assert.strictEqual(Math.min(fileSize, fragmentEnd) - suffixFragmentOffset <= MAX_PART_SIZE, true)\n            const suffixRange = `bytes=${suffixFragmentOffset}-${Math.min(fileSize, fragmentEnd) - 1}`\n            const copySuffixParams = { ...copyMultipartParams, PartNumber: partNumber++, CopySourceRange: suffixRange }\n            const suffixPart = (await this._s3.uploadPartCopy(copySuffixParams)).CopyPartResult\n            parts.push({ ETag: suffixPart.ETag, PartNumber: copySuffixParams.PartNumber })\n            suffixFragmentOffset = fragmentEnd\n          }\n        }\n        await this._s3.completeMultipartUpload({\n          ...multipartParams,\n          MultipartUpload: { Parts: parts },\n        })\n      } catch (e) {\n        await this._s3.abortMultipartUpload(multipartParams)\n        throw e\n      }\n    }\n  }\n\n  async _openFile(path, flags) {\n    return path\n  }\n\n  async _closeFile(fd) {}\n}\n"],"file":"s3.js"}

package/package.json

@@ -1,7 +1,7 @@
 {
   "private": false,
   "name": "@xen-orchestra/fs",
-  "version": "0.19.3",
+  "version": "0.20.0",
   "license": "AGPL-3.0-or-later",
   "description": "The File System for Xen Orchestra backups.",
   "homepage": "https://github.com/vatesfr/xen-orchestra/tree/master/@xen-orchestra/fs",
@@ -21,7 +21,9 @@
     "@sindresorhus/df": "^3.1.1",
     "@sullux/aws-sdk": "^1.0.5",
     "@vates/coalesce-calls": "^0.1.0",
+    "@vates/decorate-with": "^1.0.0",
     "@xen-orchestra/async-map": "^0.1.2",
+    "@xen-orchestra/log": "^0.3.0",
     "aws-sdk": "^2.686.0",
     "decorator-synchronized": "^0.6.0",
     "execa": "^5.0.0",