@xen-orchestra/fs 0.19.2 → 1.0.0

package/dist/s3.js

@@ -5,27 +5,72 @@ Object.defineProperty(exports, "__esModule", {
 });
 exports.default = void 0;
 
-var _awsSdk = _interopRequireDefault(require("@sullux/aws-sdk"));
+var _clientS = require("@aws-sdk/client-s3");
+
+var _libStorage = require("@aws-sdk/lib-storage");
+
+var _nodeHttpHandler = require("@aws-sdk/node-http-handler");
 
 var _assert = _interopRequireDefault(require("assert"));
 
-var _http = _interopRequireDefault(require("http"));
+var _http = require("http");
+
+var _https = require("https");
+
+var _retry = _interopRequireDefault(require("promise-toolbox/retry"));
+
+var _log = require("@xen-orchestra/log");
+
+var _decorateWith = require("@vates/decorate-with");
 
-var _https = _interopRequireDefault(require("https"));
+var _stream = require("stream");
 
 var _xoRemoteParser = require("xo-remote-parser");
 
+var _copyStreamToBuffer = _interopRequireDefault(require("./_copyStreamToBuffer.js"));
+
+var _createBufferFromStream = _interopRequireDefault(require("./_createBufferFromStream.js"));
+
+var _guessAwsRegion = _interopRequireDefault(require("./_guessAwsRegion.js"));
+
 var _abstract = _interopRequireDefault(require("./abstract"));
 
+var _path = require("./_path");
+
+var _asyncEach = require("@vates/async-each");
+
+var _dec, _class;
+
 function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; }
 
+function _applyDecoratedDescriptor(target, property, decorators, descriptor, context) { var desc = {}; Object.keys(descriptor).forEach(function (key) { desc[key] = descriptor[key]; }); desc.enumerable = !!desc.enumerable; desc.configurable = !!desc.configurable; if ('value' in desc || desc.initializer) { desc.writable = true; } desc = decorators.slice().reverse().reduce(function (desc, decorator) { return decorator(target, property, desc) || desc; }, desc); if (context && desc.initializer !== void 0) { desc.value = desc.initializer ? desc.initializer.call(context) : void 0; desc.initializer = undefined; } if (desc.initializer === void 0) { Object.defineProperty(target, property, desc); desc = null; } return desc; }
+
 const MIN_PART_SIZE = 1024 * 1024 * 5;
 const MAX_PART_SIZE = 1024 * 1024 * 1024 * 5;
 const MAX_PARTS_COUNT = 10000;
 const MAX_OBJECT_SIZE = 1024 * 1024 * 1024 * 1024 * 5;
 const IDEAL_FRAGMENT_SIZE = Math.ceil(MAX_OBJECT_SIZE / MAX_PARTS_COUNT);
+const {
+  warn
+} = (0, _log.createLogger)('xo:fs:s3');
+let S3Handler = (_dec = (0, _decorateWith.decorateWith)(_retry.default.wrap, {
+  delays: [100, 200, 500, 1000, 2000],
+  when: e => {
+    var _e$$metadata;
+
+    return ((_e$$metadata = e.$metadata) === null || _e$$metadata === void 0 ? void 0 : _e$$metadata.httpStatusCode) === 500;
+  },
+
+  onRetry(error) {
+    warn('retrying writing file', {
+      attemptNumber: this.attemptNumber,
+      delay: this.delay,
+      error,
+      file: this.arguments[0]
+    });
+  }
 
-class S3Handler extends _abstract.default {
+}), (_class = class S3Handler extends _abstract.default {
   constructor(remote, _opts) {
     super(remote);
     const {
@@ -35,105 +80,127 @@ class S3Handler extends _abstract.default {
       username,
       password,
       protocol,
-      region
+      region = (0, _guessAwsRegion.default)(host)
     } = (0, _xoRemoteParser.parse)(remote.url);
-    const params = {
-      accessKeyId: username,
+    this._s3 = new _clientS.S3Client({
       apiVersion: '2006-03-01',
-      endpoint: host,
-      s3ForcePathStyle: true,
-      secretAccessKey: password,
-      signatureVersion: 'v4',
-      httpOptions: {
-        timeout: 600000
-      }
-    };
+      endpoint: `${protocol}://${host}`,
+      forcePathStyle: true,
+      credentials: {
+        accessKeyId: username,
+        secretAccessKey: password
+      },
+      tls: protocol === 'https',
+      region,
+      requestHandler: new _nodeHttpHandler.NodeHttpHandler({
+        socketTimeout: 600000,
+        httpAgent: new _http.Agent({
+          keepAlive: true
+        }),
+        httpsAgent: new _https.Agent({
+          rejectUnauthorized: !allowUnauthorized,
+          keepAlive: true
+        })
+      })
+    });
+    const parts = (0, _path.split)(path);
+    this._bucket = parts.shift();
+    this._dir = (0, _path.join)(...parts);
+  }
 
-    if (protocol === 'http') {
-      params.httpOptions.agent = new _http.default.Agent({
-        keepAlive: true
-      });
-      params.sslEnabled = false;
-    } else if (protocol === 'https') {
-      params.httpOptions.agent = new _https.default.Agent({
-        rejectUnauthorized: !allowUnauthorized,
-        keepAlive: true
-      });
-    }
+  get type() {
+    return 's3';
+  }
 
-    if (region !== undefined) {
-      params.region = region;
-    }
+  _makeCopySource(path) {
+    return (0, _path.join)(this._bucket, this._dir, path);
+  }
 
-    this._s3 = (0, _awsSdk.default)(params).s3;
-    const splitPath = path.split('/').filter(s => s.length);
-    this._bucket = splitPath.shift();
-    this._dir = splitPath.join('/');
+  _makeKey(file) {
+    return (0, _path.join)(this._dir, file);
   }
 
-  get type() {
-    return 's3';
+  _makePrefix(dir) {
+    return (0, _path.join)(this._dir, dir, '/');
   }
 
   _createParams(file) {
     return {
       Bucket: this._bucket,
-      Key: this._dir + file
+      Key: this._makeKey(file)
     };
   }
 
-  async _copy(oldPath, newPath) {
+  async _multipartCopy(oldPath, newPath) {
     const size = await this._getSize(oldPath);
-    const multipartParams = await this._s3.createMultipartUpload({ ...this._createParams(newPath)
-    });
-    const param2 = { ...multipartParams,
-      CopySource: `/${this._bucket}/${this._dir}${oldPath}`
-    };
+
+    const CopySource = this._makeCopySource(oldPath);
+
+    const multipartParams = await this._s3.send(new _clientS.CreateMultipartUploadCommand({ ...this._createParams(newPath)
+    }));
 
     try {
       const parts = [];
       let start = 0;
 
       while (start < size) {
-        const range = `bytes=${start}-${Math.min(start + MAX_PART_SIZE, size) - 1}`;
-        const partParams = { ...param2,
-          PartNumber: parts.length + 1,
-          CopySourceRange: range
-        };
-        const upload = await this._s3.uploadPartCopy(partParams);
+        const partNumber = parts.length + 1;
+        const upload = await this._s3.send(new _clientS.UploadPartCopyCommand({ ...multipartParams,
+          CopySource,
+          CopySourceRange: `bytes=${start}-${Math.min(start + MAX_PART_SIZE, size) - 1}`,
+          PartNumber: partNumber
+        }));
         parts.push({
           ETag: upload.CopyPartResult.ETag,
-          PartNumber: partParams.PartNumber
+          PartNumber: partNumber
         });
         start += MAX_PART_SIZE;
       }
 
-      await this._s3.completeMultipartUpload({ ...multipartParams,
+      await this._s3.send(new _clientS.CompleteMultipartUploadCommand({ ...multipartParams,
         MultipartUpload: {
           Parts: parts
         }
-      });
+      }));
     } catch (e) {
-      await this._s3.abortMultipartUpload(multipartParams);
+      await this._s3.send(new _clientS.AbortMultipartUploadCommand(multipartParams));
+      throw e;
+    }
+  }
+
+  async _copy(oldPath, newPath) {
+    const CopySource = this._makeCopySource(oldPath);
+
+    try {
+      await this._s3.send(new _clientS.CopyObjectCommand({ ...this._createParams(newPath),
+        CopySource
+      }));
+    } catch (e) {
+      if (e.name === 'EntityTooLarge') {
+        return this._multipartCopy(oldPath, newPath);
+      }
+
       throw e;
     }
   }
 
   async _isNotEmptyDir(path) {
-    const result = await this._s3.listObjectsV2({
+    var _result$Contents;
+
+    const result = await this._s3.send(new _clientS.ListObjectsV2Command({
       Bucket: this._bucket,
       MaxKeys: 1,
-      Prefix: this._dir + path + '/'
-    });
-    return result.Contents.length !== 0;
+      Prefix: this._makePrefix(path)
+    }));
+    return ((_result$Contents = result.Contents) === null || _result$Contents === void 0 ? void 0 : _result$Contents.length) > 0;
   }
 
   async _isFile(path) {
     try {
-      await this._s3.headObject(this._createParams(path));
+      await this._s3.send(new _clientS.HeadObjectCommand(this._createParams(path)));
       return true;
     } catch (error) {
-      if (error.code === 'NotFound') {
+      if (error.name === 'NotFound') {
         return false;
       }
 
@@ -144,12 +211,17 @@ class S3Handler extends _abstract.default {
   async _outputStream(path, input, {
     validator
   }) {
-    await this._s3.upload({ ...this._createParams(path),
-      Body: input
-    }, {
+    const Body = new _stream.PassThrough();
+    (0, _stream.pipeline)(input, Body, () => {});
+    const upload = new _libStorage.Upload({
+      client: this._s3,
+      queueSize: 1,
       partSize: IDEAL_FRAGMENT_SIZE,
-      queueSize: 1
+      params: { ...this._createParams(path),
+        Body
+      }
     });
+    await upload.done();
 
     if (validator !== undefined) {
       try {
@@ -162,9 +234,9 @@ class S3Handler extends _abstract.default {
   }
 
   async _writeFile(file, data, options) {
-    return this._s3.putObject({ ...this._createParams(file),
+    return this._s3.send(new _clientS.PutObjectCommand({ ...this._createParams(file),
       Body: data
-    });
+    }));
   }
 
   async _createReadStream(path, options) {
@@ -175,11 +247,11 @@ class S3Handler extends _abstract.default {
       throw error;
     }
 
-    return this._s3.getObject.raw(this._createParams(path)).createReadStream();
+    return (await this._s3.send(new _clientS.GetObjectCommand(this._createParams(path)))).Body;
   }
 
   async _unlink(path) {
-    await this._s3.deleteObject(this._createParams(path));
+    await this._s3.send(new _clientS.DeleteObjectCommand(this._createParams(path)));
 
     if (await this._isNotEmptyDir(path)) {
       const error = new Error(`EISDIR: illegal operation on a directory, unlink '${path}'`);
@@ -190,37 +262,36 @@ class S3Handler extends _abstract.default {
   }
 
   async _list(dir) {
-    function splitPath(path) {
-      return path.split('/').filter(d => d.length);
-    }
+    let NextContinuationToken;
+    const uniq = new Set();
 
-    const prefix = [this._dir, dir].join('/');
-    const splitPrefix = splitPath(prefix);
-    const result = await this._s3.listObjectsV2({
-      Bucket: this._bucket,
-      Prefix: splitPrefix.join('/') + '/',
-      Delimiter: '/'
-    });
+    const Prefix = this._makePrefix(dir);
 
-    if (result.isTruncated) {
-      const error = new Error('more than 1000 objects, unsupported in this implementation');
-      error.dir = dir;
-      throw error;
-    }
+    do {
+      const result = await this._s3.send(new _clientS.ListObjectsV2Command({
+        Bucket: this._bucket,
+        Prefix,
+        Delimiter: '/',
+        ContinuationToken: NextContinuationToken
+      }));
 
-    const uniq = [];
+      if (result.IsTruncated) {
+        warn(`need pagination to browse the directory ${dir} completely`);
+        NextContinuationToken = result.NextContinuationToken;
+      } else {
+        NextContinuationToken = undefined;
+      }
 
-    for (const entry of result.CommonPrefixes) {
-      const line = splitPath(entry.Prefix);
-      uniq.push(line[line.length - 1]);
-    }
+      for (const entry of result.CommonPrefixes ?? []) {
+        uniq.add((0, _path.basename)(entry.Prefix));
+      }
 
-    for (const entry of result.Contents) {
-      const line = splitPath(entry.Key);
-      uniq.push(line[line.length - 1]);
-    }
+      for (const entry of result.Contents ?? []) {
+        uniq.add((0, _path.basename)(entry.Key));
+      }
+    } while (NextContinuationToken !== undefined);
 
-    return uniq;
+    return [...uniq];
   }
 
   async _mkdir(path) {
@@ -234,7 +305,7 @@ class S3Handler extends _abstract.default {
 
   async _rename(oldPath, newPath) {
     await this.copy(oldPath, newPath);
-    await this._s3.deleteObject(this._createParams(oldPath));
+    await this._s3.send(new _clientS.DeleteObjectCommand(this._createParams(oldPath)));
   }
 
   async _getSize(file) {
@@ -242,7 +313,7 @@ class S3Handler extends _abstract.default {
       file = file.fd;
     }
 
-    const result = await this._s3.headObject(this._createParams(file));
+    const result = await this._s3.send(new _clientS.HeadObjectCommand(this._createParams(file)));
     return +result.ContentLength;
   }
 
@@ -256,14 +327,14 @@ class S3Handler extends _abstract.default {
     params.Range = `bytes=${position}-${position + buffer.length - 1}`;
 
     try {
-      const result = await this._s3.getObject(params);
-      result.Body.copy(buffer);
+      const result = await this._s3.send(new _clientS.GetObjectCommand(params));
+      const bytesRead = await (0, _copyStreamToBuffer.default)(result.Body, buffer);
       return {
-        bytesRead: result.Body.length,
+        bytesRead,
         buffer
       };
     } catch (e) {
-      if (e.code === 'NoSuchKey') {
+      if (e.name === 'NoSuchKey') {
         if (await this._isNotEmptyDir(file)) {
           const error = new Error(`${file} is a directory`);
           error.code = 'EISDIR';
@@ -288,23 +359,26 @@ class S3Handler extends _abstract.default {
   async _rmtree(path) {
     let NextContinuationToken;
 
+    const Prefix = this._makePrefix(path);
+
     do {
-      const result = await this._s3.listObjectsV2({
+      const result = await this._s3.send(new _clientS.ListObjectsV2Command({
         Bucket: this._bucket,
-        Prefix: this._dir + path + '/',
+        Prefix,
         ContinuationToken: NextContinuationToken
-      });
-      NextContinuationToken = result.isTruncated ? null : result.NextContinuationToken;
-
-      for (const {
+      }));
+      NextContinuationToken = result.IsTruncated ? result.NextContinuationToken : undefined;
+      await (0, _asyncEach.asyncEach)(result.Contents ?? [], async ({
         Key
-      } of result.Contents) {
-        await this._s3.deleteObject({
+      }) => {
+        await this._s3.send(new _clientS.DeleteObjectCommand({
           Bucket: this._bucket,
           Key
-        });
-      }
-    } while (NextContinuationToken !== null);
+        }));
+      }, {
+        concurrency: 16
+      });
+    } while (NextContinuationToken !== undefined);
   }
 
   async _write(file, buffer, position) {
@@ -317,9 +391,9 @@ class S3Handler extends _abstract.default {
     let fileSize;
 
     try {
-      fileSize = +(await this._s3.headObject(uploadParams)).ContentLength;
+      fileSize = +(await this._s3.send(new _clientS.HeadObjectCommand(uploadParams))).ContentLength;
     } catch (e) {
-      if (e.code === 'NotFound') {
+      if (e.name === 'NotFound') {
         fileSize = 0;
       } else {
         throw e;
@@ -328,20 +402,26 @@ class S3Handler extends _abstract.default {
 
     if (fileSize < MIN_PART_SIZE) {
       const resultBuffer = Buffer.alloc(Math.max(fileSize, position + buffer.length));
-      const fileContent = fileSize !== 0 ? (await this._s3.getObject(uploadParams)).Body : Buffer.alloc(0);
-      fileContent.copy(resultBuffer);
+
+      if (fileSize !== 0) {
+        const result = await this._s3.send(new _clientS.GetObjectCommand(uploadParams));
+        await (0, _copyStreamToBuffer.default)(result.Body, resultBuffer);
+      } else {
+        Buffer.alloc(0).copy(resultBuffer);
+      }
+
       buffer.copy(resultBuffer, position);
-      await this._s3.putObject({ ...uploadParams,
+      await this._s3.send(new _clientS.PutObjectCommand({ ...uploadParams,
         Body: resultBuffer
-      });
+      }));
       return {
         buffer,
         bytesWritten: buffer.length
       };
     } else {
-      const multipartParams = await this._s3.createMultipartUpload(uploadParams);
+      const multipartParams = await this._s3.send(new _clientS.CreateMultipartUploadCommand(uploadParams));
       const copyMultipartParams = { ...multipartParams,
-        CopySource: `/${this._bucket}/${this._dir + file}`
+        CopySource: this._makeCopySource(file)
       };
 
       try {
@@ -373,7 +453,7 @@ class S3Handler extends _abstract.default {
             PartNumber: partNumber++,
             CopySourceRange: range
           };
-          const part = await this._s3.uploadPartCopy(copyPrefixParams);
+          const part = await this._s3.send(new _clientS.UploadPartCopyCommand(copyPrefixParams));
           parts.push({
             ETag: part.CopyPartResult.ETag,
             PartNumber: copyPrefixParams.PartNumber
@@ -385,7 +465,15 @@ class S3Handler extends _abstract.default {
           const downloadParams = { ...uploadParams,
             Range: `bytes=${prefixPosition}-${prefixSize - 1}`
           };
-          const prefixBuffer = prefixSize > 0 ? (await this._s3.getObject(downloadParams)).Body : Buffer.alloc(0);
+          let prefixBuffer;
+
+          if (prefixSize > 0) {
+            const result = await this._s3.send(new _clientS.GetObjectCommand(downloadParams));
+            prefixBuffer = await (0, _createBufferFromStream.default)(result.Body);
+          } else {
+            prefixBuffer = Buffer.alloc(0);
+          }
+
           editBuffer = Buffer.concat([prefixBuffer, buffer]);
           editBufferOffset -= prefixLastFragmentSize;
         }
@@ -400,7 +488,8 @@ class S3Handler extends _abstract.default {
           const downloadParams = { ...uploadParams,
             Range: prefixRange
           };
-          const complementBuffer = (await this._s3.getObject(downloadParams)).Body;
+          const result = await this._s3.send(new _clientS.GetObjectCommand(downloadParams));
+          const complementBuffer = await (0, _createBufferFromStream.default)(result.Body);
           editBuffer = Buffer.concat([editBuffer, complementBuffer]);
         }
 
@@ -408,7 +497,7 @@ class S3Handler extends _abstract.default {
           Body: editBuffer,
           PartNumber: partNumber++
         };
-        const editPart = await this._s3.uploadPart(editParams);
+        const editPart = await this._s3.send(new _clientS.UploadPartCommand(editParams));
         parts.push({
           ETag: editPart.ETag,
           PartNumber: editParams.PartNumber
@@ -428,7 +517,7 @@ class S3Handler extends _abstract.default {
               PartNumber: partNumber++,
               CopySourceRange: suffixRange
             };
-            const suffixPart = (await this._s3.uploadPartCopy(copySuffixParams)).CopyPartResult;
+            const suffixPart = (await this._s3.send(new _clientS.UploadPartCopyCommand(copySuffixParams))).CopyPartResult;
             parts.push({
               ETag: suffixPart.ETag,
               PartNumber: copySuffixParams.PartNumber
@@ -437,13 +526,13 @@ class S3Handler extends _abstract.default {
           }
         }
 
-        await this._s3.completeMultipartUpload({ ...multipartParams,
+        await this._s3.send(new _clientS.CompleteMultipartUploadCommand({ ...multipartParams,
           MultipartUpload: {
             Parts: parts
           }
-        });
+        }));
       } catch (e) {
-        await this._s3.abortMultipartUpload(multipartParams);
+        await this._s3.send(new _clientS.AbortMultipartUploadCommand(multipartParams));
         throw e;
       }
     }
@@ -455,7 +544,6 @@ class S3Handler extends _abstract.default {
 
   async _closeFile(fd) {}
 
-}
-
+}, (_applyDecoratedDescriptor(_class.prototype, "_writeFile", [_dec], Object.getOwnPropertyDescriptor(_class.prototype, "_writeFile"), _class.prototype)), _class));
 exports.default = S3Handler;
 //# sourceMappingURL=s3.js.map