@cloudant/couchbackup 2.9.12-SNAPSHOT.136 → 2.9.12

package/.scannerwork/scanner-report/source-6.txt

@@ -13,7 +13,7 @@
 // limitations under the License.
 
 // Small script which backs up a Cloudant or CouchDB database to an S3
-// bucket via a stream rather than on-disk file.
+// bucket, using an intermediary file on disk.
 //
 // The script generates the backup object name by combining together the path
 // part of the database URL and the current time.
@@ -21,11 +21,13 @@
 'use strict';
 
 const stream = require('stream');
+const fs = require('fs');
 const url = require('url');
 
 const AWS = require('aws-sdk');
 const couchbackup = require('@cloudant/couchbackup');
 const debug = require('debug')('s3-backup');
+const tmp = require('tmp');
 const VError = require('verror').VError;
 
 /*
@@ -47,12 +49,12 @@ function main() {
     .argv;
 
   const sourceUrl = argv.source;
-  const backupName = new url.URL(sourceUrl).pathname.split('/').filter(function(x) { return x; }).join('-');
   const backupBucket = argv.bucket;
+  const backupName = new url.URL(sourceUrl).pathname.split('/').filter(function(x) { return x; }).join('-');
   const backupKeyPrefix = `${argv.prefix}-${backupName}`;
-  const shallow = argv.shallow;
 
   const backupKey = `${backupKeyPrefix}-${new Date().toISOString()}`;
+  const backupTmpFile = tmp.fileSync();
 
   const s3Endpoint = argv.s3url;
   const awsProfile = argv.awsprofile;
@@ -70,14 +72,18 @@ function main() {
   debug(`Creating a new backup of ${s(sourceUrl)} at ${backupBucket}/${backupKey}...`);
   bucketAccessible(s3, backupBucket)
     .then(() => {
-      return backupToS3(sourceUrl, s3, backupBucket, backupKey, shallow);
+      return createBackupFile(sourceUrl, backupTmpFile.name);
+    })
+    .then(() => {
+      return uploadNewBackup(s3, backupTmpFile.name, backupBucket, backupKey);
     })
     .then(() => {
+      debug('Backup successful!');
+      backupTmpFile.removeCallback();
       debug('done.');
     })
     .catch((reason) => {
       debug(`Error: ${reason}`);
-      process.exit(1);
     });
 }
 
@@ -105,63 +111,72 @@ function bucketAccessible(s3, bucketName) {
 }
 
 /**
- * Backup directly from Cloudant to an object store object via a stream.
+ * Use couchbackup to create a backup of the specified database to a file path.
  *
- * @param {any} sourceUrl URL of database
- * @param {any} s3Client Object store client
- * @param {any} s3Bucket Backup destination bucket
- * @param {any} s3Key Backup destination key name (shouldn't exist)
- * @param {any} shallow Whether to use the couchbackup `shallow` mode
+ * @param {any} sourceUrl Database URL
+ * @param {any} backupTmpFilePath Path to write file
  * @returns Promise
  */
-function backupToS3(sourceUrl, s3Client, s3Bucket, s3Key, shallow) {
+function createBackupFile(sourceUrl, backupTmpFilePath) {
   return new Promise((resolve, reject) => {
-    debug(`Setting up S3 upload to ${s3Bucket}/${s3Key}`);
-
-    // A pass through stream that has couchbackup's output
-    // written to it and it then read by the S3 upload client.
-    // It has a 64MB highwater mark to allow for fairly
-    // uneven network connectivity.
-    const streamToUpload = new stream.PassThrough({ highWaterMark: 67108864 });
-
-    // Set up S3 upload.
-    const params = {
-      Bucket: s3Bucket,
-      Key: s3Key,
-      Body: streamToUpload
-    };
-    s3Client.upload(params, function(err, data) {
-      debug('Object store upload done');
-      if (err) {
-        debug(err);
-        reject(new VError(err, 'Object store upload failed'));
-        return;
-      }
-      debug('Object store upload succeeded');
-      debug(data);
-      resolve();
-    }).httpUploadProgress = (progress) => {
-      debug(`Object store upload progress: ${progress}`);
-    };
-
-    debug(`Starting streaming data from ${s(sourceUrl)}`);
     couchbackup.backup(
       sourceUrl,
-      streamToUpload,
-      (err, obj) => {
+      fs.createWriteStream(backupTmpFilePath),
+      (err) => {
         if (err) {
-          debug(err);
-          reject(new VError(err, 'CouchBackup failed with an error'));
-          return;
+          return reject(new VError(err, 'CouchBackup process failed'));
         }
-        debug(`Download from ${s(sourceUrl)} complete.`);
-        streamToUpload.end(); // must call end() to complete upload.
-        // resolve() is called by the upload
+        debug('couchbackup to file done; uploading to S3');
+        resolve('creating backup file complete');
       }
     );
   });
 }
 
+/**
+ * Upload a backup file to an S3 bucket.
+ *
+ * @param {any} s3 Object store client
+ * @param {any} backupTmpFilePath Path of backup file to write.
+ * @param {any} bucket Object store bucket name
+ * @param {any} key Object store key name
+ * @returns Promise
+ */
+function uploadNewBackup(s3, backupTmpFilePath, bucket, key) {
+  return new Promise((resolve, reject) => {
+    debug(`Uploading from ${backupTmpFilePath} to ${bucket}/${key}`);
+
+    function uploadFromStream(s3, bucket, key) {
+      const pass = new stream.PassThrough();
+
+      const params = {
+        Bucket: bucket,
+        Key: key,
+        Body: pass
+      };
+      s3.upload(params, function(err, data) {
+        debug('S3 upload done');
+        if (err) {
+          debug(err);
+          reject(new VError(err, 'Upload failed'));
+          return;
+        }
+        debug('Upload succeeded');
+        debug(data);
+        resolve();
+      }).httpUploadProgress = (progress) => {
+        debug(`S3 upload progress: ${progress}`);
+      };
+
+      return pass;
+    }
+
+    const inputStream = fs.createReadStream(backupTmpFilePath);
+    const s3Stream = uploadFromStream(s3, bucket, key);
+    inputStream.pipe(s3Stream);
+  });
+}
+
 /**
  * Remove creds from a URL, e.g., before logging
  *