@cloudant/couchbackup 2.9.13-SNAPSHOT.143 → 2.9.13

package/.scannerwork/scanner-report/source-11.txt

@@ -1,4 +1,4 @@
-// Copyright © 2018, 2021 IBM Corp. All rights reserved.
+// Copyright © 2017, 2018 IBM Corp. All rights reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -12,81 +12,164 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-/* global describe it */
+// Small script which backs up a Cloudant or CouchDB database to an S3
+// bucket via a stream rather than on-disk file.
+//
+// The script generates the backup object name by combining together the path
+// part of the database URL and the current time.
+
 'use strict';
 
-const fs = require('fs');
-const readline = require('readline');
-const u = require('./citestutils.js');
-const uuid = require('uuid').v4;
+const stream = require('stream');
+const url = require('url');
 
-const params = { useApi: true };
+const AWS = require('aws-sdk');
+const couchbackup = require('@cloudant/couchbackup');
+const debug = require('debug')('s3-backup');
+const VError = require('verror').VError;
 
-describe(u.scenario('Concurrent database backups', params), function() {
-  it('should run concurrent API database backups correctly #slower', function(done) {
-    // Allow up to 900 s to backup and compare (it should be much faster)!
-    u.setTimeout(this, 900);
+/*
+  Main function, run from base of file.
+*/
+function main() {
+  const argv = require('yargs')
+    .usage('Usage: $0 [options]')
+    .example('$0 -s https://user:pass@host/db -b <bucket>', 'Backup db to bucket')
+    .options({
+      source: { alias: 's', nargs: 1, demandOption: true, describe: 'Source database URL' },
+      bucket: { alias: 'b', nargs: 1, demandOption: true, describe: 'Destination bucket' },
+      prefix: { alias: 'p', nargs: 1, describe: 'Prefix for backup object key', default: 'couchbackup' },
+      s3url: { nargs: 1, describe: 'S3 endpoint URL' },
+      awsprofile: { nargs: 1, describe: 'The profile section to use in the ~/.aws/credentials file', default: 'default' }
+    })
+    .help('h').alias('h', 'help')
+    .epilog('Copyright (C) IBM 2017')
+    .argv;
 
-    let doneCount = 0;
-    let doneErr;
-    const finished = function(err) {
-      doneCount++;
-      if (doneCount === 2) {
-        done(doneErr || err);
-      }
-      doneErr = err;
-    };
+  const sourceUrl = argv.source;
+  const backupName = new url.URL(sourceUrl).pathname.split('/').filter(function(x) { return x; }).join('-');
+  const backupBucket = argv.bucket;
+  const backupKeyPrefix = `${argv.prefix}-${backupName}`;
+  const shallow = argv.shallow;
 
-    const checkForEmptyBatches = function(fileName, cb) {
-      let foundEmptyBatch = false;
+  const backupKey = `${backupKeyPrefix}-${new Date().toISOString()}`;
 
-      const rd = readline.createInterface({
-        input: fs.createReadStream(fileName),
-        output: fs.createWriteStream('/dev/null'),
-        terminal: false
-      });
+  const s3Endpoint = argv.s3url;
+  const awsProfile = argv.awsprofile;
 
-      rd.on('line', function(line) {
-        if (JSON.parse(line).length === 0) {
-          // Note: Empty batch arrays indicate that the running backup is
-          // incorrectly sharing a log file with another ongoing backup job.
-          foundEmptyBatch = true;
-        }
-      });
+  // Creds are from ~/.aws/credentials, environment etc. (see S3 docs).
+  const awsOpts = {
+    signatureVersion: 'v4',
+    credentials: new AWS.SharedIniFileCredentials({ profile: awsProfile })
+  };
+  if (typeof s3Endpoint !== 'undefined') {
+    awsOpts.endpoint = new AWS.Endpoint(s3Endpoint);
+  }
+  const s3 = new AWS.S3(awsOpts);
 
-      rd.on('close', function() {
-        if (foundEmptyBatch) {
-          cb(new Error(`Log file '${fileName}' contains empty batches`));
-        } else {
-          cb();
-        }
-      });
-    };
+  debug(`Creating a new backup of ${s(sourceUrl)} at ${backupBucket}/${backupKey}...`);
+  bucketAccessible(s3, backupBucket)
+    .then(() => {
+      return backupToS3(sourceUrl, s3, backupBucket, backupKey, shallow);
+    })
+    .then(() => {
+      debug('done.');
+    })
+    .catch((reason) => {
+      debug(`Error: ${reason}`);
+      process.exit(1);
+    });
+}
 
-    // [1] Run 'largedb2g' database backup
-    const actualBackup1 = `./${uuid()}`;
-    const output1 = fs.createWriteStream(actualBackup1);
-    output1.on('open', function() {
-      u.testBackup(params, 'largedb2g', output1, function(err) {
-        if (err) {
-          finished(err);
-        } else {
-          checkForEmptyBatches(actualBackup1, finished);
-        }
-      });
+/**
+ * Return a promise that resolves if the bucket is available and
+ * rejects if not.
+ *
+ * @param {any} s3 S3 client object
+ * @param {any} bucketName Bucket name
+ * @returns Promise
+ */
+function bucketAccessible(s3, bucketName) {
+  return new Promise(function(resolve, reject) {
+    const params = {
+      Bucket: bucketName
+    };
+    s3.headBucket(params, function(err, data) {
+      if (err) {
+        reject(new VError(err, 'S3 bucket not accessible'));
+      } else {
+        resolve();
+      }
     });
+  });
+}
+
+/**
+ * Backup directly from Cloudant to an object store object via a stream.
+ *
+ * @param {any} sourceUrl URL of database
+ * @param {any} s3Client Object store client
+ * @param {any} s3Bucket Backup destination bucket
+ * @param {any} s3Key Backup destination key name (shouldn't exist)
+ * @param {any} shallow Whether to use the couchbackup `shallow` mode
+ * @returns Promise
+ */
+function backupToS3(sourceUrl, s3Client, s3Bucket, s3Key, shallow) {
+  return new Promise((resolve, reject) => {
+    debug(`Setting up S3 upload to ${s3Bucket}/${s3Key}`);
+
+    // A pass through stream that has couchbackup's output
+    // written to it and it then read by the S3 upload client.
+    // It has a 64MB highwater mark to allow for fairly
+    // uneven network connectivity.
+    const streamToUpload = new stream.PassThrough({ highWaterMark: 67108864 });
 
-    // [2] Run 'largedb1g' database backup
-    const actualBackup2 = `./${uuid()}`;
-    const output2 = fs.createWriteStream(actualBackup2);
-    output2.on('open', function() {
-      u.testBackup(params, 'largedb1g', output2, function(err) {
+    // Set up S3 upload.
+    const params = {
+      Bucket: s3Bucket,
+      Key: s3Key,
+      Body: streamToUpload
+    };
+    s3Client.upload(params, function(err, data) {
+      debug('Object store upload done');
+      if (err) {
+        debug(err);
+        reject(new VError(err, 'Object store upload failed'));
+        return;
+      }
+      debug('Object store upload succeeded');
+      debug(data);
+      resolve();
+    }).httpUploadProgress = (progress) => {
+      debug(`Object store upload progress: ${progress}`);
+    };
+
+    debug(`Starting streaming data from ${s(sourceUrl)}`);
+    couchbackup.backup(
+      sourceUrl,
+      streamToUpload,
+      (err, obj) => {
         if (err) {
-          finished(err);
-        } else {
-          checkForEmptyBatches(actualBackup2, finished);
+          debug(err);
+          reject(new VError(err, 'CouchBackup failed with an error'));
+          return;
         }
-      });
-    });
+        debug(`Download from ${s(sourceUrl)} complete.`);
+        streamToUpload.end(); // must call end() to complete upload.
+        // resolve() is called by the upload
+      }
+    );
   });
-});
+}
+
+/**
+ * Remove creds from a URL, e.g., before logging
+ *
+ * @param {string} url URL to safen
+ */
+function s(originalUrl) {
+  const parts = new url.URL(originalUrl);
+  return url.format(parts, { auth: false });
+}
+
+main();

package/.scannerwork/scanner-report/source-12.txt

@@ -1,4 +1,4 @@
-// Copyright © 2017, 2021 IBM Corp. All rights reserved.
+// Copyright © 2017, 2018 IBM Corp. All rights reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -12,57 +12,179 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-/* global beforeEach afterEach */
+// Small script which backs up a Cloudant or CouchDB database to an S3
+// bucket, using an intermediary file on disk.
+//
+// The script generates the backup object name by combining together the path
+// part of the database URL and the current time.
+
 'use strict';
 
-const { CloudantV1 } = require('@ibm-cloud/cloudant');
-const url = new URL((process.env.COUCH_BACKEND_URL) ? process.env.COUCH_BACKEND_URL : 'https://no-couch-backend-url-set.test');
-const { BasicAuthenticator, NoAuthAuthenticator } = require('ibm-cloud-sdk-core');
-const authenticator = (url.username) ? new BasicAuthenticator({ username: url.username, password: decodeURIComponent(url.password) }) : new NoAuthAuthenticator();
-const serviceOpts = {
-  authenticator: authenticator
-};
-const cloudant = new CloudantV1(serviceOpts);
-// Remove auth from URL before using for service
-cloudant.setServiceUrl(new URL(url.pathname, url.origin).toString());
-const uuid = require('uuid').v4;
+const stream = require('stream');
 const fs = require('fs');
+const url = require('url');
 
-// Mocha hooks that will be at the root context so run for all tests
-
-beforeEach('Create test database', function(done) {
-  // Don't run hook for unit tests, just for CI
-  if (!this.currentTest.fullTitle().includes('#unit')) {
-    // Allow 10 seconds to create the DB
-    this.timeout(10 * 1000);
-    const unique = uuid();
-    this.fileName = `${unique}`;
-    this.dbName = 'couchbackup_test_' + unique;
-    cloudant.putDatabase({ db: this.dbName }).then(() => { done(); }).catch((err) => { done(err); });
-  } else {
-    done();
-  }
-});
-
-afterEach('Delete test database', function(done) {
-  // Don't run hook for unit tests, just for CI
-  if (!this.currentTest.fullTitle().includes('#unit')) {
-    // Allow 10 seconds to delete the DB
-    this.timeout(10 * 1000);
-    deleteIfExists(this.fileName);
-    deleteIfExists(`${this.fileName}.log`);
-    cloudant.deleteDatabase({ db: this.dbName }).then(() => { done(); }).catch((err) => { done(err); });
-  } else {
-    done();
+const AWS = require('aws-sdk');
+const couchbackup = require('@cloudant/couchbackup');
+const debug = require('debug')('s3-backup');
+const tmp = require('tmp');
+const VError = require('verror').VError;
+
+/*
+  Main function, run from base of file.
+*/
+function main() {
+  const argv = require('yargs')
+    .usage('Usage: $0 [options]')
+    .example('$0 -s https://user:pass@host/db -b <bucket>', 'Backup db to bucket')
+    .options({
+      source: { alias: 's', nargs: 1, demandOption: true, describe: 'Source database URL' },
+      bucket: { alias: 'b', nargs: 1, demandOption: true, describe: 'Destination bucket' },
+      prefix: { alias: 'p', nargs: 1, describe: 'Prefix for backup object key', default: 'couchbackup' },
+      s3url: { nargs: 1, describe: 'S3 endpoint URL' },
+      awsprofile: { nargs: 1, describe: 'The profile section to use in the ~/.aws/credentials file', default: 'default' }
+    })
+    .help('h').alias('h', 'help')
+    .epilog('Copyright (C) IBM 2017')
+    .argv;
+
+  const sourceUrl = argv.source;
+  const backupBucket = argv.bucket;
+  const backupName = new url.URL(sourceUrl).pathname.split('/').filter(function(x) { return x; }).join('-');
+  const backupKeyPrefix = `${argv.prefix}-${backupName}`;
+
+  const backupKey = `${backupKeyPrefix}-${new Date().toISOString()}`;
+  const backupTmpFile = tmp.fileSync();
+
+  const s3Endpoint = argv.s3url;
+  const awsProfile = argv.awsprofile;
+
+  // Creds are from ~/.aws/credentials, environment etc. (see S3 docs).
+  const awsOpts = {
+    signatureVersion: 'v4',
+    credentials: new AWS.SharedIniFileCredentials({ profile: awsProfile })
+  };
+  if (typeof s3Endpoint !== 'undefined') {
+    awsOpts.endpoint = new AWS.Endpoint(s3Endpoint);
   }
-});
+  const s3 = new AWS.S3(awsOpts);
+
+  debug(`Creating a new backup of ${s(sourceUrl)} at ${backupBucket}/${backupKey}...`);
+  bucketAccessible(s3, backupBucket)
+    .then(() => {
+      return createBackupFile(sourceUrl, backupTmpFile.name);
+    })
+    .then(() => {
+      return uploadNewBackup(s3, backupTmpFile.name, backupBucket, backupKey);
+    })
+    .then(() => {
+      debug('Backup successful!');
+      backupTmpFile.removeCallback();
+      debug('done.');
+    })
+    .catch((reason) => {
+      debug(`Error: ${reason}`);
+    });
+}
+
+/**
+ * Return a promise that resolves if the bucket is available and
+ * rejects if not.
+ *
+ * @param {any} s3 S3 client object
+ * @param {any} bucketName Bucket name
+ * @returns Promise
+ */
+function bucketAccessible(s3, bucketName) {
+  return new Promise(function(resolve, reject) {
+    const params = {
+      Bucket: bucketName
+    };
+    s3.headBucket(params, function(err, data) {
+      if (err) {
+        reject(new VError(err, 'S3 bucket not accessible'));
+      } else {
+        resolve();
+      }
+    });
+  });
+}
 
-function deleteIfExists(fileName) {
-  fs.unlink(fileName, function(err) {
-    if (err) {
-      if (err.code !== 'ENOENT') {
-        console.error(`${err.code} ${err.message}`);
+/**
+ * Use couchbackup to create a backup of the specified database to a file path.
+ *
+ * @param {any} sourceUrl Database URL
+ * @param {any} backupTmpFilePath Path to write file
+ * @returns Promise
+ */
+function createBackupFile(sourceUrl, backupTmpFilePath) {
+  return new Promise((resolve, reject) => {
+    couchbackup.backup(
+      sourceUrl,
+      fs.createWriteStream(backupTmpFilePath),
+      (err) => {
+        if (err) {
+          return reject(new VError(err, 'CouchBackup process failed'));
+        }
+        debug('couchbackup to file done; uploading to S3');
+        resolve('creating backup file complete');
       }
+    );
+  });
+}
+
+/**
+ * Upload a backup file to an S3 bucket.
+ *
+ * @param {any} s3 Object store client
+ * @param {any} backupTmpFilePath Path of backup file to write.
+ * @param {any} bucket Object store bucket name
+ * @param {any} key Object store key name
+ * @returns Promise
+ */
+function uploadNewBackup(s3, backupTmpFilePath, bucket, key) {
+  return new Promise((resolve, reject) => {
+    debug(`Uploading from ${backupTmpFilePath} to ${bucket}/${key}`);
+
+    function uploadFromStream(s3, bucket, key) {
+      const pass = new stream.PassThrough();
+
+      const params = {
+        Bucket: bucket,
+        Key: key,
+        Body: pass
+      };
+      s3.upload(params, function(err, data) {
+        debug('S3 upload done');
+        if (err) {
+          debug(err);
+          reject(new VError(err, 'Upload failed'));
+          return;
+        }
+        debug('Upload succeeded');
+        debug(data);
+        resolve();
+      }).httpUploadProgress = (progress) => {
+        debug(`S3 upload progress: ${progress}`);
+      };
+
+      return pass;
     }
+
+    const inputStream = fs.createReadStream(backupTmpFilePath);
+    const s3Stream = uploadFromStream(s3, bucket, key);
+    inputStream.pipe(s3Stream);
   });
 }
+
+/**
+ * Remove creds from a URL, e.g., before logging
+ *
+ * @param {string} url URL to safen
+ */
+function s(originalUrl) {
+  const parts = new url.URL(originalUrl);
+  return url.format(parts, { auth: false });
+}
+
+main();

package/.scannerwork/scanner-report/source-14.txt

@@ -1,4 +1,4 @@
-// Copyright © 2017, 2018 IBM Corp. All rights reserved.
+// Copyright © 2017, 2021 IBM Corp. All rights reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -12,57 +12,57 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-/* global describe it */
+/* global beforeEach afterEach */
 'use strict';
 
+const { CloudantV1 } = require('@ibm-cloud/cloudant');
+const url = new URL((process.env.COUCH_BACKEND_URL) ? process.env.COUCH_BACKEND_URL : 'https://no-couch-backend-url-set.test');
+const { BasicAuthenticator, NoAuthAuthenticator } = require('ibm-cloud-sdk-core');
+const authenticator = (url.username) ? new BasicAuthenticator({ username: url.username, password: decodeURIComponent(url.password) }) : new NoAuthAuthenticator();
+const serviceOpts = {
+  authenticator: authenticator
+};
+const cloudant = new CloudantV1(serviceOpts);
+// Remove auth from URL before using for service
+cloudant.setServiceUrl(new URL(url.pathname, url.origin).toString());
+const uuid = require('uuid').v4;
 const fs = require('fs');
-const u = require('./citestutils.js');
 
-[{ useApi: true }, { useApi: false }].forEach(function(params) {
-  describe(u.scenario('Compression tests', params), function() {
-    const p = u.p(params, { compression: true });
+// Mocha hooks that will be at the root context so run for all tests
 
-    it('should backup animaldb to a compressed file', function(done) {
-      // Allow up to 60 s for backup of animaldb
-      u.setTimeout(this, 60);
-      const compressedBackup = `./${this.fileName}`;
-      const output = fs.createWriteStream(compressedBackup);
-      output.on('open', function() {
-        u.testBackup(p, 'animaldb', output, function(err) {
-          if (err) {
-            done(err);
-          } else {
-            u.assertGzipFile(compressedBackup, done);
-          }
-        });
-      });
-    });
-
-    it('should backup and restore animaldb via a compressed file', function(done) {
-      // Allow up to 60 s for backup and restore of animaldb
-      u.setTimeout(this, 60);
-      const compressedBackup = `./${this.fileName}`;
-      u.testBackupAndRestoreViaFile(p, 'animaldb', compressedBackup, this.dbName, function(err) {
-        if (err) {
-          done(err);
-        } else {
-          u.assertGzipFile(compressedBackup, done);
-        }
-      });
-    });
+beforeEach('Create test database', function(done) {
+  // Don't run hook for unit tests, just for CI
+  if (!this.currentTest.fullTitle().includes('#unit')) {
+    // Allow 10 seconds to create the DB
+    this.timeout(10 * 1000);
+    const unique = uuid();
+    this.fileName = `${unique}`;
+    this.dbName = 'couchbackup_test_' + unique;
+    cloudant.putDatabase({ db: this.dbName }).then(() => { done(); }).catch((err) => { done(err); });
+  } else {
+    done();
+  }
+});
 
-    it('should backup and restore animaldb via a compressed stream', function(done) {
-      // Allow up to 60 s for backup and restore of animaldb
-      u.setTimeout(this, 60);
-      u.testDirectBackupAndRestore(p, 'animaldb', this.dbName, done);
-    });
+afterEach('Delete test database', function(done) {
+  // Don't run hook for unit tests, just for CI
+  if (!this.currentTest.fullTitle().includes('#unit')) {
+    // Allow 10 seconds to delete the DB
+    this.timeout(10 * 1000);
+    deleteIfExists(this.fileName);
+    deleteIfExists(`${this.fileName}.log`);
+    cloudant.deleteDatabase({ db: this.dbName }).then(() => { done(); }).catch((err) => { done(err); });
+  } else {
+    done();
+  }
+});
 
-    it('should backup and restore largedb2g via a compressed file #slower', function(done) {
-      // Takes ~ 25 min using CLI, but sometimes over an hour with API
-      u.setTimeout(this, 180 * 60);
-      const compressedBackup = `./${this.fileName}`;
-      params.compression = true;
-      u.testBackupAndRestoreViaFile(p, 'largedb2g', compressedBackup, this.dbName, done);
-    });
+function deleteIfExists(fileName) {
+  fs.unlink(fileName, function(err) {
+    if (err) {
+      if (err.code !== 'ENOENT') {
+        console.error(`${err.code} ${err.message}`);
+      }
+    }
   });
-});
+}