@cloudant/couchbackup 2.9.12 → 2.9.13-SNAPSHOT.139

package/.scannerwork/scanner-report/source-20.txt

@@ -11,271 +11,356 @@
 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 // See the License for the specific language governing permissions and
 // limitations under the License.
+
+/* global describe afterEach before after it */
 'use strict';
 
-const async = require('async');
-const events = require('events');
-const fs = require('fs');
-const error = require('./error.js');
-const spoolchanges = require('./spoolchanges.js');
-const logfilesummary = require('./logfilesummary.js');
-const logfilegetbatches = require('./logfilegetbatches.js');
-
-/**
- * Read documents from a database to be backed up.
- *
- * @param {string} db - `@cloudant/cloudant` DB object for source database.
- * @param {number} blocksize - number of documents to download in single request
- * @param {number} parallelism - number of concurrent downloads
- * @param {string} log - path to log file to use
- * @param {boolean} resume - whether to resume from an existing log file
- * @returns EventEmitter with following events:
- *  - `received` - called with a block of documents to write to backup
- *  - `error` - on error
- *  - `finished` - when backup process is finished (either complete or errored)
- */
-module.exports = function(db, options) {
-  const ee = new events.EventEmitter();
-  const start = new Date().getTime(); // backup start time
-  const batchesPerDownloadSession = 50; // max batches to read from log file for download at a time (prevent OOM)
-
-  function proceedWithBackup() {
-    if (options.resume) {
-      // pick up from existing log file from previous run
-      downloadRemainingBatches(options.log, db, ee, start, batchesPerDownloadSession, options.parallelism);
-    } else {
-      // create new log file and process
-      spoolchanges(db, options.log, options.bufferSize, ee, function(err) {
-        if (err) {
-          ee.emit('error', err);
-        } else {
-          downloadRemainingBatches(options.log, db, ee, start, batchesPerDownloadSession, options.parallelism);
-        }
-      });
-    }
-  }
-
-  validateBulkGetSupport(db, function(err) {
-    if (err) {
-      return ee.emit('error', err);
-    } else {
-      proceedWithBackup();
-    }
+const assert = require('assert');
+const parser = require('../includes/parser.js');
+
+describe('#unit Default parameters', function() {
+  let processEnvCopy;
+  let processArgvCopy;
+
+  before('Set process data for test', function() {
+    // Copy env and argv so we can reset them after the tests
+    processEnvCopy = JSON.parse(JSON.stringify(process.env));
+    processArgvCopy = JSON.parse(JSON.stringify(process.argv));
+
+    // setup environment variables
+    process.env.COUCH_URL = 'http://user:pass@myurl.com';
+    process.env.COUCH_DATABASE = 'mydb';
+    process.env.COUCH_BUFFER_SIZE = '1000';
+    process.env.COUCH_PARALLELISM = '20';
+    process.env.COUCH_REQUEST_TIMEOUT = '20000';
+    process.env.COUCH_LOG = 'my.log';
+    process.env.COUCH_RESUME = 'true';
+    process.env.COUCH_OUTPUT = 'myfile.txt';
+    process.env.COUCH_MODE = 'shallow';
+    process.env.CLOUDANT_IAM_API_KEY = 'ABC123-ZYX987_cba789-xyz321';
+    process.env.COUCH_QUIET = 'true';
   });
 
-  return ee;
-};
-
-/**
- * Validate /_bulk_get support for a specified database.
- *
- * @param {string} db - nodejs-cloudant db
- * @param {function} callback - called on completion with signature (err)
- */
-function validateBulkGetSupport(db, callback) {
-  db.service.postBulkGet({ db: db.db, docs: [] }).then(() => { callback(); }).catch(err => {
-    err = error.convertResponseError(err, function(err) {
-      switch (err.status) {
-        case undefined:
-          // There was no status code on the error
-          return err;
-        case 404:
-          return new error.BackupError('BulkGetError', 'Database does not support /_bulk_get endpoint');
-        default:
-          return new error.HTTPError(err);
-      }
-    });
-    callback(err);
+  after('Reset process data', function() {
+    process.env = processEnvCopy;
+    process.argv = processArgvCopy;
   });
-}
-
-/**
- * Download remaining batches in a log file, splitting batches into sets
- * to avoid enqueueing too many in one go.
- *
- * @param {string} log - log file name to maintain download state
- * @param {string} db - nodejs-cloudant db
- * @param {events.EventEmitter} ee - event emitter to emit received events on
- * @param {time} startTime - start time for backup process
- * @param {number} batchesPerDownloadSession - max batches to enqueue for
- *  download at a time. As batches contain many doc IDs, this helps avoid
- *  exhausting memory.
- * @param {number} parallelism - number of concurrent downloads
- * @returns function to call do download remaining batches with signature
- *  (err, {batches: batch, docs: doccount}) {@see spoolchanges}.
- */
-function downloadRemainingBatches(log, db, ee, startTime, batchesPerDownloadSession, parallelism) {
-  let total = 0; // running total of documents downloaded so far
-  let noRemainingBatches = false;
-
-  // Generate a set of batches (up to batchesPerDownloadSession) to download from the
-  // log file and download them. Set noRemainingBatches to `true` for last batch.
-  function downloadSingleBatchSet(done) {
-    // Fetch the doc IDs for the batches in the current set to
-    // download them.
-    function batchSetComplete(err, data) {
-      if (!err) {
-        total = data.total;
-      }
-      done(err);
-    }
-    function processRetrievedBatches(err, batches) {
-      if (!err) {
-        // process them in parallelised queue
-        processBatchSet(db, parallelism, log, batches, ee, startTime, total, batchSetComplete);
-      } else {
-        batchSetComplete(err);
-      }
-    }
-
-    readBatchSetIdsFromLogFile(log, batchesPerDownloadSession, function(err, batchSetIds) {
-      if (err) {
-        ee.emit('error', err);
-        // Stop processing changes file for fatal errors
-        noRemainingBatches = true;
-        done();
-      } else {
-        if (batchSetIds.length === 0) {
-          noRemainingBatches = true;
-          return done();
-        }
-        logfilegetbatches(log, batchSetIds, processRetrievedBatches);
-      }
-    });
-  }
-
-  // Return true if all batches in log file have been downloaded
-  function isFinished(callback) { callback(null, noRemainingBatches); }
-
-  function onComplete() {
-    ee.emit('finished', { total: total });
-  }
-
-  async.doUntil(downloadSingleBatchSet, isFinished, onComplete);
-}
-
-/**
- * Return a set of uncompleted download batch IDs from the log file.
- *
- * @param {string} log - log file path
- * @param {number} batchesPerDownloadSession - maximum IDs to return
- * @param {function} callback - sign (err, batchSetIds array)
- */
-function readBatchSetIdsFromLogFile(log, batchesPerDownloadSession, callback) {
-  logfilesummary(log, function processSummary(err, summary) {
-    if (!err) {
-      if (!summary.changesComplete) {
-        callback(new error.BackupError('IncompleteChangesInLogFile',
-          'WARNING: Changes did not finish spooling'));
-        return;
-      }
-      if (Object.keys(summary.batches).length === 0) {
-        return callback(null, []);
-      }
-
-      // batch IDs are the property names of summary.batches
-      const batchSetIds = getPropertyNames(summary.batches, batchesPerDownloadSession);
-      callback(null, batchSetIds);
-    } else {
-      callback(err);
-    }
+
+  afterEach(function() {
+    delete require.cache[require.resolve('commander')];
   });
-}
-
-/**
- * Download a set of batches retrieved from a log file. When a download is
- * complete, add a line to the logfile indicating such.
- *
- * @param {any} db - nodejs-cloudant database
- * @param {any} parallelism - number of concurrent requests to make
- * @param {any} log - log file to drive downloads from
- * @param {any} batches - batches to download
- * @param {any} ee - event emitter for progress. This funciton emits
- *  received and error events.
- * @param {any} start - time backup started, to report deltas
- * @param {any} grandtotal - count of documents downloaded prior to this set
- *  of batches
- * @param {any} callback - completion callback, (err, {total: number}).
- */
-function processBatchSet(db, parallelism, log, batches, ee, start, grandtotal, callback) {
-  let hasErrored = false;
-  let total = grandtotal;
-
-  // queue to process the fetch requests in an orderly fashion using _bulk_get
-  const q = async.queue(function(payload, done) {
-    const output = [];
-    const thisBatch = payload.batch;
-    delete payload.batch;
-    delete payload.command;
-
-    function logCompletedBatch(batch) {
-      if (log) {
-        fs.appendFile(log, ':d batch' + thisBatch + '\n', done);
-      } else {
-        done();
-      }
-    }
-
-    // do the /db/_bulk_get request
-    db.service.postBulkGet({
-      db: db.db,
-      revs: true,
-      docs: payload.docs
-    }).then(response => {
-      // create an output array with the docs returned
-      response.result.results.forEach(function(d) {
-        if (d.docs) {
-          d.docs.forEach(function(doc) {
-            if (doc.ok) {
-              output.push(doc.ok);
-            }
-          });
-        }
-      });
-      total += output.length;
-      const t = (new Date().getTime() - start) / 1000;
-      ee.emit('received', {
-        batch: thisBatch,
-        data: output,
-        length: output.length,
-        time: t,
-        total: total
-      }, q, logCompletedBatch);
-    }).catch(err => {
-      if (!hasErrored) {
-        hasErrored = true;
-        err = error.convertResponseError(err);
-        // Kill the queue for fatal errors
-        q.kill();
-        ee.emit('error', err);
-      }
-      done();
-    });
-  }, parallelism);
-
-  for (const i in batches) {
-    q.push(batches[i]);
-  }
-
-  q.drain(function() {
-    callback(null, { total: total });
+
+  describe('Backup command-line', function() {
+    it('respects the COUCH_URL env variable if the --url backup command-line parameter is missing', function(done) {
+      process.argv = ['node', 'test'];
+      const program = parser.parseBackupArgs();
+      assert.strictEqual(typeof program.url, 'string');
+      assert.strictEqual(program.url, process.env.COUCH_URL);
+      done();
+    });
+
+    it('respects the COUCH_DATABASE env variable if the --db backup command-line parameter is missing', function(done) {
+      process.argv = ['node', 'test'];
+      const program = parser.parseBackupArgs();
+      assert.strictEqual(typeof program.db, 'string');
+      assert.strictEqual(program.db, process.env.COUCH_DATABASE);
+      done();
+    });
+
+    it('respects the COUCH_BUFFER_SIZE env variable if the --buffer-size backup command-line parameter is missing', function(done) {
+      process.argv = ['node', 'test'];
+      const program = parser.parseBackupArgs();
+      assert.strictEqual(typeof program.bufferSize, 'number');
+      assert.strictEqual(program.bufferSize, parseInt(process.env.COUCH_BUFFER_SIZE, 10));
+      done();
+    });
+
+    it('respects the COUCH_PARALLELISM env variable if the --parallelism backup command-line parameter is missing', function(done) {
+      process.argv = ['node', 'test'];
+      const program = parser.parseBackupArgs();
+      assert.strictEqual(typeof program.parallelism, 'number');
+      assert.strictEqual(program.parallelism, parseInt(process.env.COUCH_PARALLELISM, 10));
+      done();
+    });
+
+    it('respects the COUCH_REQUEST_TIMEOUT env variable if the --request-timeout backup command-line parameter is missing', function(done) {
+      process.argv = ['node', 'test'];
+      const program = parser.parseBackupArgs();
+      assert.strictEqual(typeof program.requestTimeout, 'number');
+      assert.strictEqual(program.requestTimeout, parseInt(process.env.COUCH_REQUEST_TIMEOUT, 10));
+      done();
+    });
+
+    it('respects the CLOUDANT_IAM_API_KEY env variable if the --iam-api-key backup command-line parameter is missing', function(done) {
+      process.argv = ['node', 'test'];
+      const program = parser.parseBackupArgs();
+      assert.strictEqual(typeof program.iamApiKey, 'string');
+      assert.strictEqual(program.iamApiKey, process.env.CLOUDANT_IAM_API_KEY);
+      done();
+    });
+
+    it('respects the COUCH_LOG env variable if the --log backup command-line parameter is missing', function(done) {
+      process.argv = ['node', 'test'];
+      const program = parser.parseBackupArgs();
+      assert.strictEqual(typeof program.log, 'string');
+      assert.strictEqual(program.log, process.env.COUCH_LOG);
+      done();
+    });
+
+    it('respects the COUCH_RESUME env variable if the --resume backup command-line parameter is missing', function(done) {
+      process.argv = ['node', 'test'];
+      const program = parser.parseBackupArgs();
+      assert.strictEqual(typeof program.resume, 'boolean');
+      assert.strictEqual(program.resume, true);
+      done();
+    });
+
+    it('respects the COUCH_OUTPUT env variable if the --output backup command-line parameter is missing', function(done) {
+      process.argv = ['node', 'test'];
+      const program = parser.parseBackupArgs();
+      assert.strictEqual(typeof program.output, 'string');
+      assert.strictEqual(program.output, process.env.COUCH_OUTPUT);
+      done();
+    });
+
+    it('respects the COUCH_MODE env variable if the --mode backup command-line parameter is missing', function(done) {
+      process.argv = ['node', 'test'];
+      const program = parser.parseBackupArgs();
+      assert.strictEqual(typeof program.mode, 'string');
+      assert.strictEqual(program.mode, process.env.COUCH_MODE);
+      done();
+    });
+
+    it('respects the COUCH_QUIET env variable if the --quiet backup command-line parameter is missing', function(done) {
+      process.argv = ['node', 'test'];
+      const program = parser.parseBackupArgs();
+      assert.strictEqual(typeof program.quiet, 'boolean');
+      assert.strictEqual(program.quiet, true);
+      done();
+    });
+
+    it('respects the backup --url command-line parameter', function(done) {
+      const url = 'http://user:pass@myurl2.com';
+      process.argv = ['node', 'test', '--url', url];
+      const program = parser.parseBackupArgs();
+      assert.strictEqual(typeof program.url, 'string');
+      assert.strictEqual(program.url, url);
+      done();
+    });
+
+    it('respects the backup --db command-line parameter', function(done) {
+      const db = 'mydb2';
+      process.argv = ['node', 'test', '--db', db];
+      const program = parser.parseBackupArgs();
+      assert.strictEqual(typeof program.db, 'string');
+      assert.strictEqual(program.db, db);
+      done();
+    });
+
+    it('respects the backup --buffer-size command-line parameter', function(done) {
+      const bufferSize = 500;
+      process.argv = ['node', 'test', '--buffer-size', bufferSize];
+      const program = parser.parseBackupArgs();
+      assert.strictEqual(typeof program.bufferSize, 'number');
+      assert.strictEqual(program.bufferSize, bufferSize);
+      done();
+    });
+
+    it('respects the backup --parallelism command-line parameter', function(done) {
+      const parallelism = 10;
+      process.argv = ['node', 'test', '--parallelism', parallelism];
+      const program = parser.parseBackupArgs();
+      assert.strictEqual(typeof program.parallelism, 'number');
+      assert.strictEqual(program.parallelism, parallelism);
+      done();
+    });
+
+    it('respects the backup --request-timeout command-line parameter', function(done) {
+      const requestTimeout = 10000;
+      process.argv = ['node', 'test', '--request-timeout', requestTimeout];
+      const program = parser.parseBackupArgs();
+      assert.strictEqual(typeof program.requestTimeout, 'number');
+      assert.strictEqual(program.requestTimeout, requestTimeout);
+      done();
+    });
+
+    it('respects the backup --iam-api-key command-line parameter', function(done) {
+      const key = '123abc-789zyx_CBA987-XYZ321';
+      process.argv = ['node', 'test', '--iam-api-key', key];
+      const program = parser.parseBackupArgs();
+      assert.strictEqual(typeof program.iamApiKey, 'string');
+      assert.strictEqual(program.iamApiKey, key);
+      done();
+    });
+
+    it('respects the backup --log command-line parameter', function(done) {
+      const filename = 'my2.log';
+      process.argv = ['node', 'test', '--log', filename];
+      const program = parser.parseBackupArgs();
+      assert.strictEqual(typeof program.log, 'string');
+      assert.strictEqual(program.log, filename);
+      done();
+    });
+
+    it('respects the backup --resume command-line parameter', function(done) {
+      process.argv = ['node', 'test', '--resume'];
+      const program = parser.parseBackupArgs();
+      assert.strictEqual(typeof program.resume, 'boolean');
+      assert.strictEqual(program.resume, true);
+      done();
+    });
+
+    it('respects the backup --output command-line parameter', function(done) {
+      const filename = 'myfile2.txt';
+      process.argv = ['node', 'test', '--output', filename];
+      const program = parser.parseBackupArgs();
+      assert.strictEqual(typeof program.output, 'string');
+      assert.strictEqual(program.output, filename);
+      done();
+    });
+
+    it('respects the backup --mode full command-line parameter', function(done) {
+      process.argv = ['node', 'test', '--mode', 'full'];
+      const program = parser.parseBackupArgs();
+      assert.strictEqual(typeof program.mode, 'string');
+      assert.strictEqual(program.mode, 'full');
+      done();
+    });
+
+    it('respects the backup --mode shallow command-line parameter', function(done) {
+      process.argv = ['node', 'test', '--mode', 'shallow'];
+      const program = parser.parseBackupArgs();
+      assert.strictEqual(typeof program.mode, 'string');
+      assert.strictEqual(program.mode, 'shallow');
+      done();
+    });
+
+    it('respects the backup --quiet command-line parameter', function(done) {
+      process.argv = ['node', 'test', '--quiet'];
+      const program = parser.parseBackupArgs();
+      assert.strictEqual(typeof program.quiet, 'boolean');
+      assert.strictEqual(program.quiet, true);
+      done();
+    });
+  });
+
+  describe('Restore command-line', function() {
+    it('respects the COUCH_URL env variable if the --url restore command-line parameter is missing', function(done) {
+      process.argv = ['node', 'test'];
+      const program = parser.parseRestoreArgs();
+      assert.strictEqual(typeof program.url, 'string');
+      assert.strictEqual(program.url, process.env.COUCH_URL);
+      done();
+    });
+
+    it('respects the COUCH_DATABASE env variable if the --db restore command-line parameter is missing', function(done) {
+      process.argv = ['node', 'test'];
+      const program = parser.parseRestoreArgs();
+      assert.strictEqual(typeof program.db, 'string');
+      assert.strictEqual(program.db, process.env.COUCH_DATABASE);
+      done();
+    });
+
+    it('respects the COUCH_BUFFER_SIZE env variable if the --buffer-size restore command-line parameter is missing', function(done) {
+      process.argv = ['node', 'test'];
+      const program = parser.parseRestoreArgs();
+      assert.strictEqual(typeof program.bufferSize, 'number');
+      assert.strictEqual(program.bufferSize, parseInt(process.env.COUCH_BUFFER_SIZE, 10));
+      done();
+    });
+
+    it('respects the COUCH_PARALLELISM env variable if the --parallelism restore command-line parameter is missing', function(done) {
+      process.argv = ['node', 'test'];
+      const program = parser.parseRestoreArgs();
+      assert.strictEqual(typeof program.parallelism, 'number');
+      assert.strictEqual(program.parallelism, parseInt(process.env.COUCH_PARALLELISM, 10));
+      done();
+    });
+
+    it('respects the COUCH_REQUEST_TIMEOUT env variable if the --request-timeout restore command-line parameter is missing', function(done) {
+      process.argv = ['node', 'test'];
+      const program = parser.parseRestoreArgs();
+      assert.strictEqual(typeof program.requestTimeout, 'number');
+      assert.strictEqual(program.requestTimeout, parseInt(process.env.COUCH_REQUEST_TIMEOUT, 10));
+      done();
+    });
+
+    it('respects the CLOUDANT_IAM_API_KEY env variable if the --iam-api-key restore command-line parameter is missing', function(done) {
+      process.argv = ['node', 'test'];
+      const program = parser.parseRestoreArgs();
+      assert.strictEqual(typeof program.iamApiKey, 'string');
+      assert.strictEqual(program.iamApiKey, process.env.CLOUDANT_IAM_API_KEY);
+      done();
+    });
+
+    it('respects the COUCH_QUIET env variable if the --quiet restorer command-line parameter is missing', function(done) {
+      process.argv = ['node', 'test'];
+      const program = parser.parseBackupArgs();
+      assert.strictEqual(typeof program.quiet, 'boolean');
+      assert.strictEqual(program.quiet, true);
+      done();
+    });
+
+    it('respects the restore --url command-line parameter', function(done) {
+      const url = 'https://a:b@myurl3.com';
+      process.argv = ['node', 'test', '--url', url];
+      const program = parser.parseRestoreArgs();
+      assert.strictEqual(typeof program.url, 'string');
+      assert.strictEqual(program.url, url);
+      done();
+    });
+
+    it('respects the restore --db command-line parameter', function(done) {
+      const db = 'mydb3';
+      process.argv = ['node', 'test', '--db', db];
+      const program = parser.parseRestoreArgs();
+      assert.strictEqual(typeof program.db, 'string');
+      assert.strictEqual(program.db, db);
+      done();
+    });
+
+    it('respects the restore --buffer-size command-line parameter', function(done) {
+      const bufferSize = 250;
+      process.argv = ['node', 'test', '--buffer-size', bufferSize];
+      const program = parser.parseRestoreArgs();
+      assert.strictEqual(typeof program.bufferSize, 'number');
+      assert.strictEqual(program.bufferSize, bufferSize);
+      done();
+    });
+
+    it('respects the restore --parallelism command-line parameter', function(done) {
+      const parallelism = 5;
+      process.argv = ['node', 'test', '--parallelism', parallelism];
+      const program = parser.parseRestoreArgs();
+      assert.strictEqual(typeof program.parallelism, 'number');
+      assert.strictEqual(program.parallelism, parallelism);
+      done();
+    });
+
+    it('respects the restore --request-timeout command-line parameter', function(done) {
+      const requestTimeout = 10000;
+      process.argv = ['node', 'test', '--request-timeout', requestTimeout];
+      const program = parser.parseRestoreArgs();
+      assert.strictEqual(typeof program.requestTimeout, 'number');
+      assert.strictEqual(program.requestTimeout, requestTimeout);
+      done();
+    });
+
+    it('respects the restore --iam-api-key command-line parameter', function(done) {
+      const key = '123abc-789zyx_CBA987-XYZ321';
+      process.argv = ['node', 'test', '--iam-api-key', key];
+      const program = parser.parseRestoreArgs();
+      assert.strictEqual(typeof program.iamApiKey, 'string');
+      assert.strictEqual(program.iamApiKey, key);
+      done();
+    });
+
+    it('respects the restore --quiet command-line parameter', function(done) {
+      process.argv = ['node', 'test', '--quiet'];
+      const program = parser.parseRestoreArgs();
+      assert.strictEqual(typeof program.quiet, 'boolean');
+      assert.strictEqual(program.quiet, true);
+      done();
+    });
   });
-}
-
-/**
- * Returns first N properties on an object.
- *
- * @param {object} obj - object with properties
- * @param {number} count - number of properties to return
- */
-function getPropertyNames(obj, count) {
-  // decide which batch numbers to deal with
-  const batchestofetch = [];
-  let j = 0;
-  for (const i in obj) {
-    batchestofetch.push(parseInt(i));
-    j++;
-    if (j >= count) break;
-  }
-  return batchestofetch;
-}
+});