@cloudant/couchbackup 2.9.12-SNAPSHOT.135 → 2.9.12

package/.scannerwork/scanner-report/source-55.txt

@@ -1,4 +1,4 @@
-// Copyright © 2017, 2021 IBM Corp. All rights reserved.
+// Copyright © 2017, 2018 IBM Corp. All rights reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -11,271 +11,215 @@
 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 // See the License for the specific language governing permissions and
 // limitations under the License.
+
+/* global describe it */
 'use strict';
 
-const async = require('async');
-const events = require('events');
+const assert = require('assert');
+const backup = require('../app.js').backup;
 const fs = require('fs');
-const error = require('./error.js');
-const spoolchanges = require('./spoolchanges.js');
-const logfilesummary = require('./logfilesummary.js');
-const logfilegetbatches = require('./logfilegetbatches.js');
-
-/**
- * Read documents from a database to be backed up.
- *
- * @param {string} db - `@cloudant/cloudant` DB object for source database.
- * @param {number} blocksize - number of documents to download in single request
- * @param {number} parallelism - number of concurrent downloads
- * @param {string} log - path to log file to use
- * @param {boolean} resume - whether to resume from an existing log file
- * @returns EventEmitter with following events:
- *  - `received` - called with a block of documents to write to backup
- *  - `error` - on error
- *  - `finished` - when backup process is finished (either complete or errored)
- */
-module.exports = function(db, options) {
-  const ee = new events.EventEmitter();
-  const start = new Date().getTime(); // backup start time
-  const batchesPerDownloadSession = 50; // max batches to read from log file for download at a time (prevent OOM)
-
-  function proceedWithBackup() {
-    if (options.resume) {
-      // pick up from existing log file from previous run
-      downloadRemainingBatches(options.log, db, ee, start, batchesPerDownloadSession, options.parallelism);
-    } else {
-      // create new log file and process
-      spoolchanges(db, options.log, options.bufferSize, ee, function(err) {
-        if (err) {
-          ee.emit('error', err);
-        } else {
-          downloadRemainingBatches(options.log, db, ee, start, batchesPerDownloadSession, options.parallelism);
-        }
-      });
-    }
+const nock = require('nock');
+
+const goodUrl = 'http://localhost:5984/db';
+// The real validateArgs function of app.js isn't
+// exported - so we call the exported backup method
+// instead. We don't get as far as a real backup when
+// testing error cases. For success cases we nock the
+// goodUrl and
+const validateArgs = function(url, opts, callback) {
+  const nullStream = fs.createWriteStream('/dev/null');
+  let cb = callback;
+  if (url === goodUrl) {
+    // Nock the goodUrl
+    nock(goodUrl).head('').reply(404, { error: 'not_found', reason: 'missing' });
+    // replace the callback to handle the nock response
+    // to avoid attempting a real backup
+    cb = function(err) {
+      nullStream.end();
+      if (err.name === 'DatabaseNotFound') {
+        // This is what we expect if we reached the backup
+        // This is success for valid args cases.
+        err = null;
+      }
+      callback(err);
+    };
   }
-
-  validateBulkGetSupport(db, function(err) {
-    if (err) {
-      return ee.emit('error', err);
-    } else {
-      proceedWithBackup();
-    }
-  });
-
-  return ee;
+  backup(url, nullStream, opts, cb);
+  return true;
 };
 
-/**
- * Validate /_bulk_get support for a specified database.
- *
- * @param {string} db - nodejs-cloudant db
- * @param {function} callback - called on completion with signature (err)
- */
-function validateBulkGetSupport(db, callback) {
-  db.service.postBulkGet({ db: db.db, docs: [] }).then(() => { callback(); }).catch(err => {
-    err = error.convertResponseError(err, function(err) {
-      switch (err.status) {
-        case undefined:
-          // There was no status code on the error
-          return err;
-        case 404:
-          return new error.BackupError('BulkGetError', 'Database does not support /_bulk_get endpoint');
-        default:
-          return new error.HTTPError(err);
-      }
-    });
-    callback(err);
-  });
+const stderrWriteFun = process.stderr.write;
+let capturedStderr;
+
+function captureStderr() {
+  process.stderr.write = function(string, encoding, fd) {
+    capturedStderr += string;
+  };
 }
 
-/**
- * Download remaining batches in a log file, splitting batches into sets
- * to avoid enqueueing too many in one go.
- *
- * @param {string} log - log file name to maintain download state
- * @param {string} db - nodejs-cloudant db
- * @param {events.EventEmitter} ee - event emitter to emit received events on
- * @param {time} startTime - start time for backup process
- * @param {number} batchesPerDownloadSession - max batches to enqueue for
- *  download at a time. As batches contain many doc IDs, this helps avoid
- *  exhausting memory.
- * @param {number} parallelism - number of concurrent downloads
- * @returns function to call do download remaining batches with signature
- *  (err, {batches: batch, docs: doccount}) {@see spoolchanges}.
- */
-function downloadRemainingBatches(log, db, ee, startTime, batchesPerDownloadSession, parallelism) {
-  let total = 0; // running total of documents downloaded so far
-  let noRemainingBatches = false;
+function releaseStderr() {
+  process.stderr.write = stderrWriteFun;
+  capturedStderr = null;
+}
 
-  // Generate a set of batches (up to batchesPerDownloadSession) to download from the
-  // log file and download them. Set noRemainingBatches to `true` for last batch.
-  function downloadSingleBatchSet(done) {
-    // Fetch the doc IDs for the batches in the current set to
-    // download them.
-    function batchSetComplete(err, data) {
-      if (!err) {
-        total = data.total;
-      }
-      done(err);
-    }
-    function processRetrievedBatches(err, batches) {
-      if (!err) {
-        // process them in parallelised queue
-        processBatchSet(db, parallelism, log, batches, ee, startTime, total, batchSetComplete);
-      } else {
-        batchSetComplete(err);
-      }
+function assertErrorMessage(msg, done) {
+  return function(err, data) {
+    try {
+      assert(err.message, 'There should be an error message');
+      assert(err.message.indexOf(msg) >= 0);
+      assert(data === null || data === undefined, 'There should only be an error.');
+      done();
+    } catch (e) {
+      done(e);
     }
-
-    readBatchSetIdsFromLogFile(log, batchesPerDownloadSession, function(err, batchSetIds) {
-      if (err) {
-        ee.emit('error', err);
-        // Stop processing changes file for fatal errors
-        noRemainingBatches = true;
-        done();
-      } else {
-        if (batchSetIds.length === 0) {
-          noRemainingBatches = true;
-          return done();
-        }
-        logfilegetbatches(log, batchSetIds, processRetrievedBatches);
-      }
-    });
-  }
-
-  // Return true if all batches in log file have been downloaded
-  function isFinished(callback) { callback(null, noRemainingBatches); }
-
-  function onComplete() {
-    ee.emit('finished', { total: total });
-  }
-
-  async.doUntil(downloadSingleBatchSet, isFinished, onComplete);
+  };
 }
 
-/**
- * Return a set of uncompleted download batch IDs from the log file.
- *
- * @param {string} log - log file path
- * @param {number} batchesPerDownloadSession - maximum IDs to return
- * @param {function} callback - sign (err, batchSetIds array)
- */
-function readBatchSetIdsFromLogFile(log, batchesPerDownloadSession, callback) {
-  logfilesummary(log, function processSummary(err, summary) {
-    if (!err) {
-      if (!summary.changesComplete) {
-        callback(new error.BackupError('IncompleteChangesInLogFile',
-          'WARNING: Changes did not finish spooling'));
-        return;
-      }
-      if (Object.keys(summary.batches).length === 0) {
-        return callback(null, []);
-      }
-
-      // batch IDs are the property names of summary.batches
-      const batchSetIds = getPropertyNames(summary.batches, batchesPerDownloadSession);
-      callback(null, batchSetIds);
-    } else {
-      callback(err);
+function assertNoError(done) {
+  return function(err, data) {
+    try {
+      assert(err === null, 'There should be no error message.');
+      done();
+    } catch (e) {
+      done(e);
     }
-  });
+  };
 }
 
-/**
- * Download a set of batches retrieved from a log file. When a download is
- * complete, add a line to the logfile indicating such.
- *
- * @param {any} db - nodejs-cloudant database
- * @param {any} parallelism - number of concurrent requests to make
- * @param {any} log - log file to drive downloads from
- * @param {any} batches - batches to download
- * @param {any} ee - event emitter for progress. This funciton emits
- *  received and error events.
- * @param {any} start - time backup started, to report deltas
- * @param {any} grandtotal - count of documents downloaded prior to this set
- *  of batches
- * @param {any} callback - completion callback, (err, {total: number}).
- */
-function processBatchSet(db, parallelism, log, batches, ee, start, grandtotal, callback) {
-  let hasErrored = false;
-  let total = grandtotal;
-
-  // queue to process the fetch requests in an orderly fashion using _bulk_get
-  const q = async.queue(function(payload, done) {
-    const output = [];
-    const thisBatch = payload.batch;
-    delete payload.batch;
-    delete payload.command;
-
-    function logCompletedBatch(batch) {
-      if (log) {
-        fs.appendFile(log, ':d batch' + thisBatch + '\n', done);
-      } else {
-        done();
-      }
+describe('#unit Validate arguments', function() {
+  it('returns error for invalid URL type', function(done) {
+    validateArgs(true, {}, assertErrorMessage('Invalid URL, must be type string', done));
+  });
+  it('returns no error for valid URL type', function(done) {
+    validateArgs(goodUrl, {}, assertNoError(done));
+  });
+  it('returns error for invalid (no host) URL', function(done) {
+    validateArgs('http://', {}, assertErrorMessage('Invalid URL', done));
+  });
+  it('returns error for invalid (no protocol) URL', function(done) {
+    validateArgs('invalid', {}, assertErrorMessage('Invalid URL', done));
+  });
+  it('returns error for invalid (wrong protocol) URL', function(done) {
+    validateArgs('ftp://invalid.example.com', {}, assertErrorMessage('Invalid URL protocol.', done));
+  });
+  it('returns error for invalid (no path) URL', function(done) {
+    validateArgs('https://invalid.example.com', {}, assertErrorMessage('Invalid URL, missing path element (no database).', done));
+  });
+  it('returns error for invalid (no protocol, no host) URL', function(done) {
+    validateArgs('invalid', {}, assertErrorMessage('Invalid URL', done));
+  });
+  it('returns error for invalid buffer size type', function(done) {
+    validateArgs(goodUrl, { bufferSize: '123' }, assertErrorMessage('Invalid buffer size option, must be a positive integer in the range (0, MAX_SAFE_INTEGER]', done));
+  });
+  it('returns error for zero buffer size', function(done) {
+    validateArgs(goodUrl, { bufferSize: 0 }, assertErrorMessage('Invalid buffer size option, must be a positive integer in the range (0, MAX_SAFE_INTEGER]', done));
+  });
+  it('returns error for float buffer size', function(done) {
+    validateArgs(goodUrl, { bufferSize: 1.23 }, assertErrorMessage('Invalid buffer size option, must be a positive integer in the range (0, MAX_SAFE_INTEGER]', done));
+  });
+  it('returns no error for valid buffer size type', function(done) {
+    validateArgs(goodUrl, { bufferSize: 123 }, assertNoError(done));
+  });
+  it('returns error for invalid log type', function(done) {
+    validateArgs(goodUrl, { log: true }, assertErrorMessage('Invalid log option, must be type string', done));
+  });
+  it('returns no error for valid log type', function(done) {
+    validateArgs(goodUrl, { log: 'log.txt' }, assertNoError(done));
+  });
+  it('returns error for invalid mode type', function(done) {
+    validateArgs(goodUrl, { mode: true }, assertErrorMessage('Invalid mode option, must be either "full" or "shallow"', done));
+  });
+  it('returns error for invalid mode string', function(done) {
+    validateArgs(goodUrl, { mode: 'foobar' }, assertErrorMessage('Invalid mode option, must be either "full" or "shallow"', done));
+  });
+  it('returns no error for valid mode type', function(done) {
+    validateArgs(goodUrl, { mode: 'full' }, assertNoError(done));
+  });
+  it('returns error for invalid output type', function(done) {
+    validateArgs(goodUrl, { output: true }, assertErrorMessage('Invalid output option, must be type string', done));
+  });
+  it('returns no error for valid output type', function(done) {
+    validateArgs(goodUrl, { output: 'output.txt' }, assertNoError(done));
+  });
+  it('returns error for invalid parallelism type', function(done) {
+    validateArgs(goodUrl, { parallelism: '123' }, assertErrorMessage('Invalid parallelism option, must be a positive integer in the range (0, MAX_SAFE_INTEGER]', done));
+  });
+  it('returns error for zero parallelism', function(done) {
+    validateArgs(goodUrl, { parallelism: 0 }, assertErrorMessage('Invalid parallelism option, must be a positive integer in the range (0, MAX_SAFE_INTEGER]', done));
+  });
+  it('returns error for float parallelism', function(done) {
+    validateArgs(goodUrl, { parallelism: 1.23 }, assertErrorMessage('Invalid parallelism option, must be a positive integer in the range (0, MAX_SAFE_INTEGER]', done));
+  });
+  it('returns no error for valid parallelism type', function(done) {
+    validateArgs(goodUrl, { parallelism: 123 }, assertNoError(done));
+  });
+  it('returns error for invalid request timeout type', function(done) {
+    validateArgs(goodUrl, { requestTimeout: '123' }, assertErrorMessage('Invalid request timeout option, must be a positive integer in the range (0, MAX_SAFE_INTEGER]', done));
+  });
+  it('returns error for zero request timeout', function(done) {
+    validateArgs(goodUrl, { requestTimeout: 0 }, assertErrorMessage('Invalid request timeout option, must be a positive integer in the range (0, MAX_SAFE_INTEGER]', done));
+  });
+  it('returns error for float request timout', function(done) {
+    validateArgs(goodUrl, { requestTimeout: 1.23 }, assertErrorMessage('Invalid request timeout option, must be a positive integer in the range (0, MAX_SAFE_INTEGER]', done));
+  });
+  it('returns no error for valid request timeout type', function(done) {
+    validateArgs(goodUrl, { requestTimeout: 123 }, assertNoError(done));
+  });
+  it('returns error for invalid resume type', function(done) {
+    validateArgs(goodUrl, { resume: 'true' }, assertErrorMessage('Invalid resume option, must be type boolean', done));
+  });
+  it('returns no error for valid resume type', function(done) {
+    validateArgs(goodUrl, { resume: false }, assertNoError(done));
+  });
+  it('returns error for invalid key type', function(done) {
+    validateArgs(goodUrl, { iamApiKey: true }, assertErrorMessage('Invalid iamApiKey option, must be type string', done));
+  });
+  it('returns error for key and URL credentials supplied', function(done) {
+    validateArgs('https://a:b@example.com/db', { iamApiKey: 'abc123' }, assertErrorMessage('URL user information must not be supplied when using IAM API key.', done));
+  });
+  it('warns for log arg in shallow mode', function(done) {
+    captureStderr();
+    try {
+      validateArgs(goodUrl, { mode: 'shallow', log: 'test' }, function(err, data) {
+        assert.ok(err);
+        assert.ok(!data);
+        assert(capturedStderr.indexOf('The options "log" and "resume" are invalid when using shallow mode.') > -1, 'Log warning message was not present');
+      });
+      done();
+    } catch (e) {
+      done(e);
+    } finally {
+      releaseStderr();
     }
-
-    // do the /db/_bulk_get request
-    db.service.postBulkGet({
-      db: db.db,
-      revs: true,
-      docs: payload.docs
-    }).then(response => {
-      // create an output array with the docs returned
-      response.result.results.forEach(function(d) {
-        if (d.docs) {
-          d.docs.forEach(function(doc) {
-            if (doc.ok) {
-              output.push(doc.ok);
-            }
-          });
-        }
+  });
+  it('warns for resume arg in shallow mode', function(done) {
+    captureStderr();
+    try {
+      validateArgs(goodUrl, { mode: 'shallow', log: 'test', resume: true }, function(err, data) {
+        assert.ok(err);
+        assert.ok(!data);
+        assert(capturedStderr.indexOf('The options "log" and "resume" are invalid when using shallow mode.') > -1, 'Log warning message was not present');
       });
-      total += output.length;
-      const t = (new Date().getTime() - start) / 1000;
-      ee.emit('received', {
-        batch: thisBatch,
-        data: output,
-        length: output.length,
-        time: t,
-        total: total
-      }, q, logCompletedBatch);
-    }).catch(err => {
-      if (!hasErrored) {
-        hasErrored = true;
-        err = error.convertResponseError(err);
-        // Kill the queue for fatal errors
-        q.kill();
-        ee.emit('error', err);
-      }
       done();
-    });
-  }, parallelism);
-
-  for (const i in batches) {
-    q.push(batches[i]);
-  }
-
-  q.drain(function() {
-    callback(null, { total: total });
+    } catch (e) {
+      done(e);
+    } finally {
+      releaseStderr();
+    }
   });
-}
-
-/**
- * Returns first N properties on an object.
- *
- * @param {object} obj - object with properties
- * @param {number} count - number of properties to return
- */
-function getPropertyNames(obj, count) {
-  // decide which batch numbers to deal with
-  const batchestofetch = [];
-  let j = 0;
-  for (const i in obj) {
-    batchestofetch.push(parseInt(i));
-    j++;
-    if (j >= count) break;
-  }
-  return batchestofetch;
-}
+  it('warns for parallism arg in shallow mode', function(done) {
+    captureStderr();
+    try {
+      validateArgs(goodUrl, { mode: 'shallow', parallelsim: 10 }, function(err, data) {
+        assert.ok(err);
+        assert.ok(!data);
+        assert(capturedStderr.indexOf('The option "parallelism" has no effect when using shallow mode.') > -1, 'Log warning message was not present');
+      });
+      done();
+    } catch (e) {
+      done(e);
+    } finally {
+      releaseStderr();
+    }
+  });
+});

package/.scannerwork/scanner-report/source-56.txt

@@ -1,4 +1,4 @@
-// Copyright © 2017, 2021 IBM Corp. All rights reserved.
+// Copyright © 2018, 2021 IBM Corp. All rights reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -11,108 +11,82 @@
 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 // See the License for the specific language governing permissions and
 // limitations under the License.
-'use strict';
-
-const path = require('path');
-const tmp = require('tmp');
-
-/**
-  Return API default settings.
-*/
-function apiDefaults() {
-  return {
-    parallelism: 5,
-    bufferSize: 500,
-    requestTimeout: 120000,
-    log: tmp.tmpNameSync(),
-    resume: false,
-    mode: 'full'
-  };
-}
-
-/**
-  Return CLI default settings.
-*/
-function cliDefaults() {
-  const defaults = apiDefaults();
-
-  // add additional legacy settings
-  defaults.db = 'test';
-  defaults.url = 'http://localhost:5984';
-
-  // add CLI only option
-  defaults.quiet = false;
-
-  return defaults;
-}
-
-/**
-  Override settings **in-place** with environment variables.
-*/
-function applyEnvironmentVariables(opts) {
-  // if we have a custom CouchDB url
-  if (typeof process.env.COUCH_URL !== 'undefined') {
-    opts.url = process.env.COUCH_URL;
-  }
-
-  // if we have a specified databases
-  if (typeof process.env.COUCH_DATABASE !== 'undefined') {
-    opts.db = process.env.COUCH_DATABASE;
-  }
-
-  // if we have a specified buffer size
-  if (typeof process.env.COUCH_BUFFER_SIZE !== 'undefined') {
-    opts.bufferSize = parseInt(process.env.COUCH_BUFFER_SIZE);
-  }
 
-  // if we have a specified parallelism
-  if (typeof process.env.COUCH_PARALLELISM !== 'undefined') {
-    opts.parallelism = parseInt(process.env.COUCH_PARALLELISM);
-  }
-
-  // if we have a specified request timeout
-  if (typeof process.env.COUCH_REQUEST_TIMEOUT !== 'undefined') {
-    opts.requestTimeout = parseInt(process.env.COUCH_REQUEST_TIMEOUT);
-  }
-
-  // if we have a specified log file
-  if (typeof process.env.COUCH_LOG !== 'undefined') {
-    opts.log = path.normalize(process.env.COUCH_LOG);
-  }
-
-  // if we are instructed to resume
-  if (typeof process.env.COUCH_RESUME !== 'undefined' && process.env.COUCH_RESUME === 'true') {
-    opts.resume = true;
-  }
-
-  // if we are given an output filename
-  if (typeof process.env.COUCH_OUTPUT !== 'undefined') {
-    opts.output = path.normalize(process.env.COUCH_OUTPUT);
-  }
-
-  // if we only want a shallow copy
-  if (typeof process.env.COUCH_MODE !== 'undefined' && process.env.COUCH_MODE === 'shallow') {
-    opts.mode = 'shallow';
-  }
-
-  // if we are instructed to be quiet
-  if (typeof process.env.COUCH_QUIET !== 'undefined' && process.env.COUCH_QUIET === 'true') {
-    opts.quiet = true;
-  }
-
-  // if we have a specified API key
-  if (typeof process.env.CLOUDANT_IAM_API_KEY !== 'undefined') {
-    opts.iamApiKey = process.env.CLOUDANT_IAM_API_KEY;
-  }
-
-  // if we have a specified IAM token endpoint
-  if (typeof process.env.CLOUDANT_IAM_TOKEN_URL !== 'undefined') {
-    opts.iamTokenUrl = process.env.CLOUDANT_IAM_TOKEN_URL;
-  }
-}
+/* global describe it */
+'use strict';
 
-module.exports = {
-  apiDefaults: apiDefaults,
-  cliDefaults: cliDefaults,
-  applyEnvironmentVariables: applyEnvironmentVariables
-};
+const fs = require('fs');
+const readline = require('readline');
+const u = require('./citestutils.js');
+const uuid = require('uuid').v4;
+
+const params = { useApi: true };
+
+describe(u.scenario('Concurrent database backups', params), function() {
+  it('should run concurrent API database backups correctly #slower', function(done) {
+    // Allow up to 900 s to backup and compare (it should be much faster)!
+    u.setTimeout(this, 900);
+
+    let doneCount = 0;
+    let doneErr;
+    const finished = function(err) {
+      doneCount++;
+      if (doneCount === 2) {
+        done(doneErr || err);
+      }
+      doneErr = err;
+    };
+
+    const checkForEmptyBatches = function(fileName, cb) {
+      let foundEmptyBatch = false;
+
+      const rd = readline.createInterface({
+        input: fs.createReadStream(fileName),
+        output: fs.createWriteStream('/dev/null'),
+        terminal: false
+      });
+
+      rd.on('line', function(line) {
+        if (JSON.parse(line).length === 0) {
+          // Note: Empty batch arrays indicate that the running backup is
+          // incorrectly sharing a log file with another ongoing backup job.
+          foundEmptyBatch = true;
+        }
+      });
+
+      rd.on('close', function() {
+        if (foundEmptyBatch) {
+          cb(new Error(`Log file '${fileName}' contains empty batches`));
+        } else {
+          cb();
+        }
+      });
+    };
+
+    // [1] Run 'largedb2g' database backup
+    const actualBackup1 = `./${uuid()}`;
+    const output1 = fs.createWriteStream(actualBackup1);
+    output1.on('open', function() {
+      u.testBackup(params, 'largedb2g', output1, function(err) {
+        if (err) {
+          finished(err);
+        } else {
+          checkForEmptyBatches(actualBackup1, finished);
+        }
+      });
+    });
+
+    // [2] Run 'largedb1g' database backup
+    const actualBackup2 = `./${uuid()}`;
+    const output2 = fs.createWriteStream(actualBackup2);
+    output2.on('open', function() {
+      u.testBackup(params, 'largedb1g', output2, function(err) {
+        if (err) {
+          finished(err);
+        } else {
+          checkForEmptyBatches(actualBackup2, finished);
+        }
+      });
+    });
+  });
+});