@cloudant/couchbackup 2.9.13 → 2.9.14-SNAPSHOT.147

package/.scannerwork/scanner-report/source-5.txt

@@ -1,4 +1,4 @@
-// Copyright © 2017, 2021 IBM Corp. All rights reserved.
+// Copyright © 2017 IBM Corp. All rights reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -11,408 +11,104 @@
 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 // See the License for the specific language governing permissions and
 // limitations under the License.
-'use strict';
-
-/**
- * CouchBackup module.
- * @module couchbackup
- * @see module:couchbackup
- */
-
-const backupFull = require('./includes/backup.js');
-const defaults = require('./includes/config.js').apiDefaults;
-const error = require('./includes/error.js');
-const request = require('./includes/request.js');
-const restoreInternal = require('./includes/restore.js');
-const backupShallow = require('./includes/shallowbackup.js');
-const debug = require('debug')('couchbackup:app');
-const events = require('events');
-const fs = require('fs');
-const URL = require('url').URL;
-
-/**
- * Test for a positive, safe integer.
- *
- * @param {object} x - Object under test.
- */
-function isSafePositiveInteger(x) {
-  // https://developer.mozilla.org/en/docs/Web/JavaScript/Reference/Global_Objects/Number/MAX_SAFE_INTEGER
-  const MAX_SAFE_INTEGER = Number.MAX_SAFE_INTEGER || 9007199254740991;
-  // Is it a number?
-  return Object.prototype.toString.call(x) === '[object Number]' &&
-    // Is it an integer?
-    x % 1 === 0 &&
-    // Is it positive?
-    x > 0 &&
-    // Is it less than the maximum safe integer?
-    x <= MAX_SAFE_INTEGER;
-}
-
-/**
- * Validate arguments.
- *
- * @param {object} url - URL of database.
- * @param {object} opts - Options.
- * @param {function} cb - Callback to be called on error.
- */
-function validateArgs(url, opts, cb) {
-  if (typeof url !== 'string') {
-    cb(new error.BackupError('InvalidOption', 'Invalid URL, must be type string'), null);
-    return;
-  }
-  if (opts && typeof opts.bufferSize !== 'undefined' && !isSafePositiveInteger(opts.bufferSize)) {
-    cb(new error.BackupError('InvalidOption', 'Invalid buffer size option, must be a positive integer in the range (0, MAX_SAFE_INTEGER]'), null);
-    return;
-  }
-  if (opts && typeof opts.iamApiKey !== 'undefined' && typeof opts.iamApiKey !== 'string') {
-    cb(new error.BackupError('InvalidOption', 'Invalid iamApiKey option, must be type string'), null);
-    return;
-  }
-  if (opts && typeof opts.log !== 'undefined' && typeof opts.log !== 'string') {
-    cb(new error.BackupError('InvalidOption', 'Invalid log option, must be type string'), null);
-    return;
-  }
-  if (opts && typeof opts.mode !== 'undefined' && ['full', 'shallow'].indexOf(opts.mode) === -1) {
-    cb(new error.BackupError('InvalidOption', 'Invalid mode option, must be either "full" or "shallow"'), null);
-    return;
-  }
-  if (opts && typeof opts.output !== 'undefined' && typeof opts.output !== 'string') {
-    cb(new error.BackupError('InvalidOption', 'Invalid output option, must be type string'), null);
-    return;
-  }
-  if (opts && typeof opts.parallelism !== 'undefined' && !isSafePositiveInteger(opts.parallelism)) {
-    cb(new error.BackupError('InvalidOption', 'Invalid parallelism option, must be a positive integer in the range (0, MAX_SAFE_INTEGER]'), null);
-    return;
-  }
-  if (opts && typeof opts.requestTimeout !== 'undefined' && !isSafePositiveInteger(opts.requestTimeout)) {
-    cb(new error.BackupError('InvalidOption', 'Invalid request timeout option, must be a positive integer in the range (0, MAX_SAFE_INTEGER]'), null);
-    return;
-  }
-  if (opts && typeof opts.resume !== 'undefined' && typeof opts.resume !== 'boolean') {
-    cb(new error.BackupError('InvalidOption', 'Invalid resume option, must be type boolean'), null);
-    return;
-  }
 
-  // Validate URL and ensure no auth if using key
-  try {
-    const urlObject = new URL(url);
-    // We require a protocol, host and path (for db), fail if any is missing.
-    if (urlObject.protocol !== 'https:' && urlObject.protocol !== 'http:') {
-      cb(new error.BackupError('InvalidOption', 'Invalid URL protocol.'));
-      return;
-    }
-    if (!urlObject.host) {
-      cb(new error.BackupError('InvalidOption', 'Invalid URL host.'));
-      return;
-    }
-    if (!urlObject.pathname || urlObject.pathname === '/') {
-      cb(new error.BackupError('InvalidOption', 'Invalid URL, missing path element (no database).'));
-      return;
-    }
-    if (opts && opts.iamApiKey && (urlObject.username || urlObject.password)) {
-      cb(new error.BackupError('InvalidOption', 'URL user information must not be supplied when using IAM API key.'));
-      return;
-    }
-  } catch (err) {
-    cb(err);
-    return;
-  }
+/* global after before describe */
+'use strict';
 
-  // Perform validation of invalid options for shallow mode and WARN
-  // We don't error for backwards compatibility with scripts that may have been
-  // written passing complete sets of options through
-  if (opts && opts.mode === 'shallow') {
-    if (opts.log || opts.resume) {
-      console.warn('WARNING: the options "log" and "resume" are invalid when using shallow mode.');
-    }
-    if (opts.parallelism) {
-      console.warn('WARNING: the option "parallelism" has no effect when using shallow mode.');
-    }
-  }
+const url = require('url');
+const toxy = require('toxy');
+// Import the common hooks
+require('../test/hooks.js');
 
-  if (opts && opts.resume) {
-    if (!opts.log) {
-      // This is the second place we check for the presence of the log option in conjunction with resume
-      // It has to be here for the API case
-      cb(new error.BackupError('NoLogFileName', 'To resume a backup, a log file must be specified'), null);
-      return;
-    } else if (!fs.existsSync(opts.log)) {
-      cb(new error.BackupError('LogDoesNotExist', 'To resume a backup, the log file must exist'), null);
-      return;
-    }
-  }
-  return true;
-}
+const tpoisons = toxy.poisons;
+const trules = toxy.rules;
 
-function addEventListener(indicator, emitter, event, f) {
-  emitter.on(event, function(...args) {
-    if (!indicator.errored) {
-      if (event === 'error') indicator.errored = true;
-      f(...args);
-    }
+function setupProxy(poison) {
+  const backendUrl = new url.URL(process.env.COUCH_BACKEND_URL);
+  const proxy = toxy({
+    auth: `${backendUrl.username}:${backendUrl.password}`,
+    changeOrigin: true
   });
-}
 
-/*
-  Check the backup database exists and that the credentials used have
-  visibility. Callback with a fatal error if there is a problem with the DB.
-  @param {string} db - database object
-  @param {function(err)} callback - error is undefined if DB exists
-*/
-function proceedIfBackupDbValid(db, callback) {
-  db.service.headDatabase({ db: db.db }).then(() => callback()).catch(err => {
-    err = error.convertResponseError(err, err => parseIfDbValidResponseError(db, err));
-    callback(err);
-  });
-}
-
-/*
-  Check that the restore database exists, is new and is empty. Also verify that the credentials used have
-  visibility. Callback with a fatal error if there is a problem with the DB.
-  @param {string} db - database object
-  @param {function(err)} callback - error is undefined if DB exists, new and empty
-*/
-function proceedIfRestoreDbValid(db, callback) {
-  db.service.getDatabaseInformation({ db: db.db }).then(response => {
-    const { doc_count: docCount, doc_del_count: deletedDocCount } = response.result;
-    // The system databases can have a validation ddoc(s) injected in them on creation.
-    // This sets the doc count off, so we just complitely exclude the system databases from this check.
-    // The assumption here is that users restoring system databases know what they are doing.
-    if (!db.db.startsWith('_') && (docCount !== 0 || deletedDocCount !== 0)) {
-      const notEmptyDBErr = new Error(`Target database ${db.url}${db.db} is not empty.`);
-      notEmptyDBErr.name = 'DatabaseNotEmpty';
-      callback(notEmptyDBErr);
-    } else {
-      callback();
-    }
-  }).catch(err => {
-    err = error.convertResponseError(err, err => parseIfDbValidResponseError(db, err));
-    callback(err);
-  });
-}
-
-/*
-  Convert the database validation response error to a special DatabaseNotFound error
-  in case the database is missing. Otherwise delegate to the default error factory.
-  @param {object} db - database object
-  @param {object} err - HTTP response error
-*/
-function parseIfDbValidResponseError(db, err) {
-  if (err && err.status === 404) {
-    // Override the error type and message for the DB not found case
-    const msg = `Database ${db.url}` +
-    `${db.db} does not exist. ` +
-    'Check the URL and database name have been specified correctly.';
-    const noDBErr = new Error(msg);
-    noDBErr.name = 'DatabaseNotFound';
-    return noDBErr;
+  // Forward traffic to DB
+  proxy.forward(process.env.COUCH_BACKEND_URL);
+
+  switch (poison) {
+    case 'normal':
+      // No poisons to add
+      break;
+    case 'bandwidth-limit':
+      // https://github.com/h2non/toxy#bandwidth
+      // Note the implementation of bandwidth is simplistic and the threshold
+      // delay is applied to every write of the buffer, so use the smallest
+      // delay possible and adjust the rate using the bytes size instead.
+      proxy
+        .poison(tpoisons.bandwidth({ bytes: 512, threshold: 1 })); // 0.5 MB/s
+      break;
+    case 'latency':
+      // https://github.com/h2non/toxy#latency
+      proxy
+        .poison(tpoisons.latency({ max: 1500, min: 250 }))
+        .withRule(trules.probability(60));
+      break;
+    case 'slow-read':
+      // https://github.com/h2non/toxy#slow-read
+      // Note this only impacts read of data from requests so only for non-GET
+      // In practice this means that it impacts restore much more than backup
+      // since although backup POSTs to _bulk_get the content is much smaller
+      // than what is POSTed to _bulk_docs for a restore.
+      // Similarly to bandwidth-limit use a 1 ms threshold
+      proxy
+        .poison(tpoisons.slowRead({ chunk: 256, threshold: 1 }))
+        // Slow read for 10 % of the time e.g. 10 ms in every 100
+        .withRule(trules.timeThreshold({ duration: 10, period: 100 }));
+      break;
+    case 'rate-limit':
+      // https://github.com/h2non/toxy#rate-limit
+      // Simulate the Cloudant free plan with 20 lookups ps and 10 writes ps
+      proxy.post('/*/_bulk_get')
+        .poison(tpoisons.rateLimit({ limit: 20, threshold: 1000 }));
+      proxy.post('/*/_bulk_docs')
+        .poison(tpoisons.rateLimit({ limit: 10, threshold: 1000 }));
+      break;
+    default:
+      throw Error('Unknown toxy poison ' + poison);
   }
-  // Delegate to the default error factory if it wasn't a 404
-  return error.convertResponseError(err);
-}
-
-module.exports = {
-
-  /**
-   * Backup a Cloudant database to a stream.
-   *
-   * @param {string} srcUrl - URL of database to backup.
-   * @param {stream.Writable} targetStream - Stream to write content to.
-   * @param {object} opts - Backup options.
-   * @param {number} [opts.parallelism=5] - Number of parallel HTTP requests to use.
-   * @param {number} [opts.bufferSize=500] - Number of documents per batch request.
-   * @param {number} [opts.requestTimeout=120000] - Milliseconds to wait before retrying a HTTP request.
-   * @param {string} [opts.iamApiKey] - IAM API key to use to access Cloudant database.
-   * @param {string} [opts.log] - Log file name. Default uses a temporary file.
-   * @param {boolean} [opts.resume] - Whether to resume from existing log.
-   * @param {string} [opts.mode=full] - Use `full` or `shallow` mode.
-   * @param {backupRestoreCallback} callback - Called on completion.
-   */
-  backup: function(srcUrl, targetStream, opts, callback) {
-    const listenerErrorIndicator = { errored: false };
-    if (typeof callback === 'undefined' && typeof opts === 'function') {
-      callback = opts;
-      opts = {};
-    }
-    if (!validateArgs(srcUrl, opts, callback)) {
-      // bad args, bail
-      return;
-    }
-
-    // if there is an error writing to the stream, call the completion
-    // callback with the error set
-    addEventListener(listenerErrorIndicator, targetStream, 'error', function(err) {
-      debug('Error ' + JSON.stringify(err));
-      if (callback) callback(err);
-    });
-
-    opts = Object.assign({}, defaults(), opts);
 
-    const ee = new events.EventEmitter();
-
-    // Set up the DB client
-    const backupDB = request.client(srcUrl, opts);
-
-    // Validate the DB exists, before proceeding to backup
-    proceedIfBackupDbValid(backupDB, function(err) {
-      if (err) {
-        if (err.name === 'DatabaseNotFound') {
-          err.message = `${err.message} Ensure the backup source database exists.`;
-        }
-        // Didn't exist, or another fatal error, exit
-        callback(err);
-        return;
-      }
-      let backup = null;
-      if (opts.mode === 'shallow') {
-        backup = backupShallow;
-      } else { // full mode
-        backup = backupFull;
-      }
-
-      // If resuming write a newline as it's possible one would be missing from
-      // an interruption of the previous backup. If the backup was clean this
-      // will cause an empty line that will be gracefully handled by the restore.
-      if (opts.resume) {
-        targetStream.write('\n');
-      }
+  // Catch remaining traffic
+  proxy.all('/*');
+  return proxy;
+}
 
-      // Get the event emitter from the backup process so we can handle events
-      // before passing them on to the app's event emitter if needed.
-      const internalEE = backup(backupDB, opts);
-      addEventListener(listenerErrorIndicator, internalEE, 'changes', function(batch) {
-        ee.emit('changes', batch);
-      });
-      addEventListener(listenerErrorIndicator, internalEE, 'received', function(obj, q, logCompletedBatch) {
-        // this may be too verbose to have as well as the "backed up" message
-        // debug(' received batch', obj.batch, ' docs: ', obj.total, 'Time', obj.time);
-        // Callback to emit the written event when the content is flushed
-        function writeFlushed() {
-          ee.emit('written', { total: obj.total, time: obj.time, batch: obj.batch });
-          if (logCompletedBatch) {
-            logCompletedBatch(obj.batch);
-          }
-          debug(' backed up batch', obj.batch, ' docs: ', obj.total, 'Time', obj.time);
-        }
-        // Write the received content to the targetStream
-        const continueWriting = targetStream.write(JSON.stringify(obj.data) + '\n',
-          'utf8',
-          writeFlushed);
-        if (!continueWriting) {
-          // The buffer was full, pause the queue to stop the writes until we
-          // get a drain event
-          if (q && !q.paused) {
-            q.pause();
-            targetStream.once('drain', function() {
-              q.resume();
-            });
-          }
-        }
-      });
-      // For errors we expect, may or may not be fatal
-      addEventListener(listenerErrorIndicator, internalEE, 'error', function(err) {
-        debug('Error ' + JSON.stringify(err));
-        callback(err);
-      });
-      addEventListener(listenerErrorIndicator, internalEE, 'finished', function(obj) {
-        function emitFinished() {
-          debug('Backup complete - written ' + JSON.stringify(obj));
-          const summary = { total: obj.total };
-          ee.emit('finished', summary);
-          if (callback) callback(null, summary);
-        }
-        if (targetStream === process.stdout) {
-          // stdout cannot emit a finish event so use a final write + callback
-          targetStream.write('', 'utf8', emitFinished);
-        } else {
-          // If we're writing to a file, end the writes and register the
-          // emitFinished function for a callback when the file stream's finish
-          // event is emitted.
-          targetStream.end('', 'utf8', emitFinished);
-        }
-      });
+const poisons = [
+  'normal',
+  'bandwidth-limit',
+  'latency',
+  'slow-read',
+  'rate-limit'
+];
+
+poisons.forEach(function(poison) {
+  describe('unreliable network tests (using toxy poison ' + poison + ')', function() {
+    let proxy;
+
+    before('start toxy server', function() {
+      proxy = setupProxy(poison);
+      console.log('Using toxy poison ' + poison);
+
+      // For these tests COUCH_URL points to the toxy proxy on localhost whereas
+      // COUCH_BACKEND_URL is the real CouchDb instance.
+      const toxyUrl = new url.URL(process.env.COUCH_URL);
+      // Listen on the specified hostname only, so if using localhost we don't
+      // need external connections.
+      proxy.listen(toxyUrl.port, toxyUrl.hostname);
     });
-    return ee;
-  },
-
-  /**
-   * Restore a backup from a stream.
-   *
-   * @param {stream.Readable} srcStream - Stream containing backed up data.
-   * @param {string} targetUrl - Target database.
-   * @param {object} opts - Restore options.
-   * @param {number} opts.parallelism - Number of parallel HTTP requests to use. Default 5.
-   * @param {number} opts.bufferSize - Number of documents per batch request. Default 500.
-   * @param {number} opts.requestTimeout - Milliseconds to wait before retrying a HTTP request. Default 120000.
-   * @param {string} opts.iamApiKey - IAM API key to use to access Cloudant database.
-   * @param {backupRestoreCallback} callback - Called on completion.
-   */
-  restore: function(srcStream, targetUrl, opts, callback) {
-    const listenerErrorIndicator = { errored: false };
-    if (typeof callback === 'undefined' && typeof opts === 'function') {
-      callback = opts;
-      opts = {};
-    }
-    validateArgs(targetUrl, opts, callback);
-    opts = Object.assign({}, defaults(), opts);
 
-    const ee = new events.EventEmitter();
-
-    // Set up the DB client
-    const restoreDB = request.client(targetUrl, opts);
-
-    // Validate the DB exists, before proceeding to restore
-    proceedIfRestoreDbValid(restoreDB, function(err) {
-      if (err) {
-        if (err.name === 'DatabaseNotFound') {
-          err.message = `${err.message} Create the target database before restoring.`;
-        } else if (err.name === 'DatabaseNotEmpty') {
-          err.message = `${err.message} A target database must be a new and empty database.`;
-        }
-        // Didn't exist, or another fatal error, exit
-        callback(err);
-        return;
-      }
-
-      restoreInternal(
-        restoreDB,
-        opts,
-        srcStream,
-        ee,
-        function(err, writer) {
-          if (err) {
-            callback(err, null);
-            return;
-          }
-          if (writer != null) {
-            addEventListener(listenerErrorIndicator, writer, 'restored', function(obj) {
-              debug(' restored ', obj.total);
-              ee.emit('restored', { documents: obj.documents, total: obj.total });
-            });
-            addEventListener(listenerErrorIndicator, writer, 'error', function(err) {
-              debug('Error ' + JSON.stringify(err));
-              // Only call destroy if it is available on the stream
-              if (srcStream.destroy && srcStream.destroy instanceof Function) {
-                srcStream.destroy();
-              }
-              callback(err);
-            });
-            addEventListener(listenerErrorIndicator, writer, 'finished', function(obj) {
-              debug('restore complete');
-              ee.emit('finished', { total: obj.total });
-              callback(null, obj);
-            });
-          }
-        }
-      );
+    after('stop toxy server', function() {
+      proxy.close();
     });
-    return ee;
-  }
-};
 
-/**
- * Backup/restore callback
- * @callback backupRestoreCallback
- * @param {Error} err - Error object if operation failed.
- * @param {object} data - summary data for backup/restore
- */
+    delete require.cache[require.resolve('../test/ci_e2e.js')];
+    require('../test/ci_e2e.js');
+  });
+});