@cloudant/couchbackup 2.9.16-SNAPSHOT.182 → 2.9.16

package/.scannerwork/scanner-report/source-66.txt

@@ -1,4 +1,4 @@
-// Copyright © 2017, 2021 IBM Corp. All rights reserved.
+// Copyright © 2017, 2018 IBM Corp. All rights reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -11,404 +11,165 @@
 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 // See the License for the specific language governing permissions and
 // limitations under the License.
-'use strict';
-
-/**
- * CouchBackup module.
- * @module couchbackup
- * @see module:couchbackup
- */
-
-const backupFull = require('./includes/backup.js');
-const defaults = require('./includes/config.js').apiDefaults;
-const error = require('./includes/error.js');
-const request = require('./includes/request.js');
-const restoreInternal = require('./includes/restore.js');
-const backupShallow = require('./includes/shallowbackup.js');
-const debug = require('debug')('couchbackup:app');
-const events = require('events');
-const fs = require('fs');
-const URL = require('url').URL;
-
-/**
- * Test for a positive, safe integer.
- *
- * @param {object} x - Object under test.
- */
-function isSafePositiveInteger(x) {
-  // https://developer.mozilla.org/en/docs/Web/JavaScript/Reference/Global_Objects/Number/MAX_SAFE_INTEGER
-  const MAX_SAFE_INTEGER = Number.MAX_SAFE_INTEGER || 9007199254740991;
-  // Is it a number?
-  return Object.prototype.toString.call(x) === '[object Number]' &&
-    // Is it an integer?
-    x % 1 === 0 &&
-    // Is it positive?
-    x > 0 &&
-    // Is it less than the maximum safe integer?
-    x <= MAX_SAFE_INTEGER;
-}
-
-/**
- * Validate arguments.
- *
- * @param {object} url - URL of database.
- * @param {object} opts - Options.
- * @param {function} cb - Callback to be called on error.
- */
-function validateArgs(url, opts, cb) {
-  if (typeof url !== 'string') {
-    cb(new error.BackupError('InvalidOption', 'Invalid URL, must be type string'), null);
-    return;
-  }
-  if (opts && typeof opts.bufferSize !== 'undefined' && !isSafePositiveInteger(opts.bufferSize)) {
-    cb(new error.BackupError('InvalidOption', 'Invalid buffer size option, must be a positive integer in the range (0, MAX_SAFE_INTEGER]'), null);
-    return;
-  }
-  if (opts && typeof opts.iamApiKey !== 'undefined' && typeof opts.iamApiKey !== 'string') {
-    cb(new error.BackupError('InvalidOption', 'Invalid iamApiKey option, must be type string'), null);
-    return;
-  }
-  if (opts && typeof opts.log !== 'undefined' && typeof opts.log !== 'string') {
-    cb(new error.BackupError('InvalidOption', 'Invalid log option, must be type string'), null);
-    return;
-  }
-  if (opts && typeof opts.mode !== 'undefined' && ['full', 'shallow'].indexOf(opts.mode) === -1) {
-    cb(new error.BackupError('InvalidOption', 'Invalid mode option, must be either "full" or "shallow"'), null);
-    return;
-  }
-  if (opts && typeof opts.output !== 'undefined' && typeof opts.output !== 'string') {
-    cb(new error.BackupError('InvalidOption', 'Invalid output option, must be type string'), null);
-    return;
-  }
-  if (opts && typeof opts.parallelism !== 'undefined' && !isSafePositiveInteger(opts.parallelism)) {
-    cb(new error.BackupError('InvalidOption', 'Invalid parallelism option, must be a positive integer in the range (0, MAX_SAFE_INTEGER]'), null);
-    return;
-  }
-  if (opts && typeof opts.requestTimeout !== 'undefined' && !isSafePositiveInteger(opts.requestTimeout)) {
-    cb(new error.BackupError('InvalidOption', 'Invalid request timeout option, must be a positive integer in the range (0, MAX_SAFE_INTEGER]'), null);
-    return;
-  }
-  if (opts && typeof opts.resume !== 'undefined' && typeof opts.resume !== 'boolean') {
-    cb(new error.BackupError('InvalidOption', 'Invalid resume option, must be type boolean'), null);
-    return;
-  }
-
-  // Validate URL and ensure no auth if using key
-  try {
-    const urlObject = new URL(url);
-    // We require a protocol, host and path (for db), fail if any is missing.
-    if (urlObject.protocol !== 'https:' && urlObject.protocol !== 'http:') {
-      cb(new error.BackupError('InvalidOption', 'Invalid URL protocol.'));
-      return;
-    }
-    if (!urlObject.pathname || urlObject.pathname === '/') {
-      cb(new error.BackupError('InvalidOption', 'Invalid URL, missing path element (no database).'));
-      return;
-    }
-    if (opts && opts.iamApiKey && (urlObject.username || urlObject.password)) {
-      cb(new error.BackupError('InvalidOption', 'URL user information must not be supplied when using IAM API key.'));
-      return;
-    }
-  } catch (err) {
-    cb(error.wrapPossibleInvalidUrlError(err));
-    return;
-  }
 
-  // Perform validation of invalid options for shallow mode and WARN
-  // We don't error for backwards compatibility with scripts that may have been
-  // written passing complete sets of options through
-  if (opts && opts.mode === 'shallow') {
-    if (opts.log || opts.resume) {
-      console.warn('WARNING: the options "log" and "resume" are invalid when using shallow mode.');
-    }
-    if (opts.parallelism) {
-      console.warn('WARNING: the option "parallelism" has no effect when using shallow mode.');
-    }
-  }
-
-  if (opts && opts.resume) {
-    if (!opts.log) {
-      // This is the second place we check for the presence of the log option in conjunction with resume
-      // It has to be here for the API case
-      cb(new error.BackupError('NoLogFileName', 'To resume a backup, a log file must be specified'), null);
-      return;
-    } else if (!fs.existsSync(opts.log)) {
-      cb(new error.BackupError('LogDoesNotExist', 'To resume a backup, the log file must exist'), null);
-      return;
-    }
-  }
-  return true;
-}
+// Small script which backs up a Cloudant or CouchDB database to an S3
+// bucket via a stream rather than on-disk file.
+//
+// The script generates the backup object name by combining together the path
+// part of the database URL and the current time.
 
-function addEventListener(indicator, emitter, event, f) {
-  emitter.on(event, function(...args) {
-    if (!indicator.errored) {
-      if (event === 'error') indicator.errored = true;
-      f(...args);
-    }
-  });
-}
+'use strict';
 
-/*
-  Check the backup database exists and that the credentials used have
-  visibility. Callback with a fatal error if there is a problem with the DB.
-  @param {string} db - database object
-  @param {function(err)} callback - error is undefined if DB exists
-*/
-function proceedIfBackupDbValid(db, callback) {
-  db.service.headDatabase({ db: db.db }).then(() => callback()).catch(err => {
-    err = error.convertResponseError(err, err => parseIfDbValidResponseError(db, err));
-    callback(err);
-  });
-}
+const stream = require('stream');
+const url = require('url');
 
-/*
-  Check that the restore database exists, is new and is empty. Also verify that the credentials used have
-  visibility. Callback with a fatal error if there is a problem with the DB.
-  @param {string} db - database object
-  @param {function(err)} callback - error is undefined if DB exists, new and empty
-*/
-function proceedIfRestoreDbValid(db, callback) {
-  db.service.getDatabaseInformation({ db: db.db }).then(response => {
-    const { doc_count: docCount, doc_del_count: deletedDocCount } = response.result;
-    // The system databases can have a validation ddoc(s) injected in them on creation.
-    // This sets the doc count off, so we just complitely exclude the system databases from this check.
-    // The assumption here is that users restoring system databases know what they are doing.
-    if (!db.db.startsWith('_') && (docCount !== 0 || deletedDocCount !== 0)) {
-      const notEmptyDBErr = new Error(`Target database ${db.url}${db.db} is not empty.`);
-      notEmptyDBErr.name = 'DatabaseNotEmpty';
-      callback(notEmptyDBErr);
-    } else {
-      callback();
-    }
-  }).catch(err => {
-    err = error.convertResponseError(err, err => parseIfDbValidResponseError(db, err));
-    callback(err);
-  });
-}
+const AWS = require('aws-sdk');
+const couchbackup = require('@cloudant/couchbackup');
+const debug = require('debug')('s3-backup');
+const VError = require('verror').VError;
 
 /*
-  Convert the database validation response error to a special DatabaseNotFound error
-  in case the database is missing. Otherwise delegate to the default error factory.
-  @param {object} db - database object
-  @param {object} err - HTTP response error
+  Main function, run from base of file.
 */
-function parseIfDbValidResponseError(db, err) {
-  if (err && err.status === 404) {
-    // Override the error type and message for the DB not found case
-    const msg = `Database ${db.url}` +
-    `${db.db} does not exist. ` +
-    'Check the URL and database name have been specified correctly.';
-    const noDBErr = new Error(msg);
-    noDBErr.name = 'DatabaseNotFound';
-    return noDBErr;
+function main() {
+  const argv = require('yargs')
+    .usage('Usage: $0 [options]')
+    .example('$0 -s https://user:pass@host/db -b <bucket>', 'Backup db to bucket')
+    .options({
+      source: { alias: 's', nargs: 1, demandOption: true, describe: 'Source database URL' },
+      bucket: { alias: 'b', nargs: 1, demandOption: true, describe: 'Destination bucket' },
+      prefix: { alias: 'p', nargs: 1, describe: 'Prefix for backup object key', default: 'couchbackup' },
+      s3url: { nargs: 1, describe: 'S3 endpoint URL' },
+      awsprofile: { nargs: 1, describe: 'The profile section to use in the ~/.aws/credentials file', default: 'default' }
+    })
+    .help('h').alias('h', 'help')
+    .epilog('Copyright (C) IBM 2017')
+    .argv;
+
+  const sourceUrl = argv.source;
+  const backupName = new url.URL(sourceUrl).pathname.split('/').filter(function(x) { return x; }).join('-');
+  const backupBucket = argv.bucket;
+  const backupKeyPrefix = `${argv.prefix}-${backupName}`;
+  const shallow = argv.shallow;
+
+  const backupKey = `${backupKeyPrefix}-${new Date().toISOString()}`;
+
+  const s3Endpoint = argv.s3url;
+  const awsProfile = argv.awsprofile;
+
+  // Creds are from ~/.aws/credentials, environment etc. (see S3 docs).
+  const awsOpts = {
+    signatureVersion: 'v4',
+    credentials: new AWS.SharedIniFileCredentials({ profile: awsProfile })
+  };
+  if (typeof s3Endpoint !== 'undefined') {
+    awsOpts.endpoint = new AWS.Endpoint(s3Endpoint);
   }
-  // Delegate to the default error factory if it wasn't a 404
-  return error.convertResponseError(err);
-}
-
-module.exports = {
-
-  /**
-   * Backup a Cloudant database to a stream.
-   *
-   * @param {string} srcUrl - URL of database to backup.
-   * @param {stream.Writable} targetStream - Stream to write content to.
-   * @param {object} opts - Backup options.
-   * @param {number} [opts.parallelism=5] - Number of parallel HTTP requests to use.
-   * @param {number} [opts.bufferSize=500] - Number of documents per batch request.
-   * @param {number} [opts.requestTimeout=120000] - Milliseconds to wait before retrying a HTTP request.
-   * @param {string} [opts.iamApiKey] - IAM API key to use to access Cloudant database.
-   * @param {string} [opts.log] - Log file name. Default uses a temporary file.
-   * @param {boolean} [opts.resume] - Whether to resume from existing log.
-   * @param {string} [opts.mode=full] - Use `full` or `shallow` mode.
-   * @param {backupRestoreCallback} callback - Called on completion.
-   */
-  backup: function(srcUrl, targetStream, opts, callback) {
-    const listenerErrorIndicator = { errored: false };
-    if (typeof callback === 'undefined' && typeof opts === 'function') {
-      callback = opts;
-      opts = {};
-    }
-    if (!validateArgs(srcUrl, opts, callback)) {
-      // bad args, bail
-      return;
-    }
-
-    // if there is an error writing to the stream, call the completion
-    // callback with the error set
-    addEventListener(listenerErrorIndicator, targetStream, 'error', function(err) {
-      debug('Error ' + JSON.stringify(err));
-      if (callback) callback(err);
+  const s3 = new AWS.S3(awsOpts);
+
+  debug(`Creating a new backup of ${s(sourceUrl)} at ${backupBucket}/${backupKey}...`);
+  bucketAccessible(s3, backupBucket)
+    .then(() => {
+      return backupToS3(sourceUrl, s3, backupBucket, backupKey, shallow);
+    })
+    .then(() => {
+      debug('done.');
+    })
+    .catch((reason) => {
+      debug(`Error: ${reason}`);
+      process.exit(1);
     });
+}
 
-    opts = Object.assign({}, defaults(), opts);
-
-    const ee = new events.EventEmitter();
-
-    // Set up the DB client
-    const backupDB = request.client(srcUrl, opts);
-
-    // Validate the DB exists, before proceeding to backup
-    proceedIfBackupDbValid(backupDB, function(err) {
+/**
+ * Return a promise that resolves if the bucket is available and
+ * rejects if not.
+ *
+ * @param {any} s3 S3 client object
+ * @param {any} bucketName Bucket name
+ * @returns Promise
+ */
+function bucketAccessible(s3, bucketName) {
+  return new Promise(function(resolve, reject) {
+    const params = {
+      Bucket: bucketName
+    };
+    s3.headBucket(params, function(err, data) {
       if (err) {
-        if (err.name === 'DatabaseNotFound') {
-          err.message = `${err.message} Ensure the backup source database exists.`;
-        }
-        // Didn't exist, or another fatal error, exit
-        callback(err);
-        return;
-      }
-      let backup = null;
-      if (opts.mode === 'shallow') {
-        backup = backupShallow;
-      } else { // full mode
-        backup = backupFull;
+        reject(new VError(err, 'S3 bucket not accessible'));
+      } else {
+        resolve();
       }
-
-      // If resuming write a newline as it's possible one would be missing from
-      // an interruption of the previous backup. If the backup was clean this
-      // will cause an empty line that will be gracefully handled by the restore.
-      if (opts.resume) {
-        targetStream.write('\n');
-      }
-
-      // Get the event emitter from the backup process so we can handle events
-      // before passing them on to the app's event emitter if needed.
-      const internalEE = backup(backupDB, opts);
-      addEventListener(listenerErrorIndicator, internalEE, 'changes', function(batch) {
-        ee.emit('changes', batch);
-      });
-      addEventListener(listenerErrorIndicator, internalEE, 'received', function(obj, q, logCompletedBatch) {
-        // this may be too verbose to have as well as the "backed up" message
-        // debug(' received batch', obj.batch, ' docs: ', obj.total, 'Time', obj.time);
-        // Callback to emit the written event when the content is flushed
-        function writeFlushed() {
-          ee.emit('written', { total: obj.total, time: obj.time, batch: obj.batch });
-          if (logCompletedBatch) {
-            logCompletedBatch(obj.batch);
-          }
-          debug(' backed up batch', obj.batch, ' docs: ', obj.total, 'Time', obj.time);
-        }
-        // Write the received content to the targetStream
-        const continueWriting = targetStream.write(JSON.stringify(obj.data) + '\n',
-          'utf8',
-          writeFlushed);
-        if (!continueWriting) {
-          // The buffer was full, pause the queue to stop the writes until we
-          // get a drain event
-          if (q && !q.paused) {
-            q.pause();
-            targetStream.once('drain', function() {
-              q.resume();
-            });
-          }
-        }
-      });
-      // For errors we expect, may or may not be fatal
-      addEventListener(listenerErrorIndicator, internalEE, 'error', function(err) {
-        debug('Error ' + JSON.stringify(err));
-        callback(err);
-      });
-      addEventListener(listenerErrorIndicator, internalEE, 'finished', function(obj) {
-        function emitFinished() {
-          debug('Backup complete - written ' + JSON.stringify(obj));
-          const summary = { total: obj.total };
-          ee.emit('finished', summary);
-          if (callback) callback(null, summary);
-        }
-        if (targetStream === process.stdout) {
-          // stdout cannot emit a finish event so use a final write + callback
-          targetStream.write('', 'utf8', emitFinished);
-        } else {
-          // If we're writing to a file, end the writes and register the
-          // emitFinished function for a callback when the file stream's finish
-          // event is emitted.
-          targetStream.end('', 'utf8', emitFinished);
-        }
-      });
     });
-    return ee;
-  },
-
-  /**
-   * Restore a backup from a stream.
-   *
-   * @param {stream.Readable} srcStream - Stream containing backed up data.
-   * @param {string} targetUrl - Target database.
-   * @param {object} opts - Restore options.
-   * @param {number} opts.parallelism - Number of parallel HTTP requests to use. Default 5.
-   * @param {number} opts.bufferSize - Number of documents per batch request. Default 500.
-   * @param {number} opts.requestTimeout - Milliseconds to wait before retrying a HTTP request. Default 120000.
-   * @param {string} opts.iamApiKey - IAM API key to use to access Cloudant database.
-   * @param {backupRestoreCallback} callback - Called on completion.
-   */
-  restore: function(srcStream, targetUrl, opts, callback) {
-    const listenerErrorIndicator = { errored: false };
-    if (typeof callback === 'undefined' && typeof opts === 'function') {
-      callback = opts;
-      opts = {};
-    }
-    validateArgs(targetUrl, opts, callback);
-    opts = Object.assign({}, defaults(), opts);
-
-    const ee = new events.EventEmitter();
-
-    // Set up the DB client
-    const restoreDB = request.client(targetUrl, opts);
+  });
+}
 
-    // Validate the DB exists, before proceeding to restore
-    proceedIfRestoreDbValid(restoreDB, function(err) {
+/**
+ * Backup directly from Cloudant to an object store object via a stream.
+ *
+ * @param {any} sourceUrl URL of database
+ * @param {any} s3Client Object store client
+ * @param {any} s3Bucket Backup destination bucket
+ * @param {any} s3Key Backup destination key name (shouldn't exist)
+ * @param {any} shallow Whether to use the couchbackup `shallow` mode
+ * @returns Promise
+ */
+function backupToS3(sourceUrl, s3Client, s3Bucket, s3Key, shallow) {
+  return new Promise((resolve, reject) => {
+    debug(`Setting up S3 upload to ${s3Bucket}/${s3Key}`);
+
+    // A pass through stream that has couchbackup's output
+    // written to it and it then read by the S3 upload client.
+    // It has a 64MB highwater mark to allow for fairly
+    // uneven network connectivity.
+    const streamToUpload = new stream.PassThrough({ highWaterMark: 67108864 });
+
+    // Set up S3 upload.
+    const params = {
+      Bucket: s3Bucket,
+      Key: s3Key,
+      Body: streamToUpload
+    };
+    s3Client.upload(params, function(err, data) {
+      debug('Object store upload done');
       if (err) {
-        if (err.name === 'DatabaseNotFound') {
-          err.message = `${err.message} Create the target database before restoring.`;
-        } else if (err.name === 'DatabaseNotEmpty') {
-          err.message = `${err.message} A target database must be a new and empty database.`;
-        }
-        // Didn't exist, or another fatal error, exit
-        callback(err);
+        debug(err);
+        reject(new VError(err, 'Object store upload failed'));
         return;
       }
-
-      restoreInternal(
-        restoreDB,
-        opts,
-        srcStream,
-        ee,
-        function(err, writer) {
-          if (err) {
-            callback(err, null);
-            return;
-          }
-          if (writer != null) {
-            addEventListener(listenerErrorIndicator, writer, 'restored', function(obj) {
-              debug(' restored ', obj.total);
-              ee.emit('restored', { documents: obj.documents, total: obj.total });
-            });
-            addEventListener(listenerErrorIndicator, writer, 'error', function(err) {
-              debug('Error ' + JSON.stringify(err));
-              // Only call destroy if it is available on the stream
-              if (srcStream.destroy && srcStream.destroy instanceof Function) {
-                srcStream.destroy();
-              }
-              callback(err);
-            });
-            addEventListener(listenerErrorIndicator, writer, 'finished', function(obj) {
-              debug('restore complete');
-              ee.emit('finished', { total: obj.total });
-              callback(null, obj);
-            });
-          }
+      debug('Object store upload succeeded');
+      debug(data);
+      resolve();
+    }).httpUploadProgress = (progress) => {
+      debug(`Object store upload progress: ${progress}`);
+    };
+
+    debug(`Starting streaming data from ${s(sourceUrl)}`);
+    couchbackup.backup(
+      sourceUrl,
+      streamToUpload,
+      (err, obj) => {
+        if (err) {
+          debug(err);
+          reject(new VError(err, 'CouchBackup failed with an error'));
+          return;
         }
-      );
-    });
-    return ee;
-  }
-};
+        debug(`Download from ${s(sourceUrl)} complete.`);
+        streamToUpload.end(); // must call end() to complete upload.
+        // resolve() is called by the upload
+      }
+    );
+  });
+}
 
 /**
- * Backup/restore callback
- * @callback backupRestoreCallback
- * @param {Error} err - Error object if operation failed.
- * @param {object} data - summary data for backup/restore
+ * Remove creds from a URL, e.g., before logging
+ *
+ * @param {string} url URL to safen
  */
+function s(originalUrl) {
+  const parts = new url.URL(originalUrl);
+  return url.format(parts, { auth: false });
+}
+
+main();