@cloudant/couchbackup 2.9.12 → 2.9.13-SNAPSHOT.139

package/.scannerwork/scanner-report/source-54.txt

@@ -1,4 +1,4 @@
-// Copyright © 2017 IBM Corp. All rights reserved.
+// Copyright © 2017, 2021 IBM Corp. All rights reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -11,19 +11,119 @@
 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 // See the License for the specific language governing permissions and
 // limitations under the License.
-
-/* global describe it */
 'use strict';
 
-const u = require('./citestutils.js');
-
-[{ useApi: true }, { useApi: false }].forEach(function(params) {
-  describe(u.scenario('#slowest End to end backup and restore', params), function() {
-    // 10 GB is about the largest the CI can handle before getting very upset
-    // about how long things are taking
-    it('should backup and restore largedb10g', function(done) {
-      u.setTimeout(this, 350 * 60);
-      u.testDirectBackupAndRestore(params, 'largedb10g', this.dbName, done);
-    });
-  });
-});
+const cliutils = require('./cliutils.js');
+const config = require('./config.js');
+const error = require('./error.js');
+const path = require('path');
+const pkg = require('../package.json');
+
+function parseBackupArgs() {
+  const program = require('commander');
+
+  // Option CLI defaults
+  const defaults = config.cliDefaults();
+
+  // Options set by environment variables
+  const envVarOptions = {};
+  config.applyEnvironmentVariables(envVarOptions);
+
+  program
+    .version(pkg.version)
+    .description('Backup a CouchDB/Cloudant database to a backup text file.')
+    .usage('[options...]')
+    .option('-b, --buffer-size <n>',
+      cliutils.getUsage('number of documents fetched at once', defaults.bufferSize),
+      Number)
+    .option('-d, --db <db>',
+      cliutils.getUsage('name of the database to backup', defaults.db))
+    .option('-k, --iam-api-key <API key>',
+      cliutils.getUsage('IAM API key to access the Cloudant server'))
+    .option('-l, --log <file>',
+      cliutils.getUsage('file to store logging information during backup; invalid in "shallow" mode', 'a temporary file'),
+      path.normalize)
+    .option('-m, --mode <mode>',
+      cliutils.getUsage('"shallow" if only a superficial backup is done (ignoring conflicts and revision tokens), else "full" for complete backup', defaults.mode),
+      (mode) => { return mode.toLowerCase(); })
+    .option('-o, --output <file>',
+      cliutils.getUsage('file name to store the backup data', 'stdout'),
+      path.normalize)
+    .option('-p, --parallelism <n>',
+      cliutils.getUsage('number of HTTP requests to perform in parallel when performing a backup; ignored in "shallow" mode', defaults.parallelism),
+      Number)
+    .option('-q, --quiet',
+      cliutils.getUsage('suppress batch messages', defaults.quiet))
+    .option('-r, --resume',
+      cliutils.getUsage('continue a previous backup from its last known position; invalid in "shallow" mode', defaults.resume))
+    .option('-t, --request-timeout <n>',
+      cliutils.getUsage('milliseconds to wait for a response to a HTTP request before retrying the request', defaults.requestTimeout),
+      Number)
+    .option('-u, --url <url>',
+      cliutils.getUsage('URL of the CouchDB/Cloudant server', defaults.url))
+    .parse(process.argv);
+
+  // Remove defaults that don't apply when using shallow mode
+  if (program.opts().mode === 'shallow' || envVarOptions.mode === 'shallow') {
+    delete defaults.parallelism;
+    delete defaults.log;
+    delete defaults.resume;
+  }
+
+  // Apply the options in order so that the CLI overrides env vars and env variables
+  // override defaults.
+  const opts = Object.assign({}, defaults, envVarOptions, program.opts());
+
+  if (opts.resume && (opts.log === defaults.log)) {
+    // If resuming and the log file arg is the newly generated tmp name from defaults then we know that --log wasn't specified.
+    // We have to do this check here for the CLI case because of the default.
+    error.terminationCallback(new error.BackupError('NoLogFileName', 'To resume a backup, a log file must be specified'));
+  }
+
+  return opts;
+}
+
+function parseRestoreArgs() {
+  const program = require('commander');
+
+  // Option CLI defaults
+  const defaults = config.cliDefaults();
+
+  // Options set by environment variables
+  const envVarOptions = {};
+  config.applyEnvironmentVariables(envVarOptions);
+
+  program
+    .version(pkg.version)
+    .description('Restore a CouchDB/Cloudant database from a backup text file.')
+    .usage('[options...]')
+    .option('-b, --buffer-size <n>',
+      cliutils.getUsage('number of documents restored at once', defaults.bufferSize),
+      Number)
+    .option('-d, --db <db>',
+      cliutils.getUsage('name of the new, existing database to restore to', defaults.db))
+    .option('-k, --iam-api-key <API key>',
+      cliutils.getUsage('IAM API key to access the Cloudant server'))
+    .option('-p, --parallelism <n>',
+      cliutils.getUsage('number of HTTP requests to perform in parallel when restoring a backup', defaults.parallelism),
+      Number)
+    .option('-q, --quiet',
+      cliutils.getUsage('suppress batch messages', defaults.quiet))
+    .option('-t, --request-timeout <n>',
+      cliutils.getUsage('milliseconds to wait for a response to a HTTP request before retrying the request', defaults.requestTimeout),
+      Number)
+    .option('-u, --url <url>',
+      cliutils.getUsage('URL of the CouchDB/Cloudant server', defaults.url))
+    .parse(process.argv);
+
+  // Apply the options in order so that the CLI overrides env vars and env variables
+  // override defaults.
+  const opts = Object.assign({}, defaults, envVarOptions, program.opts());
+
+  return opts;
+}
+
+module.exports = {
+  parseBackupArgs: parseBackupArgs,
+  parseRestoreArgs: parseRestoreArgs
+};

package/.scannerwork/scanner-report/source-55.txt

@@ -1,4 +1,4 @@
-// Copyright © 2017, 2018 IBM Corp. All rights reserved.
+// Copyright © 2017, 2022 IBM Corp. All rights reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -11,215 +11,101 @@
 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 // See the License for the specific language governing permissions and
 // limitations under the License.
-
-/* global describe it */
 'use strict';
 
-const assert = require('assert');
-const backup = require('../app.js').backup;
 const fs = require('fs');
-const nock = require('nock');
-
-const goodUrl = 'http://localhost:5984/db';
-// The real validateArgs function of app.js isn't
-// exported - so we call the exported backup method
-// instead. We don't get as far as a real backup when
-// testing error cases. For success cases we nock the
-// goodUrl and
-const validateArgs = function(url, opts, callback) {
-  const nullStream = fs.createWriteStream('/dev/null');
-  let cb = callback;
-  if (url === goodUrl) {
-    // Nock the goodUrl
-    nock(goodUrl).head('').reply(404, { error: 'not_found', reason: 'missing' });
-    // replace the callback to handle the nock response
-    // to avoid attempting a real backup
-    cb = function(err) {
-      nullStream.end();
-      if (err.name === 'DatabaseNotFound') {
-        // This is what we expect if we reached the backup
-        // This is success for valid args cases.
-        err = null;
-      }
-      callback(err);
-    };
-  }
-  backup(url, nullStream, opts, cb);
-  return true;
-};
+const liner = require('./liner.js');
+const change = require('./change.js');
+const error = require('./error.js');
+const debug = require('debug')('couchbackup:spoolchanges');
 
-const stderrWriteFun = process.stderr.write;
-let capturedStderr;
+/**
+ * Write log file for all changes from a database, ready for downloading
+ * in batches.
+ *
+ * @param {string} dbUrl - URL of database
+ * @param {string} log - path to log file to use
+ * @param {number} bufferSize - the number of changes per batch/log line
+ * @param {function(err)} callback - a callback to run on completion
+ */
+module.exports = function(db, log, bufferSize, ee, callback) {
+  // list of document ids to process
+  const buffer = [];
+  let batch = 0;
+  let lastSeq = null;
+  const logStream = fs.createWriteStream(log);
+  let pending = 0;
+  // The number of changes to fetch per request
+  const limit = 100000;
 
-function captureStderr() {
-  process.stderr.write = function(string, encoding, fd) {
-    capturedStderr += string;
-  };
-}
-
-function releaseStderr() {
-  process.stderr.write = stderrWriteFun;
-  capturedStderr = null;
-}
-
-function assertErrorMessage(msg, done) {
-  return function(err, data) {
-    try {
-      assert(err.message, 'There should be an error message');
-      assert(err.message.indexOf(msg) >= 0);
-      assert(data === null || data === undefined, 'There should only be an error.');
-      done();
-    } catch (e) {
-      done(e);
+  // send documents ids to the queue in batches of bufferSize + the last batch
+  const processBuffer = function(lastOne) {
+    if (buffer.length >= bufferSize || (lastOne && buffer.length > 0)) {
+      debug('writing', buffer.length, 'changes to the backup file');
+      const b = { docs: buffer.splice(0, bufferSize), batch: batch };
+      logStream.write(':t batch' + batch + ' ' + JSON.stringify(b.docs) + '\n');
+      ee.emit('changes', batch);
+      batch++;
     }
   };
-}
 
-function assertNoError(done) {
-  return function(err, data) {
-    try {
-      assert(err === null, 'There should be no error message.');
-      done();
-    } catch (e) {
-      done(e);
+  // called once per received change
+  const onChange = function(c) {
+    if (c) {
+      if (c.error) {
+        ee.emit('error', new error.BackupError('InvalidChange', `Received invalid change: ${c}`));
+      } else if (c.changes) {
+        const obj = { id: c.id };
+        buffer.push(obj);
+        processBuffer(false);
+      } else if (c.last_seq) {
+        lastSeq = c.last_seq;
+        pending = c.pending;
+      }
     }
   };
-}
 
-describe('#unit Validate arguments', function() {
-  it('returns error for invalid URL type', function(done) {
-    validateArgs(true, {}, assertErrorMessage('Invalid URL, must be type string', done));
-  });
-  it('returns no error for valid URL type', function(done) {
-    validateArgs(goodUrl, {}, assertNoError(done));
-  });
-  it('returns error for invalid (no host) URL', function(done) {
-    validateArgs('http://', {}, assertErrorMessage('Invalid URL', done));
-  });
-  it('returns error for invalid (no protocol) URL', function(done) {
-    validateArgs('invalid', {}, assertErrorMessage('Invalid URL', done));
-  });
-  it('returns error for invalid (wrong protocol) URL', function(done) {
-    validateArgs('ftp://invalid.example.com', {}, assertErrorMessage('Invalid URL protocol.', done));
-  });
-  it('returns error for invalid (no path) URL', function(done) {
-    validateArgs('https://invalid.example.com', {}, assertErrorMessage('Invalid URL, missing path element (no database).', done));
-  });
-  it('returns error for invalid (no protocol, no host) URL', function(done) {
-    validateArgs('invalid', {}, assertErrorMessage('Invalid URL', done));
-  });
-  it('returns error for invalid buffer size type', function(done) {
-    validateArgs(goodUrl, { bufferSize: '123' }, assertErrorMessage('Invalid buffer size option, must be a positive integer in the range (0, MAX_SAFE_INTEGER]', done));
-  });
-  it('returns error for zero buffer size', function(done) {
-    validateArgs(goodUrl, { bufferSize: 0 }, assertErrorMessage('Invalid buffer size option, must be a positive integer in the range (0, MAX_SAFE_INTEGER]', done));
-  });
-  it('returns error for float buffer size', function(done) {
-    validateArgs(goodUrl, { bufferSize: 1.23 }, assertErrorMessage('Invalid buffer size option, must be a positive integer in the range (0, MAX_SAFE_INTEGER]', done));
-  });
-  it('returns no error for valid buffer size type', function(done) {
-    validateArgs(goodUrl, { bufferSize: 123 }, assertNoError(done));
-  });
-  it('returns error for invalid log type', function(done) {
-    validateArgs(goodUrl, { log: true }, assertErrorMessage('Invalid log option, must be type string', done));
-  });
-  it('returns no error for valid log type', function(done) {
-    validateArgs(goodUrl, { log: 'log.txt' }, assertNoError(done));
-  });
-  it('returns error for invalid mode type', function(done) {
-    validateArgs(goodUrl, { mode: true }, assertErrorMessage('Invalid mode option, must be either "full" or "shallow"', done));
-  });
-  it('returns error for invalid mode string', function(done) {
-    validateArgs(goodUrl, { mode: 'foobar' }, assertErrorMessage('Invalid mode option, must be either "full" or "shallow"', done));
-  });
-  it('returns no error for valid mode type', function(done) {
-    validateArgs(goodUrl, { mode: 'full' }, assertNoError(done));
-  });
-  it('returns error for invalid output type', function(done) {
-    validateArgs(goodUrl, { output: true }, assertErrorMessage('Invalid output option, must be type string', done));
-  });
-  it('returns no error for valid output type', function(done) {
-    validateArgs(goodUrl, { output: 'output.txt' }, assertNoError(done));
-  });
-  it('returns error for invalid parallelism type', function(done) {
-    validateArgs(goodUrl, { parallelism: '123' }, assertErrorMessage('Invalid parallelism option, must be a positive integer in the range (0, MAX_SAFE_INTEGER]', done));
-  });
-  it('returns error for zero parallelism', function(done) {
-    validateArgs(goodUrl, { parallelism: 0 }, assertErrorMessage('Invalid parallelism option, must be a positive integer in the range (0, MAX_SAFE_INTEGER]', done));
-  });
-  it('returns error for float parallelism', function(done) {
-    validateArgs(goodUrl, { parallelism: 1.23 }, assertErrorMessage('Invalid parallelism option, must be a positive integer in the range (0, MAX_SAFE_INTEGER]', done));
-  });
-  it('returns no error for valid parallelism type', function(done) {
-    validateArgs(goodUrl, { parallelism: 123 }, assertNoError(done));
-  });
-  it('returns error for invalid request timeout type', function(done) {
-    validateArgs(goodUrl, { requestTimeout: '123' }, assertErrorMessage('Invalid request timeout option, must be a positive integer in the range (0, MAX_SAFE_INTEGER]', done));
-  });
-  it('returns error for zero request timeout', function(done) {
-    validateArgs(goodUrl, { requestTimeout: 0 }, assertErrorMessage('Invalid request timeout option, must be a positive integer in the range (0, MAX_SAFE_INTEGER]', done));
-  });
-  it('returns error for float request timout', function(done) {
-    validateArgs(goodUrl, { requestTimeout: 1.23 }, assertErrorMessage('Invalid request timeout option, must be a positive integer in the range (0, MAX_SAFE_INTEGER]', done));
-  });
-  it('returns no error for valid request timeout type', function(done) {
-    validateArgs(goodUrl, { requestTimeout: 123 }, assertNoError(done));
-  });
-  it('returns error for invalid resume type', function(done) {
-    validateArgs(goodUrl, { resume: 'true' }, assertErrorMessage('Invalid resume option, must be type boolean', done));
-  });
-  it('returns no error for valid resume type', function(done) {
-    validateArgs(goodUrl, { resume: false }, assertNoError(done));
-  });
-  it('returns error for invalid key type', function(done) {
-    validateArgs(goodUrl, { iamApiKey: true }, assertErrorMessage('Invalid iamApiKey option, must be type string', done));
-  });
-  it('returns error for key and URL credentials supplied', function(done) {
-    validateArgs('https://a:b@example.com/db', { iamApiKey: 'abc123' }, assertErrorMessage('URL user information must not be supplied when using IAM API key.', done));
-  });
-  it('warns for log arg in shallow mode', function(done) {
-    captureStderr();
-    try {
-      validateArgs(goodUrl, { mode: 'shallow', log: 'test' }, function(err, data) {
-        assert.ok(err);
-        assert.ok(!data);
-        assert(capturedStderr.indexOf('The options "log" and "resume" are invalid when using shallow mode.') > -1, 'Log warning message was not present');
+  function getChanges(since = 0) {
+    debug('making changes request since ' + since);
+    return db.service.postChangesAsStream({ db: db.db, since: since, limit: limit, seqInterval: limit })
+      .then(response => {
+        response.result.pipe(liner())
+          .on('error', function(err) {
+            logStream.end();
+            callback(err);
+          })
+          .pipe(change(onChange))
+          .on('error', function(err) {
+            logStream.end();
+            callback(err);
+          })
+          .on('finish', function() {
+            processBuffer(true);
+            if (!lastSeq) {
+              logStream.end();
+              debug('changes request terminated before last_seq was sent');
+              callback(new error.BackupError('SpoolChangesError', 'Changes request terminated before last_seq was sent'));
+            } else {
+              debug(`changes request completed with last_seq: ${lastSeq} and ${pending} changes pending.`);
+              if (pending > 0) {
+                // Return the next promise
+                return getChanges(lastSeq);
+              } else {
+                debug('finished streaming database changes');
+                logStream.end(':changes_complete ' + lastSeq + '\n', 'utf8', callback);
+              }
+            }
+          });
+      })
+      .catch(err => {
+        logStream.end();
+        if (err.status && err.status >= 400) {
+          callback(error.convertResponseError(err));
+        } else if (err.name !== 'SpoolChangesError') {
+          callback(new error.BackupError('SpoolChangesError', `Failed changes request - ${err.message}`));
+        }
       });
-      done();
-    } catch (e) {
-      done(e);
-    } finally {
-      releaseStderr();
-    }
-  });
-  it('warns for resume arg in shallow mode', function(done) {
-    captureStderr();
-    try {
-      validateArgs(goodUrl, { mode: 'shallow', log: 'test', resume: true }, function(err, data) {
-        assert.ok(err);
-        assert.ok(!data);
-        assert(capturedStderr.indexOf('The options "log" and "resume" are invalid when using shallow mode.') > -1, 'Log warning message was not present');
-      });
-      done();
-    } catch (e) {
-      done(e);
-    } finally {
-      releaseStderr();
-    }
-  });
-  it('warns for parallism arg in shallow mode', function(done) {
-    captureStderr();
-    try {
-      validateArgs(goodUrl, { mode: 'shallow', parallelsim: 10 }, function(err, data) {
-        assert.ok(err);
-        assert.ok(!data);
-        assert(capturedStderr.indexOf('The option "parallelism" has no effect when using shallow mode.') > -1, 'Log warning message was not present');
-      });
-      done();
-    } catch (e) {
-      done(e);
-    } finally {
-      releaseStderr();
-    }
-  });
-});
+  }
+
+  getChanges();
+};