@cloudant/couchbackup 2.9.14 → 2.9.15-SNAPSHOT.162

package/.scannerwork/scanner-report/source-6.txt

@@ -1,60 +1,102 @@
-#!/usr/bin/env node
-// Copyright © 2017, 2021 IBM Corp. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-'use strict';
-
-const error = require('../includes/error.js');
-const cliutils = require('../includes/cliutils.js');
-const couchbackup = require('../app.js');
-const parser = require('../includes/parser.js');
-const debug = require('debug');
-const restoreDebug = debug('couchbackup:restore');
-const restoreBatchDebug = debug('couchbackup:restore:batch');
-
-restoreDebug.enabled = true;
-
-try {
-  const program = parser.parseRestoreArgs();
-  const databaseUrl = cliutils.databaseUrl(program.url, program.db);
-  const opts = {
-    bufferSize: program.bufferSize,
-    parallelism: program.parallelism,
-    requestTimeout: program.requestTimeout,
-    iamApiKey: program.iamApiKey,
-    iamTokenUrl: program.iamTokenUrl
-  };
-
-  // log configuration to console
-  console.error('='.repeat(80));
-  console.error('Performing restore on ' + databaseUrl.replace(/\/\/.+@/g, '//****:****@') + ' using configuration:');
-  console.error(JSON.stringify(opts, null, 2).replace(/"iamApiKey": "[^"]+"/, '"iamApiKey": "****"'));
-  console.error('='.repeat(80));
-
-  restoreBatchDebug.enabled = !program.quiet;
-
-  return couchbackup.restore(
-    process.stdin, // restore from stdin
-    databaseUrl,
-    opts,
-    error.terminationCallback
-  ).on('restored', function(obj) {
-    restoreBatchDebug('restored', obj.total);
-  }).on('error', function(e) {
-    restoreDebug('ERROR', e);
-  }).on('finished', function(obj) {
-    restoreDebug('finished', obj);
-  });
-} catch (err) {
-  error.terminationCallback(err);
-}
+<testsuites name="test-iam">
+  <testsuite name="Basic backup and restore using API" tests="3" errors="0" failures="0" skipped="0" timestamp="2023-11-02T17:46:06" time="4.942">
+    <testcase classname="test-iam.Basic backup and restore using API" name="should backup animaldb to a file correctly" time="1.05">
+    </testcase>
+    <testcase classname="test-iam.Basic backup and restore using API" name="should restore animaldb to a database correctly" time="1.87">
+    </testcase>
+    <testcase classname="test-iam.Basic backup and restore using API" name="should execute a shallow mode backup successfully" time="0.825">
+    </testcase>
+  </testsuite>
+  <testsuite name="Basic backup and restore using API Buffer size tests using API" tests="3" errors="0" failures="0" skipped="0" timestamp="2023-11-02T17:46:11" time="11.399">
+    <testcase classname="test-iam.Basic backup and restore using API Buffer size tests using API" name="should backup/restore animaldb with the same buffer size" time="2.655">
+    </testcase>
+    <testcase classname="test-iam.Basic backup and restore using API Buffer size tests using API" name="should backup/restore animaldb with backup buffer &gt; restore buffer" time="4.442">
+    </testcase>
+    <testcase classname="test-iam.Basic backup and restore using API Buffer size tests using API" name="should backup/restore animaldb with backup buffer &lt; restore buffer" time="3.451">
+    </testcase>
+  </testsuite>
+  <testsuite name="Basic backup and restore using CLI" tests="3" errors="0" failures="0" skipped="0" timestamp="2023-11-02T17:46:22" time="5.587">
+    <testcase classname="test-iam.Basic backup and restore using CLI" name="should backup animaldb to a file correctly" time="1.361">
+    </testcase>
+    <testcase classname="test-iam.Basic backup and restore using CLI" name="should restore animaldb to a database correctly" time="2.334">
+    </testcase>
+    <testcase classname="test-iam.Basic backup and restore using CLI" name="should execute a shallow mode backup successfully" time="1.069">
+    </testcase>
+  </testsuite>
+  <testsuite name="Basic backup and restore using CLI Buffer size tests using CLI" tests="3" errors="0" failures="0" skipped="0" timestamp="2023-11-02T17:46:28" time="13.096">
+    <testcase classname="test-iam.Basic backup and restore using CLI Buffer size tests using CLI" name="should backup/restore animaldb with the same buffer size" time="3.363">
+    </testcase>
+    <testcase classname="test-iam.Basic backup and restore using CLI Buffer size tests using CLI" name="should backup/restore animaldb with backup buffer &gt; restore buffer" time="4.437">
+    </testcase>
+    <testcase classname="test-iam.Basic backup and restore using CLI Buffer size tests using CLI" name="should backup/restore animaldb with backup buffer &lt; restore buffer" time="4.474">
+    </testcase>
+  </testsuite>
+  <testsuite name="Compression tests using API" tests="3" errors="0" failures="0" skipped="0" timestamp="2023-11-02T17:46:41" time="5.622">
+    <testcase classname="test-iam.Compression tests using API" name="should backup animaldb to a compressed file" time="0.972">
+    </testcase>
+    <testcase classname="test-iam.Compression tests using API" name="should backup and restore animaldb via a compressed file" time="1.629">
+    </testcase>
+    <testcase classname="test-iam.Compression tests using API" name="should backup and restore animaldb via a compressed stream" time="2.202">
+    </testcase>
+  </testsuite>
+  <testsuite name="Compression tests using CLI" tests="3" errors="0" failures="0" skipped="0" timestamp="2023-11-02T17:46:46" time="7.822">
+    <testcase classname="test-iam.Compression tests using CLI" name="should backup animaldb to a compressed file" time="1.339">
+    </testcase>
+    <testcase classname="test-iam.Compression tests using CLI" name="should backup and restore animaldb via a compressed file" time="2.913">
+    </testcase>
+    <testcase classname="test-iam.Compression tests using CLI" name="should backup and restore animaldb via a compressed stream" time="2.738">
+    </testcase>
+  </testsuite>
+  <testsuite name="End to end backup and restore using API" tests="2" errors="0" failures="0" skipped="0" timestamp="2023-11-02T17:46:54" time="333.956">
+    <testcase classname="test-iam.End to end backup and restore using API" name="should backup and restore animaldb" time="2.123">
+    </testcase>
+    <testcase classname="test-iam.End to end backup and restore using API" name="should backup and restore largedb1g #slow" time="331.034">
+    </testcase>
+  </testsuite>
+  <testsuite name="End to end backup and restore using CLI" tests="2" errors="0" failures="0" skipped="0" timestamp="2023-11-02T17:52:28" time="516.382">
+    <testcase classname="test-iam.End to end backup and restore using CLI" name="should backup and restore animaldb" time="2.893">
+    </testcase>
+    <testcase classname="test-iam.End to end backup and restore using CLI" name="should backup and restore largedb1g #slow" time="512.703">
+    </testcase>
+  </testsuite>
+  <testsuite name="Encryption tests" tests="1" errors="0" failures="0" skipped="0" timestamp="2023-11-02T18:01:05" time="2.849">
+    <testcase classname="test-iam.Encryption tests" name="should backup and restore animaldb via an encrypted file" time="2.582">
+    </testcase>
+  </testsuite>
+  <testsuite name="Write error tests" tests="1" errors="0" failures="0" skipped="0" timestamp="2023-11-02T18:01:07" time="0.29">
+    <testcase classname="test-iam.Write error tests" name="calls callback with error set when stream is not writeable" time="0.017">
+    </testcase>
+  </testsuite>
+  <testsuite name="Event tests" tests="2" errors="0" failures="0" skipped="0" timestamp="2023-11-02T18:01:08" time="2.336">
+    <testcase classname="test-iam.Event tests" name="should get a finished event when using stdout" time="0.827">
+    </testcase>
+    <testcase classname="test-iam.Event tests" name="should get a finished event when using file output" time="0.973">
+    </testcase>
+  </testsuite>
+  <testsuite name="Resume tests using API" tests="3" errors="0" failures="0" skipped="0" timestamp="2023-11-02T18:01:10" time="5.363">
+    <testcase classname="test-iam.Resume tests using API" name="should create a log file" time="0.859">
+    </testcase>
+    <testcase classname="test-iam.Resume tests using API" name="should restore corrupted animaldb to a database correctly" time="1.823">
+    </testcase>
+    <testcase classname="test-iam.Resume tests using API" name="should restore resumed animaldb with blank line to a database correctly" time="1.875">
+    </testcase>
+  </testsuite>
+  <testsuite name="Resume tests using CLI" tests="3" errors="0" failures="0" skipped="0" timestamp="2023-11-02T18:01:15" time="6.592">
+    <testcase classname="test-iam.Resume tests using CLI" name="should create a log file" time="1.338">
+    </testcase>
+    <testcase classname="test-iam.Resume tests using CLI" name="should restore corrupted animaldb to a database correctly" time="2.23">
+    </testcase>
+    <testcase classname="test-iam.Resume tests using CLI" name="should restore resumed animaldb with blank line to a database correctly" time="2.212">
+    </testcase>
+  </testsuite>
+  <testsuite name="Resume tests" tests="2" errors="0" failures="0" skipped="0" timestamp="2023-11-02T18:01:22" time="33.862">
+    <testcase classname="test-iam.Resume tests" name="should correctly backup and restore backup10m" time="16.648">
+    </testcase>
+    <testcase classname="test-iam.Resume tests" name="should correctly backup and restore backup10m using --output" time="16.682">
+    </testcase>
+  </testsuite>
+  <testsuite name="Longer spool changes checks" tests="1" errors="0" failures="0" skipped="0" timestamp="2023-11-02T18:01:56" time="48.25">
+    <testcase classname="test-iam.Longer spool changes checks" name="#slow should keep collecting changes (25M)" time="47.981">
+    </testcase>
+  </testsuite>
+</testsuites>

package/.scannerwork/scanner-report/source-60.txt

@@ -13,106 +13,117 @@
 // limitations under the License.
 'use strict';
 
+const cliutils = require('./cliutils.js');
+const config = require('./config.js');
+const error = require('./error.js');
 const path = require('path');
-const tmp = require('tmp');
-
-/**
-  Return API default settings.
-*/
-function apiDefaults() {
-  return {
-    parallelism: 5,
-    bufferSize: 500,
-    requestTimeout: 120000,
-    log: tmp.tmpNameSync(),
-    resume: false,
-    mode: 'full'
-  };
-}
-
-/**
-  Return CLI default settings.
-*/
-function cliDefaults() {
-  const defaults = apiDefaults();
-
-  // add additional legacy settings
-  defaults.db = 'test';
-  defaults.url = 'http://localhost:5984';
-
-  // add CLI only option
-  defaults.quiet = false;
-
-  return defaults;
-}
-
-/**
-  Override settings **in-place** with environment variables.
-*/
-function applyEnvironmentVariables(opts) {
-  // if we have a custom CouchDB url
-  if (typeof process.env.COUCH_URL !== 'undefined') {
-    opts.url = process.env.COUCH_URL;
+const pkg = require('../package.json');
+
+function parseBackupArgs() {
+  const program = require('commander');
+
+  // Option CLI defaults
+  const defaults = config.cliDefaults();
+
+  // Options set by environment variables
+  const envVarOptions = {};
+  config.applyEnvironmentVariables(envVarOptions);
+
+  program
+    .version(pkg.version)
+    .description('Backup a CouchDB/Cloudant database to a backup text file.')
+    .usage('[options...]')
+    .option('-b, --buffer-size <n>',
+      cliutils.getUsage('number of documents fetched at once', defaults.bufferSize),
+      Number)
+    .option('-d, --db <db>',
+      cliutils.getUsage('name of the database to backup', defaults.db))
+    .option('-k, --iam-api-key <API key>',
+      cliutils.getUsage('IAM API key to access the Cloudant server'))
+    .option('-l, --log <file>',
+      cliutils.getUsage('file to store logging information during backup; invalid in "shallow" mode', 'a temporary file'),
+      path.normalize)
+    .option('-m, --mode <mode>',
+      cliutils.getUsage('"shallow" if only a superficial backup is done (ignoring conflicts and revision tokens), else "full" for complete backup', defaults.mode),
+      (mode) => { return mode.toLowerCase(); })
+    .option('-o, --output <file>',
+      cliutils.getUsage('file name to store the backup data', 'stdout'),
+      path.normalize)
+    .option('-p, --parallelism <n>',
+      cliutils.getUsage('number of HTTP requests to perform in parallel when performing a backup; ignored in "shallow" mode', defaults.parallelism),
+      Number)
+    .option('-q, --quiet',
+      cliutils.getUsage('suppress batch messages', defaults.quiet))
+    .option('-r, --resume',
+      cliutils.getUsage('continue a previous backup from its last known position; invalid in "shallow" mode', defaults.resume))
+    .option('-t, --request-timeout <n>',
+      cliutils.getUsage('milliseconds to wait for a response to a HTTP request before retrying the request', defaults.requestTimeout),
+      Number)
+    .option('-u, --url <url>',
+      cliutils.getUsage('URL of the CouchDB/Cloudant server', defaults.url))
+    .parse(process.argv);
+
+  // Remove defaults that don't apply when using shallow mode
+  if (program.opts().mode === 'shallow' || envVarOptions.mode === 'shallow') {
+    delete defaults.parallelism;
+    delete defaults.log;
+    delete defaults.resume;
   }
 
-  // if we have a specified databases
-  if (typeof process.env.COUCH_DATABASE !== 'undefined') {
-    opts.db = process.env.COUCH_DATABASE;
-  }
-
-  // if we have a specified buffer size
-  if (typeof process.env.COUCH_BUFFER_SIZE !== 'undefined') {
-    opts.bufferSize = parseInt(process.env.COUCH_BUFFER_SIZE);
-  }
+  // Apply the options in order so that the CLI overrides env vars and env variables
+  // override defaults.
+  const opts = Object.assign({}, defaults, envVarOptions, program.opts());
 
-  // if we have a specified parallelism
-  if (typeof process.env.COUCH_PARALLELISM !== 'undefined') {
-    opts.parallelism = parseInt(process.env.COUCH_PARALLELISM);
+  if (opts.resume && (opts.log === defaults.log)) {
+    // If resuming and the log file arg is the newly generated tmp name from defaults then we know that --log wasn't specified.
+    // We have to do this check here for the CLI case because of the default.
+    error.terminationCallback(new error.BackupError('NoLogFileName', 'To resume a backup, a log file must be specified'));
   }
 
-  // if we have a specified request timeout
-  if (typeof process.env.COUCH_REQUEST_TIMEOUT !== 'undefined') {
-    opts.requestTimeout = parseInt(process.env.COUCH_REQUEST_TIMEOUT);
-  }
-
-  // if we have a specified log file
-  if (typeof process.env.COUCH_LOG !== 'undefined') {
-    opts.log = path.normalize(process.env.COUCH_LOG);
-  }
-
-  // if we are instructed to resume
-  if (typeof process.env.COUCH_RESUME !== 'undefined' && process.env.COUCH_RESUME === 'true') {
-    opts.resume = true;
-  }
-
-  // if we are given an output filename
-  if (typeof process.env.COUCH_OUTPUT !== 'undefined') {
-    opts.output = path.normalize(process.env.COUCH_OUTPUT);
-  }
-
-  // if we only want a shallow copy
-  if (typeof process.env.COUCH_MODE !== 'undefined' && process.env.COUCH_MODE === 'shallow') {
-    opts.mode = 'shallow';
-  }
-
-  // if we are instructed to be quiet
-  if (typeof process.env.COUCH_QUIET !== 'undefined' && process.env.COUCH_QUIET === 'true') {
-    opts.quiet = true;
-  }
-
-  // if we have a specified API key
-  if (typeof process.env.CLOUDANT_IAM_API_KEY !== 'undefined') {
-    opts.iamApiKey = process.env.CLOUDANT_IAM_API_KEY;
-  }
+  return opts;
+}
 
-  // if we have a specified IAM token endpoint
-  if (typeof process.env.CLOUDANT_IAM_TOKEN_URL !== 'undefined') {
-    opts.iamTokenUrl = process.env.CLOUDANT_IAM_TOKEN_URL;
-  }
+function parseRestoreArgs() {
+  const program = require('commander');
+
+  // Option CLI defaults
+  const defaults = config.cliDefaults();
+
+  // Options set by environment variables
+  const envVarOptions = {};
+  config.applyEnvironmentVariables(envVarOptions);
+
+  program
+    .version(pkg.version)
+    .description('Restore a CouchDB/Cloudant database from a backup text file.')
+    .usage('[options...]')
+    .option('-b, --buffer-size <n>',
+      cliutils.getUsage('number of documents restored at once', defaults.bufferSize),
+      Number)
+    .option('-d, --db <db>',
+      cliutils.getUsage('name of the new, existing database to restore to', defaults.db))
+    .option('-k, --iam-api-key <API key>',
+      cliutils.getUsage('IAM API key to access the Cloudant server'))
+    .option('-p, --parallelism <n>',
+      cliutils.getUsage('number of HTTP requests to perform in parallel when restoring a backup', defaults.parallelism),
+      Number)
+    .option('-q, --quiet',
+      cliutils.getUsage('suppress batch messages', defaults.quiet))
+    .option('-t, --request-timeout <n>',
+      cliutils.getUsage('milliseconds to wait for a response to a HTTP request before retrying the request', defaults.requestTimeout),
+      Number)
+    .option('-u, --url <url>',
+      cliutils.getUsage('URL of the CouchDB/Cloudant server', defaults.url))
+    .parse(process.argv);
+
+  // Apply the options in order so that the CLI overrides env vars and env variables
+  // override defaults.
+  const opts = Object.assign({}, defaults, envVarOptions, program.opts());
+
+  return opts;
 }
 
 module.exports = {
-  apiDefaults: apiDefaults,
-  cliDefaults: cliDefaults,
-  applyEnvironmentVariables: applyEnvironmentVariables
+  parseBackupArgs: parseBackupArgs,
+  parseRestoreArgs: parseRestoreArgs
 };

package/.scannerwork/scanner-report/source-61.txt

@@ -1,4 +1,4 @@
-// Copyright © 2017, 2018 IBM Corp. All rights reserved.
+// Copyright © 2017 IBM Corp. All rights reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -13,19 +13,29 @@
 // limitations under the License.
 'use strict';
 
-module.exports = function(db, options, readstream, ee, callback) {
-  const liner = require('../includes/liner.js')();
-  const writer = require('../includes/writer.js')(db, options.bufferSize, options.parallelism, ee);
+// stolen from http://strongloop.com/strongblog/practical-examples-of-the-new-node-js-streams-api/
+const stream = require('stream');
 
-  // pipe the input to the output, via transformation functions
-  readstream
-    .pipe(liner) // transform the input stream into per-line
-    .on('error', function(err) {
-      // Forward the error to the writer event emitter where we already have
-      // listeners on for handling errors
-      writer.emit('error', err);
-    })
-    .pipe(writer); // transform the data
+module.exports = function(onChange) {
+  const change = new stream.Transform({ objectMode: true });
 
-  callback(null, writer);
+  change._transform = function(line, encoding, done) {
+    let obj = null;
+
+    // one change per line - remove the trailing comma
+    line = line.trim().replace(/,$/, '');
+
+    // extract thee last_seq at the end of the changes feed
+    if (line.match(/^"last_seq":/)) {
+      line = '{' + line;
+    }
+    try {
+      obj = JSON.parse(line);
+    } catch (e) {
+    }
+    onChange(obj);
+    done();
+  };
+
+  return change;
 };

package/.scannerwork/scanner-report/source-62.txt

@@ -1,4 +1,4 @@
-// Copyright © 2017, 2022 IBM Corp. All rights reserved.
+// Copyright © 2017, 2018 IBM Corp. All rights reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -13,68 +13,47 @@
 // limitations under the License.
 'use strict';
 
-const async = require('async');
-const error = require('./error.js');
-const events = require('events');
-
-module.exports = function(db, options) {
-  const ee = new events.EventEmitter();
-  const start = new Date().getTime();
-  let batch = 0;
-  let hasErrored = false;
-  let startKey = null;
-  let total = 0;
+/**
+ * Utility methods for the command line interface.
+ * @module cliutils
+ * @see module:cliutils
+ */
 
-  async.doUntil(
-    function(callback) {
-      // Note, include_docs: true is set automatically when using the
-      // fetch function.
-      const opts = { db: db.db, limit: options.bufferSize, includeDocs: true };
-
-      // To avoid double fetching a document solely for the purposes of getting
-      // the next ID to use as a startKey for the next page we instead use the
-      // last ID of the current page and append the lowest unicode sort
-      // character.
-      if (startKey) opts.startKey = `${startKey}\0`;
-      db.service.postAllDocs(opts).then(response => {
-        const body = response.result;
-        if (!body.rows) {
-          ee.emit('error', new error.BackupError(
-            'AllDocsError', 'ERROR: Invalid all docs response'));
-          callback();
-        } else {
-          if (body.rows.length < opts.limit) {
-            startKey = null; // last batch
-          } else {
-            startKey = body.rows[opts.limit - 1].id;
-          }
+const url = require('url');
+const error = require('./error.js');
 
-          const docs = [];
-          body.rows.forEach(function(doc) {
-            docs.push(doc.doc);
-          });
+module.exports = {
 
-          if (docs.length > 0) {
-            ee.emit('received', {
-              batch: batch++,
-              data: docs,
-              length: docs.length,
-              time: (new Date().getTime() - start) / 1000,
-              total: total += docs.length
-            });
-          }
-          callback();
-        }
-      }).catch(err => {
-        err = error.convertResponseError(err);
-        ee.emit('error', err);
-        hasErrored = true;
-        callback();
-      });
-    },
-    function(callback) { callback(null, hasErrored || startKey == null); },
-    function() { ee.emit('finished', { total: total }); }
-  );
+  /**
+   * Combine a base URL and a database name, ensuring at least single slash
+   * between root and database name. This allows users to have Couch behind
+   * proxies that mount Couch's / endpoint at some other mount point.
+   * @param {string} root - root URL
+   * @param {string} databaseName - database name
+   * @return concatenated URL.
+   *
+   * @private
+   */
+  databaseUrl: function databaseUrl(root, databaseName) {
+    if (!root.endsWith('/')) {
+      root = root + '/';
+    }
+    try {
+      return new url.URL(encodeURIComponent(databaseName), root).toString();
+    } catch (err) {
+      throw error.wrapPossibleInvalidUrlError(err);
+    }
+  },
 
-  return ee;
+  /**
+   * Generate CLI argument usage text.
+   *
+   * @param {string} description - argument description.
+   * @param {string} defaultValue - default argument value.
+   *
+   * @private
+   */
+  getUsage: function getUsage(description, defaultValue) {
+    return `${description} ${defaultValue !== undefined ? ` (default: ${defaultValue})` : ''}`;
+  }
 };