@cloudant/couchbackup 2.9.10 → 2.9.11-SNAPSHOT.116

package/.scannerwork/scanner-report/source-51.txt

@@ -1,4 +1,4 @@
-// Copyright © 2023 IBM Corp. All rights reserved.
+// Copyright © 2017, 2021 IBM Corp. All rights reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -11,90 +11,94 @@
 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 // See the License for the specific language governing permissions and
 // limitations under the License.
-
 'use strict';
 
-const chunk = require('lodash/chunk');
-const difference = require('lodash/difference');
-const forOwn = require('lodash/forOwn');
-const isEmpty = require('lodash/isEmpty');
-const union = require('lodash/union');
-
-const compare = async(database1, database2, client) => {
-  // check docs same in both dbs
-  const allDocs1 = await getAllDocs(client, database1);
-  const allDocs2 = await getAllDocs(client, database2);
-
-  const onlyInDb1 = (difference(allDocs1, allDocs2));
-  const onlyInDb2 = (difference(allDocs2, allDocs1));
-
-  let databasesSame = isEmpty(onlyInDb1) && isEmpty(onlyInDb2);
-
-  if (!databasesSame) {
-    console.log(onlyInDb1.length + ' documents only in db 1.');
-    console.log('Document IDs only in db 1: ' + onlyInDb1);
-    console.log(onlyInDb2.length + ' documents only in db 2.');
-    console.log('Document IDs only in db 2: ' + onlyInDb2);
-  }
-
-  // check revs same in docs common to both dbs
-  const partitionSize = 500;
-  const batches = chunk(union(allDocs1, allDocs2), partitionSize);
-
-  const missingRevsInDb2 = await getMissingRevs(client, database1, database2, batches);
-  const missingRevsInDb1 = await getMissingRevs(client, database2, database1, batches);
-
-  databasesSame = databasesSame && isEmpty(missingRevsInDb1) && isEmpty(missingRevsInDb2);
-
-  if (!databasesSame) {
-    console.log('Missing revs in db 1:' + JSON.stringify(missingRevsInDb1));
-    console.log('Missing revs in db 2:' + JSON.stringify(missingRevsInDb2));
-  }
-
-  return databasesSame;
+// fatal errors
+const codes = {
+  Error: 1,
+  InvalidOption: 2,
+  DatabaseNotFound: 10,
+  Unauthorized: 11,
+  Forbidden: 12,
+  DatabaseNotEmpty: 13,
+  NoLogFileName: 20,
+  LogDoesNotExist: 21,
+  IncompleteChangesInLogFile: 22,
+  SpoolChangesError: 30,
+  HTTPFatalError: 40,
+  BulkGetError: 50
 };
 
-const getMissingRevs = async(client, databaseName1, databaseName2, batcheses) => {
-  const fakeRevisionId = '9999-a';
-
-  const missing = {};
-
-  // look in db1 - use a fake revision ID to fetch all leaf revisions
-
-  for (const batches of batcheses) {
-    const documentRevisions = {};
-    batches.forEach(id => (documentRevisions[id] = [fakeRevisionId]));
-
-    const result1 = await client.postRevsDiff({ db: databaseName1, documentRevisions });
-    const revsDiffRequestDb2 = {};
-    forOwn(result1.result, (v, k) => (revsDiffRequestDb2[k] = v.possible_ancestors));
-    // look in db2
-    const result2 = await client.postRevsDiff({ db: databaseName2, documentRevisions: revsDiffRequestDb2 });
-    forOwn(result2.result, (v, k) => {
-      if ('missing' in v) {
-        missing[k] = v.missing;
-      }
-    });
+class BackupError extends Error {
+  constructor(name, message) {
+    super(message);
+    this.name = name;
   }
-  return missing;
-};
-
-const getAllDocs = async(client, database) => {
-  let allDocIds = [];
-  const limit = 2000;
-  let startKey = '\u0000';
-  do {
-    const pageOfDocIds = (await client.postAllDocs({ db: database, startKey, limit })).result.rows.map(r => r.id);
-    allDocIds = allDocIds.concat(pageOfDocIds);
-    if (pageOfDocIds.length < limit) {
-      startKey = null;
+}
+
+class HTTPError extends BackupError {
+  constructor(responseError, name) {
+    // Special case some names for more useful error messages
+    switch (responseError.status) {
+      case 401:
+        name = 'Unauthorized';
+        break;
+      case 403:
+        name = 'Forbidden';
+        break;
+      default:
+        name = name || 'HTTPFatalError';
+    }
+    super(name, responseError.message);
+  }
+}
+
+// Default function to return an error for HTTP status codes
+// < 400 -> OK
+// 4XX (except 429) -> Fatal
+// 429 & >=500 -> Transient
+function checkResponse(err) {
+  if (err) {
+    // Construct an HTTPError if there is request information on the error
+    // Codes < 400 are considered OK
+    if (err.status >= 400) {
+      return new HTTPError(err);
     } else {
-      startKey = pageOfDocIds[limit - 1] + '\u0000';
+      // Send it back again if there was no status code, e.g. a cxn error
+      return augmentMessage(err);
     }
-  } while (startKey != null);
-  return allDocIds;
-};
+  }
+}
+
+function convertResponseError(responseError, errorFactory) {
+  if (!errorFactory) {
+    errorFactory = checkResponse;
+  }
+  return errorFactory(responseError);
+}
+
+function augmentMessage(err) {
+  // For errors that don't have a status code, we are likely looking at a cxn
+  // error.
+  // Try to augment the message with more detail (core puts the code in statusText)
+  if (err && err.statusText) {
+    err.message = `${err.message} ${err.statusText}`;
+  }
+  if (err && err.description) {
+    err.message = `${err.message} ${err.description}`;
+  }
+  return err;
+}
 
 module.exports = {
-  compare
+  BackupError: BackupError,
+  HTTPError: HTTPError,
+  convertResponseError: convertResponseError,
+  terminationCallback: function terminationCallback(err, data) {
+    if (err) {
+      console.error(`ERROR: ${err.message}`);
+      process.exitCode = codes[err.name] || 1;
+      process.exit();
+    }
+  }
 };

package/.scannerwork/scanner-report/source-52.txt

@@ -1,4 +1,4 @@
-// Copyright © 2017 IBM Corp. All rights reserved.
+// Copyright © 2017, 2021 IBM Corp. All rights reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -11,109 +11,119 @@
 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 // See the License for the specific language governing permissions and
 // limitations under the License.
-
-/* global describe it */
 'use strict';
 
-const assert = require('assert');
-const fs = require('fs');
-const u = require('./citestutils.js');
-
-[{ useApi: true }, { useApi: false }].forEach(function(params) {
-  describe(u.scenario('Resume tests', params), function() {
-    it('should create a log file', function(done) {
-      // Allow up to 90 s for this test
-      u.setTimeout(this, 60);
-
-      const actualBackup = `./${this.fileName}`;
-      const logFile = `./${this.fileName}` + '.log';
-      // Use abort parameter to terminate the backup a given number of ms after
-      // the first data write to the output file.
-      const p = u.p(params, { opts: { log: logFile } });
-      u.testBackupToFile(p, 'animaldb', actualBackup, function(err) {
-        if (err) {
-          done(err);
-        } else {
-          // Assert the log file exists
-          try {
-            assert.ok(fs.existsSync(logFile), 'The log file should exist.');
-            done();
-          } catch (err) {
-            done(err);
-          }
-        }
-      });
-    });
-
-    it('should restore corrupted animaldb to a database correctly', function(done) {
-      // Allow up to 60 s to restore and compare (again it should be faster)!
-      u.setTimeout(this, 60);
-      const input = fs.createReadStream('./test/fixtures/animaldb_corrupted.json');
-      const dbName = this.dbName;
-      input.on('open', function() {
-        u.testRestore(params, input, dbName, function(err) {
-          if (err) {
-            done(err);
-          } else {
-            u.dbCompare('animaldb', dbName, done);
-          }
-        });
-      });
-    });
-
-    it('should restore resumed animaldb with blank line to a database correctly', function(done) {
-      // Allow up to 60 s to restore and compare (again it should be faster)!
-      u.setTimeout(this, 60);
-      const input = fs.createReadStream('./test/fixtures/animaldb_resumed_blank.json');
-      const dbName = this.dbName;
-      input.on('open', function() {
-        u.testRestore(params, input, dbName, function(err) {
-          if (err) {
-            done(err);
-          } else {
-            u.dbCompare('animaldb', dbName, done);
-          }
-        });
-      });
-    });
-  });
-});
-
-describe('Resume tests', function() {
-  // Currently cannot abort API backups, when we do this test should be run for
-  // both API and CLI
-  it('should correctly backup and restore backup10m', function(done) {
-    // Allow up to 90 s for this test
-    u.setTimeout(this, 90);
-
-    const actualBackup = `./${this.fileName}`;
-    const logFile = `./${this.fileName}` + '.log';
-    // Use abort parameter to terminate the backup a given number of ms after
-    // the first data write to the output file.
-    const p = u.p(params, { abort: true }, { opts: { log: logFile } });
-    const restoreDb = this.dbName;
-    // Set the database doc count as fewer than this should be written during
-    // resumed backup.
-    p.exclusiveMaxExpected = 5096;
-
-    u.testBackupAbortResumeRestore(p, 'backup10m', actualBackup, restoreDb, done);
-  });
-  // Note --output is only valid for CLI usage, this test should only run for CLI
-  const params = { useApi: false };
-  it('should correctly backup and restore backup10m using --output', function(done) {
-    // Allow up to 90 s for this test
-    u.setTimeout(this, 90);
-
-    const actualBackup = `./${this.fileName}`;
-    const logFile = `./${this.fileName}` + '.log';
-    // Use abort parameter to terminate the backup a given number of ms after
-    // the first data write to the output file.
-    const p = u.p(params, { abort: true }, { opts: { output: actualBackup, log: logFile } });
-    const restoreDb = this.dbName;
-    // Set the database doc count as fewer than this should be written during
-    // resumed backup.
-    p.exclusiveMaxExpected = 5096;
-
-    u.testBackupAbortResumeRestore(p, 'backup10m', actualBackup, restoreDb, done);
-  });
-});
+const cliutils = require('./cliutils.js');
+const config = require('./config.js');
+const error = require('./error.js');
+const path = require('path');
+const pkg = require('../package.json');
+
+function parseBackupArgs() {
+  const program = require('commander');
+
+  // Option CLI defaults
+  const defaults = config.cliDefaults();
+
+  // Options set by environment variables
+  const envVarOptions = {};
+  config.applyEnvironmentVariables(envVarOptions);
+
+  program
+    .version(pkg.version)
+    .description('Backup a CouchDB/Cloudant database to a backup text file.')
+    .usage('[options...]')
+    .option('-b, --buffer-size <n>',
+      cliutils.getUsage('number of documents fetched at once', defaults.bufferSize),
+      Number)
+    .option('-d, --db <db>',
+      cliutils.getUsage('name of the database to backup', defaults.db))
+    .option('-k, --iam-api-key <API key>',
+      cliutils.getUsage('IAM API key to access the Cloudant server'))
+    .option('-l, --log <file>',
+      cliutils.getUsage('file to store logging information during backup; invalid in "shallow" mode', 'a temporary file'),
+      path.normalize)
+    .option('-m, --mode <mode>',
+      cliutils.getUsage('"shallow" if only a superficial backup is done (ignoring conflicts and revision tokens), else "full" for complete backup', defaults.mode),
+      (mode) => { return mode.toLowerCase(); })
+    .option('-o, --output <file>',
+      cliutils.getUsage('file name to store the backup data', 'stdout'),
+      path.normalize)
+    .option('-p, --parallelism <n>',
+      cliutils.getUsage('number of HTTP requests to perform in parallel when performing a backup; ignored in "shallow" mode', defaults.parallelism),
+      Number)
+    .option('-q, --quiet',
+      cliutils.getUsage('suppress batch messages', defaults.quiet))
+    .option('-r, --resume',
+      cliutils.getUsage('continue a previous backup from its last known position; invalid in "shallow" mode', defaults.resume))
+    .option('-t, --request-timeout <n>',
+      cliutils.getUsage('milliseconds to wait for a response to a HTTP request before retrying the request', defaults.requestTimeout),
+      Number)
+    .option('-u, --url <url>',
+      cliutils.getUsage('URL of the CouchDB/Cloudant server', defaults.url))
+    .parse(process.argv);
+
+  // Remove defaults that don't apply when using shallow mode
+  if (program.opts().mode === 'shallow' || envVarOptions.mode === 'shallow') {
+    delete defaults.parallelism;
+    delete defaults.log;
+    delete defaults.resume;
+  }
+
+  // Apply the options in order so that the CLI overrides env vars and env variables
+  // override defaults.
+  const opts = Object.assign({}, defaults, envVarOptions, program.opts());
+
+  if (opts.resume && (opts.log === defaults.log)) {
+    // If resuming and the log file arg is the newly generated tmp name from defaults then we know that --log wasn't specified.
+    // We have to do this check here for the CLI case because of the default.
+    error.terminationCallback(new error.BackupError('NoLogFileName', 'To resume a backup, a log file must be specified'));
+  }
+
+  return opts;
+}
+
+function parseRestoreArgs() {
+  const program = require('commander');
+
+  // Option CLI defaults
+  const defaults = config.cliDefaults();
+
+  // Options set by environment variables
+  const envVarOptions = {};
+  config.applyEnvironmentVariables(envVarOptions);
+
+  program
+    .version(pkg.version)
+    .description('Restore a CouchDB/Cloudant database from a backup text file.')
+    .usage('[options...]')
+    .option('-b, --buffer-size <n>',
+      cliutils.getUsage('number of documents restored at once', defaults.bufferSize),
+      Number)
+    .option('-d, --db <db>',
+      cliutils.getUsage('name of the new, existing database to restore to', defaults.db))
+    .option('-k, --iam-api-key <API key>',
+      cliutils.getUsage('IAM API key to access the Cloudant server'))
+    .option('-p, --parallelism <n>',
+      cliutils.getUsage('number of HTTP requests to perform in parallel when restoring a backup', defaults.parallelism),
+      Number)
+    .option('-q, --quiet',
+      cliutils.getUsage('suppress batch messages', defaults.quiet))
+    .option('-t, --request-timeout <n>',
+      cliutils.getUsage('milliseconds to wait for a response to a HTTP request before retrying the request', defaults.requestTimeout),
+      Number)
+    .option('-u, --url <url>',
+      cliutils.getUsage('URL of the CouchDB/Cloudant server', defaults.url))
+    .parse(process.argv);
+
+  // Apply the options in order so that the CLI overrides env vars and env variables
+  // override defaults.
+  const opts = Object.assign({}, defaults, envVarOptions, program.opts());
+
+  return opts;
+}
+
+module.exports = {
+  parseBackupArgs: parseBackupArgs,
+  parseRestoreArgs: parseRestoreArgs
+};