@cloudant/couchbackup 2.9.13-SNAPSHOT.144 → 2.9.13

package/.scannerwork/scanner-report/source-56.txt

@@ -0,0 +1,281 @@
+// Copyright © 2017, 2021 IBM Corp. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+'use strict';
+
+const async = require('async');
+const events = require('events');
+const fs = require('fs');
+const error = require('./error.js');
+const spoolchanges = require('./spoolchanges.js');
+const logfilesummary = require('./logfilesummary.js');
+const logfilegetbatches = require('./logfilegetbatches.js');
+
+/**
+ * Read documents from a database to be backed up.
+ *
+ * @param {string} db - `@cloudant/cloudant` DB object for source database.
+ * @param {number} blocksize - number of documents to download in single request
+ * @param {number} parallelism - number of concurrent downloads
+ * @param {string} log - path to log file to use
+ * @param {boolean} resume - whether to resume from an existing log file
+ * @returns EventEmitter with following events:
+ *  - `received` - called with a block of documents to write to backup
+ *  - `error` - on error
+ *  - `finished` - when backup process is finished (either complete or errored)
+ */
+module.exports = function(db, options) {
+  const ee = new events.EventEmitter();
+  const start = new Date().getTime(); // backup start time
+  const batchesPerDownloadSession = 50; // max batches to read from log file for download at a time (prevent OOM)
+
+  function proceedWithBackup() {
+    if (options.resume) {
+      // pick up from existing log file from previous run
+      downloadRemainingBatches(options.log, db, ee, start, batchesPerDownloadSession, options.parallelism);
+    } else {
+      // create new log file and process
+      spoolchanges(db, options.log, options.bufferSize, ee, function(err) {
+        if (err) {
+          ee.emit('error', err);
+        } else {
+          downloadRemainingBatches(options.log, db, ee, start, batchesPerDownloadSession, options.parallelism);
+        }
+      });
+    }
+  }
+
+  validateBulkGetSupport(db, function(err) {
+    if (err) {
+      return ee.emit('error', err);
+    } else {
+      proceedWithBackup();
+    }
+  });
+
+  return ee;
+};
+
+/**
+ * Validate /_bulk_get support for a specified database.
+ *
+ * @param {string} db - nodejs-cloudant db
+ * @param {function} callback - called on completion with signature (err)
+ */
+function validateBulkGetSupport(db, callback) {
+  db.service.postBulkGet({ db: db.db, docs: [] }).then(() => { callback(); }).catch(err => {
+    err = error.convertResponseError(err, function(err) {
+      switch (err.status) {
+        case undefined:
+          // There was no status code on the error
+          return err;
+        case 404:
+          return new error.BackupError('BulkGetError', 'Database does not support /_bulk_get endpoint');
+        default:
+          return new error.HTTPError(err);
+      }
+    });
+    callback(err);
+  });
+}
+
+/**
+ * Download remaining batches in a log file, splitting batches into sets
+ * to avoid enqueueing too many in one go.
+ *
+ * @param {string} log - log file name to maintain download state
+ * @param {string} db - nodejs-cloudant db
+ * @param {events.EventEmitter} ee - event emitter to emit received events on
+ * @param {time} startTime - start time for backup process
+ * @param {number} batchesPerDownloadSession - max batches to enqueue for
+ *  download at a time. As batches contain many doc IDs, this helps avoid
+ *  exhausting memory.
+ * @param {number} parallelism - number of concurrent downloads
+ * @returns function to call do download remaining batches with signature
+ *  (err, {batches: batch, docs: doccount}) {@see spoolchanges}.
+ */
+function downloadRemainingBatches(log, db, ee, startTime, batchesPerDownloadSession, parallelism) {
+  let total = 0; // running total of documents downloaded so far
+  let noRemainingBatches = false;
+
+  // Generate a set of batches (up to batchesPerDownloadSession) to download from the
+  // log file and download them. Set noRemainingBatches to `true` for last batch.
+  function downloadSingleBatchSet(done) {
+    // Fetch the doc IDs for the batches in the current set to
+    // download them.
+    function batchSetComplete(err, data) {
+      if (!err) {
+        total = data.total;
+      }
+      done(err);
+    }
+    function processRetrievedBatches(err, batches) {
+      if (!err) {
+        // process them in parallelised queue
+        processBatchSet(db, parallelism, log, batches, ee, startTime, total, batchSetComplete);
+      } else {
+        batchSetComplete(err);
+      }
+    }
+
+    readBatchSetIdsFromLogFile(log, batchesPerDownloadSession, function(err, batchSetIds) {
+      if (err) {
+        ee.emit('error', err);
+        // Stop processing changes file for fatal errors
+        noRemainingBatches = true;
+        done();
+      } else {
+        if (batchSetIds.length === 0) {
+          noRemainingBatches = true;
+          return done();
+        }
+        logfilegetbatches(log, batchSetIds, processRetrievedBatches);
+      }
+    });
+  }
+
+  // Return true if all batches in log file have been downloaded
+  function isFinished(callback) { callback(null, noRemainingBatches); }
+
+  function onComplete() {
+    ee.emit('finished', { total: total });
+  }
+
+  async.doUntil(downloadSingleBatchSet, isFinished, onComplete);
+}
+
+/**
+ * Return a set of uncompleted download batch IDs from the log file.
+ *
+ * @param {string} log - log file path
+ * @param {number} batchesPerDownloadSession - maximum IDs to return
+ * @param {function} callback - sign (err, batchSetIds array)
+ */
+function readBatchSetIdsFromLogFile(log, batchesPerDownloadSession, callback) {
+  logfilesummary(log, function processSummary(err, summary) {
+    if (!err) {
+      if (!summary.changesComplete) {
+        callback(new error.BackupError('IncompleteChangesInLogFile',
+          'WARNING: Changes did not finish spooling'));
+        return;
+      }
+      if (Object.keys(summary.batches).length === 0) {
+        return callback(null, []);
+      }
+
+      // batch IDs are the property names of summary.batches
+      const batchSetIds = getPropertyNames(summary.batches, batchesPerDownloadSession);
+      callback(null, batchSetIds);
+    } else {
+      callback(err);
+    }
+  });
+}
+
+/**
+ * Download a set of batches retrieved from a log file. When a download is
+ * complete, add a line to the logfile indicating such.
+ *
+ * @param {any} db - nodejs-cloudant database
+ * @param {any} parallelism - number of concurrent requests to make
+ * @param {any} log - log file to drive downloads from
+ * @param {any} batches - batches to download
+ * @param {any} ee - event emitter for progress. This funciton emits
+ *  received and error events.
+ * @param {any} start - time backup started, to report deltas
+ * @param {any} grandtotal - count of documents downloaded prior to this set
+ *  of batches
+ * @param {any} callback - completion callback, (err, {total: number}).
+ */
+function processBatchSet(db, parallelism, log, batches, ee, start, grandtotal, callback) {
+  let hasErrored = false;
+  let total = grandtotal;
+
+  // queue to process the fetch requests in an orderly fashion using _bulk_get
+  const q = async.queue(function(payload, done) {
+    const output = [];
+    const thisBatch = payload.batch;
+    delete payload.batch;
+    delete payload.command;
+
+    function logCompletedBatch(batch) {
+      if (log) {
+        fs.appendFile(log, ':d batch' + thisBatch + '\n', done);
+      } else {
+        done();
+      }
+    }
+
+    // do the /db/_bulk_get request
+    db.service.postBulkGet({
+      db: db.db,
+      revs: true,
+      docs: payload.docs
+    }).then(response => {
+      // create an output array with the docs returned
+      response.result.results.forEach(function(d) {
+        if (d.docs) {
+          d.docs.forEach(function(doc) {
+            if (doc.ok) {
+              output.push(doc.ok);
+            }
+          });
+        }
+      });
+      total += output.length;
+      const t = (new Date().getTime() - start) / 1000;
+      ee.emit('received', {
+        batch: thisBatch,
+        data: output,
+        length: output.length,
+        time: t,
+        total: total
+      }, q, logCompletedBatch);
+    }).catch(err => {
+      if (!hasErrored) {
+        hasErrored = true;
+        err = error.convertResponseError(err);
+        // Kill the queue for fatal errors
+        q.kill();
+        ee.emit('error', err);
+      }
+      done();
+    });
+  }, parallelism);
+
+  for (const i in batches) {
+    q.push(batches[i]);
+  }
+
+  q.drain(function() {
+    callback(null, { total: total });
+  });
+}
+
+/**
+ * Returns first N properties on an object.
+ *
+ * @param {object} obj - object with properties
+ * @param {number} count - number of properties to return
+ */
+function getPropertyNames(obj, count) {
+  // decide which batch numbers to deal with
+  const batchestofetch = [];
+  let j = 0;
+  for (const i in obj) {
+    batchestofetch.push(parseInt(i));
+    j++;
+    if (j >= count) break;
+  }
+  return batchestofetch;
+}

package/.scannerwork/scanner-report/source-57.txt

@@ -0,0 +1,118 @@
+// Copyright © 2017, 2021 IBM Corp. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+'use strict';
+
+const path = require('path');
+const tmp = require('tmp');
+
+/**
+  Return API default settings.
+*/
+function apiDefaults() {
+  return {
+    parallelism: 5,
+    bufferSize: 500,
+    requestTimeout: 120000,
+    log: tmp.tmpNameSync(),
+    resume: false,
+    mode: 'full'
+  };
+}
+
+/**
+  Return CLI default settings.
+*/
+function cliDefaults() {
+  const defaults = apiDefaults();
+
+  // add additional legacy settings
+  defaults.db = 'test';
+  defaults.url = 'http://localhost:5984';
+
+  // add CLI only option
+  defaults.quiet = false;
+
+  return defaults;
+}
+
+/**
+  Override settings **in-place** with environment variables.
+*/
+function applyEnvironmentVariables(opts) {
+  // if we have a custom CouchDB url
+  if (typeof process.env.COUCH_URL !== 'undefined') {
+    opts.url = process.env.COUCH_URL;
+  }
+
+  // if we have a specified databases
+  if (typeof process.env.COUCH_DATABASE !== 'undefined') {
+    opts.db = process.env.COUCH_DATABASE;
+  }
+
+  // if we have a specified buffer size
+  if (typeof process.env.COUCH_BUFFER_SIZE !== 'undefined') {
+    opts.bufferSize = parseInt(process.env.COUCH_BUFFER_SIZE);
+  }
+
+  // if we have a specified parallelism
+  if (typeof process.env.COUCH_PARALLELISM !== 'undefined') {
+    opts.parallelism = parseInt(process.env.COUCH_PARALLELISM);
+  }
+
+  // if we have a specified request timeout
+  if (typeof process.env.COUCH_REQUEST_TIMEOUT !== 'undefined') {
+    opts.requestTimeout = parseInt(process.env.COUCH_REQUEST_TIMEOUT);
+  }
+
+  // if we have a specified log file
+  if (typeof process.env.COUCH_LOG !== 'undefined') {
+    opts.log = path.normalize(process.env.COUCH_LOG);
+  }
+
+  // if we are instructed to resume
+  if (typeof process.env.COUCH_RESUME !== 'undefined' && process.env.COUCH_RESUME === 'true') {
+    opts.resume = true;
+  }
+
+  // if we are given an output filename
+  if (typeof process.env.COUCH_OUTPUT !== 'undefined') {
+    opts.output = path.normalize(process.env.COUCH_OUTPUT);
+  }
+
+  // if we only want a shallow copy
+  if (typeof process.env.COUCH_MODE !== 'undefined' && process.env.COUCH_MODE === 'shallow') {
+    opts.mode = 'shallow';
+  }
+
+  // if we are instructed to be quiet
+  if (typeof process.env.COUCH_QUIET !== 'undefined' && process.env.COUCH_QUIET === 'true') {
+    opts.quiet = true;
+  }
+
+  // if we have a specified API key
+  if (typeof process.env.CLOUDANT_IAM_API_KEY !== 'undefined') {
+    opts.iamApiKey = process.env.CLOUDANT_IAM_API_KEY;
+  }
+
+  // if we have a specified IAM token endpoint
+  if (typeof process.env.CLOUDANT_IAM_TOKEN_URL !== 'undefined') {
+    opts.iamTokenUrl = process.env.CLOUDANT_IAM_TOKEN_URL;
+  }
+}
+
+module.exports = {
+  apiDefaults: apiDefaults,
+  cliDefaults: cliDefaults,
+  applyEnvironmentVariables: applyEnvironmentVariables
+};

package/.scannerwork/scanner-report/source-58.txt

@@ -1,4 +1,3 @@
-#!/usr/bin/env node
 // Copyright © 2017, 2021 IBM Corp. All rights reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
@@ -14,43 +13,92 @@
 // limitations under the License.
 'use strict';
 
-const error = require('../includes/error.js');
-const cliutils = require('../includes/cliutils.js');
-const couchbackup = require('../app.js');
-const parser = require('../includes/parser.js');
-const debug = require('debug');
-const restoreDebug = debug('couchbackup:restore');
-const restoreBatchDebug = debug('couchbackup:restore:batch');
+// fatal errors
+const codes = {
+  Error: 1,
+  InvalidOption: 2,
+  DatabaseNotFound: 10,
+  Unauthorized: 11,
+  Forbidden: 12,
+  DatabaseNotEmpty: 13,
+  NoLogFileName: 20,
+  LogDoesNotExist: 21,
+  IncompleteChangesInLogFile: 22,
+  SpoolChangesError: 30,
+  HTTPFatalError: 40,
+  BulkGetError: 50
+};
 
-restoreDebug.enabled = true;
+class BackupError extends Error {
+  constructor(name, message) {
+    super(message);
+    this.name = name;
+  }
+}
 
-const program = parser.parseRestoreArgs();
-const databaseUrl = cliutils.databaseUrl(program.url, program.db);
-const opts = {
-  bufferSize: program.bufferSize,
-  parallelism: program.parallelism,
-  requestTimeout: program.requestTimeout,
-  iamApiKey: program.iamApiKey,
-  iamTokenUrl: program.iamTokenUrl
-};
+class HTTPError extends BackupError {
+  constructor(responseError, name) {
+    // Special case some names for more useful error messages
+    switch (responseError.status) {
+      case 401:
+        name = 'Unauthorized';
+        break;
+      case 403:
+        name = 'Forbidden';
+        break;
+      default:
+        name = name || 'HTTPFatalError';
+    }
+    super(name, responseError.message);
+  }
+}
 
-// log configuration to console
-console.error('='.repeat(80));
-console.error('Performing restore on ' + databaseUrl.replace(/\/\/.+@/g, '//****:****@') + ' using configuration:');
-console.error(JSON.stringify(opts, null, 2).replace(/"iamApiKey": "[^"]+"/, '"iamApiKey": "****"'));
-console.error('='.repeat(80));
+// Default function to return an error for HTTP status codes
+// < 400 -> OK
+// 4XX (except 429) -> Fatal
+// 429 & >=500 -> Transient
+function checkResponse(err) {
+  if (err) {
+    // Construct an HTTPError if there is request information on the error
+    // Codes < 400 are considered OK
+    if (err.status >= 400) {
+      return new HTTPError(err);
+    } else {
+      // Send it back again if there was no status code, e.g. a cxn error
+      return augmentMessage(err);
+    }
+  }
+}
 
-restoreBatchDebug.enabled = !program.quiet;
+function convertResponseError(responseError, errorFactory) {
+  if (!errorFactory) {
+    errorFactory = checkResponse;
+  }
+  return errorFactory(responseError);
+}
 
-return couchbackup.restore(
-  process.stdin, // restore from stdin
-  databaseUrl,
-  opts,
-  error.terminationCallback
-).on('restored', function(obj) {
-  restoreBatchDebug('restored', obj.total);
-}).on('error', function(e) {
-  restoreDebug('ERROR', e);
-}).on('finished', function(obj) {
-  restoreDebug('finished', obj);
-});
+function augmentMessage(err) {
+  // For errors that don't have a status code, we are likely looking at a cxn
+  // error.
+  // Try to augment the message with more detail (core puts the code in statusText)
+  if (err && err.statusText) {
+    err.message = `${err.message} ${err.statusText}`;
+  }
+  if (err && err.description) {
+    err.message = `${err.message} ${err.description}`;
+  }
+  return err;
+}
+
+module.exports = {
+  BackupError: BackupError,
+  HTTPError: HTTPError,
+  convertResponseError: convertResponseError,
+  terminationCallback: function terminationCallback(err, data) {
+    if (err) {
+      console.error(`ERROR: ${err.message}`);
+      process.exitCode = codes[err.name] || 1;
+      process.exit();
+    }
+  }
+};

package/.scannerwork/scanner-report/source-59.txt

@@ -1,5 +1,4 @@
-#!/usr/bin/env node
-// Copyright © 2017, 2021 IBM Corp. All rights reserved.
+// Copyright © 2017, 2022 IBM Corp. All rights reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -14,64 +13,68 @@
 // limitations under the License.
 'use strict';
 
-const error = require('../includes/error.js');
-const fs = require('fs');
-const cliutils = require('../includes/cliutils.js');
-const couchbackup = require('../app.js');
-const parser = require('../includes/parser.js');
-const debug = require('debug');
-const backupDebug = debug('couchbackup:backup');
-const backupBatchDebug = debug('couchbackup:backup:batch');
+const async = require('async');
+const error = require('./error.js');
+const events = require('events');
 
-backupDebug.enabled = true;
+module.exports = function(db, options) {
+  const ee = new events.EventEmitter();
+  const start = new Date().getTime();
+  let batch = 0;
+  let hasErrored = false;
+  let startKey = null;
+  let total = 0;
 
-const program = parser.parseBackupArgs();
+  async.doUntil(
+    function(callback) {
+      // Note, include_docs: true is set automatically when using the
+      // fetch function.
+      const opts = { db: db.db, limit: options.bufferSize, includeDocs: true };
 
-const databaseUrl = cliutils.databaseUrl(program.url, program.db);
-const opts = {
-  bufferSize: program.bufferSize,
-  log: program.log,
-  mode: program.mode,
-  parallelism: program.parallelism,
-  requestTimeout: program.requestTimeout,
-  resume: program.resume,
-  iamApiKey: program.iamApiKey,
-  iamTokenUrl: program.iamTokenUrl
-};
-
-// log configuration to console
-console.error('='.repeat(80));
-console.error('Performing backup on ' + databaseUrl.replace(/\/\/.+@/g, '//****:****@') + ' using configuration:');
-console.error(JSON.stringify(opts, null, 2).replace(/"iamApiKey": "[^"]+"/, '"iamApiKey": "****"'));
-console.error('='.repeat(80));
-
-backupBatchDebug.enabled = !program.quiet;
+      // To avoid double fetching a document solely for the purposes of getting
+      // the next ID to use as a startKey for the next page we instead use the
+      // last ID of the current page and append the lowest unicode sort
+      // character.
+      if (startKey) opts.startKey = `${startKey}\0`;
+      db.service.postAllDocs(opts).then(response => {
+        const body = response.result;
+        if (!body.rows) {
+          ee.emit('error', new error.BackupError(
+            'AllDocsError', 'ERROR: Invalid all docs response'));
+          callback();
+        } else {
+          if (body.rows.length < opts.limit) {
+            startKey = null; // last batch
+          } else {
+            startKey = body.rows[opts.limit - 1].id;
+          }
 
-let ws = process.stdout;
+          const docs = [];
+          body.rows.forEach(function(doc) {
+            docs.push(doc.doc);
+          });
 
-// open output file
-if (program.output) {
-  let flags = 'w';
-  if (program.log && program.resume) {
-    flags = 'a';
-  }
-  const fd = fs.openSync(program.output, flags);
-  ws = fs.createWriteStream(null, { fd: fd });
-}
+          if (docs.length > 0) {
+            ee.emit('received', {
+              batch: batch++,
+              data: docs,
+              length: docs.length,
+              time: (new Date().getTime() - start) / 1000,
+              total: total += docs.length
+            });
+          }
+          callback();
+        }
+      }).catch(err => {
+        err = error.convertResponseError(err);
+        ee.emit('error', err);
+        hasErrored = true;
+        callback();
+      });
+    },
+    function(callback) { callback(null, hasErrored || startKey == null); },
+    function() { ee.emit('finished', { total: total }); }
+  );
 
-backupDebug('Fetching all database changes...');
-
-return couchbackup.backup(
-  databaseUrl,
-  ws,
-  opts,
-  error.terminationCallback
-).on('changes', function(batch) {
-  backupBatchDebug('Total batches received:', batch + 1);
-}).on('written', function(obj) {
-  backupBatchDebug('Written batch ID:', obj.batch, 'Total document revisions written:', obj.total, 'Time:', obj.time);
-}).on('error', function(e) {
-  backupDebug('ERROR', e);
-}).on('finished', function(obj) {
-  backupDebug('Finished - Total document revisions written:', obj.total);
-});
+  return ee;
+};