@cloudant/couchbackup 2.9.8-SNAPSHOT.94 → 2.9.9-SNAPSHOT.100

package/.scannerwork/scanner-report/source-52.txt

@@ -0,0 +1,54 @@
+// Copyright © 2017, 2018 IBM Corp. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+'use strict';
+
+/**
+ * Utility methods for the command line interface.
+ * @module cliutils
+ * @see module:cliutils
+ */
+
+const url = require('url');
+
+module.exports = {
+
+  /**
+   * Combine a base URL and a database name, ensuring at least single slash
+   * between root and database name. This allows users to have Couch behind
+   * proxies that mount Couch's / endpoint at some other mount point.
+   * @param {string} root - root URL
+   * @param {string} databaseName - database name
+   * @return concatenated URL.
+   *
+   * @private
+   */
+  databaseUrl: function databaseUrl(root, databaseName) {
+    if (!root.endsWith('/')) {
+      root = root + '/';
+    }
+    return new url.URL(encodeURIComponent(databaseName), root).toString();
+  },
+
+  /**
+   * Generate CLI argument usage text.
+   *
+   * @param {string} description - argument description.
+   * @param {string} defaultValue - default argument value.
+   *
+   * @private
+   */
+  getUsage: function getUsage(description, defaultValue) {
+    return `${description} ${defaultValue !== undefined ? ` (default: ${defaultValue})` : ''}`;
+  }
+};

package/.scannerwork/scanner-report/source-53.txt

@@ -1,4 +1,4 @@
-// Copyright © 2017, 2021 IBM Corp. All rights reserved.
+// Copyright © 2017 IBM Corp. All rights reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -13,152 +13,34 @@
 // limitations under the License.
 'use strict';
 
-const async = require('async');
+// stolen from http://strongloop.com/strongblog/practical-examples-of-the-new-node-js-streams-api/
 const stream = require('stream');
-const error = require('./error.js');
-const debug = require('debug')('couchbackup:writer');
 
-module.exports = function(db, bufferSize, parallelism, ee) {
-  const writer = new stream.Transform({ objectMode: true });
-  let buffer = [];
-  let written = 0;
-  let linenumber = 0;
+module.exports = function() {
+  const liner = new stream.Transform({ objectMode: true });
 
-  // this is the queue of chunks that are written to the database
-  // the queue's payload will be an array of documents to be written,
-  // the size of the array will be bufferSize. The variable parallelism
-  // determines how many HTTP requests will occur at any one time.
-  const q = async.queue(function(payload, cb) {
-    // if we are restoring known revisions, we need to supply new_edits=false
-    if (payload.docs && payload.docs[0] && payload.docs[0]._rev) {
-      payload.new_edits = false;
-      debug('Using new_edits false mode.');
+  liner._transform = function(chunk, encoding, done) {
+    let data = chunk.toString();
+    if (this._lastLineData) {
+      data = this._lastLineData + data;
     }
 
-    if (!didError) {
-      db.service.postBulkDocs({
-        db: db.db,
-        bulkDocs: payload
-      }).then(response => {
-        if (!response.result || (payload.new_edits === false && response.result.length > 0)) {
-          throw new Error(`Error writing batch with new_edits:${payload.new_edits !== false}` +
-            ` and ${response.result ? response.result.length : 'unavailable'} items`);
-        }
-        written += payload.docs.length;
-        writer.emit('restored', { documents: payload.docs.length, total: written });
-        cb();
-      }).catch(err => {
-        err = error.convertResponseError(err);
-        debug(`Error writing docs ${err.name} ${err.message}`);
-        cb(err, payload);
-      });
-    }
-  }, parallelism);
-
-  let didError = false;
+    const lines = data.split('\n');
+    this._lastLineData = lines.splice(lines.length - 1, 1)[0];
 
-  // write the contents of the buffer to CouchDB in blocks of bufferSize
-  function processBuffer(flush, callback) {
-    function taskCallback(err, payload) {
-      if (err && !didError) {
-        debug(`Queue task failed with error ${err.name}`);
-        didError = true;
-        q.kill();
-        writer.emit('error', err);
-      }
+    for (const i in lines) {
+      this.push(lines[i]);
     }
+    done();
+  };
 
-    if (flush || buffer.length >= bufferSize) {
-      // work through the buffer to break off bufferSize chunks
-      // and feed the chunks to the queue
-      do {
-        // split the buffer into bufferSize chunks
-        const toSend = buffer.splice(0, bufferSize);
-
-        // and add the chunk to the queue
-        debug(`Adding ${toSend.length} to the write queue.`);
-        q.push({ docs: toSend }, taskCallback);
-      } while (buffer.length >= bufferSize);
-
-      // send any leftover documents to the queue
-      if (flush && buffer.length > 0) {
-        debug(`Adding remaining ${buffer.length} to the write queue.`);
-        q.push({ docs: buffer }, taskCallback);
-      }
-
-      // wait until the queue size falls to a reasonable level
-      async.until(
-        // wait until the queue length drops to twice the paralellism
-        // or until empty on the last write
-        function(callback) {
-          // if we encountered an error, stop this until loop
-          if (didError) {
-            return callback(null, true);
-          }
-          if (flush) {
-            callback(null, q.idle() && q.length() === 0);
-          } else {
-            callback(null, q.length() <= parallelism * 2);
-          }
-        },
-        function(cb) {
-          setTimeout(cb, 20);
-        },
-
-        function() {
-          if (flush && !didError) {
-            writer.emit('finished', { total: written });
-          }
-          // callback when we're happy with the queue size
-          callback();
-        });
-    } else {
-      callback();
-    }
-  }
-
-  // take an object
-  writer._transform = function(obj, encoding, done) {
-    // each obj that arrives here is a line from the backup file
-    // it should contain an array of objects. The length of the array
-    // depends on the bufferSize at backup time.
-    linenumber++;
-    if (!didError && obj !== '') {
-      // see if it parses as JSON
-      try {
-        const arr = JSON.parse(obj);
-
-        // if it's an array with a length
-        if (typeof arr === 'object' && arr.length > 0) {
-          // push each document into a buffer
-          buffer = buffer.concat(arr);
-
-          // pause the stream
-          // it's likely that the speed with which data can be read from disk
-          // may exceed the rate it can be written to CouchDB. To prevent
-          // the whole file being buffered in memory, we pause the stream here.
-          // it is resumed, when processBuffer calls back and we call done()
-          this.pause();
-
-          // break the buffer in to bufferSize chunks to be written to the database
-          processBuffer(false, done);
-        } else {
-          ee.emit('error', new error.BackupError('BackupFileJsonError', `Error on line ${linenumber} of backup file - not an array`));
-          done();
-        }
-      } catch (e) {
-        ee.emit('error', new error.BackupError('BackupFileJsonError', `Error on line ${linenumber} of backup file - cannot parse as JSON`));
-        // Could be an incomplete write that was subsequently resumed
-        done();
-      }
-    } else {
-      done();
+  liner._flush = function(done) {
+    if (this._lastLineData) {
+      this.push(this._lastLineData);
     }
+    this._lastLineData = null;
+    done();
   };
 
-  // called when we need to flush everything
-  writer._flush = function(done) {
-    processBuffer(true, done);
-  };
-  return writer;
+  return liner;
 };

package/.scannerwork/scanner-report/source-54.txt

@@ -1,4 +1,4 @@
-// Copyright © 2017 IBM Corp. All rights reserved.
+// Copyright © 2017, 2021 IBM Corp. All rights reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -13,34 +13,269 @@
 // limitations under the License.
 'use strict';
 
-// stolen from http://strongloop.com/strongblog/practical-examples-of-the-new-node-js-streams-api/
-const stream = require('stream');
+const async = require('async');
+const events = require('events');
+const fs = require('fs');
+const error = require('./error.js');
+const spoolchanges = require('./spoolchanges.js');
+const logfilesummary = require('./logfilesummary.js');
+const logfilegetbatches = require('./logfilegetbatches.js');
 
-module.exports = function() {
-  const liner = new stream.Transform({ objectMode: true });
+/**
+ * Read documents from a database to be backed up.
+ *
+ * @param {string} db - `@cloudant/cloudant` DB object for source database.
+ * @param {number} blocksize - number of documents to download in single request
+ * @param {number} parallelism - number of concurrent downloads
+ * @param {string} log - path to log file to use
+ * @param {boolean} resume - whether to resume from an existing log file
+ * @returns EventEmitter with following events:
+ *  - `received` - called with a block of documents to write to backup
+ *  - `error` - on error
+ *  - `finished` - when backup process is finished (either complete or errored)
+ */
+module.exports = function(db, options) {
+  const ee = new events.EventEmitter();
+  const start = new Date().getTime(); // backup start time
+  const batchesPerDownloadSession = 50; // max batches to read from log file for download at a time (prevent OOM)
 
-  liner._transform = function(chunk, encoding, done) {
-    let data = chunk.toString();
-    if (this._lastLineData) {
-      data = this._lastLineData + data;
+  function proceedWithBackup() {
+    if (options.resume) {
+      // pick up from existing log file from previous run
+      downloadRemainingBatches(options.log, db, ee, start, batchesPerDownloadSession, options.parallelism);
+    } else {
+      // create new log file and process
+      spoolchanges(db, options.log, options.bufferSize, ee, function(err) {
+        if (err) {
+          ee.emit('error', err);
+        } else {
+          downloadRemainingBatches(options.log, db, ee, start, batchesPerDownloadSession, options.parallelism);
+        }
+      });
     }
+  }
 
-    const lines = data.split('\n');
-    this._lastLineData = lines.splice(lines.length - 1, 1)[0];
+  validateBulkGetSupport(db, function(err) {
+    if (err) {
+      return ee.emit('error', err);
+    } else {
+      proceedWithBackup();
+    }
+  });
+
+  return ee;
+};
+
+/**
+ * Validate /_bulk_get support for a specified database.
+ *
+ * @param {string} db - nodejs-cloudant db
+ * @param {function} callback - called on completion with signature (err)
+ */
+function validateBulkGetSupport(db, callback) {
+  db.service.postBulkGet({ db: db.db, docs: [] }).then(() => { callback(); }).catch(err => {
+    err = error.convertResponseError(err, function(err) {
+      switch (err.status) {
+        case undefined:
+          // There was no status code on the error
+          return err;
+        case 404:
+          return new error.BackupError('BulkGetError', 'Database does not support /_bulk_get endpoint');
+        default:
+          return new error.HTTPError(err);
+      }
+    });
+    callback(err);
+  });
+}
+
+/**
+ * Download remaining batches in a log file, splitting batches into sets
+ * to avoid enqueueing too many in one go.
+ *
+ * @param {string} log - log file name to maintain download state
+ * @param {string} db - nodejs-cloudant db
+ * @param {events.EventEmitter} ee - event emitter to emit received events on
+ * @param {time} startTime - start time for backup process
+ * @param {number} batchesPerDownloadSession - max batches to enqueue for
+ *  download at a time. As batches contain many doc IDs, this helps avoid
+ *  exhausting memory.
+ * @param {number} parallelism - number of concurrent downloads
+ * @returns function to call do download remaining batches with signature
+ *  (err, {batches: batch, docs: doccount}) {@see spoolchanges}.
+ */
+function downloadRemainingBatches(log, db, ee, startTime, batchesPerDownloadSession, parallelism) {
+  let total = 0; // running total of documents downloaded so far
+  let noRemainingBatches = false;
 
-    for (const i in lines) {
-      this.push(lines[i]);
+  // Generate a set of batches (up to batchesPerDownloadSession) to download from the
+  // log file and download them. Set noRemainingBatches to `true` for last batch.
+  function downloadSingleBatchSet(done) {
+    // Fetch the doc IDs for the batches in the current set to
+    // download them.
+    function batchSetComplete(err, data) {
+      if (!err) {
+        total = data.total;
+      }
+      done(err);
     }
-    done();
-  };
+    function processRetrievedBatches(err, batches) {
+      if (!err) {
+        // process them in parallelised queue
+        processBatchSet(db, parallelism, log, batches, ee, startTime, total, batchSetComplete);
+      } else {
+        batchSetComplete(err);
+      }
+    }
+
+    readBatchSetIdsFromLogFile(log, batchesPerDownloadSession, function(err, batchSetIds) {
+      if (err) {
+        ee.emit('error', err);
+        // Stop processing changes file for fatal errors
+        noRemainingBatches = true;
+        done();
+      } else {
+        if (batchSetIds.length === 0) {
+          noRemainingBatches = true;
+          return done();
+        }
+        logfilegetbatches(log, batchSetIds, processRetrievedBatches);
+      }
+    });
+  }
+
+  // Return true if all batches in log file have been downloaded
+  function isFinished(callback) { callback(null, noRemainingBatches); }
+
+  function onComplete() {
+    ee.emit('finished', { total: total });
+  }
+
+  async.doUntil(downloadSingleBatchSet, isFinished, onComplete);
+}
 
-  liner._flush = function(done) {
-    if (this._lastLineData) {
-      this.push(this._lastLineData);
+/**
+ * Return a set of uncompleted download batch IDs from the log file.
+ *
+ * @param {string} log - log file path
+ * @param {number} batchesPerDownloadSession - maximum IDs to return
+ * @param {function} callback - sign (err, batchSetIds array)
+ */
+function readBatchSetIdsFromLogFile(log, batchesPerDownloadSession, callback) {
+  logfilesummary(log, function processSummary(err, summary) {
+    if (!err) {
+      if (!summary.changesComplete) {
+        callback(new error.BackupError('IncompleteChangesInLogFile',
+          'WARNING: Changes did not finish spooling'));
+        return;
+      }
+      if (Object.keys(summary.batches).length === 0) {
+        return callback(null, []);
+      }
+
+      // batch IDs are the property names of summary.batches
+      const batchSetIds = getPropertyNames(summary.batches, batchesPerDownloadSession);
+      callback(null, batchSetIds);
+    } else {
+      callback(err);
     }
-    this._lastLineData = null;
-    done();
-  };
+  });
+}
 
-  return liner;
-};
+/**
+ * Download a set of batches retrieved from a log file. When a download is
+ * complete, add a line to the logfile indicating such.
+ *
+ * @param {any} db - nodejs-cloudant database
+ * @param {any} parallelism - number of concurrent requests to make
+ * @param {any} log - log file to drive downloads from
+ * @param {any} batches - batches to download
+ * @param {any} ee - event emitter for progress. This funciton emits
+ *  received and error events.
+ * @param {any} start - time backup started, to report deltas
+ * @param {any} grandtotal - count of documents downloaded prior to this set
+ *  of batches
+ * @param {any} callback - completion callback, (err, {total: number}).
+ */
+function processBatchSet(db, parallelism, log, batches, ee, start, grandtotal, callback) {
+  let hasErrored = false;
+  let total = grandtotal;
+
+  // queue to process the fetch requests in an orderly fashion using _bulk_get
+  const q = async.queue(function(payload, done) {
+    const output = [];
+    const thisBatch = payload.batch;
+    delete payload.batch;
+    delete payload.command;
+
+    function logCompletedBatch(batch) {
+      if (log) {
+        fs.appendFile(log, ':d batch' + thisBatch + '\n', done);
+      } else {
+        done();
+      }
+    }
+
+    // do the /db/_bulk_get request
+    db.service.postBulkGet({
+      db: db.db,
+      revs: true,
+      docs: payload.docs
+    }).then(response => {
+      // create an output array with the docs returned
+      response.result.results.forEach(function(d) {
+        if (d.docs) {
+          d.docs.forEach(function(doc) {
+            if (doc.ok) {
+              output.push(doc.ok);
+            }
+          });
+        }
+      });
+      total += output.length;
+      const t = (new Date().getTime() - start) / 1000;
+      ee.emit('received', {
+        batch: thisBatch,
+        data: output,
+        length: output.length,
+        time: t,
+        total: total
+      }, q, logCompletedBatch);
+    }).catch(err => {
+      if (!hasErrored) {
+        hasErrored = true;
+        err = error.convertResponseError(err);
+        // Kill the queue for fatal errors
+        q.kill();
+        ee.emit('error', err);
+      }
+      done();
+    });
+  }, parallelism);
+
+  for (const i in batches) {
+    q.push(batches[i]);
+  }
+
+  q.drain(function() {
+    callback(null, { total: total });
+  });
+}
+
+/**
+ * Returns first N properties on an object.
+ *
+ * @param {object} obj - object with properties
+ * @param {number} count - number of properties to return
+ */
+function getPropertyNames(obj, count) {
+  // decide which batch numbers to deal with
+  const batchestofetch = [];
+  let j = 0;
+  for (const i in obj) {
+    batchestofetch.push(parseInt(i));
+    j++;
+    if (j >= count) break;
+  }
+  return batchestofetch;
+}