@cloudant/couchbackup 2.9.12-SNAPSHOT.136 → 2.9.12

package/.scannerwork/scanner-report/source-14.txt

@@ -1,4 +1,4 @@
-// Copyright © 2017 IBM Corp. All rights reserved.
+// Copyright © 2017, 2021 IBM Corp. All rights reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -11,26 +11,154 @@
 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 // See the License for the specific language governing permissions and
 // limitations under the License.
-
-/* global describe it */
 'use strict';
 
-const u = require('./citestutils.js');
-
-describe('Encryption tests', function() {
-  // Note CLI only to use openssl command
-  const p = { useApi: false, encryption: true };
-
-  it('should backup and restore animaldb via an encrypted file', function(done) {
-    // Allow up to 60 s for backup and restore of animaldb
-    u.setTimeout(this, 60);
-    const encryptedBackup = `./${this.fileName}`;
-    u.testBackupAndRestoreViaFile(p, 'animaldb', encryptedBackup, this.dbName, function(err) {
-      if (err) {
-        done(err);
-      } else {
-        u.assertEncryptedFile(encryptedBackup, done);
+const async = require('async');
+const stream = require('stream');
+const error = require('./error.js');
+const debug = require('debug')('couchbackup:writer');
+
+module.exports = function(db, bufferSize, parallelism, ee) {
+  const writer = new stream.Transform({ objectMode: true });
+  let buffer = [];
+  let written = 0;
+  let linenumber = 0;
+
+  // this is the queue of chunks that are written to the database
+  // the queue's payload will be an array of documents to be written,
+  // the size of the array will be bufferSize. The variable parallelism
+  // determines how many HTTP requests will occur at any one time.
+  const q = async.queue(function(payload, cb) {
+    // if we are restoring known revisions, we need to supply new_edits=false
+    if (payload.docs && payload.docs[0] && payload.docs[0]._rev) {
+      payload.new_edits = false;
+      debug('Using new_edits false mode.');
+    }
+
+    if (!didError) {
+      db.service.postBulkDocs({
+        db: db.db,
+        bulkDocs: payload
+      }).then(response => {
+        if (!response.result || (payload.new_edits === false && response.result.length > 0)) {
+          throw new Error(`Error writing batch with new_edits:${payload.new_edits !== false}` +
+            ` and ${response.result ? response.result.length : 'unavailable'} items`);
+        }
+        written += payload.docs.length;
+        writer.emit('restored', { documents: payload.docs.length, total: written });
+        cb();
+      }).catch(err => {
+        err = error.convertResponseError(err);
+        debug(`Error writing docs ${err.name} ${err.message}`);
+        cb(err, payload);
+      });
+    }
+  }, parallelism);
+
+  let didError = false;
+
+  // write the contents of the buffer to CouchDB in blocks of bufferSize
+  function processBuffer(flush, callback) {
+    function taskCallback(err, payload) {
+      if (err && !didError) {
+        debug(`Queue task failed with error ${err.name}`);
+        didError = true;
+        q.kill();
+        writer.emit('error', err);
+      }
+    }
+
+    if (flush || buffer.length >= bufferSize) {
+      // work through the buffer to break off bufferSize chunks
+      // and feed the chunks to the queue
+      do {
+        // split the buffer into bufferSize chunks
+        const toSend = buffer.splice(0, bufferSize);
+
+        // and add the chunk to the queue
+        debug(`Adding ${toSend.length} to the write queue.`);
+        q.push({ docs: toSend }, taskCallback);
+      } while (buffer.length >= bufferSize);
+
+      // send any leftover documents to the queue
+      if (flush && buffer.length > 0) {
+        debug(`Adding remaining ${buffer.length} to the write queue.`);
+        q.push({ docs: buffer }, taskCallback);
       }
-    });
-  });
-});
+
+      // wait until the queue size falls to a reasonable level
+      async.until(
+        // wait until the queue length drops to twice the paralellism
+        // or until empty on the last write
+        function(callback) {
+          // if we encountered an error, stop this until loop
+          if (didError) {
+            return callback(null, true);
+          }
+          if (flush) {
+            callback(null, q.idle() && q.length() === 0);
+          } else {
+            callback(null, q.length() <= parallelism * 2);
+          }
+        },
+        function(cb) {
+          setTimeout(cb, 20);
+        },
+
+        function() {
+          if (flush && !didError) {
+            writer.emit('finished', { total: written });
+          }
+          // callback when we're happy with the queue size
+          callback();
+        });
+    } else {
+      callback();
+    }
+  }
+
+  // take an object
+  writer._transform = function(obj, encoding, done) {
+    // each obj that arrives here is a line from the backup file
+    // it should contain an array of objects. The length of the array
+    // depends on the bufferSize at backup time.
+    linenumber++;
+    if (!didError && obj !== '') {
+      // see if it parses as JSON
+      try {
+        const arr = JSON.parse(obj);
+
+        // if it's an array with a length
+        if (typeof arr === 'object' && arr.length > 0) {
+          // push each document into a buffer
+          buffer = buffer.concat(arr);
+
+          // pause the stream
+          // it's likely that the speed with which data can be read from disk
+          // may exceed the rate it can be written to CouchDB. To prevent
+          // the whole file being buffered in memory, we pause the stream here.
+          // it is resumed, when processBuffer calls back and we call done()
+          this.pause();
+
+          // break the buffer in to bufferSize chunks to be written to the database
+          processBuffer(false, done);
+        } else {
+          ee.emit('error', new error.BackupError('BackupFileJsonError', `Error on line ${linenumber} of backup file - not an array`));
+          done();
+        }
+      } catch (e) {
+        ee.emit('error', new error.BackupError('BackupFileJsonError', `Error on line ${linenumber} of backup file - cannot parse as JSON`));
+        // Could be an incomplete write that was subsequently resumed
+        done();
+      }
+    } else {
+      done();
+    }
+  };
+
+  // called when we need to flush everything
+  writer._flush = function(done) {
+    processBuffer(true, done);
+  };
+  return writer;
+};

package/.scannerwork/scanner-report/source-15.txt

@@ -1,4 +1,4 @@
-// Copyright © 2017, 2021 IBM Corp. All rights reserved.
+// Copyright © 2017 IBM Corp. All rights reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -11,356 +11,65 @@
 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 // See the License for the specific language governing permissions and
 // limitations under the License.
-
-/* global describe afterEach before after it */
 'use strict';
 
-const assert = require('assert');
-const parser = require('../includes/parser.js');
-
-describe('#unit Default parameters', function() {
-  let processEnvCopy;
-  let processArgvCopy;
-
-  before('Set process data for test', function() {
-    // Copy env and argv so we can reset them after the tests
-    processEnvCopy = JSON.parse(JSON.stringify(process.env));
-    processArgvCopy = JSON.parse(JSON.stringify(process.argv));
-
-    // setup environment variables
-    process.env.COUCH_URL = 'http://user:pass@myurl.com';
-    process.env.COUCH_DATABASE = 'mydb';
-    process.env.COUCH_BUFFER_SIZE = '1000';
-    process.env.COUCH_PARALLELISM = '20';
-    process.env.COUCH_REQUEST_TIMEOUT = '20000';
-    process.env.COUCH_LOG = 'my.log';
-    process.env.COUCH_RESUME = 'true';
-    process.env.COUCH_OUTPUT = 'myfile.txt';
-    process.env.COUCH_MODE = 'shallow';
-    process.env.CLOUDANT_IAM_API_KEY = 'ABC123-ZYX987_cba789-xyz321';
-    process.env.COUCH_QUIET = 'true';
-  });
-
-  after('Reset process data', function() {
-    process.env = processEnvCopy;
-    process.argv = processArgvCopy;
-  });
-
-  afterEach(function() {
-    delete require.cache[require.resolve('commander')];
-  });
-
-  describe('Backup command-line', function() {
-    it('respects the COUCH_URL env variable if the --url backup command-line parameter is missing', function(done) {
-      process.argv = ['node', 'test'];
-      const program = parser.parseBackupArgs();
-      assert.strictEqual(typeof program.url, 'string');
-      assert.strictEqual(program.url, process.env.COUCH_URL);
-      done();
-    });
-
-    it('respects the COUCH_DATABASE env variable if the --db backup command-line parameter is missing', function(done) {
-      process.argv = ['node', 'test'];
-      const program = parser.parseBackupArgs();
-      assert.strictEqual(typeof program.db, 'string');
-      assert.strictEqual(program.db, process.env.COUCH_DATABASE);
-      done();
-    });
-
-    it('respects the COUCH_BUFFER_SIZE env variable if the --buffer-size backup command-line parameter is missing', function(done) {
-      process.argv = ['node', 'test'];
-      const program = parser.parseBackupArgs();
-      assert.strictEqual(typeof program.bufferSize, 'number');
-      assert.strictEqual(program.bufferSize, parseInt(process.env.COUCH_BUFFER_SIZE, 10));
-      done();
-    });
-
-    it('respects the COUCH_PARALLELISM env variable if the --parallelism backup command-line parameter is missing', function(done) {
-      process.argv = ['node', 'test'];
-      const program = parser.parseBackupArgs();
-      assert.strictEqual(typeof program.parallelism, 'number');
-      assert.strictEqual(program.parallelism, parseInt(process.env.COUCH_PARALLELISM, 10));
-      done();
-    });
-
-    it('respects the COUCH_REQUEST_TIMEOUT env variable if the --request-timeout backup command-line parameter is missing', function(done) {
-      process.argv = ['node', 'test'];
-      const program = parser.parseBackupArgs();
-      assert.strictEqual(typeof program.requestTimeout, 'number');
-      assert.strictEqual(program.requestTimeout, parseInt(process.env.COUCH_REQUEST_TIMEOUT, 10));
-      done();
-    });
-
-    it('respects the CLOUDANT_IAM_API_KEY env variable if the --iam-api-key backup command-line parameter is missing', function(done) {
-      process.argv = ['node', 'test'];
-      const program = parser.parseBackupArgs();
-      assert.strictEqual(typeof program.iamApiKey, 'string');
-      assert.strictEqual(program.iamApiKey, process.env.CLOUDANT_IAM_API_KEY);
-      done();
-    });
-
-    it('respects the COUCH_LOG env variable if the --log backup command-line parameter is missing', function(done) {
-      process.argv = ['node', 'test'];
-      const program = parser.parseBackupArgs();
-      assert.strictEqual(typeof program.log, 'string');
-      assert.strictEqual(program.log, process.env.COUCH_LOG);
-      done();
-    });
-
-    it('respects the COUCH_RESUME env variable if the --resume backup command-line parameter is missing', function(done) {
-      process.argv = ['node', 'test'];
-      const program = parser.parseBackupArgs();
-      assert.strictEqual(typeof program.resume, 'boolean');
-      assert.strictEqual(program.resume, true);
-      done();
-    });
-
-    it('respects the COUCH_OUTPUT env variable if the --output backup command-line parameter is missing', function(done) {
-      process.argv = ['node', 'test'];
-      const program = parser.parseBackupArgs();
-      assert.strictEqual(typeof program.output, 'string');
-      assert.strictEqual(program.output, process.env.COUCH_OUTPUT);
-      done();
-    });
-
-    it('respects the COUCH_MODE env variable if the --mode backup command-line parameter is missing', function(done) {
-      process.argv = ['node', 'test'];
-      const program = parser.parseBackupArgs();
-      assert.strictEqual(typeof program.mode, 'string');
-      assert.strictEqual(program.mode, process.env.COUCH_MODE);
-      done();
-    });
-
-    it('respects the COUCH_QUIET env variable if the --quiet backup command-line parameter is missing', function(done) {
-      process.argv = ['node', 'test'];
-      const program = parser.parseBackupArgs();
-      assert.strictEqual(typeof program.quiet, 'boolean');
-      assert.strictEqual(program.quiet, true);
-      done();
-    });
-
-    it('respects the backup --url command-line parameter', function(done) {
-      const url = 'http://user:pass@myurl2.com';
-      process.argv = ['node', 'test', '--url', url];
-      const program = parser.parseBackupArgs();
-      assert.strictEqual(typeof program.url, 'string');
-      assert.strictEqual(program.url, url);
-      done();
-    });
-
-    it('respects the backup --db command-line parameter', function(done) {
-      const db = 'mydb2';
-      process.argv = ['node', 'test', '--db', db];
-      const program = parser.parseBackupArgs();
-      assert.strictEqual(typeof program.db, 'string');
-      assert.strictEqual(program.db, db);
-      done();
-    });
-
-    it('respects the backup --buffer-size command-line parameter', function(done) {
-      const bufferSize = 500;
-      process.argv = ['node', 'test', '--buffer-size', bufferSize];
-      const program = parser.parseBackupArgs();
-      assert.strictEqual(typeof program.bufferSize, 'number');
-      assert.strictEqual(program.bufferSize, bufferSize);
-      done();
-    });
-
-    it('respects the backup --parallelism command-line parameter', function(done) {
-      const parallelism = 10;
-      process.argv = ['node', 'test', '--parallelism', parallelism];
-      const program = parser.parseBackupArgs();
-      assert.strictEqual(typeof program.parallelism, 'number');
-      assert.strictEqual(program.parallelism, parallelism);
-      done();
-    });
-
-    it('respects the backup --request-timeout command-line parameter', function(done) {
-      const requestTimeout = 10000;
-      process.argv = ['node', 'test', '--request-timeout', requestTimeout];
-      const program = parser.parseBackupArgs();
-      assert.strictEqual(typeof program.requestTimeout, 'number');
-      assert.strictEqual(program.requestTimeout, requestTimeout);
-      done();
-    });
-
-    it('respects the backup --iam-api-key command-line parameter', function(done) {
-      const key = '123abc-789zyx_CBA987-XYZ321';
-      process.argv = ['node', 'test', '--iam-api-key', key];
-      const program = parser.parseBackupArgs();
-      assert.strictEqual(typeof program.iamApiKey, 'string');
-      assert.strictEqual(program.iamApiKey, key);
-      done();
-    });
-
-    it('respects the backup --log command-line parameter', function(done) {
-      const filename = 'my2.log';
-      process.argv = ['node', 'test', '--log', filename];
-      const program = parser.parseBackupArgs();
-      assert.strictEqual(typeof program.log, 'string');
-      assert.strictEqual(program.log, filename);
-      done();
-    });
-
-    it('respects the backup --resume command-line parameter', function(done) {
-      process.argv = ['node', 'test', '--resume'];
-      const program = parser.parseBackupArgs();
-      assert.strictEqual(typeof program.resume, 'boolean');
-      assert.strictEqual(program.resume, true);
-      done();
-    });
-
-    it('respects the backup --output command-line parameter', function(done) {
-      const filename = 'myfile2.txt';
-      process.argv = ['node', 'test', '--output', filename];
-      const program = parser.parseBackupArgs();
-      assert.strictEqual(typeof program.output, 'string');
-      assert.strictEqual(program.output, filename);
-      done();
-    });
-
-    it('respects the backup --mode full command-line parameter', function(done) {
-      process.argv = ['node', 'test', '--mode', 'full'];
-      const program = parser.parseBackupArgs();
-      assert.strictEqual(typeof program.mode, 'string');
-      assert.strictEqual(program.mode, 'full');
-      done();
-    });
-
-    it('respects the backup --mode shallow command-line parameter', function(done) {
-      process.argv = ['node', 'test', '--mode', 'shallow'];
-      const program = parser.parseBackupArgs();
-      assert.strictEqual(typeof program.mode, 'string');
-      assert.strictEqual(program.mode, 'shallow');
-      done();
-    });
-
-    it('respects the backup --quiet command-line parameter', function(done) {
-      process.argv = ['node', 'test', '--quiet'];
-      const program = parser.parseBackupArgs();
-      assert.strictEqual(typeof program.quiet, 'boolean');
-      assert.strictEqual(program.quiet, true);
-      done();
-    });
-  });
-
-  describe('Restore command-line', function() {
-    it('respects the COUCH_URL env variable if the --url restore command-line parameter is missing', function(done) {
-      process.argv = ['node', 'test'];
-      const program = parser.parseRestoreArgs();
-      assert.strictEqual(typeof program.url, 'string');
-      assert.strictEqual(program.url, process.env.COUCH_URL);
-      done();
-    });
-
-    it('respects the COUCH_DATABASE env variable if the --db restore command-line parameter is missing', function(done) {
-      process.argv = ['node', 'test'];
-      const program = parser.parseRestoreArgs();
-      assert.strictEqual(typeof program.db, 'string');
-      assert.strictEqual(program.db, process.env.COUCH_DATABASE);
-      done();
-    });
-
-    it('respects the COUCH_BUFFER_SIZE env variable if the --buffer-size restore command-line parameter is missing', function(done) {
-      process.argv = ['node', 'test'];
-      const program = parser.parseRestoreArgs();
-      assert.strictEqual(typeof program.bufferSize, 'number');
-      assert.strictEqual(program.bufferSize, parseInt(process.env.COUCH_BUFFER_SIZE, 10));
-      done();
-    });
-
-    it('respects the COUCH_PARALLELISM env variable if the --parallelism restore command-line parameter is missing', function(done) {
-      process.argv = ['node', 'test'];
-      const program = parser.parseRestoreArgs();
-      assert.strictEqual(typeof program.parallelism, 'number');
-      assert.strictEqual(program.parallelism, parseInt(process.env.COUCH_PARALLELISM, 10));
-      done();
-    });
-
-    it('respects the COUCH_REQUEST_TIMEOUT env variable if the --request-timeout restore command-line parameter is missing', function(done) {
-      process.argv = ['node', 'test'];
-      const program = parser.parseRestoreArgs();
-      assert.strictEqual(typeof program.requestTimeout, 'number');
-      assert.strictEqual(program.requestTimeout, parseInt(process.env.COUCH_REQUEST_TIMEOUT, 10));
-      done();
-    });
-
-    it('respects the CLOUDANT_IAM_API_KEY env variable if the --iam-api-key restore command-line parameter is missing', function(done) {
-      process.argv = ['node', 'test'];
-      const program = parser.parseRestoreArgs();
-      assert.strictEqual(typeof program.iamApiKey, 'string');
-      assert.strictEqual(program.iamApiKey, process.env.CLOUDANT_IAM_API_KEY);
-      done();
-    });
-
-    it('respects the COUCH_QUIET env variable if the --quiet restorer command-line parameter is missing', function(done) {
-      process.argv = ['node', 'test'];
-      const program = parser.parseBackupArgs();
-      assert.strictEqual(typeof program.quiet, 'boolean');
-      assert.strictEqual(program.quiet, true);
-      done();
-    });
-
-    it('respects the restore --url command-line parameter', function(done) {
-      const url = 'https://a:b@myurl3.com';
-      process.argv = ['node', 'test', '--url', url];
-      const program = parser.parseRestoreArgs();
-      assert.strictEqual(typeof program.url, 'string');
-      assert.strictEqual(program.url, url);
-      done();
-    });
-
-    it('respects the restore --db command-line parameter', function(done) {
-      const db = 'mydb3';
-      process.argv = ['node', 'test', '--db', db];
-      const program = parser.parseRestoreArgs();
-      assert.strictEqual(typeof program.db, 'string');
-      assert.strictEqual(program.db, db);
-      done();
-    });
-
-    it('respects the restore --buffer-size command-line parameter', function(done) {
-      const bufferSize = 250;
-      process.argv = ['node', 'test', '--buffer-size', bufferSize];
-      const program = parser.parseRestoreArgs();
-      assert.strictEqual(typeof program.bufferSize, 'number');
-      assert.strictEqual(program.bufferSize, bufferSize);
-      done();
-    });
-
-    it('respects the restore --parallelism command-line parameter', function(done) {
-      const parallelism = 5;
-      process.argv = ['node', 'test', '--parallelism', parallelism];
-      const program = parser.parseRestoreArgs();
-      assert.strictEqual(typeof program.parallelism, 'number');
-      assert.strictEqual(program.parallelism, parallelism);
-      done();
-    });
-
-    it('respects the restore --request-timeout command-line parameter', function(done) {
-      const requestTimeout = 10000;
-      process.argv = ['node', 'test', '--request-timeout', requestTimeout];
-      const program = parser.parseRestoreArgs();
-      assert.strictEqual(typeof program.requestTimeout, 'number');
-      assert.strictEqual(program.requestTimeout, requestTimeout);
-      done();
-    });
-
-    it('respects the restore --iam-api-key command-line parameter', function(done) {
-      const key = '123abc-789zyx_CBA987-XYZ321';
-      process.argv = ['node', 'test', '--iam-api-key', key];
-      const program = parser.parseRestoreArgs();
-      assert.strictEqual(typeof program.iamApiKey, 'string');
-      assert.strictEqual(program.iamApiKey, key);
-      done();
-    });
-
-    it('respects the restore --quiet command-line parameter', function(done) {
-      process.argv = ['node', 'test', '--quiet'];
-      const program = parser.parseRestoreArgs();
-      assert.strictEqual(typeof program.quiet, 'boolean');
-      assert.strictEqual(program.quiet, true);
-      done();
-    });
-  });
-});
+const fs = require('fs');
+const stream = require('stream');
+const liner = require('./liner.js');
+
+const onLine = function(onCommand, batches) {
+  const change = new stream.Transform({ objectMode: true });
+  change._transform = function(line, encoding, done) {
+    if (line && line[0] === ':') {
+      const obj = {
+        command: null,
+        batch: null,
+        docs: []
+      };
+
+      let matches;
+
+      // extract command
+      matches = line.match(/^:([a-z_]+) ?/);
+      if (matches) {
+        obj.command = matches[1];
+      }
+
+      // extract batch
+      matches = line.match(/ batch([0-9]+)/);
+      if (matches) {
+        obj.batch = parseInt(matches[1]);
+      }
+
+      // if this is one we want
+      if (obj.command === 't' && batches.indexOf(obj.batch) > -1) {
+        const json = line.replace(/^.* batch[0-9]+ /, '').trim();
+        obj.docs = JSON.parse(json);
+        onCommand(obj);
+      }
+    }
+    done();
+  };
+  return change;
+};
+
+module.exports = function(log, batches, callback) {
+  // our sense of state
+  const retval = { };
+
+  // called with each line from the log file
+  const onCommand = function(obj) {
+    retval[obj.batch] = obj;
+  };
+
+  // stream through the previous log file
+  fs.createReadStream(log)
+    .pipe(liner())
+    .pipe(onLine(onCommand, batches))
+    .on('error', function(err) {
+      callback(err);
+    })
+    .on('finish', function() {
+      callback(null, retval);
+    });
+};