@cloudant/couchbackup 2.9.16 → 2.9.17-SNAPSHOT.185

package/.scannerwork/scanner-report/source-46.txt

@@ -1,4 +1,4 @@
-// Copyright © 2017 IBM Corp. All rights reserved.
+// Copyright © 2017, 2023 IBM Corp. All rights reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -11,31 +11,19 @@
 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 // See the License for the specific language governing permissions and
 // limitations under the License.
-'use strict';
-
-// stolen from http://strongloop.com/strongblog/practical-examples-of-the-new-node-js-streams-api/
-const stream = require('stream');
-
-module.exports = function(onChange) {
-  const change = new stream.Transform({ objectMode: true });
 
-  change._transform = function(line, encoding, done) {
-    let obj = null;
-
-    // one change per line - remove the trailing comma
-    line = line.trim().replace(/,$/, '');
+/* global describe it */
+'use strict';
 
-    // extract thee last_seq at the end of the changes feed
-    if (line.match(/^"last_seq":/)) {
-      line = '{' + line;
-    }
-    try {
-      obj = JSON.parse(line);
-    } catch (e) {
-    }
-    onChange(obj);
-    done();
-  };
+const u = require('./citestutils.js');
 
-  return change;
-};
+[{ useApi: true }, { useApi: false }].forEach(function(params) {
+  describe(u.scenario('#slowest End to end backup and restore', params), function() {
+    // 10 GB is about the largest the CI can handle before getting very upset
+    // about how long things are taking
+    it('should backup and restore largedb10g', async function() {
+      u.setTimeout(this, 350 * 60);
+      return u.testDirectBackupAndRestore(params, 'largedb10g', this.dbName);
+    });
+  });
+});

package/.scannerwork/scanner-report/source-47.txt

@@ -1,4 +1,4 @@
-// Copyright © 2017, 2021 IBM Corp. All rights reserved.
+// Copyright © 2017, 2023 IBM Corp. All rights reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -11,103 +11,52 @@
 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 // See the License for the specific language governing permissions and
 // limitations under the License.
-'use strict';
-
-// fatal errors
-const codes = {
-  Error: 1,
-  InvalidOption: 2,
-  DatabaseNotFound: 10,
-  Unauthorized: 11,
-  Forbidden: 12,
-  DatabaseNotEmpty: 13,
-  NoLogFileName: 20,
-  LogDoesNotExist: 21,
-  IncompleteChangesInLogFile: 22,
-  SpoolChangesError: 30,
-  HTTPFatalError: 40,
-  BulkGetError: 50
-};
-
-class BackupError extends Error {
-  constructor(name, message) {
-    super(message);
-    this.name = name;
-  }
-}
-
-class HTTPError extends BackupError {
-  constructor(responseError, name) {
-    // Special case some names for more useful error messages
-    switch (responseError.status) {
-      case 401:
-        name = 'Unauthorized';
-        break;
-      case 403:
-        name = 'Forbidden';
-        break;
-      default:
-        name = name || 'HTTPFatalError';
-    }
-    super(name, responseError.message);
-  }
-}
 
-// Default function to return an error for HTTP status codes
-// < 400 -> OK
-// 4XX (except 429) -> Fatal
-// 429 & >=500 -> Transient
-function checkResponse(err) {
-  if (err) {
-    // Construct an HTTPError if there is request information on the error
-    // Codes < 400 are considered OK
-    if (err.status >= 400) {
-      return new HTTPError(err);
-    } else {
-      // Send it back again if there was no status code, e.g. a cxn error
-      return augmentMessage(err);
-    }
-  }
-}
-
-function convertResponseError(responseError, errorFactory) {
-  if (!errorFactory) {
-    errorFactory = checkResponse;
-  }
-  return errorFactory(responseError);
-}
-
-function augmentMessage(err) {
-  // For errors that don't have a status code, we are likely looking at a cxn
-  // error.
-  // Try to augment the message with more detail (core puts the code in statusText)
-  if (err && err.statusText) {
-    err.message = `${err.message} ${err.statusText}`;
-  }
-  if (err && err.description) {
-    err.message = `${err.message} ${err.description}`;
-  }
-  return err;
-}
-
-function wrapPossibleInvalidUrlError(err) {
-  if (err.code === 'ERR_INVALID_URL') {
-    // Wrap ERR_INVALID_URL in our own InvalidOption
-    return new BackupError('InvalidOption', err.message);
-  }
-  return err;
-}
+/* global describe it */
+'use strict';
 
-module.exports = {
-  BackupError,
-  HTTPError,
-  wrapPossibleInvalidUrlError,
-  convertResponseError,
-  terminationCallback: function terminationCallback(err, data) {
-    if (err) {
-      console.error(`ERROR: ${err.message}`);
-      process.exitCode = codes[err.name] || 1;
-      process.exit();
-    }
-  }
-};
+const fs = require('fs');
+const { once } = require('node:events');
+const u = require('./citestutils.js');
+
+[{ useApi: true }, { useApi: false }].forEach(function(params) {
+  describe(u.scenario('Compression tests', params), function() {
+    const p = u.p(params, { compression: true });
+
+    it('should backup animaldb to a compressed file', async function() {
+      // Allow up to 60 s for backup of animaldb
+      u.setTimeout(this, 60);
+      const compressedBackup = `./${this.fileName}`;
+      const output = fs.createWriteStream(compressedBackup);
+      return once(output, 'open')
+        .then(() => {
+          return u.testBackup(p, 'animaldb', output);
+        }).then(() => {
+          return u.assertGzipFile(compressedBackup);
+        });
+    });
+
+    it('should backup and restore animaldb via a compressed file', async function() {
+      // Allow up to 60 s for backup and restore of animaldb
+      u.setTimeout(this, 60);
+      const compressedBackup = `./${this.fileName}`;
+      return u.testBackupAndRestoreViaFile(p, 'animaldb', compressedBackup, this.dbName).then(() => {
+        return u.assertGzipFile(compressedBackup);
+      });
+    });
+
+    it('should backup and restore animaldb via a compressed stream', async function() {
+      // Allow up to 60 s for backup and restore of animaldb
+      u.setTimeout(this, 60);
+      return u.testDirectBackupAndRestore(p, 'animaldb', this.dbName);
+    });
+
+    it('should backup and restore largedb2g via a compressed file #slower', async function() {
+      // Takes ~ 25 min using CLI, but sometimes over an hour with API
+      u.setTimeout(this, 180 * 60);
+      const compressedBackup = `./${this.fileName}`;
+      params.compression = true;
+      return u.testBackupAndRestoreViaFile(p, 'largedb2g', compressedBackup, this.dbName);
+    });
+  });
+});

package/.scannerwork/scanner-report/source-48.txt

@@ -1,4 +1,4 @@
-// Copyright © 2017, 2021 IBM Corp. All rights reserved.
+// Copyright © 2018, 2023 IBM Corp. All rights reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -11,154 +11,65 @@
 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 // See the License for the specific language governing permissions and
 // limitations under the License.
-'use strict';
-
-const async = require('async');
-const stream = require('stream');
-const error = require('./error.js');
-const debug = require('debug')('couchbackup:writer');
-
-module.exports = function(db, bufferSize, parallelism, ee) {
-  const writer = new stream.Transform({ objectMode: true });
-  let buffer = [];
-  let written = 0;
-  let linenumber = 0;
-
-  // this is the queue of chunks that are written to the database
-  // the queue's payload will be an array of documents to be written,
-  // the size of the array will be bufferSize. The variable parallelism
-  // determines how many HTTP requests will occur at any one time.
-  const q = async.queue(function(payload, cb) {
-    // if we are restoring known revisions, we need to supply new_edits=false
-    if (payload.docs && payload.docs[0] && payload.docs[0]._rev) {
-      payload.new_edits = false;
-      debug('Using new_edits false mode.');
-    }
-
-    if (!didError) {
-      db.service.postBulkDocs({
-        db: db.db,
-        bulkDocs: payload
-      }).then(response => {
-        if (!response.result || (payload.new_edits === false && response.result.length > 0)) {
-          throw new Error(`Error writing batch with new_edits:${payload.new_edits !== false}` +
-            ` and ${response.result ? response.result.length : 'unavailable'} items`);
-        }
-        written += payload.docs.length;
-        writer.emit('restored', { documents: payload.docs.length, total: written });
-        cb();
-      }).catch(err => {
-        err = error.convertResponseError(err);
-        debug(`Error writing docs ${err.name} ${err.message}`);
-        cb(err, payload);
-      });
-    }
-  }, parallelism);
-
-  let didError = false;
-
-  // write the contents of the buffer to CouchDB in blocks of bufferSize
-  function processBuffer(flush, callback) {
-    function taskCallback(err, payload) {
-      if (err && !didError) {
-        debug(`Queue task failed with error ${err.name}`);
-        didError = true;
-        q.kill();
-        writer.emit('error', err);
-      }
-    }
 
-    if (flush || buffer.length >= bufferSize) {
-      // work through the buffer to break off bufferSize chunks
-      // and feed the chunks to the queue
-      do {
-        // split the buffer into bufferSize chunks
-        const toSend = buffer.splice(0, bufferSize);
-
-        // and add the chunk to the queue
-        debug(`Adding ${toSend.length} to the write queue.`);
-        q.push({ docs: toSend }, taskCallback);
-      } while (buffer.length >= bufferSize);
+/* global describe it */
+'use strict';
 
-      // send any leftover documents to the queue
-      if (flush && buffer.length > 0) {
-        debug(`Adding remaining ${buffer.length} to the write queue.`);
-        q.push({ docs: buffer }, taskCallback);
-      }
+const fs = require('fs');
+const { once } = require('node:events');
+const readline = require('readline');
+const u = require('./citestutils.js');
+const uuid = require('uuid').v4;
 
-      // wait until the queue size falls to a reasonable level
-      async.until(
-        // wait until the queue length drops to twice the paralellism
-        // or until empty on the last write
-        function(callback) {
-          // if we encountered an error, stop this until loop
-          if (didError) {
-            return callback(null, true);
-          }
-          if (flush) {
-            callback(null, q.idle() && q.length() === 0);
-          } else {
-            callback(null, q.length() <= parallelism * 2);
-          }
-        },
-        function(cb) {
-          setTimeout(cb, 20);
-        },
+const params = { useApi: true };
 
-        function() {
-          if (flush && !didError) {
-            writer.emit('finished', { total: written });
-          }
-          // callback when we're happy with the queue size
-          callback();
-        });
-    } else {
-      callback();
-    }
-  }
+describe(u.scenario('Concurrent database backups', params), function() {
+  it('should run concurrent API database backups correctly #slower', async function() {
+    // Allow up to 900 s to backup and compare (it should be much faster)!
+    u.setTimeout(this, 900);
 
-  // take an object
-  writer._transform = function(obj, encoding, done) {
-    // each obj that arrives here is a line from the backup file
-    // it should contain an array of objects. The length of the array
-    // depends on the bufferSize at backup time.
-    linenumber++;
-    if (!didError && obj !== '') {
-      // see if it parses as JSON
-      try {
-        const arr = JSON.parse(obj);
+    const checkForEmptyBatches = async function(fileName) {
+      let foundEmptyBatch = false;
 
-        // if it's an array with a length
-        if (typeof arr === 'object' && arr.length > 0) {
-          // push each document into a buffer
-          buffer = buffer.concat(arr);
+      const rd = readline.createInterface({
+        input: fs.createReadStream(fileName),
+        output: fs.createWriteStream('/dev/null'),
+        terminal: false
+      });
 
-          // pause the stream
-          // it's likely that the speed with which data can be read from disk
-          // may exceed the rate it can be written to CouchDB. To prevent
-          // the whole file being buffered in memory, we pause the stream here.
-          // it is resumed, when processBuffer calls back and we call done()
-          this.pause();
+      rd.on('line', function(line) {
+        if (JSON.parse(line).length === 0) {
+          // Note: Empty batch arrays indicate that the running backup is
+          // incorrectly sharing a log file with another ongoing backup job.
+          foundEmptyBatch = true;
+        }
+      });
 
-          // break the buffer in to bufferSize chunks to be written to the database
-          processBuffer(false, done);
+      rd.on('close', function() {
+        if (foundEmptyBatch) {
+          return Promise.reject(new Error(`Log file '${fileName}' contains empty batches`));
         } else {
-          ee.emit('error', new error.BackupError('BackupFileJsonError', `Error on line ${linenumber} of backup file - not an array`));
-          done();
+          return Promise.resolve();
         }
-      } catch (e) {
-        ee.emit('error', new error.BackupError('BackupFileJsonError', `Error on line ${linenumber} of backup file - cannot parse as JSON`));
-        // Could be an incomplete write that was subsequently resumed
-        done();
-      }
-    } else {
-      done();
-    }
-  };
+      });
+    };
+
+    const backupPromise = async function() {
+      const actualBackup = `./${uuid()}`;
+      const output = fs.createWriteStream(actualBackup);
+      return once(output, 'open').then(() => {
+        return u.testBackup(params, 'largedb1g', output);
+      }).then(() => {
+        return checkForEmptyBatches(actualBackup);
+      });
+    };
+
+    // [1] Run 'largedb1g' database backup
+    const backup1 = backupPromise();
+
+    // [2] Run 'largedb1g' database backup
+    const backup2 = backupPromise();
 
-  // called when we need to flush everything
-  writer._flush = function(done) {
-    processBuffer(true, done);
-  };
-  return writer;
-};
+    return Promise.all([backup1, backup2]);
+  });
+});