@cloudant/couchbackup 2.9.16-SNAPSHOT.183 → 2.9.16

package/.scannerwork/scanner-report/source-7.txt

@@ -1,4 +1,4 @@
-// Copyright © 2017, 2021 IBM Corp. All rights reserved.
+// Copyright © 2017, 2023 IBM Corp. All rights reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -11,404 +11,181 @@
 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 // See the License for the specific language governing permissions and
 // limitations under the License.
-'use strict';
-
-/**
- * CouchBackup module.
- * @module couchbackup
- * @see module:couchbackup
- */
-
-const backupFull = require('./includes/backup.js');
-const defaults = require('./includes/config.js').apiDefaults;
-const error = require('./includes/error.js');
-const request = require('./includes/request.js');
-const restoreInternal = require('./includes/restore.js');
-const backupShallow = require('./includes/shallowbackup.js');
-const debug = require('debug')('couchbackup:app');
-const events = require('events');
-const fs = require('fs');
-const URL = require('url').URL;
-
-/**
- * Test for a positive, safe integer.
- *
- * @param {object} x - Object under test.
- */
-function isSafePositiveInteger(x) {
-  // https://developer.mozilla.org/en/docs/Web/JavaScript/Reference/Global_Objects/Number/MAX_SAFE_INTEGER
-  const MAX_SAFE_INTEGER = Number.MAX_SAFE_INTEGER || 9007199254740991;
-  // Is it a number?
-  return Object.prototype.toString.call(x) === '[object Number]' &&
-    // Is it an integer?
-    x % 1 === 0 &&
-    // Is it positive?
-    x > 0 &&
-    // Is it less than the maximum safe integer?
-    x <= MAX_SAFE_INTEGER;
-}
-
-/**
- * Validate arguments.
- *
- * @param {object} url - URL of database.
- * @param {object} opts - Options.
- * @param {function} cb - Callback to be called on error.
- */
-function validateArgs(url, opts, cb) {
-  if (typeof url !== 'string') {
-    cb(new error.BackupError('InvalidOption', 'Invalid URL, must be type string'), null);
-    return;
-  }
-  if (opts && typeof opts.bufferSize !== 'undefined' && !isSafePositiveInteger(opts.bufferSize)) {
-    cb(new error.BackupError('InvalidOption', 'Invalid buffer size option, must be a positive integer in the range (0, MAX_SAFE_INTEGER]'), null);
-    return;
-  }
-  if (opts && typeof opts.iamApiKey !== 'undefined' && typeof opts.iamApiKey !== 'string') {
-    cb(new error.BackupError('InvalidOption', 'Invalid iamApiKey option, must be type string'), null);
-    return;
-  }
-  if (opts && typeof opts.log !== 'undefined' && typeof opts.log !== 'string') {
-    cb(new error.BackupError('InvalidOption', 'Invalid log option, must be type string'), null);
-    return;
-  }
-  if (opts && typeof opts.mode !== 'undefined' && ['full', 'shallow'].indexOf(opts.mode) === -1) {
-    cb(new error.BackupError('InvalidOption', 'Invalid mode option, must be either "full" or "shallow"'), null);
-    return;
-  }
-  if (opts && typeof opts.output !== 'undefined' && typeof opts.output !== 'string') {
-    cb(new error.BackupError('InvalidOption', 'Invalid output option, must be type string'), null);
-    return;
-  }
-  if (opts && typeof opts.parallelism !== 'undefined' && !isSafePositiveInteger(opts.parallelism)) {
-    cb(new error.BackupError('InvalidOption', 'Invalid parallelism option, must be a positive integer in the range (0, MAX_SAFE_INTEGER]'), null);
-    return;
-  }
-  if (opts && typeof opts.requestTimeout !== 'undefined' && !isSafePositiveInteger(opts.requestTimeout)) {
-    cb(new error.BackupError('InvalidOption', 'Invalid request timeout option, must be a positive integer in the range (0, MAX_SAFE_INTEGER]'), null);
-    return;
-  }
-  if (opts && typeof opts.resume !== 'undefined' && typeof opts.resume !== 'boolean') {
-    cb(new error.BackupError('InvalidOption', 'Invalid resume option, must be type boolean'), null);
-    return;
-  }
 
-  // Validate URL and ensure no auth if using key
-  try {
-    const urlObject = new URL(url);
-    // We require a protocol, host and path (for db), fail if any is missing.
-    if (urlObject.protocol !== 'https:' && urlObject.protocol !== 'http:') {
-      cb(new error.BackupError('InvalidOption', 'Invalid URL protocol.'));
-      return;
-    }
-    if (!urlObject.pathname || urlObject.pathname === '/') {
-      cb(new error.BackupError('InvalidOption', 'Invalid URL, missing path element (no database).'));
-      return;
-    }
-    if (opts && opts.iamApiKey && (urlObject.username || urlObject.password)) {
-      cb(new error.BackupError('InvalidOption', 'URL user information must not be supplied when using IAM API key.'));
-      return;
-    }
-  } catch (err) {
-    cb(error.wrapPossibleInvalidUrlError(err));
-    return;
-  }
+/* global describe it */
+'use strict';
 
-  // Perform validation of invalid options for shallow mode and WARN
-  // We don't error for backwards compatibility with scripts that may have been
-  // written passing complete sets of options through
-  if (opts && opts.mode === 'shallow') {
-    if (opts.log || opts.resume) {
-      console.warn('WARNING: the options "log" and "resume" are invalid when using shallow mode.');
-    }
-    if (opts.parallelism) {
-      console.warn('WARNING: the option "parallelism" has no effect when using shallow mode.');
-    }
-  }
+const assert = require('assert');
+const nock = require('nock');
+const request = require('../includes/request.js');
+const changes = require('../includes/spoolchanges.js');
 
-  if (opts && opts.resume) {
-    if (!opts.log) {
-      // This is the second place we check for the presence of the log option in conjunction with resume
-      // It has to be here for the API case
-      cb(new error.BackupError('NoLogFileName', 'To resume a backup, a log file must be specified'), null);
-      return;
-    } else if (!fs.existsSync(opts.log)) {
-      cb(new error.BackupError('LogDoesNotExist', 'To resume a backup, the log file must exist'), null);
-      return;
-    }
-  }
-  return true;
-}
+const url = 'http://localhost:7777';
+const dbName = 'fakenockdb';
+const longTestTimeout = 3000;
 
-function addEventListener(indicator, emitter, event, f) {
-  emitter.on(event, function(...args) {
-    if (!indicator.errored) {
-      if (event === 'error') indicator.errored = true;
-      f(...args);
-    }
-  });
-}
+const db = request.client(`${url}/${dbName}`, { parallelism: 1 });
 
-/*
-  Check the backup database exists and that the credentials used have
-  visibility. Callback with a fatal error if there is a problem with the DB.
-  @param {string} db - database object
-  @param {function(err)} callback - error is undefined if DB exists
-*/
-function proceedIfBackupDbValid(db, callback) {
-  db.service.headDatabase({ db: db.db }).then(() => callback()).catch(err => {
-    err = error.convertResponseError(err, err => parseIfDbValidResponseError(db, err));
-    callback(err);
+const seqSuffix = Buffer.alloc(124, 'abc123').toString('base64');
+function provideChanges(batchSize, totalChanges, fullResponse = false) {
+  let pending = totalChanges;
+  const sparseResultsArray = Array(batchSize).fill({
+    seq: null,
+    id: 'doc',
+    changes: [{ rev: '1-abcdef0123456789abcdef0123456789' }]
   });
-}
-
-/*
-  Check that the restore database exists, is new and is empty. Also verify that the credentials used have
-  visibility. Callback with a fatal error if there is a problem with the DB.
-  @param {string} db - database object
-  @param {function(err)} callback - error is undefined if DB exists, new and empty
-*/
-function proceedIfRestoreDbValid(db, callback) {
-  db.service.getDatabaseInformation({ db: db.db }).then(response => {
-    const { doc_count: docCount, doc_del_count: deletedDocCount } = response.result;
-    // The system databases can have a validation ddoc(s) injected in them on creation.
-    // This sets the doc count off, so we just complitely exclude the system databases from this check.
-    // The assumption here is that users restoring system databases know what they are doing.
-    if (!db.db.startsWith('_') && (docCount !== 0 || deletedDocCount !== 0)) {
-      const notEmptyDBErr = new Error(`Target database ${db.url}${db.db} is not empty.`);
-      notEmptyDBErr.name = 'DatabaseNotEmpty';
-      callback(notEmptyDBErr);
-    } else {
-      callback();
-    }
-  }).catch(err => {
-    err = error.convertResponseError(err, err => parseIfDbValidResponseError(db, err));
-    callback(err);
-  });
-}
-
-/*
-  Convert the database validation response error to a special DatabaseNotFound error
-  in case the database is missing. Otherwise delegate to the default error factory.
-  @param {object} db - database object
-  @param {object} err - HTTP response error
-*/
-function parseIfDbValidResponseError(db, err) {
-  if (err && err.status === 404) {
-    // Override the error type and message for the DB not found case
-    const msg = `Database ${db.url}` +
-    `${db.db} does not exist. ` +
-    'Check the URL and database name have been specified correctly.';
-    const noDBErr = new Error(msg);
-    noDBErr.name = 'DatabaseNotFound';
-    return noDBErr;
-  }
-  // Delegate to the default error factory if it wasn't a 404
-  return error.convertResponseError(err);
-}
-
-module.exports = {
-
-  /**
-   * Backup a Cloudant database to a stream.
-   *
-   * @param {string} srcUrl - URL of database to backup.
-   * @param {stream.Writable} targetStream - Stream to write content to.
-   * @param {object} opts - Backup options.
-   * @param {number} [opts.parallelism=5] - Number of parallel HTTP requests to use.
-   * @param {number} [opts.bufferSize=500] - Number of documents per batch request.
-   * @param {number} [opts.requestTimeout=120000] - Milliseconds to wait before retrying a HTTP request.
-   * @param {string} [opts.iamApiKey] - IAM API key to use to access Cloudant database.
-   * @param {string} [opts.log] - Log file name. Default uses a temporary file.
-   * @param {boolean} [opts.resume] - Whether to resume from existing log.
-   * @param {string} [opts.mode=full] - Use `full` or `shallow` mode.
-   * @param {backupRestoreCallback} callback - Called on completion.
-   */
-  backup: function(srcUrl, targetStream, opts, callback) {
-    const listenerErrorIndicator = { errored: false };
-    if (typeof callback === 'undefined' && typeof opts === 'function') {
-      callback = opts;
-      opts = {};
-    }
-    if (!validateArgs(srcUrl, opts, callback)) {
-      // bad args, bail
-      return;
-    }
-
-    // if there is an error writing to the stream, call the completion
-    // callback with the error set
-    addEventListener(listenerErrorIndicator, targetStream, 'error', function(err) {
-      debug('Error ' + JSON.stringify(err));
-      if (callback) callback(err);
+  nock(url)
+    .post(`/${dbName}/_changes`)
+    .query(true)
+    .times(totalChanges / batchSize + (totalChanges % batchSize > 0 ? 1 : 0))
+    .reply(200, (uri, requestBody) => {
+      pending -= batchSize;
+      const lastSeq = (totalChanges - pending);
+      const seq = lastSeq - batchSize;
+      return {
+        results: fullResponse
+          ? Array.from(Array(batchSize), (_, i) => {
+            return {
+              seq: `${seq + i}-${seqSuffix}`,
+              id: `doc${seq + i}`,
+              changes: [{ rev: '1-abcdef0123456789abcdef0123456789' }]
+            };
+          })
+          : sparseResultsArray,
+        pending: pending,
+        last_seq: `${lastSeq}-abc`
+      };
     });
+}
 
-    opts = Object.assign({}, defaults(), opts);
-
-    const ee = new events.EventEmitter();
-
-    // Set up the DB client
-    const backupDB = request.client(srcUrl, opts);
-
-    // Validate the DB exists, before proceeding to backup
-    proceedIfBackupDbValid(backupDB, function(err) {
-      if (err) {
-        if (err.name === 'DatabaseNotFound') {
-          err.message = `${err.message} Ensure the backup source database exists.`;
-        }
-        // Didn't exist, or another fatal error, exit
-        callback(err);
-        return;
-      }
-      let backup = null;
-      if (opts.mode === 'shallow') {
-        backup = backupShallow;
-      } else { // full mode
-        backup = backupFull;
-      }
-
-      // If resuming write a newline as it's possible one would be missing from
-      // an interruption of the previous backup. If the backup was clean this
-      // will cause an empty line that will be gracefully handled by the restore.
-      if (opts.resume) {
-        targetStream.write('\n');
-      }
-
-      // Get the event emitter from the backup process so we can handle events
-      // before passing them on to the app's event emitter if needed.
-      const internalEE = backup(backupDB, opts);
-      addEventListener(listenerErrorIndicator, internalEE, 'changes', function(batch) {
-        ee.emit('changes', batch);
-      });
-      addEventListener(listenerErrorIndicator, internalEE, 'received', function(obj, q, logCompletedBatch) {
-        // this may be too verbose to have as well as the "backed up" message
-        // debug(' received batch', obj.batch, ' docs: ', obj.total, 'Time', obj.time);
-        // Callback to emit the written event when the content is flushed
-        function writeFlushed() {
-          ee.emit('written', { total: obj.total, time: obj.time, batch: obj.batch });
-          if (logCompletedBatch) {
-            logCompletedBatch(obj.batch);
-          }
-          debug(' backed up batch', obj.batch, ' docs: ', obj.total, 'Time', obj.time);
-        }
-        // Write the received content to the targetStream
-        const continueWriting = targetStream.write(JSON.stringify(obj.data) + '\n',
-          'utf8',
-          writeFlushed);
-        if (!continueWriting) {
-          // The buffer was full, pause the queue to stop the writes until we
-          // get a drain event
-          if (q && !q.paused) {
-            q.pause();
-            targetStream.once('drain', function() {
-              q.resume();
-            });
-          }
+describe('#unit Check spool changes', function() {
+  it('should terminate on request error', async function() {
+    nock(url)
+      .post(`/${dbName}/_changes`)
+      .query(true)
+      .times(3)
+      .replyWithError({ code: 'ECONNRESET', message: 'socket hang up' });
+
+    return new Promise((resolve, reject) => {
+      changes(db, '/dev/null', 500, null, function(err) {
+        try {
+          assert.strictEqual(err.name, 'SpoolChangesError');
+          assert.strictEqual(err.message, `Failed changes request - socket hang up: post ${url}/${dbName}/_changes`);
+          assert.ok(nock.isDone());
+          resolve();
+        } catch (err) {
+          reject(err);
         }
       });
-      // For errors we expect, may or may not be fatal
-      addEventListener(listenerErrorIndicator, internalEE, 'error', function(err) {
-        debug('Error ' + JSON.stringify(err));
-        callback(err);
+    });
+  }).timeout(longTestTimeout);
+
+  it('should terminate on bad HTTP status code response', async function() {
+    nock(url)
+      .post(`/${dbName}/_changes`)
+      .query(true)
+      .times(3)
+      .reply(500, function(uri, requestBody) {
+        this.req.response.statusMessage = 'Internal Server Error';
+        return { error: 'foo', reason: 'bar' };
       });
-      addEventListener(listenerErrorIndicator, internalEE, 'finished', function(obj) {
-        function emitFinished() {
-          debug('Backup complete - written ' + JSON.stringify(obj));
-          const summary = { total: obj.total };
-          ee.emit('finished', summary);
-          if (callback) callback(null, summary);
+    return new Promise((resolve, reject) => {
+      changes(db, '/dev/null', 500, null, function(err) {
+        try {
+          assert.strictEqual(err.name, 'HTTPFatalError');
+          assert.strictEqual(err.message, `500 Internal Server Error: post ${url}/${dbName}/_changes - Error: foo, Reason: bar`);
+          assert.ok(nock.isDone());
+          resolve();
+        } catch (err) {
+          reject(err);
         }
-        if (targetStream === process.stdout) {
-          // stdout cannot emit a finish event so use a final write + callback
-          targetStream.write('', 'utf8', emitFinished);
-        } else {
-          // If we're writing to a file, end the writes and register the
-          // emitFinished function for a callback when the file stream's finish
-          // event is emitted.
-          targetStream.end('', 'utf8', emitFinished);
+      });
+    });
+  }).timeout(longTestTimeout);
+
+  it('should keep collecting changes', async function() {
+    // This test validates that spooling will correctly
+    // continue across multiple requests
+    // (4 batches of 100000 to be precise).
+    // This test might take up to 10 seconds
+    this.timeout(10 * 1000);
+
+    // Use full changes for this test
+    provideChanges(100000, 400000, true);
+    return new Promise((resolve, reject) => {
+      changes(db, '/dev/null', 500, null, function(err) {
+        try {
+          assert.ok(!err);
+          assert.ok(nock.isDone());
+          resolve();
+        } catch (err) {
+          reject(err);
         }
       });
     });
-    return ee;
-  },
-
-  /**
-   * Restore a backup from a stream.
-   *
-   * @param {stream.Readable} srcStream - Stream containing backed up data.
-   * @param {string} targetUrl - Target database.
-   * @param {object} opts - Restore options.
-   * @param {number} opts.parallelism - Number of parallel HTTP requests to use. Default 5.
-   * @param {number} opts.bufferSize - Number of documents per batch request. Default 500.
-   * @param {number} opts.requestTimeout - Milliseconds to wait before retrying a HTTP request. Default 120000.
-   * @param {string} opts.iamApiKey - IAM API key to use to access Cloudant database.
-   * @param {backupRestoreCallback} callback - Called on completion.
-   */
-  restore: function(srcStream, targetUrl, opts, callback) {
-    const listenerErrorIndicator = { errored: false };
-    if (typeof callback === 'undefined' && typeof opts === 'function') {
-      callback = opts;
-      opts = {};
-    }
-    validateArgs(targetUrl, opts, callback);
-    opts = Object.assign({}, defaults(), opts);
-
-    const ee = new events.EventEmitter();
-
-    // Set up the DB client
-    const restoreDB = request.client(targetUrl, opts);
+  });
 
-    // Validate the DB exists, before proceeding to restore
-    proceedIfRestoreDbValid(restoreDB, function(err) {
-      if (err) {
-        if (err.name === 'DatabaseNotFound') {
-          err.message = `${err.message} Create the target database before restoring.`;
-        } else if (err.name === 'DatabaseNotEmpty') {
-          err.message = `${err.message} A target database must be a new and empty database.`;
+  it('should keep collecting sparse changes', async function() {
+    // This test checks that making thousands of requests doesn't
+    // make anything bad happen.
+    // This test might take up to 25 seconds
+    this.timeout(25 * 1000);
+    // Use sparse changes for this test and a batch size of 1
+    provideChanges(1, 2500);
+    return new Promise((resolve, reject) => {
+      changes(db, '/dev/null', 500, null, function(err) {
+        try {
+          assert.ok(!err);
+          assert.ok(nock.isDone());
+          resolve();
+        } catch (err) {
+          reject(err);
         }
-        // Didn't exist, or another fatal error, exit
-        callback(err);
-        return;
-      }
-
-      restoreInternal(
-        restoreDB,
-        opts,
-        srcStream,
-        ee,
-        function(err, writer) {
-          if (err) {
-            callback(err, null);
-            return;
-          }
-          if (writer != null) {
-            addEventListener(listenerErrorIndicator, writer, 'restored', function(obj) {
-              debug(' restored ', obj.total);
-              ee.emit('restored', { documents: obj.documents, total: obj.total });
-            });
-            addEventListener(listenerErrorIndicator, writer, 'error', function(err) {
-              debug('Error ' + JSON.stringify(err));
-              // Only call destroy if it is available on the stream
-              if (srcStream.destroy && srcStream.destroy instanceof Function) {
-                srcStream.destroy();
-              }
-              callback(err);
-            });
-            addEventListener(listenerErrorIndicator, writer, 'finished', function(obj) {
-              debug('restore complete');
-              ee.emit('finished', { total: obj.total });
-              callback(null, obj);
-            });
-          }
+      });
+    });
+  });
+});
+
+describe('Longer spool changes checks', function() {
+  it('#slow should keep collecting changes (25M)', async function() {
+    // This test might take up to 5 minutes
+    this.timeout(5 * 60 * 1000);
+    // Note changes spooling uses a constant batch size, we are setting
+    // a test value here and setting the buffer to match
+    const batch = 100000;
+    // Use sparse changes for this test
+    provideChanges(batch, 25000000);
+    return new Promise((resolve, reject) => {
+      changes(db, '/dev/null', batch, null, function(err) {
+        try {
+          assert.ok(!err);
+          assert.ok(nock.isDone());
+          resolve();
+        } catch (err) {
+          reject(err);
         }
-      );
+      });
     });
-    return ee;
-  }
-};
+  });
 
-/**
- * Backup/restore callback
- * @callback backupRestoreCallback
- * @param {Error} err - Error object if operation failed.
- * @param {object} data - summary data for backup/restore
- */
+  it('#slower should keep collecting changes (500M)', async function() {
+    // This test might take up to 90 minutes
+    this.timeout(90 * 60 * 1000);
+    // Note changes spooling uses a constant batch size, we are setting
+    // a test value here and setting the buffer to match
+    const batch = 1000000;
+    // Use full changes for this test to exercise load
+    provideChanges(batch, 500000000, true);
+    return new Promise((resolve, reject) => {
+      changes(db, '/dev/null', batch, null, function(err) {
+        try {
+          assert.ok(!err);
+          assert.ok(nock.isDone());
+          resolve();
+        } catch (err) {
+          reject(err);
+        }
+      });
+    });
+  });
+});

package/.scannerwork/scanner-report/source-8.txt

@@ -0,0 +1,62 @@
+// Copyright © 2017, 2023 IBM Corp. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+/* global describe it */
+'use strict';
+
+const fs = require('fs');
+const { once } = require('node:events');
+const u = require('./citestutils.js');
+
+[{ useApi: true }, { useApi: false }].forEach(function(params) {
+  describe(u.scenario('Compression tests', params), function() {
+    const p = u.p(params, { compression: true });
+
+    it('should backup animaldb to a compressed file', async function() {
+      // Allow up to 60 s for backup of animaldb
+      u.setTimeout(this, 60);
+      const compressedBackup = `./${this.fileName}`;
+      const output = fs.createWriteStream(compressedBackup);
+      return once(output, 'open')
+        .then(() => {
+          return u.testBackup(p, 'animaldb', output);
+        }).then(() => {
+          return u.assertGzipFile(compressedBackup);
+        });
+    });
+
+    it('should backup and restore animaldb via a compressed file', async function() {
+      // Allow up to 60 s for backup and restore of animaldb
+      u.setTimeout(this, 60);
+      const compressedBackup = `./${this.fileName}`;
+      return u.testBackupAndRestoreViaFile(p, 'animaldb', compressedBackup, this.dbName).then(() => {
+        return u.assertGzipFile(compressedBackup);
+      });
+    });
+
+    it('should backup and restore animaldb via a compressed stream', async function() {
+      // Allow up to 60 s for backup and restore of animaldb
+      u.setTimeout(this, 60);
+      return u.testDirectBackupAndRestore(p, 'animaldb', this.dbName);
+    });
+
+    it('should backup and restore largedb2g via a compressed file #slower', async function() {
+      // Takes ~ 25 min using CLI, but sometimes over an hour with API
+      u.setTimeout(this, 180 * 60);
+      const compressedBackup = `./${this.fileName}`;
+      params.compression = true;
+      return u.testBackupAndRestoreViaFile(p, 'largedb2g', compressedBackup, this.dbName);
+    });
+  });
+});