@cloudant/couchbackup 2.9.16 → 2.9.17-SNAPSHOT.185

package/.scannerwork/scanner-report/source-49.txt

@@ -1,4 +1,4 @@
-// Copyright © 2017, 2021 IBM Corp. All rights reserved.
+// Copyright © 2017, 2023 IBM Corp. All rights reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -11,271 +11,283 @@
 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 // See the License for the specific language governing permissions and
 // limitations under the License.
+
+/* global describe it before after beforeEach */
 'use strict';
 
-const async = require('async');
-const events = require('events');
+const assert = require('assert');
 const fs = require('fs');
-const error = require('./error.js');
-const spoolchanges = require('./spoolchanges.js');
-const logfilesummary = require('./logfilesummary.js');
-const logfilegetbatches = require('./logfilegetbatches.js');
-
-/**
- * Read documents from a database to be backed up.
- *
- * @param {string} db - `@cloudant/cloudant` DB object for source database.
- * @param {number} blocksize - number of documents to download in single request
- * @param {number} parallelism - number of concurrent downloads
- * @param {string} log - path to log file to use
- * @param {boolean} resume - whether to resume from an existing log file
- * @returns EventEmitter with following events:
- *  - `received` - called with a block of documents to write to backup
- *  - `error` - on error
- *  - `finished` - when backup process is finished (either complete or errored)
- */
-module.exports = function(db, options) {
-  const ee = new events.EventEmitter();
-  const start = new Date().getTime(); // backup start time
-  const batchesPerDownloadSession = 50; // max batches to read from log file for download at a time (prevent OOM)
-
-  function proceedWithBackup() {
-    if (options.resume) {
-      // pick up from existing log file from previous run
-      downloadRemainingBatches(options.log, db, ee, start, batchesPerDownloadSession, options.parallelism);
-    } else {
-      // create new log file and process
-      spoolchanges(db, options.log, options.bufferSize, ee, function(err) {
-        if (err) {
-          ee.emit('error', err);
-        } else {
-          downloadRemainingBatches(options.log, db, ee, start, batchesPerDownloadSession, options.parallelism);
-        }
-      });
-    }
+const u = require('./citestutils.js');
+const mockServerPort = +process.env.COUCHBACKUP_MOCK_SERVER_PORT || 7777;
+const { once } = require('node:events');
+const url = `http://localhost:${mockServerPort}`;
+const nock = require('nock');
+const httpProxy = require('http-proxy');
+const Readable = require('stream').Readable;
+
+// Create an infinite stream to read.
+// It just keeps sending a backup line, useful for testing cases of
+// termination while a stream has content remaining (the animaldb backup
+// is too small for that).
+class InfiniteBackupStream extends Readable {
+  constructor(opt) {
+    super(opt);
+    this.contents = Buffer.from('[{"_id":"giraffe","_rev":"3-7665c3e66315ff40616cceef62886bd8","min_weight":830,"min_length":5,"max_weight":1600,"max_length":6,"wiki_page":"http://en.wikipedia.org/wiki/Giraffe","class":"mammal","diet":"herbivore","_revisions":{"start":3,"ids":["7665c3e66315ff40616cceef62886bd8","aaaf10d5a68cdf22d95a5482a0e95549","967a00dff5e02add41819138abb3284d"]}}]\n', 'utf8');
   }
 
-  validateBulkGetSupport(db, function(err) {
-    if (err) {
-      return ee.emit('error', err);
-    } else {
-      proceedWithBackup();
-    }
-  });
+  _read() {
+    let proceed;
+    do {
+      proceed = this.push(this.contents);
+    } while (proceed);
+  }
+}
 
-  return ee;
-};
-
-/**
- * Validate /_bulk_get support for a specified database.
- *
- * @param {string} db - nodejs-cloudant db
- * @param {function} callback - called on completion with signature (err)
- */
-function validateBulkGetSupport(db, callback) {
-  db.service.postBulkGet({ db: db.db, docs: [] }).then(() => { callback(); }).catch(err => {
-    err = error.convertResponseError(err, function(err) {
-      switch (err.status) {
-        case undefined:
-          // There was no status code on the error
-          return err;
-        case 404:
-          return new error.BackupError('BulkGetError', 'Database does not support /_bulk_get endpoint');
-        default:
-          return new error.HTTPError(err);
-      }
-    });
-    callback(err);
+function assertNock() {
+  try {
+    assert.ok(nock.isDone());
+  } catch (err) {
+    console.error('pending mocks: %j', nock.pendingMocks());
+    throw err;
+  }
+}
+
+function testPromiseWithAssertNock(testPromise) {
+  return testPromise.finally(() => {
+    assertNock();
   });
 }
 
-/**
- * Download remaining batches in a log file, splitting batches into sets
- * to avoid enqueueing too many in one go.
- *
- * @param {string} log - log file name to maintain download state
- * @param {string} db - nodejs-cloudant db
- * @param {events.EventEmitter} ee - event emitter to emit received events on
- * @param {time} startTime - start time for backup process
- * @param {number} batchesPerDownloadSession - max batches to enqueue for
- *  download at a time. As batches contain many doc IDs, this helps avoid
- *  exhausting memory.
- * @param {number} parallelism - number of concurrent downloads
- * @returns function to call do download remaining batches with signature
- *  (err, {batches: batch, docs: doccount}) {@see spoolchanges}.
- */
-function downloadRemainingBatches(log, db, ee, startTime, batchesPerDownloadSession, parallelism) {
-  let total = 0; // running total of documents downloaded so far
-  let noRemainingBatches = false;
-
-  // Generate a set of batches (up to batchesPerDownloadSession) to download from the
-  // log file and download them. Set noRemainingBatches to `true` for last batch.
-  function downloadSingleBatchSet(done) {
-    // Fetch the doc IDs for the batches in the current set to
-    // download them.
-    function batchSetComplete(err, data) {
-      if (!err) {
-        total = data.total;
-      }
-      done(err);
-    }
-    function processRetrievedBatches(err, batches) {
-      if (!err) {
-        // process them in parallelised queue
-        processBatchSet(db, parallelism, log, batches, ee, startTime, total, batchSetComplete);
-      } else {
-        batchSetComplete(err);
-      }
-    }
+async function backupHttpError(opts, errorName, errorCode) {
+  const p = u.p(opts, { expectedBackupError: { name: errorName, code: errorCode } });
 
-    readBatchSetIdsFromLogFile(log, batchesPerDownloadSession, function(err, batchSetIds) {
-      if (err) {
-        ee.emit('error', err);
-        // Stop processing changes file for fatal errors
-        noRemainingBatches = true;
-        done();
-      } else {
-        if (batchSetIds.length === 0) {
-          noRemainingBatches = true;
-          return done();
-        }
-        logfilegetbatches(log, batchSetIds, processRetrievedBatches);
-      }
+  // Create a file and attempt a backup to it
+  const output = fs.createWriteStream('/dev/null');
+  return once(output, 'open')
+    .then(() => {
+      return testPromiseWithAssertNock(u.testBackup(p, 'fakenockdb', output));
     });
-  }
+}
 
-  // Return true if all batches in log file have been downloaded
-  function isFinished(callback) { callback(null, noRemainingBatches); }
+async function restoreHttpError(opts, errorName, errorCode) {
+  const q = u.p(opts, { expectedRestoreError: { name: errorName, code: errorCode } });
+  return testPromiseWithAssertNock(u.testRestoreFromFile(q, './test/fixtures/animaldb_expected.json', 'fakenockdb'));
+}
 
-  function onComplete() {
-    ee.emit('finished', { total: total });
-  }
+[{ useApi: true }, { useApi: false }].forEach(function(params) {
+  describe(u.scenario('#unit Fatal errors', params), function() {
+    // These tests do real requests with mocks and if they run slowly
+    // the 2 second default mocha timeout can be insufficient, use 10 s
+    this.timeout(10000);
 
-  async.doUntil(downloadSingleBatchSet, isFinished, onComplete);
-}
+    let processEnvCopy;
+    let proxy;
 
-/**
- * Return a set of uncompleted download batch IDs from the log file.
- *
- * @param {string} log - log file path
- * @param {number} batchesPerDownloadSession - maximum IDs to return
- * @param {function} callback - sign (err, batchSetIds array)
- */
-function readBatchSetIdsFromLogFile(log, batchesPerDownloadSession, callback) {
-  logfilesummary(log, function processSummary(err, summary) {
-    if (!err) {
-      if (!summary.changesComplete) {
-        callback(new error.BackupError('IncompleteChangesInLogFile',
-          'WARNING: Changes did not finish spooling'));
-        return;
-      }
-      if (Object.keys(summary.batches).length === 0) {
-        return callback(null, []);
+    before('Set process data for test', function() {
+      const proxyPort = mockServerPort + 1000;
+      // Copy env and argv so we can reset them after the tests
+      processEnvCopy = JSON.parse(JSON.stringify(process.env));
+
+      // Set up a proxy to point to our nock server because the nock override
+      // isn't visible to the spawned CLI process
+      if (!params.useApi) {
+        proxy = httpProxy.createProxyServer({ target: url }).listen(proxyPort, 'localhost');
+        proxy.on('error', (err, req, res) => {
+          console.log(`Proxy received error ${err}`);
+          res.writeHead(400, {
+            'Content-Type': 'application/json'
+          });
+          res.end(JSON.stringify(err));
+        });
       }
 
-      // batch IDs are the property names of summary.batches
-      const batchSetIds = getPropertyNames(summary.batches, batchesPerDownloadSession);
-      callback(null, batchSetIds);
-    } else {
-      callback(err);
-    }
-  });
-}
+      // setup environment variables
+      process.env.COUCH_URL = (params.useApi) ? url : `http://localhost:${proxyPort}`;
 
-/**
- * Download a set of batches retrieved from a log file. When a download is
- * complete, add a line to the logfile indicating such.
- *
- * @param {any} db - nodejs-cloudant database
- * @param {any} parallelism - number of concurrent requests to make
- * @param {any} log - log file to drive downloads from
- * @param {any} batches - batches to download
- * @param {any} ee - event emitter for progress. This funciton emits
- *  received and error events.
- * @param {any} start - time backup started, to report deltas
- * @param {any} grandtotal - count of documents downloaded prior to this set
- *  of batches
- * @param {any} callback - completion callback, (err, {total: number}).
- */
-function processBatchSet(db, parallelism, log, batches, ee, start, grandtotal, callback) {
-  let hasErrored = false;
-  let total = grandtotal;
-
-  // queue to process the fetch requests in an orderly fashion using _bulk_get
-  const q = async.queue(function(payload, done) {
-    const output = [];
-    const thisBatch = payload.batch;
-    delete payload.batch;
-    delete payload.command;
-
-    function logCompletedBatch(batch) {
-      if (log) {
-        fs.appendFile(log, ':d batch' + thisBatch + '\n', done);
+      nock.emitter.on('no match', (req, opts) => {
+        console.error(`Unmatched nock request ${opts.method} ${opts.protocol}${opts.host}${opts.path}`);
+      });
+    });
+
+    after('Reset process data', function(done) {
+      process.env = processEnvCopy;
+      nock.emitter.removeAllListeners();
+      if (!params.useApi) {
+        proxy.close(done);
       } else {
         done();
       }
-    }
-
-    // do the /db/_bulk_get request
-    db.service.postBulkGet({
-      db: db.db,
-      revs: true,
-      docs: payload.docs
-    }).then(response => {
-      // create an output array with the docs returned
-      response.result.results.forEach(function(d) {
-        if (d.docs) {
-          d.docs.forEach(function(doc) {
-            if (doc.ok) {
-              output.push(doc.ok);
-            }
+    });
+
+    beforeEach('Reset nocks', function() {
+      nock.cleanAll();
+    });
+
+    describe('for backup', function() {
+      it('should terminate when DB does not exist', function() {
+        // Simulate existence check
+        nock(url).head('/fakenockdb').reply(404, { error: 'not_found', reason: 'missing' });
+        return backupHttpError(params, 'DatabaseNotFound', 10);
+      });
+
+      it('should terminate on BulkGetError', function() {
+        // Simulate existence check
+        const n = nock(url).head('/fakenockdb').reply(200);
+        // Simulate _bulk_get not available
+        n.post('/fakenockdb/_bulk_get').reply(404, { error: 'not_found', reason: 'missing' });
+        return backupHttpError(params, 'BulkGetError', 50);
+      });
+
+      it('should terminate on Unauthorized existence check', function() {
+        // Simulate a 401
+        nock(url).head('/fakenockdb').reply(401, { error: 'unauthorized', reason: '_reader access is required for this request' });
+        return backupHttpError(params, 'Unauthorized', 11);
+      });
+
+      it('should terminate on Forbidden no _reader', function() {
+        // Simulate a 403
+        nock(url).head('/fakenockdb').reply(403, { error: 'forbidden', reason: '_reader access is required for this request' });
+        return backupHttpError(params, 'Forbidden', 12);
+      });
+
+      it('should terminate on _bulk_get HTTPFatalError', function() {
+        // Provide a mock complete changes log to allow a resume to skip ahead
+        const p = u.p(params, { opts: { resume: true, log: './test/fixtures/test.log' } });
+        // Allow the existence and _bulk_get checks to pass
+        const n = nock(url).head('/fakenockdb').reply(200);
+        n.post('/fakenockdb/_bulk_get').reply(200, '{"results": []}');
+        // Simulate a fatal HTTP error when trying to fetch docs
+        // Note: 2 outstanding batches, so 2 responses, 1 mock is optional because we can't guarantee timing
+        n.post('/fakenockdb/_bulk_get').query(true).reply(400, { error: 'bad_request', reason: 'testing bad response' });
+        n.post('/fakenockdb/_bulk_get').query(true).optionally().reply(400, { error: 'bad_request', reason: 'testing bad response' });
+        return backupHttpError(p, 'HTTPFatalError', 40);
+      });
+
+      it('should terminate on NoLogFileName', function() {
+        // Don't supply a log file name with resume
+        const p = u.p(params, { opts: { resume: true } });
+        return backupHttpError(p, 'NoLogFileName', 20);
+      });
+
+      it('should terminate on LogDoesNotExist', function() {
+        // Use a non-existent log file
+        const p = u.p(params, { opts: { resume: true, log: './test/fixtures/doesnotexist.log' } });
+        return backupHttpError(p, 'LogDoesNotExist', 21);
+      });
+
+      it('should terminate on IncompleteChangesInLogFile', function() {
+        // Use an incomplete changes log file
+        const p = u.p(params, { opts: { resume: true, log: './test/fixtures/incomplete_changes.log' } });
+        // Allow the existence and _bulk_get checks to pass
+        const n = nock(url).head('/fakenockdb').reply(200);
+        n.post('/fakenockdb/_bulk_get').reply(200, '{"results": []}');
+        // Should fail when it reads the incomplete changes
+        return backupHttpError(p, 'IncompleteChangesInLogFile', 22);
+      });
+
+      it('should terminate on _changes HTTPFatalError', function() {
+        // Allow the existence and _bulk_get checks to pass
+        const n = nock(url).head('/fakenockdb').reply(200);
+        n.post('/fakenockdb/_bulk_get').reply(200, '{"results": []}');
+        // Simulate a fatal HTTP error when trying to fetch docs (note 2 outstanding batches)
+        n.post('/fakenockdb/_changes').query(true).reply(400, { error: 'bad_request', reason: 'testing bad response' });
+        return backupHttpError(params, 'HTTPFatalError', 40);
+      });
+
+      it('should terminate on SpoolChangesError', function() {
+        // Allow the existence and _bulk_get checks to pass
+        const n = nock(url).head('/fakenockdb').reply(200);
+        n.post('/fakenockdb/_bulk_get').reply(200, '{"results": []}');
+        // Simulate a changes without a last_seq
+        n.post('/fakenockdb/_changes').query(true).reply(200,
+          {
+            results: [{
+              seq: '2-g1AAAAEbeJzLYWBgYMlgTmFQSElKzi9KdUhJstTLTS3KLElMT9VLzskvTUnMK9HLSy3JAapkSmRIsv___39WBnMiUy5QgN3MzDIxOdEMWb85dv0gSxThigyN8diS5AAkk-pBFiUyoOkzxKMvjwVIMjQAKaDW_Zh6TQnqPQDRC7I3CwDPDV1k',
+              id: 'badger',
+              changes: [{ rev: '4-51aa94e4b0ef37271082033bba52b850' }]
+            }]
           });
-        }
+        return backupHttpError(params, 'SpoolChangesError', 30);
       });
-      total += output.length;
-      const t = (new Date().getTime() - start) / 1000;
-      ee.emit('received', {
-        batch: thisBatch,
-        data: output,
-        length: output.length,
-        time: t,
-        total: total
-      }, q, logCompletedBatch);
-    }).catch(err => {
-      if (!hasErrored) {
-        hasErrored = true;
-        err = error.convertResponseError(err);
-        // Kill the queue for fatal errors
-        q.kill();
-        ee.emit('error', err);
-      }
-      done();
     });
-  }, parallelism);
 
-  for (const i in batches) {
-    q.push(batches[i]);
-  }
+    describe('for restore', function() {
+      it('should terminate on Unauthorized db existence check', function() {
+        // Simulate a 401
+        nock(url).get('/fakenockdb').reply(401, { error: 'unauthorized', reason: '_reader access is required for this request' });
+        return restoreHttpError(params, 'Unauthorized', 11);
+      });
 
-  q.drain(function() {
-    callback(null, { total: total });
-  });
-}
+      it('should terminate on Forbidden no _writer', function() {
+        // Simulate the DB exists (i.e. you can read it)
+        const n = nock(url).get('/fakenockdb').reply(200, { doc_count: 0, doc_del_count: 0 });
+        // Simulate a 403 trying to write
+        n.post('/fakenockdb/_bulk_docs').reply(403, { error: 'forbidden', reason: '_writer access is required for this request' });
+        return restoreHttpError(params, 'Forbidden', 12);
+      });
 
-/**
- * Returns first N properties on an object.
- *
- * @param {object} obj - object with properties
- * @param {number} count - number of properties to return
- */
-function getPropertyNames(obj, count) {
-  // decide which batch numbers to deal with
-  const batchestofetch = [];
-  let j = 0;
-  for (const i in obj) {
-    batchestofetch.push(parseInt(i));
-    j++;
-    if (j >= count) break;
-  }
-  return batchestofetch;
-}
+      it('should terminate on RestoreDatabaseNotFound', function() {
+        // Simulate the DB does not exist
+        nock(url).get('/fakenockdb').reply(404, { error: 'not_found', reason: 'Database does not exist.' });
+        return restoreHttpError(params, 'DatabaseNotFound', 10);
+      });
+
+      it('should terminate on notEmptyDBErr when database is not empty', function() {
+        // Simulate the DB that does exist and not empty
+        nock(url).get('/fakenockdb').reply(200, { doc_count: 10, doc_del_count: 0 });
+        return restoreHttpError(params, 'DatabaseNotEmpty', 13);
+      });
+
+      it('should terminate on notEmptyDBErr when database is not new', function() {
+        // Simulate the DB that does exist and not new
+        nock(url).get('/fakenockdb').reply(200, { doc_count: 0, doc_del_count: 10 });
+        return restoreHttpError(params, 'DatabaseNotEmpty', 13);
+      });
+
+      it('should terminate on _bulk_docs HTTPFatalError', function() {
+        // Simulate the DB exists
+        const n = nock(url).get('/fakenockdb').reply(200, { doc_count: 0, doc_del_count: 0 });
+        // Use a parallelism of one and mock one response
+        const p = u.p(params, { opts: { parallelism: 1 } });
+        // Simulate a 400 trying to write
+        n.post('/fakenockdb/_bulk_docs').reply(400, { error: 'bad_request', reason: 'testing bad response' });
+        return restoreHttpError(p, 'HTTPFatalError', 40);
+      });
+
+      it('should terminate on _bulk_docs HTTPFatalError from system database', function() {
+        // Simulate that target database exists and is _not_ empty.
+        // This should pass validator as we exclude system databases from the check.
+        const n = nock(url).get('/_replicator').reply(200, { doc_count: 1, doc_del_count: 0 });
+        // Simulate a 400 trying to write
+        n.post('/_replicator/_bulk_docs').reply(400, { error: 'bad_request', reason: 'testing bad response' });
+        // Use a parallelism of one and mock one response
+        const q = u.p(params, { opts: { parallelism: 1 }, expectedRestoreError: { name: 'HTTPFatalError', code: 40 } });
+        return testPromiseWithAssertNock(u.testRestore(q, new InfiniteBackupStream(), '_replicator'));
+      });
+
+      it('should terminate on _bulk_docs HTTPFatalError large stream', function() {
+        // Simulate the DB exists
+        const n = nock(url).get('/fakenockdb').reply(200, { doc_count: 0, doc_del_count: 0 });
+        // Simulate a 400 trying to write
+        // Provide a body function to handle the stream, but allow any body
+        n.post('/fakenockdb/_bulk_docs', function(body) { return true; }).reply(400, { error: 'bad_request', reason: 'testing bad response' });
+        // Use only parallelism 1 so we don't have to mock up loads of responses
+        const q = u.p(params, { opts: { parallelism: 1 }, expectedRestoreError: { name: 'HTTPFatalError', code: 40 } });
+        return testPromiseWithAssertNock(u.testRestore(q, new InfiniteBackupStream(), 'fakenockdb'));
+      });
+
+      it('should terminate on multiple _bulk_docs HTTPFatalError', function() {
+        // Simulate the DB exists
+        const n = nock(url).get('/fakenockdb').reply(200, { doc_count: 0, doc_del_count: 0 });
+        // Simulate a 400 trying to write docs, 5 times because of default parallelism
+        // Provide a body function to handle the stream, but allow any body
+        // Four of the mocks are optional because of parallelism 5 we can't guarantee that the exit will happen
+        // after all 5 requests, but we must get at least one of them
+        n.post('/fakenockdb/_bulk_docs', function(body) { return true; }).reply(400, { error: 'bad_request', reason: 'testing bad response' });
+        n.post('/fakenockdb/_bulk_docs', function(body) { return true; }).times(4).optionally().reply(400, { error: 'bad_request', reason: 'testing bad response' });
+        const q = u.p(params, { opts: { bufferSize: 1 }, expectedRestoreError: { name: 'HTTPFatalError', code: 40 } });
+        return restoreHttpError(q, 'HTTPFatalError', 40);
+      });
+    });
+  });
+});