@cloudant/couchbackup 2.9.16-SNAPSHOT.183 → 2.9.16

package/.scannerwork/scanner-report/source-16.txt

@@ -12,176 +12,282 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-/* global describe it */
+/* global describe it before after beforeEach */
 'use strict';
 
 const assert = require('assert');
-const backup = require('../app.js').backup;
 const fs = require('fs');
+const u = require('./citestutils.js');
+const mockServerPort = +process.env.COUCHBACKUP_MOCK_SERVER_PORT || 7777;
+const { once } = require('node:events');
+const url = `http://localhost:${mockServerPort}`;
 const nock = require('nock');
-const util = require('util');
-const backupPromise = util.promisify(backup);
-
-const goodUrl = 'http://localhost:5984/db';
-// The real validateArgs function of app.js isn't
-// exported - so we call the exported backup method
-// instead. We don't get as far as a real backup when
-// testing error cases. For success cases we nock the
-// goodUrl and
-const validateArgs = async function(url, opts, errorValidationForAssertRejects) {
-  const nullStream = fs.createWriteStream('/dev/null');
-  if (url === goodUrl) {
-    // Nock the goodUrl
-    nock(goodUrl).head('').reply(404, { error: 'not_found', reason: 'missing' });
+const httpProxy = require('http-proxy');
+const Readable = require('stream').Readable;
+
+// Create an infinite stream to read.
+// It just keeps sending a backup line, useful for testing cases of
+// termination while a stream has content remaining (the animaldb backup
+// is too small for that).
+class InfiniteBackupStream extends Readable {
+  constructor(opt) {
+    super(opt);
+    this.contents = Buffer.from('[{"_id":"giraffe","_rev":"3-7665c3e66315ff40616cceef62886bd8","min_weight":830,"min_length":5,"max_weight":1600,"max_length":6,"wiki_page":"http://en.wikipedia.org/wiki/Giraffe","class":"mammal","diet":"herbivore","_revisions":{"start":3,"ids":["7665c3e66315ff40616cceef62886bd8","aaaf10d5a68cdf22d95a5482a0e95549","967a00dff5e02add41819138abb3284d"]}}]\n', 'utf8');
   }
-  return assert.rejects(backupPromise(url, nullStream, opts), errorValidationForAssertRejects);
-};
-
-const validateShallowModeArgs = async function(url, opts, msg) {
-  // We pass assertNoValidationError because for these shallow opts
-  // we are expecting only a stderr warning
-  return validateArgs(url, opts, assertNoValidationError()).then(() => {
-    // Assert the warning message was in stderr
-    assert(capturedStderr.indexOf(msg) > -1, 'Log warning message was not present');
-  });
-};
 
-const stderrWriteFun = process.stderr.write;
-let capturedStderr;
+  _read() {
+    let proceed;
+    do {
+      proceed = this.push(this.contents);
+    } while (proceed);
+  }
+}
 
-function captureStderr() {
-  process.stderr.write = function(string, encoding, fd) {
-    capturedStderr += string;
-  };
+function assertNock() {
+  try {
+    assert.ok(nock.isDone());
+  } catch (err) {
+    console.error('pending mocks: %j', nock.pendingMocks());
+    throw err;
+  }
 }
 
-function releaseStderr() {
-  process.stderr.write = stderrWriteFun;
-  capturedStderr = null;
+function testPromiseWithAssertNock(testPromise) {
+  return testPromise.finally(() => {
+    assertNock();
+  });
 }
 
-// Return a validation object for use with assert.rejects
-function assertErrorMessage(msg) {
-  return { name: 'InvalidOption', message: msg };
+async function backupHttpError(opts, errorName, errorCode) {
+  const p = u.p(opts, { expectedBackupError: { name: errorName, code: errorCode } });
+
+  // Create a file and attempt a backup to it
+  const output = fs.createWriteStream('/dev/null');
+  return once(output, 'open')
+    .then(() => {
+      return testPromiseWithAssertNock(u.testBackup(p, 'fakenockdb', output));
+    });
 }
 
-// For cases where validation should pass we reach a real backup that hits a 404
-// mock for a DatabaseNotFound, so that it is the expected in the case assertNoValidationError
-function assertNoValidationError() { return { name: 'DatabaseNotFound' }; }
+async function restoreHttpError(opts, errorName, errorCode) {
+  const q = u.p(opts, { expectedRestoreError: { name: errorName, code: errorCode } });
+  return testPromiseWithAssertNock(u.testRestoreFromFile(q, './test/fixtures/animaldb_expected.json', 'fakenockdb'));
+}
 
-describe('#unit Validate arguments', function() {
-  it('returns error for invalid URL type', async function() {
-    return validateArgs(true, {}, assertErrorMessage('Invalid URL, must be type string'));
-  });
-  it('returns no error for valid URL type', async function() {
-    return validateArgs(goodUrl, {}, assertNoValidationError());
-  });
-  it('returns error for invalid (no host) URL', async function() {
-    return validateArgs('http://', {}, assertErrorMessage('Invalid URL'));
-  });
-  it('returns error for invalid (no protocol) URL', async function() {
-    return validateArgs('invalid', {}, assertErrorMessage('Invalid URL'));
-  });
-  it('returns error for invalid (wrong protocol) URL', async function() {
-    return validateArgs('ftp://invalid.example.com', {}, assertErrorMessage('Invalid URL protocol.'));
-  });
-  it('returns error for invalid (no path) URL', async function() {
-    return validateArgs('https://invalid.example.com', {}, assertErrorMessage('Invalid URL, missing path element (no database).'));
-  });
-  it('returns error for invalid (no protocol, no host) URL', async function() {
-    return validateArgs('invalid', {}, assertErrorMessage('Invalid URL'));
-  });
-  it('returns error for invalid buffer size type', async function() {
-    return validateArgs(goodUrl, { bufferSize: '123' }, assertErrorMessage('Invalid buffer size option, must be a positive integer in the range (0, MAX_SAFE_INTEGER]'));
-  });
-  it('returns error for zero buffer size', async function() {
-    return validateArgs(goodUrl, { bufferSize: 0 }, assertErrorMessage('Invalid buffer size option, must be a positive integer in the range (0, MAX_SAFE_INTEGER]'));
-  });
-  it('returns error for float buffer size', async function() {
-    return validateArgs(goodUrl, { bufferSize: 1.23 }, assertErrorMessage('Invalid buffer size option, must be a positive integer in the range (0, MAX_SAFE_INTEGER]'));
-  });
-  it('returns no error for valid buffer size type', async function() {
-    return validateArgs(goodUrl, { bufferSize: 123 }, assertNoValidationError());
-  });
-  it('returns error for invalid log type', async function() {
-    return validateArgs(goodUrl, { log: true }, assertErrorMessage('Invalid log option, must be type string'));
-  });
-  it('returns no error for valid log type', async function() {
-    return validateArgs(goodUrl, { log: 'log.txt' }, assertNoValidationError());
-  });
-  it('returns error for invalid mode type', async function() {
-    return validateArgs(goodUrl, { mode: true }, assertErrorMessage('Invalid mode option, must be either "full" or "shallow"'));
-  });
-  it('returns error for invalid mode string', async function() {
-    return validateArgs(goodUrl, { mode: 'foobar' }, assertErrorMessage('Invalid mode option, must be either "full" or "shallow"'));
-  });
-  it('returns no error for valid mode type', async function() {
-    return validateArgs(goodUrl, { mode: 'full' }, assertNoValidationError());
-  });
-  it('returns error for invalid output type', async function() {
-    return validateArgs(goodUrl, { output: true }, assertErrorMessage('Invalid output option, must be type string'));
-  });
-  it('returns no error for valid output type', async function() {
-    return validateArgs(goodUrl, { output: 'output.txt' }, assertNoValidationError());
-  });
-  it('returns error for invalid parallelism type', async function() {
-    return validateArgs(goodUrl, { parallelism: '123' }, assertErrorMessage('Invalid parallelism option, must be a positive integer in the range (0, MAX_SAFE_INTEGER]'));
-  });
-  it('returns error for zero parallelism', async function() {
-    return validateArgs(goodUrl, { parallelism: 0 }, assertErrorMessage('Invalid parallelism option, must be a positive integer in the range (0, MAX_SAFE_INTEGER]'));
-  });
-  it('returns error for float parallelism', async function() {
-    return validateArgs(goodUrl, { parallelism: 1.23 }, assertErrorMessage('Invalid parallelism option, must be a positive integer in the range (0, MAX_SAFE_INTEGER]'));
-  });
-  it('returns no error for valid parallelism type', async function() {
-    return validateArgs(goodUrl, { parallelism: 123 }, assertNoValidationError());
-  });
-  it('returns error for invalid request timeout type', async function() {
-    return validateArgs(goodUrl, { requestTimeout: '123' }, assertErrorMessage('Invalid request timeout option, must be a positive integer in the range (0, MAX_SAFE_INTEGER]'));
-  });
-  it('returns error for zero request timeout', async function() {
-    return validateArgs(goodUrl, { requestTimeout: 0 }, assertErrorMessage('Invalid request timeout option, must be a positive integer in the range (0, MAX_SAFE_INTEGER]'));
-  });
-  it('returns error for float request timout', async function() {
-    return validateArgs(goodUrl, { requestTimeout: 1.23 }, assertErrorMessage('Invalid request timeout option, must be a positive integer in the range (0, MAX_SAFE_INTEGER]'));
-  });
-  it('returns no error for valid request timeout type', async function() {
-    return validateArgs(goodUrl, { requestTimeout: 123 }, assertNoValidationError());
-  });
-  it('returns error for invalid resume type', async function() {
-    return validateArgs(goodUrl, { resume: 'true' }, assertErrorMessage('Invalid resume option, must be type boolean'));
-  });
-  it('returns no error for valid resume type', async function() {
-    return validateArgs(goodUrl, { resume: false }, assertNoValidationError());
-  });
-  it('returns error for invalid key type', async function() {
-    return validateArgs(goodUrl, { iamApiKey: true }, assertErrorMessage('Invalid iamApiKey option, must be type string'));
-  });
-  it('returns error for key and URL credentials supplied', async function() {
-    return validateArgs('https://a:b@example.com/db', { iamApiKey: 'abc123' }, assertErrorMessage('URL user information must not be supplied when using IAM API key.'));
-  });
-  it('warns for log arg in shallow mode', async function() {
-    captureStderr();
-    return validateShallowModeArgs(goodUrl, { mode: 'shallow', log: 'test' },
-      'the options "log" and "resume" are invalid when using shallow mode.').finally(
-      () => {
-        releaseStderr();
+[{ useApi: true }, { useApi: false }].forEach(function(params) {
+  describe(u.scenario('#unit Fatal errors', params), function() {
+    // These tests do real requests with mocks and if they run slowly
+    // the 2 second default mocha timeout can be insufficient, use 10 s
+    this.timeout(10000);
+
+    let processEnvCopy;
+    let proxy;
+
+    before('Set process data for test', function() {
+      const proxyPort = mockServerPort + 1000;
+      // Copy env and argv so we can reset them after the tests
+      processEnvCopy = JSON.parse(JSON.stringify(process.env));
+
+      // Set up a proxy to point to our nock server because the nock override
+      // isn't visible to the spawned CLI process
+      if (!params.useApi) {
+        proxy = httpProxy.createProxyServer({ target: url }).listen(proxyPort, 'localhost');
+        proxy.on('error', (err, req, res) => {
+          console.log(`Proxy received error ${err}`);
+          res.writeHead(400, {
+            'Content-Type': 'application/json'
+          });
+          res.end(JSON.stringify(err));
+        });
+      }
+
+      // setup environment variables
+      process.env.COUCH_URL = (params.useApi) ? url : `http://localhost:${proxyPort}`;
+
+      nock.emitter.on('no match', (req, opts) => {
+        console.error(`Unmatched nock request ${opts.method} ${opts.protocol}${opts.host}${opts.path}`);
       });
-  });
-  it('warns for resume arg in shallow mode', async function() {
-    captureStderr();
-    return validateShallowModeArgs(goodUrl, { mode: 'shallow', log: 'test', resume: true },
-      'the options "log" and "resume" are invalid when using shallow mode.').finally(
-      () => {
-        releaseStderr();
+    });
+
+    after('Reset process data', function(done) {
+      process.env = processEnvCopy;
+      nock.emitter.removeAllListeners();
+      if (!params.useApi) {
+        proxy.close(done);
+      } else {
+        done();
+      }
+    });
+
+    beforeEach('Reset nocks', function() {
+      nock.cleanAll();
+    });
+
+    describe('for backup', function() {
+      it('should terminate when DB does not exist', function() {
+        // Simulate existence check
+        nock(url).head('/fakenockdb').reply(404, { error: 'not_found', reason: 'missing' });
+        return backupHttpError(params, 'DatabaseNotFound', 10);
       });
-  });
-  it('warns for parallelism arg in shallow mode', async function() {
-    captureStderr();
-    return validateShallowModeArgs(goodUrl, { mode: 'shallow', parallelism: 10 },
-      'the option "parallelism" has no effect when using shallow mode.').finally(
-      () => {
-        releaseStderr();
+
+      it('should terminate on BulkGetError', function() {
+        // Simulate existence check
+        const n = nock(url).head('/fakenockdb').reply(200);
+        // Simulate _bulk_get not available
+        n.post('/fakenockdb/_bulk_get').reply(404, { error: 'not_found', reason: 'missing' });
+        return backupHttpError(params, 'BulkGetError', 50);
+      });
+
+      it('should terminate on Unauthorized existence check', function() {
+        // Simulate a 401
+        nock(url).head('/fakenockdb').reply(401, { error: 'unauthorized', reason: '_reader access is required for this request' });
+        return backupHttpError(params, 'Unauthorized', 11);
+      });
+
+      it('should terminate on Forbidden no _reader', function() {
+        // Simulate a 403
+        nock(url).head('/fakenockdb').reply(403, { error: 'forbidden', reason: '_reader access is required for this request' });
+        return backupHttpError(params, 'Forbidden', 12);
+      });
+
+      it('should terminate on _bulk_get HTTPFatalError', function() {
+        // Provide a mock complete changes log to allow a resume to skip ahead
+        const p = u.p(params, { opts: { resume: true, log: './test/fixtures/test.log' } });
+        // Allow the existence and _bulk_get checks to pass
+        const n = nock(url).head('/fakenockdb').reply(200);
+        n.post('/fakenockdb/_bulk_get').reply(200, '{"results": []}');
+        // Simulate a fatal HTTP error when trying to fetch docs
+        // Note: 2 outstanding batches, so 2 responses, 1 mock is optional because we can't guarantee timing
+        n.post('/fakenockdb/_bulk_get').query(true).reply(400, { error: 'bad_request', reason: 'testing bad response' });
+        n.post('/fakenockdb/_bulk_get').query(true).optionally().reply(400, { error: 'bad_request', reason: 'testing bad response' });
+        return backupHttpError(p, 'HTTPFatalError', 40);
+      });
+
+      it('should terminate on NoLogFileName', function() {
+        // Don't supply a log file name with resume
+        const p = u.p(params, { opts: { resume: true } });
+        return backupHttpError(p, 'NoLogFileName', 20);
+      });
+
+      it('should terminate on LogDoesNotExist', function() {
+        // Use a non-existent log file
+        const p = u.p(params, { opts: { resume: true, log: './test/fixtures/doesnotexist.log' } });
+        return backupHttpError(p, 'LogDoesNotExist', 21);
+      });
+
+      it('should terminate on IncompleteChangesInLogFile', function() {
+        // Use an incomplete changes log file
+        const p = u.p(params, { opts: { resume: true, log: './test/fixtures/incomplete_changes.log' } });
+        // Allow the existence and _bulk_get checks to pass
+        const n = nock(url).head('/fakenockdb').reply(200);
+        n.post('/fakenockdb/_bulk_get').reply(200, '{"results": []}');
+        // Should fail when it reads the incomplete changes
+        return backupHttpError(p, 'IncompleteChangesInLogFile', 22);
+      });
+
+      it('should terminate on _changes HTTPFatalError', function() {
+        // Allow the existence and _bulk_get checks to pass
+        const n = nock(url).head('/fakenockdb').reply(200);
+        n.post('/fakenockdb/_bulk_get').reply(200, '{"results": []}');
+        // Simulate a fatal HTTP error when trying to fetch docs (note 2 outstanding batches)
+        n.post('/fakenockdb/_changes').query(true).reply(400, { error: 'bad_request', reason: 'testing bad response' });
+        return backupHttpError(params, 'HTTPFatalError', 40);
+      });
+
+      it('should terminate on SpoolChangesError', function() {
+        // Allow the existence and _bulk_get checks to pass
+        const n = nock(url).head('/fakenockdb').reply(200);
+        n.post('/fakenockdb/_bulk_get').reply(200, '{"results": []}');
+        // Simulate a changes without a last_seq
+        n.post('/fakenockdb/_changes').query(true).reply(200,
+          {
+            results: [{
+              seq: '2-g1AAAAEbeJzLYWBgYMlgTmFQSElKzi9KdUhJstTLTS3KLElMT9VLzskvTUnMK9HLSy3JAapkSmRIsv___39WBnMiUy5QgN3MzDIxOdEMWb85dv0gSxThigyN8diS5AAkk-pBFiUyoOkzxKMvjwVIMjQAKaDW_Zh6TQnqPQDRC7I3CwDPDV1k',
+              id: 'badger',
+              changes: [{ rev: '4-51aa94e4b0ef37271082033bba52b850' }]
+            }]
+          });
+        return backupHttpError(params, 'SpoolChangesError', 30);
+      });
+    });
+
+    describe('for restore', function() {
+      it('should terminate on Unauthorized db existence check', function() {
+        // Simulate a 401
+        nock(url).get('/fakenockdb').reply(401, { error: 'unauthorized', reason: '_reader access is required for this request' });
+        return restoreHttpError(params, 'Unauthorized', 11);
+      });
+
+      it('should terminate on Forbidden no _writer', function() {
+        // Simulate the DB exists (i.e. you can read it)
+        const n = nock(url).get('/fakenockdb').reply(200, { doc_count: 0, doc_del_count: 0 });
+        // Simulate a 403 trying to write
+        n.post('/fakenockdb/_bulk_docs').reply(403, { error: 'forbidden', reason: '_writer access is required for this request' });
+        return restoreHttpError(params, 'Forbidden', 12);
+      });
+
+      it('should terminate on RestoreDatabaseNotFound', function() {
+        // Simulate the DB does not exist
+        nock(url).get('/fakenockdb').reply(404, { error: 'not_found', reason: 'Database does not exist.' });
+        return restoreHttpError(params, 'DatabaseNotFound', 10);
+      });
+
+      it('should terminate on notEmptyDBErr when database is not empty', function() {
+        // Simulate the DB that does exist and not empty
+        nock(url).get('/fakenockdb').reply(200, { doc_count: 10, doc_del_count: 0 });
+        return restoreHttpError(params, 'DatabaseNotEmpty', 13);
+      });
+
+      it('should terminate on notEmptyDBErr when database is not new', function() {
+        // Simulate the DB that does exist and not new
+        nock(url).get('/fakenockdb').reply(200, { doc_count: 0, doc_del_count: 10 });
+        return restoreHttpError(params, 'DatabaseNotEmpty', 13);
+      });
+
+      it('should terminate on _bulk_docs HTTPFatalError', function() {
+        // Simulate the DB exists
+        const n = nock(url).get('/fakenockdb').reply(200, { doc_count: 0, doc_del_count: 0 });
+        // Use a parallelism of one and mock one response
+        const p = u.p(params, { opts: { parallelism: 1 } });
+        // Simulate a 400 trying to write
+        n.post('/fakenockdb/_bulk_docs').reply(400, { error: 'bad_request', reason: 'testing bad response' });
+        return restoreHttpError(p, 'HTTPFatalError', 40);
+      });
+
+      it('should terminate on _bulk_docs HTTPFatalError from system database', function() {
+        // Simulate that target database exists and is _not_ empty.
+        // This should pass validator as we exclude system databases from the check.
+        const n = nock(url).get('/_replicator').reply(200, { doc_count: 1, doc_del_count: 0 });
+        // Simulate a 400 trying to write
+        n.post('/_replicator/_bulk_docs').reply(400, { error: 'bad_request', reason: 'testing bad response' });
+        // Use a parallelism of one and mock one response
+        const q = u.p(params, { opts: { parallelism: 1 }, expectedRestoreError: { name: 'HTTPFatalError', code: 40 } });
+        return testPromiseWithAssertNock(u.testRestore(q, new InfiniteBackupStream(), '_replicator'));
+      });
+
+      it('should terminate on _bulk_docs HTTPFatalError large stream', function() {
+        // Simulate the DB exists
+        const n = nock(url).get('/fakenockdb').reply(200, { doc_count: 0, doc_del_count: 0 });
+        // Simulate a 400 trying to write
+        // Provide a body function to handle the stream, but allow any body
+        n.post('/fakenockdb/_bulk_docs', function(body) { return true; }).reply(400, { error: 'bad_request', reason: 'testing bad response' });
+        // Use only parallelism 1 so we don't have to mock up loads of responses
+        const q = u.p(params, { opts: { parallelism: 1 }, expectedRestoreError: { name: 'HTTPFatalError', code: 40 } });
+        return testPromiseWithAssertNock(u.testRestore(q, new InfiniteBackupStream(), 'fakenockdb'));
+      });
+
+      it('should terminate on multiple _bulk_docs HTTPFatalError', function() {
+        // Simulate the DB exists
+        const n = nock(url).get('/fakenockdb').reply(200, { doc_count: 0, doc_del_count: 0 });
+        // Simulate a 400 trying to write docs, 5 times because of default parallelism
+        // Provide a body function to handle the stream, but allow any body
+        // Four of the mocks are optional because of parallelism 5 we can't guarantee that the exit will happen
+        // after all 5 requests, but we must get at least one of them
+        n.post('/fakenockdb/_bulk_docs', function(body) { return true; }).reply(400, { error: 'bad_request', reason: 'testing bad response' });
+        n.post('/fakenockdb/_bulk_docs', function(body) { return true; }).times(4).optionally().reply(400, { error: 'bad_request', reason: 'testing bad response' });
+        const q = u.p(params, { opts: { bufferSize: 1 }, expectedRestoreError: { name: 'HTTPFatalError', code: 40 } });
+        return restoreHttpError(q, 'HTTPFatalError', 40);
       });
+    });
   });
 });