@cloudant/couchbackup 2.9.8 → 2.9.9-SNAPSHOT.96

package/.scannerwork/scanner-report/source-35.txt

@@ -1,4 +1,4 @@
-// Copyright © 2017, 2021 IBM Corp. All rights reserved.
+// Copyright © 2017 IBM Corp. All rights reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -12,136 +12,24 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-/* global describe it beforeEach */
+/* global describe it */
 'use strict';
 
 const assert = require('assert');
-const fs = require('fs');
-const nock = require('nock');
-const request = require('../includes/request.js');
-const writer = require('../includes/writer.js');
-const noopEmitter = new (require('events')).EventEmitter();
-const liner = require('../includes/liner.js');
-
-const longTestTimeout = 3000;
-
-// The writer expects a line-by-line stream so this utility function does that
-// processing for the tests (which normally happens in the internal restore
-// function).
-function testLinestream(fixture = './test/fixtures/animaldb_expected.json') {
-  return fs.createReadStream(fixture)
-    .pipe(liner());
-}
-
-describe('#unit Check database restore writer', function() {
-  const dbUrl = 'http://localhost:5984/animaldb';
-  const db = request.client(dbUrl, { parallelism: 1 });
-
-  beforeEach('Reset nocks', function() {
-    nock.cleanAll();
-  });
-
-  it('should complete successfully', function(done) {
-    nock(dbUrl)
-      .post('/_bulk_docs')
-      .reply(200, []); // success
-
-    testLinestream()
-      .pipe(writer(db, 500, 1, noopEmitter))
-      .on('error', function(err) {
-        done(err);
-      })
-      .on('finished', function(data) {
-        assert.strictEqual(data.total, 15);
-        assert.ok(nock.isDone());
-        done();
-      });
-  });
-
-  it('should terminate on a fatal error', function(done) {
-    nock(dbUrl)
-      .post('/_bulk_docs')
-      .reply(401, { error: 'Unauthorized' }); // fatal error
-
-    testLinestream()
-      .pipe(writer(db, 500, 1, noopEmitter))
-      .on('error', function(err) {
-        assert.strictEqual(err.name, 'Unauthorized');
-        assert.strictEqual(err.message, 'Access is denied due to invalid credentials.');
-        assert.ok(nock.isDone());
-        done();
-      });
-  });
-
-  it('should retry on transient errors', function(done) {
-    nock(dbUrl)
-      .post('/_bulk_docs')
-      .reply(429, { error: 'Too Many Requests' }) // transient error
-      .post('/_bulk_docs')
-      .reply(500, { error: 'Internal Server Error' }) // transient error
-      .post('/_bulk_docs')
-      .reply(200, { ok: true }); // third time lucky success
-
-    testLinestream()
-      .pipe(writer(db, 500, 1, noopEmitter))
-      .on('error', function(err) {
-        done(err);
-      })
-      .on('finished', function(data) {
-        assert.strictEqual(data.total, 15);
-        assert.ok(nock.isDone());
-        done();
-      });
-  }).timeout(longTestTimeout);
-
-  it('should fail after 3 transient errors', function(done) {
-    nock(dbUrl)
-      .post('/_bulk_docs')
-      .reply(429, { error: 'Too Many Requests' }) // transient error
-      .post('/_bulk_docs')
-      .reply(500, { error: 'Internal Server Error' }) // transient error
-      .post('/_bulk_docs')
-      .reply(503, { error: 'Service Unavailable' }); // Final transient error
-
-    testLinestream()
-      .pipe(writer(db, 500, 1, noopEmitter))
-      .on('error', function(err) {
-        assert.strictEqual(err.name, 'HTTPFatalError');
-        assert.strictEqual(err.message, `503 : post ${dbUrl}/_bulk_docs - Error: Service Unavailable`);
-        assert.ok(nock.isDone());
-        done();
-      });
-  }).timeout(longTestTimeout);
-
-  it('should restore shallow backups without rev info successfully', function(done) {
-    nock(dbUrl)
-      .post('/_bulk_docs')
-      .reply(200, [{ ok: true, id: 'foo', rev: '1-abc' }]); // success
-
-    testLinestream('./test/fixtures/animaldb_old_shallow.json')
-      .pipe(writer(db, 500, 1, noopEmitter))
-      .on('error', function(err) {
-        done(err);
-      })
-      .on('finished', function(data) {
-        assert.strictEqual(data.total, 11);
-        assert.ok(nock.isDone());
-        done();
-      });
-  });
-
-  it('should get a batch error for non-empty array response with new_edits false', function(done) {
-    nock(dbUrl)
-      .post('/_bulk_docs')
-      .reply(200, [{ id: 'foo', error: 'foo', reason: 'bar' }]);
-
-    testLinestream()
-      .pipe(writer(db, 500, 1, noopEmitter))
-      .on('error', function(err) {
-        assert.strictEqual(err.name, 'Error');
-        assert.strictEqual(err.message, 'Error writing batch with new_edits:false and 1 items');
-        assert.ok(nock.isDone());
-        done();
-      });
+const logfilegetbatches = require('../includes/logfilegetbatches.js');
+
+describe('#unit Fetching batches from a log file', function() {
+  it('should fetch multiple batches correctly', function(done) {
+    logfilegetbatches('./test/fixtures/test.log', [1, 4], function(err, data) {
+      assert.ok(!err);
+      assert.ok(data);
+      assert.strictEqual(typeof data, 'object');
+      assert.strictEqual(Object.keys(data).length, 2);
+      assert.deepStrictEqual(data['1'].docs, [{ id: '6' }, { id: '7' }, { id: '8' }, { id: '9' }, { id: '10' }]);
+      assert.strictEqual(data['1'].batch, 1);
+      assert.deepStrictEqual(data['4'].docs, [{ id: '21' }, { id: '22' }]);
+      assert.strictEqual(data['4'].batch, 4);
+      done();
+    });
   });
 });

package/.scannerwork/scanner-report/source-36.txt

@@ -15,211 +15,54 @@
 /* global describe it */
 'use strict';
 
-const assert = require('assert');
-const backup = require('../app.js').backup;
 const fs = require('fs');
-const nock = require('nock');
+const u = require('./citestutils.js');
 
-const goodUrl = 'http://localhost:5984/db';
-// The real validateArgs function of app.js isn't
-// exported - so we call the exported backup method
-// instead. We don't get as far as a real backup when
-// testing error cases. For success cases we nock the
-// goodUrl and
-const validateArgs = function(url, opts, callback) {
-  const nullStream = fs.createWriteStream('/dev/null');
-  let cb = callback;
-  if (url === goodUrl) {
-    // Nock the goodUrl
-    nock(goodUrl).head('').reply(404, { error: 'not_found', reason: 'missing' });
-    // replace the callback to handle the nock response
-    // to avoid attempting a real backup
-    cb = function(err) {
-      nullStream.end();
-      if (err.name === 'DatabaseNotFound') {
-        // This is what we expect if we reached the backup
-        // This is success for valid args cases.
-        err = null;
-      }
-      callback(err);
-    };
-  }
-  backup(url, nullStream, opts, cb);
-  return true;
-};
+[{ useApi: true }, { useApi: false }].forEach(function(params) {
+  describe(u.scenario('Compression tests', params), function() {
+    const p = u.p(params, { compression: true });
 
-const stderrWriteFun = process.stderr.write;
-let capturedStderr;
-
-function captureStderr() {
-  process.stderr.write = function(string, encoding, fd) {
-    capturedStderr += string;
-  };
-}
-
-function releaseStderr() {
-  process.stderr.write = stderrWriteFun;
-  capturedStderr = null;
-}
+    it('should backup animaldb to a compressed file', function(done) {
+      // Allow up to 60 s for backup of animaldb
+      u.setTimeout(this, 60);
+      const compressedBackup = `./${this.fileName}`;
+      const output = fs.createWriteStream(compressedBackup);
+      output.on('open', function() {
+        u.testBackup(p, 'animaldb', output, function(err) {
+          if (err) {
+            done(err);
+          } else {
+            u.assertGzipFile(compressedBackup, done);
+          }
+        });
+      });
+    });
 
-function assertErrorMessage(msg, done) {
-  return function(err, data) {
-    try {
-      assert(err.message, 'There should be an error message');
-      assert(err.message.indexOf(msg) >= 0);
-      assert(data === null || data === undefined, 'There should only be an error.');
-      done();
-    } catch (e) {
-      done(e);
-    }
-  };
-}
+    it('should backup and restore animaldb via a compressed file', function(done) {
+      // Allow up to 60 s for backup and restore of animaldb
+      u.setTimeout(this, 60);
+      const compressedBackup = `./${this.fileName}`;
+      u.testBackupAndRestoreViaFile(p, 'animaldb', compressedBackup, this.dbName, function(err) {
+        if (err) {
+          done(err);
+        } else {
+          u.assertGzipFile(compressedBackup, done);
+        }
+      });
+    });
 
-function assertNoError(done) {
-  return function(err, data) {
-    try {
-      assert(err === null, 'There should be no error message.');
-      done();
-    } catch (e) {
-      done(e);
-    }
-  };
-}
+    it('should backup and restore animaldb via a compressed stream', function(done) {
+      // Allow up to 60 s for backup and restore of animaldb
+      u.setTimeout(this, 60);
+      u.testDirectBackupAndRestore(p, 'animaldb', this.dbName, done);
+    });
 
-describe('#unit Validate arguments', function() {
-  it('returns error for invalid URL type', function(done) {
-    validateArgs(true, {}, assertErrorMessage('Invalid URL, must be type string', done));
-  });
-  it('returns no error for valid URL type', function(done) {
-    validateArgs(goodUrl, {}, assertNoError(done));
-  });
-  it('returns error for invalid (no host) URL', function(done) {
-    validateArgs('http://', {}, assertErrorMessage('Invalid URL', done));
-  });
-  it('returns error for invalid (no protocol) URL', function(done) {
-    validateArgs('invalid', {}, assertErrorMessage('Invalid URL', done));
-  });
-  it('returns error for invalid (wrong protocol) URL', function(done) {
-    validateArgs('ftp://invalid.example.com', {}, assertErrorMessage('Invalid URL protocol.', done));
-  });
-  it('returns error for invalid (no path) URL', function(done) {
-    validateArgs('https://invalid.example.com', {}, assertErrorMessage('Invalid URL, missing path element (no database).', done));
-  });
-  it('returns error for invalid (no protocol, no host) URL', function(done) {
-    validateArgs('invalid', {}, assertErrorMessage('Invalid URL', done));
-  });
-  it('returns error for invalid buffer size type', function(done) {
-    validateArgs(goodUrl, { bufferSize: '123' }, assertErrorMessage('Invalid buffer size option, must be a positive integer in the range (0, MAX_SAFE_INTEGER]', done));
-  });
-  it('returns error for zero buffer size', function(done) {
-    validateArgs(goodUrl, { bufferSize: 0 }, assertErrorMessage('Invalid buffer size option, must be a positive integer in the range (0, MAX_SAFE_INTEGER]', done));
-  });
-  it('returns error for float buffer size', function(done) {
-    validateArgs(goodUrl, { bufferSize: 1.23 }, assertErrorMessage('Invalid buffer size option, must be a positive integer in the range (0, MAX_SAFE_INTEGER]', done));
-  });
-  it('returns no error for valid buffer size type', function(done) {
-    validateArgs(goodUrl, { bufferSize: 123 }, assertNoError(done));
-  });
-  it('returns error for invalid log type', function(done) {
-    validateArgs(goodUrl, { log: true }, assertErrorMessage('Invalid log option, must be type string', done));
-  });
-  it('returns no error for valid log type', function(done) {
-    validateArgs(goodUrl, { log: 'log.txt' }, assertNoError(done));
-  });
-  it('returns error for invalid mode type', function(done) {
-    validateArgs(goodUrl, { mode: true }, assertErrorMessage('Invalid mode option, must be either "full" or "shallow"', done));
-  });
-  it('returns error for invalid mode string', function(done) {
-    validateArgs(goodUrl, { mode: 'foobar' }, assertErrorMessage('Invalid mode option, must be either "full" or "shallow"', done));
-  });
-  it('returns no error for valid mode type', function(done) {
-    validateArgs(goodUrl, { mode: 'full' }, assertNoError(done));
-  });
-  it('returns error for invalid output type', function(done) {
-    validateArgs(goodUrl, { output: true }, assertErrorMessage('Invalid output option, must be type string', done));
-  });
-  it('returns no error for valid output type', function(done) {
-    validateArgs(goodUrl, { output: 'output.txt' }, assertNoError(done));
-  });
-  it('returns error for invalid parallelism type', function(done) {
-    validateArgs(goodUrl, { parallelism: '123' }, assertErrorMessage('Invalid parallelism option, must be a positive integer in the range (0, MAX_SAFE_INTEGER]', done));
-  });
-  it('returns error for zero parallelism', function(done) {
-    validateArgs(goodUrl, { parallelism: 0 }, assertErrorMessage('Invalid parallelism option, must be a positive integer in the range (0, MAX_SAFE_INTEGER]', done));
-  });
-  it('returns error for float parallelism', function(done) {
-    validateArgs(goodUrl, { parallelism: 1.23 }, assertErrorMessage('Invalid parallelism option, must be a positive integer in the range (0, MAX_SAFE_INTEGER]', done));
-  });
-  it('returns no error for valid parallelism type', function(done) {
-    validateArgs(goodUrl, { parallelism: 123 }, assertNoError(done));
-  });
-  it('returns error for invalid request timeout type', function(done) {
-    validateArgs(goodUrl, { requestTimeout: '123' }, assertErrorMessage('Invalid request timeout option, must be a positive integer in the range (0, MAX_SAFE_INTEGER]', done));
-  });
-  it('returns error for zero request timeout', function(done) {
-    validateArgs(goodUrl, { requestTimeout: 0 }, assertErrorMessage('Invalid request timeout option, must be a positive integer in the range (0, MAX_SAFE_INTEGER]', done));
-  });
-  it('returns error for float request timout', function(done) {
-    validateArgs(goodUrl, { requestTimeout: 1.23 }, assertErrorMessage('Invalid request timeout option, must be a positive integer in the range (0, MAX_SAFE_INTEGER]', done));
-  });
-  it('returns no error for valid request timeout type', function(done) {
-    validateArgs(goodUrl, { requestTimeout: 123 }, assertNoError(done));
-  });
-  it('returns error for invalid resume type', function(done) {
-    validateArgs(goodUrl, { resume: 'true' }, assertErrorMessage('Invalid resume option, must be type boolean', done));
-  });
-  it('returns no error for valid resume type', function(done) {
-    validateArgs(goodUrl, { resume: false }, assertNoError(done));
-  });
-  it('returns error for invalid key type', function(done) {
-    validateArgs(goodUrl, { iamApiKey: true }, assertErrorMessage('Invalid iamApiKey option, must be type string', done));
-  });
-  it('returns error for key and URL credentials supplied', function(done) {
-    validateArgs('https://a:b@example.com/db', { iamApiKey: 'abc123' }, assertErrorMessage('URL user information must not be supplied when using IAM API key.', done));
-  });
-  it('warns for log arg in shallow mode', function(done) {
-    captureStderr();
-    try {
-      validateArgs(goodUrl, { mode: 'shallow', log: 'test' }, function(err, data) {
-        assert.ok(err);
-        assert.ok(!data);
-        assert(capturedStderr.indexOf('The options "log" and "resume" are invalid when using shallow mode.') > -1, 'Log warning message was not present');
-      });
-      done();
-    } catch (e) {
-      done(e);
-    } finally {
-      releaseStderr();
-    }
-  });
-  it('warns for resume arg in shallow mode', function(done) {
-    captureStderr();
-    try {
-      validateArgs(goodUrl, { mode: 'shallow', log: 'test', resume: true }, function(err, data) {
-        assert.ok(err);
-        assert.ok(!data);
-        assert(capturedStderr.indexOf('The options "log" and "resume" are invalid when using shallow mode.') > -1, 'Log warning message was not present');
-      });
-      done();
-    } catch (e) {
-      done(e);
-    } finally {
-      releaseStderr();
-    }
-  });
-  it('warns for parallism arg in shallow mode', function(done) {
-    captureStderr();
-    try {
-      validateArgs(goodUrl, { mode: 'shallow', parallelsim: 10 }, function(err, data) {
-        assert.ok(err);
-        assert.ok(!data);
-        assert(capturedStderr.indexOf('The option "parallelism" has no effect when using shallow mode.') > -1, 'Log warning message was not present');
-      });
-      done();
-    } catch (e) {
-      done(e);
-    } finally {
-      releaseStderr();
-    }
+    it('should backup and restore largedb2g via a compressed file #slower', function(done) {
+      // Takes ~ 25 min using CLI, but sometimes over an hour with API
+      u.setTimeout(this, 180 * 60);
+      const compressedBackup = `./${this.fileName}`;
+      params.compression = true;
+      u.testBackupAndRestoreViaFile(p, 'largedb2g', compressedBackup, this.dbName, done);
+    });
   });
 });

package/.scannerwork/scanner-report/source-37.txt

@@ -12,103 +12,25 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-/* global after before describe */
+/* global describe it */
 'use strict';
 
-const url = require('url');
-const toxy = require('toxy');
-// Import the common hooks
-require('../test/hooks.js');
+delete require.cache[require.resolve('./citestutils.js')];
+const u = require('./citestutils.js');
 
-const tpoisons = toxy.poisons;
-const trules = toxy.rules;
-
-function setupProxy(poison) {
-  const backendUrl = new url.URL(process.env.COUCH_BACKEND_URL);
-  const proxy = toxy({
-    auth: `${backendUrl.username}:${backendUrl.password}`,
-    changeOrigin: true
-  });
-
-  // Forward traffic to DB
-  proxy.forward(process.env.COUCH_BACKEND_URL);
-
-  switch (poison) {
-    case 'normal':
-      // No poisons to add
-      break;
-    case 'bandwidth-limit':
-      // https://github.com/h2non/toxy#bandwidth
-      // Note the implementation of bandwidth is simplistic and the threshold
-      // delay is applied to every write of the buffer, so use the smallest
-      // delay possible and adjust the rate using the bytes size instead.
-      proxy
-        .poison(tpoisons.bandwidth({ bytes: 512, threshold: 1 })); // 0.5 MB/s
-      break;
-    case 'latency':
-      // https://github.com/h2non/toxy#latency
-      proxy
-        .poison(tpoisons.latency({ max: 1500, min: 250 }))
-        .withRule(trules.probability(60));
-      break;
-    case 'slow-read':
-      // https://github.com/h2non/toxy#slow-read
-      // Note this only impacts read of data from requests so only for non-GET
-      // In practice this means that it impacts restore much more than backup
-      // since although backup POSTs to _bulk_get the content is much smaller
-      // than what is POSTed to _bulk_docs for a restore.
-      // Similarly to bandwidth-limit use a 1 ms threshold
-      proxy
-        .poison(tpoisons.slowRead({ chunk: 256, threshold: 1 }))
-        // Slow read for 10 % of the time e.g. 10 ms in every 100
-        .withRule(trules.timeThreshold({ duration: 10, period: 100 }));
-      break;
-    case 'rate-limit':
-      // https://github.com/h2non/toxy#rate-limit
-      // Simulate the Cloudant free plan with 20 lookups ps and 10 writes ps
-      proxy.post('/*/_bulk_get')
-        .poison(tpoisons.rateLimit({ limit: 20, threshold: 1000 }));
-      proxy.post('/*/_bulk_docs')
-        .poison(tpoisons.rateLimit({ limit: 10, threshold: 1000 }));
-      break;
-    default:
-      throw Error('Unknown toxy poison ' + poison);
-  }
-
-  // Catch remaining traffic
-  proxy.all('/*');
-  return proxy;
-}
-
-const poisons = [
-  'normal',
-  'bandwidth-limit',
-  'latency',
-  'slow-read',
-  'rate-limit'
-];
-
-poisons.forEach(function(poison) {
-  describe('unreliable network tests (using toxy poison ' + poison + ')', function() {
-    let proxy;
-
-    before('start toxy server', function() {
-      proxy = setupProxy(poison);
-      console.log('Using toxy poison ' + poison);
-
-      // For these tests COUCH_URL points to the toxy proxy on localhost whereas
-      // COUCH_BACKEND_URL is the real CouchDb instance.
-      const toxyUrl = new url.URL(process.env.COUCH_URL);
-      // Listen on the specified hostname only, so if using localhost we don't
-      // need external connections.
-      proxy.listen(toxyUrl.port, toxyUrl.hostname);
+[{ useApi: true }, { useApi: false }].forEach(function(params) {
+  describe(u.scenario('End to end backup and restore', params), function() {
+    it('should backup and restore animaldb', function(done) {
+      // Allow up to 60 s for backup and restore of animaldb
+      u.setTimeout(this, 60);
+      u.testDirectBackupAndRestore(params, 'animaldb', this.dbName, done);
     });
-
-    after('stop toxy server', function() {
-      proxy.close();
+    it('should backup and restore largedb1g #slow', function(done) {
+      // Allow up to 30 m for backup and restore of largedb1g
+      // This is a long time but when many builds run in parallel it can take a
+      // while to get this done.
+      u.setTimeout(this, 30 * 60);
+      u.testDirectBackupAndRestore(params, 'largedb1g', this.dbName, done);
     });
-
-    delete require.cache[require.resolve('../test/ci_e2e.js')];
-    require('../test/ci_e2e.js');
   });
 });