@cloudant/couchbackup 2.9.10-SNAPSHOT.112 → 2.9.10

package/.scannerwork/scanner-report/source-19.txt

@@ -1,4 +1,4 @@
-// Copyright © 2017, 2021 IBM Corp. All rights reserved.
+// Copyright © 2017 IBM Corp. All rights reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -12,131 +12,103 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-/* global describe it before after */
+/* global after before describe */
 'use strict';
 
-const assert = require('assert');
-const applyEnvVars = require('../includes/config.js').applyEnvironmentVariables;
+const url = require('url');
+const toxy = require('toxy');
+// Import the common hooks
+require('../test/hooks.js');
 
-describe('#unit Configuration', function() {
-  let processEnvCopy;
+const tpoisons = toxy.poisons;
+const trules = toxy.rules;
 
-  before('Save env', function() {
-    // Copy env so we can reset it after the tests
-    processEnvCopy = JSON.parse(JSON.stringify(process.env));
+function setupProxy(poison) {
+  const backendUrl = new url.URL(process.env.COUCH_BACKEND_URL);
+  const proxy = toxy({
+    auth: `${backendUrl.username}:${backendUrl.password}`,
+    changeOrigin: true
   });
 
-  after('Reset env', function() {
-    process.env = processEnvCopy;
-  });
-
-  it('respects the COUCH_URL env variable', function(done) {
-    process.env.COUCH_URL = 'http://user:pass@myurl.com';
-    const config = {};
-    applyEnvVars(config);
-    assert.strictEqual(typeof config.url, 'string');
-    assert.strictEqual(config.url, process.env.COUCH_URL);
-    done();
-  });
-
-  it('respects the COUCH_DATABASE env variable', function(done) {
-    process.env.COUCH_DATABASE = 'mydb';
-    const config = {};
-    applyEnvVars(config);
-    assert.strictEqual(typeof config.db, 'string');
-    assert.strictEqual(config.db, process.env.COUCH_DATABASE);
-    done();
-  });
-
-  it('respects the COUCH_BUFFER_SIZE env variable', function(done) {
-    process.env.COUCH_BUFFER_SIZE = '1000';
-    const config = {};
-    applyEnvVars(config);
-    assert.strictEqual(typeof config.bufferSize, 'number');
-    assert.strictEqual(config.bufferSize, 1000);
-    done();
-  });
-
-  it('respects the COUCH_PARALLELISM env variable', function(done) {
-    process.env.COUCH_PARALLELISM = '20';
-    const config = {};
-    applyEnvVars(config);
-    assert.strictEqual(typeof config.parallelism, 'number');
-    assert.strictEqual(config.parallelism, 20);
-    done();
-  });
-
-  it('respects the COUCH_REQUEST_TIMEOUT env variable', function(done) {
-    process.env.COUCH_REQUEST_TIMEOUT = '10000';
-    const config = {};
-    applyEnvVars(config);
-    assert.strictEqual(typeof config.requestTimeout, 'number');
-    assert.strictEqual(config.requestTimeout, 10000);
-    done();
-  });
-
-  it('respects the CLOUDANT_IAM_API_KEY env variable', function(done) {
-    const key = 'ABC123-ZYX987_cba789-xyz321';
-    process.env.CLOUDANT_IAM_API_KEY = key;
-    const config = {};
-    applyEnvVars(config);
-    assert.strictEqual(typeof config.iamApiKey, 'string');
-    assert.strictEqual(config.iamApiKey, key);
-    done();
-  });
-
-  it('respects the CLOUDANT_IAM_TOKEN_URL env variable', function(done) {
-    const u = 'https://testhost.example:1234/identity/token';
-    process.env.CLOUDANT_IAM_TOKEN_URL = u;
-    const config = {};
-    applyEnvVars(config);
-    assert.strictEqual(typeof config.iamTokenUrl, 'string');
-    assert.strictEqual(config.iamTokenUrl, u);
-    done();
-  });
-
-  it('respects the COUCH_LOG env variable', function(done) {
-    process.env.COUCH_LOG = 'my.log';
-    const config = {};
-    applyEnvVars(config);
-    assert.strictEqual(typeof config.log, 'string');
-    assert.strictEqual(config.log, process.env.COUCH_LOG);
-    done();
-  });
-
-  it('respects the COUCH_RESUME env variable', function(done) {
-    process.env.COUCH_RESUME = 'true';
-    const config = {};
-    applyEnvVars(config);
-    assert.strictEqual(typeof config.resume, 'boolean');
-    assert.strictEqual(config.resume, true);
-    done();
-  });
-
-  it('respects the COUCH_OUTPUT env variable', function(done) {
-    process.env.COUCH_OUTPUT = 'myfile.txt';
-    const config = {};
-    applyEnvVars(config);
-    assert.strictEqual(typeof config.output, 'string');
-    assert.strictEqual(config.output, process.env.COUCH_OUTPUT);
-    done();
-  });
-
-  it('respects the COUCH_MODE env variable', function(done) {
-    process.env.COUCH_MODE = 'shallow';
-    const config = {};
-    applyEnvVars(config);
-    assert.strictEqual(typeof config.mode, 'string');
-    assert.strictEqual(config.mode, 'shallow');
-    done();
-  });
-
-  it('respects the COUCH_QUIET env variable', function(done) {
-    process.env.COUCH_QUIET = 'true';
-    const config = {};
-    applyEnvVars(config);
-    assert.strictEqual(typeof config.quiet, 'boolean');
-    assert.strictEqual(config.quiet, true);
-    done();
+  // Forward traffic to DB
+  proxy.forward(process.env.COUCH_BACKEND_URL);
+
+  switch (poison) {
+    case 'normal':
+      // No poisons to add
+      break;
+    case 'bandwidth-limit':
+      // https://github.com/h2non/toxy#bandwidth
+      // Note the implementation of bandwidth is simplistic and the threshold
+      // delay is applied to every write of the buffer, so use the smallest
+      // delay possible and adjust the rate using the bytes size instead.
+      proxy
+        .poison(tpoisons.bandwidth({ bytes: 512, threshold: 1 })); // 0.5 MB/s
+      break;
+    case 'latency':
+      // https://github.com/h2non/toxy#latency
+      proxy
+        .poison(tpoisons.latency({ max: 1500, min: 250 }))
+        .withRule(trules.probability(60));
+      break;
+    case 'slow-read':
+      // https://github.com/h2non/toxy#slow-read
+      // Note this only impacts read of data from requests so only for non-GET
+      // In practice this means that it impacts restore much more than backup
+      // since although backup POSTs to _bulk_get the content is much smaller
+      // than what is POSTed to _bulk_docs for a restore.
+      // Similarly to bandwidth-limit use a 1 ms threshold
+      proxy
+        .poison(tpoisons.slowRead({ chunk: 256, threshold: 1 }))
+        // Slow read for 10 % of the time e.g. 10 ms in every 100
+        .withRule(trules.timeThreshold({ duration: 10, period: 100 }));
+      break;
+    case 'rate-limit':
+      // https://github.com/h2non/toxy#rate-limit
+      // Simulate the Cloudant free plan with 20 lookups ps and 10 writes ps
+      proxy.post('/*/_bulk_get')
+        .poison(tpoisons.rateLimit({ limit: 20, threshold: 1000 }));
+      proxy.post('/*/_bulk_docs')
+        .poison(tpoisons.rateLimit({ limit: 10, threshold: 1000 }));
+      break;
+    default:
+      throw Error('Unknown toxy poison ' + poison);
+  }
+
+  // Catch remaining traffic
+  proxy.all('/*');
+  return proxy;
+}
+
+const poisons = [
+  'normal',
+  'bandwidth-limit',
+  'latency',
+  'slow-read',
+  'rate-limit'
+];
+
+poisons.forEach(function(poison) {
+  describe('unreliable network tests (using toxy poison ' + poison + ')', function() {
+    let proxy;
+
+    before('start toxy server', function() {
+      proxy = setupProxy(poison);
+      console.log('Using toxy poison ' + poison);
+
+      // For these tests COUCH_URL points to the toxy proxy on localhost whereas
+      // COUCH_BACKEND_URL is the real CouchDb instance.
+      const toxyUrl = new url.URL(process.env.COUCH_URL);
+      // Listen on the specified hostname only, so if using localhost we don't
+      // need external connections.
+      proxy.listen(toxyUrl.port, toxyUrl.hostname);
+    });
+
+    after('stop toxy server', function() {
+      proxy.close();
+    });
+
+    delete require.cache[require.resolve('../test/ci_e2e.js')];
+    require('../test/ci_e2e.js');
   });
 });

package/.scannerwork/scanner-report/source-20.txt

@@ -15,15 +15,21 @@
 /* global describe it */
 'use strict';
 
-const u = require('./citestutils.js');
+const assert = require('assert');
+const logfilegetbatches = require('../includes/logfilegetbatches.js');
 
-[{ useApi: true }, { useApi: false }].forEach(function(params) {
-  describe(u.scenario('#slowest End to end backup and restore', params), function() {
-    // 10 GB is about the largest the CI can handle before getting very upset
-    // about how long things are taking
-    it('should backup and restore largedb10g', function(done) {
-      u.setTimeout(this, 350 * 60);
-      u.testDirectBackupAndRestore(params, 'largedb10g', this.dbName, done);
+describe('#unit Fetching batches from a log file', function() {
+  it('should fetch multiple batches correctly', function(done) {
+    logfilegetbatches('./test/fixtures/test.log', [1, 4], function(err, data) {
+      assert.ok(!err);
+      assert.ok(data);
+      assert.strictEqual(typeof data, 'object');
+      assert.strictEqual(Object.keys(data).length, 2);
+      assert.deepStrictEqual(data['1'].docs, [{ id: '6' }, { id: '7' }, { id: '8' }, { id: '9' }, { id: '10' }]);
+      assert.strictEqual(data['1'].batch, 1);
+      assert.deepStrictEqual(data['4'].docs, [{ id: '21' }, { id: '22' }]);
+      assert.strictEqual(data['4'].batch, 4);
+      done();
     });
   });
 });

package/.scannerwork/scanner-report/source-21.txt

@@ -1,4 +1,4 @@
-// Copyright © 2017 IBM Corp. All rights reserved.
+// Copyright © 2017, 2021 IBM Corp. All rights reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -12,23 +12,233 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-/* global describe it */
+/* global describe it beforeEach */
 'use strict';
 
 const assert = require('assert');
-const logfilesummary = require('../includes/logfilesummary.js');
-
-describe('#unit Fetching summary from the log file', function() {
-  it('should fetch a summary correctly', function(done) {
-    logfilesummary('./test/fixtures/test.log', function(err, data) {
-      assert.ok(!err);
-      assert.ok(data);
-      assert.strictEqual(data.changesComplete, true);
-      assert.strictEqual(typeof data.batches, 'object');
-      assert.strictEqual(Object.keys(data.batches).length, 2);
-      assert.deepStrictEqual(data.batches['1'], true);
-      assert.deepStrictEqual(data.batches['4'], true);
+const nock = require('nock');
+const request = require('../includes/request.js');
+const error = require('../includes/error.js');
+
+const url = 'http://localhost:7777/testdb';
+const db = request.client(url, { parallelism: 1 });
+const timeoutDb = request.client(url, { parallelism: 1, requestTimeout: 500 });
+const longTestTimeout = 3000;
+
+beforeEach('Clean nock', function() {
+  nock.cleanAll();
+});
+
+describe('#unit Check request headers', function() {
+  it('should have a couchbackup user-agent', function(done) {
+    const couch = nock(url)
+      .matchHeader('user-agent', /couchbackup-cloudant\/\d+\.\d+\.\d+(?:-SNAPSHOT)? \(Node.js v\d+\.\d+\.\d+\)/)
+      .head('/good')
+      .reply(200);
+
+    db.service.headDocument({ db: db.db, docId: 'good' }).then(response => {
+      assert.ok(couch.isDone());
+      done();
+    }).catch(err => {
+      done(err);
+    });
+  });
+});
+
+describe('#unit Check request response error callback', function() {
+  it('should not callback with error for 200 response', function(done) {
+    const couch = nock(url)
+      .get('/good')
+      .reply(200, { ok: true });
+
+    db.service.getDocument({ db: db.db, docId: 'good' }).then(response => {
+      assert.ok(response.result);
+      assert.ok(couch.isDone());
+      done();
+    }).catch(err => {
+      err = error.convertResponseError(err);
+      done(err);
+    });
+  });
+
+  it('should callback with error after 3 500 responses', function(done) {
+    const couch = nock(url)
+      .get('/bad')
+      .times(3)
+      .reply(500, function(uri, requestBody) {
+        this.req.response.statusMessage = 'Internal Server Error';
+        return { error: 'foo', reason: 'bar' };
+      });
+
+    db.service.getDocument({ db: db.db, docId: 'bad' }).then(response => {
+      done(new Error('Successful response when error expected.'));
+    }).catch(err => {
+      err = error.convertResponseError(err);
+      assert.strictEqual(err.name, 'HTTPFatalError');
+      assert.strictEqual(err.message, `500 Internal Server Error: get ${url}/bad - Error: foo, Reason: bar`);
+      assert.ok(couch.isDone());
+      done();
+    }).catch(err => {
+      // Handle assertion errors
+      done(err);
+    });
+  }).timeout(longTestTimeout);
+
+  it('should callback with error after 3 POST 503 responses', function(done) {
+    const couch = nock(url)
+      .post('/_bulk_get')
+      .query(true)
+      .times(3)
+      .reply(503, function(uri, requestBody) {
+        this.req.response.statusMessage = 'Service Unavailable';
+        return { error: 'service_unavailable', reason: 'Service unavailable' };
+      });
+
+    db.service.postBulkGet({ db: db.db, revs: true, docs: [] }).then(response => {
+      done(new Error('Successful response when error expected.'));
+    }).catch(err => {
+      err = error.convertResponseError(err);
+      assert.strictEqual(err.name, 'HTTPFatalError');
+      assert.strictEqual(err.message, `503 Service Unavailable: post ${url}/_bulk_get - Error: service_unavailable, Reason: Service unavailable`);
+      assert.ok(couch.isDone());
+      done();
+    }).catch(err => {
+      // Handle assertion errors
+      done(err);
+    });
+  }).timeout(longTestTimeout);
+
+  it('should callback with error after 3 429 responses', function(done) {
+    const couch = nock(url)
+      .get('/bad')
+      .times(3)
+      .reply(429, function(uri, requestBody) {
+        this.req.response.statusMessage = 'Too Many Requests';
+        return { error: 'foo', reason: 'bar' };
+      });
+
+    db.service.getDocument({ db: db.db, docId: 'bad' }).then(response => {
+      done(new Error('Successful response when error expected.'));
+    }).catch(err => {
+      err = error.convertResponseError(err);
+      assert.strictEqual(err.name, 'HTTPFatalError');
+      assert.strictEqual(err.message, `429 Too Many Requests: get ${url}/bad - Error: foo, Reason: bar`);
+      assert.ok(couch.isDone());
+      done();
+    }).catch(err => {
+      // Handle assertion errors
+      done(err);
+    });
+  }).timeout(longTestTimeout);
+
+  it('should callback with fatal error for 404 response', function(done) {
+    const couch = nock(url)
+      .get('/bad')
+      .reply(404, function(uri, requestBody) {
+        this.req.response.statusMessage = 'Not Found';
+        return { error: 'foo', reason: 'bar' };
+      });
+
+    db.service.getDocument({ db: db.db, docId: 'bad' }).then(response => {
+      done(new Error('Successful response when error expected.'));
+    }).catch(err => {
+      err = error.convertResponseError(err);
+      assert.strictEqual(err.name, 'HTTPFatalError');
+      assert.strictEqual(err.message, `404 Not Found: get ${url}/bad - Error: foo, Reason: bar`);
+      assert.ok(couch.isDone());
+      done();
+    }).catch(err => {
+      // Handle assertion errors
+      done(err);
+    });
+  });
+
+  it('should callback with same error for no status code error response', function(done) {
+    const couch = nock(url)
+      .get('/bad')
+      .times(3)
+      .replyWithError('testing badness');
+
+    db.service.getDocument({ db: db.db, docId: 'bad' }).then(response => {
+      done(new Error('Successful response when error expected.'));
+    }).catch(err => {
+      const err2 = error.convertResponseError(err);
+      assert.strictEqual(err, err2);
+      assert.ok(couch.isDone());
       done();
+    }).catch(err => {
+      // Handle assertion errors
+      done(err);
+    });
+  }).timeout(longTestTimeout);
+
+  it('should retry request if HTTP request gets timed out', function(done) {
+    const couch = nock(url)
+      .post('/_bulk_get')
+      .query(true)
+      .delay(1000)
+      .reply(200, { results: { docs: [{ id: '1', ok: { _id: '1' } }] } })
+      .post('/_bulk_get')
+      .query(true)
+      .reply(200, { results: { docs: [{ id: '1', ok: { _id: '1' } }, { id: '2', ok: { _id: '2' } }] } });
+
+    timeoutDb.service.postBulkGet({ db: db.db, revs: true, docs: [] }).then(response => {
+      assert.ok(response);
+      assert.ok(response.result);
+      assert.ok(response.result.results);
+      assert.ok(response.result.results.docs);
+      assert.strictEqual(response.result.results.docs.length, 2);
+      assert.ok(couch.isDone());
+      done();
+    }).catch(err => {
+      done(err);
+    });
+  });
+
+  it('should callback with error code ESOCKETTIMEDOUT if 3 HTTP requests gets timed out', function(done) {
+    // Increase the timeout for this test to allow for the delays
+    this.timeout(3000);
+    const couch = nock(url)
+      .post('/_bulk_get')
+      .query(true)
+      .delay(1000)
+      .times(3)
+      .reply(200, { ok: true });
+
+    timeoutDb.service.postBulkGet({ db: db.db, revs: true, docs: [] }).then(response => {
+      done(new Error('Successful response when error expected.'));
+    }).catch(err => {
+      err = error.convertResponseError(err);
+      // Note axios returns ECONNABORTED rather than ESOCKETTIMEDOUT
+      // See https://github.com/axios/axios/issues/2710 via https://github.com/axios/axios/issues/1543`
+      assert.strictEqual(err.statusText, 'ECONNABORTED');
+      assert.strictEqual(err.message, `timeout of 500ms exceeded: post ${url}/_bulk_get ECONNABORTED`);
+      assert.ok(couch.isDone());
+      done();
+    }).catch(err => {
+      // Handle assertion errors
+      done(err);
+    });
+  });
+  describe('#unit Check credentials', function() {
+    it('should properly decode username and password', function(done) {
+      const username = 'user%123';
+      const password = 'colon:at@321';
+      const url = `http://${encodeURIComponent(username)}:${encodeURIComponent(password)}@localhost:7777/testdb`;
+      const sessionUrl = 'http://localhost:7777';
+      const couch = nock(sessionUrl)
+        .post('/_session', { username: username, password: password })
+        .reply(200, { ok: true }, { 'Set-Cookie': 'AuthSession=ABC123DEF4356;' })
+        .get('/')
+        .reply(200);
+      const db = request.client(url, { parallelism: 1 });
+      db.service.getServerInformation().then(response => {
+        assert.ok(response);
+        assert.ok(couch.isDone());
+        done();
+      }).catch(err => {
+        done(err);
+      });
     });
   });
 });