@cloudant/couchbackup 2.9.8-SNAPSHOT.93 → 2.9.8-SNAPSHOT.94

package/.scannerwork/scanner-report/source-32.txt

@@ -0,0 +1,119 @@
+// Copyright © 2017 IBM Corp. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+/* global describe it */
+'use strict';
+
+const assert = require('assert');
+const fs = require('fs');
+const u = require('./citestutils.js');
+
+[{ useApi: true }, { useApi: false }].forEach(function(params) {
+  describe(u.scenario('Resume tests', params), function() {
+    it('should create a log file', function(done) {
+      // Allow up to 90 s for this test
+      u.setTimeout(this, 60);
+
+      const actualBackup = `./${this.fileName}`;
+      const logFile = `./${this.fileName}` + '.log';
+      // Use abort parameter to terminate the backup a given number of ms after
+      // the first data write to the output file.
+      const p = u.p(params, { opts: { log: logFile } });
+      u.testBackupToFile(p, 'animaldb', actualBackup, function(err) {
+        if (err) {
+          done(err);
+        } else {
+          // Assert the log file exists
+          try {
+            assert.ok(fs.existsSync(logFile), 'The log file should exist.');
+            done();
+          } catch (err) {
+            done(err);
+          }
+        }
+      });
+    });
+
+    it('should restore corrupted animaldb to a database correctly', function(done) {
+      // Allow up to 60 s to restore and compare (again it should be faster)!
+      u.setTimeout(this, 60);
+      const input = fs.createReadStream('./test/fixtures/animaldb_corrupted.json');
+      const dbName = this.dbName;
+      input.on('open', function() {
+        u.testRestore(params, input, dbName, function(err) {
+          if (err) {
+            done(err);
+          } else {
+            u.dbCompare('animaldb', dbName, done);
+          }
+        });
+      });
+    });
+
+    it('should restore resumed animaldb with blank line to a database correctly', function(done) {
+      // Allow up to 60 s to restore and compare (again it should be faster)!
+      u.setTimeout(this, 60);
+      const input = fs.createReadStream('./test/fixtures/animaldb_resumed_blank.json');
+      const dbName = this.dbName;
+      input.on('open', function() {
+        u.testRestore(params, input, dbName, function(err) {
+          if (err) {
+            done(err);
+          } else {
+            u.dbCompare('animaldb', dbName, done);
+          }
+        });
+      });
+    });
+  });
+});
+
+describe('Resume tests', function() {
+  // Currently cannot abort API backups, when we do this test should be run for
+  // both API and CLI
+  it('should correctly backup and restore backup10m', function(done) {
+    // Allow up to 90 s for this test
+    u.setTimeout(this, 90);
+
+    const actualBackup = `./${this.fileName}`;
+    const logFile = `./${this.fileName}` + '.log';
+    // Use abort parameter to terminate the backup a given number of ms after
+    // the first data write to the output file.
+    const p = u.p(params, { abort: true }, { opts: { log: logFile } });
+    const restoreDb = this.dbName;
+    // Set the database doc count as fewer than this should be written during
+    // resumed backup.
+    p.exclusiveMaxExpected = 5096;
+
+    u.testBackupAbortResumeRestore(p, 'backup10m', actualBackup, restoreDb, done);
+  });
+  // Note --output is only valid for CLI usage, this test should only run for CLI
+  const params = { useApi: false };
+  it('should correctly backup and restore backup10m using --output', function(done) {
+    // Allow up to 90 s for this test
+    u.setTimeout(this, 90);
+
+    const actualBackup = `./${this.fileName}`;
+    const logFile = `./${this.fileName}` + '.log';
+    // Use abort parameter to terminate the backup a given number of ms after
+    // the first data write to the output file.
+    const p = u.p(params, { abort: true }, { opts: { output: actualBackup, log: logFile } });
+    const restoreDb = this.dbName;
+    // Set the database doc count as fewer than this should be written during
+    // resumed backup.
+    p.exclusiveMaxExpected = 5096;
+
+    u.testBackupAbortResumeRestore(p, 'backup10m', actualBackup, restoreDb, done);
+  });
+});

package/.scannerwork/scanner-report/source-33.txt

@@ -0,0 +1,29 @@
+// Copyright © 2017 IBM Corp. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+/* global describe it */
+'use strict';
+
+const u = require('./citestutils.js');
+
+[{ useApi: true }, { useApi: false }].forEach(function(params) {
+  describe(u.scenario('#slowest End to end backup and restore', params), function() {
+    // 10 GB is about the largest the CI can handle before getting very upset
+    // about how long things are taking
+    it('should backup and restore largedb10g', function(done) {
+      u.setTimeout(this, 350 * 60);
+      u.testDirectBackupAndRestore(params, 'largedb10g', this.dbName, done);
+    });
+  });
+});

package/.scannerwork/scanner-report/source-34.txt

@@ -0,0 +1,225 @@
+// Copyright © 2017, 2018 IBM Corp. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+/* global describe it */
+'use strict';
+
+const assert = require('assert');
+const backup = require('../app.js').backup;
+const fs = require('fs');
+const nock = require('nock');
+
+const goodUrl = 'http://localhost:5984/db';
+// The real validateArgs function of app.js isn't
+// exported - so we call the exported backup method
+// instead. We don't get as far as a real backup when
+// testing error cases. For success cases we nock the
+// goodUrl and
+const validateArgs = function(url, opts, callback) {
+  const nullStream = fs.createWriteStream('/dev/null');
+  let cb = callback;
+  if (url === goodUrl) {
+    // Nock the goodUrl
+    nock(goodUrl).head('').reply(404, { error: 'not_found', reason: 'missing' });
+    // replace the callback to handle the nock response
+    // to avoid attempting a real backup
+    cb = function(err) {
+      nullStream.end();
+      if (err.name === 'DatabaseNotFound') {
+        // This is what we expect if we reached the backup
+        // This is success for valid args cases.
+        err = null;
+      }
+      callback(err);
+    };
+  }
+  backup(url, nullStream, opts, cb);
+  return true;
+};
+
+const stderrWriteFun = process.stderr.write;
+let capturedStderr;
+
+function captureStderr() {
+  process.stderr.write = function(string, encoding, fd) {
+    capturedStderr += string;
+  };
+}
+
+function releaseStderr() {
+  process.stderr.write = stderrWriteFun;
+  capturedStderr = null;
+}
+
+function assertErrorMessage(msg, done) {
+  return function(err, data) {
+    try {
+      assert(err.message, 'There should be an error message');
+      assert(err.message.indexOf(msg) >= 0);
+      assert(data === null || data === undefined, 'There should only be an error.');
+      done();
+    } catch (e) {
+      done(e);
+    }
+  };
+}
+
+function assertNoError(done) {
+  return function(err, data) {
+    try {
+      assert(err === null, 'There should be no error message.');
+      done();
+    } catch (e) {
+      done(e);
+    }
+  };
+}
+
+describe('#unit Validate arguments', function() {
+  it('returns error for invalid URL type', function(done) {
+    validateArgs(true, {}, assertErrorMessage('Invalid URL, must be type string', done));
+  });
+  it('returns no error for valid URL type', function(done) {
+    validateArgs(goodUrl, {}, assertNoError(done));
+  });
+  it('returns error for invalid (no host) URL', function(done) {
+    validateArgs('http://', {}, assertErrorMessage('Invalid URL', done));
+  });
+  it('returns error for invalid (no protocol) URL', function(done) {
+    validateArgs('invalid', {}, assertErrorMessage('Invalid URL', done));
+  });
+  it('returns error for invalid (wrong protocol) URL', function(done) {
+    validateArgs('ftp://invalid.example.com', {}, assertErrorMessage('Invalid URL protocol.', done));
+  });
+  it('returns error for invalid (no path) URL', function(done) {
+    validateArgs('https://invalid.example.com', {}, assertErrorMessage('Invalid URL, missing path element (no database).', done));
+  });
+  it('returns error for invalid (no protocol, no host) URL', function(done) {
+    validateArgs('invalid', {}, assertErrorMessage('Invalid URL', done));
+  });
+  it('returns error for invalid buffer size type', function(done) {
+    validateArgs(goodUrl, { bufferSize: '123' }, assertErrorMessage('Invalid buffer size option, must be a positive integer in the range (0, MAX_SAFE_INTEGER]', done));
+  });
+  it('returns error for zero buffer size', function(done) {
+    validateArgs(goodUrl, { bufferSize: 0 }, assertErrorMessage('Invalid buffer size option, must be a positive integer in the range (0, MAX_SAFE_INTEGER]', done));
+  });
+  it('returns error for float buffer size', function(done) {
+    validateArgs(goodUrl, { bufferSize: 1.23 }, assertErrorMessage('Invalid buffer size option, must be a positive integer in the range (0, MAX_SAFE_INTEGER]', done));
+  });
+  it('returns no error for valid buffer size type', function(done) {
+    validateArgs(goodUrl, { bufferSize: 123 }, assertNoError(done));
+  });
+  it('returns error for invalid log type', function(done) {
+    validateArgs(goodUrl, { log: true }, assertErrorMessage('Invalid log option, must be type string', done));
+  });
+  it('returns no error for valid log type', function(done) {
+    validateArgs(goodUrl, { log: 'log.txt' }, assertNoError(done));
+  });
+  it('returns error for invalid mode type', function(done) {
+    validateArgs(goodUrl, { mode: true }, assertErrorMessage('Invalid mode option, must be either "full" or "shallow"', done));
+  });
+  it('returns error for invalid mode string', function(done) {
+    validateArgs(goodUrl, { mode: 'foobar' }, assertErrorMessage('Invalid mode option, must be either "full" or "shallow"', done));
+  });
+  it('returns no error for valid mode type', function(done) {
+    validateArgs(goodUrl, { mode: 'full' }, assertNoError(done));
+  });
+  it('returns error for invalid output type', function(done) {
+    validateArgs(goodUrl, { output: true }, assertErrorMessage('Invalid output option, must be type string', done));
+  });
+  it('returns no error for valid output type', function(done) {
+    validateArgs(goodUrl, { output: 'output.txt' }, assertNoError(done));
+  });
+  it('returns error for invalid parallelism type', function(done) {
+    validateArgs(goodUrl, { parallelism: '123' }, assertErrorMessage('Invalid parallelism option, must be a positive integer in the range (0, MAX_SAFE_INTEGER]', done));
+  });
+  it('returns error for zero parallelism', function(done) {
+    validateArgs(goodUrl, { parallelism: 0 }, assertErrorMessage('Invalid parallelism option, must be a positive integer in the range (0, MAX_SAFE_INTEGER]', done));
+  });
+  it('returns error for float parallelism', function(done) {
+    validateArgs(goodUrl, { parallelism: 1.23 }, assertErrorMessage('Invalid parallelism option, must be a positive integer in the range (0, MAX_SAFE_INTEGER]', done));
+  });
+  it('returns no error for valid parallelism type', function(done) {
+    validateArgs(goodUrl, { parallelism: 123 }, assertNoError(done));
+  });
+  it('returns error for invalid request timeout type', function(done) {
+    validateArgs(goodUrl, { requestTimeout: '123' }, assertErrorMessage('Invalid request timeout option, must be a positive integer in the range (0, MAX_SAFE_INTEGER]', done));
+  });
+  it('returns error for zero request timeout', function(done) {
+    validateArgs(goodUrl, { requestTimeout: 0 }, assertErrorMessage('Invalid request timeout option, must be a positive integer in the range (0, MAX_SAFE_INTEGER]', done));
+  });
+  it('returns error for float request timout', function(done) {
+    validateArgs(goodUrl, { requestTimeout: 1.23 }, assertErrorMessage('Invalid request timeout option, must be a positive integer in the range (0, MAX_SAFE_INTEGER]', done));
+  });
+  it('returns no error for valid request timeout type', function(done) {
+    validateArgs(goodUrl, { requestTimeout: 123 }, assertNoError(done));
+  });
+  it('returns error for invalid resume type', function(done) {
+    validateArgs(goodUrl, { resume: 'true' }, assertErrorMessage('Invalid resume option, must be type boolean', done));
+  });
+  it('returns no error for valid resume type', function(done) {
+    validateArgs(goodUrl, { resume: false }, assertNoError(done));
+  });
+  it('returns error for invalid key type', function(done) {
+    validateArgs(goodUrl, { iamApiKey: true }, assertErrorMessage('Invalid iamApiKey option, must be type string', done));
+  });
+  it('returns error for key and URL credentials supplied', function(done) {
+    validateArgs('https://a:b@example.com/db', { iamApiKey: 'abc123' }, assertErrorMessage('URL user information must not be supplied when using IAM API key.', done));
+  });
+  it('warns for log arg in shallow mode', function(done) {
+    captureStderr();
+    try {
+      validateArgs(goodUrl, { mode: 'shallow', log: 'test' }, function(err, data) {
+        assert.ok(err);
+        assert.ok(!data);
+        assert(capturedStderr.indexOf('The options "log" and "resume" are invalid when using shallow mode.') > -1, 'Log warning message was not present');
+      });
+      done();
+    } catch (e) {
+      done(e);
+    } finally {
+      releaseStderr();
+    }
+  });
+  it('warns for resume arg in shallow mode', function(done) {
+    captureStderr();
+    try {
+      validateArgs(goodUrl, { mode: 'shallow', log: 'test', resume: true }, function(err, data) {
+        assert.ok(err);
+        assert.ok(!data);
+        assert(capturedStderr.indexOf('The options "log" and "resume" are invalid when using shallow mode.') > -1, 'Log warning message was not present');
+      });
+      done();
+    } catch (e) {
+      done(e);
+    } finally {
+      releaseStderr();
+    }
+  });
+  it('warns for parallism arg in shallow mode', function(done) {
+    captureStderr();
+    try {
+      validateArgs(goodUrl, { mode: 'shallow', parallelsim: 10 }, function(err, data) {
+        assert.ok(err);
+        assert.ok(!data);
+        assert(capturedStderr.indexOf('The option "parallelism" has no effect when using shallow mode.') > -1, 'Log warning message was not present');
+      });
+      done();
+    } catch (e) {
+      done(e);
+    } finally {
+      releaseStderr();
+    }
+  });
+});

package/.scannerwork/scanner-report/source-35.txt

@@ -0,0 +1,114 @@
+// Copyright © 2017 IBM Corp. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+/* global after before describe */
+'use strict';
+
+const url = require('url');
+const toxy = require('toxy');
+// Import the common hooks
+require('../test/hooks.js');
+
+const tpoisons = toxy.poisons;
+const trules = toxy.rules;
+
+function setupProxy(poison) {
+  const backendUrl = new url.URL(process.env.COUCH_BACKEND_URL);
+  const proxy = toxy({
+    auth: `${backendUrl.username}:${backendUrl.password}`,
+    changeOrigin: true
+  });
+
+  // Forward traffic to DB
+  proxy.forward(process.env.COUCH_BACKEND_URL);
+
+  switch (poison) {
+    case 'normal':
+      // No poisons to add
+      break;
+    case 'bandwidth-limit':
+      // https://github.com/h2non/toxy#bandwidth
+      // Note the implementation of bandwidth is simplistic and the threshold
+      // delay is applied to every write of the buffer, so use the smallest
+      // delay possible and adjust the rate using the bytes size instead.
+      proxy
+        .poison(tpoisons.bandwidth({ bytes: 512, threshold: 1 })); // 0.5 MB/s
+      break;
+    case 'latency':
+      // https://github.com/h2non/toxy#latency
+      proxy
+        .poison(tpoisons.latency({ max: 1500, min: 250 }))
+        .withRule(trules.probability(60));
+      break;
+    case 'slow-read':
+      // https://github.com/h2non/toxy#slow-read
+      // Note this only impacts read of data from requests so only for non-GET
+      // In practice this means that it impacts restore much more than backup
+      // since although backup POSTs to _bulk_get the content is much smaller
+      // than what is POSTed to _bulk_docs for a restore.
+      // Similarly to bandwidth-limit use a 1 ms threshold
+      proxy
+        .poison(tpoisons.slowRead({ chunk: 256, threshold: 1 }))
+        // Slow read for 10 % of the time e.g. 10 ms in every 100
+        .withRule(trules.timeThreshold({ duration: 10, period: 100 }));
+      break;
+    case 'rate-limit':
+      // https://github.com/h2non/toxy#rate-limit
+      // Simulate the Cloudant free plan with 20 lookups ps and 10 writes ps
+      proxy.post('/*/_bulk_get')
+        .poison(tpoisons.rateLimit({ limit: 20, threshold: 1000 }));
+      proxy.post('/*/_bulk_docs')
+        .poison(tpoisons.rateLimit({ limit: 10, threshold: 1000 }));
+      break;
+    default:
+      throw Error('Unknown toxy poison ' + poison);
+  }
+
+  // Catch remaining traffic
+  proxy.all('/*');
+  return proxy;
+}
+
+const poisons = [
+  'normal',
+  'bandwidth-limit',
+  'latency',
+  'slow-read',
+  'rate-limit'
+];
+
+poisons.forEach(function(poison) {
+  describe('unreliable network tests (using toxy poison ' + poison + ')', function() {
+    let proxy;
+
+    before('start toxy server', function() {
+      proxy = setupProxy(poison);
+      console.log('Using toxy poison ' + poison);
+
+      // For these tests COUCH_URL points to the toxy proxy on localhost whereas
+      // COUCH_BACKEND_URL is the real CouchDb instance.
+      const toxyUrl = new url.URL(process.env.COUCH_URL);
+      // Listen on the specified hostname only, so if using localhost we don't
+      // need external connections.
+      proxy.listen(toxyUrl.port, toxyUrl.hostname);
+    });
+
+    after('stop toxy server', function() {
+      proxy.close();
+    });
+
+    delete require.cache[require.resolve('../test/ci_e2e.js')];
+    require('../test/ci_e2e.js');
+  });
+});