@cloudant/couchbackup 2.9.9-SNAPSHOT.99 → 2.9.9

package/.scannerwork/scanner-report/source-36.txt

@@ -0,0 +1,36 @@
+// Copyright © 2017 IBM Corp. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+/* global describe it */
+'use strict';
+
+delete require.cache[require.resolve('./citestutils.js')];
+const u = require('./citestutils.js');
+
+[{ useApi: true }, { useApi: false }].forEach(function(params) {
+  describe(u.scenario('End to end backup and restore', params), function() {
+    it('should backup and restore animaldb', function(done) {
+      // Allow up to 60 s for backup and restore of animaldb
+      u.setTimeout(this, 60);
+      u.testDirectBackupAndRestore(params, 'animaldb', this.dbName, done);
+    });
+    it('should backup and restore largedb1g #slow', function(done) {
+      // Allow up to 30 m for backup and restore of largedb1g
+      // This is a long time but when many builds run in parallel it can take a
+      // while to get this done.
+      u.setTimeout(this, 30 * 60);
+      u.testDirectBackupAndRestore(params, 'largedb1g', this.dbName, done);
+    });
+  });
+});

package/.scannerwork/scanner-report/source-37.txt

@@ -1,5 +1,4 @@
-#!/usr/bin/env node
-// Copyright © 2017, 2021 IBM Corp. All rights reserved.
+// Copyright © 2017, 2022 IBM Corp. All rights reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -12,66 +11,135 @@
 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 // See the License for the specific language governing permissions and
 // limitations under the License.
+
+/* global describe it beforeEach */
 'use strict';
 
-const error = require('../includes/error.js');
+const assert = require('assert');
+const backup = require('../includes/shallowbackup.js');
+const request = require('../includes/request.js');
 const fs = require('fs');
-const cliutils = require('../includes/cliutils.js');
-const couchbackup = require('../app.js');
-const parser = require('../includes/parser.js');
-const debug = require('debug');
-const backupDebug = debug('couchbackup:backup');
-const backupBatchDebug = debug('couchbackup:backup:batch');
+const nock = require('nock');
+
+// Function to create a DB object and call the shallow backup function
+// This is normally done by app.js
+function shallowBackup(dbUrl, opts) {
+  const db = request.client(dbUrl, opts);
+  // Disable compression to make body assertions easier
+  db.service.setEnableGzipCompression(false);
+  return backup(db, opts);
+}
 
-backupDebug.enabled = true;
+// Note all these tests include a body parameter of include_docs and a query
+// string of include_docs because of a quirk of nano that when using the fetch
+// method always adds the include_docs query string.
+describe('#unit Perform backup using shallow backup', function() {
+  const dbUrl = 'http://localhost:5984/animaldb';
+  // Query string keys are stringified by Nano
+  const badgerKey = 'badger\0';
+  const kookaburraKey = 'kookaburra\0';
+  const snipeKey = 'snipe\0';
 
-const program = parser.parseBackupArgs();
+  beforeEach('Reset nocks', function() {
+    nock.cleanAll();
+  });
 
-const databaseUrl = cliutils.databaseUrl(program.url, program.db);
-const opts = {
-  bufferSize: program.bufferSize,
-  log: program.log,
-  mode: program.mode,
-  parallelism: program.parallelism,
-  requestTimeout: program.requestTimeout,
-  resume: program.resume,
-  iamApiKey: program.iamApiKey,
-  iamTokenUrl: program.iamTokenUrl
-};
+  it('should perform a shallow backup', function(done) {
+    const couch = nock(dbUrl)
+      // batch 1
+      .post('/_all_docs', { limit: 3, include_docs: true })
+      .reply(200, JSON.parse(fs.readFileSync('./test/fixtures/animaldb_all_docs_1.json', 'utf8')))
+      // batch 2
+      .post('/_all_docs', { limit: 3, start_key: badgerKey, include_docs: true })
+      .reply(200, JSON.parse(fs.readFileSync('./test/fixtures/animaldb_all_docs_2.json', 'utf8')))
+      // batch 3
+      .post('/_all_docs', { limit: 3, start_key: kookaburraKey, include_docs: true })
+      .reply(200, JSON.parse(fs.readFileSync('./test/fixtures/animaldb_all_docs_3.json', 'utf8')))
+      // batch 4
+      .post('/_all_docs', { limit: 3, start_key: snipeKey, include_docs: true })
+      .reply(200, JSON.parse(fs.readFileSync('./test/fixtures/animaldb_all_docs_4.json', 'utf8')));
 
-// log configuration to console
-console.error('='.repeat(80));
-console.error('Performing backup on ' + databaseUrl.replace(/\/\/.+@/g, '//****:****@') + ' using configuration:');
-console.error(JSON.stringify(opts, null, 2).replace(/"iamApiKey": "[^"]+"/, '"iamApiKey": "****"'));
-console.error('='.repeat(80));
+    shallowBackup(dbUrl, { bufferSize: 3, parallelism: 1 })
+      .on('error', function(err) {
+        assert.fail(err);
+      })
+      .on('received', function(data) {
+        if (data.batch === 3) {
+          assert.strictEqual(data.length, 2); // smaller last batch
+        } else {
+          assert.strictEqual(data.length, 3);
+        }
+      })
+      .on('finished', function(data) {
+        assert.strictEqual(data.total, 11);
+        assert.ok(couch.isDone());
+        done();
+      });
+  });
 
-backupBatchDebug.enabled = !program.quiet;
+  it('should perform a shallow backup with transient error', function(done) {
+    const couch = nock(dbUrl)
+      // batch 1
+      .post('/_all_docs', { limit: 3, include_docs: true })
+      .reply(200, JSON.parse(fs.readFileSync('./test/fixtures/animaldb_all_docs_1.json', 'utf8')))
+      // batch 2
+      .post('/_all_docs', { limit: 3, start_key: badgerKey, include_docs: true })
+      .reply(200, JSON.parse(fs.readFileSync('./test/fixtures/animaldb_all_docs_2.json', 'utf8')))
+      // batch 3 - transient error
+      .post('/_all_docs', { limit: 3, start_key: kookaburraKey, include_docs: true })
+      .reply(500, { error: 'Internal Server Error' })
+      // batch 3 - retry
+      .post('/_all_docs', { limit: 3, start_key: kookaburraKey, include_docs: true })
+      .reply(200, JSON.parse(fs.readFileSync('./test/fixtures/animaldb_all_docs_3.json', 'utf8')))
+      // batch 4
+      .post('/_all_docs', { limit: 3, start_key: snipeKey, include_docs: true })
+      .reply(200, JSON.parse(fs.readFileSync('./test/fixtures/animaldb_all_docs_4.json', 'utf8')));
 
-let ws = process.stdout;
+    shallowBackup(dbUrl, { bufferSize: 3, parallelism: 1 })
+      .on('error', function(err) {
+        assert.strictEqual(err.name, 'HTTPError');
+      })
+      .on('received', function(data) {
+        if (data.batch === 3) {
+          assert.strictEqual(data.length, 2); // smaller last batch
+        } else {
+          assert.strictEqual(data.length, 3);
+        }
+      })
+      .on('finished', function(data) {
+        assert.strictEqual(data.total, 11);
+        assert.ok(couch.isDone());
+        done();
+      });
+  });
 
-// open output file
-if (program.output) {
-  let flags = 'w';
-  if (program.log && program.resume) {
-    flags = 'a';
-  }
-  const fd = fs.openSync(program.output, flags);
-  ws = fs.createWriteStream(null, { fd: fd });
-}
+  it('should fail to perform a shallow backup on fatal error', function(done) {
+    const couch = nock(dbUrl)
+      // batch 1
+      .post('/_all_docs', { limit: 3, include_docs: true })
+      .reply(200, JSON.parse(fs.readFileSync('./test/fixtures/animaldb_all_docs_1.json', 'utf8')))
+      // batch 2
+      .post('/_all_docs', { limit: 3, start_key: badgerKey, include_docs: true })
+      .reply(200, JSON.parse(fs.readFileSync('./test/fixtures/animaldb_all_docs_2.json', 'utf8')))
+      // batch 3 - fatal error
+      .post('/_all_docs', { limit: 3, start_key: kookaburraKey, include_docs: true })
+      .reply(401, { error: 'Unauthorized' });
 
-backupDebug('Fetching all database changes...');
+    let errCount = 0;
 
-return couchbackup.backup(
-  databaseUrl,
-  ws,
-  opts,
-  error.terminationCallback
-).on('changes', function(batch) {
-  backupBatchDebug('Total batches received:', batch + 1);
-}).on('written', function(obj) {
-  backupBatchDebug('Written batch ID:', obj.batch, 'Total document revisions written:', obj.total, 'Time:', obj.time);
-}).on('error', function(e) {
-  backupDebug('ERROR', e);
-}).on('finished', function(obj) {
-  backupDebug('Finished - Total document revisions written:', obj.total);
+    shallowBackup(dbUrl, { bufferSize: 3, parallelism: 1 })
+      .on('error', function(err) {
+        errCount++;
+        assert.strictEqual(err.name, 'Unauthorized');
+      })
+      .on('received', function(data) {
+        assert.strictEqual(data.length, 3);
+      })
+      .on('finished', function(data) {
+        assert.strictEqual(data.total, 6);
+        assert.ok(couch.isDone());
+        assert.strictEqual(errCount, 1);
+        done();
+      });
+  });
 });

package/.scannerwork/scanner-report/source-38.txt

@@ -1,4 +1,3 @@
-#!/usr/bin/env node
 // Copyright © 2017, 2021 IBM Corp. All rights reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
@@ -12,45 +11,58 @@
 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 // See the License for the specific language governing permissions and
 // limitations under the License.
-'use strict';
-
-const error = require('../includes/error.js');
-const cliutils = require('../includes/cliutils.js');
-const couchbackup = require('../app.js');
-const parser = require('../includes/parser.js');
-const debug = require('debug');
-const restoreDebug = debug('couchbackup:restore');
-const restoreBatchDebug = debug('couchbackup:restore:batch');
 
-restoreDebug.enabled = true;
+/* global beforeEach afterEach */
+'use strict';
 
-const program = parser.parseRestoreArgs();
-const databaseUrl = cliutils.databaseUrl(program.url, program.db);
-const opts = {
-  bufferSize: program.bufferSize,
-  parallelism: program.parallelism,
-  requestTimeout: program.requestTimeout,
-  iamApiKey: program.iamApiKey,
-  iamTokenUrl: program.iamTokenUrl
+const { CloudantV1 } = require('@ibm-cloud/cloudant');
+const url = new URL((process.env.COUCH_BACKEND_URL) ? process.env.COUCH_BACKEND_URL : 'https://no-couch-backend-url-set.test');
+const { BasicAuthenticator, NoAuthAuthenticator } = require('ibm-cloud-sdk-core');
+const authenticator = (url.username) ? new BasicAuthenticator({ username: url.username, password: decodeURIComponent(url.password) }) : new NoAuthAuthenticator();
+const serviceOpts = {
+  authenticator: authenticator
 };
+const cloudant = new CloudantV1(serviceOpts);
+// Remove auth from URL before using for service
+cloudant.setServiceUrl(new URL(url.pathname, url.origin).toString());
+const uuid = require('uuid').v4;
+const fs = require('fs');
 
-// log configuration to console
-console.error('='.repeat(80));
-console.error('Performing restore on ' + databaseUrl.replace(/\/\/.+@/g, '//****:****@') + ' using configuration:');
-console.error(JSON.stringify(opts, null, 2).replace(/"iamApiKey": "[^"]+"/, '"iamApiKey": "****"'));
-console.error('='.repeat(80));
+// Mocha hooks that will be at the root context so run for all tests
 
-restoreBatchDebug.enabled = !program.quiet;
+beforeEach('Create test database', function(done) {
+  // Don't run hook for unit tests, just for CI
+  if (!this.currentTest.fullTitle().includes('#unit')) {
+    // Allow 10 seconds to create the DB
+    this.timeout(10 * 1000);
+    const unique = uuid();
+    this.fileName = `${unique}`;
+    this.dbName = 'couchbackup_test_' + unique;
+    cloudant.putDatabase({ db: this.dbName }).then(() => { done(); }).catch((err) => { done(err); });
+  } else {
+    done();
+  }
+});
 
-return couchbackup.restore(
-  process.stdin, // restore from stdin
-  databaseUrl,
-  opts,
-  error.terminationCallback
-).on('restored', function(obj) {
-  restoreBatchDebug('restored', obj.total);
-}).on('error', function(e) {
-  restoreDebug('ERROR', e);
-}).on('finished', function(obj) {
-  restoreDebug('finished', obj);
+afterEach('Delete test database', function(done) {
+  // Don't run hook for unit tests, just for CI
+  if (!this.currentTest.fullTitle().includes('#unit')) {
+    // Allow 10 seconds to delete the DB
+    this.timeout(10 * 1000);
+    deleteIfExists(this.fileName);
+    deleteIfExists(`${this.fileName}.log`);
+    cloudant.deleteDatabase({ db: this.dbName }).then(() => { done(); }).catch((err) => { done(err); });
+  } else {
+    done();
+  }
 });
+
+function deleteIfExists(fileName) {
+  fs.unlink(fileName, function(err) {
+    if (err) {
+      if (err.code !== 'ENOENT') {
+        console.error(`${err.code} ${err.message}`);
+      }
+    }
+  });
+}

package/.scannerwork/scanner-report/source-39.txt

@@ -1,4 +1,4 @@
-// Copyright © 2017 IBM Corp. All rights reserved.
+// Copyright © 2017, 2021 IBM Corp. All rights reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -12,103 +12,136 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-/* global after before describe */
+/* global describe it beforeEach */
 'use strict';
 
-const url = require('url');
-const toxy = require('toxy');
-// Import the common hooks
-require('../test/hooks.js');
+const assert = require('assert');
+const fs = require('fs');
+const nock = require('nock');
+const request = require('../includes/request.js');
+const writer = require('../includes/writer.js');
+const noopEmitter = new (require('events')).EventEmitter();
+const liner = require('../includes/liner.js');
 
-const tpoisons = toxy.poisons;
-const trules = toxy.rules;
+const longTestTimeout = 3000;
 
-function setupProxy(poison) {
-  const backendUrl = new url.URL(process.env.COUCH_BACKEND_URL);
-  const proxy = toxy({
-    auth: `${backendUrl.username}:${backendUrl.password}`,
-    changeOrigin: true
+// The writer expects a line-by-line stream so this utility function does that
+// processing for the tests (which normally happens in the internal restore
+// function).
+function testLinestream(fixture = './test/fixtures/animaldb_expected.json') {
+  return fs.createReadStream(fixture)
+    .pipe(liner());
+}
+
+describe('#unit Check database restore writer', function() {
+  const dbUrl = 'http://localhost:5984/animaldb';
+  const db = request.client(dbUrl, { parallelism: 1 });
+
+  beforeEach('Reset nocks', function() {
+    nock.cleanAll();
   });
 
-  // Forward traffic to DB
-  proxy.forward(process.env.COUCH_BACKEND_URL);
-
-  switch (poison) {
-    case 'normal':
-      // No poisons to add
-      break;
-    case 'bandwidth-limit':
-      // https://github.com/h2non/toxy#bandwidth
-      // Note the implementation of bandwidth is simplistic and the threshold
-      // delay is applied to every write of the buffer, so use the smallest
-      // delay possible and adjust the rate using the bytes size instead.
-      proxy
-        .poison(tpoisons.bandwidth({ bytes: 512, threshold: 1 })); // 0.5 MB/s
-      break;
-    case 'latency':
-      // https://github.com/h2non/toxy#latency
-      proxy
-        .poison(tpoisons.latency({ max: 1500, min: 250 }))
-        .withRule(trules.probability(60));
-      break;
-    case 'slow-read':
-      // https://github.com/h2non/toxy#slow-read
-      // Note this only impacts read of data from requests so only for non-GET
-      // In practice this means that it impacts restore much more than backup
-      // since although backup POSTs to _bulk_get the content is much smaller
-      // than what is POSTed to _bulk_docs for a restore.
-      // Similarly to bandwidth-limit use a 1 ms threshold
-      proxy
-        .poison(tpoisons.slowRead({ chunk: 256, threshold: 1 }))
-        // Slow read for 10 % of the time e.g. 10 ms in every 100
-        .withRule(trules.timeThreshold({ duration: 10, period: 100 }));
-      break;
-    case 'rate-limit':
-      // https://github.com/h2non/toxy#rate-limit
-      // Simulate the Cloudant free plan with 20 lookups ps and 10 writes ps
-      proxy.post('/*/_bulk_get')
-        .poison(tpoisons.rateLimit({ limit: 20, threshold: 1000 }));
-      proxy.post('/*/_bulk_docs')
-        .poison(tpoisons.rateLimit({ limit: 10, threshold: 1000 }));
-      break;
-    default:
-      throw Error('Unknown toxy poison ' + poison);
-  }
-
-  // Catch remaining traffic
-  proxy.all('/*');
-  return proxy;
-}
+  it('should complete successfully', function(done) {
+    nock(dbUrl)
+      .post('/_bulk_docs')
+      .reply(200, []); // success
+
+    testLinestream()
+      .pipe(writer(db, 500, 1, noopEmitter))
+      .on('error', function(err) {
+        done(err);
+      })
+      .on('finished', function(data) {
+        assert.strictEqual(data.total, 15);
+        assert.ok(nock.isDone());
+        done();
+      });
+  });
+
+  it('should terminate on a fatal error', function(done) {
+    nock(dbUrl)
+      .post('/_bulk_docs')
+      .reply(401, { error: 'Unauthorized' }); // fatal error
+
+    testLinestream()
+      .pipe(writer(db, 500, 1, noopEmitter))
+      .on('error', function(err) {
+        assert.strictEqual(err.name, 'Unauthorized');
+        assert.strictEqual(err.message, 'Access is denied due to invalid credentials.');
+        assert.ok(nock.isDone());
+        done();
+      });
+  });
+
+  it('should retry on transient errors', function(done) {
+    nock(dbUrl)
+      .post('/_bulk_docs')
+      .reply(429, { error: 'Too Many Requests' }) // transient error
+      .post('/_bulk_docs')
+      .reply(500, { error: 'Internal Server Error' }) // transient error
+      .post('/_bulk_docs')
+      .reply(200, { ok: true }); // third time lucky success
+
+    testLinestream()
+      .pipe(writer(db, 500, 1, noopEmitter))
+      .on('error', function(err) {
+        done(err);
+      })
+      .on('finished', function(data) {
+        assert.strictEqual(data.total, 15);
+        assert.ok(nock.isDone());
+        done();
+      });
+  }).timeout(longTestTimeout);
+
+  it('should fail after 3 transient errors', function(done) {
+    nock(dbUrl)
+      .post('/_bulk_docs')
+      .reply(429, { error: 'Too Many Requests' }) // transient error
+      .post('/_bulk_docs')
+      .reply(500, { error: 'Internal Server Error' }) // transient error
+      .post('/_bulk_docs')
+      .reply(503, { error: 'Service Unavailable' }); // Final transient error
+
+    testLinestream()
+      .pipe(writer(db, 500, 1, noopEmitter))
+      .on('error', function(err) {
+        assert.strictEqual(err.name, 'HTTPFatalError');
+        assert.strictEqual(err.message, `503 : post ${dbUrl}/_bulk_docs - Error: Service Unavailable`);
+        assert.ok(nock.isDone());
+        done();
+      });
+  }).timeout(longTestTimeout);
+
+  it('should restore shallow backups without rev info successfully', function(done) {
+    nock(dbUrl)
+      .post('/_bulk_docs')
+      .reply(200, [{ ok: true, id: 'foo', rev: '1-abc' }]); // success
+
+    testLinestream('./test/fixtures/animaldb_old_shallow.json')
+      .pipe(writer(db, 500, 1, noopEmitter))
+      .on('error', function(err) {
+        done(err);
+      })
+      .on('finished', function(data) {
+        assert.strictEqual(data.total, 11);
+        assert.ok(nock.isDone());
+        done();
+      });
+  });
+
+  it('should get a batch error for non-empty array response with new_edits false', function(done) {
+    nock(dbUrl)
+      .post('/_bulk_docs')
+      .reply(200, [{ id: 'foo', error: 'foo', reason: 'bar' }]);
 
-const poisons = [
-  'normal',
-  'bandwidth-limit',
-  'latency',
-  'slow-read',
-  'rate-limit'
-];
-
-poisons.forEach(function(poison) {
-  describe('unreliable network tests (using toxy poison ' + poison + ')', function() {
-    let proxy;
-
-    before('start toxy server', function() {
-      proxy = setupProxy(poison);
-      console.log('Using toxy poison ' + poison);
-
-      // For these tests COUCH_URL points to the toxy proxy on localhost whereas
-      // COUCH_BACKEND_URL is the real CouchDb instance.
-      const toxyUrl = new url.URL(process.env.COUCH_URL);
-      // Listen on the specified hostname only, so if using localhost we don't
-      // need external connections.
-      proxy.listen(toxyUrl.port, toxyUrl.hostname);
-    });
-
-    after('stop toxy server', function() {
-      proxy.close();
-    });
-
-    delete require.cache[require.resolve('../test/ci_e2e.js')];
-    require('../test/ci_e2e.js');
+    testLinestream()
+      .pipe(writer(db, 500, 1, noopEmitter))
+      .on('error', function(err) {
+        assert.strictEqual(err.name, 'Error');
+        assert.strictEqual(err.message, 'Error writing batch with new_edits:false and 1 items');
+        assert.ok(nock.isDone());
+        done();
+      });
   });
 });