@cloudant/couchbackup 2.9.9-SNAPSHOT.98 → 2.9.9

package/.scannerwork/scanner-report/source-37.txt

@@ -1,4 +1,4 @@
-// Copyright © 2017, 2021 IBM Corp. All rights reserved.
+// Copyright © 2017, 2022 IBM Corp. All rights reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -12,131 +12,134 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-/* global describe it before after */
+/* global describe it beforeEach */
 'use strict';
 
 const assert = require('assert');
-const applyEnvVars = require('../includes/config.js').applyEnvironmentVariables;
-
-describe('#unit Configuration', function() {
-  let processEnvCopy;
-
-  before('Save env', function() {
-    // Copy env so we can reset it after the tests
-    processEnvCopy = JSON.parse(JSON.stringify(process.env));
-  });
-
-  after('Reset env', function() {
-    process.env = processEnvCopy;
-  });
-
-  it('respects the COUCH_URL env variable', function(done) {
-    process.env.COUCH_URL = 'http://user:pass@myurl.com';
-    const config = {};
-    applyEnvVars(config);
-    assert.strictEqual(typeof config.url, 'string');
-    assert.strictEqual(config.url, process.env.COUCH_URL);
-    done();
-  });
-
-  it('respects the COUCH_DATABASE env variable', function(done) {
-    process.env.COUCH_DATABASE = 'mydb';
-    const config = {};
-    applyEnvVars(config);
-    assert.strictEqual(typeof config.db, 'string');
-    assert.strictEqual(config.db, process.env.COUCH_DATABASE);
-    done();
-  });
-
-  it('respects the COUCH_BUFFER_SIZE env variable', function(done) {
-    process.env.COUCH_BUFFER_SIZE = '1000';
-    const config = {};
-    applyEnvVars(config);
-    assert.strictEqual(typeof config.bufferSize, 'number');
-    assert.strictEqual(config.bufferSize, 1000);
-    done();
-  });
-
-  it('respects the COUCH_PARALLELISM env variable', function(done) {
-    process.env.COUCH_PARALLELISM = '20';
-    const config = {};
-    applyEnvVars(config);
-    assert.strictEqual(typeof config.parallelism, 'number');
-    assert.strictEqual(config.parallelism, 20);
-    done();
-  });
-
-  it('respects the COUCH_REQUEST_TIMEOUT env variable', function(done) {
-    process.env.COUCH_REQUEST_TIMEOUT = '10000';
-    const config = {};
-    applyEnvVars(config);
-    assert.strictEqual(typeof config.requestTimeout, 'number');
-    assert.strictEqual(config.requestTimeout, 10000);
-    done();
-  });
-
-  it('respects the CLOUDANT_IAM_API_KEY env variable', function(done) {
-    const key = 'ABC123-ZYX987_cba789-xyz321';
-    process.env.CLOUDANT_IAM_API_KEY = key;
-    const config = {};
-    applyEnvVars(config);
-    assert.strictEqual(typeof config.iamApiKey, 'string');
-    assert.strictEqual(config.iamApiKey, key);
-    done();
-  });
-
-  it('respects the CLOUDANT_IAM_TOKEN_URL env variable', function(done) {
-    const u = 'https://testhost.example:1234/identity/token';
-    process.env.CLOUDANT_IAM_TOKEN_URL = u;
-    const config = {};
-    applyEnvVars(config);
-    assert.strictEqual(typeof config.iamTokenUrl, 'string');
-    assert.strictEqual(config.iamTokenUrl, u);
-    done();
-  });
-
-  it('respects the COUCH_LOG env variable', function(done) {
-    process.env.COUCH_LOG = 'my.log';
-    const config = {};
-    applyEnvVars(config);
-    assert.strictEqual(typeof config.log, 'string');
-    assert.strictEqual(config.log, process.env.COUCH_LOG);
-    done();
-  });
-
-  it('respects the COUCH_RESUME env variable', function(done) {
-    process.env.COUCH_RESUME = 'true';
-    const config = {};
-    applyEnvVars(config);
-    assert.strictEqual(typeof config.resume, 'boolean');
-    assert.strictEqual(config.resume, true);
-    done();
+const backup = require('../includes/shallowbackup.js');
+const request = require('../includes/request.js');
+const fs = require('fs');
+const nock = require('nock');
+
+// Function to create a DB object and call the shallow backup function
+// This is normally done by app.js
+function shallowBackup(dbUrl, opts) {
+  const db = request.client(dbUrl, opts);
+  // Disable compression to make body assertions easier
+  db.service.setEnableGzipCompression(false);
+  return backup(db, opts);
+}
+
+// Note all these tests include a body parameter of include_docs and a query
+// string of include_docs because of a quirk of nano that when using the fetch
+// method always adds the include_docs query string.
+describe('#unit Perform backup using shallow backup', function() {
+  const dbUrl = 'http://localhost:5984/animaldb';
+  // Query string keys are stringified by Nano
+  const badgerKey = 'badger\0';
+  const kookaburraKey = 'kookaburra\0';
+  const snipeKey = 'snipe\0';
+
+  beforeEach('Reset nocks', function() {
+    nock.cleanAll();
   });
 
-  it('respects the COUCH_OUTPUT env variable', function(done) {
-    process.env.COUCH_OUTPUT = 'myfile.txt';
-    const config = {};
-    applyEnvVars(config);
-    assert.strictEqual(typeof config.output, 'string');
-    assert.strictEqual(config.output, process.env.COUCH_OUTPUT);
-    done();
+  it('should perform a shallow backup', function(done) {
+    const couch = nock(dbUrl)
+      // batch 1
+      .post('/_all_docs', { limit: 3, include_docs: true })
+      .reply(200, JSON.parse(fs.readFileSync('./test/fixtures/animaldb_all_docs_1.json', 'utf8')))
+      // batch 2
+      .post('/_all_docs', { limit: 3, start_key: badgerKey, include_docs: true })
+      .reply(200, JSON.parse(fs.readFileSync('./test/fixtures/animaldb_all_docs_2.json', 'utf8')))
+      // batch 3
+      .post('/_all_docs', { limit: 3, start_key: kookaburraKey, include_docs: true })
+      .reply(200, JSON.parse(fs.readFileSync('./test/fixtures/animaldb_all_docs_3.json', 'utf8')))
+      // batch 4
+      .post('/_all_docs', { limit: 3, start_key: snipeKey, include_docs: true })
+      .reply(200, JSON.parse(fs.readFileSync('./test/fixtures/animaldb_all_docs_4.json', 'utf8')));
+
+    shallowBackup(dbUrl, { bufferSize: 3, parallelism: 1 })
+      .on('error', function(err) {
+        assert.fail(err);
+      })
+      .on('received', function(data) {
+        if (data.batch === 3) {
+          assert.strictEqual(data.length, 2); // smaller last batch
+        } else {
+          assert.strictEqual(data.length, 3);
+        }
+      })
+      .on('finished', function(data) {
+        assert.strictEqual(data.total, 11);
+        assert.ok(couch.isDone());
+        done();
+      });
   });
 
-  it('respects the COUCH_MODE env variable', function(done) {
-    process.env.COUCH_MODE = 'shallow';
-    const config = {};
-    applyEnvVars(config);
-    assert.strictEqual(typeof config.mode, 'string');
-    assert.strictEqual(config.mode, 'shallow');
-    done();
+  it('should perform a shallow backup with transient error', function(done) {
+    const couch = nock(dbUrl)
+      // batch 1
+      .post('/_all_docs', { limit: 3, include_docs: true })
+      .reply(200, JSON.parse(fs.readFileSync('./test/fixtures/animaldb_all_docs_1.json', 'utf8')))
+      // batch 2
+      .post('/_all_docs', { limit: 3, start_key: badgerKey, include_docs: true })
+      .reply(200, JSON.parse(fs.readFileSync('./test/fixtures/animaldb_all_docs_2.json', 'utf8')))
+      // batch 3 - transient error
+      .post('/_all_docs', { limit: 3, start_key: kookaburraKey, include_docs: true })
+      .reply(500, { error: 'Internal Server Error' })
+      // batch 3 - retry
+      .post('/_all_docs', { limit: 3, start_key: kookaburraKey, include_docs: true })
+      .reply(200, JSON.parse(fs.readFileSync('./test/fixtures/animaldb_all_docs_3.json', 'utf8')))
+      // batch 4
+      .post('/_all_docs', { limit: 3, start_key: snipeKey, include_docs: true })
+      .reply(200, JSON.parse(fs.readFileSync('./test/fixtures/animaldb_all_docs_4.json', 'utf8')));
+
+    shallowBackup(dbUrl, { bufferSize: 3, parallelism: 1 })
+      .on('error', function(err) {
+        assert.strictEqual(err.name, 'HTTPError');
+      })
+      .on('received', function(data) {
+        if (data.batch === 3) {
+          assert.strictEqual(data.length, 2); // smaller last batch
+        } else {
+          assert.strictEqual(data.length, 3);
+        }
+      })
+      .on('finished', function(data) {
+        assert.strictEqual(data.total, 11);
+        assert.ok(couch.isDone());
+        done();
+      });
   });
 
-  it('respects the COUCH_QUIET env variable', function(done) {
-    process.env.COUCH_QUIET = 'true';
-    const config = {};
-    applyEnvVars(config);
-    assert.strictEqual(typeof config.quiet, 'boolean');
-    assert.strictEqual(config.quiet, true);
-    done();
+  it('should fail to perform a shallow backup on fatal error', function(done) {
+    const couch = nock(dbUrl)
+      // batch 1
+      .post('/_all_docs', { limit: 3, include_docs: true })
+      .reply(200, JSON.parse(fs.readFileSync('./test/fixtures/animaldb_all_docs_1.json', 'utf8')))
+      // batch 2
+      .post('/_all_docs', { limit: 3, start_key: badgerKey, include_docs: true })
+      .reply(200, JSON.parse(fs.readFileSync('./test/fixtures/animaldb_all_docs_2.json', 'utf8')))
+      // batch 3 - fatal error
+      .post('/_all_docs', { limit: 3, start_key: kookaburraKey, include_docs: true })
+      .reply(401, { error: 'Unauthorized' });
+
+    let errCount = 0;
+
+    shallowBackup(dbUrl, { bufferSize: 3, parallelism: 1 })
+      .on('error', function(err) {
+        errCount++;
+        assert.strictEqual(err.name, 'Unauthorized');
+      })
+      .on('received', function(data) {
+        assert.strictEqual(data.length, 3);
+      })
+      .on('finished', function(data) {
+        assert.strictEqual(data.total, 6);
+        assert.ok(couch.isDone());
+        assert.strictEqual(errCount, 1);
+        done();
+      });
   });
 });

package/.scannerwork/scanner-report/source-38.txt

@@ -0,0 +1,68 @@
+// Copyright © 2017, 2021 IBM Corp. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+/* global beforeEach afterEach */
+'use strict';
+
+const { CloudantV1 } = require('@ibm-cloud/cloudant');
+const url = new URL((process.env.COUCH_BACKEND_URL) ? process.env.COUCH_BACKEND_URL : 'https://no-couch-backend-url-set.test');
+const { BasicAuthenticator, NoAuthAuthenticator } = require('ibm-cloud-sdk-core');
+const authenticator = (url.username) ? new BasicAuthenticator({ username: url.username, password: decodeURIComponent(url.password) }) : new NoAuthAuthenticator();
+const serviceOpts = {
+  authenticator: authenticator
+};
+const cloudant = new CloudantV1(serviceOpts);
+// Remove auth from URL before using for service
+cloudant.setServiceUrl(new URL(url.pathname, url.origin).toString());
+const uuid = require('uuid').v4;
+const fs = require('fs');
+
+// Mocha hooks that will be at the root context so run for all tests
+
+beforeEach('Create test database', function(done) {
+  // Don't run hook for unit tests, just for CI
+  if (!this.currentTest.fullTitle().includes('#unit')) {
+    // Allow 10 seconds to create the DB
+    this.timeout(10 * 1000);
+    const unique = uuid();
+    this.fileName = `${unique}`;
+    this.dbName = 'couchbackup_test_' + unique;
+    cloudant.putDatabase({ db: this.dbName }).then(() => { done(); }).catch((err) => { done(err); });
+  } else {
+    done();
+  }
+});
+
+afterEach('Delete test database', function(done) {
+  // Don't run hook for unit tests, just for CI
+  if (!this.currentTest.fullTitle().includes('#unit')) {
+    // Allow 10 seconds to delete the DB
+    this.timeout(10 * 1000);
+    deleteIfExists(this.fileName);
+    deleteIfExists(`${this.fileName}.log`);
+    cloudant.deleteDatabase({ db: this.dbName }).then(() => { done(); }).catch((err) => { done(err); });
+  } else {
+    done();
+  }
+});
+
+function deleteIfExists(fileName) {
+  fs.unlink(fileName, function(err) {
+    if (err) {
+      if (err.code !== 'ENOENT') {
+        console.error(`${err.code} ${err.message}`);
+      }
+    }
+  });
+}

package/.scannerwork/scanner-report/source-39.txt

@@ -0,0 +1,147 @@
+// Copyright © 2017, 2021 IBM Corp. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+/* global describe it beforeEach */
+'use strict';
+
+const assert = require('assert');
+const fs = require('fs');
+const nock = require('nock');
+const request = require('../includes/request.js');
+const writer = require('../includes/writer.js');
+const noopEmitter = new (require('events')).EventEmitter();
+const liner = require('../includes/liner.js');
+
+const longTestTimeout = 3000;
+
+// The writer expects a line-by-line stream so this utility function does that
+// processing for the tests (which normally happens in the internal restore
+// function).
+function testLinestream(fixture = './test/fixtures/animaldb_expected.json') {
+  return fs.createReadStream(fixture)
+    .pipe(liner());
+}
+
+describe('#unit Check database restore writer', function() {
+  const dbUrl = 'http://localhost:5984/animaldb';
+  const db = request.client(dbUrl, { parallelism: 1 });
+
+  beforeEach('Reset nocks', function() {
+    nock.cleanAll();
+  });
+
+  it('should complete successfully', function(done) {
+    nock(dbUrl)
+      .post('/_bulk_docs')
+      .reply(200, []); // success
+
+    testLinestream()
+      .pipe(writer(db, 500, 1, noopEmitter))
+      .on('error', function(err) {
+        done(err);
+      })
+      .on('finished', function(data) {
+        assert.strictEqual(data.total, 15);
+        assert.ok(nock.isDone());
+        done();
+      });
+  });
+
+  it('should terminate on a fatal error', function(done) {
+    nock(dbUrl)
+      .post('/_bulk_docs')
+      .reply(401, { error: 'Unauthorized' }); // fatal error
+
+    testLinestream()
+      .pipe(writer(db, 500, 1, noopEmitter))
+      .on('error', function(err) {
+        assert.strictEqual(err.name, 'Unauthorized');
+        assert.strictEqual(err.message, 'Access is denied due to invalid credentials.');
+        assert.ok(nock.isDone());
+        done();
+      });
+  });
+
+  it('should retry on transient errors', function(done) {
+    nock(dbUrl)
+      .post('/_bulk_docs')
+      .reply(429, { error: 'Too Many Requests' }) // transient error
+      .post('/_bulk_docs')
+      .reply(500, { error: 'Internal Server Error' }) // transient error
+      .post('/_bulk_docs')
+      .reply(200, { ok: true }); // third time lucky success
+
+    testLinestream()
+      .pipe(writer(db, 500, 1, noopEmitter))
+      .on('error', function(err) {
+        done(err);
+      })
+      .on('finished', function(data) {
+        assert.strictEqual(data.total, 15);
+        assert.ok(nock.isDone());
+        done();
+      });
+  }).timeout(longTestTimeout);
+
+  it('should fail after 3 transient errors', function(done) {
+    nock(dbUrl)
+      .post('/_bulk_docs')
+      .reply(429, { error: 'Too Many Requests' }) // transient error
+      .post('/_bulk_docs')
+      .reply(500, { error: 'Internal Server Error' }) // transient error
+      .post('/_bulk_docs')
+      .reply(503, { error: 'Service Unavailable' }); // Final transient error
+
+    testLinestream()
+      .pipe(writer(db, 500, 1, noopEmitter))
+      .on('error', function(err) {
+        assert.strictEqual(err.name, 'HTTPFatalError');
+        assert.strictEqual(err.message, `503 : post ${dbUrl}/_bulk_docs - Error: Service Unavailable`);
+        assert.ok(nock.isDone());
+        done();
+      });
+  }).timeout(longTestTimeout);
+
+  it('should restore shallow backups without rev info successfully', function(done) {
+    nock(dbUrl)
+      .post('/_bulk_docs')
+      .reply(200, [{ ok: true, id: 'foo', rev: '1-abc' }]); // success
+
+    testLinestream('./test/fixtures/animaldb_old_shallow.json')
+      .pipe(writer(db, 500, 1, noopEmitter))
+      .on('error', function(err) {
+        done(err);
+      })
+      .on('finished', function(data) {
+        assert.strictEqual(data.total, 11);
+        assert.ok(nock.isDone());
+        done();
+      });
+  });
+
+  it('should get a batch error for non-empty array response with new_edits false', function(done) {
+    nock(dbUrl)
+      .post('/_bulk_docs')
+      .reply(200, [{ id: 'foo', error: 'foo', reason: 'bar' }]);
+
+    testLinestream()
+      .pipe(writer(db, 500, 1, noopEmitter))
+      .on('error', function(err) {
+        assert.strictEqual(err.name, 'Error');
+        assert.strictEqual(err.message, 'Error writing batch with new_edits:false and 1 items');
+        assert.ok(nock.isDone());
+        done();
+      });
+  });
+});