@cloudant/couchbackup 2.9.16-SNAPSHOT.183 → 2.9.16

package/.scannerwork/scanner-report/source-10.txt

@@ -15,26 +15,23 @@
 /* global describe it */
 'use strict';
 
-const assert = require('assert');
-const logfilesummary = require('../includes/logfilesummary.js');
+delete require.cache[require.resolve('./citestutils.js')];
+const u = require('./citestutils.js');
 
-describe('#unit Fetching summary from the log file', function() {
-  it('should fetch a summary correctly', function() {
-    return new Promise((resolve, reject) => {
-      logfilesummary('./test/fixtures/test.log', function(err, data) {
-        try {
-          assert.ok(!err);
-          assert.ok(data);
-          assert.strictEqual(data.changesComplete, true);
-          assert.strictEqual(typeof data.batches, 'object');
-          assert.strictEqual(Object.keys(data.batches).length, 2);
-          assert.deepStrictEqual(data.batches['1'], true);
-          assert.deepStrictEqual(data.batches['4'], true);
-          resolve();
-        } catch (err) {
-          reject(err);
-        }
-      });
+[{ useApi: true }, { useApi: false }].forEach(function(params) {
+  describe(u.scenario('End to end backup and restore', params), function() {
+    it('should backup and restore animaldb', async function() {
+      // Allow up to 60 s for backup and restore of animaldb
+      u.setTimeout(this, 60);
+      return u.testDirectBackupAndRestore(params, 'animaldb', this.dbName);
+    });
+
+    it('should backup and restore largedb1g #slow', async function() {
+      // Allow up to 30 m for backup and restore of largedb1g
+      // This is a long time but when many builds run in parallel it can take a
+      // while to get this done.
+      u.setTimeout(this, 30 * 60);
+      return u.testDirectBackupAndRestore(params, 'largedb1g', this.dbName);
     });
   });
 });

package/.scannerwork/scanner-report/source-12.txt

@@ -12,31 +12,202 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-/* global describe it */
+/* global describe it beforeEach */
 'use strict';
 
 const assert = require('assert');
-const fs = require('fs');
-const u = require('./citestutils.js');
-
-describe('Write error tests', function() {
-  it('calls callback with error set when stream is not writeable', async function() {
-    u.setTimeout(this, 10);
-    const dirname = fs.mkdtempSync('test_backup_');
-    // make temp dir read only
-    fs.chmodSync(dirname, 0o444);
-    const filename = dirname + '/test.backup';
-    const backupStream = fs.createWriteStream(filename, { flags: 'w' });
-    const params = { useApi: true };
-    // try to do backup and check err was set in callback
-    return u.testBackup(params, 'animaldb', backupStream).then(() => {
-      assert.fail('Should throw an "EACCES" error');
-    }).catch((resultErr) => {
-      // cleanup temp dir
-      fs.rmdirSync(dirname);
-      // error should have been set
-      assert.ok(resultErr);
-      assert.strictEqual(resultErr.code, 'EACCES');
+const nock = require('nock');
+const request = require('../includes/request.js');
+const error = require('../includes/error.js');
+
+const url = 'http://localhost:7777/testdb';
+const db = request.client(url, { parallelism: 1 });
+const timeoutDb = request.client(url, { parallelism: 1, requestTimeout: 500 });
+const longTestTimeout = 3000;
+
+beforeEach('Clean nock', function() {
+  nock.cleanAll();
+});
+
+describe('#unit Check request headers', function() {
+  it('should have a couchbackup user-agent', async function() {
+    const couch = nock(url)
+      .matchHeader('user-agent', /couchbackup-cloudant\/\d+\.\d+\.\d+(?:-SNAPSHOT)? \(Node.js v\d+\.\d+\.\d+\)/)
+      .head('/good')
+      .reply(200);
+
+    return db.service.headDocument({ db: db.db, docId: 'good' }).then(() => {
+      assert.ok(couch.isDone());
+    });
+  });
+});
+
+describe('#unit Check request response error callback', function() {
+  it('should not callback with error for 200 response', async function() {
+    const couch = nock(url)
+      .get('/good')
+      .reply(200, { ok: true });
+
+    return db.service.getDocument({ db: db.db, docId: 'good' }).then(response => {
+      assert.ok(response.result);
+      assert.ok(couch.isDone());
+    });
+  });
+
+  it('should callback with error after 3 500 responses', async function() {
+    const couch = nock(url)
+      .get('/bad')
+      .times(3)
+      .reply(500, function(uri, requestBody) {
+        this.req.response.statusMessage = 'Internal Server Error';
+        return { error: 'foo', reason: 'bar' };
+      });
+
+    return assert.rejects(
+      db.service.getDocument({ db: db.db, docId: 'bad' }),
+      (err) => {
+        err = error.convertResponseError(err);
+        assert.strictEqual(err.name, 'HTTPFatalError');
+        assert.strictEqual(err.message, `500 Internal Server Error: get ${url}/bad - Error: foo, Reason: bar`);
+        assert.ok(couch.isDone());
+        return true;
+      });
+  }).timeout(longTestTimeout);
+
+  it('should callback with error after 3 POST 503 responses', async function() {
+    const couch = nock(url)
+      .post('/_bulk_get')
+      .query(true)
+      .times(3)
+      .reply(503, function(uri, requestBody) {
+        this.req.response.statusMessage = 'Service Unavailable';
+        return { error: 'service_unavailable', reason: 'Service unavailable' };
+      });
+
+    return assert.rejects(
+      db.service.postBulkGet({ db: db.db, revs: true, docs: [] }),
+      (err) => {
+        err = error.convertResponseError(err);
+        assert.strictEqual(err.name, 'HTTPFatalError');
+        assert.strictEqual(err.message, `503 Service Unavailable: post ${url}/_bulk_get - Error: service_unavailable, Reason: Service unavailable`);
+        assert.ok(couch.isDone());
+        return true;
+      });
+  }).timeout(longTestTimeout);
+
+  it('should callback with error after 3 429 responses', async function() {
+    const couch = nock(url)
+      .get('/bad')
+      .times(3)
+      .reply(429, function(uri, requestBody) {
+        this.req.response.statusMessage = 'Too Many Requests';
+        return { error: 'foo', reason: 'bar' };
+      });
+
+    return assert.rejects(
+      db.service.getDocument({ db: db.db, docId: 'bad' }),
+      (err) => {
+        err = error.convertResponseError(err);
+        assert.strictEqual(err.name, 'HTTPFatalError');
+        assert.strictEqual(err.message, `429 Too Many Requests: get ${url}/bad - Error: foo, Reason: bar`);
+        assert.ok(couch.isDone());
+        return true;
+      });
+  }).timeout(longTestTimeout);
+
+  it('should callback with fatal error for 404 response', async function() {
+    const couch = nock(url)
+      .get('/bad')
+      .reply(404, function(uri, requestBody) {
+        this.req.response.statusMessage = 'Not Found';
+        return { error: 'foo', reason: 'bar' };
+      });
+
+    return assert.rejects(
+      db.service.getDocument({ db: db.db, docId: 'bad' }),
+      (err) => {
+        err = error.convertResponseError(err);
+        assert.strictEqual(err.name, 'HTTPFatalError');
+        assert.strictEqual(err.message, `404 Not Found: get ${url}/bad - Error: foo, Reason: bar`);
+        assert.ok(couch.isDone());
+        return true;
+      });
+  });
+
+  it('should callback with same error for no status code error response', async function() {
+    const couch = nock(url)
+      .get('/bad')
+      .times(3)
+      .replyWithError('testing badness');
+
+    return assert.rejects(
+      db.service.getDocument({ db: db.db, docId: 'bad' }),
+      (err) => {
+        const err2 = error.convertResponseError(err);
+        assert.strictEqual(err, err2);
+        assert.ok(couch.isDone());
+        return true;
+      });
+  }).timeout(longTestTimeout);
+
+  it('should retry request if HTTP request gets timed out', async function() {
+    const couch = nock(url)
+      .post('/_bulk_get')
+      .query(true)
+      .delay(1000)
+      .reply(200, { results: { docs: [{ id: '1', ok: { _id: '1' } }] } })
+      .post('/_bulk_get')
+      .query(true)
+      .reply(200, { results: { docs: [{ id: '1', ok: { _id: '1' } }, { id: '2', ok: { _id: '2' } }] } });
+
+    return timeoutDb.service.postBulkGet({ db: db.db, revs: true, docs: [] }).then((response) => {
+      assert.ok(response);
+      assert.ok(response.result);
+      assert.ok(response.result.results);
+      assert.ok(response.result.results.docs);
+      assert.strictEqual(response.result.results.docs.length, 2);
+      assert.ok(couch.isDone());
+    });
+  });
+
+  it('should callback with error code ESOCKETTIMEDOUT if 3 HTTP requests gets timed out', async function() {
+    // Increase the timeout for this test to allow for the delays
+    this.timeout(3000);
+    const couch = nock(url)
+      .post('/_bulk_get')
+      .query(true)
+      .delay(1000)
+      .times(3)
+      .reply(200, { ok: true });
+
+    return assert.rejects(
+      timeoutDb.service.postBulkGet({ db: db.db, revs: true, docs: [] }),
+      (err) => {
+        err = error.convertResponseError(err);
+        // Note axios returns ECONNABORTED rather than ESOCKETTIMEDOUT
+        // See https://github.com/axios/axios/issues/2710 via https://github.com/axios/axios/issues/1543`
+        assert.strictEqual(err.statusText, 'ECONNABORTED');
+        assert.strictEqual(err.message, `timeout of 500ms exceeded: post ${url}/_bulk_get ECONNABORTED`);
+        assert.ok(couch.isDone());
+        return true;
+      });
+  });
+  describe('#unit Check credentials', async function() {
+    it('should properly decode username and password', async function() {
+      const username = 'user%123';
+      const password = 'colon:at@321';
+      const url = `http://${encodeURIComponent(username)}:${encodeURIComponent(password)}@localhost:7777/testdb`;
+      const sessionUrl = 'http://localhost:7777';
+      const couch = nock(sessionUrl)
+        .post('/_session', { username: username, password: password })
+        .reply(200, { ok: true }, { 'Set-Cookie': 'AuthSession=ABC123DEF4356;' })
+        .get('/')
+        .reply(200);
+      const db = request.client(url, { parallelism: 1 });
+      return db.service.getServerInformation().then(response => {
+        assert.ok(response);
+        assert.ok(couch.isDone());
+      });
     });
   });
 });

package/.scannerwork/scanner-report/source-13.txt

@@ -15,15 +15,87 @@
 /* global describe it */
 'use strict';
 
+const assert = require('assert');
+const fs = require('fs');
+const { once } = require('node:events');
 const u = require('./citestutils.js');
 
 [{ useApi: true }, { useApi: false }].forEach(function(params) {
-  describe(u.scenario('#slowest End to end backup and restore', params), function() {
-    // 10 GB is about the largest the CI can handle before getting very upset
-    // about how long things are taking
-    it('should backup and restore largedb10g', async function() {
-      u.setTimeout(this, 350 * 60);
-      return u.testDirectBackupAndRestore(params, 'largedb10g', this.dbName);
+  describe(u.scenario('Resume tests', params), function() {
+    it('should create a log file', async function() {
+      // Allow up to 90 s for this test
+      u.setTimeout(this, 60);
+
+      const actualBackup = `./${this.fileName}`;
+      const logFile = `./${this.fileName}` + '.log';
+      const p = u.p(params, { opts: { log: logFile } });
+      return u.testBackupToFile(p, 'animaldb', actualBackup).then(() => {
+        assert.ok(fs.existsSync(logFile), 'The log file should exist.');
+      });
+    });
+
+    it('should restore corrupted animaldb to a database correctly', async function() {
+      // Allow up to 60 s to restore and compare (again it should be faster)!
+      u.setTimeout(this, 60);
+      const input = fs.createReadStream('./test/fixtures/animaldb_corrupted.json');
+      const dbName = this.dbName;
+      const p = u.p(params, { expectedRestoreErrorRecoverable: { name: 'BackupFileJsonError' } });
+      return once(input, 'open')
+        .then(() => {
+          return u.testRestore(p, input, dbName);
+        }).then(() => {
+          return u.dbCompare('animaldb', dbName);
+        });
     });
+
+    it('should restore resumed animaldb with blank line to a database correctly', async function() {
+      // Allow up to 60 s to restore and compare (again it should be faster)!
+      u.setTimeout(this, 60);
+      const input = fs.createReadStream('./test/fixtures/animaldb_resumed_blank.json');
+      const dbName = this.dbName;
+      return once(input, 'open')
+        .then(() => {
+          return u.testRestore(params, input, dbName);
+        }).then(() => {
+          return u.dbCompare('animaldb', dbName);
+        });
+    });
+  });
+});
+
+describe('Resume tests', function() {
+  // Currently cannot abort API backups, when we do this test should be run for
+  // both API and CLI
+  it('should correctly backup and restore backup10m', async function() {
+    // Allow up to 90 s for this test
+    u.setTimeout(this, 90);
+
+    const actualBackup = `./${this.fileName}`;
+    const logFile = `./${this.fileName}` + '.log';
+    // Use abort parameter to terminate the backup
+    const p = u.p(params, { abort: true }, { opts: { log: logFile } });
+    const restoreDb = this.dbName;
+    // Set the database doc count as fewer than this should be written during
+    // resumed backup.
+    p.exclusiveMaxExpected = 5096;
+
+    return u.testBackupAbortResumeRestore(p, 'backup10m', actualBackup, restoreDb);
+  });
+  // Note --output is only valid for CLI usage, this test should only run for CLI
+  const params = { useApi: false };
+  it('should correctly backup and restore backup10m using --output', async function() {
+    // Allow up to 90 s for this test
+    u.setTimeout(this, 90);
+
+    const actualBackup = `./${this.fileName}`;
+    const logFile = `./${this.fileName}` + '.log';
+    // Use abort parameter to terminate the backup
+    const p = u.p(params, { abort: true }, { opts: { output: actualBackup, log: logFile } });
+    const restoreDb = this.dbName;
+    // Set the database doc count as fewer than this should be written during
+    // resumed backup.
+    p.exclusiveMaxExpected = 5096;
+
+    return await u.testBackupAbortResumeRestore(p, 'backup10m', actualBackup, restoreDb);
   });
 });

package/.scannerwork/scanner-report/source-14.txt

@@ -12,51 +12,119 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-/* global describe it */
+/* global describe it before after */
 'use strict';
 
-const fs = require('fs');
-const { once } = require('node:events');
-const u = require('./citestutils.js');
-
-[{ useApi: true }, { useApi: false }].forEach(function(params) {
-  describe(u.scenario('Compression tests', params), function() {
-    const p = u.p(params, { compression: true });
-
-    it('should backup animaldb to a compressed file', async function() {
-      // Allow up to 60 s for backup of animaldb
-      u.setTimeout(this, 60);
-      const compressedBackup = `./${this.fileName}`;
-      const output = fs.createWriteStream(compressedBackup);
-      return once(output, 'open')
-        .then(() => {
-          return u.testBackup(p, 'animaldb', output);
-        }).then(() => {
-          return u.assertGzipFile(compressedBackup);
-        });
-    });
-
-    it('should backup and restore animaldb via a compressed file', async function() {
-      // Allow up to 60 s for backup and restore of animaldb
-      u.setTimeout(this, 60);
-      const compressedBackup = `./${this.fileName}`;
-      return u.testBackupAndRestoreViaFile(p, 'animaldb', compressedBackup, this.dbName).then(() => {
-        return u.assertGzipFile(compressedBackup);
-      });
-    });
-
-    it('should backup and restore animaldb via a compressed stream', async function() {
-      // Allow up to 60 s for backup and restore of animaldb
-      u.setTimeout(this, 60);
-      return u.testDirectBackupAndRestore(p, 'animaldb', this.dbName);
-    });
-
-    it('should backup and restore largedb2g via a compressed file #slower', async function() {
-      // Takes ~ 25 min using CLI, but sometimes over an hour with API
-      u.setTimeout(this, 180 * 60);
-      const compressedBackup = `./${this.fileName}`;
-      params.compression = true;
-      return u.testBackupAndRestoreViaFile(p, 'largedb2g', compressedBackup, this.dbName);
-    });
+const assert = require('assert');
+const applyEnvVars = require('../includes/config.js').applyEnvironmentVariables;
+
+describe('#unit Configuration', function() {
+  let processEnvCopy;
+
+  before('Save env', function() {
+    // Copy env so we can reset it after the tests
+    processEnvCopy = JSON.parse(JSON.stringify(process.env));
+  });
+
+  after('Reset env', function() {
+    process.env = processEnvCopy;
+  });
+
+  it('respects the COUCH_URL env variable', function() {
+    process.env.COUCH_URL = 'http://user:pass@myurl.com';
+    const config = {};
+    applyEnvVars(config);
+    assert.strictEqual(typeof config.url, 'string');
+    assert.strictEqual(config.url, process.env.COUCH_URL);
+  });
+
+  it('respects the COUCH_DATABASE env variable', function() {
+    process.env.COUCH_DATABASE = 'mydb';
+    const config = {};
+    applyEnvVars(config);
+    assert.strictEqual(typeof config.db, 'string');
+    assert.strictEqual(config.db, process.env.COUCH_DATABASE);
+  });
+
+  it('respects the COUCH_BUFFER_SIZE env variable', function() {
+    process.env.COUCH_BUFFER_SIZE = '1000';
+    const config = {};
+    applyEnvVars(config);
+    assert.strictEqual(typeof config.bufferSize, 'number');
+    assert.strictEqual(config.bufferSize, 1000);
+  });
+
+  it('respects the COUCH_PARALLELISM env variable', function() {
+    process.env.COUCH_PARALLELISM = '20';
+    const config = {};
+    applyEnvVars(config);
+    assert.strictEqual(typeof config.parallelism, 'number');
+    assert.strictEqual(config.parallelism, 20);
+  });
+
+  it('respects the COUCH_REQUEST_TIMEOUT env variable', function() {
+    process.env.COUCH_REQUEST_TIMEOUT = '10000';
+    const config = {};
+    applyEnvVars(config);
+    assert.strictEqual(typeof config.requestTimeout, 'number');
+    assert.strictEqual(config.requestTimeout, 10000);
+  });
+
+  it('respects the CLOUDANT_IAM_API_KEY env variable', function() {
+    const key = 'ABC123-ZYX987_cba789-xyz321';
+    process.env.CLOUDANT_IAM_API_KEY = key;
+    const config = {};
+    applyEnvVars(config);
+    assert.strictEqual(typeof config.iamApiKey, 'string');
+    assert.strictEqual(config.iamApiKey, key);
+  });
+
+  it('respects the CLOUDANT_IAM_TOKEN_URL env variable', function() {
+    const u = 'https://testhost.example:1234/identity/token';
+    process.env.CLOUDANT_IAM_TOKEN_URL = u;
+    const config = {};
+    applyEnvVars(config);
+    assert.strictEqual(typeof config.iamTokenUrl, 'string');
+    assert.strictEqual(config.iamTokenUrl, u);
+  });
+
+  it('respects the COUCH_LOG env variable', function() {
+    process.env.COUCH_LOG = 'my.log';
+    const config = {};
+    applyEnvVars(config);
+    assert.strictEqual(typeof config.log, 'string');
+    assert.strictEqual(config.log, process.env.COUCH_LOG);
+  });
+
+  it('respects the COUCH_RESUME env variable', function() {
+    process.env.COUCH_RESUME = 'true';
+    const config = {};
+    applyEnvVars(config);
+    assert.strictEqual(typeof config.resume, 'boolean');
+    assert.strictEqual(config.resume, true);
+  });
+
+  it('respects the COUCH_OUTPUT env variable', function() {
+    process.env.COUCH_OUTPUT = 'myfile.txt';
+    const config = {};
+    applyEnvVars(config);
+    assert.strictEqual(typeof config.output, 'string');
+    assert.strictEqual(config.output, process.env.COUCH_OUTPUT);
+  });
+
+  it('respects the COUCH_MODE env variable', function() {
+    process.env.COUCH_MODE = 'shallow';
+    const config = {};
+    applyEnvVars(config);
+    assert.strictEqual(typeof config.mode, 'string');
+    assert.strictEqual(config.mode, 'shallow');
+  });
+
+  it('respects the COUCH_QUIET env variable', function() {
+    process.env.COUCH_QUIET = 'true';
+    const config = {};
+    applyEnvVars(config);
+    assert.strictEqual(typeof config.quiet, 'boolean');
+    assert.strictEqual(config.quiet, true);
   });
 });