@cloudant/couchbackup 2.9.13-SNAPSHOT.143 → 2.9.13

package/.scannerwork/scanner-report/source-37.txt

@@ -1,4 +1,4 @@
-// Copyright © 2017, 2021 IBM Corp. All rights reserved.
+// Copyright © 2017, 2022 IBM Corp. All rights reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -12,131 +12,134 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-/* global describe it before after */
+/* global describe it beforeEach */
 'use strict';
 
 const assert = require('assert');
-const applyEnvVars = require('../includes/config.js').applyEnvironmentVariables;
-
-describe('#unit Configuration', function() {
-  let processEnvCopy;
-
-  before('Save env', function() {
-    // Copy env so we can reset it after the tests
-    processEnvCopy = JSON.parse(JSON.stringify(process.env));
-  });
-
-  after('Reset env', function() {
-    process.env = processEnvCopy;
-  });
-
-  it('respects the COUCH_URL env variable', function(done) {
-    process.env.COUCH_URL = 'http://user:pass@myurl.com';
-    const config = {};
-    applyEnvVars(config);
-    assert.strictEqual(typeof config.url, 'string');
-    assert.strictEqual(config.url, process.env.COUCH_URL);
-    done();
-  });
-
-  it('respects the COUCH_DATABASE env variable', function(done) {
-    process.env.COUCH_DATABASE = 'mydb';
-    const config = {};
-    applyEnvVars(config);
-    assert.strictEqual(typeof config.db, 'string');
-    assert.strictEqual(config.db, process.env.COUCH_DATABASE);
-    done();
-  });
-
-  it('respects the COUCH_BUFFER_SIZE env variable', function(done) {
-    process.env.COUCH_BUFFER_SIZE = '1000';
-    const config = {};
-    applyEnvVars(config);
-    assert.strictEqual(typeof config.bufferSize, 'number');
-    assert.strictEqual(config.bufferSize, 1000);
-    done();
-  });
-
-  it('respects the COUCH_PARALLELISM env variable', function(done) {
-    process.env.COUCH_PARALLELISM = '20';
-    const config = {};
-    applyEnvVars(config);
-    assert.strictEqual(typeof config.parallelism, 'number');
-    assert.strictEqual(config.parallelism, 20);
-    done();
-  });
-
-  it('respects the COUCH_REQUEST_TIMEOUT env variable', function(done) {
-    process.env.COUCH_REQUEST_TIMEOUT = '10000';
-    const config = {};
-    applyEnvVars(config);
-    assert.strictEqual(typeof config.requestTimeout, 'number');
-    assert.strictEqual(config.requestTimeout, 10000);
-    done();
-  });
-
-  it('respects the CLOUDANT_IAM_API_KEY env variable', function(done) {
-    const key = 'ABC123-ZYX987_cba789-xyz321';
-    process.env.CLOUDANT_IAM_API_KEY = key;
-    const config = {};
-    applyEnvVars(config);
-    assert.strictEqual(typeof config.iamApiKey, 'string');
-    assert.strictEqual(config.iamApiKey, key);
-    done();
-  });
-
-  it('respects the CLOUDANT_IAM_TOKEN_URL env variable', function(done) {
-    const u = 'https://testhost.example:1234/identity/token';
-    process.env.CLOUDANT_IAM_TOKEN_URL = u;
-    const config = {};
-    applyEnvVars(config);
-    assert.strictEqual(typeof config.iamTokenUrl, 'string');
-    assert.strictEqual(config.iamTokenUrl, u);
-    done();
-  });
-
-  it('respects the COUCH_LOG env variable', function(done) {
-    process.env.COUCH_LOG = 'my.log';
-    const config = {};
-    applyEnvVars(config);
-    assert.strictEqual(typeof config.log, 'string');
-    assert.strictEqual(config.log, process.env.COUCH_LOG);
-    done();
-  });
-
-  it('respects the COUCH_RESUME env variable', function(done) {
-    process.env.COUCH_RESUME = 'true';
-    const config = {};
-    applyEnvVars(config);
-    assert.strictEqual(typeof config.resume, 'boolean');
-    assert.strictEqual(config.resume, true);
-    done();
+const backup = require('../includes/shallowbackup.js');
+const request = require('../includes/request.js');
+const fs = require('fs');
+const nock = require('nock');
+
+// Function to create a DB object and call the shallow backup function
+// This is normally done by app.js
+function shallowBackup(dbUrl, opts) {
+  const db = request.client(dbUrl, opts);
+  // Disable compression to make body assertions easier
+  db.service.setEnableGzipCompression(false);
+  return backup(db, opts);
+}
+
+// Note all these tests include a body parameter of include_docs and a query
+// string of include_docs because of a quirk of nano that when using the fetch
+// method always adds the include_docs query string.
+describe('#unit Perform backup using shallow backup', function() {
+  const dbUrl = 'http://localhost:5984/animaldb';
+  // Query string keys are stringified by Nano
+  const badgerKey = 'badger\0';
+  const kookaburraKey = 'kookaburra\0';
+  const snipeKey = 'snipe\0';
+
+  beforeEach('Reset nocks', function() {
+    nock.cleanAll();
   });
 
-  it('respects the COUCH_OUTPUT env variable', function(done) {
-    process.env.COUCH_OUTPUT = 'myfile.txt';
-    const config = {};
-    applyEnvVars(config);
-    assert.strictEqual(typeof config.output, 'string');
-    assert.strictEqual(config.output, process.env.COUCH_OUTPUT);
-    done();
+  it('should perform a shallow backup', function(done) {
+    const couch = nock(dbUrl)
+      // batch 1
+      .post('/_all_docs', { limit: 3, include_docs: true })
+      .reply(200, JSON.parse(fs.readFileSync('./test/fixtures/animaldb_all_docs_1.json', 'utf8')))
+      // batch 2
+      .post('/_all_docs', { limit: 3, start_key: badgerKey, include_docs: true })
+      .reply(200, JSON.parse(fs.readFileSync('./test/fixtures/animaldb_all_docs_2.json', 'utf8')))
+      // batch 3
+      .post('/_all_docs', { limit: 3, start_key: kookaburraKey, include_docs: true })
+      .reply(200, JSON.parse(fs.readFileSync('./test/fixtures/animaldb_all_docs_3.json', 'utf8')))
+      // batch 4
+      .post('/_all_docs', { limit: 3, start_key: snipeKey, include_docs: true })
+      .reply(200, JSON.parse(fs.readFileSync('./test/fixtures/animaldb_all_docs_4.json', 'utf8')));
+
+    shallowBackup(dbUrl, { bufferSize: 3, parallelism: 1 })
+      .on('error', function(err) {
+        assert.fail(err);
+      })
+      .on('received', function(data) {
+        if (data.batch === 3) {
+          assert.strictEqual(data.length, 2); // smaller last batch
+        } else {
+          assert.strictEqual(data.length, 3);
+        }
+      })
+      .on('finished', function(data) {
+        assert.strictEqual(data.total, 11);
+        assert.ok(couch.isDone());
+        done();
+      });
   });
 
-  it('respects the COUCH_MODE env variable', function(done) {
-    process.env.COUCH_MODE = 'shallow';
-    const config = {};
-    applyEnvVars(config);
-    assert.strictEqual(typeof config.mode, 'string');
-    assert.strictEqual(config.mode, 'shallow');
-    done();
+  it('should perform a shallow backup with transient error', function(done) {
+    const couch = nock(dbUrl)
+      // batch 1
+      .post('/_all_docs', { limit: 3, include_docs: true })
+      .reply(200, JSON.parse(fs.readFileSync('./test/fixtures/animaldb_all_docs_1.json', 'utf8')))
+      // batch 2
+      .post('/_all_docs', { limit: 3, start_key: badgerKey, include_docs: true })
+      .reply(200, JSON.parse(fs.readFileSync('./test/fixtures/animaldb_all_docs_2.json', 'utf8')))
+      // batch 3 - transient error
+      .post('/_all_docs', { limit: 3, start_key: kookaburraKey, include_docs: true })
+      .reply(500, { error: 'Internal Server Error' })
+      // batch 3 - retry
+      .post('/_all_docs', { limit: 3, start_key: kookaburraKey, include_docs: true })
+      .reply(200, JSON.parse(fs.readFileSync('./test/fixtures/animaldb_all_docs_3.json', 'utf8')))
+      // batch 4
+      .post('/_all_docs', { limit: 3, start_key: snipeKey, include_docs: true })
+      .reply(200, JSON.parse(fs.readFileSync('./test/fixtures/animaldb_all_docs_4.json', 'utf8')));
+
+    shallowBackup(dbUrl, { bufferSize: 3, parallelism: 1 })
+      .on('error', function(err) {
+        assert.strictEqual(err.name, 'HTTPError');
+      })
+      .on('received', function(data) {
+        if (data.batch === 3) {
+          assert.strictEqual(data.length, 2); // smaller last batch
+        } else {
+          assert.strictEqual(data.length, 3);
+        }
+      })
+      .on('finished', function(data) {
+        assert.strictEqual(data.total, 11);
+        assert.ok(couch.isDone());
+        done();
+      });
   });
 
-  it('respects the COUCH_QUIET env variable', function(done) {
-    process.env.COUCH_QUIET = 'true';
-    const config = {};
-    applyEnvVars(config);
-    assert.strictEqual(typeof config.quiet, 'boolean');
-    assert.strictEqual(config.quiet, true);
-    done();
+  it('should fail to perform a shallow backup on fatal error', function(done) {
+    const couch = nock(dbUrl)
+      // batch 1
+      .post('/_all_docs', { limit: 3, include_docs: true })
+      .reply(200, JSON.parse(fs.readFileSync('./test/fixtures/animaldb_all_docs_1.json', 'utf8')))
+      // batch 2
+      .post('/_all_docs', { limit: 3, start_key: badgerKey, include_docs: true })
+      .reply(200, JSON.parse(fs.readFileSync('./test/fixtures/animaldb_all_docs_2.json', 'utf8')))
+      // batch 3 - fatal error
+      .post('/_all_docs', { limit: 3, start_key: kookaburraKey, include_docs: true })
+      .reply(401, { error: 'Unauthorized' });
+
+    let errCount = 0;
+
+    shallowBackup(dbUrl, { bufferSize: 3, parallelism: 1 })
+      .on('error', function(err) {
+        errCount++;
+        assert.strictEqual(err.name, 'Unauthorized');
+      })
+      .on('received', function(data) {
+        assert.strictEqual(data.length, 3);
+      })
+      .on('finished', function(data) {
+        assert.strictEqual(data.total, 6);
+        assert.ok(couch.isDone());
+        assert.strictEqual(errCount, 1);
+        done();
+      });
   });
 });

package/.scannerwork/scanner-report/{source-34.txt → source-38.txt}

@@ -16,18 +16,19 @@
 'use strict';
 
 const assert = require('assert');
-const logfilesummary = require('../includes/logfilesummary.js');
+const logfilegetbatches = require('../includes/logfilegetbatches.js');
 
-describe('#unit Fetching summary from the log file', function() {
-  it('should fetch a summary correctly', function(done) {
-    logfilesummary('./test/fixtures/test.log', function(err, data) {
+describe('#unit Fetching batches from a log file', function() {
+  it('should fetch multiple batches correctly', function(done) {
+    logfilegetbatches('./test/fixtures/test.log', [1, 4], function(err, data) {
       assert.ok(!err);
       assert.ok(data);
-      assert.strictEqual(data.changesComplete, true);
-      assert.strictEqual(typeof data.batches, 'object');
-      assert.strictEqual(Object.keys(data.batches).length, 2);
-      assert.deepStrictEqual(data.batches['1'], true);
-      assert.deepStrictEqual(data.batches['4'], true);
+      assert.strictEqual(typeof data, 'object');
+      assert.strictEqual(Object.keys(data).length, 2);
+      assert.deepStrictEqual(data['1'].docs, [{ id: '6' }, { id: '7' }, { id: '8' }, { id: '9' }, { id: '10' }]);
+      assert.strictEqual(data['1'].batch, 1);
+      assert.deepStrictEqual(data['4'].docs, [{ id: '21' }, { id: '22' }]);
+      assert.strictEqual(data['4'].batch, 4);
       done();
     });
   });

package/.scannerwork/scanner-report/source-39.txt

@@ -1,5 +1,4 @@
-#!/usr/bin/env node
-// Copyright © 2017, 2021 IBM Corp. All rights reserved.
+// Copyright © 2017, 2018 IBM Corp. All rights reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -12,45 +11,37 @@
 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 // See the License for the specific language governing permissions and
 // limitations under the License.
-'use strict';
-
-const error = require('../includes/error.js');
-const cliutils = require('../includes/cliutils.js');
-const couchbackup = require('../app.js');
-const parser = require('../includes/parser.js');
-const debug = require('debug');
-const restoreDebug = debug('couchbackup:restore');
-const restoreBatchDebug = debug('couchbackup:restore:batch');
-
-restoreDebug.enabled = true;
 
-const program = parser.parseRestoreArgs();
-const databaseUrl = cliutils.databaseUrl(program.url, program.db);
-const opts = {
-  bufferSize: program.bufferSize,
-  parallelism: program.parallelism,
-  requestTimeout: program.requestTimeout,
-  iamApiKey: program.iamApiKey,
-  iamTokenUrl: program.iamTokenUrl
-};
-
-// log configuration to console
-console.error('='.repeat(80));
-console.error('Performing restore on ' + databaseUrl.replace(/\/\/.+@/g, '//****:****@') + ' using configuration:');
-console.error(JSON.stringify(opts, null, 2).replace(/"iamApiKey": "[^"]+"/, '"iamApiKey": "****"'));
-console.error('='.repeat(80));
+/* global describe it */
+'use strict';
 
-restoreBatchDebug.enabled = !program.quiet;
+const assert = require('assert');
+const fs = require('fs');
+const u = require('./citestutils.js');
 
-return couchbackup.restore(
-  process.stdin, // restore from stdin
-  databaseUrl,
-  opts,
-  error.terminationCallback
-).on('restored', function(obj) {
-  restoreBatchDebug('restored', obj.total);
-}).on('error', function(e) {
-  restoreDebug('ERROR', e);
-}).on('finished', function(obj) {
-  restoreDebug('finished', obj);
+describe('Write error tests', function() {
+  it('calls callback with error set when stream is not writeable', function(done) {
+    u.setTimeout(this, 10);
+    const dirname = fs.mkdtempSync('test_backup_');
+    // make temp dir read only
+    fs.chmodSync(dirname, 0o444);
+    const filename = dirname + '/test.backup';
+    const backupStream = fs.createWriteStream(filename, { flags: 'w' });
+    const params = { useApi: true };
+    // try to do backup and check err was set in callback
+    u.testBackup(params, 'animaldb', backupStream, function(resultErr) {
+      let err = null;
+      try {
+        // cleanup temp dir
+        fs.rmdirSync(dirname);
+        // error should have been set
+        assert.ok(resultErr);
+        assert.strictEqual(resultErr.code, 'EACCES');
+      } catch (thrownErr) {
+        err = thrownErr;
+      } finally {
+        done(err);
+      }
+    });
+  });
 });

package/.scannerwork/scanner-report/source-40.txt

@@ -1,5 +1,4 @@
-#!/usr/bin/env node
-// Copyright © 2017, 2021 IBM Corp. All rights reserved.
+// Copyright © 2017 IBM Corp. All rights reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -12,66 +11,24 @@
 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 // See the License for the specific language governing permissions and
 // limitations under the License.
-'use strict';
-
-const error = require('../includes/error.js');
-const fs = require('fs');
-const cliutils = require('../includes/cliutils.js');
-const couchbackup = require('../app.js');
-const parser = require('../includes/parser.js');
-const debug = require('debug');
-const backupDebug = debug('couchbackup:backup');
-const backupBatchDebug = debug('couchbackup:backup:batch');
-
-backupDebug.enabled = true;
-
-const program = parser.parseBackupArgs();
-
-const databaseUrl = cliutils.databaseUrl(program.url, program.db);
-const opts = {
-  bufferSize: program.bufferSize,
-  log: program.log,
-  mode: program.mode,
-  parallelism: program.parallelism,
-  requestTimeout: program.requestTimeout,
-  resume: program.resume,
-  iamApiKey: program.iamApiKey,
-  iamTokenUrl: program.iamTokenUrl
-};
 
-// log configuration to console
-console.error('='.repeat(80));
-console.error('Performing backup on ' + databaseUrl.replace(/\/\/.+@/g, '//****:****@') + ' using configuration:');
-console.error(JSON.stringify(opts, null, 2).replace(/"iamApiKey": "[^"]+"/, '"iamApiKey": "****"'));
-console.error('='.repeat(80));
-
-backupBatchDebug.enabled = !program.quiet;
-
-let ws = process.stdout;
-
-// open output file
-if (program.output) {
-  let flags = 'w';
-  if (program.log && program.resume) {
-    flags = 'a';
-  }
-  const fd = fs.openSync(program.output, flags);
-  ws = fs.createWriteStream(null, { fd: fd });
-}
-
-backupDebug('Fetching all database changes...');
+/* global describe it */
+'use strict';
 
-return couchbackup.backup(
-  databaseUrl,
-  ws,
-  opts,
-  error.terminationCallback
-).on('changes', function(batch) {
-  backupBatchDebug('Total batches received:', batch + 1);
-}).on('written', function(obj) {
-  backupBatchDebug('Written batch ID:', obj.batch, 'Total document revisions written:', obj.total, 'Time:', obj.time);
-}).on('error', function(e) {
-  backupDebug('ERROR', e);
-}).on('finished', function(obj) {
-  backupDebug('Finished - Total document revisions written:', obj.total);
+const assert = require('assert');
+const logfilesummary = require('../includes/logfilesummary.js');
+
+describe('#unit Fetching summary from the log file', function() {
+  it('should fetch a summary correctly', function(done) {
+    logfilesummary('./test/fixtures/test.log', function(err, data) {
+      assert.ok(!err);
+      assert.ok(data);
+      assert.strictEqual(data.changesComplete, true);
+      assert.strictEqual(typeof data.batches, 'object');
+      assert.strictEqual(Object.keys(data.batches).length, 2);
+      assert.deepStrictEqual(data.batches['1'], true);
+      assert.deepStrictEqual(data.batches['4'], true);
+      done();
+    });
+  });
 });

package/.scannerwork/scanner-report/{source-13.txt → source-41.txt}

@@ -17,20 +17,13 @@
 
 const u = require('./citestutils.js');
 
-describe('Encryption tests', function() {
-  // Note CLI only to use openssl command
-  const p = { useApi: false, encryption: true };
-
-  it('should backup and restore animaldb via an encrypted file', function(done) {
-    // Allow up to 60 s for backup and restore of animaldb
-    u.setTimeout(this, 60);
-    const encryptedBackup = `./${this.fileName}`;
-    u.testBackupAndRestoreViaFile(p, 'animaldb', encryptedBackup, this.dbName, function(err) {
-      if (err) {
-        done(err);
-      } else {
-        u.assertEncryptedFile(encryptedBackup, done);
-      }
+[{ useApi: true }, { useApi: false }].forEach(function(params) {
+  describe(u.scenario('#slowest End to end backup and restore', params), function() {
+    // 10 GB is about the largest the CI can handle before getting very upset
+    // about how long things are taking
+    it('should backup and restore largedb10g', function(done) {
+      u.setTimeout(this, 350 * 60);
+      u.testDirectBackupAndRestore(params, 'largedb10g', this.dbName, done);
     });
   });
 });