@cloudant/couchbackup 2.9.10-SNAPSHOT.112 → 2.9.10

package/.scannerwork/scanner-report/source-4.txt

@@ -1,4 +1,4 @@
-// Copyright © 2017, 2021 IBM Corp. All rights reserved.
+// Copyright © 2017 IBM Corp. All rights reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -13,106 +13,80 @@
 // limitations under the License.
 'use strict';
 
-const path = require('path');
-const tmp = require('tmp');
-
-/**
-  Return API default settings.
-*/
-function apiDefaults() {
-  return {
-    parallelism: 5,
-    bufferSize: 500,
-    requestTimeout: 120000,
-    log: tmp.tmpNameSync(),
-    resume: false,
-    mode: 'full'
+const fs = require('fs');
+const stream = require('stream');
+const liner = require('./liner.js');
+
+const onLine = function(onCommand, getDocs) {
+  const change = new stream.Transform({ objectMode: true });
+
+  change._transform = function(line, encoding, done) {
+    if (line && line[0] === ':') {
+      const obj = {
+        command: null,
+        batch: null,
+        docs: []
+      };
+
+      let matches;
+
+      // extract command
+      matches = line.match(/^:([a-z_]+) ?/);
+      if (matches) {
+        obj.command = matches[1];
+      }
+
+      // extract batch
+      matches = line.match(/ batch([0-9]+)/);
+      if (matches) {
+        obj.batch = parseInt(matches[1]);
+      }
+
+      // extract doc ids
+      if (getDocs && obj.command === 't') {
+        const json = line.replace(/^.* batch[0-9]+ /, '').trim();
+        obj.docs = JSON.parse(json);
+      }
+      onCommand(obj);
+    }
+    done();
   };
-}
-
-/**
-  Return CLI default settings.
-*/
-function cliDefaults() {
-  const defaults = apiDefaults();
-
-  // add additional legacy settings
-  defaults.db = 'test';
-  defaults.url = 'http://localhost:5984';
-
-  // add CLI only option
-  defaults.quiet = false;
-
-  return defaults;
-}
+  return change;
+};
 
 /**
-  Override settings **in-place** with environment variables.
-*/
-function applyEnvironmentVariables(opts) {
-  // if we have a custom CouchDB url
-  if (typeof process.env.COUCH_URL !== 'undefined') {
-    opts.url = process.env.COUCH_URL;
-  }
-
-  // if we have a specified databases
-  if (typeof process.env.COUCH_DATABASE !== 'undefined') {
-    opts.db = process.env.COUCH_DATABASE;
-  }
-
-  // if we have a specified buffer size
-  if (typeof process.env.COUCH_BUFFER_SIZE !== 'undefined') {
-    opts.bufferSize = parseInt(process.env.COUCH_BUFFER_SIZE);
-  }
+ * Generate a list of remaining batches from a download file.
+ *
+ * @param {string} log - log file name
+ * @param {function} callback - callback with err, {changesComplete: N, batches: N}.
+ *  changesComplete signifies whether the log file appeared to
+ *  have completed reading the changes feed (contains :changes_complete).
+ *  batches are remaining batch IDs for download.
+ */
+module.exports = function(log, callback) {
+  // our sense of state
+  const state = {
 
-  // if we have a specified parallelism
-  if (typeof process.env.COUCH_PARALLELISM !== 'undefined') {
-    opts.parallelism = parseInt(process.env.COUCH_PARALLELISM);
-  }
-
-  // if we have a specified request timeout
-  if (typeof process.env.COUCH_REQUEST_TIMEOUT !== 'undefined') {
-    opts.requestTimeout = parseInt(process.env.COUCH_REQUEST_TIMEOUT);
-  }
-
-  // if we have a specified log file
-  if (typeof process.env.COUCH_LOG !== 'undefined') {
-    opts.log = path.normalize(process.env.COUCH_LOG);
-  }
-
-  // if we are instructed to resume
-  if (typeof process.env.COUCH_RESUME !== 'undefined' && process.env.COUCH_RESUME === 'true') {
-    opts.resume = true;
-  }
-
-  // if we are given an output filename
-  if (typeof process.env.COUCH_OUTPUT !== 'undefined') {
-    opts.output = path.normalize(process.env.COUCH_OUTPUT);
-  }
-
-  // if we only want a shallow copy
-  if (typeof process.env.COUCH_MODE !== 'undefined' && process.env.COUCH_MODE === 'shallow') {
-    opts.mode = 'shallow';
-  }
-
-  // if we are instructed to be quiet
-  if (typeof process.env.COUCH_QUIET !== 'undefined' && process.env.COUCH_QUIET === 'true') {
-    opts.quiet = true;
-  }
-
-  // if we have a specified API key
-  if (typeof process.env.CLOUDANT_IAM_API_KEY !== 'undefined') {
-    opts.iamApiKey = process.env.CLOUDANT_IAM_API_KEY;
-  }
-
-  // if we have a specified IAM token endpoint
-  if (typeof process.env.CLOUDANT_IAM_TOKEN_URL !== 'undefined') {
-    opts.iamTokenUrl = process.env.CLOUDANT_IAM_TOKEN_URL;
-  }
-}
+  };
+  let changesComplete = false;
+
+  // called with each line from the log file
+  const onCommand = function(obj) {
+    if (obj.command === 't') {
+      state[obj.batch] = true;
+    } else if (obj.command === 'd') {
+      delete state[obj.batch];
+    } else if (obj.command === 'changes_complete') {
+      changesComplete = true;
+    }
+  };
 
-module.exports = {
-  apiDefaults: apiDefaults,
-  cliDefaults: cliDefaults,
-  applyEnvironmentVariables: applyEnvironmentVariables
+  // stream through the previous log file
+  fs.createReadStream(log)
+    .pipe(liner())
+    .pipe(onLine(onCommand, false))
+    .on('finish', function() {
+      const obj = { changesComplete: changesComplete, batches: state };
+      callback(null, obj);
+    });
 };

package/.scannerwork/scanner-report/source-47.txt

@@ -1,4 +1,4 @@
-// Copyright © 2023 IBM Corp. All rights reserved.
+// Copyright © 2017 IBM Corp. All rights reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -12,89 +12,52 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+/* global describe it */
 'use strict';
 
-const chunk = require('lodash/chunk');
-const difference = require('lodash/difference');
-const forOwn = require('lodash/forOwn');
-const isEmpty = require('lodash/isEmpty');
-const union = require('lodash/union');
-
-const compare = async(database1, database2, client) => {
-  // check docs same in both dbs
-  const allDocs1 = await getAllDocs(client, database1);
-  const allDocs2 = await getAllDocs(client, database2);
-
-  const onlyInDb1 = (difference(allDocs1, allDocs2));
-  const onlyInDb2 = (difference(allDocs2, allDocs1));
-
-  let databasesSame = isEmpty(onlyInDb1) && isEmpty(onlyInDb2);
-
-  if (!databasesSame) {
-    console.log(onlyInDb1.length + ' documents only in db 1.');
-    console.log('Document IDs only in db 1: ' + onlyInDb1);
-    console.log(onlyInDb2.length + ' documents only in db 2.');
-    console.log('Document IDs only in db 2: ' + onlyInDb2);
-  }
-
-  // check revs same in docs common to both dbs
-  const partitionSize = 500;
-  const batches = chunk(union(allDocs1, allDocs2), partitionSize);
-
-  const missingRevsInDb2 = await getMissingRevs(client, database1, database2, batches);
-  const missingRevsInDb1 = await getMissingRevs(client, database2, database1, batches);
-
-  databasesSame = databasesSame && isEmpty(missingRevsInDb1) && isEmpty(missingRevsInDb2);
-
-  if (!databasesSame) {
-    console.log('Missing revs in db 1:' + JSON.stringify(missingRevsInDb1));
-    console.log('Missing revs in db 2:' + JSON.stringify(missingRevsInDb2));
-  }
-
-  return databasesSame;
-};
-
-const getMissingRevs = async(client, databaseName1, databaseName2, batcheses) => {
-  const fakeRevisionId = '9999-a';
-
-  const missing = {};
-
-  // look in db1 - use a fake revision ID to fetch all leaf revisions
-
-  for (const batches of batcheses) {
-    const documentRevisions = {};
-    batches.forEach(id => (documentRevisions[id] = [fakeRevisionId]));
-
-    const result1 = await client.postRevsDiff({ db: databaseName1, documentRevisions });
-    const revsDiffRequestDb2 = {};
-    forOwn(result1.result, (v, k) => (revsDiffRequestDb2[k] = v.possible_ancestors));
-    // look in db2
-    const result2 = await client.postRevsDiff({ db: databaseName2, documentRevisions: revsDiffRequestDb2 });
-    forOwn(result2.result, (v, k) => {
-      if ('missing' in v) {
-        missing[k] = v.missing;
+const fs = require('fs');
+const u = require('./citestutils.js');
+
+describe('Event tests', function() {
+  it('should get a finished event when using stdout', function(done) {
+    u.setTimeout(this, 40);
+    // Use the API so we can get events
+    const params = { useApi: true };
+    const backup = u.testBackup(params, 'animaldb', process.stdout, function(err) {
+      if (err) {
+        done(err);
       }
     });
-  }
-  return missing;
-};
-
-const getAllDocs = async(client, database) => {
-  let allDocIds = [];
-  const limit = 2000;
-  let startKey = '\u0000';
-  do {
-    const pageOfDocIds = (await client.postAllDocs({ db: database, startKey, limit })).result.rows.map(r => r.id);
-    allDocIds = allDocIds.concat(pageOfDocIds);
-    if (pageOfDocIds.length < limit) {
-      startKey = null;
-    } else {
-      startKey = pageOfDocIds[limit - 1] + '\u0000';
-    }
-  } while (startKey != null);
-  return allDocIds;
-};
-
-module.exports = {
-  compare
-};
+    backup.on('finished', function() {
+      try {
+        // Test will time out if the finished event is not emitted
+        done();
+      } catch (err) {
+        done(err);
+      }
+    });
+  });
+  it('should get a finished event when using file output', function(done) {
+    u.setTimeout(this, 40);
+    // Use the API so we can get events
+    const params = { useApi: true };
+    const actualBackup = `./${this.fileName}`;
+    // Create a file and backup to it
+    const output = fs.createWriteStream(actualBackup);
+    output.on('open', function() {
+      const backup = u.testBackup(params, 'animaldb', output, function(err) {
+        if (err) {
+          done(err);
+        }
+      });
+      backup.on('finished', function() {
+        try {
+          // Test will time out if the finished event is not emitted
+          done();
+        } catch (err) {
+          done(err);
+        }
+      });
+    });
+  });
+});

package/.scannerwork/scanner-report/source-48.txt

@@ -1,4 +1,4 @@
-// Copyright © 2017 IBM Corp. All rights reserved.
+// Copyright © 2017, 2021 IBM Corp. All rights reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -12,120 +12,57 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-/* global describe it */
+/* global beforeEach afterEach */
 'use strict';
 
+const { CloudantV1 } = require('@ibm-cloud/cloudant');
+const url = new URL((process.env.COUCH_BACKEND_URL) ? process.env.COUCH_BACKEND_URL : 'https://no-couch-backend-url-set.test');
+const { BasicAuthenticator, NoAuthAuthenticator } = require('ibm-cloud-sdk-core');
+const authenticator = (url.username) ? new BasicAuthenticator({ username: url.username, password: decodeURIComponent(url.password) }) : new NoAuthAuthenticator();
+const serviceOpts = {
+  authenticator: authenticator
+};
+const cloudant = new CloudantV1(serviceOpts);
+// Remove auth from URL before using for service
+cloudant.setServiceUrl(new URL(url.pathname, url.origin).toString());
+const uuid = require('uuid').v4;
 const fs = require('fs');
-const u = require('./citestutils.js');
 
-[{ useApi: true }, { useApi: false }].forEach(function(params) {
-  describe(u.scenario('Basic backup and restore', params), function() {
-    it('should backup animaldb to a file correctly', function(done) {
-      // Allow up to 40 s to backup and compare (it should be much faster)!
-      u.setTimeout(this, 40);
-      const actualBackup = `./${this.fileName}`;
-      // Create a file and backup to it
-      const output = fs.createWriteStream(actualBackup);
-      output.on('open', function() {
-        u.testBackup(params, 'animaldb', output, function(err) {
-          if (err) {
-            done(err);
-          } else {
-            u.readSortAndDeepEqual(actualBackup, './test/fixtures/animaldb_expected.json', done);
-          }
-        });
-      });
-    });
+// Mocha hooks that will be at the root context so run for all tests
 
-    it('should restore animaldb to a database correctly', function(done) {
-      // Allow up to 60 s to restore and compare (again it should be faster)!
-      u.setTimeout(this, 60);
-      const input = fs.createReadStream('./test/fixtures/animaldb_expected.json');
-      const dbName = this.dbName;
-      input.on('open', function() {
-        u.testRestore(params, input, dbName, function(err) {
-          if (err) {
-            done(err);
-          } else {
-            u.dbCompare('animaldb', dbName, done);
-          }
-        });
-      });
-    });
+beforeEach('Create test database', function(done) {
+  // Don't run hook for unit tests, just for CI
+  if (!this.currentTest.fullTitle().includes('#unit')) {
+    // Allow 10 seconds to create the DB
+    this.timeout(10 * 1000);
+    const unique = uuid();
+    this.fileName = `${unique}`;
+    this.dbName = 'couchbackup_test_' + unique;
+    cloudant.putDatabase({ db: this.dbName }).then(() => { done(); }).catch((err) => { done(err); });
+  } else {
+    done();
+  }
+});
 
-    it('should execute a shallow mode backup successfully', function(done) {
-      // Allow 30 s
-      u.setTimeout(this, 30);
-      const actualBackup = `./${this.fileName}`;
-      const output = fs.createWriteStream(actualBackup);
-      // Add the shallow mode option
-      const p = u.p(params, { opts: { mode: 'shallow' } });
-      output.on('open', function() {
-        u.testBackup(p, 'animaldb', output, function(err) {
-          if (err) {
-            done(err);
-          } else {
-            u.readSortAndDeepEqual(actualBackup, './test/fixtures/animaldb_expected_shallow.json', done);
-          }
-        });
-      });
-    });
+afterEach('Delete test database', function(done) {
+  // Don't run hook for unit tests, just for CI
+  if (!this.currentTest.fullTitle().includes('#unit')) {
+    // Allow 10 seconds to delete the DB
+    this.timeout(10 * 1000);
+    deleteIfExists(this.fileName);
+    deleteIfExists(`${this.fileName}.log`);
+    cloudant.deleteDatabase({ db: this.dbName }).then(() => { done(); }).catch((err) => { done(err); });
+  } else {
+    done();
+  }
+});
 
-    describe(u.scenario('Buffer size tests', params), function() {
-      it('should backup/restore animaldb with the same buffer size', function(done) {
-        // Allow up to 60 s for backup and restore of animaldb
-        u.setTimeout(this, 60);
-        const actualBackup = `./${this.fileName}`;
-        const logFile = `./${this.fileName}` + '.log';
-        const p = u.p(params, { opts: { log: logFile, bufferSize: 1 } });
-        u.testBackupAndRestoreViaFile(p, 'animaldb', actualBackup, this.dbName, done);
-      });
-      it('should backup/restore animaldb with backup buffer > restore buffer', function(done) {
-        // Allow up to 60 s for backup and restore of animaldb
-        u.setTimeout(this, 60);
-        const actualBackup = `./${this.fileName}`;
-        const logFile = `./${this.fileName}` + '.log';
-        const dbName = this.dbName;
-        const p = u.p(params, { opts: { log: logFile, bufferSize: 2 } }); // backup
-        const q = u.p(params, { opts: { bufferSize: 1 } }); // restore
-        u.testBackupToFile(p, 'animaldb', actualBackup, function(err) {
-          if (err) {
-            done(err);
-          } else {
-            // restore
-            u.testRestoreFromFile(q, actualBackup, dbName, function(err) {
-              if (!err) {
-                u.dbCompare('animaldb', dbName, done);
-              } else {
-                done(err);
-              }
-            });
-          }
-        });
-      });
-      it('should backup/restore animaldb with backup buffer < restore buffer', function(done) {
-        // Allow up to 60 s for backup and restore of animaldb
-        u.setTimeout(this, 60);
-        const actualBackup = `./${this.fileName}`;
-        const logFile = `./${this.fileName}` + '.log';
-        const dbName = this.dbName;
-        const p = u.p(params, { opts: { log: logFile, bufferSize: 1 } }); // backup
-        const q = u.p(params, { opts: { bufferSize: 2 } }); // restore
-        u.testBackupToFile(p, 'animaldb', actualBackup, function(err) {
-          if (err) {
-            done(err);
-          } else {
-            // restore
-            u.testRestoreFromFile(q, actualBackup, dbName, function(err) {
-              if (!err) {
-                u.dbCompare('animaldb', dbName, done);
-              } else {
-                done(err);
-              }
-            });
-          }
-        });
-      });
-    });
+function deleteIfExists(fileName) {
+  fs.unlink(fileName, function(err) {
+    if (err) {
+      if (err.code !== 'ENOENT') {
+        console.error(`${err.code} ${err.message}`);
+      }
+    }
   });
-});
+}