@cloudant/couchbackup 2.9.16 → 2.9.17-SNAPSHOT.185

package/.scannerwork/scanner-report/source-19.txt

@@ -1,4 +1,4 @@
-// Copyright © 2017, 2023 IBM Corp. All rights reserved.
+// Copyright © 2017, 2018 IBM Corp. All rights reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -11,124 +11,49 @@
 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 // See the License for the specific language governing permissions and
 // limitations under the License.
-
-/* global describe it beforeEach */
 'use strict';
 
-const assert = require('assert');
-const fs = require('fs');
-const nock = require('nock');
-const request = require('../includes/request.js');
-const writer = require('../includes/writer.js');
-const noopEmitter = new (require('events')).EventEmitter();
-const liner = require('../includes/liner.js');
-const { once } = require('node:events');
-const { pipeline } = require('node:stream/promises');
-const longTestTimeout = 3000;
-
-describe('#unit Check database restore writer', function() {
-  const dbUrl = 'http://localhost:5984/animaldb';
-  const db = request.client(dbUrl, { parallelism: 1 });
-
-  beforeEach('Reset nocks', function() {
-    nock.cleanAll();
-  });
-
-  it('should complete successfully', async function() {
-    nock(dbUrl)
-      .post('/_bulk_docs')
-      .reply(200, []); // success
-
-    const w = writer(db, 500, 1, noopEmitter);
-    return Promise.all([pipeline(fs.createReadStream('./test/fixtures/animaldb_expected.json'), liner(), w),
-      once(w, 'finished').then((data) => {
-        assert.strictEqual(data[0].total, 15);
-        assert.ok(nock.isDone());
-      })]);
-  });
-
-  it('should terminate on a fatal error', async function() {
-    nock(dbUrl)
-      .post('/_bulk_docs')
-      .reply(401, { error: 'Unauthorized' }); // fatal error
-
-    const w = writer(db, 500, 1, noopEmitter);
-    return assert.rejects(
-      pipeline(fs.createReadStream('./test/fixtures/animaldb_expected.json'), liner(), w),
-      (err) => {
-        assert.strictEqual(err.name, 'Unauthorized');
-        assert.strictEqual(err.message, 'Access is denied due to invalid credentials.');
-        assert.ok(nock.isDone());
-        return true;
-      }
-    );
-  });
-
-  it('should retry on transient errors', async function() {
-    nock(dbUrl)
-      .post('/_bulk_docs')
-      .reply(429, { error: 'Too Many Requests' }) // transient error
-      .post('/_bulk_docs')
-      .reply(500, { error: 'Internal Server Error' }) // transient error
-      .post('/_bulk_docs')
-      .reply(200, { ok: true }); // third time lucky success
-
-    const w = writer(db, 500, 1, noopEmitter);
-    return Promise.all([pipeline(fs.createReadStream('./test/fixtures/animaldb_expected.json'), liner(), w),
-      once(w, 'finished').then((data) => {
-        assert.strictEqual(data[0].total, 15);
-        assert.ok(nock.isDone());
-      })]);
-  }).timeout(longTestTimeout);
-
-  it('should fail after 3 transient errors', async function() {
-    nock(dbUrl)
-      .post('/_bulk_docs')
-      .reply(429, { error: 'Too Many Requests' }) // transient error
-      .post('/_bulk_docs')
-      .reply(500, { error: 'Internal Server Error' }) // transient error
-      .post('/_bulk_docs')
-      .reply(503, { error: 'Service Unavailable' }); // Final transient error
-
-    const w = writer(db, 500, 1, noopEmitter);
-    return assert.rejects(
-      pipeline(fs.createReadStream('./test/fixtures/animaldb_expected.json'), liner(), w),
-      (err) => {
-        assert.strictEqual(err.name, 'HTTPFatalError');
-        assert.strictEqual(err.message, `503 : post ${dbUrl}/_bulk_docs - Error: Service Unavailable`);
-        assert.ok(nock.isDone());
-        return true;
-      }
-    );
-  }).timeout(longTestTimeout);
-
-  it('should restore shallow backups without rev info successfully', async function() {
-    nock(dbUrl)
-      .post('/_bulk_docs')
-      .reply(200, [{ ok: true, id: 'foo', rev: '1-abc' }]); // success
-
-    const w = writer(db, 500, 1, noopEmitter);
-    return Promise.all([pipeline(fs.createReadStream('./test/fixtures/animaldb_old_shallow.json'), liner(), w),
-      once(w, 'finished').then((data) => {
-        assert.strictEqual(data[0].total, 11);
-        assert.ok(nock.isDone());
-      })]);
-  });
-
-  it('should get a batch error for non-empty array response with new_edits false', async function() {
-    nock(dbUrl)
-      .post('/_bulk_docs')
-      .reply(200, [{ id: 'foo', error: 'foo', reason: 'bar' }]);
-
-    const w = writer(db, 500, 1, noopEmitter);
-    return assert.rejects(
-      pipeline(fs.createReadStream('./test/fixtures/animaldb_expected.json'), liner(), w),
-      (err) => {
-        assert.strictEqual(err.name, 'Error');
-        assert.strictEqual(err.message, 'Error writing batch with new_edits:false and 1 items');
-        assert.ok(nock.isDone());
-        return true;
-      }
-    );
-  });
-});
+/**
+ * Utility methods for the command line interface.
+ * @module cliutils
+ * @see module:cliutils
+ */
+
+const url = require('url');
+const error = require('./error.js');
+
+module.exports = {
+
+  /**
+   * Combine a base URL and a database name, ensuring at least single slash
+   * between root and database name. This allows users to have Couch behind
+   * proxies that mount Couch's / endpoint at some other mount point.
+   * @param {string} root - root URL
+   * @param {string} databaseName - database name
+   * @return concatenated URL.
+   *
+   * @private
+   */
+  databaseUrl: function databaseUrl(root, databaseName) {
+    if (!root.endsWith('/')) {
+      root = root + '/';
+    }
+    try {
+      return new url.URL(encodeURIComponent(databaseName), root).toString();
+    } catch (err) {
+      throw error.wrapPossibleInvalidUrlError(err);
+    }
+  },
+
+  /**
+   * Generate CLI argument usage text.
+   *
+   * @param {string} description - argument description.
+   * @param {string} defaultValue - default argument value.
+   *
+   * @private
+   */
+  getUsage: function getUsage(description, defaultValue) {
+    return `${description} ${defaultValue !== undefined ? ` (default: ${defaultValue})` : ''}`;
+  }
+};

package/.scannerwork/scanner-report/source-20.txt

@@ -1,4 +1,4 @@
-// Copyright © 2017, 2023 IBM Corp. All rights reserved.
+// Copyright © 2017, 2021 IBM Corp. All rights reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -11,55 +11,103 @@
 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 // See the License for the specific language governing permissions and
 // limitations under the License.
-
-/* global beforeEach afterEach */
 'use strict';
 
-const { CloudantV1 } = require('@ibm-cloud/cloudant');
-const url = new URL((process.env.COUCH_BACKEND_URL) ? process.env.COUCH_BACKEND_URL : 'https://no-couch-backend-url-set.test');
-const { BasicAuthenticator, NoAuthAuthenticator } = require('ibm-cloud-sdk-core');
-const authenticator = (url.username) ? new BasicAuthenticator({ username: url.username, password: decodeURIComponent(url.password) }) : new NoAuthAuthenticator();
-const serviceOpts = {
-  authenticator: authenticator
+// fatal errors
+const codes = {
+  Error: 1,
+  InvalidOption: 2,
+  DatabaseNotFound: 10,
+  Unauthorized: 11,
+  Forbidden: 12,
+  DatabaseNotEmpty: 13,
+  NoLogFileName: 20,
+  LogDoesNotExist: 21,
+  IncompleteChangesInLogFile: 22,
+  SpoolChangesError: 30,
+  HTTPFatalError: 40,
+  BulkGetError: 50
 };
-const cloudant = new CloudantV1(serviceOpts);
-// Remove auth from URL before using for service
-cloudant.setServiceUrl(new URL(url.pathname, url.origin).toString());
-const uuid = require('uuid').v4;
-const fs = require('fs');
 
-// Mocha hooks that will be at the root context so run for all tests
+class BackupError extends Error {
+  constructor(name, message) {
+    super(message);
+    this.name = name;
+  }
+}
 
-beforeEach('Create test database', async function() {
-  // Don't run hook for unit tests, just for CI
-  if (!this.currentTest.fullTitle().includes('#unit')) {
-    // Allow 10 seconds to create the DB
-    this.timeout(10 * 1000);
-    const unique = uuid();
-    this.fileName = `${unique}`;
-    this.dbName = 'couchbackup_test_' + unique;
+class HTTPError extends BackupError {
+  constructor(responseError, name) {
+    // Special case some names for more useful error messages
+    switch (responseError.status) {
+      case 401:
+        name = 'Unauthorized';
+        break;
+      case 403:
+        name = 'Forbidden';
+        break;
+      default:
+        name = name || 'HTTPFatalError';
+    }
+    super(name, responseError.message);
+  }
+}
 
-    return cloudant.putDatabase({ db: this.dbName });
+// Default function to return an error for HTTP status codes
+// < 400 -> OK
+// 4XX (except 429) -> Fatal
+// 429 & >=500 -> Transient
+function checkResponse(err) {
+  if (err) {
+    // Construct an HTTPError if there is request information on the error
+    // Codes < 400 are considered OK
+    if (err.status >= 400) {
+      return new HTTPError(err);
+    } else {
+      // Send it back again if there was no status code, e.g. a cxn error
+      return augmentMessage(err);
+    }
   }
-});
+}
 
-afterEach('Delete test database', async function() {
-  // Don't run hook for unit tests, just for CI
-  if (!this.currentTest.fullTitle().includes('#unit')) {
-    // Allow 10 seconds to delete the DB
-    this.timeout(10 * 1000);
-    deleteIfExists(this.fileName);
-    deleteIfExists(`${this.fileName}.log`);
-    return cloudant.deleteDatabase({ db: this.dbName });
+function convertResponseError(responseError, errorFactory) {
+  if (!errorFactory) {
+    errorFactory = checkResponse;
   }
-});
+  return errorFactory(responseError);
+}
 
-function deleteIfExists(fileName) {
-  fs.unlink(fileName, function(err) {
+function augmentMessage(err) {
+  // For errors that don't have a status code, we are likely looking at a cxn
+  // error.
+  // Try to augment the message with more detail (core puts the code in statusText)
+  if (err && err.statusText) {
+    err.message = `${err.message} ${err.statusText}`;
+  }
+  if (err && err.description) {
+    err.message = `${err.message} ${err.description}`;
+  }
+  return err;
+}
+
+function wrapPossibleInvalidUrlError(err) {
+  if (err.code === 'ERR_INVALID_URL') {
+    // Wrap ERR_INVALID_URL in our own InvalidOption
+    return new BackupError('InvalidOption', err.message);
+  }
+  return err;
+}
+
+module.exports = {
+  BackupError,
+  HTTPError,
+  wrapPossibleInvalidUrlError,
+  convertResponseError,
+  terminationCallback: function terminationCallback(err, data) {
     if (err) {
-      if (err.code !== 'ENOENT') {
-        console.error(`${err.code} ${err.message}`);
-      }
+      console.error(`ERROR: ${err.message}`);
+      process.exitCode = codes[err.name] || 1;
+      process.exit();
     }
-  });
-}
+  }
+};

package/.scannerwork/scanner-report/source-21.txt

@@ -1,4 +1,4 @@
-// Copyright © 2017, 2023 IBM Corp. All rights reserved.
+// Copyright © 2017, 2021 IBM Corp. All rights reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -11,30 +11,154 @@
 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 // See the License for the specific language governing permissions and
 // limitations under the License.
-
-/* global describe it */
 'use strict';
 
-const assert = require('assert');
-const logfilesummary = require('../includes/logfilesummary.js');
-
-describe('#unit Fetching summary from the log file', function() {
-  it('should fetch a summary correctly', function() {
-    return new Promise((resolve, reject) => {
-      logfilesummary('./test/fixtures/test.log', function(err, data) {
-        try {
-          assert.ok(!err);
-          assert.ok(data);
-          assert.strictEqual(data.changesComplete, true);
-          assert.strictEqual(typeof data.batches, 'object');
-          assert.strictEqual(Object.keys(data.batches).length, 2);
-          assert.deepStrictEqual(data.batches['1'], true);
-          assert.deepStrictEqual(data.batches['4'], true);
-          resolve();
-        } catch (err) {
-          reject(err);
+const async = require('async');
+const stream = require('stream');
+const error = require('./error.js');
+const debug = require('debug')('couchbackup:writer');
+
+module.exports = function(db, bufferSize, parallelism, ee) {
+  const writer = new stream.Transform({ objectMode: true });
+  let buffer = [];
+  let written = 0;
+  let linenumber = 0;
+
+  // this is the queue of chunks that are written to the database
+  // the queue's payload will be an array of documents to be written,
+  // the size of the array will be bufferSize. The variable parallelism
+  // determines how many HTTP requests will occur at any one time.
+  const q = async.queue(function(payload, cb) {
+    // if we are restoring known revisions, we need to supply new_edits=false
+    if (payload.docs && payload.docs[0] && payload.docs[0]._rev) {
+      payload.new_edits = false;
+      debug('Using new_edits false mode.');
+    }
+
+    if (!didError) {
+      db.service.postBulkDocs({
+        db: db.db,
+        bulkDocs: payload
+      }).then(response => {
+        if (!response.result || (payload.new_edits === false && response.result.length > 0)) {
+          throw new Error(`Error writing batch with new_edits:${payload.new_edits !== false}` +
+            ` and ${response.result ? response.result.length : 'unavailable'} items`);
         }
+        written += payload.docs.length;
+        writer.emit('restored', { documents: payload.docs.length, total: written });
+        cb();
+      }).catch(err => {
+        err = error.convertResponseError(err);
+        debug(`Error writing docs ${err.name} ${err.message}`);
+        cb(err, payload);
       });
-    });
-  });
-});
+    }
+  }, parallelism);
+
+  let didError = false;
+
+  // write the contents of the buffer to CouchDB in blocks of bufferSize
+  function processBuffer(flush, callback) {
+    function taskCallback(err, payload) {
+      if (err && !didError) {
+        debug(`Queue task failed with error ${err.name}`);
+        didError = true;
+        q.kill();
+        writer.emit('error', err);
+      }
+    }
+
+    if (flush || buffer.length >= bufferSize) {
+      // work through the buffer to break off bufferSize chunks
+      // and feed the chunks to the queue
+      do {
+        // split the buffer into bufferSize chunks
+        const toSend = buffer.splice(0, bufferSize);
+
+        // and add the chunk to the queue
+        debug(`Adding ${toSend.length} to the write queue.`);
+        q.push({ docs: toSend }, taskCallback);
+      } while (buffer.length >= bufferSize);
+
+      // send any leftover documents to the queue
+      if (flush && buffer.length > 0) {
+        debug(`Adding remaining ${buffer.length} to the write queue.`);
+        q.push({ docs: buffer }, taskCallback);
+      }
+
+      // wait until the queue size falls to a reasonable level
+      async.until(
+        // wait until the queue length drops to twice the paralellism
+        // or until empty on the last write
+        function(callback) {
+          // if we encountered an error, stop this until loop
+          if (didError) {
+            return callback(null, true);
+          }
+          if (flush) {
+            callback(null, q.idle() && q.length() === 0);
+          } else {
+            callback(null, q.length() <= parallelism * 2);
+          }
+        },
+        function(cb) {
+          setTimeout(cb, 20);
+        },
+
+        function() {
+          if (flush && !didError) {
+            writer.emit('finished', { total: written });
+          }
+          // callback when we're happy with the queue size
+          callback();
+        });
+    } else {
+      callback();
+    }
+  }
+
+  // take an object
+  writer._transform = function(obj, encoding, done) {
+    // each obj that arrives here is a line from the backup file
+    // it should contain an array of objects. The length of the array
+    // depends on the bufferSize at backup time.
+    linenumber++;
+    if (!didError && obj !== '') {
+      // see if it parses as JSON
+      try {
+        const arr = JSON.parse(obj);
+
+        // if it's an array with a length
+        if (typeof arr === 'object' && arr.length > 0) {
+          // push each document into a buffer
+          buffer = buffer.concat(arr);
+
+          // pause the stream
+          // it's likely that the speed with which data can be read from disk
+          // may exceed the rate it can be written to CouchDB. To prevent
+          // the whole file being buffered in memory, we pause the stream here.
+          // it is resumed, when processBuffer calls back and we call done()
+          this.pause();
+
+          // break the buffer in to bufferSize chunks to be written to the database
+          processBuffer(false, done);
+        } else {
+          ee.emit('error', new error.BackupError('BackupFileJsonError', `Error on line ${linenumber} of backup file - not an array`));
+          done();
+        }
+      } catch (e) {
+        ee.emit('error', new error.BackupError('BackupFileJsonError', `Error on line ${linenumber} of backup file - cannot parse as JSON`));
+        // Could be an incomplete write that was subsequently resumed
+        done();
+      }
+    } else {
+      done();
+    }
+  };
+
+  // called when we need to flush everything
+  writer._flush = function(done) {
+    processBuffer(true, done);
+  };
+  return writer;
+};