@cloudant/couchbackup 2.9.16 → 2.9.17-SNAPSHOT.186

package/.scannerwork/scanner-report/source-46.txt

@@ -1,4 +1,4 @@
-// Copyright © 2017 IBM Corp. All rights reserved.
+// Copyright © 2017, 2023 IBM Corp. All rights reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -11,31 +11,203 @@
 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 // See the License for the specific language governing permissions and
 // limitations under the License.
+
+/* global describe it beforeEach */
 'use strict';
 
-// stolen from http://strongloop.com/strongblog/practical-examples-of-the-new-node-js-streams-api/
-const stream = require('stream');
+const assert = require('assert');
+const nock = require('nock');
+const request = require('../includes/request.js');
+const error = require('../includes/error.js');
+
+const url = 'http://localhost:7777/testdb';
+const db = request.client(url, { parallelism: 1 });
+const timeoutDb = request.client(url, { parallelism: 1, requestTimeout: 500 });
+const longTestTimeout = 3000;
+
+beforeEach('Clean nock', function() {
+  nock.cleanAll();
+});
+
+describe('#unit Check request headers', function() {
+  it('should have a couchbackup user-agent', async function() {
+    const couch = nock(url)
+      .matchHeader('user-agent', /couchbackup-cloudant\/\d+\.\d+\.\d+(?:-SNAPSHOT)? \(Node.js v\d+\.\d+\.\d+\)/)
+      .head('/good')
+      .reply(200);
+
+    return db.service.headDocument({ db: db.db, docId: 'good' }).then(() => {
+      assert.ok(couch.isDone());
+    });
+  });
+});
+
+describe('#unit Check request response error callback', function() {
+  it('should not callback with error for 200 response', async function() {
+    const couch = nock(url)
+      .get('/good')
+      .reply(200, { ok: true });
+
+    return db.service.getDocument({ db: db.db, docId: 'good' }).then(response => {
+      assert.ok(response.result);
+      assert.ok(couch.isDone());
+    });
+  });
+
+  it('should callback with error after 3 500 responses', async function() {
+    const couch = nock(url)
+      .get('/bad')
+      .times(3)
+      .reply(500, function(uri, requestBody) {
+        this.req.response.statusMessage = 'Internal Server Error';
+        return { error: 'foo', reason: 'bar' };
+      });
+
+    return assert.rejects(
+      db.service.getDocument({ db: db.db, docId: 'bad' }),
+      (err) => {
+        err = error.convertResponseError(err);
+        assert.strictEqual(err.name, 'HTTPFatalError');
+        assert.strictEqual(err.message, `500 Internal Server Error: get ${url}/bad - Error: foo, Reason: bar`);
+        assert.ok(couch.isDone());
+        return true;
+      });
+  }).timeout(longTestTimeout);
+
+  it('should callback with error after 3 POST 503 responses', async function() {
+    const couch = nock(url)
+      .post('/_bulk_get')
+      .query(true)
+      .times(3)
+      .reply(503, function(uri, requestBody) {
+        this.req.response.statusMessage = 'Service Unavailable';
+        return { error: 'service_unavailable', reason: 'Service unavailable' };
+      });
+
+    return assert.rejects(
+      db.service.postBulkGet({ db: db.db, revs: true, docs: [] }),
+      (err) => {
+        err = error.convertResponseError(err);
+        assert.strictEqual(err.name, 'HTTPFatalError');
+        assert.strictEqual(err.message, `503 Service Unavailable: post ${url}/_bulk_get - Error: service_unavailable, Reason: Service unavailable`);
+        assert.ok(couch.isDone());
+        return true;
+      });
+  }).timeout(longTestTimeout);
+
+  it('should callback with error after 3 429 responses', async function() {
+    const couch = nock(url)
+      .get('/bad')
+      .times(3)
+      .reply(429, function(uri, requestBody) {
+        this.req.response.statusMessage = 'Too Many Requests';
+        return { error: 'foo', reason: 'bar' };
+      });
+
+    return assert.rejects(
+      db.service.getDocument({ db: db.db, docId: 'bad' }),
+      (err) => {
+        err = error.convertResponseError(err);
+        assert.strictEqual(err.name, 'HTTPFatalError');
+        assert.strictEqual(err.message, `429 Too Many Requests: get ${url}/bad - Error: foo, Reason: bar`);
+        assert.ok(couch.isDone());
+        return true;
+      });
+  }).timeout(longTestTimeout);
+
+  it('should callback with fatal error for 404 response', async function() {
+    const couch = nock(url)
+      .get('/bad')
+      .reply(404, function(uri, requestBody) {
+        this.req.response.statusMessage = 'Not Found';
+        return { error: 'foo', reason: 'bar' };
+      });
+
+    return assert.rejects(
+      db.service.getDocument({ db: db.db, docId: 'bad' }),
+      (err) => {
+        err = error.convertResponseError(err);
+        assert.strictEqual(err.name, 'HTTPFatalError');
+        assert.strictEqual(err.message, `404 Not Found: get ${url}/bad - Error: foo, Reason: bar`);
+        assert.ok(couch.isDone());
+        return true;
+      });
+  });
+
+  it('should callback with same error for no status code error response', async function() {
+    const couch = nock(url)
+      .get('/bad')
+      .times(3)
+      .replyWithError('testing badness');
 
-module.exports = function(onChange) {
-  const change = new stream.Transform({ objectMode: true });
+    return assert.rejects(
+      db.service.getDocument({ db: db.db, docId: 'bad' }),
+      (err) => {
+        const err2 = error.convertResponseError(err);
+        assert.strictEqual(err, err2);
+        assert.ok(couch.isDone());
+        return true;
+      });
+  }).timeout(longTestTimeout);
 
-  change._transform = function(line, encoding, done) {
-    let obj = null;
+  it('should retry request if HTTP request gets timed out', async function() {
+    const couch = nock(url)
+      .post('/_bulk_get')
+      .query(true)
+      .delay(1000)
+      .reply(200, { results: { docs: [{ id: '1', ok: { _id: '1' } }] } })
+      .post('/_bulk_get')
+      .query(true)
+      .reply(200, { results: { docs: [{ id: '1', ok: { _id: '1' } }, { id: '2', ok: { _id: '2' } }] } });
 
-    // one change per line - remove the trailing comma
-    line = line.trim().replace(/,$/, '');
+    return timeoutDb.service.postBulkGet({ db: db.db, revs: true, docs: [] }).then((response) => {
+      assert.ok(response);
+      assert.ok(response.result);
+      assert.ok(response.result.results);
+      assert.ok(response.result.results.docs);
+      assert.strictEqual(response.result.results.docs.length, 2);
+      assert.ok(couch.isDone());
+    });
+  });
 
-    // extract thee last_seq at the end of the changes feed
-    if (line.match(/^"last_seq":/)) {
-      line = '{' + line;
-    }
-    try {
-      obj = JSON.parse(line);
-    } catch (e) {
-    }
-    onChange(obj);
-    done();
-  };
+  it('should callback with error code ESOCKETTIMEDOUT if 3 HTTP requests gets timed out', async function() {
+    // Increase the timeout for this test to allow for the delays
+    this.timeout(3000);
+    const couch = nock(url)
+      .post('/_bulk_get')
+      .query(true)
+      .delay(1000)
+      .times(3)
+      .reply(200, { ok: true });
 
-  return change;
-};
+    return assert.rejects(
+      timeoutDb.service.postBulkGet({ db: db.db, revs: true, docs: [] }),
+      (err) => {
+        err = error.convertResponseError(err);
+        // Note axios returns ECONNABORTED rather than ESOCKETTIMEDOUT
+        // See https://github.com/axios/axios/issues/2710 via https://github.com/axios/axios/issues/1543`
+        assert.strictEqual(err.statusText, 'ECONNABORTED');
+        assert.strictEqual(err.message, `timeout of 500ms exceeded: post ${url}/_bulk_get ECONNABORTED`);
+        assert.ok(couch.isDone());
+        return true;
+      });
+  });
+  describe('#unit Check credentials', async function() {
+    it('should properly decode username and password', async function() {
+      const username = 'user%123';
+      const password = 'colon:at@321';
+      const url = `http://${encodeURIComponent(username)}:${encodeURIComponent(password)}@localhost:7777/testdb`;
+      const sessionUrl = 'http://localhost:7777';
+      const couch = nock(sessionUrl)
+        .post('/_session', { username: username, password: password })
+        .reply(200, { ok: true }, { 'Set-Cookie': 'AuthSession=ABC123DEF4356;' })
+        .get('/')
+        .reply(200);
+      const db = request.client(url, { parallelism: 1 });
+      return db.service.getServerInformation().then(response => {
+        assert.ok(response);
+        assert.ok(couch.isDone());
+      });
+    });
+  });
+});

package/.scannerwork/scanner-report/source-47.txt

@@ -1,4 +1,4 @@
-// Copyright © 2017, 2021 IBM Corp. All rights reserved.
+// Copyright © 2017, 2023 IBM Corp. All rights reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -11,103 +11,27 @@
 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 // See the License for the specific language governing permissions and
 // limitations under the License.
-'use strict';
-
-// fatal errors
-const codes = {
-  Error: 1,
-  InvalidOption: 2,
-  DatabaseNotFound: 10,
-  Unauthorized: 11,
-  Forbidden: 12,
-  DatabaseNotEmpty: 13,
-  NoLogFileName: 20,
-  LogDoesNotExist: 21,
-  IncompleteChangesInLogFile: 22,
-  SpoolChangesError: 30,
-  HTTPFatalError: 40,
-  BulkGetError: 50
-};
-
-class BackupError extends Error {
-  constructor(name, message) {
-    super(message);
-    this.name = name;
-  }
-}
-
-class HTTPError extends BackupError {
-  constructor(responseError, name) {
-    // Special case some names for more useful error messages
-    switch (responseError.status) {
-      case 401:
-        name = 'Unauthorized';
-        break;
-      case 403:
-        name = 'Forbidden';
-        break;
-      default:
-        name = name || 'HTTPFatalError';
-    }
-    super(name, responseError.message);
-  }
-}
 
-// Default function to return an error for HTTP status codes
-// < 400 -> OK
-// 4XX (except 429) -> Fatal
-// 429 & >=500 -> Transient
-function checkResponse(err) {
-  if (err) {
-    // Construct an HTTPError if there is request information on the error
-    // Codes < 400 are considered OK
-    if (err.status >= 400) {
-      return new HTTPError(err);
-    } else {
-      // Send it back again if there was no status code, e.g. a cxn error
-      return augmentMessage(err);
-    }
-  }
-}
-
-function convertResponseError(responseError, errorFactory) {
-  if (!errorFactory) {
-    errorFactory = checkResponse;
-  }
-  return errorFactory(responseError);
-}
-
-function augmentMessage(err) {
-  // For errors that don't have a status code, we are likely looking at a cxn
-  // error.
-  // Try to augment the message with more detail (core puts the code in statusText)
-  if (err && err.statusText) {
-    err.message = `${err.message} ${err.statusText}`;
-  }
-  if (err && err.description) {
-    err.message = `${err.message} ${err.description}`;
-  }
-  return err;
-}
-
-function wrapPossibleInvalidUrlError(err) {
-  if (err.code === 'ERR_INVALID_URL') {
-    // Wrap ERR_INVALID_URL in our own InvalidOption
-    return new BackupError('InvalidOption', err.message);
-  }
-  return err;
-}
+/* global describe it */
+'use strict';
 
-module.exports = {
-  BackupError,
-  HTTPError,
-  wrapPossibleInvalidUrlError,
-  convertResponseError,
-  terminationCallback: function terminationCallback(err, data) {
-    if (err) {
-      console.error(`ERROR: ${err.message}`);
-      process.exitCode = codes[err.name] || 1;
-      process.exit();
-    }
-  }
-};
+const u = require('./citestutils.js');
+
+describe('Event tests', function() {
+  it('should get a finished event when using stdout', async function() {
+    u.setTimeout(this, 40);
+    // Use the API so we can get events, pass eventEmitter so we get the emitter back
+    const params = { useApi: true, useStdOut: true };
+    // All API backups now set an event listener for finished and it is part of the backup
+    // promise, so if the backup passes the finished event fired.
+    return u.testBackup(params, 'animaldb', process.stdout);
+  });
+
+  it('should get a finished event when using file output', async function() {
+    u.setTimeout(this, 40);
+    // Use the API so we can get events, pass eventEmitter so we get the emitter back
+    const params = { useApi: true };
+    const actualBackup = `./${this.fileName}`;
+    return u.testBackupToFile(params, 'animaldb', actualBackup);
+  });
+});

package/.scannerwork/scanner-report/source-48.txt

@@ -1,4 +1,4 @@
-// Copyright © 2017, 2021 IBM Corp. All rights reserved.
+// Copyright © 2017, 2023 IBM Corp. All rights reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -11,154 +11,31 @@
 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 // See the License for the specific language governing permissions and
 // limitations under the License.
-'use strict';
-
-const async = require('async');
-const stream = require('stream');
-const error = require('./error.js');
-const debug = require('debug')('couchbackup:writer');
-
-module.exports = function(db, bufferSize, parallelism, ee) {
-  const writer = new stream.Transform({ objectMode: true });
-  let buffer = [];
-  let written = 0;
-  let linenumber = 0;
 
-  // this is the queue of chunks that are written to the database
-  // the queue's payload will be an array of documents to be written,
-  // the size of the array will be bufferSize. The variable parallelism
-  // determines how many HTTP requests will occur at any one time.
-  const q = async.queue(function(payload, cb) {
-    // if we are restoring known revisions, we need to supply new_edits=false
-    if (payload.docs && payload.docs[0] && payload.docs[0]._rev) {
-      payload.new_edits = false;
-      debug('Using new_edits false mode.');
-    }
+/* global describe it */
+'use strict';
 
-    if (!didError) {
-      db.service.postBulkDocs({
-        db: db.db,
-        bulkDocs: payload
-      }).then(response => {
-        if (!response.result || (payload.new_edits === false && response.result.length > 0)) {
-          throw new Error(`Error writing batch with new_edits:${payload.new_edits !== false}` +
-            ` and ${response.result ? response.result.length : 'unavailable'} items`);
+const assert = require('assert');
+const logfilegetbatches = require('../includes/logfilegetbatches.js');
+
+describe('#unit Fetching batches from a log file', function() {
+  it('should fetch multiple batches correctly', async function() {
+    return new Promise((resolve, reject) => {
+      logfilegetbatches('./test/fixtures/test.log', [1, 4], function(err, data) {
+        try {
+          assert.ok(!err);
+          assert.ok(data);
+          assert.strictEqual(typeof data, 'object');
+          assert.strictEqual(Object.keys(data).length, 2);
+          assert.deepStrictEqual(data['1'].docs, [{ id: '6' }, { id: '7' }, { id: '8' }, { id: '9' }, { id: '10' }]);
+          assert.strictEqual(data['1'].batch, 1);
+          assert.deepStrictEqual(data['4'].docs, [{ id: '21' }, { id: '22' }]);
+          assert.strictEqual(data['4'].batch, 4);
+          resolve();
+        } catch (err) {
+          reject(err);
         }
-        written += payload.docs.length;
-        writer.emit('restored', { documents: payload.docs.length, total: written });
-        cb();
-      }).catch(err => {
-        err = error.convertResponseError(err);
-        debug(`Error writing docs ${err.name} ${err.message}`);
-        cb(err, payload);
       });
-    }
-  }, parallelism);
-
-  let didError = false;
-
-  // write the contents of the buffer to CouchDB in blocks of bufferSize
-  function processBuffer(flush, callback) {
-    function taskCallback(err, payload) {
-      if (err && !didError) {
-        debug(`Queue task failed with error ${err.name}`);
-        didError = true;
-        q.kill();
-        writer.emit('error', err);
-      }
-    }
-
-    if (flush || buffer.length >= bufferSize) {
-      // work through the buffer to break off bufferSize chunks
-      // and feed the chunks to the queue
-      do {
-        // split the buffer into bufferSize chunks
-        const toSend = buffer.splice(0, bufferSize);
-
-        // and add the chunk to the queue
-        debug(`Adding ${toSend.length} to the write queue.`);
-        q.push({ docs: toSend }, taskCallback);
-      } while (buffer.length >= bufferSize);
-
-      // send any leftover documents to the queue
-      if (flush && buffer.length > 0) {
-        debug(`Adding remaining ${buffer.length} to the write queue.`);
-        q.push({ docs: buffer }, taskCallback);
-      }
-
-      // wait until the queue size falls to a reasonable level
-      async.until(
-        // wait until the queue length drops to twice the paralellism
-        // or until empty on the last write
-        function(callback) {
-          // if we encountered an error, stop this until loop
-          if (didError) {
-            return callback(null, true);
-          }
-          if (flush) {
-            callback(null, q.idle() && q.length() === 0);
-          } else {
-            callback(null, q.length() <= parallelism * 2);
-          }
-        },
-        function(cb) {
-          setTimeout(cb, 20);
-        },
-
-        function() {
-          if (flush && !didError) {
-            writer.emit('finished', { total: written });
-          }
-          // callback when we're happy with the queue size
-          callback();
-        });
-    } else {
-      callback();
-    }
-  }
-
-  // take an object
-  writer._transform = function(obj, encoding, done) {
-    // each obj that arrives here is a line from the backup file
-    // it should contain an array of objects. The length of the array
-    // depends on the bufferSize at backup time.
-    linenumber++;
-    if (!didError && obj !== '') {
-      // see if it parses as JSON
-      try {
-        const arr = JSON.parse(obj);
-
-        // if it's an array with a length
-        if (typeof arr === 'object' && arr.length > 0) {
-          // push each document into a buffer
-          buffer = buffer.concat(arr);
-
-          // pause the stream
-          // it's likely that the speed with which data can be read from disk
-          // may exceed the rate it can be written to CouchDB. To prevent
-          // the whole file being buffered in memory, we pause the stream here.
-          // it is resumed, when processBuffer calls back and we call done()
-          this.pause();
-
-          // break the buffer in to bufferSize chunks to be written to the database
-          processBuffer(false, done);
-        } else {
-          ee.emit('error', new error.BackupError('BackupFileJsonError', `Error on line ${linenumber} of backup file - not an array`));
-          done();
-        }
-      } catch (e) {
-        ee.emit('error', new error.BackupError('BackupFileJsonError', `Error on line ${linenumber} of backup file - cannot parse as JSON`));
-        // Could be an incomplete write that was subsequently resumed
-        done();
-      }
-    } else {
-      done();
-    }
-  };
-
-  // called when we need to flush everything
-  writer._flush = function(done) {
-    processBuffer(true, done);
-  };
-  return writer;
-};
+    });
+  });
+});