@cloudant/couchbackup 2.9.16 → 2.9.17-SNAPSHOT.185

package/.scannerwork/scanner-report/source-50.txt

@@ -1,4 +1,4 @@
-// Copyright © 2017 IBM Corp. All rights reserved.
+// Copyright © 2017, 2023 IBM Corp. All rights reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -11,82 +11,124 @@
 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 // See the License for the specific language governing permissions and
 // limitations under the License.
+
+/* global describe it beforeEach */
 'use strict';
 
+const assert = require('assert');
 const fs = require('fs');
-const stream = require('stream');
-const liner = require('./liner.js');
-
-const onLine = function(onCommand, getDocs) {
-  const change = new stream.Transform({ objectMode: true });
-
-  change._transform = function(line, encoding, done) {
-    if (line && line[0] === ':') {
-      const obj = {
-        command: null,
-        batch: null,
-        docs: []
-      };
-
-      let matches;
-
-      // extract command
-      matches = line.match(/^:([a-z_]+) ?/);
-      if (matches) {
-        obj.command = matches[1];
+const nock = require('nock');
+const request = require('../includes/request.js');
+const writer = require('../includes/writer.js');
+const noopEmitter = new (require('events')).EventEmitter();
+const liner = require('../includes/liner.js');
+const { once } = require('node:events');
+const { pipeline } = require('node:stream/promises');
+const longTestTimeout = 3000;
+
+describe('#unit Check database restore writer', function() {
+  const dbUrl = 'http://localhost:5984/animaldb';
+  const db = request.client(dbUrl, { parallelism: 1 });
+
+  beforeEach('Reset nocks', function() {
+    nock.cleanAll();
+  });
+
+  it('should complete successfully', async function() {
+    nock(dbUrl)
+      .post('/_bulk_docs')
+      .reply(200, []); // success
+
+    const w = writer(db, 500, 1, noopEmitter);
+    return Promise.all([pipeline(fs.createReadStream('./test/fixtures/animaldb_expected.json'), liner(), w),
+      once(w, 'finished').then((data) => {
+        assert.strictEqual(data[0].total, 15);
+        assert.ok(nock.isDone());
+      })]);
+  });
+
+  it('should terminate on a fatal error', async function() {
+    nock(dbUrl)
+      .post('/_bulk_docs')
+      .reply(401, { error: 'Unauthorized' }); // fatal error
+
+    const w = writer(db, 500, 1, noopEmitter);
+    return assert.rejects(
+      pipeline(fs.createReadStream('./test/fixtures/animaldb_expected.json'), liner(), w),
+      (err) => {
+        assert.strictEqual(err.name, 'Unauthorized');
+        assert.strictEqual(err.message, 'Access is denied due to invalid credentials.');
+        assert.ok(nock.isDone());
+        return true;
       }
+    );
+  });
 
-      // extract batch
-      matches = line.match(/ batch([0-9]+)/);
-      if (matches) {
-        obj.batch = parseInt(matches[1]);
+  it('should retry on transient errors', async function() {
+    nock(dbUrl)
+      .post('/_bulk_docs')
+      .reply(429, { error: 'Too Many Requests' }) // transient error
+      .post('/_bulk_docs')
+      .reply(500, { error: 'Internal Server Error' }) // transient error
+      .post('/_bulk_docs')
+      .reply(200, { ok: true }); // third time lucky success
+
+    const w = writer(db, 500, 1, noopEmitter);
+    return Promise.all([pipeline(fs.createReadStream('./test/fixtures/animaldb_expected.json'), liner(), w),
+      once(w, 'finished').then((data) => {
+        assert.strictEqual(data[0].total, 15);
+        assert.ok(nock.isDone());
+      })]);
+  }).timeout(longTestTimeout);
+
+  it('should fail after 3 transient errors', async function() {
+    nock(dbUrl)
+      .post('/_bulk_docs')
+      .reply(429, { error: 'Too Many Requests' }) // transient error
+      .post('/_bulk_docs')
+      .reply(500, { error: 'Internal Server Error' }) // transient error
+      .post('/_bulk_docs')
+      .reply(503, { error: 'Service Unavailable' }); // Final transient error
+
+    const w = writer(db, 500, 1, noopEmitter);
+    return assert.rejects(
+      pipeline(fs.createReadStream('./test/fixtures/animaldb_expected.json'), liner(), w),
+      (err) => {
+        assert.strictEqual(err.name, 'HTTPFatalError');
+        assert.strictEqual(err.message, `503 : post ${dbUrl}/_bulk_docs - Error: Service Unavailable`);
+        assert.ok(nock.isDone());
+        return true;
       }
+    );
+  }).timeout(longTestTimeout);
+
+  it('should restore shallow backups without rev info successfully', async function() {
+    nock(dbUrl)
+      .post('/_bulk_docs')
+      .reply(200, [{ ok: true, id: 'foo', rev: '1-abc' }]); // success
+
+    const w = writer(db, 500, 1, noopEmitter);
+    return Promise.all([pipeline(fs.createReadStream('./test/fixtures/animaldb_old_shallow.json'), liner(), w),
+      once(w, 'finished').then((data) => {
+        assert.strictEqual(data[0].total, 11);
+        assert.ok(nock.isDone());
+      })]);
+  });
+
+  it('should get a batch error for non-empty array response with new_edits false', async function() {
+    nock(dbUrl)
+      .post('/_bulk_docs')
+      .reply(200, [{ id: 'foo', error: 'foo', reason: 'bar' }]);
 
-      // extract doc ids
-      if (getDocs && obj.command === 't') {
-        const json = line.replace(/^.* batch[0-9]+ /, '').trim();
-        obj.docs = JSON.parse(json);
+    const w = writer(db, 500, 1, noopEmitter);
+    return assert.rejects(
+      pipeline(fs.createReadStream('./test/fixtures/animaldb_expected.json'), liner(), w),
+      (err) => {
+        assert.strictEqual(err.name, 'Error');
+        assert.strictEqual(err.message, 'Error writing batch with new_edits:false and 1 items');
+        assert.ok(nock.isDone());
+        return true;
       }
-      onCommand(obj);
-    }
-    done();
-  };
-  return change;
-};
-
-/**
- * Generate a list of remaining batches from a download file.
- *
- * @param {string} log - log file name
- * @param {function} callback - callback with err, {changesComplete: N, batches: N}.
- *  changesComplete signifies whether the log file appeared to
- *  have completed reading the changes feed (contains :changes_complete).
- *  batches are remaining batch IDs for download.
- */
-module.exports = function(log, callback) {
-  // our sense of state
-  const state = {
-
-  };
-  let changesComplete = false;
-
-  // called with each line from the log file
-  const onCommand = function(obj) {
-    if (obj.command === 't') {
-      state[obj.batch] = true;
-    } else if (obj.command === 'd') {
-      delete state[obj.batch];
-    } else if (obj.command === 'changes_complete') {
-      changesComplete = true;
-    }
-  };
-
-  // stream through the previous log file
-  fs.createReadStream(log)
-    .pipe(liner())
-    .pipe(onLine(onCommand, false))
-    .on('finish', function() {
-      const obj = { changesComplete: changesComplete, batches: state };
-      callback(null, obj);
-    });
-};
+    );
+  });
+});

package/.scannerwork/scanner-report/source-51.txt

@@ -1,4 +1,4 @@
-// Copyright © 2017 IBM Corp. All rights reserved.
+// Copyright © 2017, 2023 IBM Corp. All rights reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -11,65 +11,203 @@
 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 // See the License for the specific language governing permissions and
 // limitations under the License.
+
+/* global describe it beforeEach */
 'use strict';
 
-const fs = require('fs');
-const stream = require('stream');
-const liner = require('./liner.js');
-
-const onLine = function(onCommand, batches) {
-  const change = new stream.Transform({ objectMode: true });
-  change._transform = function(line, encoding, done) {
-    if (line && line[0] === ':') {
-      const obj = {
-        command: null,
-        batch: null,
-        docs: []
-      };
-
-      let matches;
-
-      // extract command
-      matches = line.match(/^:([a-z_]+) ?/);
-      if (matches) {
-        obj.command = matches[1];
-      }
-
-      // extract batch
-      matches = line.match(/ batch([0-9]+)/);
-      if (matches) {
-        obj.batch = parseInt(matches[1]);
-      }
-
-      // if this is one we want
-      if (obj.command === 't' && batches.indexOf(obj.batch) > -1) {
-        const json = line.replace(/^.* batch[0-9]+ /, '').trim();
-        obj.docs = JSON.parse(json);
-        onCommand(obj);
-      }
-    }
-    done();
-  };
-  return change;
-};
-
-module.exports = function(log, batches, callback) {
-  // our sense of state
-  const retval = { };
-
-  // called with each line from the log file
-  const onCommand = function(obj) {
-    retval[obj.batch] = obj;
-  };
-
-  // stream through the previous log file
-  fs.createReadStream(log)
-    .pipe(liner())
-    .pipe(onLine(onCommand, batches))
-    .on('error', function(err) {
-      callback(err);
-    })
-    .on('finish', function() {
-      callback(null, retval);
+const assert = require('assert');
+const nock = require('nock');
+const request = require('../includes/request.js');
+const error = require('../includes/error.js');
+
+const url = 'http://localhost:7777/testdb';
+const db = request.client(url, { parallelism: 1 });
+const timeoutDb = request.client(url, { parallelism: 1, requestTimeout: 500 });
+const longTestTimeout = 3000;
+
+beforeEach('Clean nock', function() {
+  nock.cleanAll();
+});
+
+describe('#unit Check request headers', function() {
+  it('should have a couchbackup user-agent', async function() {
+    const couch = nock(url)
+      .matchHeader('user-agent', /couchbackup-cloudant\/\d+\.\d+\.\d+(?:-SNAPSHOT)? \(Node.js v\d+\.\d+\.\d+\)/)
+      .head('/good')
+      .reply(200);
+
+    return db.service.headDocument({ db: db.db, docId: 'good' }).then(() => {
+      assert.ok(couch.isDone());
+    });
+  });
+});
+
+describe('#unit Check request response error callback', function() {
+  it('should not callback with error for 200 response', async function() {
+    const couch = nock(url)
+      .get('/good')
+      .reply(200, { ok: true });
+
+    return db.service.getDocument({ db: db.db, docId: 'good' }).then(response => {
+      assert.ok(response.result);
+      assert.ok(couch.isDone());
+    });
+  });
+
+  it('should callback with error after 3 500 responses', async function() {
+    const couch = nock(url)
+      .get('/bad')
+      .times(3)
+      .reply(500, function(uri, requestBody) {
+        this.req.response.statusMessage = 'Internal Server Error';
+        return { error: 'foo', reason: 'bar' };
+      });
+
+    return assert.rejects(
+      db.service.getDocument({ db: db.db, docId: 'bad' }),
+      (err) => {
+        err = error.convertResponseError(err);
+        assert.strictEqual(err.name, 'HTTPFatalError');
+        assert.strictEqual(err.message, `500 Internal Server Error: get ${url}/bad - Error: foo, Reason: bar`);
+        assert.ok(couch.isDone());
+        return true;
+      });
+  }).timeout(longTestTimeout);
+
+  it('should callback with error after 3 POST 503 responses', async function() {
+    const couch = nock(url)
+      .post('/_bulk_get')
+      .query(true)
+      .times(3)
+      .reply(503, function(uri, requestBody) {
+        this.req.response.statusMessage = 'Service Unavailable';
+        return { error: 'service_unavailable', reason: 'Service unavailable' };
+      });
+
+    return assert.rejects(
+      db.service.postBulkGet({ db: db.db, revs: true, docs: [] }),
+      (err) => {
+        err = error.convertResponseError(err);
+        assert.strictEqual(err.name, 'HTTPFatalError');
+        assert.strictEqual(err.message, `503 Service Unavailable: post ${url}/_bulk_get - Error: service_unavailable, Reason: Service unavailable`);
+        assert.ok(couch.isDone());
+        return true;
+      });
+  }).timeout(longTestTimeout);
+
+  it('should callback with error after 3 429 responses', async function() {
+    const couch = nock(url)
+      .get('/bad')
+      .times(3)
+      .reply(429, function(uri, requestBody) {
+        this.req.response.statusMessage = 'Too Many Requests';
+        return { error: 'foo', reason: 'bar' };
+      });
+
+    return assert.rejects(
+      db.service.getDocument({ db: db.db, docId: 'bad' }),
+      (err) => {
+        err = error.convertResponseError(err);
+        assert.strictEqual(err.name, 'HTTPFatalError');
+        assert.strictEqual(err.message, `429 Too Many Requests: get ${url}/bad - Error: foo, Reason: bar`);
+        assert.ok(couch.isDone());
+        return true;
+      });
+  }).timeout(longTestTimeout);
+
+  it('should callback with fatal error for 404 response', async function() {
+    const couch = nock(url)
+      .get('/bad')
+      .reply(404, function(uri, requestBody) {
+        this.req.response.statusMessage = 'Not Found';
+        return { error: 'foo', reason: 'bar' };
+      });
+
+    return assert.rejects(
+      db.service.getDocument({ db: db.db, docId: 'bad' }),
+      (err) => {
+        err = error.convertResponseError(err);
+        assert.strictEqual(err.name, 'HTTPFatalError');
+        assert.strictEqual(err.message, `404 Not Found: get ${url}/bad - Error: foo, Reason: bar`);
+        assert.ok(couch.isDone());
+        return true;
+      });
+  });
+
+  it('should callback with same error for no status code error response', async function() {
+    const couch = nock(url)
+      .get('/bad')
+      .times(3)
+      .replyWithError('testing badness');
+
+    return assert.rejects(
+      db.service.getDocument({ db: db.db, docId: 'bad' }),
+      (err) => {
+        const err2 = error.convertResponseError(err);
+        assert.strictEqual(err, err2);
+        assert.ok(couch.isDone());
+        return true;
+      });
+  }).timeout(longTestTimeout);
+
+  it('should retry request if HTTP request gets timed out', async function() {
+    const couch = nock(url)
+      .post('/_bulk_get')
+      .query(true)
+      .delay(1000)
+      .reply(200, { results: { docs: [{ id: '1', ok: { _id: '1' } }] } })
+      .post('/_bulk_get')
+      .query(true)
+      .reply(200, { results: { docs: [{ id: '1', ok: { _id: '1' } }, { id: '2', ok: { _id: '2' } }] } });
+
+    return timeoutDb.service.postBulkGet({ db: db.db, revs: true, docs: [] }).then((response) => {
+      assert.ok(response);
+      assert.ok(response.result);
+      assert.ok(response.result.results);
+      assert.ok(response.result.results.docs);
+      assert.strictEqual(response.result.results.docs.length, 2);
+      assert.ok(couch.isDone());
+    });
+  });
+
+  it('should callback with error code ESOCKETTIMEDOUT if 3 HTTP requests gets timed out', async function() {
+    // Increase the timeout for this test to allow for the delays
+    this.timeout(3000);
+    const couch = nock(url)
+      .post('/_bulk_get')
+      .query(true)
+      .delay(1000)
+      .times(3)
+      .reply(200, { ok: true });
+
+    return assert.rejects(
+      timeoutDb.service.postBulkGet({ db: db.db, revs: true, docs: [] }),
+      (err) => {
+        err = error.convertResponseError(err);
+        // Note axios returns ECONNABORTED rather than ESOCKETTIMEDOUT
+        // See https://github.com/axios/axios/issues/2710 via https://github.com/axios/axios/issues/1543`
+        assert.strictEqual(err.statusText, 'ECONNABORTED');
+        assert.strictEqual(err.message, `timeout of 500ms exceeded: post ${url}/_bulk_get ECONNABORTED`);
+        assert.ok(couch.isDone());
+        return true;
+      });
+  });
+  describe('#unit Check credentials', async function() {
+    it('should properly decode username and password', async function() {
+      const username = 'user%123';
+      const password = 'colon:at@321';
+      const url = `http://${encodeURIComponent(username)}:${encodeURIComponent(password)}@localhost:7777/testdb`;
+      const sessionUrl = 'http://localhost:7777';
+      const couch = nock(sessionUrl)
+        .post('/_session', { username: username, password: password })
+        .reply(200, { ok: true }, { 'Set-Cookie': 'AuthSession=ABC123DEF4356;' })
+        .get('/')
+        .reply(200);
+      const db = request.client(url, { parallelism: 1 });
+      return db.service.getServerInformation().then(response => {
+        assert.ok(response);
+        assert.ok(couch.isDone());
+      });
     });
-};
+  });
+});

package/.scannerwork/scanner-report/source-52.txt

@@ -1,4 +1,4 @@
-// Copyright © 2017, 2018 IBM Corp. All rights reserved.
+// Copyright © 2023 IBM Corp. All rights reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -11,21 +11,141 @@
 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 // See the License for the specific language governing permissions and
 // limitations under the License.
+
+/* global */
 'use strict';
 
-module.exports = function(db, options, readstream, ee, callback) {
-  const liner = require('../includes/liner.js')();
-  const writer = require('../includes/writer.js')(db, options.bufferSize, options.parallelism, ee);
+const { fork, spawn } = require('node:child_process');
+const { once } = require('node:events');
+const { Duplex } = require('node:stream');
+const debug = require('debug');
+const logProcess = debug('couchbackup:test:process');
+
+class TestProcess {
+  constructor(cmd, args, mode) {
+    this.cmd = cmd;
+    // Child process stdio [stdin, stdout, stderr, ...extra channels]
+    const childProcessOptions = { stdio: [] };
+    switch (mode) {
+      case 'readable':
+        // Readable only, no writing to stdin so ignore it
+        childProcessOptions.stdio = ['ignore', 'pipe', 'inherit'];
+        break;
+      case 'writable':
+        // Writable only, no reading from stdout so ignore it
+        childProcessOptions.stdio = ['pipe', 'ignore', 'inherit'];
+        break;
+      default:
+        // Default Duplex mode pipe both stdin and stdout
+        childProcessOptions.stdio = ['pipe', 'pipe', 'inherit'];
+        break;
+    }
+    if (cmd.endsWith('.js')) {
+      // Add Node fork ipc channel
+      childProcessOptions.stdio.push('ipc');
+      logProcess(`Forking Node process for ${cmd} with stdio:[${childProcessOptions.stdio}]`);
+      this.childProcess = fork(cmd, args, childProcessOptions);
+    } else {
+      logProcess(`Spawning process for ${cmd} with stdio:[${childProcessOptions.stdio}]`);
+      this.childProcess = spawn(cmd, args, childProcessOptions);
+    }
+
+    this.childProcessPromise = once(this.childProcess, 'close').then(() => {
+      const code = this.childProcess.exitCode;
+      const signal = this.childProcess.signalCode;
+      logProcess(`Test process ${cmd} closed with code ${code} and signal ${signal}`);
+      if (code === 0) {
+        logProcess(`Resolving process promise for ${cmd}`);
+        return Promise.resolve(code);
+      } else {
+        const e = new Error(`Test child process ${cmd} exited with code ${code} and ${signal}. This may be normal for error case testing.`);
+        e.code = code;
+        e.signal = signal;
+        logProcess(`Will reject process promise for ${cmd} with ${e}`);
+        return Promise.reject(e);
+      }
+    });
+
+    switch (mode) {
+      case 'readable':
+        this.duplexFrom = this.childProcess.stdout;
+        break;
+      case 'writable':
+        this.duplexFrom = this.childProcess.stdin;
+        break;
+      default:
+        // Default is duplex
+        this.duplexFrom = { writable: this.childProcess.stdin, readable: this.childProcess.stdout };
+    }
 
-  // pipe the input to the output, via transformation functions
-  readstream
-    .pipe(liner) // transform the input stream into per-line
-    .on('error', function(err) {
-      // Forward the error to the writer event emitter where we already have
-      // listeners on for handling errors
-      writer.emit('error', err);
-    })
-    .pipe(writer); // transform the data
+    this.stream = Duplex.from(this.duplexFrom);
+  }
+}
 
-  callback(null, writer);
+module.exports = {
+  TestProcess,
+  cliBackup: function(databaseName, params = {}) {
+    const args = ['--db', databaseName];
+    if (params.opts) {
+      if (params.opts.mode) {
+        args.push('--mode');
+        args.push(params.opts.mode);
+      }
+      if (params.opts.output) {
+        args.push('--output');
+        args.push(params.opts.output);
+      }
+      if (params.opts.log) {
+        args.push('--log');
+        args.push(params.opts.log);
+      }
+      if (params.opts.resume) {
+        args.push('--resume');
+        args.push(params.opts.resume);
+      }
+      if (params.opts.bufferSize) {
+        args.push('--buffer-size');
+        args.push(params.opts.bufferSize);
+      }
+      if (params.opts.iamApiKey) {
+        args.push('--iam-api-key');
+        args.push(params.opts.iamApiKey);
+      }
+    }
+    return new TestProcess('./bin/couchbackup.bin.js', args, 'readable');
+  },
+  cliRestore: function(databaseName, params) {
+    const args = ['--db', databaseName];
+    if (params.opts) {
+      if (params.opts.bufferSize) {
+        args.push('--buffer-size');
+        args.push(params.opts.bufferSize);
+      }
+      if (params.opts.parallelism) {
+        args.push('--parallelism');
+        args.push(params.opts.parallelism);
+      }
+      if (params.opts.requestTimeout) {
+        args.push('--request-timeout');
+        args.push(params.opts.requestTimeout);
+      }
+      if (params.opts.iamApiKey) {
+        args.push('--iam-api-key');
+        args.push(params.opts.iamApiKey);
+      }
+    }
+    return new TestProcess('./bin/couchrestore.bin.js', args, 'writable');
+  },
+  cliGzip: function() {
+    return new TestProcess('gzip', []);
+  },
+  cliGunzip: function() {
+    return new TestProcess('gunzip', []);
+  },
+  cliEncrypt: function() {
+    return new TestProcess('openssl', ['aes-128-cbc', '-pass', 'pass:12345']);
+  },
+  cliDecrypt: function() {
+    return new TestProcess('openssl', ['aes-128-cbc', '-d', '-pass', 'pass:12345']);
+  }
 };