@cloudant/couchbackup 2.9.16 → 2.9.17-SNAPSHOT.185

package/.scannerwork/scanner-report/source-12.txt

@@ -1,4 +1,4 @@
-// Copyright © 2017, 2023 IBM Corp. All rights reserved.
+// Copyright © 2017 IBM Corp. All rights reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -11,203 +11,36 @@
 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 // See the License for the specific language governing permissions and
 // limitations under the License.
-
-/* global describe it beforeEach */
 'use strict';
 
-const assert = require('assert');
-const nock = require('nock');
-const request = require('../includes/request.js');
-const error = require('../includes/error.js');
-
-const url = 'http://localhost:7777/testdb';
-const db = request.client(url, { parallelism: 1 });
-const timeoutDb = request.client(url, { parallelism: 1, requestTimeout: 500 });
-const longTestTimeout = 3000;
-
-beforeEach('Clean nock', function() {
-  nock.cleanAll();
-});
-
-describe('#unit Check request headers', function() {
-  it('should have a couchbackup user-agent', async function() {
-    const couch = nock(url)
-      .matchHeader('user-agent', /couchbackup-cloudant\/\d+\.\d+\.\d+(?:-SNAPSHOT)? \(Node.js v\d+\.\d+\.\d+\)/)
-      .head('/good')
-      .reply(200);
-
-    return db.service.headDocument({ db: db.db, docId: 'good' }).then(() => {
-      assert.ok(couch.isDone());
-    });
-  });
-});
-
-describe('#unit Check request response error callback', function() {
-  it('should not callback with error for 200 response', async function() {
-    const couch = nock(url)
-      .get('/good')
-      .reply(200, { ok: true });
-
-    return db.service.getDocument({ db: db.db, docId: 'good' }).then(response => {
-      assert.ok(response.result);
-      assert.ok(couch.isDone());
-    });
-  });
-
-  it('should callback with error after 3 500 responses', async function() {
-    const couch = nock(url)
-      .get('/bad')
-      .times(3)
-      .reply(500, function(uri, requestBody) {
-        this.req.response.statusMessage = 'Internal Server Error';
-        return { error: 'foo', reason: 'bar' };
-      });
-
-    return assert.rejects(
-      db.service.getDocument({ db: db.db, docId: 'bad' }),
-      (err) => {
-        err = error.convertResponseError(err);
-        assert.strictEqual(err.name, 'HTTPFatalError');
-        assert.strictEqual(err.message, `500 Internal Server Error: get ${url}/bad - Error: foo, Reason: bar`);
-        assert.ok(couch.isDone());
-        return true;
-      });
-  }).timeout(longTestTimeout);
-
-  it('should callback with error after 3 POST 503 responses', async function() {
-    const couch = nock(url)
-      .post('/_bulk_get')
-      .query(true)
-      .times(3)
-      .reply(503, function(uri, requestBody) {
-        this.req.response.statusMessage = 'Service Unavailable';
-        return { error: 'service_unavailable', reason: 'Service unavailable' };
-      });
-
-    return assert.rejects(
-      db.service.postBulkGet({ db: db.db, revs: true, docs: [] }),
-      (err) => {
-        err = error.convertResponseError(err);
-        assert.strictEqual(err.name, 'HTTPFatalError');
-        assert.strictEqual(err.message, `503 Service Unavailable: post ${url}/_bulk_get - Error: service_unavailable, Reason: Service unavailable`);
-        assert.ok(couch.isDone());
-        return true;
-      });
-  }).timeout(longTestTimeout);
-
-  it('should callback with error after 3 429 responses', async function() {
-    const couch = nock(url)
-      .get('/bad')
-      .times(3)
-      .reply(429, function(uri, requestBody) {
-        this.req.response.statusMessage = 'Too Many Requests';
-        return { error: 'foo', reason: 'bar' };
-      });
-
-    return assert.rejects(
-      db.service.getDocument({ db: db.db, docId: 'bad' }),
-      (err) => {
-        err = error.convertResponseError(err);
-        assert.strictEqual(err.name, 'HTTPFatalError');
-        assert.strictEqual(err.message, `429 Too Many Requests: get ${url}/bad - Error: foo, Reason: bar`);
-        assert.ok(couch.isDone());
-        return true;
-      });
-  }).timeout(longTestTimeout);
-
-  it('should callback with fatal error for 404 response', async function() {
-    const couch = nock(url)
-      .get('/bad')
-      .reply(404, function(uri, requestBody) {
-        this.req.response.statusMessage = 'Not Found';
-        return { error: 'foo', reason: 'bar' };
-      });
-
-    return assert.rejects(
-      db.service.getDocument({ db: db.db, docId: 'bad' }),
-      (err) => {
-        err = error.convertResponseError(err);
-        assert.strictEqual(err.name, 'HTTPFatalError');
-        assert.strictEqual(err.message, `404 Not Found: get ${url}/bad - Error: foo, Reason: bar`);
-        assert.ok(couch.isDone());
-        return true;
-      });
-  });
-
-  it('should callback with same error for no status code error response', async function() {
-    const couch = nock(url)
-      .get('/bad')
-      .times(3)
-      .replyWithError('testing badness');
-
-    return assert.rejects(
-      db.service.getDocument({ db: db.db, docId: 'bad' }),
-      (err) => {
-        const err2 = error.convertResponseError(err);
-        assert.strictEqual(err, err2);
-        assert.ok(couch.isDone());
-        return true;
-      });
-  }).timeout(longTestTimeout);
-
-  it('should retry request if HTTP request gets timed out', async function() {
-    const couch = nock(url)
-      .post('/_bulk_get')
-      .query(true)
-      .delay(1000)
-      .reply(200, { results: { docs: [{ id: '1', ok: { _id: '1' } }] } })
-      .post('/_bulk_get')
-      .query(true)
-      .reply(200, { results: { docs: [{ id: '1', ok: { _id: '1' } }, { id: '2', ok: { _id: '2' } }] } });
-
-    return timeoutDb.service.postBulkGet({ db: db.db, revs: true, docs: [] }).then((response) => {
-      assert.ok(response);
-      assert.ok(response.result);
-      assert.ok(response.result.results);
-      assert.ok(response.result.results.docs);
-      assert.strictEqual(response.result.results.docs.length, 2);
-      assert.ok(couch.isDone());
-    });
-  });
-
-  it('should callback with error code ESOCKETTIMEDOUT if 3 HTTP requests gets timed out', async function() {
-    // Increase the timeout for this test to allow for the delays
-    this.timeout(3000);
-    const couch = nock(url)
-      .post('/_bulk_get')
-      .query(true)
-      .delay(1000)
-      .times(3)
-      .reply(200, { ok: true });
-
-    return assert.rejects(
-      timeoutDb.service.postBulkGet({ db: db.db, revs: true, docs: [] }),
-      (err) => {
-        err = error.convertResponseError(err);
-        // Note axios returns ECONNABORTED rather than ESOCKETTIMEDOUT
-        // See https://github.com/axios/axios/issues/2710 via https://github.com/axios/axios/issues/1543`
-        assert.strictEqual(err.statusText, 'ECONNABORTED');
-        assert.strictEqual(err.message, `timeout of 500ms exceeded: post ${url}/_bulk_get ECONNABORTED`);
-        assert.ok(couch.isDone());
-        return true;
-      });
-  });
-  describe('#unit Check credentials', async function() {
-    it('should properly decode username and password', async function() {
-      const username = 'user%123';
-      const password = 'colon:at@321';
-      const url = `http://${encodeURIComponent(username)}:${encodeURIComponent(password)}@localhost:7777/testdb`;
-      const sessionUrl = 'http://localhost:7777';
-      const couch = nock(sessionUrl)
-        .post('/_session', { username: username, password: password })
-        .reply(200, { ok: true }, { 'Set-Cookie': 'AuthSession=ABC123DEF4356;' })
-        .get('/')
-        .reply(200);
-      const db = request.client(url, { parallelism: 1 });
-      return db.service.getServerInformation().then(response => {
-        assert.ok(response);
-        assert.ok(couch.isDone());
-      });
-    });
-  });
-});
+// stolen from http://strongloop.com/strongblog/practical-examples-of-the-new-node-js-streams-api/
+const stream = require('stream');
+
+module.exports = function() {
+  const liner = new stream.Transform({ objectMode: true });
+
+  liner._transform = function(chunk, encoding, done) {
+    let data = chunk.toString();
+    if (this._lastLineData) {
+      data = this._lastLineData + data;
+    }
+
+    const lines = data.split('\n');
+    this._lastLineData = lines.splice(lines.length - 1, 1)[0];
+
+    for (const i in lines) {
+      this.push(lines[i]);
+    }
+    done();
+  };
+
+  liner._flush = function(done) {
+    if (this._lastLineData) {
+      this.push(this._lastLineData);
+    }
+    this._lastLineData = null;
+    done();
+  };
+
+  return liner;
+};

package/.scannerwork/scanner-report/source-13.txt

@@ -1,4 +1,4 @@
-// Copyright © 2017, 2023 IBM Corp. All rights reserved.
+// Copyright © 2017, 2018 IBM Corp. All rights reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -11,91 +11,21 @@
 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 // See the License for the specific language governing permissions and
 // limitations under the License.
-
-/* global describe it */
 'use strict';
 
-const assert = require('assert');
-const fs = require('fs');
-const { once } = require('node:events');
-const u = require('./citestutils.js');
-
-[{ useApi: true }, { useApi: false }].forEach(function(params) {
-  describe(u.scenario('Resume tests', params), function() {
-    it('should create a log file', async function() {
-      // Allow up to 90 s for this test
-      u.setTimeout(this, 60);
-
-      const actualBackup = `./${this.fileName}`;
-      const logFile = `./${this.fileName}` + '.log';
-      const p = u.p(params, { opts: { log: logFile } });
-      return u.testBackupToFile(p, 'animaldb', actualBackup).then(() => {
-        assert.ok(fs.existsSync(logFile), 'The log file should exist.');
-      });
-    });
-
-    it('should restore corrupted animaldb to a database correctly', async function() {
-      // Allow up to 60 s to restore and compare (again it should be faster)!
-      u.setTimeout(this, 60);
-      const input = fs.createReadStream('./test/fixtures/animaldb_corrupted.json');
-      const dbName = this.dbName;
-      const p = u.p(params, { expectedRestoreErrorRecoverable: { name: 'BackupFileJsonError' } });
-      return once(input, 'open')
-        .then(() => {
-          return u.testRestore(p, input, dbName);
-        }).then(() => {
-          return u.dbCompare('animaldb', dbName);
-        });
-    });
-
-    it('should restore resumed animaldb with blank line to a database correctly', async function() {
-      // Allow up to 60 s to restore and compare (again it should be faster)!
-      u.setTimeout(this, 60);
-      const input = fs.createReadStream('./test/fixtures/animaldb_resumed_blank.json');
-      const dbName = this.dbName;
-      return once(input, 'open')
-        .then(() => {
-          return u.testRestore(params, input, dbName);
-        }).then(() => {
-          return u.dbCompare('animaldb', dbName);
-        });
-    });
-  });
-});
-
-describe('Resume tests', function() {
-  // Currently cannot abort API backups, when we do this test should be run for
-  // both API and CLI
-  it('should correctly backup and restore backup10m', async function() {
-    // Allow up to 90 s for this test
-    u.setTimeout(this, 90);
-
-    const actualBackup = `./${this.fileName}`;
-    const logFile = `./${this.fileName}` + '.log';
-    // Use abort parameter to terminate the backup
-    const p = u.p(params, { abort: true }, { opts: { log: logFile } });
-    const restoreDb = this.dbName;
-    // Set the database doc count as fewer than this should be written during
-    // resumed backup.
-    p.exclusiveMaxExpected = 5096;
-
-    return u.testBackupAbortResumeRestore(p, 'backup10m', actualBackup, restoreDb);
-  });
-  // Note --output is only valid for CLI usage, this test should only run for CLI
-  const params = { useApi: false };
-  it('should correctly backup and restore backup10m using --output', async function() {
-    // Allow up to 90 s for this test
-    u.setTimeout(this, 90);
-
-    const actualBackup = `./${this.fileName}`;
-    const logFile = `./${this.fileName}` + '.log';
-    // Use abort parameter to terminate the backup
-    const p = u.p(params, { abort: true }, { opts: { output: actualBackup, log: logFile } });
-    const restoreDb = this.dbName;
-    // Set the database doc count as fewer than this should be written during
-    // resumed backup.
-    p.exclusiveMaxExpected = 5096;
-
-    return await u.testBackupAbortResumeRestore(p, 'backup10m', actualBackup, restoreDb);
-  });
-});
+module.exports = function(db, options, readstream, ee, callback) {
+  const liner = require('../includes/liner.js')();
+  const writer = require('../includes/writer.js')(db, options.bufferSize, options.parallelism, ee);
+
+  // pipe the input to the output, via transformation functions
+  readstream
+    .pipe(liner) // transform the input stream into per-line
+    .on('error', function(err) {
+      // Forward the error to the writer event emitter where we already have
+      // listeners on for handling errors
+      writer.emit('error', err);
+    })
+    .pipe(writer); // transform the data
+
+  callback(null, writer);
+};

package/.scannerwork/scanner-report/source-14.txt

@@ -1,4 +1,4 @@
-// Copyright © 2017, 2023 IBM Corp. All rights reserved.
+// Copyright © 2017, 2021 IBM Corp. All rights reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -11,120 +11,119 @@
 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 // See the License for the specific language governing permissions and
 // limitations under the License.
-
-/* global describe it before after */
 'use strict';
 
-const assert = require('assert');
-const applyEnvVars = require('../includes/config.js').applyEnvironmentVariables;
-
-describe('#unit Configuration', function() {
-  let processEnvCopy;
-
-  before('Save env', function() {
-    // Copy env so we can reset it after the tests
-    processEnvCopy = JSON.parse(JSON.stringify(process.env));
-  });
-
-  after('Reset env', function() {
-    process.env = processEnvCopy;
-  });
-
-  it('respects the COUCH_URL env variable', function() {
-    process.env.COUCH_URL = 'http://user:pass@myurl.com';
-    const config = {};
-    applyEnvVars(config);
-    assert.strictEqual(typeof config.url, 'string');
-    assert.strictEqual(config.url, process.env.COUCH_URL);
-  });
-
-  it('respects the COUCH_DATABASE env variable', function() {
-    process.env.COUCH_DATABASE = 'mydb';
-    const config = {};
-    applyEnvVars(config);
-    assert.strictEqual(typeof config.db, 'string');
-    assert.strictEqual(config.db, process.env.COUCH_DATABASE);
-  });
-
-  it('respects the COUCH_BUFFER_SIZE env variable', function() {
-    process.env.COUCH_BUFFER_SIZE = '1000';
-    const config = {};
-    applyEnvVars(config);
-    assert.strictEqual(typeof config.bufferSize, 'number');
-    assert.strictEqual(config.bufferSize, 1000);
-  });
-
-  it('respects the COUCH_PARALLELISM env variable', function() {
-    process.env.COUCH_PARALLELISM = '20';
-    const config = {};
-    applyEnvVars(config);
-    assert.strictEqual(typeof config.parallelism, 'number');
-    assert.strictEqual(config.parallelism, 20);
-  });
-
-  it('respects the COUCH_REQUEST_TIMEOUT env variable', function() {
-    process.env.COUCH_REQUEST_TIMEOUT = '10000';
-    const config = {};
-    applyEnvVars(config);
-    assert.strictEqual(typeof config.requestTimeout, 'number');
-    assert.strictEqual(config.requestTimeout, 10000);
-  });
-
-  it('respects the CLOUDANT_IAM_API_KEY env variable', function() {
-    const key = 'ABC123-ZYX987_cba789-xyz321';
-    process.env.CLOUDANT_IAM_API_KEY = key;
-    const config = {};
-    applyEnvVars(config);
-    assert.strictEqual(typeof config.iamApiKey, 'string');
-    assert.strictEqual(config.iamApiKey, key);
-  });
-
-  it('respects the CLOUDANT_IAM_TOKEN_URL env variable', function() {
-    const u = 'https://testhost.example:1234/identity/token';
-    process.env.CLOUDANT_IAM_TOKEN_URL = u;
-    const config = {};
-    applyEnvVars(config);
-    assert.strictEqual(typeof config.iamTokenUrl, 'string');
-    assert.strictEqual(config.iamTokenUrl, u);
-  });
-
-  it('respects the COUCH_LOG env variable', function() {
-    process.env.COUCH_LOG = 'my.log';
-    const config = {};
-    applyEnvVars(config);
-    assert.strictEqual(typeof config.log, 'string');
-    assert.strictEqual(config.log, process.env.COUCH_LOG);
-  });
-
-  it('respects the COUCH_RESUME env variable', function() {
-    process.env.COUCH_RESUME = 'true';
-    const config = {};
-    applyEnvVars(config);
-    assert.strictEqual(typeof config.resume, 'boolean');
-    assert.strictEqual(config.resume, true);
-  });
-
-  it('respects the COUCH_OUTPUT env variable', function() {
-    process.env.COUCH_OUTPUT = 'myfile.txt';
-    const config = {};
-    applyEnvVars(config);
-    assert.strictEqual(typeof config.output, 'string');
-    assert.strictEqual(config.output, process.env.COUCH_OUTPUT);
-  });
-
-  it('respects the COUCH_MODE env variable', function() {
-    process.env.COUCH_MODE = 'shallow';
-    const config = {};
-    applyEnvVars(config);
-    assert.strictEqual(typeof config.mode, 'string');
-    assert.strictEqual(config.mode, 'shallow');
-  });
-
-  it('respects the COUCH_QUIET env variable', function() {
-    process.env.COUCH_QUIET = 'true';
-    const config = {};
-    applyEnvVars(config);
-    assert.strictEqual(typeof config.quiet, 'boolean');
-    assert.strictEqual(config.quiet, true);
-  });
-});
+const cliutils = require('./cliutils.js');
+const config = require('./config.js');
+const error = require('./error.js');
+const path = require('path');
+const pkg = require('../package.json');
+
+function parseBackupArgs() {
+  const program = require('commander');
+
+  // Option CLI defaults
+  const defaults = config.cliDefaults();
+
+  // Options set by environment variables
+  const envVarOptions = {};
+  config.applyEnvironmentVariables(envVarOptions);
+
+  program
+    .version(pkg.version)
+    .description('Backup a CouchDB/Cloudant database to a backup text file.')
+    .usage('[options...]')
+    .option('-b, --buffer-size <n>',
+      cliutils.getUsage('number of documents fetched at once', defaults.bufferSize),
+      Number)
+    .option('-d, --db <db>',
+      cliutils.getUsage('name of the database to backup', defaults.db))
+    .option('-k, --iam-api-key <API key>',
+      cliutils.getUsage('IAM API key to access the Cloudant server'))
+    .option('-l, --log <file>',
+      cliutils.getUsage('file to store logging information during backup; invalid in "shallow" mode', 'a temporary file'),
+      path.normalize)
+    .option('-m, --mode <mode>',
+      cliutils.getUsage('"shallow" if only a superficial backup is done (ignoring conflicts and revision tokens), else "full" for complete backup', defaults.mode),
+      (mode) => { return mode.toLowerCase(); })
+    .option('-o, --output <file>',
+      cliutils.getUsage('file name to store the backup data', 'stdout'),
+      path.normalize)
+    .option('-p, --parallelism <n>',
+      cliutils.getUsage('number of HTTP requests to perform in parallel when performing a backup; ignored in "shallow" mode', defaults.parallelism),
+      Number)
+    .option('-q, --quiet',
+      cliutils.getUsage('suppress batch messages', defaults.quiet))
+    .option('-r, --resume',
+      cliutils.getUsage('continue a previous backup from its last known position; invalid in "shallow" mode', defaults.resume))
+    .option('-t, --request-timeout <n>',
+      cliutils.getUsage('milliseconds to wait for a response to a HTTP request before retrying the request', defaults.requestTimeout),
+      Number)
+    .option('-u, --url <url>',
+      cliutils.getUsage('URL of the CouchDB/Cloudant server', defaults.url))
+    .parse(process.argv);
+
+  // Remove defaults that don't apply when using shallow mode
+  if (program.opts().mode === 'shallow' || envVarOptions.mode === 'shallow') {
+    delete defaults.parallelism;
+    delete defaults.log;
+    delete defaults.resume;
+  }
+
+  // Apply the options in order so that the CLI overrides env vars and env variables
+  // override defaults.
+  const opts = Object.assign({}, defaults, envVarOptions, program.opts());
+
+  if (opts.resume && (opts.log === defaults.log)) {
+    // If resuming and the log file arg is the newly generated tmp name from defaults then we know that --log wasn't specified.
+    // We have to do this check here for the CLI case because of the default.
+    error.terminationCallback(new error.BackupError('NoLogFileName', 'To resume a backup, a log file must be specified'));
+  }
+
+  return opts;
+}
+
+function parseRestoreArgs() {
+  const program = require('commander');
+
+  // Option CLI defaults
+  const defaults = config.cliDefaults();
+
+  // Options set by environment variables
+  const envVarOptions = {};
+  config.applyEnvironmentVariables(envVarOptions);
+
+  program
+    .version(pkg.version)
+    .description('Restore a CouchDB/Cloudant database from a backup text file.')
+    .usage('[options...]')
+    .option('-b, --buffer-size <n>',
+      cliutils.getUsage('number of documents restored at once', defaults.bufferSize),
+      Number)
+    .option('-d, --db <db>',
+      cliutils.getUsage('name of the new, existing database to restore to', defaults.db))
+    .option('-k, --iam-api-key <API key>',
+      cliutils.getUsage('IAM API key to access the Cloudant server'))
+    .option('-p, --parallelism <n>',
+      cliutils.getUsage('number of HTTP requests to perform in parallel when restoring a backup', defaults.parallelism),
+      Number)
+    .option('-q, --quiet',
+      cliutils.getUsage('suppress batch messages', defaults.quiet))
+    .option('-t, --request-timeout <n>',
+      cliutils.getUsage('milliseconds to wait for a response to a HTTP request before retrying the request', defaults.requestTimeout),
+      Number)
+    .option('-u, --url <url>',
+      cliutils.getUsage('URL of the CouchDB/Cloudant server', defaults.url))
+    .parse(process.argv);
+
+  // Apply the options in order so that the CLI overrides env vars and env variables
+  // override defaults.
+  const opts = Object.assign({}, defaults, envVarOptions, program.opts());
+
+  return opts;
+}
+
+module.exports = {
+  parseBackupArgs: parseBackupArgs,
+  parseRestoreArgs: parseRestoreArgs
+};