@cloudant/couchbackup 2.9.14-SNAPSHOT.159 → 2.9.14

package/.scannerwork/scanner-report/source-7.txt

@@ -1,4 +1,5 @@
-// Copyright © 2017 IBM Corp. All rights reserved.
+#!/usr/bin/env node
+// Copyright © 2017, 2021 IBM Corp. All rights reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -13,80 +14,68 @@
 // limitations under the License.
 'use strict';
 
+const error = require('../includes/error.js');
 const fs = require('fs');
-const stream = require('stream');
-const liner = require('./liner.js');
+const cliutils = require('../includes/cliutils.js');
+const couchbackup = require('../app.js');
+const parser = require('../includes/parser.js');
+const debug = require('debug');
+const backupDebug = debug('couchbackup:backup');
+const backupBatchDebug = debug('couchbackup:backup:batch');
 
-const onLine = function(onCommand, getDocs) {
-  const change = new stream.Transform({ objectMode: true });
+backupDebug.enabled = true;
 
-  change._transform = function(line, encoding, done) {
-    if (line && line[0] === ':') {
-      const obj = {
-        command: null,
-        batch: null,
-        docs: []
-      };
+try {
+  const program = parser.parseBackupArgs();
+  const databaseUrl = cliutils.databaseUrl(program.url, program.db);
 
-      let matches;
-
-      // extract command
-      matches = line.match(/^:([a-z_]+) ?/);
-      if (matches) {
-        obj.command = matches[1];
-      }
-
-      // extract batch
-      matches = line.match(/ batch([0-9]+)/);
-      if (matches) {
-        obj.batch = parseInt(matches[1]);
-      }
-
-      // extract doc ids
-      if (getDocs && obj.command === 't') {
-        const json = line.replace(/^.* batch[0-9]+ /, '').trim();
-        obj.docs = JSON.parse(json);
-      }
-      onCommand(obj);
-    }
-    done();
+  const opts = {
+    bufferSize: program.bufferSize,
+    log: program.log,
+    mode: program.mode,
+    parallelism: program.parallelism,
+    requestTimeout: program.requestTimeout,
+    resume: program.resume,
+    iamApiKey: program.iamApiKey,
+    iamTokenUrl: program.iamTokenUrl
   };
-  return change;
-};
 
-/**
- * Generate a list of remaining batches from a download file.
- *
- * @param {string} log - log file name
- * @param {function} callback - callback with err, {changesComplete: N, batches: N}.
- *  changesComplete signifies whether the log file appeared to
- *  have completed reading the changes feed (contains :changes_complete).
- *  batches are remaining batch IDs for download.
- */
-module.exports = function(log, callback) {
-  // our sense of state
-  const state = {
+  // log configuration to console
+  console.error('='.repeat(80));
+  console.error('Performing backup on ' + databaseUrl.replace(/\/\/.+@/g, '//****:****@') + ' using configuration:');
+  console.error(JSON.stringify(opts, null, 2).replace(/"iamApiKey": "[^"]+"/, '"iamApiKey": "****"'));
+  console.error('='.repeat(80));
 
-  };
-  let changesComplete = false;
+  backupBatchDebug.enabled = !program.quiet;
+
+  let ws = process.stdout;
 
-  // called with each line from the log file
-  const onCommand = function(obj) {
-    if (obj.command === 't') {
-      state[obj.batch] = true;
-    } else if (obj.command === 'd') {
-      delete state[obj.batch];
-    } else if (obj.command === 'changes_complete') {
-      changesComplete = true;
+  // open output file
+  if (program.output) {
+    let flags = 'w';
+    if (program.log && program.resume) {
+      flags = 'a';
     }
-  };
+    const fd = fs.openSync(program.output, flags);
+    ws = fs.createWriteStream(null, { fd });
+  }
+
+  backupDebug('Fetching all database changes...');
 
-  // stream through the previous log file
-  fs.createReadStream(log)
-    .pipe(liner())
-    .pipe(onLine(onCommand, false))
-    .on('finish', function() {
-      const obj = { changesComplete: changesComplete, batches: state };
-      callback(null, obj);
-    });
-};
+  return couchbackup.backup(
+    databaseUrl,
+    ws,
+    opts,
+    error.terminationCallback
+  ).on('changes', function(batch) {
+    backupBatchDebug('Total batches received:', batch + 1);
+  }).on('written', function(obj) {
+    backupBatchDebug('Written batch ID:', obj.batch, 'Total document revisions written:', obj.total, 'Time:', obj.time);
+  }).on('error', function(e) {
+    backupDebug('ERROR', e);
+  }).on('finished', function(obj) {
+    backupDebug('Finished - Total document revisions written:', obj.total);
+  });
+} catch (err) {
+  error.terminationCallback(err);
+}

package/.scannerwork/scanner-report/source-8.txt

@@ -1,4 +1,4 @@
-// Copyright © 2017, 2018 IBM Corp. All rights reserved.
+// Copyright © 2017 IBM Corp. All rights reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -11,49 +11,104 @@
 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 // See the License for the specific language governing permissions and
 // limitations under the License.
-'use strict';
 
-/**
- * Utility methods for the command line interface.
- * @module cliutils
- * @see module:cliutils
- */
+/* global after before describe */
+'use strict';
 
 const url = require('url');
-const error = require('./error.js');
-
-module.exports = {
-
-  /**
-   * Combine a base URL and a database name, ensuring at least single slash
-   * between root and database name. This allows users to have Couch behind
-   * proxies that mount Couch's / endpoint at some other mount point.
-   * @param {string} root - root URL
-   * @param {string} databaseName - database name
-   * @return concatenated URL.
-   *
-   * @private
-   */
-  databaseUrl: function databaseUrl(root, databaseName) {
-    if (!root.endsWith('/')) {
-      root = root + '/';
-    }
-    try {
-      return new url.URL(encodeURIComponent(databaseName), root).toString();
-    } catch (err) {
-      throw error.wrapPossibleInvalidUrlError(err);
-    }
-  },
-
-  /**
-   * Generate CLI argument usage text.
-   *
-   * @param {string} description - argument description.
-   * @param {string} defaultValue - default argument value.
-   *
-   * @private
-   */
-  getUsage: function getUsage(description, defaultValue) {
-    return `${description} ${defaultValue !== undefined ? ` (default: ${defaultValue})` : ''}`;
+const toxy = require('toxy');
+// Import the common hooks
+require('../test/hooks.js');
+
+const tpoisons = toxy.poisons;
+const trules = toxy.rules;
+
+function setupProxy(poison) {
+  const backendUrl = new url.URL(process.env.COUCH_BACKEND_URL);
+  const proxy = toxy({
+    auth: `${backendUrl.username}:${backendUrl.password}`,
+    changeOrigin: true
+  });
+
+  // Forward traffic to DB
+  proxy.forward(process.env.COUCH_BACKEND_URL);
+
+  switch (poison) {
+    case 'normal':
+      // No poisons to add
+      break;
+    case 'bandwidth-limit':
+      // https://github.com/h2non/toxy#bandwidth
+      // Note the implementation of bandwidth is simplistic and the threshold
+      // delay is applied to every write of the buffer, so use the smallest
+      // delay possible and adjust the rate using the bytes size instead.
+      proxy
+        .poison(tpoisons.bandwidth({ bytes: 512, threshold: 1 })); // 0.5 MB/s
+      break;
+    case 'latency':
+      // https://github.com/h2non/toxy#latency
+      proxy
+        .poison(tpoisons.latency({ max: 1500, min: 250 }))
+        .withRule(trules.probability(60));
+      break;
+    case 'slow-read':
+      // https://github.com/h2non/toxy#slow-read
+      // Note this only impacts read of data from requests so only for non-GET
+      // In practice this means that it impacts restore much more than backup
+      // since although backup POSTs to _bulk_get the content is much smaller
+      // than what is POSTed to _bulk_docs for a restore.
+      // Similarly to bandwidth-limit use a 1 ms threshold
+      proxy
+        .poison(tpoisons.slowRead({ chunk: 256, threshold: 1 }))
+        // Slow read for 10 % of the time e.g. 10 ms in every 100
+        .withRule(trules.timeThreshold({ duration: 10, period: 100 }));
+      break;
+    case 'rate-limit':
+      // https://github.com/h2non/toxy#rate-limit
+      // Simulate the Cloudant free plan with 20 lookups ps and 10 writes ps
+      proxy.post('/*/_bulk_get')
+        .poison(tpoisons.rateLimit({ limit: 20, threshold: 1000 }));
+      proxy.post('/*/_bulk_docs')
+        .poison(tpoisons.rateLimit({ limit: 10, threshold: 1000 }));
+      break;
+    default:
+      throw Error('Unknown toxy poison ' + poison);
   }
-};
+
+  // Catch remaining traffic
+  proxy.all('/*');
+  return proxy;
+}
+
+const poisons = [
+  'normal',
+  'bandwidth-limit',
+  'latency',
+  'slow-read',
+  'rate-limit'
+];
+
+poisons.forEach(function(poison) {
+  describe('unreliable network tests (using toxy poison ' + poison + ')', function() {
+    let proxy;
+
+    before('start toxy server', function() {
+      proxy = setupProxy(poison);
+      console.log('Using toxy poison ' + poison);
+
+      // For these tests COUCH_URL points to the toxy proxy on localhost whereas
+      // COUCH_BACKEND_URL is the real CouchDb instance.
+      const toxyUrl = new url.URL(process.env.COUCH_URL);
+      // Listen on the specified hostname only, so if using localhost we don't
+      // need external connections.
+      proxy.listen(toxyUrl.port, toxyUrl.hostname);
+    });
+
+    after('stop toxy server', function() {
+      proxy.close();
+    });
+
+    delete require.cache[require.resolve('../test/ci_e2e.js')];
+    require('../test/ci_e2e.js');
+  });
+});

package/.scannerwork/scanner-report/source-9.txt

@@ -1,4 +1,4 @@
-// Copyright © 2017 IBM Corp. All rights reserved.
+// Copyright © 2017, 2023 IBM Corp. All rights reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -11,31 +11,177 @@
 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 // See the License for the specific language governing permissions and
 // limitations under the License.
+
+/* global describe it */
 'use strict';
 
-// stolen from http://strongloop.com/strongblog/practical-examples-of-the-new-node-js-streams-api/
-const stream = require('stream');
+const assert = require('assert');
+const backup = require('../app.js').backup;
+const fs = require('fs');
+const nock = require('nock');
+const util = require('util');
+const backupPromise = util.promisify(backup);
 
-module.exports = function(onChange) {
-  const change = new stream.Transform({ objectMode: true });
+const goodUrl = 'http://localhost:5984/db';
+// The real validateArgs function of app.js isn't
+// exported - so we call the exported backup method
+// instead. We don't get as far as a real backup when
+// testing error cases. For success cases we nock the
+// goodUrl and
+const validateArgs = async function(url, opts, errorValidationForAssertRejects) {
+  const nullStream = fs.createWriteStream('/dev/null');
+  if (url === goodUrl) {
+    // Nock the goodUrl
+    nock(goodUrl).head('').reply(404, { error: 'not_found', reason: 'missing' });
+  }
+  return assert.rejects(backupPromise(url, nullStream, opts), errorValidationForAssertRejects);
+};
 
-  change._transform = function(line, encoding, done) {
-    let obj = null;
+const validateShallowModeArgs = async function(url, opts, msg) {
+  // We pass assertNoValidationError because for these shallow opts
+  // we are expecting only a stderr warning
+  return validateArgs(url, opts, assertNoValidationError()).then(() => {
+    // Assert the warning message was in stderr
+    assert(capturedStderr.indexOf(msg) > -1, 'Log warning message was not present');
+  });
+};
 
-    // one change per line - remove the trailing comma
-    line = line.trim().replace(/,$/, '');
+const stderrWriteFun = process.stderr.write;
+let capturedStderr;
 
-    // extract thee last_seq at the end of the changes feed
-    if (line.match(/^"last_seq":/)) {
-      line = '{' + line;
-    }
-    try {
-      obj = JSON.parse(line);
-    } catch (e) {
-    }
-    onChange(obj);
-    done();
+function captureStderr() {
+  process.stderr.write = function(string, encoding, fd) {
+    capturedStderr += string;
   };
+}
 
-  return change;
-};
+function releaseStderr() {
+  process.stderr.write = stderrWriteFun;
+  capturedStderr = null;
+}
+
+// Return a validation object for use with assert.rejects
+function assertErrorMessage(msg) {
+  return { name: 'InvalidOption', message: msg };
+}
+
+// For cases where validation should pass we reach a real backup that hits a 404
+// mock for a DatabaseNotFound, so that it is the expected in the case assertNoValidationError
+function assertNoValidationError() { return { name: 'DatabaseNotFound' }; }
+
+describe('#unit Validate arguments', function() {
+  it('returns error for invalid URL type', async function() {
+    return validateArgs(true, {}, assertErrorMessage('Invalid URL, must be type string'));
+  });
+  it('returns no error for valid URL type', async function() {
+    return validateArgs(goodUrl, {}, assertNoValidationError());
+  });
+  it('returns error for invalid (no host) URL', async function() {
+    return validateArgs('http://', {}, assertErrorMessage('Invalid URL'));
+  });
+  it('returns error for invalid (no protocol) URL', async function() {
+    return validateArgs('invalid', {}, assertErrorMessage('Invalid URL'));
+  });
+  it('returns error for invalid (wrong protocol) URL', async function() {
+    return validateArgs('ftp://invalid.example.com', {}, assertErrorMessage('Invalid URL protocol.'));
+  });
+  it('returns error for invalid (no path) URL', async function() {
+    return validateArgs('https://invalid.example.com', {}, assertErrorMessage('Invalid URL, missing path element (no database).'));
+  });
+  it('returns error for invalid (no protocol, no host) URL', async function() {
+    return validateArgs('invalid', {}, assertErrorMessage('Invalid URL'));
+  });
+  it('returns error for invalid buffer size type', async function() {
+    return validateArgs(goodUrl, { bufferSize: '123' }, assertErrorMessage('Invalid buffer size option, must be a positive integer in the range (0, MAX_SAFE_INTEGER]'));
+  });
+  it('returns error for zero buffer size', async function() {
+    return validateArgs(goodUrl, { bufferSize: 0 }, assertErrorMessage('Invalid buffer size option, must be a positive integer in the range (0, MAX_SAFE_INTEGER]'));
+  });
+  it('returns error for float buffer size', async function() {
+    return validateArgs(goodUrl, { bufferSize: 1.23 }, assertErrorMessage('Invalid buffer size option, must be a positive integer in the range (0, MAX_SAFE_INTEGER]'));
+  });
+  it('returns no error for valid buffer size type', async function() {
+    return validateArgs(goodUrl, { bufferSize: 123 }, assertNoValidationError());
+  });
+  it('returns error for invalid log type', async function() {
+    return validateArgs(goodUrl, { log: true }, assertErrorMessage('Invalid log option, must be type string'));
+  });
+  it('returns no error for valid log type', async function() {
+    return validateArgs(goodUrl, { log: 'log.txt' }, assertNoValidationError());
+  });
+  it('returns error for invalid mode type', async function() {
+    return validateArgs(goodUrl, { mode: true }, assertErrorMessage('Invalid mode option, must be either "full" or "shallow"'));
+  });
+  it('returns error for invalid mode string', async function() {
+    return validateArgs(goodUrl, { mode: 'foobar' }, assertErrorMessage('Invalid mode option, must be either "full" or "shallow"'));
+  });
+  it('returns no error for valid mode type', async function() {
+    return validateArgs(goodUrl, { mode: 'full' }, assertNoValidationError());
+  });
+  it('returns error for invalid output type', async function() {
+    return validateArgs(goodUrl, { output: true }, assertErrorMessage('Invalid output option, must be type string'));
+  });
+  it('returns no error for valid output type', async function() {
+    return validateArgs(goodUrl, { output: 'output.txt' }, assertNoValidationError());
+  });
+  it('returns error for invalid parallelism type', async function() {
+    return validateArgs(goodUrl, { parallelism: '123' }, assertErrorMessage('Invalid parallelism option, must be a positive integer in the range (0, MAX_SAFE_INTEGER]'));
+  });
+  it('returns error for zero parallelism', async function() {
+    return validateArgs(goodUrl, { parallelism: 0 }, assertErrorMessage('Invalid parallelism option, must be a positive integer in the range (0, MAX_SAFE_INTEGER]'));
+  });
+  it('returns error for float parallelism', async function() {
+    return validateArgs(goodUrl, { parallelism: 1.23 }, assertErrorMessage('Invalid parallelism option, must be a positive integer in the range (0, MAX_SAFE_INTEGER]'));
+  });
+  it('returns no error for valid parallelism type', async function() {
+    return validateArgs(goodUrl, { parallelism: 123 }, assertNoValidationError());
+  });
+  it('returns error for invalid request timeout type', async function() {
+    return validateArgs(goodUrl, { requestTimeout: '123' }, assertErrorMessage('Invalid request timeout option, must be a positive integer in the range (0, MAX_SAFE_INTEGER]'));
+  });
+  it('returns error for zero request timeout', async function() {
+    return validateArgs(goodUrl, { requestTimeout: 0 }, assertErrorMessage('Invalid request timeout option, must be a positive integer in the range (0, MAX_SAFE_INTEGER]'));
+  });
+  it('returns error for float request timout', async function() {
+    return validateArgs(goodUrl, { requestTimeout: 1.23 }, assertErrorMessage('Invalid request timeout option, must be a positive integer in the range (0, MAX_SAFE_INTEGER]'));
+  });
+  it('returns no error for valid request timeout type', async function() {
+    return validateArgs(goodUrl, { requestTimeout: 123 }, assertNoValidationError());
+  });
+  it('returns error for invalid resume type', async function() {
+    return validateArgs(goodUrl, { resume: 'true' }, assertErrorMessage('Invalid resume option, must be type boolean'));
+  });
+  it('returns no error for valid resume type', async function() {
+    return validateArgs(goodUrl, { resume: false }, assertNoValidationError());
+  });
+  it('returns error for invalid key type', async function() {
+    return validateArgs(goodUrl, { iamApiKey: true }, assertErrorMessage('Invalid iamApiKey option, must be type string'));
+  });
+  it('returns error for key and URL credentials supplied', async function() {
+    return validateArgs('https://a:b@example.com/db', { iamApiKey: 'abc123' }, assertErrorMessage('URL user information must not be supplied when using IAM API key.'));
+  });
+  it('warns for log arg in shallow mode', async function() {
+    captureStderr();
+    return validateShallowModeArgs(goodUrl, { mode: 'shallow', log: 'test' },
+      'the options "log" and "resume" are invalid when using shallow mode.').finally(
+      () => {
+        releaseStderr();
+      });
+  });
+  it('warns for resume arg in shallow mode', async function() {
+    captureStderr();
+    return validateShallowModeArgs(goodUrl, { mode: 'shallow', log: 'test', resume: true },
+      'the options "log" and "resume" are invalid when using shallow mode.').finally(
+      () => {
+        releaseStderr();
+      });
+  });
+  it('warns for parallelism arg in shallow mode', async function() {
+    captureStderr();
+    return validateShallowModeArgs(goodUrl, { mode: 'shallow', parallelism: 10 },
+      'the option "parallelism" has no effect when using shallow mode.').finally(
+      () => {
+        releaseStderr();
+      });
+  });
+});

package/.scannerwork/scanner-report/symbols-10.pb

@@ -1,30 +1,69 @@
-
- 
-PP ZZ 
- 		 
-	 %% ss 
-$$	 tt 2
-%% (( 
-)) 
-,, 
-..	 
-44	 "uu 6�

-44# '77 << AA FF KK PP UU ZZ __ dd ii nn 
-	 
- 
-  

-$$ // 

-664 588 
-;;9 :== 
-@@< =BB 
-EE< =GG 
-JJ@ ALL 
-OO4 5QQ 
-TT^ _VV 
-YY7 8[[ 
-^^] ^`` 
-cc\ ]ee 
-hh? @jj 
-mmA Boo 
-44) *pp 

-rr vv 

+
+ ** 
+   
+ FF 
+II 
+2
+ 
+ ** 
+ 
+ (, 5,,8 A--C L(
+* 3, 5,,C L--8 A2
+5 ;$ *$ *,,0 6--0 6(
+   (* 2** &(
+ * 2  (**( 0(
+  '"" ##0 9(
+ 4 =$$ %%0 9<
+ !! // // 11 66	 
+)) **3 @
+** ,,N U--N U
+,, //H X339 I
+-- //+ ;229 I
+,,! /--! /99 
+99 #DD  HH  
+99% 2DD4 A
+994 AHH4 A
+99C L@@ !
+:: BB4 B
+<< KK OO	 
+@@
 BB 
+AA
+ BB ,DDC T
+BB BB- /
+DD
+ FF 
+EE
+ FF& 8HHV h
+FF FF> ?
+FF  FF9 :
+HH
+ II 
+II JJ KK 
+II  KK 
+ # #RR 
+RR" (WW  &
+RR* 2WW9 A(
+SS XX 
+TT 
+UU WWC KZZ \\ ^^ (
+WW
+ XX! -YY \\ 
+WWg hWWl m
+!! && 
+11 44 
+= >77 

+<< << 
+AA AA  
+DD. /DDU V
+EE  EE  !
+HH. /HHi j
+JJ LL 
+II% &MM 
+@@# $NN 
+99Q RPP 

+WW3 4WWS T
+YY% &[[ 
+[[ ]] 
+VV ^^ 
+RR4 5`` 

+bb dd