@cloudant/couchbackup 2.9.16 → 2.9.17-SNAPSHOT.185

package/.scannerwork/scanner-report/source-43.txt

@@ -1,4 +1,4 @@
-// Copyright © 2017, 2018 IBM Corp. All rights reserved.
+// Copyright © 2017, 2023 IBM Corp. All rights reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -11,49 +11,173 @@
 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 // See the License for the specific language governing permissions and
 // limitations under the License.
+
+/* global describe it beforeEach */
 'use strict';
 
-/**
- * Utility methods for the command line interface.
- * @module cliutils
- * @see module:cliutils
- */
-
-const url = require('url');
-const error = require('./error.js');
-
-module.exports = {
-
-  /**
-   * Combine a base URL and a database name, ensuring at least single slash
-   * between root and database name. This allows users to have Couch behind
-   * proxies that mount Couch's / endpoint at some other mount point.
-   * @param {string} root - root URL
-   * @param {string} databaseName - database name
-   * @return concatenated URL.
-   *
-   * @private
-   */
-  databaseUrl: function databaseUrl(root, databaseName) {
-    if (!root.endsWith('/')) {
-      root = root + '/';
-    }
-    try {
-      return new url.URL(encodeURIComponent(databaseName), root).toString();
-    } catch (err) {
-      throw error.wrapPossibleInvalidUrlError(err);
-    }
-  },
-
-  /**
-   * Generate CLI argument usage text.
-   *
-   * @param {string} description - argument description.
-   * @param {string} defaultValue - default argument value.
-   *
-   * @private
-   */
-  getUsage: function getUsage(description, defaultValue) {
-    return `${description} ${defaultValue !== undefined ? ` (default: ${defaultValue})` : ''}`;
-  }
-};
+const assert = require('assert');
+const backup = require('../includes/shallowbackup.js');
+const request = require('../includes/request.js');
+const fs = require('fs');
+const nock = require('nock');
+
+// Function to create a DB object and call the shallow backup function
+// This is normally done by app.js
+function shallowBackup(dbUrl, opts) {
+  const db = request.client(dbUrl, opts);
+  // Disable compression to make body assertions easier
+  db.service.setEnableGzipCompression(false);
+  return backup(db, opts);
+}
+
+// Note all these tests include a body parameter of include_docs and a query
+// string of include_docs because of a quirk of nano that when using the fetch
+// method always adds the include_docs query string.
+describe('#unit Perform backup using shallow backup', function() {
+  const dbUrl = 'http://localhost:5984/animaldb';
+  // Query string keys are stringified by Nano
+  const badgerKey = 'badger\0';
+  const kookaburraKey = 'kookaburra\0';
+  const snipeKey = 'snipe\0';
+
+  beforeEach('Reset nocks', function() {
+    nock.cleanAll();
+  });
+
+  it('should perform a shallow backup', async function() {
+    const couch = nock(dbUrl)
+      // batch 1
+      .post('/_all_docs', { limit: 3, include_docs: true })
+      .reply(200, JSON.parse(fs.readFileSync('./test/fixtures/animaldb_all_docs_1.json', 'utf8')))
+      // batch 2
+      .post('/_all_docs', { limit: 3, start_key: badgerKey, include_docs: true })
+      .reply(200, JSON.parse(fs.readFileSync('./test/fixtures/animaldb_all_docs_2.json', 'utf8')))
+      // batch 3
+      .post('/_all_docs', { limit: 3, start_key: kookaburraKey, include_docs: true })
+      .reply(200, JSON.parse(fs.readFileSync('./test/fixtures/animaldb_all_docs_3.json', 'utf8')))
+      // batch 4
+      .post('/_all_docs', { limit: 3, start_key: snipeKey, include_docs: true })
+      .reply(200, JSON.parse(fs.readFileSync('./test/fixtures/animaldb_all_docs_4.json', 'utf8')));
+
+    return new Promise((resolve, reject) => {
+      shallowBackup(dbUrl, { bufferSize: 3, parallelism: 1 })
+        .on('error', function(err) {
+          reject(err);
+        })
+        .on('received', function(data) {
+          try {
+            if (data.batch === 3) {
+              assert.strictEqual(data.length, 2); // smaller last batch
+            } else {
+              assert.strictEqual(data.length, 3);
+            }
+          } catch (err) {
+            reject(err);
+          }
+        })
+        .on('finished', function(data) {
+          try {
+            assert.strictEqual(data.total, 11);
+            assert.ok(couch.isDone());
+            resolve();
+          } catch (err) {
+            reject(err);
+          }
+        });
+    });
+  });
+
+  it('should perform a shallow backup with transient error', async function() {
+    const couch = nock(dbUrl)
+      // batch 1
+      .post('/_all_docs', { limit: 3, include_docs: true })
+      .reply(200, JSON.parse(fs.readFileSync('./test/fixtures/animaldb_all_docs_1.json', 'utf8')))
+      // batch 2
+      .post('/_all_docs', { limit: 3, start_key: badgerKey, include_docs: true })
+      .reply(200, JSON.parse(fs.readFileSync('./test/fixtures/animaldb_all_docs_2.json', 'utf8')))
+      // batch 3 - transient error
+      .post('/_all_docs', { limit: 3, start_key: kookaburraKey, include_docs: true })
+      .reply(500, { error: 'Internal Server Error' })
+      // batch 3 - retry
+      .post('/_all_docs', { limit: 3, start_key: kookaburraKey, include_docs: true })
+      .reply(200, JSON.parse(fs.readFileSync('./test/fixtures/animaldb_all_docs_3.json', 'utf8')))
+      // batch 4
+      .post('/_all_docs', { limit: 3, start_key: snipeKey, include_docs: true })
+      .reply(200, JSON.parse(fs.readFileSync('./test/fixtures/animaldb_all_docs_4.json', 'utf8')));
+
+    return new Promise((resolve, reject) => {
+      shallowBackup(dbUrl, { bufferSize: 3, parallelism: 1 })
+        .on('error', function(err) {
+          try {
+            assert.strictEqual(err.name, 'HTTPError');
+          } catch (err) {
+            reject(err);
+          }
+        })
+        .on('received', function(data) {
+          try {
+            if (data.batch === 3) {
+              assert.strictEqual(data.length, 2); // smaller last batch
+            } else {
+              assert.strictEqual(data.length, 3);
+            }
+          } catch (err) {
+            reject(err);
+          }
+        })
+        .on('finished', function(data) {
+          try {
+            assert.strictEqual(data.total, 11);
+            assert.ok(couch.isDone());
+            resolve();
+          } catch (err) {
+            reject(err);
+          }
+        });
+    });
+  });
+
+  it('should fail to perform a shallow backup on fatal error', async function() {
+    const couch = nock(dbUrl)
+      // batch 1
+      .post('/_all_docs', { limit: 3, include_docs: true })
+      .reply(200, JSON.parse(fs.readFileSync('./test/fixtures/animaldb_all_docs_1.json', 'utf8')))
+      // batch 2
+      .post('/_all_docs', { limit: 3, start_key: badgerKey, include_docs: true })
+      .reply(200, JSON.parse(fs.readFileSync('./test/fixtures/animaldb_all_docs_2.json', 'utf8')))
+      // batch 3 - fatal error
+      .post('/_all_docs', { limit: 3, start_key: kookaburraKey, include_docs: true })
+      .reply(401, { error: 'Unauthorized' });
+
+    let errCount = 0;
+
+    return new Promise((resolve, reject) => {
+      shallowBackup(dbUrl, { bufferSize: 3, parallelism: 1 })
+        .on('error', function(err) {
+          try {
+            errCount++;
+            assert.strictEqual(err.name, 'Unauthorized');
+          } catch (err) {
+            reject(err);
+          }
+        })
+        .on('received', function(data) {
+          try {
+            assert.strictEqual(data.length, 3);
+          } catch (err) {
+            reject(err);
+          }
+        })
+        .on('finished', function(data) {
+          try {
+            assert.strictEqual(data.total, 6);
+            assert.ok(couch.isDone());
+            assert.strictEqual(errCount, 1);
+            resolve();
+          } catch (err) {
+            reject(err);
+          }
+        });
+    });
+  });
+});

package/.scannerwork/scanner-report/source-44.txt

@@ -1,4 +1,4 @@
-// Copyright © 2017, 2021 IBM Corp. All rights reserved.
+// Copyright © 2017, 2023 IBM Corp. All rights reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -11,168 +11,27 @@
 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 // See the License for the specific language governing permissions and
 // limitations under the License.
-'use strict';
-
-const pkg = require('../package.json');
-const stream = require('stream');
-const { CloudantV1, CouchdbSessionAuthenticator } = require('@ibm-cloud/cloudant');
-const { IamAuthenticator, NoAuthAuthenticator } = require('ibm-cloud-sdk-core');
-const retryPlugin = require('retry-axios');
-
-const userAgent = 'couchbackup-cloudant/' + pkg.version + ' (Node.js ' +
-      process.version + ')';
-
-// Class for streaming _changes error responses into
-// In general the response is a small error/reason JSON object
-// so it is OK to have this in memory.
-class ResponseWriteable extends stream.Writable {
-  constructor(options) {
-    super(options);
-    this.data = [];
-  }
-
-  _write(chunk, encoding, callback) {
-    this.data.push(chunk);
-    callback();
-  }
-
-  stringBody() {
-    return Buffer.concat(this.data).toString();
-  }
-}
 
-// An interceptor function to help augment error bodies with a little
-// extra information so we can continue to use consistent messaging
-// after the ugprade to @ibm-cloud/cloudant
-const errorHelper = async function(err) {
-  let method;
-  let requestUrl;
-  if (err.response) {
-    if (err.response.config.url) {
-      requestUrl = err.response.config.url;
-      method = err.response.config.method;
-    }
-    // Override the status text with an improved message
-    let errorMsg = `${err.response.status} ${err.response.statusText || ''}: ` +
-    `${method} ${requestUrl}`;
-    if (err.response.data) {
-      // Check if we have a JSON response and try to get the error/reason
-      if (err.response.headers['content-type'] === 'application/json') {
-        if (!err.response.data.error && err.response.data.pipe) {
-          // If we didn't find a JSON object with `error` then we might have a stream response.
-          // Detect the stream by the presence of `pipe` and use it to get the body and parse
-          // the error information.
-          const p = new Promise((resolve, reject) => {
-            const errorBody = new ResponseWriteable();
-            err.response.data.pipe(errorBody)
-              .on('finish', () => { resolve(JSON.parse(errorBody.stringBody())); })
-              .on('error', () => { reject(err); });
-          });
-          // Replace the stream on the response with the parsed object
-          err.response.data = await p;
-        }
-        // Append the error/reason if available
-        if (err.response.data.error) {
-          // Override the status text with our more complete message
-          errorMsg += ` - Error: ${err.response.data.error}`;
-          if (err.response.data.reason) {
-            errorMsg += `, Reason: ${err.response.data.reason}`;
-          }
-        }
-      } else {
-        errorMsg += err.response.data;
-      }
-      // Set a new message for use by the node-sdk-core
-      // We use the errors array because it gets processed
-      // ahead of all other service errors.
-      err.response.data.errors = [{ message: errorMsg }];
-    }
-  } else if (err.request) {
-    if (!err.message.includes(err.config.url)) {
-      // Augment the message with the URL and method
-      // but don't do it again if we already have the URL.
-      err.message = `${err.message}: ${err.config.method} ${err.config.url}`;
-    }
-  }
-  return Promise.reject(err);
-};
-
-module.exports = {
-  client: function(rawUrl, opts) {
-    const url = new URL(rawUrl);
-    // Split the URL to separate service from database
-    // Use origin as the "base" to remove auth elements
-    const actUrl = new URL(url.pathname.substring(0, url.pathname.lastIndexOf('/')), url.origin);
-    const dbName = url.pathname.substring(url.pathname.lastIndexOf('/') + 1);
-    let authenticator;
-    // Default to cookieauth unless an IAM key is provided
-    if (opts.iamApiKey) {
-      const iamAuthOpts = { apikey: opts.iamApiKey };
-      if (opts.iamTokenUrl) {
-        iamAuthOpts.url = opts.iamTokenUrl;
-      }
-      authenticator = new IamAuthenticator(iamAuthOpts);
-    } else if (url.username) {
-      authenticator = new CouchdbSessionAuthenticator({
-        username: decodeURIComponent(url.username),
-        password: decodeURIComponent(url.password)
-      });
-    } else {
-      authenticator = new NoAuthAuthenticator();
-    }
-    const serviceOpts = {
-      authenticator: authenticator,
-      timeout: opts.requestTimeout,
-      // Axios performance options
-      maxContentLength: -1
-    };
-
-    const service = new CloudantV1(serviceOpts);
-    // Configure retries
-    const maxRetries = 2; // for 3 total attempts
-    service.getHttpClient().defaults.raxConfig = {
-      // retries for status codes
-      retry: maxRetries,
-      // retries for non-response e.g. ETIMEDOUT
-      noResponseRetries: maxRetries,
-      backoffType: 'exponential',
-      httpMethodsToRetry: ['GET', 'HEAD', 'POST'],
-      statusCodesToRetry: [
-        [429, 429],
-        [500, 599]
-      ],
-      shouldRetry: err => {
-        const cfg = retryPlugin.getConfig(err);
-        // cap at max retries regardless of response/non-response type
-        if (cfg.currentRetryAttempt >= maxRetries) {
-          return false;
-        } else {
-          return retryPlugin.shouldRetryRequest(err);
-        }
-      },
-      instance: service.getHttpClient()
-    };
-    retryPlugin.attach(service.getHttpClient());
-
-    service.setServiceUrl(actUrl.toString());
-    if (authenticator instanceof CouchdbSessionAuthenticator) {
-      // Awkward workaround for known Couch issue with compression on _session requests
-      // It is not feasible to disable compression on all requests with the amount of
-      // data this lib needs to move, so override the property in the tokenManager instance.
-      authenticator.tokenManager.requestWrapperInstance.compressRequestData = false;
-    }
-    if (authenticator.tokenManager && authenticator.tokenManager.requestWrapperInstance) {
-      authenticator.tokenManager.requestWrapperInstance.axiosInstance.interceptors.response.use(null, errorHelper);
-    }
-    // Add error interceptors to put URLs in error messages
-    service.getHttpClient().interceptors.response.use(null, errorHelper);
-
-    // Add request interceptor to add user-agent (adding it with custom request headers gets overwritten)
-    service.getHttpClient().interceptors.request.use(function(requestConfig) {
-      requestConfig.headers['User-Agent'] = userAgent;
-      return requestConfig;
-    }, null);
+/* global describe it */
+'use strict';
 
-    return { service: service, db: dbName, url: actUrl.toString() };
-  }
-};
+delete require.cache[require.resolve('./citestutils.js')];
+const u = require('./citestutils.js');
+
+[{ useApi: true }, { useApi: false }].forEach(function(params) {
+  describe(u.scenario('End to end backup and restore', params), function() {
+    it('should backup and restore animaldb', async function() {
+      // Allow up to 60 s for backup and restore of animaldb
+      u.setTimeout(this, 60);
+      return u.testDirectBackupAndRestore(params, 'animaldb', this.dbName);
+    });
+
+    it('should backup and restore largedb1g #slow', async function() {
+      // Allow up to 30 m for backup and restore of largedb1g
+      // This is a long time but when many builds run in parallel it can take a
+      // while to get this done.
+      u.setTimeout(this, 30 * 60);
+      return u.testDirectBackupAndRestore(params, 'largedb1g', this.dbName);
+    });
+  });
+});

package/.scannerwork/scanner-report/source-45.txt

@@ -1,4 +1,4 @@
-// Copyright © 2017, 2021 IBM Corp. All rights reserved.
+// Copyright © 2017, 2023 IBM Corp. All rights reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -11,108 +11,120 @@
 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 // See the License for the specific language governing permissions and
 // limitations under the License.
+
+/* global describe it before after */
 'use strict';
 
-const path = require('path');
-const tmp = require('tmp');
-
-/**
-  Return API default settings.
-*/
-function apiDefaults() {
-  return {
-    parallelism: 5,
-    bufferSize: 500,
-    requestTimeout: 120000,
-    log: tmp.tmpNameSync(),
-    resume: false,
-    mode: 'full'
-  };
-}
-
-/**
-  Return CLI default settings.
-*/
-function cliDefaults() {
-  const defaults = apiDefaults();
-
-  // add additional legacy settings
-  defaults.db = 'test';
-  defaults.url = 'http://localhost:5984';
-
-  // add CLI only option
-  defaults.quiet = false;
-
-  return defaults;
-}
-
-/**
-  Override settings **in-place** with environment variables.
-*/
-function applyEnvironmentVariables(opts) {
-  // if we have a custom CouchDB url
-  if (typeof process.env.COUCH_URL !== 'undefined') {
-    opts.url = process.env.COUCH_URL;
-  }
-
-  // if we have a specified databases
-  if (typeof process.env.COUCH_DATABASE !== 'undefined') {
-    opts.db = process.env.COUCH_DATABASE;
-  }
-
-  // if we have a specified buffer size
-  if (typeof process.env.COUCH_BUFFER_SIZE !== 'undefined') {
-    opts.bufferSize = parseInt(process.env.COUCH_BUFFER_SIZE);
-  }
-
-  // if we have a specified parallelism
-  if (typeof process.env.COUCH_PARALLELISM !== 'undefined') {
-    opts.parallelism = parseInt(process.env.COUCH_PARALLELISM);
-  }
-
-  // if we have a specified request timeout
-  if (typeof process.env.COUCH_REQUEST_TIMEOUT !== 'undefined') {
-    opts.requestTimeout = parseInt(process.env.COUCH_REQUEST_TIMEOUT);
-  }
-
-  // if we have a specified log file
-  if (typeof process.env.COUCH_LOG !== 'undefined') {
-    opts.log = path.normalize(process.env.COUCH_LOG);
-  }
-
-  // if we are instructed to resume
-  if (typeof process.env.COUCH_RESUME !== 'undefined' && process.env.COUCH_RESUME === 'true') {
-    opts.resume = true;
-  }
-
-  // if we are given an output filename
-  if (typeof process.env.COUCH_OUTPUT !== 'undefined') {
-    opts.output = path.normalize(process.env.COUCH_OUTPUT);
-  }
-
-  // if we only want a shallow copy
-  if (typeof process.env.COUCH_MODE !== 'undefined' && process.env.COUCH_MODE === 'shallow') {
-    opts.mode = 'shallow';
-  }
-
-  // if we are instructed to be quiet
-  if (typeof process.env.COUCH_QUIET !== 'undefined' && process.env.COUCH_QUIET === 'true') {
-    opts.quiet = true;
-  }
-
-  // if we have a specified API key
-  if (typeof process.env.CLOUDANT_IAM_API_KEY !== 'undefined') {
-    opts.iamApiKey = process.env.CLOUDANT_IAM_API_KEY;
-  }
-
-  // if we have a specified IAM token endpoint
-  if (typeof process.env.CLOUDANT_IAM_TOKEN_URL !== 'undefined') {
-    opts.iamTokenUrl = process.env.CLOUDANT_IAM_TOKEN_URL;
-  }
-}
-
-module.exports = {
-  apiDefaults: apiDefaults,
-  cliDefaults: cliDefaults,
-  applyEnvironmentVariables: applyEnvironmentVariables
-};
+const assert = require('assert');
+const applyEnvVars = require('../includes/config.js').applyEnvironmentVariables;
+
+describe('#unit Configuration', function() {
+  let processEnvCopy;
+
+  before('Save env', function() {
+    // Copy env so we can reset it after the tests
+    processEnvCopy = JSON.parse(JSON.stringify(process.env));
+  });
+
+  after('Reset env', function() {
+    process.env = processEnvCopy;
+  });
+
+  it('respects the COUCH_URL env variable', function() {
+    process.env.COUCH_URL = 'http://user:pass@myurl.com';
+    const config = {};
+    applyEnvVars(config);
+    assert.strictEqual(typeof config.url, 'string');
+    assert.strictEqual(config.url, process.env.COUCH_URL);
+  });
+
+  it('respects the COUCH_DATABASE env variable', function() {
+    process.env.COUCH_DATABASE = 'mydb';
+    const config = {};
+    applyEnvVars(config);
+    assert.strictEqual(typeof config.db, 'string');
+    assert.strictEqual(config.db, process.env.COUCH_DATABASE);
+  });
+
+  it('respects the COUCH_BUFFER_SIZE env variable', function() {
+    process.env.COUCH_BUFFER_SIZE = '1000';
+    const config = {};
+    applyEnvVars(config);
+    assert.strictEqual(typeof config.bufferSize, 'number');
+    assert.strictEqual(config.bufferSize, 1000);
+  });
+
+  it('respects the COUCH_PARALLELISM env variable', function() {
+    process.env.COUCH_PARALLELISM = '20';
+    const config = {};
+    applyEnvVars(config);
+    assert.strictEqual(typeof config.parallelism, 'number');
+    assert.strictEqual(config.parallelism, 20);
+  });
+
+  it('respects the COUCH_REQUEST_TIMEOUT env variable', function() {
+    process.env.COUCH_REQUEST_TIMEOUT = '10000';
+    const config = {};
+    applyEnvVars(config);
+    assert.strictEqual(typeof config.requestTimeout, 'number');
+    assert.strictEqual(config.requestTimeout, 10000);
+  });
+
+  it('respects the CLOUDANT_IAM_API_KEY env variable', function() {
+    const key = 'ABC123-ZYX987_cba789-xyz321';
+    process.env.CLOUDANT_IAM_API_KEY = key;
+    const config = {};
+    applyEnvVars(config);
+    assert.strictEqual(typeof config.iamApiKey, 'string');
+    assert.strictEqual(config.iamApiKey, key);
+  });
+
+  it('respects the CLOUDANT_IAM_TOKEN_URL env variable', function() {
+    const u = 'https://testhost.example:1234/identity/token';
+    process.env.CLOUDANT_IAM_TOKEN_URL = u;
+    const config = {};
+    applyEnvVars(config);
+    assert.strictEqual(typeof config.iamTokenUrl, 'string');
+    assert.strictEqual(config.iamTokenUrl, u);
+  });
+
+  it('respects the COUCH_LOG env variable', function() {
+    process.env.COUCH_LOG = 'my.log';
+    const config = {};
+    applyEnvVars(config);
+    assert.strictEqual(typeof config.log, 'string');
+    assert.strictEqual(config.log, process.env.COUCH_LOG);
+  });
+
+  it('respects the COUCH_RESUME env variable', function() {
+    process.env.COUCH_RESUME = 'true';
+    const config = {};
+    applyEnvVars(config);
+    assert.strictEqual(typeof config.resume, 'boolean');
+    assert.strictEqual(config.resume, true);
+  });
+
+  it('respects the COUCH_OUTPUT env variable', function() {
+    process.env.COUCH_OUTPUT = 'myfile.txt';
+    const config = {};
+    applyEnvVars(config);
+    assert.strictEqual(typeof config.output, 'string');
+    assert.strictEqual(config.output, process.env.COUCH_OUTPUT);
+  });
+
+  it('respects the COUCH_MODE env variable', function() {
+    process.env.COUCH_MODE = 'shallow';
+    const config = {};
+    applyEnvVars(config);
+    assert.strictEqual(typeof config.mode, 'string');
+    assert.strictEqual(config.mode, 'shallow');
+  });
+
+  it('respects the COUCH_QUIET env variable', function() {
+    process.env.COUCH_QUIET = 'true';
+    const config = {};
+    applyEnvVars(config);
+    assert.strictEqual(typeof config.quiet, 'boolean');
+    assert.strictEqual(config.quiet, true);
+  });
+});