@cloudant/couchbackup 2.9.9-SNAPSHOT.98 → 2.9.9

package/.scannerwork/scanner-report/source-60.txt

@@ -1,4 +1,4 @@
-// Copyright © 2017, 2022 IBM Corp. All rights reserved.
+// Copyright © 2017, 2021 IBM Corp. All rights reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -11,135 +11,168 @@
 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 // See the License for the specific language governing permissions and
 // limitations under the License.
-
-/* global describe it beforeEach */
 'use strict';
 
-const assert = require('assert');
-const backup = require('../includes/shallowbackup.js');
-const request = require('../includes/request.js');
-const fs = require('fs');
-const nock = require('nock');
+const pkg = require('../package.json');
+const stream = require('stream');
+const { CloudantV1, CouchdbSessionAuthenticator } = require('@ibm-cloud/cloudant');
+const { IamAuthenticator, NoAuthAuthenticator } = require('ibm-cloud-sdk-core');
+const retryPlugin = require('retry-axios');
 
-// Function to create a DB object and call the shallow backup function
-// This is normally done by app.js
-function shallowBackup(dbUrl, opts) {
-  const db = request.client(dbUrl, opts);
-  // Disable compression to make body assertions easier
-  db.service.setEnableGzipCompression(false);
-  return backup(db, opts);
-}
+const userAgent = 'couchbackup-cloudant/' + pkg.version + ' (Node.js ' +
+      process.version + ')';
 
-// Note all these tests include a body parameter of include_docs and a query
-// string of include_docs because of a quirk of nano that when using the fetch
-// method always adds the include_docs query string.
-describe('#unit Perform backup using shallow backup', function() {
-  const dbUrl = 'http://localhost:5984/animaldb';
-  // Query string keys are stringified by Nano
-  const badgerKey = 'badger\0';
-  const kookaburraKey = 'kookaburra\0';
-  const snipeKey = 'snipe\0';
+// Class for streaming _changes error responses into
+// In general the response is a small error/reason JSON object
+// so it is OK to have this in memory.
+class ResponseWriteable extends stream.Writable {
+  constructor(options) {
+    super(options);
+    this.data = [];
+  }
 
-  beforeEach('Reset nocks', function() {
-    nock.cleanAll();
-  });
+  _write(chunk, encoding, callback) {
+    this.data.push(chunk);
+    callback();
+  }
 
-  it('should perform a shallow backup', function(done) {
-    const couch = nock(dbUrl)
-      // batch 1
-      .post('/_all_docs', { limit: 3, include_docs: true })
-      .reply(200, JSON.parse(fs.readFileSync('./test/fixtures/animaldb_all_docs_1.json', 'utf8')))
-      // batch 2
-      .post('/_all_docs', { limit: 3, start_key: badgerKey, include_docs: true })
-      .reply(200, JSON.parse(fs.readFileSync('./test/fixtures/animaldb_all_docs_2.json', 'utf8')))
-      // batch 3
-      .post('/_all_docs', { limit: 3, start_key: kookaburraKey, include_docs: true })
-      .reply(200, JSON.parse(fs.readFileSync('./test/fixtures/animaldb_all_docs_3.json', 'utf8')))
-      // batch 4
-      .post('/_all_docs', { limit: 3, start_key: snipeKey, include_docs: true })
-      .reply(200, JSON.parse(fs.readFileSync('./test/fixtures/animaldb_all_docs_4.json', 'utf8')));
+  stringBody() {
+    return Buffer.concat(this.data).toString();
+  }
+}
 
-    shallowBackup(dbUrl, { bufferSize: 3, parallelism: 1 })
-      .on('error', function(err) {
-        assert.fail(err);
-      })
-      .on('received', function(data) {
-        if (data.batch === 3) {
-          assert.strictEqual(data.length, 2); // smaller last batch
-        } else {
-          assert.strictEqual(data.length, 3);
+// An interceptor function to help augment error bodies with a little
+// extra information so we can continue to use consistent messaging
+// after the ugprade to @ibm-cloud/cloudant
+const errorHelper = async function(err) {
+  let method;
+  let requestUrl;
+  if (err.response) {
+    if (err.response.config.url) {
+      requestUrl = err.response.config.url;
+      method = err.response.config.method;
+    }
+    // Override the status text with an improved message
+    let errorMsg = `${err.response.status} ${err.response.statusText || ''}: ` +
+    `${method} ${requestUrl}`;
+    if (err.response.data) {
+      // Check if we have a JSON response and try to get the error/reason
+      if (err.response.headers['content-type'] === 'application/json') {
+        if (!err.response.data.error && err.response.data.pipe) {
+          // If we didn't find a JSON object with `error` then we might have a stream response.
+          // Detect the stream by the presence of `pipe` and use it to get the body and parse
+          // the error information.
+          const p = new Promise((resolve, reject) => {
+            const errorBody = new ResponseWriteable();
+            err.response.data.pipe(errorBody)
+              .on('finish', () => { resolve(JSON.parse(errorBody.stringBody())); })
+              .on('error', () => { reject(err); });
+          });
+          // Replace the stream on the response with the parsed object
+          err.response.data = await p;
         }
-      })
-      .on('finished', function(data) {
-        assert.strictEqual(data.total, 11);
-        assert.ok(couch.isDone());
-        done();
-      });
-  });
+        // Append the error/reason if available
+        if (err.response.data.error) {
+          // Override the status text with our more complete message
+          errorMsg += ` - Error: ${err.response.data.error}`;
+          if (err.response.data.reason) {
+            errorMsg += `, Reason: ${err.response.data.reason}`;
+          }
+        }
+      } else {
+        errorMsg += err.response.data;
+      }
+      // Set a new message for use by the node-sdk-core
+      // We use the errors array because it gets processed
+      // ahead of all other service errors.
+      err.response.data.errors = [{ message: errorMsg }];
+    }
+  } else if (err.request) {
+    if (!err.message.includes(err.config.url)) {
+      // Augment the message with the URL and method
+      // but don't do it again if we already have the URL.
+      err.message = `${err.message}: ${err.config.method} ${err.config.url}`;
+    }
+  }
+  return Promise.reject(err);
+};
 
-  it('should perform a shallow backup with transient error', function(done) {
-    const couch = nock(dbUrl)
-      // batch 1
-      .post('/_all_docs', { limit: 3, include_docs: true })
-      .reply(200, JSON.parse(fs.readFileSync('./test/fixtures/animaldb_all_docs_1.json', 'utf8')))
-      // batch 2
-      .post('/_all_docs', { limit: 3, start_key: badgerKey, include_docs: true })
-      .reply(200, JSON.parse(fs.readFileSync('./test/fixtures/animaldb_all_docs_2.json', 'utf8')))
-      // batch 3 - transient error
-      .post('/_all_docs', { limit: 3, start_key: kookaburraKey, include_docs: true })
-      .reply(500, { error: 'Internal Server Error' })
-      // batch 3 - retry
-      .post('/_all_docs', { limit: 3, start_key: kookaburraKey, include_docs: true })
-      .reply(200, JSON.parse(fs.readFileSync('./test/fixtures/animaldb_all_docs_3.json', 'utf8')))
-      // batch 4
-      .post('/_all_docs', { limit: 3, start_key: snipeKey, include_docs: true })
-      .reply(200, JSON.parse(fs.readFileSync('./test/fixtures/animaldb_all_docs_4.json', 'utf8')));
+module.exports = {
+  client: function(rawUrl, opts) {
+    const url = new URL(rawUrl);
+    // Split the URL to separate service from database
+    // Use origin as the "base" to remove auth elements
+    const actUrl = new URL(url.pathname.substring(0, url.pathname.lastIndexOf('/')), url.origin);
+    const dbName = url.pathname.substring(url.pathname.lastIndexOf('/') + 1);
+    let authenticator;
+    // Default to cookieauth unless an IAM key is provided
+    if (opts.iamApiKey) {
+      const iamAuthOpts = { apikey: opts.iamApiKey };
+      if (opts.iamTokenUrl) {
+        iamAuthOpts.url = opts.iamTokenUrl;
+      }
+      authenticator = new IamAuthenticator(iamAuthOpts);
+    } else if (url.username) {
+      authenticator = new CouchdbSessionAuthenticator({
+        username: decodeURIComponent(url.username),
+        password: decodeURIComponent(url.password)
+      });
+    } else {
+      authenticator = new NoAuthAuthenticator();
+    }
+    const serviceOpts = {
+      authenticator: authenticator,
+      timeout: opts.requestTimeout,
+      // Axios performance options
+      maxContentLength: -1
+    };
 
-    shallowBackup(dbUrl, { bufferSize: 3, parallelism: 1 })
-      .on('error', function(err) {
-        assert.strictEqual(err.name, 'HTTPError');
-      })
-      .on('received', function(data) {
-        if (data.batch === 3) {
-          assert.strictEqual(data.length, 2); // smaller last batch
+    const service = new CloudantV1(serviceOpts);
+    // Configure retries
+    const maxRetries = 2; // for 3 total attempts
+    service.getHttpClient().defaults.raxConfig = {
+      // retries for status codes
+      retry: maxRetries,
+      // retries for non-response e.g. ETIMEDOUT
+      noResponseRetries: maxRetries,
+      backoffType: 'exponential',
+      httpMethodsToRetry: ['GET', 'HEAD', 'POST'],
+      statusCodesToRetry: [
+        [429, 429],
+        [500, 599]
+      ],
+      shouldRetry: err => {
+        const cfg = retryPlugin.getConfig(err);
+        // cap at max retries regardless of response/non-response type
+        if (cfg.currentRetryAttempt >= maxRetries) {
+          return false;
         } else {
-          assert.strictEqual(data.length, 3);
+          return retryPlugin.shouldRetryRequest(err);
         }
-      })
-      .on('finished', function(data) {
-        assert.strictEqual(data.total, 11);
-        assert.ok(couch.isDone());
-        done();
-      });
-  });
+      },
+      instance: service.getHttpClient()
+    };
+    retryPlugin.attach(service.getHttpClient());
 
-  it('should fail to perform a shallow backup on fatal error', function(done) {
-    const couch = nock(dbUrl)
-      // batch 1
-      .post('/_all_docs', { limit: 3, include_docs: true })
-      .reply(200, JSON.parse(fs.readFileSync('./test/fixtures/animaldb_all_docs_1.json', 'utf8')))
-      // batch 2
-      .post('/_all_docs', { limit: 3, start_key: badgerKey, include_docs: true })
-      .reply(200, JSON.parse(fs.readFileSync('./test/fixtures/animaldb_all_docs_2.json', 'utf8')))
-      // batch 3 - fatal error
-      .post('/_all_docs', { limit: 3, start_key: kookaburraKey, include_docs: true })
-      .reply(401, { error: 'Unauthorized' });
+    service.setServiceUrl(actUrl.toString());
+    if (authenticator instanceof CouchdbSessionAuthenticator) {
+      // Awkward workaround for known Couch issue with compression on _session requests
+      // It is not feasible to disable compression on all requests with the amount of
+      // data this lib needs to move, so override the property in the tokenManager instance.
+      authenticator.tokenManager.requestWrapperInstance.compressRequestData = false;
+    }
+    if (authenticator.tokenManager && authenticator.tokenManager.requestWrapperInstance) {
+      authenticator.tokenManager.requestWrapperInstance.axiosInstance.interceptors.response.use(null, errorHelper);
+    }
+    // Add error interceptors to put URLs in error messages
+    service.getHttpClient().interceptors.response.use(null, errorHelper);
 
-    let errCount = 0;
+    // Add request interceptor to add user-agent (adding it with custom request headers gets overwritten)
+    service.getHttpClient().interceptors.request.use(function(requestConfig) {
+      requestConfig.headers['User-Agent'] = userAgent;
+      return requestConfig;
+    }, null);
 
-    shallowBackup(dbUrl, { bufferSize: 3, parallelism: 1 })
-      .on('error', function(err) {
-        errCount++;
-        assert.strictEqual(err.name, 'Unauthorized');
-      })
-      .on('received', function(data) {
-        assert.strictEqual(data.length, 3);
-      })
-      .on('finished', function(data) {
-        assert.strictEqual(data.total, 6);
-        assert.ok(couch.isDone());
-        assert.strictEqual(errCount, 1);
-        done();
-      });
-  });
-});
+    return { service: service, db: dbName, url: actUrl.toString() };
+  }
+};

package/.scannerwork/scanner-report/source-61.txt

@@ -1,4 +1,4 @@
-// Copyright © 2017 IBM Corp. All rights reserved.
+// Copyright © 2017, 2021 IBM Corp. All rights reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -11,24 +11,94 @@
 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 // See the License for the specific language governing permissions and
 // limitations under the License.
-
-/* global describe it */
 'use strict';
 
-const assert = require('assert');
-const logfilesummary = require('../includes/logfilesummary.js');
+// fatal errors
+const codes = {
+  Error: 1,
+  InvalidOption: 2,
+  DatabaseNotFound: 10,
+  Unauthorized: 11,
+  Forbidden: 12,
+  DatabaseNotEmpty: 13,
+  NoLogFileName: 20,
+  LogDoesNotExist: 21,
+  IncompleteChangesInLogFile: 22,
+  SpoolChangesError: 30,
+  HTTPFatalError: 40,
+  BulkGetError: 50
+};
+
+class BackupError extends Error {
+  constructor(name, message) {
+    super(message);
+    this.name = name;
+  }
+}
+
+class HTTPError extends BackupError {
+  constructor(responseError, name) {
+    // Special case some names for more useful error messages
+    switch (responseError.status) {
+      case 401:
+        name = 'Unauthorized';
+        break;
+      case 403:
+        name = 'Forbidden';
+        break;
+      default:
+        name = name || 'HTTPFatalError';
+    }
+    super(name, responseError.message);
+  }
+}
+
+// Default function to return an error for HTTP status codes
+// < 400 -> OK
+// 4XX (except 429) -> Fatal
+// 429 & >=500 -> Transient
+function checkResponse(err) {
+  if (err) {
+    // Construct an HTTPError if there is request information on the error
+    // Codes < 400 are considered OK
+    if (err.status >= 400) {
+      return new HTTPError(err);
+    } else {
+      // Send it back again if there was no status code, e.g. a cxn error
+      return augmentMessage(err);
+    }
+  }
+}
+
+function convertResponseError(responseError, errorFactory) {
+  if (!errorFactory) {
+    errorFactory = checkResponse;
+  }
+  return errorFactory(responseError);
+}
+
+function augmentMessage(err) {
+  // For errors that don't have a status code, we are likely looking at a cxn
+  // error.
+  // Try to augment the message with more detail (core puts the code in statusText)
+  if (err && err.statusText) {
+    err.message = `${err.message} ${err.statusText}`;
+  }
+  if (err && err.description) {
+    err.message = `${err.message} ${err.description}`;
+  }
+  return err;
+}
 
-describe('#unit Fetching summary from the log file', function() {
-  it('should fetch a summary correctly', function(done) {
-    logfilesummary('./test/fixtures/test.log', function(err, data) {
-      assert.ok(!err);
-      assert.ok(data);
-      assert.strictEqual(data.changesComplete, true);
-      assert.strictEqual(typeof data.batches, 'object');
-      assert.strictEqual(Object.keys(data.batches).length, 2);
-      assert.deepStrictEqual(data.batches['1'], true);
-      assert.deepStrictEqual(data.batches['4'], true);
-      done();
-    });
-  });
-});
+module.exports = {
+  BackupError: BackupError,
+  HTTPError: HTTPError,
+  convertResponseError: convertResponseError,
+  terminationCallback: function terminationCallback(err, data) {
+    if (err) {
+      console.error(`ERROR: ${err.message}`);
+      process.exitCode = codes[err.name] || 1;
+      process.exit();
+    }
+  }
+};

package/.scannerwork/scanner-report/source-62.txt

@@ -1,4 +1,4 @@
-// Copyright © 2017 IBM Corp. All rights reserved.
+// Copyright © 2017, 2018 IBM Corp. All rights reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -11,109 +11,44 @@
 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 // See the License for the specific language governing permissions and
 // limitations under the License.
-
-/* global describe it */
 'use strict';
 
-const assert = require('assert');
-const fs = require('fs');
-const u = require('./citestutils.js');
-
-[{ useApi: true }, { useApi: false }].forEach(function(params) {
-  describe(u.scenario('Resume tests', params), function() {
-    it('should create a log file', function(done) {
-      // Allow up to 90 s for this test
-      u.setTimeout(this, 60);
-
-      const actualBackup = `./${this.fileName}`;
-      const logFile = `./${this.fileName}` + '.log';
-      // Use abort parameter to terminate the backup a given number of ms after
-      // the first data write to the output file.
-      const p = u.p(params, { opts: { log: logFile } });
-      u.testBackupToFile(p, 'animaldb', actualBackup, function(err) {
-        if (err) {
-          done(err);
-        } else {
-          // Assert the log file exists
-          try {
-            assert.ok(fs.existsSync(logFile), 'The log file should exist.');
-            done();
-          } catch (err) {
-            done(err);
-          }
-        }
-      });
-    });
-
-    it('should restore corrupted animaldb to a database correctly', function(done) {
-      // Allow up to 60 s to restore and compare (again it should be faster)!
-      u.setTimeout(this, 60);
-      const input = fs.createReadStream('./test/fixtures/animaldb_corrupted.json');
-      const dbName = this.dbName;
-      input.on('open', function() {
-        u.testRestore(params, input, dbName, function(err) {
-          if (err) {
-            done(err);
-          } else {
-            u.dbCompare('animaldb', dbName, done);
-          }
-        });
-      });
-    });
-
-    it('should restore resumed animaldb with blank line to a database correctly', function(done) {
-      // Allow up to 60 s to restore and compare (again it should be faster)!
-      u.setTimeout(this, 60);
-      const input = fs.createReadStream('./test/fixtures/animaldb_resumed_blank.json');
-      const dbName = this.dbName;
-      input.on('open', function() {
-        u.testRestore(params, input, dbName, function(err) {
-          if (err) {
-            done(err);
-          } else {
-            u.dbCompare('animaldb', dbName, done);
-          }
-        });
-      });
-    });
-  });
-});
-
-describe('Resume tests', function() {
-  // Currently cannot abort API backups, when we do this test should be run for
-  // both API and CLI
-  it('should correctly backup and restore backup10m', function(done) {
-    // Allow up to 90 s for this test
-    u.setTimeout(this, 90);
-
-    const actualBackup = `./${this.fileName}`;
-    const logFile = `./${this.fileName}` + '.log';
-    // Use abort parameter to terminate the backup a given number of ms after
-    // the first data write to the output file.
-    const p = u.p(params, { abort: true }, { opts: { log: logFile } });
-    const restoreDb = this.dbName;
-    // Set the database doc count as fewer than this should be written during
-    // resumed backup.
-    p.exclusiveMaxExpected = 5096;
-
-    u.testBackupAbortResumeRestore(p, 'backup10m', actualBackup, restoreDb, done);
-  });
-  // Note --output is only valid for CLI usage, this test should only run for CLI
-  const params = { useApi: false };
-  it('should correctly backup and restore backup10m using --output', function(done) {
-    // Allow up to 90 s for this test
-    u.setTimeout(this, 90);
-
-    const actualBackup = `./${this.fileName}`;
-    const logFile = `./${this.fileName}` + '.log';
-    // Use abort parameter to terminate the backup a given number of ms after
-    // the first data write to the output file.
-    const p = u.p(params, { abort: true }, { opts: { output: actualBackup, log: logFile } });
-    const restoreDb = this.dbName;
-    // Set the database doc count as fewer than this should be written during
-    // resumed backup.
-    p.exclusiveMaxExpected = 5096;
-
-    u.testBackupAbortResumeRestore(p, 'backup10m', actualBackup, restoreDb, done);
-  });
-});
+/**
+ * Utility methods for the command line interface.
+ * @module cliutils
+ * @see module:cliutils
+ */
+
+const url = require('url');
+
+module.exports = {
+
+  /**
+   * Combine a base URL and a database name, ensuring at least single slash
+   * between root and database name. This allows users to have Couch behind
+   * proxies that mount Couch's / endpoint at some other mount point.
+   * @param {string} root - root URL
+   * @param {string} databaseName - database name
+   * @return concatenated URL.
+   *
+   * @private
+   */
+  databaseUrl: function databaseUrl(root, databaseName) {
+    if (!root.endsWith('/')) {
+      root = root + '/';
+    }
+    return new url.URL(encodeURIComponent(databaseName), root).toString();
+  },
+
+  /**
+   * Generate CLI argument usage text.
+   *
+   * @param {string} description - argument description.
+   * @param {string} defaultValue - default argument value.
+   *
+   * @private
+   */
+  getUsage: function getUsage(description, defaultValue) {
+    return `${description} ${defaultValue !== undefined ? ` (default: ${defaultValue})` : ''}`;
+  }
+};