@cloudant/couchbackup 2.9.10-SNAPSHOT.113 → 2.9.10

package/.scannerwork/scanner-report/source-20.txt

@@ -1,4 +1,4 @@
-// Copyright © 2017, 2022 IBM Corp. All rights reserved.
+// Copyright © 2017 IBM Corp. All rights reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -11,101 +11,25 @@
 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 // See the License for the specific language governing permissions and
 // limitations under the License.
-'use strict';
-
-const fs = require('fs');
-const liner = require('./liner.js');
-const change = require('./change.js');
-const error = require('./error.js');
-const debug = require('debug')('couchbackup:spoolchanges');
-
-/**
- * Write log file for all changes from a database, ready for downloading
- * in batches.
- *
- * @param {string} dbUrl - URL of database
- * @param {string} log - path to log file to use
- * @param {number} bufferSize - the number of changes per batch/log line
- * @param {function(err)} callback - a callback to run on completion
- */
-module.exports = function(db, log, bufferSize, ee, callback) {
-  // list of document ids to process
-  const buffer = [];
-  let batch = 0;
-  let lastSeq = null;
-  const logStream = fs.createWriteStream(log);
-  let pending = 0;
-  // The number of changes to fetch per request
-  const limit = 100000;
 
-  // send documents ids to the queue in batches of bufferSize + the last batch
-  const processBuffer = function(lastOne) {
-    if (buffer.length >= bufferSize || (lastOne && buffer.length > 0)) {
-      debug('writing', buffer.length, 'changes to the backup file');
-      const b = { docs: buffer.splice(0, bufferSize), batch: batch };
-      logStream.write(':t batch' + batch + ' ' + JSON.stringify(b.docs) + '\n');
-      ee.emit('changes', batch);
-      batch++;
-    }
-  };
-
-  // called once per received change
-  const onChange = function(c) {
-    if (c) {
-      if (c.error) {
-        ee.emit('error', new error.BackupError('InvalidChange', `Received invalid change: ${c}`));
-      } else if (c.changes) {
-        const obj = { id: c.id };
-        buffer.push(obj);
-        processBuffer(false);
-      } else if (c.last_seq) {
-        lastSeq = c.last_seq;
-        pending = c.pending;
-      }
-    }
-  };
+/* global describe it */
+'use strict';
 
-  function getChanges(since = 0) {
-    debug('making changes request since ' + since);
-    return db.service.postChangesAsStream({ db: db.db, since: since, limit: limit, seqInterval: limit })
-      .then(response => {
-        response.result.pipe(liner())
-          .on('error', function(err) {
-            logStream.end();
-            callback(err);
-          })
-          .pipe(change(onChange))
-          .on('error', function(err) {
-            logStream.end();
-            callback(err);
-          })
-          .on('finish', function() {
-            processBuffer(true);
-            if (!lastSeq) {
-              logStream.end();
-              debug('changes request terminated before last_seq was sent');
-              callback(new error.BackupError('SpoolChangesError', 'Changes request terminated before last_seq was sent'));
-            } else {
-              debug(`changes request completed with last_seq: ${lastSeq} and ${pending} changes pending.`);
-              if (pending > 0) {
-                // Return the next promise
-                return getChanges(lastSeq);
-              } else {
-                debug('finished streaming database changes');
-                logStream.end(':changes_complete ' + lastSeq + '\n', 'utf8', callback);
-              }
-            }
-          });
-      })
-      .catch(err => {
-        logStream.end();
-        if (err.status && err.status >= 400) {
-          callback(error.convertResponseError(err));
-        } else if (err.name !== 'SpoolChangesError') {
-          callback(new error.BackupError('SpoolChangesError', `Failed changes request - ${err.message}`));
-        }
-      });
-  }
+const assert = require('assert');
+const logfilegetbatches = require('../includes/logfilegetbatches.js');
 
-  getChanges();
-};
+describe('#unit Fetching batches from a log file', function() {
+  it('should fetch multiple batches correctly', function(done) {
+    logfilegetbatches('./test/fixtures/test.log', [1, 4], function(err, data) {
+      assert.ok(!err);
+      assert.ok(data);
+      assert.strictEqual(typeof data, 'object');
+      assert.strictEqual(Object.keys(data).length, 2);
+      assert.deepStrictEqual(data['1'].docs, [{ id: '6' }, { id: '7' }, { id: '8' }, { id: '9' }, { id: '10' }]);
+      assert.strictEqual(data['1'].batch, 1);
+      assert.deepStrictEqual(data['4'].docs, [{ id: '21' }, { id: '22' }]);
+      assert.strictEqual(data['4'].batch, 4);
+      done();
+    });
+  });
+});

package/.scannerwork/scanner-report/source-21.txt

@@ -11,168 +11,234 @@
 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 // See the License for the specific language governing permissions and
 // limitations under the License.
+
+/* global describe it beforeEach */
 'use strict';
 
-const pkg = require('../package.json');
-const stream = require('stream');
-const { CloudantV1, CouchdbSessionAuthenticator } = require('@ibm-cloud/cloudant');
-const { IamAuthenticator, NoAuthAuthenticator } = require('ibm-cloud-sdk-core');
-const retryPlugin = require('retry-axios');
-
-const userAgent = 'couchbackup-cloudant/' + pkg.version + ' (Node.js ' +
-      process.version + ')';
-
-// Class for streaming _changes error responses into
-// In general the response is a small error/reason JSON object
-// so it is OK to have this in memory.
-class ResponseWriteable extends stream.Writable {
-  constructor(options) {
-    super(options);
-    this.data = [];
-  }
-
-  _write(chunk, encoding, callback) {
-    this.data.push(chunk);
-    callback();
-  }
-
-  stringBody() {
-    return Buffer.concat(this.data).toString();
-  }
-}
-
-// An interceptor function to help augment error bodies with a little
-// extra information so we can continue to use consistent messaging
-// after the ugprade to @ibm-cloud/cloudant
-const errorHelper = async function(err) {
-  let method;
-  let requestUrl;
-  if (err.response) {
-    if (err.response.config.url) {
-      requestUrl = err.response.config.url;
-      method = err.response.config.method;
-    }
-    // Override the status text with an improved message
-    let errorMsg = `${err.response.status} ${err.response.statusText || ''}: ` +
-    `${method} ${requestUrl}`;
-    if (err.response.data) {
-      // Check if we have a JSON response and try to get the error/reason
-      if (err.response.headers['content-type'] === 'application/json') {
-        if (!err.response.data.error && err.response.data.pipe) {
-          // If we didn't find a JSON object with `error` then we might have a stream response.
-          // Detect the stream by the presence of `pipe` and use it to get the body and parse
-          // the error information.
-          const p = new Promise((resolve, reject) => {
-            const errorBody = new ResponseWriteable();
-            err.response.data.pipe(errorBody)
-              .on('finish', () => { resolve(JSON.parse(errorBody.stringBody())); })
-              .on('error', () => { reject(err); });
-          });
-          // Replace the stream on the response with the parsed object
-          err.response.data = await p;
-        }
-        // Append the error/reason if available
-        if (err.response.data.error) {
-          // Override the status text with our more complete message
-          errorMsg += ` - Error: ${err.response.data.error}`;
-          if (err.response.data.reason) {
-            errorMsg += `, Reason: ${err.response.data.reason}`;
-          }
-        }
-      } else {
-        errorMsg += err.response.data;
-      }
-      // Set a new message for use by the node-sdk-core
-      // We use the errors array because it gets processed
-      // ahead of all other service errors.
-      err.response.data.errors = [{ message: errorMsg }];
-    }
-  } else if (err.request) {
-    if (!err.message.includes(err.config.url)) {
-      // Augment the message with the URL and method
-      // but don't do it again if we already have the URL.
-      err.message = `${err.message}: ${err.config.method} ${err.config.url}`;
-    }
-  }
-  return Promise.reject(err);
-};
-
-module.exports = {
-  client: function(rawUrl, opts) {
-    const url = new URL(rawUrl);
-    // Split the URL to separate service from database
-    // Use origin as the "base" to remove auth elements
-    const actUrl = new URL(url.pathname.substring(0, url.pathname.lastIndexOf('/')), url.origin);
-    const dbName = url.pathname.substring(url.pathname.lastIndexOf('/') + 1);
-    let authenticator;
-    // Default to cookieauth unless an IAM key is provided
-    if (opts.iamApiKey) {
-      const iamAuthOpts = { apikey: opts.iamApiKey };
-      if (opts.iamTokenUrl) {
-        iamAuthOpts.url = opts.iamTokenUrl;
-      }
-      authenticator = new IamAuthenticator(iamAuthOpts);
-    } else if (url.username) {
-      authenticator = new CouchdbSessionAuthenticator({
-        username: decodeURIComponent(url.username),
-        password: decodeURIComponent(url.password)
+const assert = require('assert');
+const nock = require('nock');
+const request = require('../includes/request.js');
+const error = require('../includes/error.js');
+
+const url = 'http://localhost:7777/testdb';
+const db = request.client(url, { parallelism: 1 });
+const timeoutDb = request.client(url, { parallelism: 1, requestTimeout: 500 });
+const longTestTimeout = 3000;
+
+beforeEach('Clean nock', function() {
+  nock.cleanAll();
+});
+
+describe('#unit Check request headers', function() {
+  it('should have a couchbackup user-agent', function(done) {
+    const couch = nock(url)
+      .matchHeader('user-agent', /couchbackup-cloudant\/\d+\.\d+\.\d+(?:-SNAPSHOT)? \(Node.js v\d+\.\d+\.\d+\)/)
+      .head('/good')
+      .reply(200);
+
+    db.service.headDocument({ db: db.db, docId: 'good' }).then(response => {
+      assert.ok(couch.isDone());
+      done();
+    }).catch(err => {
+      done(err);
+    });
+  });
+});
+
+describe('#unit Check request response error callback', function() {
+  it('should not callback with error for 200 response', function(done) {
+    const couch = nock(url)
+      .get('/good')
+      .reply(200, { ok: true });
+
+    db.service.getDocument({ db: db.db, docId: 'good' }).then(response => {
+      assert.ok(response.result);
+      assert.ok(couch.isDone());
+      done();
+    }).catch(err => {
+      err = error.convertResponseError(err);
+      done(err);
+    });
+  });
+
+  it('should callback with error after 3 500 responses', function(done) {
+    const couch = nock(url)
+      .get('/bad')
+      .times(3)
+      .reply(500, function(uri, requestBody) {
+        this.req.response.statusMessage = 'Internal Server Error';
+        return { error: 'foo', reason: 'bar' };
+      });
+
+    db.service.getDocument({ db: db.db, docId: 'bad' }).then(response => {
+      done(new Error('Successful response when error expected.'));
+    }).catch(err => {
+      err = error.convertResponseError(err);
+      assert.strictEqual(err.name, 'HTTPFatalError');
+      assert.strictEqual(err.message, `500 Internal Server Error: get ${url}/bad - Error: foo, Reason: bar`);
+      assert.ok(couch.isDone());
+      done();
+    }).catch(err => {
+      // Handle assertion errors
+      done(err);
+    });
+  }).timeout(longTestTimeout);
+
+  it('should callback with error after 3 POST 503 responses', function(done) {
+    const couch = nock(url)
+      .post('/_bulk_get')
+      .query(true)
+      .times(3)
+      .reply(503, function(uri, requestBody) {
+        this.req.response.statusMessage = 'Service Unavailable';
+        return { error: 'service_unavailable', reason: 'Service unavailable' };
+      });
+
+    db.service.postBulkGet({ db: db.db, revs: true, docs: [] }).then(response => {
+      done(new Error('Successful response when error expected.'));
+    }).catch(err => {
+      err = error.convertResponseError(err);
+      assert.strictEqual(err.name, 'HTTPFatalError');
+      assert.strictEqual(err.message, `503 Service Unavailable: post ${url}/_bulk_get - Error: service_unavailable, Reason: Service unavailable`);
+      assert.ok(couch.isDone());
+      done();
+    }).catch(err => {
+      // Handle assertion errors
+      done(err);
+    });
+  }).timeout(longTestTimeout);
+
+  it('should callback with error after 3 429 responses', function(done) {
+    const couch = nock(url)
+      .get('/bad')
+      .times(3)
+      .reply(429, function(uri, requestBody) {
+        this.req.response.statusMessage = 'Too Many Requests';
+        return { error: 'foo', reason: 'bar' };
+      });
+
+    db.service.getDocument({ db: db.db, docId: 'bad' }).then(response => {
+      done(new Error('Successful response when error expected.'));
+    }).catch(err => {
+      err = error.convertResponseError(err);
+      assert.strictEqual(err.name, 'HTTPFatalError');
+      assert.strictEqual(err.message, `429 Too Many Requests: get ${url}/bad - Error: foo, Reason: bar`);
+      assert.ok(couch.isDone());
+      done();
+    }).catch(err => {
+      // Handle assertion errors
+      done(err);
+    });
+  }).timeout(longTestTimeout);
+
+  it('should callback with fatal error for 404 response', function(done) {
+    const couch = nock(url)
+      .get('/bad')
+      .reply(404, function(uri, requestBody) {
+        this.req.response.statusMessage = 'Not Found';
+        return { error: 'foo', reason: 'bar' };
+      });
+
+    db.service.getDocument({ db: db.db, docId: 'bad' }).then(response => {
+      done(new Error('Successful response when error expected.'));
+    }).catch(err => {
+      err = error.convertResponseError(err);
+      assert.strictEqual(err.name, 'HTTPFatalError');
+      assert.strictEqual(err.message, `404 Not Found: get ${url}/bad - Error: foo, Reason: bar`);
+      assert.ok(couch.isDone());
+      done();
+    }).catch(err => {
+      // Handle assertion errors
+      done(err);
+    });
+  });
+
+  it('should callback with same error for no status code error response', function(done) {
+    const couch = nock(url)
+      .get('/bad')
+      .times(3)
+      .replyWithError('testing badness');
+
+    db.service.getDocument({ db: db.db, docId: 'bad' }).then(response => {
+      done(new Error('Successful response when error expected.'));
+    }).catch(err => {
+      const err2 = error.convertResponseError(err);
+      assert.strictEqual(err, err2);
+      assert.ok(couch.isDone());
+      done();
+    }).catch(err => {
+      // Handle assertion errors
+      done(err);
+    });
+  }).timeout(longTestTimeout);
+
+  it('should retry request if HTTP request gets timed out', function(done) {
+    const couch = nock(url)
+      .post('/_bulk_get')
+      .query(true)
+      .delay(1000)
+      .reply(200, { results: { docs: [{ id: '1', ok: { _id: '1' } }] } })
+      .post('/_bulk_get')
+      .query(true)
+      .reply(200, { results: { docs: [{ id: '1', ok: { _id: '1' } }, { id: '2', ok: { _id: '2' } }] } });
+
+    timeoutDb.service.postBulkGet({ db: db.db, revs: true, docs: [] }).then(response => {
+      assert.ok(response);
+      assert.ok(response.result);
+      assert.ok(response.result.results);
+      assert.ok(response.result.results.docs);
+      assert.strictEqual(response.result.results.docs.length, 2);
+      assert.ok(couch.isDone());
+      done();
+    }).catch(err => {
+      done(err);
+    });
+  });
+
+  it('should callback with error code ESOCKETTIMEDOUT if 3 HTTP requests gets timed out', function(done) {
+    // Increase the timeout for this test to allow for the delays
+    this.timeout(3000);
+    const couch = nock(url)
+      .post('/_bulk_get')
+      .query(true)
+      .delay(1000)
+      .times(3)
+      .reply(200, { ok: true });
+
+    timeoutDb.service.postBulkGet({ db: db.db, revs: true, docs: [] }).then(response => {
+      done(new Error('Successful response when error expected.'));
+    }).catch(err => {
+      err = error.convertResponseError(err);
+      // Note axios returns ECONNABORTED rather than ESOCKETTIMEDOUT
+      // See https://github.com/axios/axios/issues/2710 via https://github.com/axios/axios/issues/1543`
+      assert.strictEqual(err.statusText, 'ECONNABORTED');
+      assert.strictEqual(err.message, `timeout of 500ms exceeded: post ${url}/_bulk_get ECONNABORTED`);
+      assert.ok(couch.isDone());
+      done();
+    }).catch(err => {
+      // Handle assertion errors
+      done(err);
+    });
+  });
+  describe('#unit Check credentials', function() {
+    it('should properly decode username and password', function(done) {
+      const username = 'user%123';
+      const password = 'colon:at@321';
+      const url = `http://${encodeURIComponent(username)}:${encodeURIComponent(password)}@localhost:7777/testdb`;
+      const sessionUrl = 'http://localhost:7777';
+      const couch = nock(sessionUrl)
+        .post('/_session', { username: username, password: password })
+        .reply(200, { ok: true }, { 'Set-Cookie': 'AuthSession=ABC123DEF4356;' })
+        .get('/')
+        .reply(200);
+      const db = request.client(url, { parallelism: 1 });
+      db.service.getServerInformation().then(response => {
+        assert.ok(response);
+        assert.ok(couch.isDone());
+        done();
+      }).catch(err => {
+        done(err);
       });
-    } else {
-      authenticator = new NoAuthAuthenticator();
-    }
-    const serviceOpts = {
-      authenticator: authenticator,
-      timeout: opts.requestTimeout,
-      // Axios performance options
-      maxContentLength: -1
-    };
-
-    const service = new CloudantV1(serviceOpts);
-    // Configure retries
-    const maxRetries = 2; // for 3 total attempts
-    service.getHttpClient().defaults.raxConfig = {
-      // retries for status codes
-      retry: maxRetries,
-      // retries for non-response e.g. ETIMEDOUT
-      noResponseRetries: maxRetries,
-      backoffType: 'exponential',
-      httpMethodsToRetry: ['GET', 'HEAD', 'POST'],
-      statusCodesToRetry: [
-        [429, 429],
-        [500, 599]
-      ],
-      shouldRetry: err => {
-        const cfg = retryPlugin.getConfig(err);
-        // cap at max retries regardless of response/non-response type
-        if (cfg.currentRetryAttempt >= maxRetries) {
-          return false;
-        } else {
-          return retryPlugin.shouldRetryRequest(err);
-        }
-      },
-      instance: service.getHttpClient()
-    };
-    retryPlugin.attach(service.getHttpClient());
-
-    service.setServiceUrl(actUrl.toString());
-    if (authenticator instanceof CouchdbSessionAuthenticator) {
-      // Awkward workaround for known Couch issue with compression on _session requests
-      // It is not feasible to disable compression on all requests with the amount of
-      // data this lib needs to move, so override the property in the tokenManager instance.
-      authenticator.tokenManager.requestWrapperInstance.compressRequestData = false;
-    }
-    if (authenticator.tokenManager && authenticator.tokenManager.requestWrapperInstance) {
-      authenticator.tokenManager.requestWrapperInstance.axiosInstance.interceptors.response.use(null, errorHelper);
-    }
-    // Add error interceptors to put URLs in error messages
-    service.getHttpClient().interceptors.response.use(null, errorHelper);
-
-    // Add request interceptor to add user-agent (adding it with custom request headers gets overwritten)
-    service.getHttpClient().interceptors.request.use(function(requestConfig) {
-      requestConfig.headers['User-Agent'] = userAgent;
-      return requestConfig;
-    }, null);
-
-    return { service: service, db: dbName, url: actUrl.toString() };
-  }
-};
+    });
+  });
+});