@cloudant/couchbackup 2.9.13 → 2.9.14-SNAPSHOT.147

package/.scannerwork/scanner-report/source-60.txt

@@ -1,4 +1,4 @@
-// Copyright © 2017 IBM Corp. All rights reserved.
+// Copyright © 2017, 2021 IBM Corp. All rights reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -13,63 +13,166 @@
 // limitations under the License.
 'use strict';
 
-const fs = require('fs');
+const pkg = require('../package.json');
 const stream = require('stream');
-const liner = require('./liner.js');
+const { CloudantV1, CouchdbSessionAuthenticator } = require('@ibm-cloud/cloudant');
+const { IamAuthenticator, NoAuthAuthenticator } = require('ibm-cloud-sdk-core');
+const retryPlugin = require('retry-axios');
 
-const onLine = function(onCommand, batches) {
-  const change = new stream.Transform({ objectMode: true });
-  change._transform = function(line, encoding, done) {
-    if (line && line[0] === ':') {
-      const obj = {
-        command: null,
-        batch: null,
-        docs: []
-      };
+const userAgent = 'couchbackup-cloudant/' + pkg.version + ' (Node.js ' +
+      process.version + ')';
 
-      let matches;
+// Class for streaming _changes error responses into
+// In general the response is a small error/reason JSON object
+// so it is OK to have this in memory.
+class ResponseWriteable extends stream.Writable {
+  constructor(options) {
+    super(options);
+    this.data = [];
+  }
 
-      // extract command
-      matches = line.match(/^:([a-z_]+) ?/);
-      if (matches) {
-        obj.command = matches[1];
-      }
+  _write(chunk, encoding, callback) {
+    this.data.push(chunk);
+    callback();
+  }
 
-      // extract batch
-      matches = line.match(/ batch([0-9]+)/);
-      if (matches) {
-        obj.batch = parseInt(matches[1]);
-      }
+  stringBody() {
+    return Buffer.concat(this.data).toString();
+  }
+}
 
-      // if this is one we want
-      if (obj.command === 't' && batches.indexOf(obj.batch) > -1) {
-        const json = line.replace(/^.* batch[0-9]+ /, '').trim();
-        obj.docs = JSON.parse(json);
-        onCommand(obj);
+// An interceptor function to help augment error bodies with a little
+// extra information so we can continue to use consistent messaging
+// after the ugprade to @ibm-cloud/cloudant
+const errorHelper = async function(err) {
+  let method;
+  let requestUrl;
+  if (err.response) {
+    if (err.response.config.url) {
+      requestUrl = err.response.config.url;
+      method = err.response.config.method;
+    }
+    // Override the status text with an improved message
+    let errorMsg = `${err.response.status} ${err.response.statusText || ''}: ` +
+    `${method} ${requestUrl}`;
+    if (err.response.data) {
+      // Check if we have a JSON response and try to get the error/reason
+      if (err.response.headers['content-type'] === 'application/json') {
+        if (!err.response.data.error && err.response.data.pipe) {
+          // If we didn't find a JSON object with `error` then we might have a stream response.
+          // Detect the stream by the presence of `pipe` and use it to get the body and parse
+          // the error information.
+          const p = new Promise((resolve, reject) => {
+            const errorBody = new ResponseWriteable();
+            err.response.data.pipe(errorBody)
+              .on('finish', () => { resolve(JSON.parse(errorBody.stringBody())); })
+              .on('error', () => { reject(err); });
+          });
+          // Replace the stream on the response with the parsed object
+          err.response.data = await p;
+        }
+        // Append the error/reason if available
+        if (err.response.data.error) {
+          // Override the status text with our more complete message
+          errorMsg += ` - Error: ${err.response.data.error}`;
+          if (err.response.data.reason) {
+            errorMsg += `, Reason: ${err.response.data.reason}`;
+          }
+        }
+      } else {
+        errorMsg += err.response.data;
       }
+      // Set a new message for use by the node-sdk-core
+      // We use the errors array because it gets processed
+      // ahead of all other service errors.
+      err.response.data.errors = [{ message: errorMsg }];
     }
-    done();
-  };
-  return change;
+  } else if (err.request) {
+    if (!err.message.includes(err.config.url)) {
+      // Augment the message with the URL and method
+      // but don't do it again if we already have the URL.
+      err.message = `${err.message}: ${err.config.method} ${err.config.url}`;
+    }
+  }
+  return Promise.reject(err);
 };
 
-module.exports = function(log, batches, callback) {
-  // our sense of state
-  const retval = { };
+module.exports = {
+  client: function(rawUrl, opts) {
+    const url = new URL(rawUrl);
+    // Split the URL to separate service from database
+    // Use origin as the "base" to remove auth elements
+    const actUrl = new URL(url.pathname.substring(0, url.pathname.lastIndexOf('/')), url.origin);
+    const dbName = url.pathname.substring(url.pathname.lastIndexOf('/') + 1);
+    let authenticator;
+    // Default to cookieauth unless an IAM key is provided
+    if (opts.iamApiKey) {
+      const iamAuthOpts = { apikey: opts.iamApiKey };
+      if (opts.iamTokenUrl) {
+        iamAuthOpts.url = opts.iamTokenUrl;
+      }
+      authenticator = new IamAuthenticator(iamAuthOpts);
+    } else if (url.username) {
+      authenticator = new CouchdbSessionAuthenticator({
+        username: decodeURIComponent(url.username),
+        password: decodeURIComponent(url.password)
+      });
+    } else {
+      authenticator = new NoAuthAuthenticator();
+    }
+    const serviceOpts = {
+      authenticator: authenticator,
+      timeout: opts.requestTimeout,
+      // Axios performance options
+      maxContentLength: -1
+    };
+
+    const service = new CloudantV1(serviceOpts);
+    // Configure retries
+    const maxRetries = 2; // for 3 total attempts
+    service.getHttpClient().defaults.raxConfig = {
+      // retries for status codes
+      retry: maxRetries,
+      // retries for non-response e.g. ETIMEDOUT
+      noResponseRetries: maxRetries,
+      backoffType: 'exponential',
+      httpMethodsToRetry: ['GET', 'HEAD', 'POST'],
+      statusCodesToRetry: [
+        [429, 429],
+        [500, 599]
+      ],
+      shouldRetry: err => {
+        const cfg = retryPlugin.getConfig(err);
+        // cap at max retries regardless of response/non-response type
+        if (cfg.currentRetryAttempt >= maxRetries) {
+          return false;
+        } else {
+          return retryPlugin.shouldRetryRequest(err);
+        }
+      },
+      instance: service.getHttpClient()
+    };
+    retryPlugin.attach(service.getHttpClient());
+
+    service.setServiceUrl(actUrl.toString());
+    if (authenticator instanceof CouchdbSessionAuthenticator) {
+      // Awkward workaround for known Couch issue with compression on _session requests
+      // It is not feasible to disable compression on all requests with the amount of
+      // data this lib needs to move, so override the property in the tokenManager instance.
+      authenticator.tokenManager.requestWrapperInstance.compressRequestData = false;
+    }
+    if (authenticator.tokenManager && authenticator.tokenManager.requestWrapperInstance) {
+      authenticator.tokenManager.requestWrapperInstance.axiosInstance.interceptors.response.use(null, errorHelper);
+    }
+    // Add error interceptors to put URLs in error messages
+    service.getHttpClient().interceptors.response.use(null, errorHelper);
 
-  // called with each line from the log file
-  const onCommand = function(obj) {
-    retval[obj.batch] = obj;
-  };
+    // Add request interceptor to add user-agent (adding it with custom request headers gets overwritten)
+    service.getHttpClient().interceptors.request.use(function(requestConfig) {
+      requestConfig.headers['User-Agent'] = userAgent;
+      return requestConfig;
+    }, null);
 
-  // stream through the previous log file
-  fs.createReadStream(log)
-    .pipe(liner())
-    .pipe(onLine(onCommand, batches))
-    .on('error', function(err) {
-      callback(err);
-    })
-    .on('finish', function() {
-      callback(null, retval);
-    });
+    return { service: service, db: dbName, url: actUrl.toString() };
+  }
 };

package/.scannerwork/scanner-report/source-61.txt

@@ -1,4 +1,4 @@
-// Copyright © 2017 IBM Corp. All rights reserved.
+// Copyright © 2017, 2018 IBM Corp. All rights reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -13,80 +13,42 @@
 // limitations under the License.
 'use strict';
 
-const fs = require('fs');
-const stream = require('stream');
-const liner = require('./liner.js');
-
-const onLine = function(onCommand, getDocs) {
-  const change = new stream.Transform({ objectMode: true });
-
-  change._transform = function(line, encoding, done) {
-    if (line && line[0] === ':') {
-      const obj = {
-        command: null,
-        batch: null,
-        docs: []
-      };
-
-      let matches;
-
-      // extract command
-      matches = line.match(/^:([a-z_]+) ?/);
-      if (matches) {
-        obj.command = matches[1];
-      }
-
-      // extract batch
-      matches = line.match(/ batch([0-9]+)/);
-      if (matches) {
-        obj.batch = parseInt(matches[1]);
-      }
-
-      // extract doc ids
-      if (getDocs && obj.command === 't') {
-        const json = line.replace(/^.* batch[0-9]+ /, '').trim();
-        obj.docs = JSON.parse(json);
-      }
-      onCommand(obj);
-    }
-    done();
-  };
-  return change;
-};
-
 /**
- * Generate a list of remaining batches from a download file.
- *
- * @param {string} log - log file name
- * @param {function} callback - callback with err, {changesComplete: N, batches: N}.
- *  changesComplete signifies whether the log file appeared to
- *  have completed reading the changes feed (contains :changes_complete).
- *  batches are remaining batch IDs for download.
+ * Utility methods for the command line interface.
+ * @module cliutils
+ * @see module:cliutils
  */
-module.exports = function(log, callback) {
-  // our sense of state
-  const state = {
 
-  };
-  let changesComplete = false;
-
-  // called with each line from the log file
-  const onCommand = function(obj) {
-    if (obj.command === 't') {
-      state[obj.batch] = true;
-    } else if (obj.command === 'd') {
-      delete state[obj.batch];
-    } else if (obj.command === 'changes_complete') {
-      changesComplete = true;
+const url = require('url');
+
+module.exports = {
+
+  /**
+   * Combine a base URL and a database name, ensuring at least single slash
+   * between root and database name. This allows users to have Couch behind
+   * proxies that mount Couch's / endpoint at some other mount point.
+   * @param {string} root - root URL
+   * @param {string} databaseName - database name
+   * @return concatenated URL.
+   *
+   * @private
+   */
+  databaseUrl: function databaseUrl(root, databaseName) {
+    if (!root.endsWith('/')) {
+      root = root + '/';
     }
-  };
-
-  // stream through the previous log file
-  fs.createReadStream(log)
-    .pipe(liner())
-    .pipe(onLine(onCommand, false))
-    .on('finish', function() {
-      const obj = { changesComplete: changesComplete, batches: state };
-      callback(null, obj);
-    });
+    return new url.URL(encodeURIComponent(databaseName), root).toString();
+  },
+
+  /**
+   * Generate CLI argument usage text.
+   *
+   * @param {string} description - argument description.
+   * @param {string} defaultValue - default argument value.
+   *
+   * @private
+   */
+  getUsage: function getUsage(description, defaultValue) {
+    return `${description} ${defaultValue !== undefined ? ` (default: ${defaultValue})` : ''}`;
+  }
 };

package/.scannerwork/scanner-report/source-62.txt

@@ -1,4 +1,4 @@
-// Copyright © 2017, 2018 IBM Corp. All rights reserved.
+// Copyright © 2017 IBM Corp. All rights reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -13,42 +13,29 @@
 // limitations under the License.
 'use strict';
 
-/**
- * Utility methods for the command line interface.
- * @module cliutils
- * @see module:cliutils
- */
+// stolen from http://strongloop.com/strongblog/practical-examples-of-the-new-node-js-streams-api/
+const stream = require('stream');
 
-const url = require('url');
+module.exports = function(onChange) {
+  const change = new stream.Transform({ objectMode: true });
 
-module.exports = {
+  change._transform = function(line, encoding, done) {
+    let obj = null;
 
-  /**
-   * Combine a base URL and a database name, ensuring at least single slash
-   * between root and database name. This allows users to have Couch behind
-   * proxies that mount Couch's / endpoint at some other mount point.
-   * @param {string} root - root URL
-   * @param {string} databaseName - database name
-   * @return concatenated URL.
-   *
-   * @private
-   */
-  databaseUrl: function databaseUrl(root, databaseName) {
-    if (!root.endsWith('/')) {
-      root = root + '/';
+    // one change per line - remove the trailing comma
+    line = line.trim().replace(/,$/, '');
+
+    // extract thee last_seq at the end of the changes feed
+    if (line.match(/^"last_seq":/)) {
+      line = '{' + line;
+    }
+    try {
+      obj = JSON.parse(line);
+    } catch (e) {
     }
-    return new url.URL(encodeURIComponent(databaseName), root).toString();
-  },
+    onChange(obj);
+    done();
+  };
 
-  /**
-   * Generate CLI argument usage text.
-   *
-   * @param {string} description - argument description.
-   * @param {string} defaultValue - default argument value.
-   *
-   * @private
-   */
-  getUsage: function getUsage(description, defaultValue) {
-    return `${description} ${defaultValue !== undefined ? ` (default: ${defaultValue})` : ''}`;
-  }
+  return change;
 };

package/.scannerwork/scanner-report/source-63.txt

@@ -1,4 +1,5 @@
-// Copyright © 2017, 2022 IBM Corp. All rights reserved.
+#!/usr/bin/env node
+// Copyright © 2017, 2021 IBM Corp. All rights reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -13,99 +14,43 @@
 // limitations under the License.
 'use strict';
 
-const fs = require('fs');
-const liner = require('./liner.js');
-const change = require('./change.js');
-const error = require('./error.js');
-const debug = require('debug')('couchbackup:spoolchanges');
+const error = require('../includes/error.js');
+const cliutils = require('../includes/cliutils.js');
+const couchbackup = require('../app.js');
+const parser = require('../includes/parser.js');
+const debug = require('debug');
+const restoreDebug = debug('couchbackup:restore');
+const restoreBatchDebug = debug('couchbackup:restore:batch');
 
-/**
- * Write log file for all changes from a database, ready for downloading
- * in batches.
- *
- * @param {string} dbUrl - URL of database
- * @param {string} log - path to log file to use
- * @param {number} bufferSize - the number of changes per batch/log line
- * @param {function(err)} callback - a callback to run on completion
- */
-module.exports = function(db, log, bufferSize, ee, callback) {
-  // list of document ids to process
-  const buffer = [];
-  let batch = 0;
-  let lastSeq = null;
-  const logStream = fs.createWriteStream(log);
-  let pending = 0;
-  // The number of changes to fetch per request
-  const limit = 100000;
+restoreDebug.enabled = true;
 
-  // send documents ids to the queue in batches of bufferSize + the last batch
-  const processBuffer = function(lastOne) {
-    if (buffer.length >= bufferSize || (lastOne && buffer.length > 0)) {
-      debug('writing', buffer.length, 'changes to the backup file');
-      const b = { docs: buffer.splice(0, bufferSize), batch: batch };
-      logStream.write(':t batch' + batch + ' ' + JSON.stringify(b.docs) + '\n');
-      ee.emit('changes', batch);
-      batch++;
-    }
-  };
+const program = parser.parseRestoreArgs();
+const databaseUrl = cliutils.databaseUrl(program.url, program.db);
+const opts = {
+  bufferSize: program.bufferSize,
+  parallelism: program.parallelism,
+  requestTimeout: program.requestTimeout,
+  iamApiKey: program.iamApiKey,
+  iamTokenUrl: program.iamTokenUrl
+};
 
-  // called once per received change
-  const onChange = function(c) {
-    if (c) {
-      if (c.error) {
-        ee.emit('error', new error.BackupError('InvalidChange', `Received invalid change: ${c}`));
-      } else if (c.changes) {
-        const obj = { id: c.id };
-        buffer.push(obj);
-        processBuffer(false);
-      } else if (c.last_seq) {
-        lastSeq = c.last_seq;
-        pending = c.pending;
-      }
-    }
-  };
+// log configuration to console
+console.error('='.repeat(80));
+console.error('Performing restore on ' + databaseUrl.replace(/\/\/.+@/g, '//****:****@') + ' using configuration:');
+console.error(JSON.stringify(opts, null, 2).replace(/"iamApiKey": "[^"]+"/, '"iamApiKey": "****"'));
+console.error('='.repeat(80));
 
-  function getChanges(since = 0) {
-    debug('making changes request since ' + since);
-    return db.service.postChangesAsStream({ db: db.db, since: since, limit: limit, seqInterval: limit })
-      .then(response => {
-        response.result.pipe(liner())
-          .on('error', function(err) {
-            logStream.end();
-            callback(err);
-          })
-          .pipe(change(onChange))
-          .on('error', function(err) {
-            logStream.end();
-            callback(err);
-          })
-          .on('finish', function() {
-            processBuffer(true);
-            if (!lastSeq) {
-              logStream.end();
-              debug('changes request terminated before last_seq was sent');
-              callback(new error.BackupError('SpoolChangesError', 'Changes request terminated before last_seq was sent'));
-            } else {
-              debug(`changes request completed with last_seq: ${lastSeq} and ${pending} changes pending.`);
-              if (pending > 0) {
-                // Return the next promise
-                return getChanges(lastSeq);
-              } else {
-                debug('finished streaming database changes');
-                logStream.end(':changes_complete ' + lastSeq + '\n', 'utf8', callback);
-              }
-            }
-          });
-      })
-      .catch(err => {
-        logStream.end();
-        if (err.status && err.status >= 400) {
-          callback(error.convertResponseError(err));
-        } else if (err.name !== 'SpoolChangesError') {
-          callback(new error.BackupError('SpoolChangesError', `Failed changes request - ${err.message}`));
-        }
-      });
-  }
+restoreBatchDebug.enabled = !program.quiet;
 
-  getChanges();
-};
+return couchbackup.restore(
+  process.stdin, // restore from stdin
+  databaseUrl,
+  opts,
+  error.terminationCallback
+).on('restored', function(obj) {
+  restoreBatchDebug('restored', obj.total);
+}).on('error', function(e) {
+  restoreDebug('ERROR', e);
+}).on('finished', function(obj) {
+  restoreDebug('finished', obj);
+});