@cloudant/couchbackup 2.9.8-SNAPSHOT.94 → 2.9.8

package/.scannerwork/scanner-report/source-65.txt

@@ -11,21 +11,180 @@
 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 // See the License for the specific language governing permissions and
 // limitations under the License.
+
+// Small script which backs up a Cloudant or CouchDB database to an S3
+// bucket, using an intermediary file on disk.
+//
+// The script generates the backup object name by combining together the path
+// part of the database URL and the current time.
+
 'use strict';
 
-module.exports = function(db, options, readstream, ee, callback) {
-  const liner = require('../includes/liner.js')();
-  const writer = require('../includes/writer.js')(db, options.bufferSize, options.parallelism, ee);
-
-  // pipe the input to the output, via transformation functions
-  readstream
-    .pipe(liner) // transform the input stream into per-line
-    .on('error', function(err) {
-      // Forward the error to the writer event emitter where we already have
-      // listeners on for handling errors
-      writer.emit('error', err);
+const stream = require('stream');
+const fs = require('fs');
+const url = require('url');
+
+const AWS = require('aws-sdk');
+const couchbackup = require('@cloudant/couchbackup');
+const debug = require('debug')('s3-backup');
+const tmp = require('tmp');
+const VError = require('verror').VError;
+
+/*
+  Main function, run from base of file.
+*/
+function main() {
+  const argv = require('yargs')
+    .usage('Usage: $0 [options]')
+    .example('$0 -s https://user:pass@host/db -b <bucket>', 'Backup db to bucket')
+    .options({
+      source: { alias: 's', nargs: 1, demandOption: true, describe: 'Source database URL' },
+      bucket: { alias: 'b', nargs: 1, demandOption: true, describe: 'Destination bucket' },
+      prefix: { alias: 'p', nargs: 1, describe: 'Prefix for backup object key', default: 'couchbackup' },
+      s3url: { nargs: 1, describe: 'S3 endpoint URL' },
+      awsprofile: { nargs: 1, describe: 'The profile section to use in the ~/.aws/credentials file', default: 'default' }
+    })
+    .help('h').alias('h', 'help')
+    .epilog('Copyright (C) IBM 2017')
+    .argv;
+
+  const sourceUrl = argv.source;
+  const backupBucket = argv.bucket;
+  const backupName = new url.URL(sourceUrl).pathname.split('/').filter(function(x) { return x; }).join('-');
+  const backupKeyPrefix = `${argv.prefix}-${backupName}`;
+
+  const backupKey = `${backupKeyPrefix}-${new Date().toISOString()}`;
+  const backupTmpFile = tmp.fileSync();
+
+  const s3Endpoint = argv.s3url;
+  const awsProfile = argv.awsprofile;
+
+  // Creds are from ~/.aws/credentials, environment etc. (see S3 docs).
+  const awsOpts = {
+    signatureVersion: 'v4',
+    credentials: new AWS.SharedIniFileCredentials({ profile: awsProfile })
+  };
+  if (typeof s3Endpoint !== 'undefined') {
+    awsOpts.endpoint = new AWS.Endpoint(s3Endpoint);
+  }
+  const s3 = new AWS.S3(awsOpts);
+
+  debug(`Creating a new backup of ${s(sourceUrl)} at ${backupBucket}/${backupKey}...`);
+  bucketAccessible(s3, backupBucket)
+    .then(() => {
+      return createBackupFile(sourceUrl, backupTmpFile.name);
     })
-    .pipe(writer); // transform the data
+    .then(() => {
+      return uploadNewBackup(s3, backupTmpFile.name, backupBucket, backupKey);
+    })
+    .then(() => {
+      debug('Backup successful!');
+      backupTmpFile.removeCallback();
+      debug('done.');
+    })
+    .catch((reason) => {
+      debug(`Error: ${reason}`);
+    });
+}
+
+/**
+ * Return a promise that resolves if the bucket is available and
+ * rejects if not.
+ *
+ * @param {any} s3 S3 client object
+ * @param {any} bucketName Bucket name
+ * @returns Promise
+ */
+function bucketAccessible(s3, bucketName) {
+  return new Promise(function(resolve, reject) {
+    const params = {
+      Bucket: bucketName
+    };
+    s3.headBucket(params, function(err, data) {
+      if (err) {
+        reject(new VError(err, 'S3 bucket not accessible'));
+      } else {
+        resolve();
+      }
+    });
+  });
+}
+
+/**
+ * Use couchbackup to create a backup of the specified database to a file path.
+ *
+ * @param {any} sourceUrl Database URL
+ * @param {any} backupTmpFilePath Path to write file
+ * @returns Promise
+ */
+function createBackupFile(sourceUrl, backupTmpFilePath) {
+  return new Promise((resolve, reject) => {
+    couchbackup.backup(
+      sourceUrl,
+      fs.createWriteStream(backupTmpFilePath),
+      (err) => {
+        if (err) {
+          return reject(new VError(err, 'CouchBackup process failed'));
+        }
+        debug('couchbackup to file done; uploading to S3');
+        resolve('creating backup file complete');
+      }
+    );
+  });
+}
+
+/**
+ * Upload a backup file to an S3 bucket.
+ *
+ * @param {any} s3 Object store client
+ * @param {any} backupTmpFilePath Path of backup file to write.
+ * @param {any} bucket Object store bucket name
+ * @param {any} key Object store key name
+ * @returns Promise
+ */
+function uploadNewBackup(s3, backupTmpFilePath, bucket, key) {
+  return new Promise((resolve, reject) => {
+    debug(`Uploading from ${backupTmpFilePath} to ${bucket}/${key}`);
+
+    function uploadFromStream(s3, bucket, key) {
+      const pass = new stream.PassThrough();
+
+      const params = {
+        Bucket: bucket,
+        Key: key,
+        Body: pass
+      };
+      s3.upload(params, function(err, data) {
+        debug('S3 upload done');
+        if (err) {
+          debug(err);
+          reject(new VError(err, 'Upload failed'));
+          return;
+        }
+        debug('Upload succeeded');
+        debug(data);
+        resolve();
+      }).httpUploadProgress = (progress) => {
+        debug(`S3 upload progress: ${progress}`);
+      };
+
+      return pass;
+    }
+
+    const inputStream = fs.createReadStream(backupTmpFilePath);
+    const s3Stream = uploadFromStream(s3, bucket, key);
+    inputStream.pipe(s3Stream);
+  });
+}
+
+/**
+ * Remove creds from a URL, e.g., before logging
+ *
+ * @param {string} url URL to safen
+ */
+function s(originalUrl) {
+  const parts = new url.URL(originalUrl);
+  return url.format(parts, { auth: false });
+}
 
-  callback(null, writer);
-};
+main();

package/.scannerwork/scanner-report/source-66.txt

@@ -1,4 +1,4 @@
-// Copyright © 2017, 2021 IBM Corp. All rights reserved.
+// Copyright © 2017, 2018 IBM Corp. All rights reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -11,271 +11,165 @@
 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 // See the License for the specific language governing permissions and
 // limitations under the License.
-'use strict';
 
-const async = require('async');
-const events = require('events');
-const fs = require('fs');
-const error = require('./error.js');
-const spoolchanges = require('./spoolchanges.js');
-const logfilesummary = require('./logfilesummary.js');
-const logfilegetbatches = require('./logfilegetbatches.js');
+// Small script which backs up a Cloudant or CouchDB database to an S3
+// bucket via a stream rather than on-disk file.
+//
+// The script generates the backup object name by combining together the path
+// part of the database URL and the current time.
 
-/**
- * Read documents from a database to be backed up.
- *
- * @param {string} db - `@cloudant/cloudant` DB object for source database.
- * @param {number} blocksize - number of documents to download in single request
- * @param {number} parallelism - number of concurrent downloads
- * @param {string} log - path to log file to use
- * @param {boolean} resume - whether to resume from an existing log file
- * @returns EventEmitter with following events:
- *  - `received` - called with a block of documents to write to backup
- *  - `error` - on error
- *  - `finished` - when backup process is finished (either complete or errored)
- */
-module.exports = function(db, options) {
-  const ee = new events.EventEmitter();
-  const start = new Date().getTime(); // backup start time
-  const batchesPerDownloadSession = 50; // max batches to read from log file for download at a time (prevent OOM)
+'use strict';
 
-  function proceedWithBackup() {
-    if (options.resume) {
-      // pick up from existing log file from previous run
-      downloadRemainingBatches(options.log, db, ee, start, batchesPerDownloadSession, options.parallelism);
-    } else {
-      // create new log file and process
-      spoolchanges(db, options.log, options.bufferSize, ee, function(err) {
-        if (err) {
-          ee.emit('error', err);
-        } else {
-          downloadRemainingBatches(options.log, db, ee, start, batchesPerDownloadSession, options.parallelism);
-        }
-      });
-    }
+const stream = require('stream');
+const url = require('url');
+
+const AWS = require('aws-sdk');
+const couchbackup = require('@cloudant/couchbackup');
+const debug = require('debug')('s3-backup');
+const VError = require('verror').VError;
+
+/*
+  Main function, run from base of file.
+*/
+function main() {
+  const argv = require('yargs')
+    .usage('Usage: $0 [options]')
+    .example('$0 -s https://user:pass@host/db -b <bucket>', 'Backup db to bucket')
+    .options({
+      source: { alias: 's', nargs: 1, demandOption: true, describe: 'Source database URL' },
+      bucket: { alias: 'b', nargs: 1, demandOption: true, describe: 'Destination bucket' },
+      prefix: { alias: 'p', nargs: 1, describe: 'Prefix for backup object key', default: 'couchbackup' },
+      s3url: { nargs: 1, describe: 'S3 endpoint URL' },
+      awsprofile: { nargs: 1, describe: 'The profile section to use in the ~/.aws/credentials file', default: 'default' }
+    })
+    .help('h').alias('h', 'help')
+    .epilog('Copyright (C) IBM 2017')
+    .argv;
+
+  const sourceUrl = argv.source;
+  const backupName = new url.URL(sourceUrl).pathname.split('/').filter(function(x) { return x; }).join('-');
+  const backupBucket = argv.bucket;
+  const backupKeyPrefix = `${argv.prefix}-${backupName}`;
+  const shallow = argv.shallow;
+
+  const backupKey = `${backupKeyPrefix}-${new Date().toISOString()}`;
+
+  const s3Endpoint = argv.s3url;
+  const awsProfile = argv.awsprofile;
+
+  // Creds are from ~/.aws/credentials, environment etc. (see S3 docs).
+  const awsOpts = {
+    signatureVersion: 'v4',
+    credentials: new AWS.SharedIniFileCredentials({ profile: awsProfile })
+  };
+  if (typeof s3Endpoint !== 'undefined') {
+    awsOpts.endpoint = new AWS.Endpoint(s3Endpoint);
   }
-
-  validateBulkGetSupport(db, function(err) {
-    if (err) {
-      return ee.emit('error', err);
-    } else {
-      proceedWithBackup();
-    }
-  });
-
-  return ee;
-};
-
-/**
- * Validate /_bulk_get support for a specified database.
- *
- * @param {string} db - nodejs-cloudant db
- * @param {function} callback - called on completion with signature (err)
- */
-function validateBulkGetSupport(db, callback) {
-  db.service.postBulkGet({ db: db.db, docs: [] }).then(() => { callback(); }).catch(err => {
-    err = error.convertResponseError(err, function(err) {
-      switch (err.status) {
-        case undefined:
-          // There was no status code on the error
-          return err;
-        case 404:
-          return new error.BackupError('BulkGetError', 'Database does not support /_bulk_get endpoint');
-        default:
-          return new error.HTTPError(err);
-      }
+  const s3 = new AWS.S3(awsOpts);
+
+  debug(`Creating a new backup of ${s(sourceUrl)} at ${backupBucket}/${backupKey}...`);
+  bucketAccessible(s3, backupBucket)
+    .then(() => {
+      return backupToS3(sourceUrl, s3, backupBucket, backupKey, shallow);
+    })
+    .then(() => {
+      debug('done.');
+    })
+    .catch((reason) => {
+      debug(`Error: ${reason}`);
+      process.exit(1);
     });
-    callback(err);
-  });
 }
 
 /**
- * Download remaining batches in a log file, splitting batches into sets
- * to avoid enqueueing too many in one go.
+ * Return a promise that resolves if the bucket is available and
+ * rejects if not.
  *
- * @param {string} log - log file name to maintain download state
- * @param {string} db - nodejs-cloudant db
- * @param {events.EventEmitter} ee - event emitter to emit received events on
- * @param {time} startTime - start time for backup process
- * @param {number} batchesPerDownloadSession - max batches to enqueue for
- *  download at a time. As batches contain many doc IDs, this helps avoid
- *  exhausting memory.
- * @param {number} parallelism - number of concurrent downloads
- * @returns function to call do download remaining batches with signature
- *  (err, {batches: batch, docs: doccount}) {@see spoolchanges}.
+ * @param {any} s3 S3 client object
+ * @param {any} bucketName Bucket name
+ * @returns Promise
  */
-function downloadRemainingBatches(log, db, ee, startTime, batchesPerDownloadSession, parallelism) {
-  let total = 0; // running total of documents downloaded so far
-  let noRemainingBatches = false;
-
-  // Generate a set of batches (up to batchesPerDownloadSession) to download from the
-  // log file and download them. Set noRemainingBatches to `true` for last batch.
-  function downloadSingleBatchSet(done) {
-    // Fetch the doc IDs for the batches in the current set to
-    // download them.
-    function batchSetComplete(err, data) {
-      if (!err) {
-        total = data.total;
-      }
-      done(err);
-    }
-    function processRetrievedBatches(err, batches) {
-      if (!err) {
-        // process them in parallelised queue
-        processBatchSet(db, parallelism, log, batches, ee, startTime, total, batchSetComplete);
-      } else {
-        batchSetComplete(err);
-      }
-    }
-
-    readBatchSetIdsFromLogFile(log, batchesPerDownloadSession, function(err, batchSetIds) {
+function bucketAccessible(s3, bucketName) {
+  return new Promise(function(resolve, reject) {
+    const params = {
+      Bucket: bucketName
+    };
+    s3.headBucket(params, function(err, data) {
       if (err) {
-        ee.emit('error', err);
-        // Stop processing changes file for fatal errors
-        noRemainingBatches = true;
-        done();
+        reject(new VError(err, 'S3 bucket not accessible'));
       } else {
-        if (batchSetIds.length === 0) {
-          noRemainingBatches = true;
-          return done();
-        }
-        logfilegetbatches(log, batchSetIds, processRetrievedBatches);
+        resolve();
       }
     });
-  }
-
-  // Return true if all batches in log file have been downloaded
-  function isFinished(callback) { callback(null, noRemainingBatches); }
-
-  function onComplete() {
-    ee.emit('finished', { total: total });
-  }
-
-  async.doUntil(downloadSingleBatchSet, isFinished, onComplete);
-}
-
-/**
- * Return a set of uncompleted download batch IDs from the log file.
- *
- * @param {string} log - log file path
- * @param {number} batchesPerDownloadSession - maximum IDs to return
- * @param {function} callback - sign (err, batchSetIds array)
- */
-function readBatchSetIdsFromLogFile(log, batchesPerDownloadSession, callback) {
-  logfilesummary(log, function processSummary(err, summary) {
-    if (!err) {
-      if (!summary.changesComplete) {
-        callback(new error.BackupError('IncompleteChangesInLogFile',
-          'WARNING: Changes did not finish spooling'));
-        return;
-      }
-      if (Object.keys(summary.batches).length === 0) {
-        return callback(null, []);
-      }
-
-      // batch IDs are the property names of summary.batches
-      const batchSetIds = getPropertyNames(summary.batches, batchesPerDownloadSession);
-      callback(null, batchSetIds);
-    } else {
-      callback(err);
-    }
   });
 }
 
 /**
- * Download a set of batches retrieved from a log file. When a download is
- * complete, add a line to the logfile indicating such.
+ * Backup directly from Cloudant to an object store object via a stream.
  *
- * @param {any} db - nodejs-cloudant database
- * @param {any} parallelism - number of concurrent requests to make
- * @param {any} log - log file to drive downloads from
- * @param {any} batches - batches to download
- * @param {any} ee - event emitter for progress. This funciton emits
- *  received and error events.
- * @param {any} start - time backup started, to report deltas
- * @param {any} grandtotal - count of documents downloaded prior to this set
- *  of batches
- * @param {any} callback - completion callback, (err, {total: number}).
+ * @param {any} sourceUrl URL of database
+ * @param {any} s3Client Object store client
+ * @param {any} s3Bucket Backup destination bucket
+ * @param {any} s3Key Backup destination key name (shouldn't exist)
+ * @param {any} shallow Whether to use the couchbackup `shallow` mode
+ * @returns Promise
  */
-function processBatchSet(db, parallelism, log, batches, ee, start, grandtotal, callback) {
-  let hasErrored = false;
-  let total = grandtotal;
-
-  // queue to process the fetch requests in an orderly fashion using _bulk_get
-  const q = async.queue(function(payload, done) {
-    const output = [];
-    const thisBatch = payload.batch;
-    delete payload.batch;
-    delete payload.command;
-
-    function logCompletedBatch(batch) {
-      if (log) {
-        fs.appendFile(log, ':d batch' + thisBatch + '\n', done);
-      } else {
-        done();
+function backupToS3(sourceUrl, s3Client, s3Bucket, s3Key, shallow) {
+  return new Promise((resolve, reject) => {
+    debug(`Setting up S3 upload to ${s3Bucket}/${s3Key}`);
+
+    // A pass through stream that has couchbackup's output
+    // written to it and it then read by the S3 upload client.
+    // It has a 64MB highwater mark to allow for fairly
+    // uneven network connectivity.
+    const streamToUpload = new stream.PassThrough({ highWaterMark: 67108864 });
+
+    // Set up S3 upload.
+    const params = {
+      Bucket: s3Bucket,
+      Key: s3Key,
+      Body: streamToUpload
+    };
+    s3Client.upload(params, function(err, data) {
+      debug('Object store upload done');
+      if (err) {
+        debug(err);
+        reject(new VError(err, 'Object store upload failed'));
+        return;
       }
-    }
-
-    // do the /db/_bulk_get request
-    db.service.postBulkGet({
-      db: db.db,
-      revs: true,
-      docs: payload.docs
-    }).then(response => {
-      // create an output array with the docs returned
-      response.result.results.forEach(function(d) {
-        if (d.docs) {
-          d.docs.forEach(function(doc) {
-            if (doc.ok) {
-              output.push(doc.ok);
-            }
-          });
+      debug('Object store upload succeeded');
+      debug(data);
+      resolve();
+    }).httpUploadProgress = (progress) => {
+      debug(`Object store upload progress: ${progress}`);
+    };
+
+    debug(`Starting streaming data from ${s(sourceUrl)}`);
+    couchbackup.backup(
+      sourceUrl,
+      streamToUpload,
+      (err, obj) => {
+        if (err) {
+          debug(err);
+          reject(new VError(err, 'CouchBackup failed with an error'));
+          return;
         }
-      });
-      total += output.length;
-      const t = (new Date().getTime() - start) / 1000;
-      ee.emit('received', {
-        batch: thisBatch,
-        data: output,
-        length: output.length,
-        time: t,
-        total: total
-      }, q, logCompletedBatch);
-    }).catch(err => {
-      if (!hasErrored) {
-        hasErrored = true;
-        err = error.convertResponseError(err);
-        // Kill the queue for fatal errors
-        q.kill();
-        ee.emit('error', err);
+        debug(`Download from ${s(sourceUrl)} complete.`);
+        streamToUpload.end(); // must call end() to complete upload.
+        // resolve() is called by the upload
       }
-      done();
-    });
-  }, parallelism);
-
-  for (const i in batches) {
-    q.push(batches[i]);
-  }
-
-  q.drain(function() {
-    callback(null, { total: total });
+    );
   });
 }
 
 /**
- * Returns first N properties on an object.
+ * Remove creds from a URL, e.g., before logging
  *
- * @param {object} obj - object with properties
- * @param {number} count - number of properties to return
+ * @param {string} url URL to safen
  */
-function getPropertyNames(obj, count) {
-  // decide which batch numbers to deal with
-  const batchestofetch = [];
-  let j = 0;
-  for (const i in obj) {
-    batchestofetch.push(parseInt(i));
-    j++;
-    if (j >= count) break;
-  }
-  return batchestofetch;
+function s(originalUrl) {
+  const parts = new url.URL(originalUrl);
+  return url.format(parts, { auth: false });
 }
+
+main();