@cloudant/couchbackup 2.9.9-SNAPSHOT.99 → 2.9.9

package/.scannerwork/scanner-report/source-65.txt

@@ -1,4 +1,4 @@
-// Copyright © 2017, 2018 IBM Corp. All rights reserved.
+// Copyright © 2017, 2022 IBM Corp. All rights reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -11,165 +11,70 @@
 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 // See the License for the specific language governing permissions and
 // limitations under the License.
-
-// Small script which backs up a Cloudant or CouchDB database to an S3
-// bucket via a stream rather than on-disk file.
-//
-// The script generates the backup object name by combining together the path
-// part of the database URL and the current time.
-
 'use strict';
 
-const stream = require('stream');
-const url = require('url');
-
-const AWS = require('aws-sdk');
-const couchbackup = require('@cloudant/couchbackup');
-const debug = require('debug')('s3-backup');
-const VError = require('verror').VError;
-
-/*
-  Main function, run from base of file.
-*/
-function main() {
-  const argv = require('yargs')
-    .usage('Usage: $0 [options]')
-    .example('$0 -s https://user:pass@host/db -b <bucket>', 'Backup db to bucket')
-    .options({
-      source: { alias: 's', nargs: 1, demandOption: true, describe: 'Source database URL' },
-      bucket: { alias: 'b', nargs: 1, demandOption: true, describe: 'Destination bucket' },
-      prefix: { alias: 'p', nargs: 1, describe: 'Prefix for backup object key', default: 'couchbackup' },
-      s3url: { nargs: 1, describe: 'S3 endpoint URL' },
-      awsprofile: { nargs: 1, describe: 'The profile section to use in the ~/.aws/credentials file', default: 'default' }
-    })
-    .help('h').alias('h', 'help')
-    .epilog('Copyright (C) IBM 2017')
-    .argv;
-
-  const sourceUrl = argv.source;
-  const backupName = new url.URL(sourceUrl).pathname.split('/').filter(function(x) { return x; }).join('-');
-  const backupBucket = argv.bucket;
-  const backupKeyPrefix = `${argv.prefix}-${backupName}`;
-  const shallow = argv.shallow;
-
-  const backupKey = `${backupKeyPrefix}-${new Date().toISOString()}`;
-
-  const s3Endpoint = argv.s3url;
-  const awsProfile = argv.awsprofile;
-
-  // Creds are from ~/.aws/credentials, environment etc. (see S3 docs).
-  const awsOpts = {
-    signatureVersion: 'v4',
-    credentials: new AWS.SharedIniFileCredentials({ profile: awsProfile })
-  };
-  if (typeof s3Endpoint !== 'undefined') {
-    awsOpts.endpoint = new AWS.Endpoint(s3Endpoint);
-  }
-  const s3 = new AWS.S3(awsOpts);
-
-  debug(`Creating a new backup of ${s(sourceUrl)} at ${backupBucket}/${backupKey}...`);
-  bucketAccessible(s3, backupBucket)
-    .then(() => {
-      return backupToS3(sourceUrl, s3, backupBucket, backupKey, shallow);
-    })
-    .then(() => {
-      debug('done.');
-    })
-    .catch((reason) => {
-      debug(`Error: ${reason}`);
-      process.exit(1);
-    });
-}
-
-/**
- * Return a promise that resolves if the bucket is available and
- * rejects if not.
- *
- * @param {any} s3 S3 client object
- * @param {any} bucketName Bucket name
- * @returns Promise
- */
-function bucketAccessible(s3, bucketName) {
-  return new Promise(function(resolve, reject) {
-    const params = {
-      Bucket: bucketName
-    };
-    s3.headBucket(params, function(err, data) {
-      if (err) {
-        reject(new VError(err, 'S3 bucket not accessible'));
-      } else {
-        resolve();
-      }
-    });
-  });
-}
-
-/**
- * Backup directly from Cloudant to an object store object via a stream.
- *
- * @param {any} sourceUrl URL of database
- * @param {any} s3Client Object store client
- * @param {any} s3Bucket Backup destination bucket
- * @param {any} s3Key Backup destination key name (shouldn't exist)
- * @param {any} shallow Whether to use the couchbackup `shallow` mode
- * @returns Promise
- */
-function backupToS3(sourceUrl, s3Client, s3Bucket, s3Key, shallow) {
-  return new Promise((resolve, reject) => {
-    debug(`Setting up S3 upload to ${s3Bucket}/${s3Key}`);
-
-    // A pass through stream that has couchbackup's output
-    // written to it and it then read by the S3 upload client.
-    // It has a 64MB highwater mark to allow for fairly
-    // uneven network connectivity.
-    const streamToUpload = new stream.PassThrough({ highWaterMark: 67108864 });
-
-    // Set up S3 upload.
-    const params = {
-      Bucket: s3Bucket,
-      Key: s3Key,
-      Body: streamToUpload
-    };
-    s3Client.upload(params, function(err, data) {
-      debug('Object store upload done');
-      if (err) {
-        debug(err);
-        reject(new VError(err, 'Object store upload failed'));
-        return;
-      }
-      debug('Object store upload succeeded');
-      debug(data);
-      resolve();
-    }).httpUploadProgress = (progress) => {
-      debug(`Object store upload progress: ${progress}`);
-    };
-
-    debug(`Starting streaming data from ${s(sourceUrl)}`);
-    couchbackup.backup(
-      sourceUrl,
-      streamToUpload,
-      (err, obj) => {
-        if (err) {
-          debug(err);
-          reject(new VError(err, 'CouchBackup failed with an error'));
-          return;
+const async = require('async');
+const error = require('./error.js');
+const events = require('events');
+
+module.exports = function(db, options) {
+  const ee = new events.EventEmitter();
+  const start = new Date().getTime();
+  let batch = 0;
+  let hasErrored = false;
+  let startKey = null;
+  let total = 0;
+
+  async.doUntil(
+    function(callback) {
+      // Note, include_docs: true is set automatically when using the
+      // fetch function.
+      const opts = { db: db.db, limit: options.bufferSize, includeDocs: true };
+
+      // To avoid double fetching a document solely for the purposes of getting
+      // the next ID to use as a startKey for the next page we instead use the
+      // last ID of the current page and append the lowest unicode sort
+      // character.
+      if (startKey) opts.startKey = `${startKey}\0`;
+      db.service.postAllDocs(opts).then(response => {
+        const body = response.result;
+        if (!body.rows) {
+          ee.emit('error', new error.BackupError(
+            'AllDocsError', 'ERROR: Invalid all docs response'));
+          callback();
+        } else {
+          if (body.rows.length < opts.limit) {
+            startKey = null; // last batch
+          } else {
+            startKey = body.rows[opts.limit - 1].id;
+          }
+
+          const docs = [];
+          body.rows.forEach(function(doc) {
+            docs.push(doc.doc);
+          });
+
+          if (docs.length > 0) {
+            ee.emit('received', {
+              batch: batch++,
+              data: docs,
+              length: docs.length,
+              time: (new Date().getTime() - start) / 1000,
+              total: total += docs.length
+            });
+          }
+          callback();
         }
-        debug(`Download from ${s(sourceUrl)} complete.`);
-        streamToUpload.end(); // must call end() to complete upload.
-        // resolve() is called by the upload
-      }
-    );
-  });
-}
-
-/**
- * Remove creds from a URL, e.g., before logging
- *
- * @param {string} url URL to safen
- */
-function s(originalUrl) {
-  const parts = new url.URL(originalUrl);
-  return url.format(parts, { auth: false });
-}
-
-main();
+      }).catch(err => {
+        err = error.convertResponseError(err);
+        ee.emit('error', err);
+        hasErrored = true;
+        callback();
+      });
+    },
+    function(callback) { callback(null, hasErrored || startKey == null); },
+    function() { ee.emit('finished', { total: total }); }
+  );
+
+  return ee;
+};

package/.scannerwork/scanner-report/source-66.txt

@@ -0,0 +1,31 @@
+// Copyright © 2017, 2018 IBM Corp. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+'use strict';
+
+module.exports = function(db, options, readstream, ee, callback) {
+  const liner = require('../includes/liner.js')();
+  const writer = require('../includes/writer.js')(db, options.bufferSize, options.parallelism, ee);
+
+  // pipe the input to the output, via transformation functions
+  readstream
+    .pipe(liner) // transform the input stream into per-line
+    .on('error', function(err) {
+      // Forward the error to the writer event emitter where we already have
+      // listeners on for handling errors
+      writer.emit('error', err);
+    })
+    .pipe(writer); // transform the data
+
+  callback(null, writer);
+};

package/.scannerwork/scanner-report/source-67.txt

@@ -1,4 +1,4 @@
-// Copyright © 2017, 2018 IBM Corp. All rights reserved.
+// Copyright © 2017, 2021 IBM Corp. All rights reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -11,180 +11,154 @@
 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 // See the License for the specific language governing permissions and
 // limitations under the License.
-
-// Small script which backs up a Cloudant or CouchDB database to an S3
-// bucket, using an intermediary file on disk.
-//
-// The script generates the backup object name by combining together the path
-// part of the database URL and the current time.
-
 'use strict';
 
+const async = require('async');
 const stream = require('stream');
-const fs = require('fs');
-const url = require('url');
-
-const AWS = require('aws-sdk');
-const couchbackup = require('@cloudant/couchbackup');
-const debug = require('debug')('s3-backup');
-const tmp = require('tmp');
-const VError = require('verror').VError;
-
-/*
-  Main function, run from base of file.
-*/
-function main() {
-  const argv = require('yargs')
-    .usage('Usage: $0 [options]')
-    .example('$0 -s https://user:pass@host/db -b <bucket>', 'Backup db to bucket')
-    .options({
-      source: { alias: 's', nargs: 1, demandOption: true, describe: 'Source database URL' },
-      bucket: { alias: 'b', nargs: 1, demandOption: true, describe: 'Destination bucket' },
-      prefix: { alias: 'p', nargs: 1, describe: 'Prefix for backup object key', default: 'couchbackup' },
-      s3url: { nargs: 1, describe: 'S3 endpoint URL' },
-      awsprofile: { nargs: 1, describe: 'The profile section to use in the ~/.aws/credentials file', default: 'default' }
-    })
-    .help('h').alias('h', 'help')
-    .epilog('Copyright (C) IBM 2017')
-    .argv;
-
-  const sourceUrl = argv.source;
-  const backupBucket = argv.bucket;
-  const backupName = new url.URL(sourceUrl).pathname.split('/').filter(function(x) { return x; }).join('-');
-  const backupKeyPrefix = `${argv.prefix}-${backupName}`;
-
-  const backupKey = `${backupKeyPrefix}-${new Date().toISOString()}`;
-  const backupTmpFile = tmp.fileSync();
-
-  const s3Endpoint = argv.s3url;
-  const awsProfile = argv.awsprofile;
-
-  // Creds are from ~/.aws/credentials, environment etc. (see S3 docs).
-  const awsOpts = {
-    signatureVersion: 'v4',
-    credentials: new AWS.SharedIniFileCredentials({ profile: awsProfile })
-  };
-  if (typeof s3Endpoint !== 'undefined') {
-    awsOpts.endpoint = new AWS.Endpoint(s3Endpoint);
-  }
-  const s3 = new AWS.S3(awsOpts);
-
-  debug(`Creating a new backup of ${s(sourceUrl)} at ${backupBucket}/${backupKey}...`);
-  bucketAccessible(s3, backupBucket)
-    .then(() => {
-      return createBackupFile(sourceUrl, backupTmpFile.name);
-    })
-    .then(() => {
-      return uploadNewBackup(s3, backupTmpFile.name, backupBucket, backupKey);
-    })
-    .then(() => {
-      debug('Backup successful!');
-      backupTmpFile.removeCallback();
-      debug('done.');
-    })
-    .catch((reason) => {
-      debug(`Error: ${reason}`);
-    });
-}
-
-/**
- * Return a promise that resolves if the bucket is available and
- * rejects if not.
- *
- * @param {any} s3 S3 client object
- * @param {any} bucketName Bucket name
- * @returns Promise
- */
-function bucketAccessible(s3, bucketName) {
-  return new Promise(function(resolve, reject) {
-    const params = {
-      Bucket: bucketName
-    };
-    s3.headBucket(params, function(err, data) {
-      if (err) {
-        reject(new VError(err, 'S3 bucket not accessible'));
-      } else {
-        resolve();
-      }
-    });
-  });
-}
-
-/**
- * Use couchbackup to create a backup of the specified database to a file path.
- *
- * @param {any} sourceUrl Database URL
- * @param {any} backupTmpFilePath Path to write file
- * @returns Promise
- */
-function createBackupFile(sourceUrl, backupTmpFilePath) {
-  return new Promise((resolve, reject) => {
-    couchbackup.backup(
-      sourceUrl,
-      fs.createWriteStream(backupTmpFilePath),
-      (err) => {
-        if (err) {
-          return reject(new VError(err, 'CouchBackup process failed'));
+const error = require('./error.js');
+const debug = require('debug')('couchbackup:writer');
+
+module.exports = function(db, bufferSize, parallelism, ee) {
+  const writer = new stream.Transform({ objectMode: true });
+  let buffer = [];
+  let written = 0;
+  let linenumber = 0;
+
+  // this is the queue of chunks that are written to the database
+  // the queue's payload will be an array of documents to be written,
+  // the size of the array will be bufferSize. The variable parallelism
+  // determines how many HTTP requests will occur at any one time.
+  const q = async.queue(function(payload, cb) {
+    // if we are restoring known revisions, we need to supply new_edits=false
+    if (payload.docs && payload.docs[0] && payload.docs[0]._rev) {
+      payload.new_edits = false;
+      debug('Using new_edits false mode.');
+    }
+
+    if (!didError) {
+      db.service.postBulkDocs({
+        db: db.db,
+        bulkDocs: payload
+      }).then(response => {
+        if (!response.result || (payload.new_edits === false && response.result.length > 0)) {
+          throw new Error(`Error writing batch with new_edits:${payload.new_edits !== false}` +
+            ` and ${response.result ? response.result.length : 'unavailable'} items`);
         }
-        debug('couchbackup to file done; uploading to S3');
-        resolve('creating backup file complete');
+        written += payload.docs.length;
+        writer.emit('restored', { documents: payload.docs.length, total: written });
+        cb();
+      }).catch(err => {
+        err = error.convertResponseError(err);
+        debug(`Error writing docs ${err.name} ${err.message}`);
+        cb(err, payload);
+      });
+    }
+  }, parallelism);
+
+  let didError = false;
+
+  // write the contents of the buffer to CouchDB in blocks of bufferSize
+  function processBuffer(flush, callback) {
+    function taskCallback(err, payload) {
+      if (err && !didError) {
+        debug(`Queue task failed with error ${err.name}`);
+        didError = true;
+        q.kill();
+        writer.emit('error', err);
       }
-    );
-  });
-}
-
-/**
- * Upload a backup file to an S3 bucket.
- *
- * @param {any} s3 Object store client
- * @param {any} backupTmpFilePath Path of backup file to write.
- * @param {any} bucket Object store bucket name
- * @param {any} key Object store key name
- * @returns Promise
- */
-function uploadNewBackup(s3, backupTmpFilePath, bucket, key) {
-  return new Promise((resolve, reject) => {
-    debug(`Uploading from ${backupTmpFilePath} to ${bucket}/${key}`);
-
-    function uploadFromStream(s3, bucket, key) {
-      const pass = new stream.PassThrough();
-
-      const params = {
-        Bucket: bucket,
-        Key: key,
-        Body: pass
-      };
-      s3.upload(params, function(err, data) {
-        debug('S3 upload done');
-        if (err) {
-          debug(err);
-          reject(new VError(err, 'Upload failed'));
-          return;
+    }
+
+    if (flush || buffer.length >= bufferSize) {
+      // work through the buffer to break off bufferSize chunks
+      // and feed the chunks to the queue
+      do {
+        // split the buffer into bufferSize chunks
+        const toSend = buffer.splice(0, bufferSize);
+
+        // and add the chunk to the queue
+        debug(`Adding ${toSend.length} to the write queue.`);
+        q.push({ docs: toSend }, taskCallback);
+      } while (buffer.length >= bufferSize);
+
+      // send any leftover documents to the queue
+      if (flush && buffer.length > 0) {
+        debug(`Adding remaining ${buffer.length} to the write queue.`);
+        q.push({ docs: buffer }, taskCallback);
+      }
+
+      // wait until the queue size falls to a reasonable level
+      async.until(
+        // wait until the queue length drops to twice the paralellism
+        // or until empty on the last write
+        function(callback) {
+          // if we encountered an error, stop this until loop
+          if (didError) {
+            return callback(null, true);
+          }
+          if (flush) {
+            callback(null, q.idle() && q.length() === 0);
+          } else {
+            callback(null, q.length() <= parallelism * 2);
+          }
+        },
+        function(cb) {
+          setTimeout(cb, 20);
+        },
+
+        function() {
+          if (flush && !didError) {
+            writer.emit('finished', { total: written });
+          }
+          // callback when we're happy with the queue size
+          callback();
+        });
+    } else {
+      callback();
+    }
+  }
+
+  // take an object
+  writer._transform = function(obj, encoding, done) {
+    // each obj that arrives here is a line from the backup file
+    // it should contain an array of objects. The length of the array
+    // depends on the bufferSize at backup time.
+    linenumber++;
+    if (!didError && obj !== '') {
+      // see if it parses as JSON
+      try {
+        const arr = JSON.parse(obj);
+
+        // if it's an array with a length
+        if (typeof arr === 'object' && arr.length > 0) {
+          // push each document into a buffer
+          buffer = buffer.concat(arr);
+
+          // pause the stream
+          // it's likely that the speed with which data can be read from disk
+          // may exceed the rate it can be written to CouchDB. To prevent
+          // the whole file being buffered in memory, we pause the stream here.
+          // it is resumed, when processBuffer calls back and we call done()
+          this.pause();
+
+          // break the buffer in to bufferSize chunks to be written to the database
+          processBuffer(false, done);
+        } else {
+          ee.emit('error', new error.BackupError('BackupFileJsonError', `Error on line ${linenumber} of backup file - not an array`));
+          done();
         }
-        debug('Upload succeeded');
-        debug(data);
-        resolve();
-      }).httpUploadProgress = (progress) => {
-        debug(`S3 upload progress: ${progress}`);
-      };
-
-      return pass;
+      } catch (e) {
+        ee.emit('error', new error.BackupError('BackupFileJsonError', `Error on line ${linenumber} of backup file - cannot parse as JSON`));
+        // Could be an incomplete write that was subsequently resumed
+        done();
+      }
+    } else {
+      done();
     }
+  };
 
-    const inputStream = fs.createReadStream(backupTmpFilePath);
-    const s3Stream = uploadFromStream(s3, bucket, key);
-    inputStream.pipe(s3Stream);
-  });
-}
-
-/**
- * Remove creds from a URL, e.g., before logging
- *
- * @param {string} url URL to safen
- */
-function s(originalUrl) {
-  const parts = new url.URL(originalUrl);
-  return url.format(parts, { auth: false });
-}
-
-main();
+  // called when we need to flush everything
+  writer._flush = function(done) {
+    processBuffer(true, done);
+  };
+  return writer;
+};