@cloudant/couchbackup 2.9.12-SNAPSHOT.136 → 2.9.12

package/.scannerwork/scanner-report/source-20.txt

@@ -0,0 +1,281 @@
+// Copyright © 2017, 2021 IBM Corp. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+'use strict';
+
+const async = require('async');
+const events = require('events');
+const fs = require('fs');
+const error = require('./error.js');
+const spoolchanges = require('./spoolchanges.js');
+const logfilesummary = require('./logfilesummary.js');
+const logfilegetbatches = require('./logfilegetbatches.js');
+
+/**
+ * Read documents from a database to be backed up.
+ *
+ * @param {string} db - `@cloudant/cloudant` DB object for source database.
+ * @param {number} blocksize - number of documents to download in single request
+ * @param {number} parallelism - number of concurrent downloads
+ * @param {string} log - path to log file to use
+ * @param {boolean} resume - whether to resume from an existing log file
+ * @returns EventEmitter with following events:
+ *  - `received` - called with a block of documents to write to backup
+ *  - `error` - on error
+ *  - `finished` - when backup process is finished (either complete or errored)
+ */
+module.exports = function(db, options) {
+  const ee = new events.EventEmitter();
+  const start = new Date().getTime(); // backup start time
+  const batchesPerDownloadSession = 50; // max batches to read from log file for download at a time (prevent OOM)
+
+  function proceedWithBackup() {
+    if (options.resume) {
+      // pick up from existing log file from previous run
+      downloadRemainingBatches(options.log, db, ee, start, batchesPerDownloadSession, options.parallelism);
+    } else {
+      // create new log file and process
+      spoolchanges(db, options.log, options.bufferSize, ee, function(err) {
+        if (err) {
+          ee.emit('error', err);
+        } else {
+          downloadRemainingBatches(options.log, db, ee, start, batchesPerDownloadSession, options.parallelism);
+        }
+      });
+    }
+  }
+
+  validateBulkGetSupport(db, function(err) {
+    if (err) {
+      return ee.emit('error', err);
+    } else {
+      proceedWithBackup();
+    }
+  });
+
+  return ee;
+};
+
+/**
+ * Validate /_bulk_get support for a specified database.
+ *
+ * @param {string} db - nodejs-cloudant db
+ * @param {function} callback - called on completion with signature (err)
+ */
+function validateBulkGetSupport(db, callback) {
+  db.service.postBulkGet({ db: db.db, docs: [] }).then(() => { callback(); }).catch(err => {
+    err = error.convertResponseError(err, function(err) {
+      switch (err.status) {
+        case undefined:
+          // There was no status code on the error
+          return err;
+        case 404:
+          return new error.BackupError('BulkGetError', 'Database does not support /_bulk_get endpoint');
+        default:
+          return new error.HTTPError(err);
+      }
+    });
+    callback(err);
+  });
+}
+
+/**
+ * Download remaining batches in a log file, splitting batches into sets
+ * to avoid enqueueing too many in one go.
+ *
+ * @param {string} log - log file name to maintain download state
+ * @param {string} db - nodejs-cloudant db
+ * @param {events.EventEmitter} ee - event emitter to emit received events on
+ * @param {time} startTime - start time for backup process
+ * @param {number} batchesPerDownloadSession - max batches to enqueue for
+ *  download at a time. As batches contain many doc IDs, this helps avoid
+ *  exhausting memory.
+ * @param {number} parallelism - number of concurrent downloads
+ * @returns function to call do download remaining batches with signature
+ *  (err, {batches: batch, docs: doccount}) {@see spoolchanges}.
+ */
+function downloadRemainingBatches(log, db, ee, startTime, batchesPerDownloadSession, parallelism) {
+  let total = 0; // running total of documents downloaded so far
+  let noRemainingBatches = false;
+
+  // Generate a set of batches (up to batchesPerDownloadSession) to download from the
+  // log file and download them. Set noRemainingBatches to `true` for last batch.
+  function downloadSingleBatchSet(done) {
+    // Fetch the doc IDs for the batches in the current set to
+    // download them.
+    function batchSetComplete(err, data) {
+      if (!err) {
+        total = data.total;
+      }
+      done(err);
+    }
+    function processRetrievedBatches(err, batches) {
+      if (!err) {
+        // process them in parallelised queue
+        processBatchSet(db, parallelism, log, batches, ee, startTime, total, batchSetComplete);
+      } else {
+        batchSetComplete(err);
+      }
+    }
+
+    readBatchSetIdsFromLogFile(log, batchesPerDownloadSession, function(err, batchSetIds) {
+      if (err) {
+        ee.emit('error', err);
+        // Stop processing changes file for fatal errors
+        noRemainingBatches = true;
+        done();
+      } else {
+        if (batchSetIds.length === 0) {
+          noRemainingBatches = true;
+          return done();
+        }
+        logfilegetbatches(log, batchSetIds, processRetrievedBatches);
+      }
+    });
+  }
+
+  // Return true if all batches in log file have been downloaded
+  function isFinished(callback) { callback(null, noRemainingBatches); }
+
+  function onComplete() {
+    ee.emit('finished', { total: total });
+  }
+
+  async.doUntil(downloadSingleBatchSet, isFinished, onComplete);
+}
+
+/**
+ * Return a set of uncompleted download batch IDs from the log file.
+ *
+ * @param {string} log - log file path
+ * @param {number} batchesPerDownloadSession - maximum IDs to return
+ * @param {function} callback - sign (err, batchSetIds array)
+ */
+function readBatchSetIdsFromLogFile(log, batchesPerDownloadSession, callback) {
+  logfilesummary(log, function processSummary(err, summary) {
+    if (!err) {
+      if (!summary.changesComplete) {
+        callback(new error.BackupError('IncompleteChangesInLogFile',
+          'WARNING: Changes did not finish spooling'));
+        return;
+      }
+      if (Object.keys(summary.batches).length === 0) {
+        return callback(null, []);
+      }
+
+      // batch IDs are the property names of summary.batches
+      const batchSetIds = getPropertyNames(summary.batches, batchesPerDownloadSession);
+      callback(null, batchSetIds);
+    } else {
+      callback(err);
+    }
+  });
+}
+
+/**
+ * Download a set of batches retrieved from a log file. When a download is
+ * complete, add a line to the logfile indicating such.
+ *
+ * @param {any} db - nodejs-cloudant database
+ * @param {any} parallelism - number of concurrent requests to make
+ * @param {any} log - log file to drive downloads from
+ * @param {any} batches - batches to download
+ * @param {any} ee - event emitter for progress. This funciton emits
+ *  received and error events.
+ * @param {any} start - time backup started, to report deltas
+ * @param {any} grandtotal - count of documents downloaded prior to this set
+ *  of batches
+ * @param {any} callback - completion callback, (err, {total: number}).
+ */
+function processBatchSet(db, parallelism, log, batches, ee, start, grandtotal, callback) {
+  let hasErrored = false;
+  let total = grandtotal;
+
+  // queue to process the fetch requests in an orderly fashion using _bulk_get
+  const q = async.queue(function(payload, done) {
+    const output = [];
+    const thisBatch = payload.batch;
+    delete payload.batch;
+    delete payload.command;
+
+    function logCompletedBatch(batch) {
+      if (log) {
+        fs.appendFile(log, ':d batch' + thisBatch + '\n', done);
+      } else {
+        done();
+      }
+    }
+
+    // do the /db/_bulk_get request
+    db.service.postBulkGet({
+      db: db.db,
+      revs: true,
+      docs: payload.docs
+    }).then(response => {
+      // create an output array with the docs returned
+      response.result.results.forEach(function(d) {
+        if (d.docs) {
+          d.docs.forEach(function(doc) {
+            if (doc.ok) {
+              output.push(doc.ok);
+            }
+          });
+        }
+      });
+      total += output.length;
+      const t = (new Date().getTime() - start) / 1000;
+      ee.emit('received', {
+        batch: thisBatch,
+        data: output,
+        length: output.length,
+        time: t,
+        total: total
+      }, q, logCompletedBatch);
+    }).catch(err => {
+      if (!hasErrored) {
+        hasErrored = true;
+        err = error.convertResponseError(err);
+        // Kill the queue for fatal errors
+        q.kill();
+        ee.emit('error', err);
+      }
+      done();
+    });
+  }, parallelism);
+
+  for (const i in batches) {
+    q.push(batches[i]);
+  }
+
+  q.drain(function() {
+    callback(null, { total: total });
+  });
+}
+
+/**
+ * Returns first N properties on an object.
+ *
+ * @param {object} obj - object with properties
+ * @param {number} count - number of properties to return
+ */
+function getPropertyNames(obj, count) {
+  // decide which batch numbers to deal with
+  const batchestofetch = [];
+  let j = 0;
+  for (const i in obj) {
+    batchestofetch.push(parseInt(i));
+    j++;
+    if (j >= count) break;
+  }
+  return batchestofetch;
+}

package/.scannerwork/scanner-report/source-23.txt

@@ -0,0 +1,77 @@
+#!/usr/bin/env node
+// Copyright © 2017, 2021 IBM Corp. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+'use strict';
+
+const error = require('../includes/error.js');
+const fs = require('fs');
+const cliutils = require('../includes/cliutils.js');
+const couchbackup = require('../app.js');
+const parser = require('../includes/parser.js');
+const debug = require('debug');
+const backupDebug = debug('couchbackup:backup');
+const backupBatchDebug = debug('couchbackup:backup:batch');
+
+backupDebug.enabled = true;
+
+const program = parser.parseBackupArgs();
+
+const databaseUrl = cliutils.databaseUrl(program.url, program.db);
+const opts = {
+  bufferSize: program.bufferSize,
+  log: program.log,
+  mode: program.mode,
+  parallelism: program.parallelism,
+  requestTimeout: program.requestTimeout,
+  resume: program.resume,
+  iamApiKey: program.iamApiKey,
+  iamTokenUrl: program.iamTokenUrl
+};
+
+// log configuration to console
+console.error('='.repeat(80));
+console.error('Performing backup on ' + databaseUrl.replace(/\/\/.+@/g, '//****:****@') + ' using configuration:');
+console.error(JSON.stringify(opts, null, 2).replace(/"iamApiKey": "[^"]+"/, '"iamApiKey": "****"'));
+console.error('='.repeat(80));
+
+backupBatchDebug.enabled = !program.quiet;
+
+let ws = process.stdout;
+
+// open output file
+if (program.output) {
+  let flags = 'w';
+  if (program.log && program.resume) {
+    flags = 'a';
+  }
+  const fd = fs.openSync(program.output, flags);
+  ws = fs.createWriteStream(null, { fd: fd });
+}
+
+backupDebug('Fetching all database changes...');
+
+return couchbackup.backup(
+  databaseUrl,
+  ws,
+  opts,
+  error.terminationCallback
+).on('changes', function(batch) {
+  backupBatchDebug('Total batches received:', batch + 1);
+}).on('written', function(obj) {
+  backupBatchDebug('Written batch ID:', obj.batch, 'Total document revisions written:', obj.total, 'Time:', obj.time);
+}).on('error', function(e) {
+  backupDebug('ERROR', e);
+}).on('finished', function(obj) {
+  backupDebug('Finished - Total document revisions written:', obj.total);
+});

package/.scannerwork/scanner-report/source-24.txt

@@ -0,0 +1,145 @@
+// Copyright © 2017, 2022 IBM Corp. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+/* global describe it beforeEach */
+'use strict';
+
+const assert = require('assert');
+const backup = require('../includes/shallowbackup.js');
+const request = require('../includes/request.js');
+const fs = require('fs');
+const nock = require('nock');
+
+// Function to create a DB object and call the shallow backup function
+// This is normally done by app.js
+function shallowBackup(dbUrl, opts) {
+  const db = request.client(dbUrl, opts);
+  // Disable compression to make body assertions easier
+  db.service.setEnableGzipCompression(false);
+  return backup(db, opts);
+}
+
+// Note all these tests include a body parameter of include_docs and a query
+// string of include_docs because of a quirk of nano that when using the fetch
+// method always adds the include_docs query string.
+describe('#unit Perform backup using shallow backup', function() {
+  const dbUrl = 'http://localhost:5984/animaldb';
+  // Query string keys are stringified by Nano
+  const badgerKey = 'badger\0';
+  const kookaburraKey = 'kookaburra\0';
+  const snipeKey = 'snipe\0';
+
+  beforeEach('Reset nocks', function() {
+    nock.cleanAll();
+  });
+
+  it('should perform a shallow backup', function(done) {
+    const couch = nock(dbUrl)
+      // batch 1
+      .post('/_all_docs', { limit: 3, include_docs: true })
+      .reply(200, JSON.parse(fs.readFileSync('./test/fixtures/animaldb_all_docs_1.json', 'utf8')))
+      // batch 2
+      .post('/_all_docs', { limit: 3, start_key: badgerKey, include_docs: true })
+      .reply(200, JSON.parse(fs.readFileSync('./test/fixtures/animaldb_all_docs_2.json', 'utf8')))
+      // batch 3
+      .post('/_all_docs', { limit: 3, start_key: kookaburraKey, include_docs: true })
+      .reply(200, JSON.parse(fs.readFileSync('./test/fixtures/animaldb_all_docs_3.json', 'utf8')))
+      // batch 4
+      .post('/_all_docs', { limit: 3, start_key: snipeKey, include_docs: true })
+      .reply(200, JSON.parse(fs.readFileSync('./test/fixtures/animaldb_all_docs_4.json', 'utf8')));
+
+    shallowBackup(dbUrl, { bufferSize: 3, parallelism: 1 })
+      .on('error', function(err) {
+        assert.fail(err);
+      })
+      .on('received', function(data) {
+        if (data.batch === 3) {
+          assert.strictEqual(data.length, 2); // smaller last batch
+        } else {
+          assert.strictEqual(data.length, 3);
+        }
+      })
+      .on('finished', function(data) {
+        assert.strictEqual(data.total, 11);
+        assert.ok(couch.isDone());
+        done();
+      });
+  });
+
+  it('should perform a shallow backup with transient error', function(done) {
+    const couch = nock(dbUrl)
+      // batch 1
+      .post('/_all_docs', { limit: 3, include_docs: true })
+      .reply(200, JSON.parse(fs.readFileSync('./test/fixtures/animaldb_all_docs_1.json', 'utf8')))
+      // batch 2
+      .post('/_all_docs', { limit: 3, start_key: badgerKey, include_docs: true })
+      .reply(200, JSON.parse(fs.readFileSync('./test/fixtures/animaldb_all_docs_2.json', 'utf8')))
+      // batch 3 - transient error
+      .post('/_all_docs', { limit: 3, start_key: kookaburraKey, include_docs: true })
+      .reply(500, { error: 'Internal Server Error' })
+      // batch 3 - retry
+      .post('/_all_docs', { limit: 3, start_key: kookaburraKey, include_docs: true })
+      .reply(200, JSON.parse(fs.readFileSync('./test/fixtures/animaldb_all_docs_3.json', 'utf8')))
+      // batch 4
+      .post('/_all_docs', { limit: 3, start_key: snipeKey, include_docs: true })
+      .reply(200, JSON.parse(fs.readFileSync('./test/fixtures/animaldb_all_docs_4.json', 'utf8')));
+
+    shallowBackup(dbUrl, { bufferSize: 3, parallelism: 1 })
+      .on('error', function(err) {
+        assert.strictEqual(err.name, 'HTTPError');
+      })
+      .on('received', function(data) {
+        if (data.batch === 3) {
+          assert.strictEqual(data.length, 2); // smaller last batch
+        } else {
+          assert.strictEqual(data.length, 3);
+        }
+      })
+      .on('finished', function(data) {
+        assert.strictEqual(data.total, 11);
+        assert.ok(couch.isDone());
+        done();
+      });
+  });
+
+  it('should fail to perform a shallow backup on fatal error', function(done) {
+    const couch = nock(dbUrl)
+      // batch 1
+      .post('/_all_docs', { limit: 3, include_docs: true })
+      .reply(200, JSON.parse(fs.readFileSync('./test/fixtures/animaldb_all_docs_1.json', 'utf8')))
+      // batch 2
+      .post('/_all_docs', { limit: 3, start_key: badgerKey, include_docs: true })
+      .reply(200, JSON.parse(fs.readFileSync('./test/fixtures/animaldb_all_docs_2.json', 'utf8')))
+      // batch 3 - fatal error
+      .post('/_all_docs', { limit: 3, start_key: kookaburraKey, include_docs: true })
+      .reply(401, { error: 'Unauthorized' });
+
+    let errCount = 0;
+
+    shallowBackup(dbUrl, { bufferSize: 3, parallelism: 1 })
+      .on('error', function(err) {
+        errCount++;
+        assert.strictEqual(err.name, 'Unauthorized');
+      })
+      .on('received', function(data) {
+        assert.strictEqual(data.length, 3);
+      })
+      .on('finished', function(data) {
+        assert.strictEqual(data.total, 6);
+        assert.ok(couch.isDone());
+        assert.strictEqual(errCount, 1);
+        done();
+      });
+  });
+});

package/.scannerwork/scanner-report/{source-34.txt → source-25.txt}

@@ -16,19 +16,18 @@
 'use strict';
 
 const assert = require('assert');
-const logfilegetbatches = require('../includes/logfilegetbatches.js');
+const logfilesummary = require('../includes/logfilesummary.js');
 
-describe('#unit Fetching batches from a log file', function() {
-  it('should fetch multiple batches correctly', function(done) {
-    logfilegetbatches('./test/fixtures/test.log', [1, 4], function(err, data) {
+describe('#unit Fetching summary from the log file', function() {
+  it('should fetch a summary correctly', function(done) {
+    logfilesummary('./test/fixtures/test.log', function(err, data) {
       assert.ok(!err);
       assert.ok(data);
-      assert.strictEqual(typeof data, 'object');
-      assert.strictEqual(Object.keys(data).length, 2);
-      assert.deepStrictEqual(data['1'].docs, [{ id: '6' }, { id: '7' }, { id: '8' }, { id: '9' }, { id: '10' }]);
-      assert.strictEqual(data['1'].batch, 1);
-      assert.deepStrictEqual(data['4'].docs, [{ id: '21' }, { id: '22' }]);
-      assert.strictEqual(data['4'].batch, 4);
+      assert.strictEqual(data.changesComplete, true);
+      assert.strictEqual(typeof data.batches, 'object');
+      assert.strictEqual(Object.keys(data.batches).length, 2);
+      assert.deepStrictEqual(data.batches['1'], true);
+      assert.deepStrictEqual(data.batches['4'], true);
       done();
     });
   });