@cloudant/couchbackup 2.9.12-SNAPSHOT.136 → 2.9.12

package/.scannerwork/scanner-report/source-10.txt

@@ -0,0 +1,111 @@
+// Copyright © 2017, 2022 IBM Corp. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+'use strict';
+
+const fs = require('fs');
+const liner = require('./liner.js');
+const change = require('./change.js');
+const error = require('./error.js');
+const debug = require('debug')('couchbackup:spoolchanges');
+
+/**
+ * Write log file for all changes from a database, ready for downloading
+ * in batches.
+ *
+ * @param {string} dbUrl - URL of database
+ * @param {string} log - path to log file to use
+ * @param {number} bufferSize - the number of changes per batch/log line
+ * @param {function(err)} callback - a callback to run on completion
+ */
+module.exports = function(db, log, bufferSize, ee, callback) {
+  // list of document ids to process
+  const buffer = [];
+  let batch = 0;
+  let lastSeq = null;
+  const logStream = fs.createWriteStream(log);
+  let pending = 0;
+  // The number of changes to fetch per request
+  const limit = 100000;
+
+  // send documents ids to the queue in batches of bufferSize + the last batch
+  const processBuffer = function(lastOne) {
+    if (buffer.length >= bufferSize || (lastOne && buffer.length > 0)) {
+      debug('writing', buffer.length, 'changes to the backup file');
+      const b = { docs: buffer.splice(0, bufferSize), batch: batch };
+      logStream.write(':t batch' + batch + ' ' + JSON.stringify(b.docs) + '\n');
+      ee.emit('changes', batch);
+      batch++;
+    }
+  };
+
+  // called once per received change
+  const onChange = function(c) {
+    if (c) {
+      if (c.error) {
+        ee.emit('error', new error.BackupError('InvalidChange', `Received invalid change: ${c}`));
+      } else if (c.changes) {
+        const obj = { id: c.id };
+        buffer.push(obj);
+        processBuffer(false);
+      } else if (c.last_seq) {
+        lastSeq = c.last_seq;
+        pending = c.pending;
+      }
+    }
+  };
+
+  function getChanges(since = 0) {
+    debug('making changes request since ' + since);
+    return db.service.postChangesAsStream({ db: db.db, since: since, limit: limit, seqInterval: limit })
+      .then(response => {
+        response.result.pipe(liner())
+          .on('error', function(err) {
+            logStream.end();
+            callback(err);
+          })
+          .pipe(change(onChange))
+          .on('error', function(err) {
+            logStream.end();
+            callback(err);
+          })
+          .on('finish', function() {
+            processBuffer(true);
+            if (!lastSeq) {
+              logStream.end();
+              debug('changes request terminated before last_seq was sent');
+              callback(new error.BackupError('SpoolChangesError', 'Changes request terminated before last_seq was sent'));
+            } else {
+              debug(`changes request completed with last_seq: ${lastSeq} and ${pending} changes pending.`);
+              if (pending > 0) {
+                // Return the next promise
+                return getChanges(lastSeq);
+              } else {
+                debug('finished streaming database changes');
+                logStream.end(':changes_complete ' + lastSeq + '\n', 'utf8', callback);
+              }
+            }
+          });
+      })
+      .catch(err => {
+        logStream.end();
+        if (err.status && err.status >= 400) {
+          callback(error.convertResponseError(err));
+        } else if (err.name !== 'SpoolChangesError') {
+          callback(new error.BackupError('SpoolChangesError', `Failed changes request - ${err.message}`));
+        }
+      });
+  }
+
+  getChanges();
+};

package/.scannerwork/scanner-report/source-11.txt

@@ -1,4 +1,4 @@
-// Copyright © 2023 IBM Corp. All rights reserved.
+// Copyright © 2017, 2018 IBM Corp. All rights reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -11,90 +11,44 @@
 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 // See the License for the specific language governing permissions and
 // limitations under the License.
-
 'use strict';
 
-const chunk = require('lodash/chunk');
-const difference = require('lodash/difference');
-const forOwn = require('lodash/forOwn');
-const isEmpty = require('lodash/isEmpty');
-const union = require('lodash/union');
-
-const compare = async(database1, database2, client) => {
-  // check docs same in both dbs
-  const allDocs1 = await getAllDocs(client, database1);
-  const allDocs2 = await getAllDocs(client, database2);
-
-  const onlyInDb1 = (difference(allDocs1, allDocs2));
-  const onlyInDb2 = (difference(allDocs2, allDocs1));
-
-  let databasesSame = isEmpty(onlyInDb1) && isEmpty(onlyInDb2);
-
-  if (!databasesSame) {
-    console.log(onlyInDb1.length + ' documents only in db 1.');
-    console.log('Document IDs only in db 1: ' + onlyInDb1);
-    console.log(onlyInDb2.length + ' documents only in db 2.');
-    console.log('Document IDs only in db 2: ' + onlyInDb2);
-  }
-
-  // check revs same in docs common to both dbs
-  const partitionSize = 500;
-  const batches = chunk(union(allDocs1, allDocs2), partitionSize);
-
-  const missingRevsInDb2 = await getMissingRevs(client, database1, database2, batches);
-  const missingRevsInDb1 = await getMissingRevs(client, database2, database1, batches);
-
-  databasesSame = databasesSame && isEmpty(missingRevsInDb1) && isEmpty(missingRevsInDb2);
+/**
+ * Utility methods for the command line interface.
+ * @module cliutils
+ * @see module:cliutils
+ */
 
-  if (!databasesSame) {
-    console.log('Missing revs in db 1:' + JSON.stringify(missingRevsInDb1));
-    console.log('Missing revs in db 2:' + JSON.stringify(missingRevsInDb2));
-  }
-
-  return databasesSame;
-};
+const url = require('url');
 
-const getMissingRevs = async(client, databaseName1, databaseName2, batcheses) => {
-  const fakeRevisionId = '9999-a';
-
-  const missing = {};
-
-  // look in db1 - use a fake revision ID to fetch all leaf revisions
-
-  for (const batches of batcheses) {
-    const documentRevisions = {};
-    batches.forEach(id => (documentRevisions[id] = [fakeRevisionId]));
-
-    const result1 = await client.postRevsDiff({ db: databaseName1, documentRevisions });
-    const revsDiffRequestDb2 = {};
-    forOwn(result1.result, (v, k) => (revsDiffRequestDb2[k] = v.possible_ancestors));
-    // look in db2
-    const result2 = await client.postRevsDiff({ db: databaseName2, documentRevisions: revsDiffRequestDb2 });
-    forOwn(result2.result, (v, k) => {
-      if ('missing' in v) {
-        missing[k] = v.missing;
-      }
-    });
-  }
-  return missing;
-};
+module.exports = {
 
-const getAllDocs = async(client, database) => {
-  let allDocIds = [];
-  const limit = 2000;
-  let startKey = '\u0000';
-  do {
-    const pageOfDocIds = (await client.postAllDocs({ db: database, startKey, limit })).result.rows.map(r => r.id);
-    allDocIds = allDocIds.concat(pageOfDocIds);
-    if (pageOfDocIds.length < limit) {
-      startKey = null;
-    } else {
-      startKey = pageOfDocIds[limit - 1] + '\u0000';
+  /**
+   * Combine a base URL and a database name, ensuring at least single slash
+   * between root and database name. This allows users to have Couch behind
+   * proxies that mount Couch's / endpoint at some other mount point.
+   * @param {string} root - root URL
+   * @param {string} databaseName - database name
+   * @return concatenated URL.
+   *
+   * @private
+   */
+  databaseUrl: function databaseUrl(root, databaseName) {
+    if (!root.endsWith('/')) {
+      root = root + '/';
     }
-  } while (startKey != null);
-  return allDocIds;
-};
-
-module.exports = {
-  compare
+    return new url.URL(encodeURIComponent(databaseName), root).toString();
+  },
+
+  /**
+   * Generate CLI argument usage text.
+   *
+   * @param {string} description - argument description.
+   * @param {string} defaultValue - default argument value.
+   *
+   * @private
+   */
+  getUsage: function getUsage(description, defaultValue) {
+    return `${description} ${defaultValue !== undefined ? ` (default: ${defaultValue})` : ''}`;
+  }
 };

package/.scannerwork/scanner-report/source-12.txt

@@ -1,4 +1,4 @@
-// Copyright © 2017, 2022 IBM Corp. All rights reserved.
+// Copyright © 2017 IBM Corp. All rights reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -11,135 +11,31 @@
 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 // See the License for the specific language governing permissions and
 // limitations under the License.
-
-/* global describe it beforeEach */
 'use strict';
 
-const assert = require('assert');
-const backup = require('../includes/shallowbackup.js');
-const request = require('../includes/request.js');
-const fs = require('fs');
-const nock = require('nock');
-
-// Function to create a DB object and call the shallow backup function
-// This is normally done by app.js
-function shallowBackup(dbUrl, opts) {
-  const db = request.client(dbUrl, opts);
-  // Disable compression to make body assertions easier
-  db.service.setEnableGzipCompression(false);
-  return backup(db, opts);
-}
-
-// Note all these tests include a body parameter of include_docs and a query
-// string of include_docs because of a quirk of nano that when using the fetch
-// method always adds the include_docs query string.
-describe('#unit Perform backup using shallow backup', function() {
-  const dbUrl = 'http://localhost:5984/animaldb';
-  // Query string keys are stringified by Nano
-  const badgerKey = 'badger\0';
-  const kookaburraKey = 'kookaburra\0';
-  const snipeKey = 'snipe\0';
-
-  beforeEach('Reset nocks', function() {
-    nock.cleanAll();
-  });
-
-  it('should perform a shallow backup', function(done) {
-    const couch = nock(dbUrl)
-      // batch 1
-      .post('/_all_docs', { limit: 3, include_docs: true })
-      .reply(200, JSON.parse(fs.readFileSync('./test/fixtures/animaldb_all_docs_1.json', 'utf8')))
-      // batch 2
-      .post('/_all_docs', { limit: 3, start_key: badgerKey, include_docs: true })
-      .reply(200, JSON.parse(fs.readFileSync('./test/fixtures/animaldb_all_docs_2.json', 'utf8')))
-      // batch 3
-      .post('/_all_docs', { limit: 3, start_key: kookaburraKey, include_docs: true })
-      .reply(200, JSON.parse(fs.readFileSync('./test/fixtures/animaldb_all_docs_3.json', 'utf8')))
-      // batch 4
-      .post('/_all_docs', { limit: 3, start_key: snipeKey, include_docs: true })
-      .reply(200, JSON.parse(fs.readFileSync('./test/fixtures/animaldb_all_docs_4.json', 'utf8')));
-
-    shallowBackup(dbUrl, { bufferSize: 3, parallelism: 1 })
-      .on('error', function(err) {
-        assert.fail(err);
-      })
-      .on('received', function(data) {
-        if (data.batch === 3) {
-          assert.strictEqual(data.length, 2); // smaller last batch
-        } else {
-          assert.strictEqual(data.length, 3);
-        }
-      })
-      .on('finished', function(data) {
-        assert.strictEqual(data.total, 11);
-        assert.ok(couch.isDone());
-        done();
-      });
-  });
+// stolen from http://strongloop.com/strongblog/practical-examples-of-the-new-node-js-streams-api/
+const stream = require('stream');
 
-  it('should perform a shallow backup with transient error', function(done) {
-    const couch = nock(dbUrl)
-      // batch 1
-      .post('/_all_docs', { limit: 3, include_docs: true })
-      .reply(200, JSON.parse(fs.readFileSync('./test/fixtures/animaldb_all_docs_1.json', 'utf8')))
-      // batch 2
-      .post('/_all_docs', { limit: 3, start_key: badgerKey, include_docs: true })
-      .reply(200, JSON.parse(fs.readFileSync('./test/fixtures/animaldb_all_docs_2.json', 'utf8')))
-      // batch 3 - transient error
-      .post('/_all_docs', { limit: 3, start_key: kookaburraKey, include_docs: true })
-      .reply(500, { error: 'Internal Server Error' })
-      // batch 3 - retry
-      .post('/_all_docs', { limit: 3, start_key: kookaburraKey, include_docs: true })
-      .reply(200, JSON.parse(fs.readFileSync('./test/fixtures/animaldb_all_docs_3.json', 'utf8')))
-      // batch 4
-      .post('/_all_docs', { limit: 3, start_key: snipeKey, include_docs: true })
-      .reply(200, JSON.parse(fs.readFileSync('./test/fixtures/animaldb_all_docs_4.json', 'utf8')));
+module.exports = function(onChange) {
+  const change = new stream.Transform({ objectMode: true });
 
-    shallowBackup(dbUrl, { bufferSize: 3, parallelism: 1 })
-      .on('error', function(err) {
-        assert.strictEqual(err.name, 'HTTPError');
-      })
-      .on('received', function(data) {
-        if (data.batch === 3) {
-          assert.strictEqual(data.length, 2); // smaller last batch
-        } else {
-          assert.strictEqual(data.length, 3);
-        }
-      })
-      .on('finished', function(data) {
-        assert.strictEqual(data.total, 11);
-        assert.ok(couch.isDone());
-        done();
-      });
-  });
+  change._transform = function(line, encoding, done) {
+    let obj = null;
 
-  it('should fail to perform a shallow backup on fatal error', function(done) {
-    const couch = nock(dbUrl)
-      // batch 1
-      .post('/_all_docs', { limit: 3, include_docs: true })
-      .reply(200, JSON.parse(fs.readFileSync('./test/fixtures/animaldb_all_docs_1.json', 'utf8')))
-      // batch 2
-      .post('/_all_docs', { limit: 3, start_key: badgerKey, include_docs: true })
-      .reply(200, JSON.parse(fs.readFileSync('./test/fixtures/animaldb_all_docs_2.json', 'utf8')))
-      // batch 3 - fatal error
-      .post('/_all_docs', { limit: 3, start_key: kookaburraKey, include_docs: true })
-      .reply(401, { error: 'Unauthorized' });
+    // one change per line - remove the trailing comma
+    line = line.trim().replace(/,$/, '');
 
-    let errCount = 0;
+    // extract thee last_seq at the end of the changes feed
+    if (line.match(/^"last_seq":/)) {
+      line = '{' + line;
+    }
+    try {
+      obj = JSON.parse(line);
+    } catch (e) {
+    }
+    onChange(obj);
+    done();
+  };
 
-    shallowBackup(dbUrl, { bufferSize: 3, parallelism: 1 })
-      .on('error', function(err) {
-        errCount++;
-        assert.strictEqual(err.name, 'Unauthorized');
-      })
-      .on('received', function(data) {
-        assert.strictEqual(data.length, 3);
-      })
-      .on('finished', function(data) {
-        assert.strictEqual(data.total, 6);
-        assert.ok(couch.isDone());
-        assert.strictEqual(errCount, 1);
-        done();
-      });
-  });
-});
+  return change;
+};

package/.scannerwork/scanner-report/source-13.txt

@@ -1,4 +1,4 @@
-// Copyright © 2017, 2018 IBM Corp. All rights reserved.
+// Copyright © 2017, 2021 IBM Corp. All rights reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -11,58 +11,119 @@
 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 // See the License for the specific language governing permissions and
 // limitations under the License.
-
-/* global describe it */
 'use strict';
 
-const fs = require('fs');
-const u = require('./citestutils.js');
-
-[{ useApi: true }, { useApi: false }].forEach(function(params) {
-  describe(u.scenario('Compression tests', params), function() {
-    const p = u.p(params, { compression: true });
-
-    it('should backup animaldb to a compressed file', function(done) {
-      // Allow up to 60 s for backup of animaldb
-      u.setTimeout(this, 60);
-      const compressedBackup = `./${this.fileName}`;
-      const output = fs.createWriteStream(compressedBackup);
-      output.on('open', function() {
-        u.testBackup(p, 'animaldb', output, function(err) {
-          if (err) {
-            done(err);
-          } else {
-            u.assertGzipFile(compressedBackup, done);
-          }
-        });
-      });
-    });
-
-    it('should backup and restore animaldb via a compressed file', function(done) {
-      // Allow up to 60 s for backup and restore of animaldb
-      u.setTimeout(this, 60);
-      const compressedBackup = `./${this.fileName}`;
-      u.testBackupAndRestoreViaFile(p, 'animaldb', compressedBackup, this.dbName, function(err) {
-        if (err) {
-          done(err);
-        } else {
-          u.assertGzipFile(compressedBackup, done);
-        }
-      });
-    });
-
-    it('should backup and restore animaldb via a compressed stream', function(done) {
-      // Allow up to 60 s for backup and restore of animaldb
-      u.setTimeout(this, 60);
-      u.testDirectBackupAndRestore(p, 'animaldb', this.dbName, done);
-    });
-
-    it('should backup and restore largedb2g via a compressed file #slower', function(done) {
-      // Takes ~ 25 min using CLI, but sometimes over an hour with API
-      u.setTimeout(this, 180 * 60);
-      const compressedBackup = `./${this.fileName}`;
-      params.compression = true;
-      u.testBackupAndRestoreViaFile(p, 'largedb2g', compressedBackup, this.dbName, done);
-    });
-  });
-});
+const cliutils = require('./cliutils.js');
+const config = require('./config.js');
+const error = require('./error.js');
+const path = require('path');
+const pkg = require('../package.json');
+
+function parseBackupArgs() {
+  const program = require('commander');
+
+  // Option CLI defaults
+  const defaults = config.cliDefaults();
+
+  // Options set by environment variables
+  const envVarOptions = {};
+  config.applyEnvironmentVariables(envVarOptions);
+
+  program
+    .version(pkg.version)
+    .description('Backup a CouchDB/Cloudant database to a backup text file.')
+    .usage('[options...]')
+    .option('-b, --buffer-size <n>',
+      cliutils.getUsage('number of documents fetched at once', defaults.bufferSize),
+      Number)
+    .option('-d, --db <db>',
+      cliutils.getUsage('name of the database to backup', defaults.db))
+    .option('-k, --iam-api-key <API key>',
+      cliutils.getUsage('IAM API key to access the Cloudant server'))
+    .option('-l, --log <file>',
+      cliutils.getUsage('file to store logging information during backup; invalid in "shallow" mode', 'a temporary file'),
+      path.normalize)
+    .option('-m, --mode <mode>',
+      cliutils.getUsage('"shallow" if only a superficial backup is done (ignoring conflicts and revision tokens), else "full" for complete backup', defaults.mode),
+      (mode) => { return mode.toLowerCase(); })
+    .option('-o, --output <file>',
+      cliutils.getUsage('file name to store the backup data', 'stdout'),
+      path.normalize)
+    .option('-p, --parallelism <n>',
+      cliutils.getUsage('number of HTTP requests to perform in parallel when performing a backup; ignored in "shallow" mode', defaults.parallelism),
+      Number)
+    .option('-q, --quiet',
+      cliutils.getUsage('suppress batch messages', defaults.quiet))
+    .option('-r, --resume',
+      cliutils.getUsage('continue a previous backup from its last known position; invalid in "shallow" mode', defaults.resume))
+    .option('-t, --request-timeout <n>',
+      cliutils.getUsage('milliseconds to wait for a response to a HTTP request before retrying the request', defaults.requestTimeout),
+      Number)
+    .option('-u, --url <url>',
+      cliutils.getUsage('URL of the CouchDB/Cloudant server', defaults.url))
+    .parse(process.argv);
+
+  // Remove defaults that don't apply when using shallow mode
+  if (program.opts().mode === 'shallow' || envVarOptions.mode === 'shallow') {
+    delete defaults.parallelism;
+    delete defaults.log;
+    delete defaults.resume;
+  }
+
+  // Apply the options in order so that the CLI overrides env vars and env variables
+  // override defaults.
+  const opts = Object.assign({}, defaults, envVarOptions, program.opts());
+
+  if (opts.resume && (opts.log === defaults.log)) {
+    // If resuming and the log file arg is the newly generated tmp name from defaults then we know that --log wasn't specified.
+    // We have to do this check here for the CLI case because of the default.
+    error.terminationCallback(new error.BackupError('NoLogFileName', 'To resume a backup, a log file must be specified'));
+  }
+
+  return opts;
+}
+
+function parseRestoreArgs() {
+  const program = require('commander');
+
+  // Option CLI defaults
+  const defaults = config.cliDefaults();
+
+  // Options set by environment variables
+  const envVarOptions = {};
+  config.applyEnvironmentVariables(envVarOptions);
+
+  program
+    .version(pkg.version)
+    .description('Restore a CouchDB/Cloudant database from a backup text file.')
+    .usage('[options...]')
+    .option('-b, --buffer-size <n>',
+      cliutils.getUsage('number of documents restored at once', defaults.bufferSize),
+      Number)
+    .option('-d, --db <db>',
+      cliutils.getUsage('name of the new, existing database to restore to', defaults.db))
+    .option('-k, --iam-api-key <API key>',
+      cliutils.getUsage('IAM API key to access the Cloudant server'))
+    .option('-p, --parallelism <n>',
+      cliutils.getUsage('number of HTTP requests to perform in parallel when restoring a backup', defaults.parallelism),
+      Number)
+    .option('-q, --quiet',
+      cliutils.getUsage('suppress batch messages', defaults.quiet))
+    .option('-t, --request-timeout <n>',
+      cliutils.getUsage('milliseconds to wait for a response to a HTTP request before retrying the request', defaults.requestTimeout),
+      Number)
+    .option('-u, --url <url>',
+      cliutils.getUsage('URL of the CouchDB/Cloudant server', defaults.url))
+    .parse(process.argv);
+
+  // Apply the options in order so that the CLI overrides env vars and env variables
+  // override defaults.
+  const opts = Object.assign({}, defaults, envVarOptions, program.opts());
+
+  return opts;
+}
+
+module.exports = {
+  parseBackupArgs: parseBackupArgs,
+  parseRestoreArgs: parseRestoreArgs
+};