@cloudant/couchbackup 2.9.16-SNAPSHOT.182 → 2.9.16

package/.scannerwork/scanner-report/source-4.txt

@@ -1,4 +1,4 @@
-// Copyright © 2018, 2023 IBM Corp. All rights reserved.
+// Copyright © 2017, 2023 IBM Corp. All rights reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -15,61 +15,173 @@
 /* global describe it */
 'use strict';
 
+const assert = require('assert');
+const backup = require('../app.js').backup;
 const fs = require('fs');
-const { once } = require('node:events');
-const readline = require('readline');
-const u = require('./citestutils.js');
-const uuid = require('uuid').v4;
+const nock = require('nock');
+const util = require('util');
+const backupPromise = util.promisify(backup);
 
-const params = { useApi: true };
+const goodUrl = 'http://localhost:5984/db';
+// The real validateArgs function of app.js isn't
+// exported - so we call the exported backup method
+// instead. We don't get as far as a real backup when
+// testing error cases. For success cases we nock the
+// goodUrl and
+const validateArgs = async function(url, opts, errorValidationForAssertRejects) {
+  const nullStream = fs.createWriteStream('/dev/null');
+  if (url === goodUrl) {
+    // Nock the goodUrl
+    nock(goodUrl).head('').reply(404, { error: 'not_found', reason: 'missing' });
+  }
+  return assert.rejects(backupPromise(url, nullStream, opts), errorValidationForAssertRejects);
+};
 
-describe(u.scenario('Concurrent database backups', params), function() {
-  it('should run concurrent API database backups correctly #slower', async function() {
-    // Allow up to 900 s to backup and compare (it should be much faster)!
-    u.setTimeout(this, 900);
-
-    const checkForEmptyBatches = async function(fileName) {
-      let foundEmptyBatch = false;
+const validateShallowModeArgs = async function(url, opts, msg) {
+  // We pass assertNoValidationError because for these shallow opts
+  // we are expecting only a stderr warning
+  return validateArgs(url, opts, assertNoValidationError()).then(() => {
+    // Assert the warning message was in stderr
+    assert(capturedStderr.indexOf(msg) > -1, 'Log warning message was not present');
+  });
+};
 
-      const rd = readline.createInterface({
-        input: fs.createReadStream(fileName),
-        output: fs.createWriteStream('/dev/null'),
-        terminal: false
-      });
+const stderrWriteFun = process.stderr.write;
+let capturedStderr;
 
-      rd.on('line', function(line) {
-        if (JSON.parse(line).length === 0) {
-          // Note: Empty batch arrays indicate that the running backup is
-          // incorrectly sharing a log file with another ongoing backup job.
-          foundEmptyBatch = true;
-        }
-      });
+function captureStderr() {
+  process.stderr.write = function(string, encoding, fd) {
+    capturedStderr += string;
+  };
+}
 
-      rd.on('close', function() {
-        if (foundEmptyBatch) {
-          return Promise.reject(new Error(`Log file '${fileName}' contains empty batches`));
-        } else {
-          return Promise.resolve();
-        }
-      });
-    };
+function releaseStderr() {
+  process.stderr.write = stderrWriteFun;
+  capturedStderr = null;
+}
 
-    const backupPromise = async function() {
-      const actualBackup = `./${uuid()}`;
-      const output = fs.createWriteStream(actualBackup);
-      return once(output, 'open').then(() => {
-        return u.testBackup(params, 'largedb1g', output);
-      }).then(() => {
-        return checkForEmptyBatches(actualBackup);
-      });
-    };
+// Return a validation object for use with assert.rejects
+function assertErrorMessage(msg) {
+  return { name: 'InvalidOption', message: msg };
+}
 
-    // [1] Run 'largedb1g' database backup
-    const backup1 = backupPromise();
+// For cases where validation should pass we reach a real backup that hits a 404
+// mock for a DatabaseNotFound, so that it is the expected in the case assertNoValidationError
+function assertNoValidationError() { return { name: 'DatabaseNotFound' }; }
 
-    // [2] Run 'largedb1g' database backup
-    const backup2 = backupPromise();
-
-    return Promise.all([backup1, backup2]);
+describe('#unit Validate arguments', function() {
+  it('returns error for invalid URL type', async function() {
+    return validateArgs(true, {}, assertErrorMessage('Invalid URL, must be type string'));
+  });
+  it('returns no error for valid URL type', async function() {
+    return validateArgs(goodUrl, {}, assertNoValidationError());
+  });
+  it('returns error for invalid (no host) URL', async function() {
+    return validateArgs('http://', {}, assertErrorMessage('Invalid URL'));
+  });
+  it('returns error for invalid (no protocol) URL', async function() {
+    return validateArgs('invalid', {}, assertErrorMessage('Invalid URL'));
+  });
+  it('returns error for invalid (wrong protocol) URL', async function() {
+    return validateArgs('ftp://invalid.example.com', {}, assertErrorMessage('Invalid URL protocol.'));
+  });
+  it('returns error for invalid (no path) URL', async function() {
+    return validateArgs('https://invalid.example.com', {}, assertErrorMessage('Invalid URL, missing path element (no database).'));
+  });
+  it('returns error for invalid (no protocol, no host) URL', async function() {
+    return validateArgs('invalid', {}, assertErrorMessage('Invalid URL'));
+  });
+  it('returns error for invalid buffer size type', async function() {
+    return validateArgs(goodUrl, { bufferSize: '123' }, assertErrorMessage('Invalid buffer size option, must be a positive integer in the range (0, MAX_SAFE_INTEGER]'));
+  });
+  it('returns error for zero buffer size', async function() {
+    return validateArgs(goodUrl, { bufferSize: 0 }, assertErrorMessage('Invalid buffer size option, must be a positive integer in the range (0, MAX_SAFE_INTEGER]'));
+  });
+  it('returns error for float buffer size', async function() {
+    return validateArgs(goodUrl, { bufferSize: 1.23 }, assertErrorMessage('Invalid buffer size option, must be a positive integer in the range (0, MAX_SAFE_INTEGER]'));
+  });
+  it('returns no error for valid buffer size type', async function() {
+    return validateArgs(goodUrl, { bufferSize: 123 }, assertNoValidationError());
+  });
+  it('returns error for invalid log type', async function() {
+    return validateArgs(goodUrl, { log: true }, assertErrorMessage('Invalid log option, must be type string'));
+  });
+  it('returns no error for valid log type', async function() {
+    return validateArgs(goodUrl, { log: 'log.txt' }, assertNoValidationError());
+  });
+  it('returns error for invalid mode type', async function() {
+    return validateArgs(goodUrl, { mode: true }, assertErrorMessage('Invalid mode option, must be either "full" or "shallow"'));
+  });
+  it('returns error for invalid mode string', async function() {
+    return validateArgs(goodUrl, { mode: 'foobar' }, assertErrorMessage('Invalid mode option, must be either "full" or "shallow"'));
+  });
+  it('returns no error for valid mode type', async function() {
+    return validateArgs(goodUrl, { mode: 'full' }, assertNoValidationError());
+  });
+  it('returns error for invalid output type', async function() {
+    return validateArgs(goodUrl, { output: true }, assertErrorMessage('Invalid output option, must be type string'));
+  });
+  it('returns no error for valid output type', async function() {
+    return validateArgs(goodUrl, { output: 'output.txt' }, assertNoValidationError());
+  });
+  it('returns error for invalid parallelism type', async function() {
+    return validateArgs(goodUrl, { parallelism: '123' }, assertErrorMessage('Invalid parallelism option, must be a positive integer in the range (0, MAX_SAFE_INTEGER]'));
+  });
+  it('returns error for zero parallelism', async function() {
+    return validateArgs(goodUrl, { parallelism: 0 }, assertErrorMessage('Invalid parallelism option, must be a positive integer in the range (0, MAX_SAFE_INTEGER]'));
+  });
+  it('returns error for float parallelism', async function() {
+    return validateArgs(goodUrl, { parallelism: 1.23 }, assertErrorMessage('Invalid parallelism option, must be a positive integer in the range (0, MAX_SAFE_INTEGER]'));
+  });
+  it('returns no error for valid parallelism type', async function() {
+    return validateArgs(goodUrl, { parallelism: 123 }, assertNoValidationError());
+  });
+  it('returns error for invalid request timeout type', async function() {
+    return validateArgs(goodUrl, { requestTimeout: '123' }, assertErrorMessage('Invalid request timeout option, must be a positive integer in the range (0, MAX_SAFE_INTEGER]'));
+  });
+  it('returns error for zero request timeout', async function() {
+    return validateArgs(goodUrl, { requestTimeout: 0 }, assertErrorMessage('Invalid request timeout option, must be a positive integer in the range (0, MAX_SAFE_INTEGER]'));
+  });
+  it('returns error for float request timout', async function() {
+    return validateArgs(goodUrl, { requestTimeout: 1.23 }, assertErrorMessage('Invalid request timeout option, must be a positive integer in the range (0, MAX_SAFE_INTEGER]'));
+  });
+  it('returns no error for valid request timeout type', async function() {
+    return validateArgs(goodUrl, { requestTimeout: 123 }, assertNoValidationError());
+  });
+  it('returns error for invalid resume type', async function() {
+    return validateArgs(goodUrl, { resume: 'true' }, assertErrorMessage('Invalid resume option, must be type boolean'));
+  });
+  it('returns no error for valid resume type', async function() {
+    return validateArgs(goodUrl, { resume: false }, assertNoValidationError());
+  });
+  it('returns error for invalid key type', async function() {
+    return validateArgs(goodUrl, { iamApiKey: true }, assertErrorMessage('Invalid iamApiKey option, must be type string'));
+  });
+  it('returns error for key and URL credentials supplied', async function() {
+    return validateArgs('https://a:b@example.com/db', { iamApiKey: 'abc123' }, assertErrorMessage('URL user information must not be supplied when using IAM API key.'));
+  });
+  it('warns for log arg in shallow mode', async function() {
+    captureStderr();
+    return validateShallowModeArgs(goodUrl, { mode: 'shallow', log: 'test' },
+      'the options "log" and "resume" are invalid when using shallow mode.').finally(
+      () => {
+        releaseStderr();
+      });
+  });
+  it('warns for resume arg in shallow mode', async function() {
+    captureStderr();
+    return validateShallowModeArgs(goodUrl, { mode: 'shallow', log: 'test', resume: true },
+      'the options "log" and "resume" are invalid when using shallow mode.').finally(
+      () => {
+        releaseStderr();
+      });
+  });
+  it('warns for parallelism arg in shallow mode', async function() {
+    captureStderr();
+    return validateShallowModeArgs(goodUrl, { mode: 'shallow', parallelism: 10 },
+      'the option "parallelism" has no effect when using shallow mode.').finally(
+      () => {
+        releaseStderr();
+      });
   });
 });

package/.scannerwork/scanner-report/source-40.txt

@@ -1,4 +1,4 @@
-// Copyright © 2017 IBM Corp. All rights reserved.
+// Copyright © 2017, 2022 IBM Corp. All rights reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -11,36 +11,101 @@
 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 // See the License for the specific language governing permissions and
 // limitations under the License.
-
-/* global after before describe */
 'use strict';
 
-// Import the common hooks
-require('../test/hooks.js');
+const fs = require('fs');
+const liner = require('./liner.js');
+const change = require('./change.js');
+const error = require('./error.js');
+const debug = require('debug')('couchbackup:spoolchanges');
 
-const poisons = [
-  'normal',
-  'bandwidth-limit',
-  'latency',
-  'slow-read',
-  'rate-limit'
-];
+/**
+ * Write log file for all changes from a database, ready for downloading
+ * in batches.
+ *
+ * @param {string} dbUrl - URL of database
+ * @param {string} log - path to log file to use
+ * @param {number} bufferSize - the number of changes per batch/log line
+ * @param {function(err)} callback - a callback to run on completion
+ */
+module.exports = function(db, log, bufferSize, ee, callback) {
+  // list of document ids to process
+  const buffer = [];
+  let batch = 0;
+  let lastSeq = null;
+  const logStream = fs.createWriteStream(log);
+  let pending = 0;
+  // The number of changes to fetch per request
+  const limit = 100000;
 
-poisons.forEach(function(poison) {
-  describe('unreliable network tests (using poison ' + poison + ')', function() {
-    before('start server', function() {
+  // send documents ids to the queue in batches of bufferSize + the last batch
+  const processBuffer = function(lastOne) {
+    if (buffer.length >= bufferSize || (lastOne && buffer.length > 0)) {
+      debug('writing', buffer.length, 'changes to the backup file');
+      const b = { docs: buffer.splice(0, bufferSize), batch: batch };
+      logStream.write(':t batch' + batch + ' ' + JSON.stringify(b.docs) + '\n');
+      ee.emit('changes', batch);
+      batch++;
+    }
+  };
 
-      // **************************
-      // Currently these tests do nothing
-      // pending resolution of https://github.com/IBM/couchbackup/issues/360
-      // to add a new toxic server
-      // **************************
-    });
+  // called once per received change
+  const onChange = function(c) {
+    if (c) {
+      if (c.error) {
+        ee.emit('error', new error.BackupError('InvalidChange', `Received invalid change: ${c}`));
+      } else if (c.changes) {
+        const obj = { id: c.id };
+        buffer.push(obj);
+        processBuffer(false);
+      } else if (c.last_seq) {
+        lastSeq = c.last_seq;
+        pending = c.pending;
+      }
+    }
+  };
 
-    after('stop server', function() {
-    });
+  function getChanges(since = 0) {
+    debug('making changes request since ' + since);
+    return db.service.postChangesAsStream({ db: db.db, since: since, limit: limit, seqInterval: limit })
+      .then(response => {
+        response.result.pipe(liner())
+          .on('error', function(err) {
+            logStream.end();
+            callback(err);
+          })
+          .pipe(change(onChange))
+          .on('error', function(err) {
+            logStream.end();
+            callback(err);
+          })
+          .on('finish', function() {
+            processBuffer(true);
+            if (!lastSeq) {
+              logStream.end();
+              debug('changes request terminated before last_seq was sent');
+              callback(new error.BackupError('SpoolChangesError', 'Changes request terminated before last_seq was sent'));
+            } else {
+              debug(`changes request completed with last_seq: ${lastSeq} and ${pending} changes pending.`);
+              if (pending > 0) {
+                // Return the next promise
+                return getChanges(lastSeq);
+              } else {
+                debug('finished streaming database changes');
+                logStream.end(':changes_complete ' + lastSeq + '\n', 'utf8', callback);
+              }
+            }
+          });
+      })
+      .catch(err => {
+        logStream.end();
+        if (err.status && err.status >= 400) {
+          callback(error.convertResponseError(err));
+        } else if (err.name !== 'SpoolChangesError') {
+          callback(new error.BackupError('SpoolChangesError', `Failed changes request - ${err.message}`));
+        }
+      });
+  }
 
-    delete require.cache[require.resolve('../test/ci_e2e.js')];
-    require('../test/ci_e2e.js');
-  });
-});
+  getChanges();
+};

package/.scannerwork/scanner-report/source-41.txt

@@ -0,0 +1,46 @@
+// Copyright © 2017 IBM Corp. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+'use strict';
+
+// stolen from http://strongloop.com/strongblog/practical-examples-of-the-new-node-js-streams-api/
+const stream = require('stream');
+
+module.exports = function() {
+  const liner = new stream.Transform({ objectMode: true });
+
+  liner._transform = function(chunk, encoding, done) {
+    let data = chunk.toString();
+    if (this._lastLineData) {
+      data = this._lastLineData + data;
+    }
+
+    const lines = data.split('\n');
+    this._lastLineData = lines.splice(lines.length - 1, 1)[0];
+
+    for (const i in lines) {
+      this.push(lines[i]);
+    }
+    done();
+  };
+
+  liner._flush = function(done) {
+    if (this._lastLineData) {
+      this.push(this._lastLineData);
+    }
+    this._lastLineData = null;
+    done();
+  };
+
+  return liner;
+};

package/.scannerwork/scanner-report/source-42.txt

@@ -1,4 +1,4 @@
-// Copyright © 2017 IBM Corp. All rights reserved.
+// Copyright © 2017, 2022 IBM Corp. All rights reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -13,63 +13,68 @@
 // limitations under the License.
 'use strict';
 
-const fs = require('fs');
-const stream = require('stream');
-const liner = require('./liner.js');
+const async = require('async');
+const error = require('./error.js');
+const events = require('events');
 
-const onLine = function(onCommand, batches) {
-  const change = new stream.Transform({ objectMode: true });
-  change._transform = function(line, encoding, done) {
-    if (line && line[0] === ':') {
-      const obj = {
-        command: null,
-        batch: null,
-        docs: []
-      };
+module.exports = function(db, options) {
+  const ee = new events.EventEmitter();
+  const start = new Date().getTime();
+  let batch = 0;
+  let hasErrored = false;
+  let startKey = null;
+  let total = 0;
 
-      let matches;
+  async.doUntil(
+    function(callback) {
+      // Note, include_docs: true is set automatically when using the
+      // fetch function.
+      const opts = { db: db.db, limit: options.bufferSize, includeDocs: true };
 
-      // extract command
-      matches = line.match(/^:([a-z_]+) ?/);
-      if (matches) {
-        obj.command = matches[1];
-      }
+      // To avoid double fetching a document solely for the purposes of getting
+      // the next ID to use as a startKey for the next page we instead use the
+      // last ID of the current page and append the lowest unicode sort
+      // character.
+      if (startKey) opts.startKey = `${startKey}\0`;
+      db.service.postAllDocs(opts).then(response => {
+        const body = response.result;
+        if (!body.rows) {
+          ee.emit('error', new error.BackupError(
+            'AllDocsError', 'ERROR: Invalid all docs response'));
+          callback();
+        } else {
+          if (body.rows.length < opts.limit) {
+            startKey = null; // last batch
+          } else {
+            startKey = body.rows[opts.limit - 1].id;
+          }
 
-      // extract batch
-      matches = line.match(/ batch([0-9]+)/);
-      if (matches) {
-        obj.batch = parseInt(matches[1]);
-      }
+          const docs = [];
+          body.rows.forEach(function(doc) {
+            docs.push(doc.doc);
+          });
 
-      // if this is one we want
-      if (obj.command === 't' && batches.indexOf(obj.batch) > -1) {
-        const json = line.replace(/^.* batch[0-9]+ /, '').trim();
-        obj.docs = JSON.parse(json);
-        onCommand(obj);
-      }
-    }
-    done();
-  };
-  return change;
-};
-
-module.exports = function(log, batches, callback) {
-  // our sense of state
-  const retval = { };
-
-  // called with each line from the log file
-  const onCommand = function(obj) {
-    retval[obj.batch] = obj;
-  };
+          if (docs.length > 0) {
+            ee.emit('received', {
+              batch: batch++,
+              data: docs,
+              length: docs.length,
+              time: (new Date().getTime() - start) / 1000,
+              total: total += docs.length
+            });
+          }
+          callback();
+        }
+      }).catch(err => {
+        err = error.convertResponseError(err);
+        ee.emit('error', err);
+        hasErrored = true;
+        callback();
+      });
+    },
+    function(callback) { callback(null, hasErrored || startKey == null); },
+    function() { ee.emit('finished', { total: total }); }
+  );
 
-  // stream through the previous log file
-  fs.createReadStream(log)
-    .pipe(liner())
-    .pipe(onLine(onCommand, batches))
-    .on('error', function(err) {
-      callback(err);
-    })
-    .on('finish', function() {
-      callback(null, retval);
-    });
+  return ee;
 };