@cloudant/couchbackup 2.9.13-SNAPSHOT.143 → 2.9.13

package/.scannerwork/scanner-report/source-62.txt

@@ -1,167 +1,54 @@
-<testsuites name="test-iam">
-  <testsuite name="Basic backup and restore using API" tests="3" errors="0" failures="0" skipped="0" timestamp="2023-09-28T09:04:42" time="4.672">
-    <testcase classname="test-iam.Basic backup and restore using API" name="should backup animaldb to a file correctly" time="0.953">
-      <system-out><![CDATA[{ total: 15 }
-]]></system-out>
-    </testcase>
-    <testcase classname="test-iam.Basic backup and restore using API" name="should restore animaldb to a database correctly" time="1.899">
-      <system-out><![CDATA[{ total: 15 }
-]]></system-out>
-    </testcase>
-    <testcase classname="test-iam.Basic backup and restore using API" name="should execute a shallow mode backup successfully" time="0.642">
-      <system-out><![CDATA[{ total: 11 }
-]]></system-out>
-    </testcase>
-  </testsuite>
-  <testsuite name="Basic backup and restore using API Buffer size tests using API" tests="3" errors="0" failures="0" skipped="0" timestamp="2023-09-28T09:04:47" time="11.406">
-    <testcase classname="test-iam.Basic backup and restore using API Buffer size tests using API" name="should backup/restore animaldb with the same buffer size" time="3.591">
-      <system-out><![CDATA[{ total: 15 }
-{ total: 15 }
-]]></system-out>
-    </testcase>
-    <testcase classname="test-iam.Basic backup and restore using API Buffer size tests using API" name="should backup/restore animaldb with backup buffer &gt; restore buffer" time="3.481">
-      <system-out><![CDATA[{ total: 15 }
-{ total: 15 }
-]]></system-out>
-    </testcase>
-    <testcase classname="test-iam.Basic backup and restore using API Buffer size tests using API" name="should backup/restore animaldb with backup buffer &lt; restore buffer" time="3.513">
-      <system-out><![CDATA[{ total: 15 }
-{ total: 15 }
-]]></system-out>
-    </testcase>
-  </testsuite>
-  <testsuite name="Basic backup and restore using CLI" tests="3" errors="0" failures="0" skipped="0" timestamp="2023-09-28T09:04:58" time="6.387">
-    <testcase classname="test-iam.Basic backup and restore using CLI" name="should backup animaldb to a file correctly" time="1.674">
-      <system-out><![CDATA[Backup process close 0 null
-]]></system-out>
-    </testcase>
-    <testcase classname="test-iam.Basic backup and restore using CLI" name="should restore animaldb to a database correctly" time="2.596">
-    </testcase>
-    <testcase classname="test-iam.Basic backup and restore using CLI" name="should execute a shallow mode backup successfully" time="1.308">
-      <system-out><![CDATA[Backup process close 0 null
-]]></system-out>
-    </testcase>
-  </testsuite>
-  <testsuite name="Basic backup and restore using CLI Buffer size tests using CLI" tests="3" errors="0" failures="0" skipped="0" timestamp="2023-09-28T09:05:04" time="15.574">
-    <testcase classname="test-iam.Basic backup and restore using CLI Buffer size tests using CLI" name="should backup/restore animaldb with the same buffer size" time="5.055">
-      <system-out><![CDATA[Backup process close 0 null
-]]></system-out>
-    </testcase>
-    <testcase classname="test-iam.Basic backup and restore using CLI Buffer size tests using CLI" name="should backup/restore animaldb with backup buffer &gt; restore buffer" time="4.771">
-      <system-out><![CDATA[Backup process close 0 null
-]]></system-out>
-    </testcase>
-    <testcase classname="test-iam.Basic backup and restore using CLI Buffer size tests using CLI" name="should backup/restore animaldb with backup buffer &lt; restore buffer" time="4.949">
-      <system-out><![CDATA[Backup process close 0 null
-]]></system-out>
-    </testcase>
-  </testsuite>
-  <testsuite name="Compression tests using API" tests="3" errors="0" failures="0" skipped="0" timestamp="2023-09-28T09:05:20" time="6.819">
-    <testcase classname="test-iam.Compression tests using API" name="should backup animaldb to a compressed file" time="0.867">
-      <system-out><![CDATA[{ total: 15 }
-]]></system-out>
-    </testcase>
-    <testcase classname="test-iam.Compression tests using API" name="should backup and restore animaldb via a compressed file" time="3.011">
-      <system-out><![CDATA[{ total: 15 }
-{ total: 15 }
-]]></system-out>
-    </testcase>
-    <testcase classname="test-iam.Compression tests using API" name="should backup and restore animaldb via a compressed stream" time="2.135">
-      <system-out><![CDATA[{ total: 15 }
-{ total: 15 }
-]]></system-out>
-    </testcase>
-  </testsuite>
-  <testsuite name="Compression tests using CLI" tests="3" errors="0" failures="0" skipped="0" timestamp="2023-09-28T09:05:27" time="10.494">
-    <testcase classname="test-iam.Compression tests using CLI" name="should backup animaldb to a compressed file" time="1.748">
-      <system-out><![CDATA[Backup process close 0 null
-]]></system-out>
-    </testcase>
-    <testcase classname="test-iam.Compression tests using CLI" name="should backup and restore animaldb via a compressed file" time="4.403">
-      <system-out><![CDATA[Backup process close 0 null
-]]></system-out>
-    </testcase>
-    <testcase classname="test-iam.Compression tests using CLI" name="should backup and restore animaldb via a compressed stream" time="3.542">
-      <system-out><![CDATA[Backup process close 0 null
-]]></system-out>
-    </testcase>
-  </testsuite>
-  <testsuite name="End to end backup and restore using API" tests="2" errors="0" failures="0" skipped="0" timestamp="2023-09-28T09:05:37" time="323.663">
-    <testcase classname="test-iam.End to end backup and restore using API" name="should backup and restore animaldb" time="2.087">
-      <system-out><![CDATA[{ total: 15 }
-{ total: 15 }
-]]></system-out>
-    </testcase>
-    <testcase classname="test-iam.End to end backup and restore using API" name="should backup and restore largedb1g #slow" time="320.79">
-      <system-out><![CDATA[{ total: 522948 }
-{ total: 522948 }
-]]></system-out>
-    </testcase>
-  </testsuite>
-  <testsuite name="End to end backup and restore using CLI" tests="2" errors="0" failures="0" skipped="0" timestamp="2023-09-28T09:11:01" time="487.18">
-    <testcase classname="test-iam.End to end backup and restore using CLI" name="should backup and restore animaldb" time="3.279">
-      <system-out><![CDATA[Backup process close 0 null
-]]></system-out>
-    </testcase>
-    <testcase classname="test-iam.End to end backup and restore using CLI" name="should backup and restore largedb1g #slow" time="480.345">
-      <system-out><![CDATA[Backup process close 0 null
-]]></system-out>
-    </testcase>
-  </testsuite>
-  <testsuite name="Encryption tests" tests="1" errors="0" failures="0" skipped="0" timestamp="2023-09-28T09:19:08" time="4.721">
-    <testcase classname="test-iam.Encryption tests" name="should backup and restore animaldb via an encrypted file" time="4.448">
-      <system-out><![CDATA[Backup process close 0 null
-]]></system-out>
-    </testcase>
-  </testsuite>
-  <testsuite name="Write error tests" tests="1" errors="0" failures="0" skipped="0" timestamp="2023-09-28T09:19:13" time="0.274">
-    <testcase classname="test-iam.Write error tests" name="calls callback with error set when stream is not writeable" time="0.012">
-    </testcase>
-  </testsuite>
-  <testsuite name="Event tests" tests="2" errors="0" failures="0" skipped="0" timestamp="2023-09-28T09:19:13" time="2.173">
-    <testcase classname="test-iam.Event tests" name="should get a finished event when using stdout" time="0.815">
-    </testcase>
-    <testcase classname="test-iam.Event tests" name="should get a finished event when using file output" time="0.827">
-    </testcase>
-  </testsuite>
-  <testsuite name="Resume tests using API" tests="3" errors="0" failures="0" skipped="0" timestamp="2023-09-28T09:19:15" time="5.097">
-    <testcase classname="test-iam.Resume tests using API" name="should create a log file" time="0.856">
-      <system-out><![CDATA[{ total: 15 }
-]]></system-out>
-    </testcase>
-    <testcase classname="test-iam.Resume tests using API" name="should restore corrupted animaldb to a database correctly" time="1.772">
-      <system-out><![CDATA[{ total: 15 }
-]]></system-out>
-    </testcase>
-    <testcase classname="test-iam.Resume tests using API" name="should restore resumed animaldb with blank line to a database correctly" time="1.674">
-      <system-out><![CDATA[{ total: 15 }
-]]></system-out>
-    </testcase>
-  </testsuite>
-  <testsuite name="Resume tests using CLI" tests="3" errors="0" failures="0" skipped="0" timestamp="2023-09-28T09:19:20" time="8.014">
-    <testcase classname="test-iam.Resume tests using CLI" name="should create a log file" time="1.844">
-      <system-out><![CDATA[Backup process close 0 null
-]]></system-out>
-    </testcase>
-    <testcase classname="test-iam.Resume tests using CLI" name="should restore corrupted animaldb to a database correctly" time="2.846">
-    </testcase>
-    <testcase classname="test-iam.Resume tests using CLI" name="should restore resumed animaldb with blank line to a database correctly" time="2.528">
-    </testcase>
-  </testsuite>
-  <testsuite name="Resume tests" tests="2" errors="0" failures="0" skipped="0" timestamp="2023-09-28T09:19:28" time="34.656">
-    <testcase classname="test-iam.Resume tests" name="should correctly backup and restore backup10m" time="17.234">
-      <system-out><![CDATA[Backup process close null SIGTERM
-Backup process close 0 null
-]]></system-out>
-    </testcase>
-    <testcase classname="test-iam.Resume tests" name="should correctly backup and restore backup10m using --output" time="16.884">
-      <system-out><![CDATA[Backup process close null SIGTERM
-Backup process close 0 null
-]]></system-out>
-    </testcase>
-  </testsuite>
-  <testsuite name="Longer spool changes checks" tests="1" errors="0" failures="0" skipped="0" timestamp="2023-09-28T09:20:03" time="45.141">
-    <testcase classname="test-iam.Longer spool changes checks" name="#slow should keep collecting changes (25M)" time="44.871">
-    </testcase>
-  </testsuite>
-</testsuites>
+// Copyright © 2017, 2018 IBM Corp. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+'use strict';
+
+/**
+ * Utility methods for the command line interface.
+ * @module cliutils
+ * @see module:cliutils
+ */
+
+const url = require('url');
+
+module.exports = {
+
+  /**
+   * Combine a base URL and a database name, ensuring at least single slash
+   * between root and database name. This allows users to have Couch behind
+   * proxies that mount Couch's / endpoint at some other mount point.
+   * @param {string} root - root URL
+   * @param {string} databaseName - database name
+   * @return concatenated URL.
+   *
+   * @private
+   */
+  databaseUrl: function databaseUrl(root, databaseName) {
+    if (!root.endsWith('/')) {
+      root = root + '/';
+    }
+    return new url.URL(encodeURIComponent(databaseName), root).toString();
+  },
+
+  /**
+   * Generate CLI argument usage text.
+   *
+   * @param {string} description - argument description.
+   * @param {string} defaultValue - default argument value.
+   *
+   * @private
+   */
+  getUsage: function getUsage(description, defaultValue) {
+    return `${description} ${defaultValue !== undefined ? ` (default: ${defaultValue})` : ''}`;
+  }
+};

package/.scannerwork/scanner-report/source-63.txt

@@ -0,0 +1,111 @@
+// Copyright © 2017, 2022 IBM Corp. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+'use strict';
+
+const fs = require('fs');
+const liner = require('./liner.js');
+const change = require('./change.js');
+const error = require('./error.js');
+const debug = require('debug')('couchbackup:spoolchanges');
+
+/**
+ * Write log file for all changes from a database, ready for downloading
+ * in batches.
+ *
+ * @param {string} dbUrl - URL of database
+ * @param {string} log - path to log file to use
+ * @param {number} bufferSize - the number of changes per batch/log line
+ * @param {function(err)} callback - a callback to run on completion
+ */
+module.exports = function(db, log, bufferSize, ee, callback) {
+  // list of document ids to process
+  const buffer = [];
+  let batch = 0;
+  let lastSeq = null;
+  const logStream = fs.createWriteStream(log);
+  let pending = 0;
+  // The number of changes to fetch per request
+  const limit = 100000;
+
+  // send documents ids to the queue in batches of bufferSize + the last batch
+  const processBuffer = function(lastOne) {
+    if (buffer.length >= bufferSize || (lastOne && buffer.length > 0)) {
+      debug('writing', buffer.length, 'changes to the backup file');
+      const b = { docs: buffer.splice(0, bufferSize), batch: batch };
+      logStream.write(':t batch' + batch + ' ' + JSON.stringify(b.docs) + '\n');
+      ee.emit('changes', batch);
+      batch++;
+    }
+  };
+
+  // called once per received change
+  const onChange = function(c) {
+    if (c) {
+      if (c.error) {
+        ee.emit('error', new error.BackupError('InvalidChange', `Received invalid change: ${c}`));
+      } else if (c.changes) {
+        const obj = { id: c.id };
+        buffer.push(obj);
+        processBuffer(false);
+      } else if (c.last_seq) {
+        lastSeq = c.last_seq;
+        pending = c.pending;
+      }
+    }
+  };
+
+  function getChanges(since = 0) {
+    debug('making changes request since ' + since);
+    return db.service.postChangesAsStream({ db: db.db, since: since, limit: limit, seqInterval: limit })
+      .then(response => {
+        response.result.pipe(liner())
+          .on('error', function(err) {
+            logStream.end();
+            callback(err);
+          })
+          .pipe(change(onChange))
+          .on('error', function(err) {
+            logStream.end();
+            callback(err);
+          })
+          .on('finish', function() {
+            processBuffer(true);
+            if (!lastSeq) {
+              logStream.end();
+              debug('changes request terminated before last_seq was sent');
+              callback(new error.BackupError('SpoolChangesError', 'Changes request terminated before last_seq was sent'));
+            } else {
+              debug(`changes request completed with last_seq: ${lastSeq} and ${pending} changes pending.`);
+              if (pending > 0) {
+                // Return the next promise
+                return getChanges(lastSeq);
+              } else {
+                debug('finished streaming database changes');
+                logStream.end(':changes_complete ' + lastSeq + '\n', 'utf8', callback);
+              }
+            }
+          });
+      })
+      .catch(err => {
+        logStream.end();
+        if (err.status && err.status >= 400) {
+          callback(error.convertResponseError(err));
+        } else if (err.name !== 'SpoolChangesError') {
+          callback(new error.BackupError('SpoolChangesError', `Failed changes request - ${err.message}`));
+        }
+      });
+  }
+
+  getChanges();
+};