@cloudant/couchbackup 2.9.10-SNAPSHOT.113 → 2.9.10

package/.scannerwork/scanner-report/source-68.txt

@@ -1,167 +1,56 @@
-<testsuites name="test-iam">
-  <testsuite name="Basic backup and restore using API" tests="3" errors="0" failures="0" skipped="0" timestamp="2023-06-06T14:08:58" time="4.976">
-    <testcase classname="test-iam.Basic backup and restore using API" name="should backup animaldb to a file correctly" time="1.038">
-      <system-out><![CDATA[{ total: 15 }
-]]></system-out>
-    </testcase>
-    <testcase classname="test-iam.Basic backup and restore using API" name="should restore animaldb to a database correctly" time="2.016">
-      <system-out><![CDATA[{ total: 15 }
-]]></system-out>
-    </testcase>
-    <testcase classname="test-iam.Basic backup and restore using API" name="should execute a shallow mode backup successfully" time="0.814">
-      <system-out><![CDATA[{ total: 11 }
-]]></system-out>
-    </testcase>
-  </testsuite>
-  <testsuite name="Basic backup and restore using API Buffer size tests using API" tests="3" errors="0" failures="0" skipped="0" timestamp="2023-06-06T14:09:03" time="11.231">
-    <testcase classname="test-iam.Basic backup and restore using API Buffer size tests using API" name="should backup/restore animaldb with the same buffer size" time="3.653">
-      <system-out><![CDATA[{ total: 15 }
-{ total: 15 }
-]]></system-out>
-    </testcase>
-    <testcase classname="test-iam.Basic backup and restore using API Buffer size tests using API" name="should backup/restore animaldb with backup buffer &gt; restore buffer" time="3.368">
-      <system-out><![CDATA[{ total: 15 }
-{ total: 15 }
-]]></system-out>
-    </testcase>
-    <testcase classname="test-iam.Basic backup and restore using API Buffer size tests using API" name="should backup/restore animaldb with backup buffer &lt; restore buffer" time="3.43">
-      <system-out><![CDATA[{ total: 15 }
-{ total: 15 }
-]]></system-out>
-    </testcase>
-  </testsuite>
-  <testsuite name="Basic backup and restore using CLI" tests="3" errors="0" failures="0" skipped="0" timestamp="2023-06-06T14:09:15" time="5.413">
-    <testcase classname="test-iam.Basic backup and restore using CLI" name="should backup animaldb to a file correctly" time="1.272">
-      <system-out><![CDATA[Backup process close 0 null
-]]></system-out>
-    </testcase>
-    <testcase classname="test-iam.Basic backup and restore using CLI" name="should restore animaldb to a database correctly" time="2.399">
-    </testcase>
-    <testcase classname="test-iam.Basic backup and restore using CLI" name="should execute a shallow mode backup successfully" time="0.961">
-      <system-out><![CDATA[Backup process close 0 null
-]]></system-out>
-    </testcase>
-  </testsuite>
-  <testsuite name="Basic backup and restore using CLI Buffer size tests using CLI" tests="3" errors="0" failures="0" skipped="0" timestamp="2023-06-06T14:09:20" time="13.746">
-    <testcase classname="test-iam.Basic backup and restore using CLI Buffer size tests using CLI" name="should backup/restore animaldb with the same buffer size" time="4.43">
-      <system-out><![CDATA[Backup process close 0 null
-]]></system-out>
-    </testcase>
-    <testcase classname="test-iam.Basic backup and restore using CLI Buffer size tests using CLI" name="should backup/restore animaldb with backup buffer &gt; restore buffer" time="4.164">
-      <system-out><![CDATA[Backup process close 0 null
-]]></system-out>
-    </testcase>
-    <testcase classname="test-iam.Basic backup and restore using CLI Buffer size tests using CLI" name="should backup/restore animaldb with backup buffer &lt; restore buffer" time="4.375">
-      <system-out><![CDATA[Backup process close 0 null
-]]></system-out>
-    </testcase>
-  </testsuite>
-  <testsuite name="Compression tests using API" tests="3" errors="0" failures="0" skipped="0" timestamp="2023-06-06T14:09:34" time="6.424">
-    <testcase classname="test-iam.Compression tests using API" name="should backup animaldb to a compressed file" time="0.808">
-      <system-out><![CDATA[{ total: 15 }
-]]></system-out>
-    </testcase>
-    <testcase classname="test-iam.Compression tests using API" name="should backup and restore animaldb via a compressed file" time="2.665">
-      <system-out><![CDATA[{ total: 15 }
-{ total: 15 }
-]]></system-out>
-    </testcase>
-    <testcase classname="test-iam.Compression tests using API" name="should backup and restore animaldb via a compressed stream" time="2.174">
-      <system-out><![CDATA[{ total: 15 }
-{ total: 15 }
-]]></system-out>
-    </testcase>
-  </testsuite>
-  <testsuite name="Compression tests using CLI" tests="3" errors="0" failures="0" skipped="0" timestamp="2023-06-06T14:09:40" time="8.23">
-    <testcase classname="test-iam.Compression tests using CLI" name="should backup animaldb to a compressed file" time="1.256">
-      <system-out><![CDATA[Backup process close 0 null
-]]></system-out>
-    </testcase>
-    <testcase classname="test-iam.Compression tests using CLI" name="should backup and restore animaldb via a compressed file" time="3.533">
-      <system-out><![CDATA[Backup process close 0 null
-]]></system-out>
-    </testcase>
-    <testcase classname="test-iam.Compression tests using CLI" name="should backup and restore animaldb via a compressed stream" time="2.661">
-      <system-out><![CDATA[Backup process close 0 null
-]]></system-out>
-    </testcase>
-  </testsuite>
-  <testsuite name="End to end backup and restore using API" tests="2" errors="0" failures="0" skipped="0" timestamp="2023-06-06T14:09:48" time="317.553">
-    <testcase classname="test-iam.End to end backup and restore using API" name="should backup and restore animaldb" time="2.106">
-      <system-out><![CDATA[{ total: 15 }
-{ total: 15 }
-]]></system-out>
-    </testcase>
-    <testcase classname="test-iam.End to end backup and restore using API" name="should backup and restore largedb1g #slow" time="314.686">
-      <system-out><![CDATA[{ total: 522948 }
-{ total: 522948 }
-]]></system-out>
-    </testcase>
-  </testsuite>
-  <testsuite name="End to end backup and restore using CLI" tests="2" errors="0" failures="0" skipped="0" timestamp="2023-06-06T14:15:06" time="519.014">
-    <testcase classname="test-iam.End to end backup and restore using CLI" name="should backup and restore animaldb" time="2.776">
-      <system-out><![CDATA[Backup process close 0 null
-]]></system-out>
-    </testcase>
-    <testcase classname="test-iam.End to end backup and restore using CLI" name="should backup and restore largedb1g #slow" time="515.489">
-      <system-out><![CDATA[Backup process close 0 null
-]]></system-out>
-    </testcase>
-  </testsuite>
-  <testsuite name="Encryption tests" tests="1" errors="0" failures="0" skipped="0" timestamp="2023-06-06T14:23:45" time="3.891">
-    <testcase classname="test-iam.Encryption tests" name="should backup and restore animaldb via an encrypted file" time="3.632">
-      <system-out><![CDATA[Backup process close 0 null
-]]></system-out>
-    </testcase>
-  </testsuite>
-  <testsuite name="Write error tests" tests="1" errors="0" failures="0" skipped="0" timestamp="2023-06-06T14:23:49" time="0.268">
-    <testcase classname="test-iam.Write error tests" name="calls callback with error set when stream is not writeable" time="0.01">
-    </testcase>
-  </testsuite>
-  <testsuite name="Event tests" tests="2" errors="0" failures="0" skipped="0" timestamp="2023-06-06T14:23:49" time="2.379">
-    <testcase classname="test-iam.Event tests" name="should get a finished event when using stdout" time="1.041">
-    </testcase>
-    <testcase classname="test-iam.Event tests" name="should get a finished event when using file output" time="0.824">
-    </testcase>
-  </testsuite>
-  <testsuite name="Resume tests using API" tests="3" errors="0" failures="0" skipped="0" timestamp="2023-06-06T14:23:52" time="5.155">
-    <testcase classname="test-iam.Resume tests using API" name="should create a log file" time="0.819">
-      <system-out><![CDATA[{ total: 15 }
-]]></system-out>
-    </testcase>
-    <testcase classname="test-iam.Resume tests using API" name="should restore corrupted animaldb to a database correctly" time="1.832">
-      <system-out><![CDATA[{ total: 15 }
-]]></system-out>
-    </testcase>
-    <testcase classname="test-iam.Resume tests using API" name="should restore resumed animaldb with blank line to a database correctly" time="1.732">
-      <system-out><![CDATA[{ total: 15 }
-]]></system-out>
-    </testcase>
-  </testsuite>
-  <testsuite name="Resume tests using CLI" tests="3" errors="0" failures="0" skipped="0" timestamp="2023-06-06T14:23:57" time="6.3">
-    <testcase classname="test-iam.Resume tests using CLI" name="should create a log file" time="1.213">
-      <system-out><![CDATA[Backup process close 0 null
-]]></system-out>
-    </testcase>
-    <testcase classname="test-iam.Resume tests using CLI" name="should restore corrupted animaldb to a database correctly" time="2.155">
-    </testcase>
-    <testcase classname="test-iam.Resume tests using CLI" name="should restore resumed animaldb with blank line to a database correctly" time="2.151">
-    </testcase>
-  </testsuite>
-  <testsuite name="Resume tests" tests="2" errors="0" failures="0" skipped="0" timestamp="2023-06-06T14:24:03" time="33.23">
-    <testcase classname="test-iam.Resume tests" name="should correctly backup and restore backup10m" time="15.911">
-      <system-out><![CDATA[Backup process close null SIGTERM
-Backup process close 0 null
-]]></system-out>
-    </testcase>
-    <testcase classname="test-iam.Resume tests" name="should correctly backup and restore backup10m using --output" time="16.808">
-      <system-out><![CDATA[Backup process close null SIGTERM
-Backup process close 0 null
-]]></system-out>
-    </testcase>
-  </testsuite>
-  <testsuite name="Longer spool changes checks" tests="1" errors="0" failures="0" skipped="0" timestamp="2023-06-06T14:24:36" time="44.268">
-    <testcase classname="test-iam.Longer spool changes checks" name="#slow should keep collecting changes (25M)" time="44.01">
-    </testcase>
-  </testsuite>
-</testsuites>
+#!/usr/bin/env node
+// Copyright © 2017, 2021 IBM Corp. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+'use strict';
+
+const error = require('../includes/error.js');
+const cliutils = require('../includes/cliutils.js');
+const couchbackup = require('../app.js');
+const parser = require('../includes/parser.js');
+const debug = require('debug');
+const restoreDebug = debug('couchbackup:restore');
+const restoreBatchDebug = debug('couchbackup:restore:batch');
+
+restoreDebug.enabled = true;
+
+const program = parser.parseRestoreArgs();
+const databaseUrl = cliutils.databaseUrl(program.url, program.db);
+const opts = {
+  bufferSize: program.bufferSize,
+  parallelism: program.parallelism,
+  requestTimeout: program.requestTimeout,
+  iamApiKey: program.iamApiKey,
+  iamTokenUrl: program.iamTokenUrl
+};
+
+// log configuration to console
+console.error('='.repeat(80));
+console.error('Performing restore on ' + databaseUrl.replace(/\/\/.+@/g, '//****:****@') + ' using configuration:');
+console.error(JSON.stringify(opts, null, 2).replace(/"iamApiKey": "[^"]+"/, '"iamApiKey": "****"'));
+console.error('='.repeat(80));
+
+restoreBatchDebug.enabled = !program.quiet;
+
+return couchbackup.restore(
+  process.stdin, // restore from stdin
+  databaseUrl,
+  opts,
+  error.terminationCallback
+).on('restored', function(obj) {
+  restoreBatchDebug('restored', obj.total);
+}).on('error', function(e) {
+  restoreDebug('ERROR', e);
+}).on('finished', function(obj) {
+  restoreDebug('finished', obj);
+});

package/.scannerwork/scanner-report/source-7.txt

@@ -0,0 +1,80 @@
+// Copyright © 2017, 2022 IBM Corp. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+'use strict';
+
+const async = require('async');
+const error = require('./error.js');
+const events = require('events');
+
+module.exports = function(db, options) {
+  const ee = new events.EventEmitter();
+  const start = new Date().getTime();
+  let batch = 0;
+  let hasErrored = false;
+  let startKey = null;
+  let total = 0;
+
+  async.doUntil(
+    function(callback) {
+      // Note, include_docs: true is set automatically when using the
+      // fetch function.
+      const opts = { db: db.db, limit: options.bufferSize, includeDocs: true };
+
+      // To avoid double fetching a document solely for the purposes of getting
+      // the next ID to use as a startKey for the next page we instead use the
+      // last ID of the current page and append the lowest unicode sort
+      // character.
+      if (startKey) opts.startKey = `${startKey}\0`;
+      db.service.postAllDocs(opts).then(response => {
+        const body = response.result;
+        if (!body.rows) {
+          ee.emit('error', new error.BackupError(
+            'AllDocsError', 'ERROR: Invalid all docs response'));
+          callback();
+        } else {
+          if (body.rows.length < opts.limit) {
+            startKey = null; // last batch
+          } else {
+            startKey = body.rows[opts.limit - 1].id;
+          }
+
+          const docs = [];
+          body.rows.forEach(function(doc) {
+            docs.push(doc.doc);
+          });
+
+          if (docs.length > 0) {
+            ee.emit('received', {
+              batch: batch++,
+              data: docs,
+              length: docs.length,
+              time: (new Date().getTime() - start) / 1000,
+              total: total += docs.length
+            });
+          }
+          callback();
+        }
+      }).catch(err => {
+        err = error.convertResponseError(err);
+        ee.emit('error', err);
+        hasErrored = true;
+        callback();
+      });
+    },
+    function(callback) { callback(null, hasErrored || startKey == null); },
+    function() { ee.emit('finished', { total: total }); }
+  );
+
+  return ee;
+};

package/.scannerwork/scanner-report/source-8.txt

@@ -0,0 +1,54 @@
+// Copyright © 2017, 2018 IBM Corp. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+'use strict';
+
+/**
+ * Utility methods for the command line interface.
+ * @module cliutils
+ * @see module:cliutils
+ */
+
+const url = require('url');
+
+module.exports = {
+
+  /**
+   * Combine a base URL and a database name, ensuring at least single slash
+   * between root and database name. This allows users to have Couch behind
+   * proxies that mount Couch's / endpoint at some other mount point.
+   * @param {string} root - root URL
+   * @param {string} databaseName - database name
+   * @return concatenated URL.
+   *
+   * @private
+   */
+  databaseUrl: function databaseUrl(root, databaseName) {
+    if (!root.endsWith('/')) {
+      root = root + '/';
+    }
+    return new url.URL(encodeURIComponent(databaseName), root).toString();
+  },
+
+  /**
+   * Generate CLI argument usage text.
+   *
+   * @param {string} description - argument description.
+   * @param {string} defaultValue - default argument value.
+   *
+   * @private
+   */
+  getUsage: function getUsage(description, defaultValue) {
+    return `${description} ${defaultValue !== undefined ? ` (default: ${defaultValue})` : ''}`;
+  }
+};

package/.scannerwork/scanner-report/source-9.txt

@@ -1,5 +1,4 @@
-#!/usr/bin/env node
-// Copyright © 2017, 2021 IBM Corp. All rights reserved.
+// Copyright © 2017, 2018 IBM Corp. All rights reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -14,43 +13,19 @@
 // limitations under the License.
 'use strict';
 
-const error = require('../includes/error.js');
-const cliutils = require('../includes/cliutils.js');
-const couchbackup = require('../app.js');
-const parser = require('../includes/parser.js');
-const debug = require('debug');
-const restoreDebug = debug('couchbackup:restore');
-const restoreBatchDebug = debug('couchbackup:restore:batch');
+module.exports = function(db, options, readstream, ee, callback) {
+  const liner = require('../includes/liner.js')();
+  const writer = require('../includes/writer.js')(db, options.bufferSize, options.parallelism, ee);
 
-restoreDebug.enabled = true;
+  // pipe the input to the output, via transformation functions
+  readstream
+    .pipe(liner) // transform the input stream into per-line
+    .on('error', function(err) {
+      // Forward the error to the writer event emitter where we already have
+      // listeners on for handling errors
+      writer.emit('error', err);
+    })
+    .pipe(writer); // transform the data
 
-const program = parser.parseRestoreArgs();
-const databaseUrl = cliutils.databaseUrl(program.url, program.db);
-const opts = {
-  bufferSize: program.bufferSize,
-  parallelism: program.parallelism,
-  requestTimeout: program.requestTimeout,
-  iamApiKey: program.iamApiKey,
-  iamTokenUrl: program.iamTokenUrl
+  callback(null, writer);
 };
-
-// log configuration to console
-console.error('='.repeat(80));
-console.error('Performing restore on ' + databaseUrl.replace(/\/\/.+@/g, '//****:****@') + ' using configuration:');
-console.error(JSON.stringify(opts, null, 2).replace(/"iamApiKey": "[^"]+"/, '"iamApiKey": "****"'));
-console.error('='.repeat(80));
-
-restoreBatchDebug.enabled = !program.quiet;
-
-return couchbackup.restore(
-  process.stdin, // restore from stdin
-  databaseUrl,
-  opts,
-  error.terminationCallback
-).on('restored', function(obj) {
-  restoreBatchDebug('restored', obj.total);
-}).on('error', function(e) {
-  restoreDebug('ERROR', e);
-}).on('finished', function(obj) {
-  restoreDebug('finished', obj);
-});