@cloudant/couchbackup 2.9.15 → 2.9.16-SNAPSHOT.179

package/.scannerwork/scanner-report/source-59.txt

@@ -1,4 +1,4 @@
-// Copyright © 2017, 2021 IBM Corp. All rights reserved.
+// Copyright © 2018, 2023 IBM Corp. All rights reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -11,168 +11,65 @@
 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 // See the License for the specific language governing permissions and
 // limitations under the License.
+
+/* global describe it */
 'use strict';
 
-const pkg = require('../package.json');
-const stream = require('stream');
-const { CloudantV1, CouchdbSessionAuthenticator } = require('@ibm-cloud/cloudant');
-const { IamAuthenticator, NoAuthAuthenticator } = require('ibm-cloud-sdk-core');
-const retryPlugin = require('retry-axios');
+const fs = require('fs');
+const { once } = require('node:events');
+const readline = require('readline');
+const u = require('./citestutils.js');
+const uuid = require('uuid').v4;
 
-const userAgent = 'couchbackup-cloudant/' + pkg.version + ' (Node.js ' +
-      process.version + ')';
+const params = { useApi: true };
 
-// Class for streaming _changes error responses into
-// In general the response is a small error/reason JSON object
-// so it is OK to have this in memory.
-class ResponseWriteable extends stream.Writable {
-  constructor(options) {
-    super(options);
-    this.data = [];
-  }
+describe(u.scenario('Concurrent database backups', params), function() {
+  it('should run concurrent API database backups correctly #slower', async function() {
+    // Allow up to 900 s to backup and compare (it should be much faster)!
+    u.setTimeout(this, 900);
 
-  _write(chunk, encoding, callback) {
-    this.data.push(chunk);
-    callback();
-  }
+    const checkForEmptyBatches = async function(fileName) {
+      let foundEmptyBatch = false;
 
-  stringBody() {
-    return Buffer.concat(this.data).toString();
-  }
-}
+      const rd = readline.createInterface({
+        input: fs.createReadStream(fileName),
+        output: fs.createWriteStream('/dev/null'),
+        terminal: false
+      });
 
-// An interceptor function to help augment error bodies with a little
-// extra information so we can continue to use consistent messaging
-// after the ugprade to @ibm-cloud/cloudant
-const errorHelper = async function(err) {
-  let method;
-  let requestUrl;
-  if (err.response) {
-    if (err.response.config.url) {
-      requestUrl = err.response.config.url;
-      method = err.response.config.method;
-    }
-    // Override the status text with an improved message
-    let errorMsg = `${err.response.status} ${err.response.statusText || ''}: ` +
-    `${method} ${requestUrl}`;
-    if (err.response.data) {
-      // Check if we have a JSON response and try to get the error/reason
-      if (err.response.headers['content-type'] === 'application/json') {
-        if (!err.response.data.error && err.response.data.pipe) {
-          // If we didn't find a JSON object with `error` then we might have a stream response.
-          // Detect the stream by the presence of `pipe` and use it to get the body and parse
-          // the error information.
-          const p = new Promise((resolve, reject) => {
-            const errorBody = new ResponseWriteable();
-            err.response.data.pipe(errorBody)
-              .on('finish', () => { resolve(JSON.parse(errorBody.stringBody())); })
-              .on('error', () => { reject(err); });
-          });
-          // Replace the stream on the response with the parsed object
-          err.response.data = await p;
-        }
-        // Append the error/reason if available
-        if (err.response.data.error) {
-          // Override the status text with our more complete message
-          errorMsg += ` - Error: ${err.response.data.error}`;
-          if (err.response.data.reason) {
-            errorMsg += `, Reason: ${err.response.data.reason}`;
-          }
+      rd.on('line', function(line) {
+        if (JSON.parse(line).length === 0) {
+          // Note: Empty batch arrays indicate that the running backup is
+          // incorrectly sharing a log file with another ongoing backup job.
+          foundEmptyBatch = true;
         }
-      } else {
-        errorMsg += err.response.data;
-      }
-      // Set a new message for use by the node-sdk-core
-      // We use the errors array because it gets processed
-      // ahead of all other service errors.
-      err.response.data.errors = [{ message: errorMsg }];
-    }
-  } else if (err.request) {
-    if (!err.message.includes(err.config.url)) {
-      // Augment the message with the URL and method
-      // but don't do it again if we already have the URL.
-      err.message = `${err.message}: ${err.config.method} ${err.config.url}`;
-    }
-  }
-  return Promise.reject(err);
-};
-
-module.exports = {
-  client: function(rawUrl, opts) {
-    const url = new URL(rawUrl);
-    // Split the URL to separate service from database
-    // Use origin as the "base" to remove auth elements
-    const actUrl = new URL(url.pathname.substring(0, url.pathname.lastIndexOf('/')), url.origin);
-    const dbName = url.pathname.substring(url.pathname.lastIndexOf('/') + 1);
-    let authenticator;
-    // Default to cookieauth unless an IAM key is provided
-    if (opts.iamApiKey) {
-      const iamAuthOpts = { apikey: opts.iamApiKey };
-      if (opts.iamTokenUrl) {
-        iamAuthOpts.url = opts.iamTokenUrl;
-      }
-      authenticator = new IamAuthenticator(iamAuthOpts);
-    } else if (url.username) {
-      authenticator = new CouchdbSessionAuthenticator({
-        username: decodeURIComponent(url.username),
-        password: decodeURIComponent(url.password)
       });
-    } else {
-      authenticator = new NoAuthAuthenticator();
-    }
-    const serviceOpts = {
-      authenticator: authenticator,
-      timeout: opts.requestTimeout,
-      // Axios performance options
-      maxContentLength: -1
-    };
 
-    const service = new CloudantV1(serviceOpts);
-    // Configure retries
-    const maxRetries = 2; // for 3 total attempts
-    service.getHttpClient().defaults.raxConfig = {
-      // retries for status codes
-      retry: maxRetries,
-      // retries for non-response e.g. ETIMEDOUT
-      noResponseRetries: maxRetries,
-      backoffType: 'exponential',
-      httpMethodsToRetry: ['GET', 'HEAD', 'POST'],
-      statusCodesToRetry: [
-        [429, 429],
-        [500, 599]
-      ],
-      shouldRetry: err => {
-        const cfg = retryPlugin.getConfig(err);
-        // cap at max retries regardless of response/non-response type
-        if (cfg.currentRetryAttempt >= maxRetries) {
-          return false;
+      rd.on('close', function() {
+        if (foundEmptyBatch) {
+          return Promise.reject(new Error(`Log file '${fileName}' contains empty batches`));
         } else {
-          return retryPlugin.shouldRetryRequest(err);
+          return Promise.resolve();
         }
-      },
-      instance: service.getHttpClient()
+      });
+    };
+
+    const backupPromise = async function() {
+      const actualBackup = `./${uuid()}`;
+      const output = fs.createWriteStream(actualBackup);
+      return once(output, 'open').then(() => {
+        return u.testBackup(params, 'largedb1g', output);
+      }).then(() => {
+        return checkForEmptyBatches(actualBackup);
+      });
     };
-    retryPlugin.attach(service.getHttpClient());
 
-    service.setServiceUrl(actUrl.toString());
-    if (authenticator instanceof CouchdbSessionAuthenticator) {
-      // Awkward workaround for known Couch issue with compression on _session requests
-      // It is not feasible to disable compression on all requests with the amount of
-      // data this lib needs to move, so override the property in the tokenManager instance.
-      authenticator.tokenManager.requestWrapperInstance.compressRequestData = false;
-    }
-    if (authenticator.tokenManager && authenticator.tokenManager.requestWrapperInstance) {
-      authenticator.tokenManager.requestWrapperInstance.axiosInstance.interceptors.response.use(null, errorHelper);
-    }
-    // Add error interceptors to put URLs in error messages
-    service.getHttpClient().interceptors.response.use(null, errorHelper);
+    // [1] Run 'largedb1g' database backup
+    const backup1 = backupPromise();
 
-    // Add request interceptor to add user-agent (adding it with custom request headers gets overwritten)
-    service.getHttpClient().interceptors.request.use(function(requestConfig) {
-      requestConfig.headers['User-Agent'] = userAgent;
-      return requestConfig;
-    }, null);
+    // [2] Run 'largedb1g' database backup
+    const backup2 = backupPromise();
 
-    return { service: service, db: dbName, url: actUrl.toString() };
-  }
-};
+    return Promise.all([backup1, backup2]);
+  });
+});

package/.scannerwork/scanner-report/source-6.txt

@@ -1,102 +1,111 @@
-<testsuites name="test-iam">
-  <testsuite name="Basic backup and restore using API" tests="3" errors="0" failures="0" skipped="0" timestamp="2023-12-04T15:06:22" time="4.609">
-    <testcase classname="test-iam.Basic backup and restore using API" name="should backup animaldb to a file correctly" time="1.06">
-    </testcase>
-    <testcase classname="test-iam.Basic backup and restore using API" name="should restore animaldb to a database correctly" time="1.802">
-    </testcase>
-    <testcase classname="test-iam.Basic backup and restore using API" name="should execute a shallow mode backup successfully" time="0.613">
-    </testcase>
-  </testsuite>
-  <testsuite name="Basic backup and restore using API Buffer size tests using API" tests="3" errors="0" failures="0" skipped="0" timestamp="2023-12-04T15:06:26" time="10.256">
-    <testcase classname="test-iam.Basic backup and restore using API Buffer size tests using API" name="should backup/restore animaldb with the same buffer size" time="2.612">
-    </testcase>
-    <testcase classname="test-iam.Basic backup and restore using API Buffer size tests using API" name="should backup/restore animaldb with backup buffer &gt; restore buffer" time="3.43">
-    </testcase>
-    <testcase classname="test-iam.Basic backup and restore using API Buffer size tests using API" name="should backup/restore animaldb with backup buffer &lt; restore buffer" time="3.411">
-    </testcase>
-  </testsuite>
-  <testsuite name="Basic backup and restore using CLI" tests="3" errors="0" failures="0" skipped="0" timestamp="2023-12-04T15:06:36" time="6.078">
-    <testcase classname="test-iam.Basic backup and restore using CLI" name="should backup animaldb to a file correctly" time="1.611">
-    </testcase>
-    <testcase classname="test-iam.Basic backup and restore using CLI" name="should restore animaldb to a database correctly" time="2.412">
-    </testcase>
-    <testcase classname="test-iam.Basic backup and restore using CLI" name="should execute a shallow mode backup successfully" time="1.265">
-    </testcase>
-  </testsuite>
-  <testsuite name="Basic backup and restore using CLI Buffer size tests using CLI" tests="3" errors="0" failures="0" skipped="0" timestamp="2023-12-04T15:06:43" time="14.512">
-    <testcase classname="test-iam.Basic backup and restore using CLI Buffer size tests using CLI" name="should backup/restore animaldb with the same buffer size" time="3.796">
-    </testcase>
-    <testcase classname="test-iam.Basic backup and restore using CLI Buffer size tests using CLI" name="should backup/restore animaldb with backup buffer &gt; restore buffer" time="4.754">
-    </testcase>
-    <testcase classname="test-iam.Basic backup and restore using CLI Buffer size tests using CLI" name="should backup/restore animaldb with backup buffer &lt; restore buffer" time="4.825">
-    </testcase>
-  </testsuite>
-  <testsuite name="Compression tests using API" tests="3" errors="0" failures="0" skipped="0" timestamp="2023-12-04T15:06:57" time="5.518">
-    <testcase classname="test-iam.Compression tests using API" name="should backup animaldb to a compressed file" time="1.187">
-    </testcase>
-    <testcase classname="test-iam.Compression tests using API" name="should backup and restore animaldb via a compressed file" time="1.382">
-    </testcase>
-    <testcase classname="test-iam.Compression tests using API" name="should backup and restore animaldb via a compressed stream" time="2.159">
-    </testcase>
-  </testsuite>
-  <testsuite name="Compression tests using CLI" tests="3" errors="0" failures="0" skipped="0" timestamp="2023-12-04T15:07:03" time="8.324">
-    <testcase classname="test-iam.Compression tests using CLI" name="should backup animaldb to a compressed file" time="1.847">
-    </testcase>
-    <testcase classname="test-iam.Compression tests using CLI" name="should backup and restore animaldb via a compressed file" time="2.808">
-    </testcase>
-    <testcase classname="test-iam.Compression tests using CLI" name="should backup and restore animaldb via a compressed stream" time="2.873">
-    </testcase>
-  </testsuite>
-  <testsuite name="End to end backup and restore using API" tests="2" errors="0" failures="0" skipped="0" timestamp="2023-12-04T15:07:11" time="313.889">
-    <testcase classname="test-iam.End to end backup and restore using API" name="should backup and restore animaldb" time="2.395">
-    </testcase>
-    <testcase classname="test-iam.End to end backup and restore using API" name="should backup and restore largedb1g #slow" time="310.737">
-    </testcase>
-  </testsuite>
-  <testsuite name="End to end backup and restore using CLI" tests="2" errors="0" failures="0" skipped="0" timestamp="2023-12-04T15:12:25" time="512.309">
-    <testcase classname="test-iam.End to end backup and restore using CLI" name="should backup and restore animaldb" time="2.913">
-    </testcase>
-    <testcase classname="test-iam.End to end backup and restore using CLI" name="should backup and restore largedb1g #slow" time="508.647">
-    </testcase>
-  </testsuite>
-  <testsuite name="Encryption tests" tests="1" errors="0" failures="0" skipped="0" timestamp="2023-12-04T15:20:57" time="3.904">
-    <testcase classname="test-iam.Encryption tests" name="should backup and restore animaldb via an encrypted file" time="3.64">
-    </testcase>
-  </testsuite>
-  <testsuite name="Write error tests" tests="1" errors="0" failures="0" skipped="0" timestamp="2023-12-04T15:21:01" time="0.27">
-    <testcase classname="test-iam.Write error tests" name="calls callback with error set when stream is not writeable" time="0.014">
-    </testcase>
-  </testsuite>
-  <testsuite name="Event tests" tests="2" errors="0" failures="0" skipped="0" timestamp="2023-12-04T15:21:01" time="2.2">
-    <testcase classname="test-iam.Event tests" name="should get a finished event when using stdout" time="0.858">
-    </testcase>
-    <testcase classname="test-iam.Event tests" name="should get a finished event when using file output" time="0.818">
-    </testcase>
-  </testsuite>
-  <testsuite name="Resume tests using API" tests="3" errors="0" failures="0" skipped="0" timestamp="2023-12-04T15:21:03" time="5.191">
-    <testcase classname="test-iam.Resume tests using API" name="should create a log file" time="0.942">
-    </testcase>
-    <testcase classname="test-iam.Resume tests using API" name="should restore corrupted animaldb to a database correctly" time="1.759">
-    </testcase>
-    <testcase classname="test-iam.Resume tests using API" name="should restore resumed animaldb with blank line to a database correctly" time="1.699">
-    </testcase>
-  </testsuite>
-  <testsuite name="Resume tests using CLI" tests="3" errors="0" failures="0" skipped="0" timestamp="2023-12-04T15:21:09" time="7.244">
-    <testcase classname="test-iam.Resume tests using CLI" name="should create a log file" time="1.801">
-    </testcase>
-    <testcase classname="test-iam.Resume tests using CLI" name="should restore corrupted animaldb to a database correctly" time="2.319">
-    </testcase>
-    <testcase classname="test-iam.Resume tests using CLI" name="should restore resumed animaldb with blank line to a database correctly" time="2.33">
-    </testcase>
-  </testsuite>
-  <testsuite name="Resume tests" tests="2" errors="0" failures="0" skipped="0" timestamp="2023-12-04T15:21:16" time="33.868">
-    <testcase classname="test-iam.Resume tests" name="should correctly backup and restore backup10m" time="16.705">
-    </testcase>
-    <testcase classname="test-iam.Resume tests" name="should correctly backup and restore backup10m using --output" time="16.625">
-    </testcase>
-  </testsuite>
-  <testsuite name="Longer spool changes checks" tests="1" errors="0" failures="0" skipped="0" timestamp="2023-12-04T15:21:50" time="47.758">
-    <testcase classname="test-iam.Longer spool changes checks" name="#slow should keep collecting changes (25M)" time="47.488">
-    </testcase>
-  </testsuite>
-</testsuites>
+// Copyright © 2017, 2022 IBM Corp. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+'use strict';
+
+const fs = require('fs');
+const liner = require('./liner.js');
+const change = require('./change.js');
+const error = require('./error.js');
+const debug = require('debug')('couchbackup:spoolchanges');
+
+/**
+ * Write log file for all changes from a database, ready for downloading
+ * in batches.
+ *
+ * @param {string} dbUrl - URL of database
+ * @param {string} log - path to log file to use
+ * @param {number} bufferSize - the number of changes per batch/log line
+ * @param {function(err)} callback - a callback to run on completion
+ */
+module.exports = function(db, log, bufferSize, ee, callback) {
+  // list of document ids to process
+  const buffer = [];
+  let batch = 0;
+  let lastSeq = null;
+  const logStream = fs.createWriteStream(log);
+  let pending = 0;
+  // The number of changes to fetch per request
+  const limit = 100000;
+
+  // send documents ids to the queue in batches of bufferSize + the last batch
+  const processBuffer = function(lastOne) {
+    if (buffer.length >= bufferSize || (lastOne && buffer.length > 0)) {
+      debug('writing', buffer.length, 'changes to the backup file');
+      const b = { docs: buffer.splice(0, bufferSize), batch: batch };
+      logStream.write(':t batch' + batch + ' ' + JSON.stringify(b.docs) + '\n');
+      ee.emit('changes', batch);
+      batch++;
+    }
+  };
+
+  // called once per received change
+  const onChange = function(c) {
+    if (c) {
+      if (c.error) {
+        ee.emit('error', new error.BackupError('InvalidChange', `Received invalid change: ${c}`));
+      } else if (c.changes) {
+        const obj = { id: c.id };
+        buffer.push(obj);
+        processBuffer(false);
+      } else if (c.last_seq) {
+        lastSeq = c.last_seq;
+        pending = c.pending;
+      }
+    }
+  };
+
+  function getChanges(since = 0) {
+    debug('making changes request since ' + since);
+    return db.service.postChangesAsStream({ db: db.db, since: since, limit: limit, seqInterval: limit })
+      .then(response => {
+        response.result.pipe(liner())
+          .on('error', function(err) {
+            logStream.end();
+            callback(err);
+          })
+          .pipe(change(onChange))
+          .on('error', function(err) {
+            logStream.end();
+            callback(err);
+          })
+          .on('finish', function() {
+            processBuffer(true);
+            if (!lastSeq) {
+              logStream.end();
+              debug('changes request terminated before last_seq was sent');
+              callback(new error.BackupError('SpoolChangesError', 'Changes request terminated before last_seq was sent'));
+            } else {
+              debug(`changes request completed with last_seq: ${lastSeq} and ${pending} changes pending.`);
+              if (pending > 0) {
+                // Return the next promise
+                return getChanges(lastSeq);
+              } else {
+                debug('finished streaming database changes');
+                logStream.end(':changes_complete ' + lastSeq + '\n', 'utf8', callback);
+              }
+            }
+          });
+      })
+      .catch(err => {
+        logStream.end();
+        if (err.status && err.status >= 400) {
+          callback(error.convertResponseError(err));
+        } else if (err.name !== 'SpoolChangesError') {
+          callback(new error.BackupError('SpoolChangesError', `Failed changes request - ${err.message}`));
+        }
+      });
+  }
+
+  getChanges();
+};