@cloudant/couchbackup 2.9.17 → 2.10.0-206

package/includes/logfilegetbatches.js

@@ -1,4 +1,4 @@
-// Copyright © 2017 IBM Corp. All rights reserved.
+// Copyright © 2017, 2023 IBM Corp. All rights reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -13,63 +13,28 @@
 // limitations under the License.
 'use strict';
 
-const fs = require('fs');
-const stream = require('stream');
-const liner = require('./liner.js');
-
-const onLine = function(onCommand, batches) {
-  const change = new stream.Transform({ objectMode: true });
-  change._transform = function(line, encoding, done) {
-    if (line && line[0] === ':') {
-      const obj = {
-        command: null,
-        batch: null,
-        docs: []
-      };
-
-      let matches;
-
-      // extract command
-      matches = line.match(/^:([a-z_]+) ?/);
-      if (matches) {
-        obj.command = matches[1];
-      }
-
-      // extract batch
-      matches = line.match(/ batch([0-9]+)/);
-      if (matches) {
-        obj.batch = parseInt(matches[1]);
-      }
-
-      // if this is one we want
-      if (obj.command === 't' && batches.indexOf(obj.batch) > -1) {
-        const json = line.replace(/^.* batch[0-9]+ /, '').trim();
-        obj.docs = JSON.parse(json);
-        onCommand(obj);
-      }
-    }
-    done();
-  };
-  return change;
-};
-
-module.exports = function(log, batches, callback) {
-  // our sense of state
-  const retval = { };
-
-  // called with each line from the log file
-  const onCommand = function(obj) {
-    retval[obj.batch] = obj;
-  };
-
-  // stream through the previous log file
-  fs.createReadStream(log)
-    .pipe(liner())
-    .pipe(onLine(onCommand, batches))
-    .on('error', function(err) {
-      callback(err);
-    })
-    .on('finish', function() {
-      callback(null, retval);
-    });
+const fs = require('node:fs');
+const { LogMapper } = require('./backupMappings.js');
+const { Liner } = require('./liner.js');
+const { FilterStream, MappingStream } = require('./transforms.js');
+
+/**
+ * Return an array of streams that when pipelined will produce
+ * pending backup batches from a log file.
+ *
+ * @param {string} log - log file name
+ * @param {Map} batches - a log summary batches Map of pending batch numbers
+ * @returns a log summary object
+ */
+module.exports = function(log, batches) {
+  const logMapper = new LogMapper();
+  return [
+    fs.createReadStream(log), // log file
+    new Liner(), // split it into lines
+    new MappingStream(logMapper.logLineToBackupBatch), // parse line to a backup batch
+    new FilterStream((metadata) => {
+      // delete returns true if the key exists, false otherwise
+      return batches.delete(metadata.batch);
+    }) // filter out already done batches
+  ];
 };

package/includes/logfilesummary.js

@@ -1,4 +1,4 @@
-// Copyright © 2017 IBM Corp. All rights reserved.
+// Copyright © 2017, 2023 IBM Corp. All rights reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -13,80 +13,50 @@
 // limitations under the License.
 'use strict';
 
-const fs = require('fs');
-const stream = require('stream');
-const liner = require('./liner.js');
-
-const onLine = function(onCommand, getDocs) {
-  const change = new stream.Transform({ objectMode: true });
-
-  change._transform = function(line, encoding, done) {
-    if (line && line[0] === ':') {
-      const obj = {
-        command: null,
-        batch: null,
-        docs: []
-      };
-
-      let matches;
-
-      // extract command
-      matches = line.match(/^:([a-z_]+) ?/);
-      if (matches) {
-        obj.command = matches[1];
-      }
-
-      // extract batch
-      matches = line.match(/ batch([0-9]+)/);
-      if (matches) {
-        obj.batch = parseInt(matches[1]);
-      }
-
-      // extract doc ids
-      if (getDocs && obj.command === 't') {
-        const json = line.replace(/^.* batch[0-9]+ /, '').trim();
-        obj.docs = JSON.parse(json);
-      }
-      onCommand(obj);
-    }
-    done();
-  };
-  return change;
-};
+const { createReadStream } = require('node:fs');
+const { Writable } = require('node:stream');
+const { pipeline } = require('node:stream/promises');
+const { Liner } = require('./liner.js');
+const { LogMapper } = require('./backupMappings.js');
+const { MappingStream } = require('./transforms.js');
 
 /**
  * Generate a list of remaining batches from a download file.
+ * Creates a summary containing a changesComplete boolean for
+ * if the :changes_complete log file entry was found and a map
+ * of pending batch numbers that have yet to be backed up
+ * (i.e. the difference of :t and :d log file entries).
  *
  * @param {string} log - log file name
- * @param {function} callback - callback with err, {changesComplete: N, batches: N}.
- *  changesComplete signifies whether the log file appeared to
- *  have completed reading the changes feed (contains :changes_complete).
- *  batches are remaining batch IDs for download.
+ * @returns a log summary object
  */
-module.exports = function(log, callback) {
-  // our sense of state
-  const state = {
-
-  };
-  let changesComplete = false;
-
-  // called with each line from the log file
-  const onCommand = function(obj) {
-    if (obj.command === 't') {
-      state[obj.batch] = true;
-    } else if (obj.command === 'd') {
-      delete state[obj.batch];
-    } else if (obj.command === 'changes_complete') {
-      changesComplete = true;
-    }
-  };
-
-  // stream through the previous log file
-  fs.createReadStream(log)
-    .pipe(liner())
-    .pipe(onLine(onCommand, false))
-    .on('finish', function() {
-      const obj = { changesComplete: changesComplete, batches: state };
-      callback(null, obj);
-    });
+module.exports = async function(log) {
+  const logMapper = new LogMapper();
+  const state = { changesComplete: false, batches: new Map() };
+
+  await pipeline(
+    createReadStream(log), // read the log file
+    new Liner(), // split it into lines
+    new MappingStream(logMapper.logLineToMetadata), // parse line to metadata
+    new Writable({
+      objectMode: true,
+      write: (metadata, encoding, callback) => {
+        switch (metadata.command) {
+          case 't':
+            state.batches.set(metadata.batch, true);
+            break;
+          case 'd':
+            state.batches.delete(metadata.batch);
+            break;
+          case 'changes_complete':
+            state.changesComplete = true;
+            break;
+          default:
+            break;
+        }
+        callback();
+      }
+    }) // Save the done batch number in an array
+  );
+  return state;
 };

package/includes/parser.js

@@ -1,4 +1,4 @@
-// Copyright © 2017, 2021 IBM Corp. All rights reserved.
+// Copyright © 2017, 2024 IBM Corp. All rights reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -124,6 +124,6 @@ function parseRestoreArgs() {
 }
 
 module.exports = {
-  parseBackupArgs: parseBackupArgs,
-  parseRestoreArgs: parseRestoreArgs
+  parseBackupArgs,
+  parseRestoreArgs
 };

package/includes/request.js

@@ -14,68 +14,44 @@
 'use strict';
 
 const pkg = require('../package.json');
-const stream = require('stream');
 const { CloudantV1, CouchdbSessionAuthenticator } = require('@ibm-cloud/cloudant');
 const { IamAuthenticator, NoAuthAuthenticator } = require('ibm-cloud-sdk-core');
 const retryPlugin = require('retry-axios');
+const debug = require('debug')('couchbackup:request');
 
 const userAgent = 'couchbackup-cloudant/' + pkg.version + ' (Node.js ' +
       process.version + ')';
 
-// Class for streaming _changes error responses into
-// In general the response is a small error/reason JSON object
-// so it is OK to have this in memory.
-class ResponseWriteable extends stream.Writable {
-  constructor(options) {
-    super(options);
-    this.data = [];
-  }
-
-  _write(chunk, encoding, callback) {
-    this.data.push(chunk);
-    callback();
-  }
-
-  stringBody() {
-    return Buffer.concat(this.data).toString();
-  }
-}
-
 // An interceptor function to help augment error bodies with a little
 // extra information so we can continue to use consistent messaging
 // after the ugprade to @ibm-cloud/cloudant
 const errorHelper = async function(err) {
+  debug('Entering error helper interceptor');
   let method;
   let requestUrl;
   if (err.response) {
+    debug('Error has a response');
     if (err.response.config.url) {
+      debug('Getting request URL and method for error');
       requestUrl = err.response.config.url;
       method = err.response.config.method;
     }
+    debug('Applying response error message with status, url, and method');
     // Override the status text with an improved message
     let errorMsg = `${err.response.status} ${err.response.statusText || ''}: ` +
     `${method} ${requestUrl}`;
     if (err.response.data) {
+      debug('Found response data');
       // Check if we have a JSON response and try to get the error/reason
       if (err.response.headers['content-type'] === 'application/json') {
-        if (!err.response.data.error && err.response.data.pipe) {
-          // If we didn't find a JSON object with `error` then we might have a stream response.
-          // Detect the stream by the presence of `pipe` and use it to get the body and parse
-          // the error information.
-          const p = new Promise((resolve, reject) => {
-            const errorBody = new ResponseWriteable();
-            err.response.data.pipe(errorBody)
-              .on('finish', () => { resolve(JSON.parse(errorBody.stringBody())); })
-              .on('error', () => { reject(err); });
-          });
-          // Replace the stream on the response with the parsed object
-          err.response.data = await p;
-        }
+        debug('Response data is JSON');
         // Append the error/reason if available
         if (err.response.data.error) {
+          debug('Augmenting error message with error property');
           // Override the status text with our more complete message
           errorMsg += ` - Error: ${err.response.data.error}`;
           if (err.response.data.reason) {
+            debug('Augmenting error message with reason property');
             errorMsg += `, Reason: ${err.response.data.reason}`;
           }
         }
@@ -88,91 +64,104 @@ const errorHelper = async function(err) {
       err.response.data.errors = [{ message: errorMsg }];
     }
   } else if (err.request) {
+    debug('Error did not include a response');
     if (!err.message.includes(err.config.url)) {
+      debug('Augmenting request error message with URL and method');
       // Augment the message with the URL and method
       // but don't do it again if we already have the URL.
       err.message = `${err.message}: ${err.config.method} ${err.config.url}`;
+    } else {
+      debug('Request error message already augmented');
     }
   }
   return Promise.reject(err);
 };
 
-module.exports = {
-  client: function(rawUrl, opts) {
-    const url = new URL(rawUrl);
-    // Split the URL to separate service from database
-    // Use origin as the "base" to remove auth elements
-    const actUrl = new URL(url.pathname.substring(0, url.pathname.lastIndexOf('/')), url.origin);
-    const dbName = url.pathname.substring(url.pathname.lastIndexOf('/') + 1);
-    let authenticator;
-    // Default to cookieauth unless an IAM key is provided
-    if (opts.iamApiKey) {
-      const iamAuthOpts = { apikey: opts.iamApiKey };
-      if (opts.iamTokenUrl) {
-        iamAuthOpts.url = opts.iamTokenUrl;
-      }
-      authenticator = new IamAuthenticator(iamAuthOpts);
-    } else if (url.username) {
-      authenticator = new CouchdbSessionAuthenticator({
-        username: decodeURIComponent(url.username),
-        password: decodeURIComponent(url.password)
-      });
-    } else {
-      authenticator = new NoAuthAuthenticator();
+function newSimpleClient(rawUrl, opts) {
+  const url = new URL(rawUrl);
+  // Split the URL to separate service from database
+  // Use origin as the "base" to remove auth elements
+  const actUrl = new URL(url.pathname.substring(0, url.pathname.lastIndexOf('/')), url.origin);
+  const dbName = url.pathname.substring(url.pathname.lastIndexOf('/') + 1);
+  let authenticator;
+  // Default to cookieauth unless an IAM key is provided
+  if (opts.iamApiKey) {
+    const iamAuthOpts = { apikey: opts.iamApiKey };
+    if (opts.iamTokenUrl) {
+      iamAuthOpts.url = opts.iamTokenUrl;
     }
-    const serviceOpts = {
-      authenticator: authenticator,
-      timeout: opts.requestTimeout,
-      // Axios performance options
-      maxContentLength: -1
-    };
-
-    const service = new CloudantV1(serviceOpts);
-    // Configure retries
-    const maxRetries = 2; // for 3 total attempts
-    service.getHttpClient().defaults.raxConfig = {
-      // retries for status codes
-      retry: maxRetries,
-      // retries for non-response e.g. ETIMEDOUT
-      noResponseRetries: maxRetries,
-      backoffType: 'exponential',
-      httpMethodsToRetry: ['GET', 'HEAD', 'POST'],
-      statusCodesToRetry: [
-        [429, 429],
-        [500, 599]
-      ],
-      shouldRetry: err => {
-        const cfg = retryPlugin.getConfig(err);
-        // cap at max retries regardless of response/non-response type
-        if (cfg.currentRetryAttempt >= maxRetries) {
-          return false;
-        } else {
-          return retryPlugin.shouldRetryRequest(err);
-        }
-      },
-      instance: service.getHttpClient()
-    };
-    retryPlugin.attach(service.getHttpClient());
+    authenticator = new IamAuthenticator(iamAuthOpts);
+  } else if (url.username) {
+    authenticator = new CouchdbSessionAuthenticator({
+      username: decodeURIComponent(url.username),
+      password: decodeURIComponent(url.password)
+    });
+  } else {
+    authenticator = new NoAuthAuthenticator();
+  }
+  const serviceOpts = {
+    authenticator,
+    timeout: opts.requestTimeout,
+    // Axios performance options
+    maxContentLength: -1
+  };
 
-    service.setServiceUrl(actUrl.toString());
-    if (authenticator instanceof CouchdbSessionAuthenticator) {
-      // Awkward workaround for known Couch issue with compression on _session requests
-      // It is not feasible to disable compression on all requests with the amount of
-      // data this lib needs to move, so override the property in the tokenManager instance.
-      authenticator.tokenManager.requestWrapperInstance.compressRequestData = false;
-    }
-    if (authenticator.tokenManager && authenticator.tokenManager.requestWrapperInstance) {
-      authenticator.tokenManager.requestWrapperInstance.axiosInstance.interceptors.response.use(null, errorHelper);
-    }
-    // Add error interceptors to put URLs in error messages
-    service.getHttpClient().interceptors.response.use(null, errorHelper);
+  const service = new CloudantV1(serviceOpts);
+  service.setServiceUrl(actUrl.toString());
+  if (authenticator instanceof CouchdbSessionAuthenticator) {
+    // Awkward workaround for known Couch issue with compression on _session requests
+    // It is not feasible to disable compression on all requests with the amount of
+    // data this lib needs to move, so override the property in the tokenManager instance.
+    authenticator.tokenManager.requestWrapperInstance.compressRequestData = false;
+  }
+  return { service, dbName, actUrl };
+}
 
-    // Add request interceptor to add user-agent (adding it with custom request headers gets overwritten)
-    service.getHttpClient().interceptors.request.use(function(requestConfig) {
-      requestConfig.headers['User-Agent'] = userAgent;
-      return requestConfig;
-    }, null);
+function newClient(rawUrl, opts) {
+  const { service, dbName, actUrl } = newSimpleClient(rawUrl, opts);
+  const authenticator = service.getAuthenticator();
+  // Configure retries
+  const maxRetries = 2; // for 3 total attempts
+  service.getHttpClient().defaults.raxConfig = {
+    // retries for status codes
+    retry: maxRetries,
+    // retries for non-response e.g. ETIMEDOUT
+    noResponseRetries: maxRetries,
+    backoffType: 'exponential',
+    httpMethodsToRetry: ['GET', 'HEAD', 'POST'],
+    statusCodesToRetry: [
+      [429, 429],
+      [500, 599]
+    ],
+    shouldRetry: err => {
+      const cfg = retryPlugin.getConfig(err);
+      // cap at max retries regardless of response/non-response type
+      if (cfg.currentRetryAttempt >= maxRetries) {
+        return false;
+      } else {
+        return retryPlugin.shouldRetryRequest(err);
+      }
+    },
+    instance: service.getHttpClient()
+  };
+  retryPlugin.attach(service.getHttpClient());
 
-    return { service: service, db: dbName, url: actUrl.toString() };
+  if (authenticator.tokenManager && authenticator.tokenManager.requestWrapperInstance) {
+    authenticator.tokenManager.requestWrapperInstance.axiosInstance.interceptors.response.use(null, errorHelper);
   }
+  // Add error interceptors to put URLs in error messages
+  service.getHttpClient().interceptors.response.use(null, errorHelper);
+
+  // Add request interceptor to add user-agent (adding it with custom request headers gets overwritten)
+  service.getHttpClient().interceptors.request.use(function(requestConfig) {
+    requestConfig.headers['User-Agent'] = userAgent;
+    return requestConfig;
+  }, null);
+
+  return { service, dbName, url: actUrl.toString() };
+}
+
+module.exports = {
+  newSimpleClient,
+  newClient
 };

package/includes/restore.js

@@ -1,4 +1,4 @@
-// Copyright © 2017, 2018 IBM Corp. All rights reserved.
+// Copyright © 2017, 2024 IBM Corp. All rights reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -13,19 +13,50 @@
 // limitations under the License.
 'use strict';
 
-module.exports = function(db, options, readstream, ee, callback) {
-  const liner = require('../includes/liner.js')();
-  const writer = require('../includes/writer.js')(db, options.bufferSize, options.parallelism, ee);
+const debug = require('debug')('couchbackup:restore');
+const { Liner } = require('../includes/liner.js');
+const { Restore } = require('../includes/restoreMappings.js');
+const { BatchingStream, MappingStream } = require('./transforms.js');
+const { Writable } = require('node:stream');
+const { pipeline } = require('node:stream/promises');
 
-  // pipe the input to the output, via transformation functions
-  readstream
-    .pipe(liner) // transform the input stream into per-line
-    .on('error', function(err) {
-      // Forward the error to the writer event emitter where we already have
-      // listeners on for handling errors
-      writer.emit('error', err);
-    })
-    .pipe(writer); // transform the data
+/**
+ * Function for performing a restore.
+ *
+ * @param {object} dbClient - object for connection to source database containing name, service and url
+ * @param {object} options - restore configuration
+ * @param {Readable} readstream - the backup file content
+ * @param {EventEmitter} ee - the user facing EventEmitter
+ * @returns a promise that resolves when the restore is complete or rejects if it errors
+ */
+module.exports = function(dbClient, options, readstream, ee) {
+  const restore = new Restore(dbClient);
+  const start = new Date().getTime(); // restore start time
+  let total = 0; // the total restored
 
-  callback(null, writer);
+  const output = new Writable({
+    objectMode: true,
+    write: (restoreBatch, encoding, cb) => {
+      debug(' restored ', restoreBatch.documents);
+      total += restoreBatch.documents;
+      const totalRunningTimeSec = (new Date().getTime() - start) / 1000;
+      try {
+        ee.emit('restored', { ...restoreBatch, total, time: totalRunningTimeSec });
+      } finally {
+        cb();
+      }
+    }
+  });
+
+  return pipeline(
+    readstream, // the backup file
+    new Liner(), // line by line
+    new MappingStream(restore.backupLineToDocsArray), // convert line to a docs array
+    new BatchingStream(options.bufferSize, true), // make new arrays of the correct buffer size
+    new MappingStream(restore.docsToRestoreBatch), // make a restore batch
+    new MappingStream(restore.pendingToRestored, options.parallelism), // do the restore at the desired level of concurrency
+    output // emit restored events
+  ).then(() => {
+    return { total };
+  });
 };