@cloudant/couchbackup 2.7.1-SNAPSHOT.58 → 2.7.1-SNAPSHOT.59

package/CHANGES.md

@@ -4,6 +4,8 @@
 - [IMPROVED] Added quiet option to backup and restore to suppress batch messages.
 - [IMPROVED] Added a preflight check for restore function to make sure that a target database is new and empty.
 - [IMPROVED] Added handling for errors reading log file.
+- [IMPROVED] Split changes spooling to improve reliability on databases with
+  millions of documents.
 
 # 2.7.0 (2021-09-14)
 - [UPGRADED] Cloudant client dependency from `@cloudant/cloudant` to `@ibm-cloud/cloudant`.

package/includes/spoolchanges.js

@@ -34,6 +34,9 @@ module.exports = function(db, log, bufferSize, ee, callback) {
   let batch = 0;
   let lastSeq = null;
   const logStream = fs.createWriteStream(log);
+  let pending = 0;
+  // The number of changes to fetch per request
+  const limit = 100000;
 
   // send documents ids to the queue in batches of bufferSize + the last batch
   const processBuffer = function(lastOne) {
@@ -57,36 +60,52 @@ module.exports = function(db, log, bufferSize, ee, callback) {
         processBuffer(false);
       } else if (c.last_seq) {
         lastSeq = c.last_seq;
+        pending = c.pending;
       }
     }
   };
 
-  // stream the changes feed to disk
-  db.service.postChangesAsStream({ db: db.db, seq_interval: 10000 }).then(response => {
-    response.result.pipe(liner())
-      .on('error', function(err) {
-        callback(err);
+  function getChanges(since = 0) {
+    debug('making changes request since ' + since);
+    return db.service.postChangesAsStream({ db: db.db, since: since, limit: limit, seq_interval: limit })
+      .then(response => {
+        response.result.pipe(liner())
+          .on('error', function(err) {
+            logStream.end();
+            callback(err);
+          })
+          .pipe(change(onChange))
+          .on('error', function(err) {
+            logStream.end();
+            callback(err);
+          })
+          .on('finish', function() {
+            processBuffer(true);
+            if (!lastSeq) {
+              logStream.end();
+              debug('changes request terminated before last_seq was sent');
+              callback(new error.BackupError('SpoolChangesError', 'Changes request terminated before last_seq was sent'));
+            } else {
+              debug(`changes request completed with last_seq: ${lastSeq} and ${pending} changes pending.`);
+              if (pending > 0) {
+                // Return the next promise
+                return getChanges(lastSeq);
+              } else {
+                debug('finished streaming database changes');
+                logStream.end(':changes_complete ' + lastSeq + '\n', 'utf8', callback);
+              }
+            }
+          });
       })
-      .pipe(change(onChange))
-      .on('error', function(err) {
-        callback(err);
-      })
-      .on('finish', function() {
-        processBuffer(true);
-        if (!lastSeq) {
-          logStream.end();
-          debug('changes request terminated before last_seq was sent');
-          callback(new error.BackupError('SpoolChangesError', 'Changes request terminated before last_seq was sent'));
-        } else {
-          debug('finished streaming database changes');
-          logStream.end(':changes_complete ' + lastSeq + '\n', 'utf8', callback);
+      .catch(err => {
+        logStream.end();
+        if (err.status && err.status >= 400) {
+          callback(error.convertResponseError(err));
+        } else if (err.name !== 'SpoolChangesError') {
+          callback(new error.BackupError('SpoolChangesError', `Failed changes request - ${err.message}`));
         }
       });
-  }).catch(err => {
-    if (err.status && err.status >= 400) {
-      callback(error.convertResponseError(err));
-    } else {
-      callback(new error.BackupError('SpoolChangesError', `Failed changes request - ${err.message}`));
-    }
-  });
+  }
+
+  getChanges();
 };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@cloudant/couchbackup",
-  "version": "2.7.1-SNAPSHOT.58",
+  "version": "2.7.1-SNAPSHOT.59",
   "description": "CouchBackup - command-line backup utility for Cloudant/CouchDB",
   "homepage": "https://github.com/cloudant/couchbackup",
   "repository": "https://github.com/cloudant/couchbackup.git",