@cloudant/couchbackup 2.9.9-SNAPSHOT.98 → 2.9.9

package/.scannerwork/scanner-report/source-63.txt

@@ -11,53 +11,65 @@
 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 // See the License for the specific language governing permissions and
 // limitations under the License.
-
-/* global describe it */
 'use strict';
 
 const fs = require('fs');
-const u = require('./citestutils.js');
-
-describe('Event tests', function() {
-  it('should get a finished event when using stdout', function(done) {
-    u.setTimeout(this, 40);
-    // Use the API so we can get events
-    const params = { useApi: true };
-    const backup = u.testBackup(params, 'animaldb', process.stdout, function(err) {
-      if (err) {
-        done(err);
+const stream = require('stream');
+const liner = require('./liner.js');
+
+const onLine = function(onCommand, batches) {
+  const change = new stream.Transform({ objectMode: true });
+  change._transform = function(line, encoding, done) {
+    if (line && line[0] === ':') {
+      const obj = {
+        command: null,
+        batch: null,
+        docs: []
+      };
+
+      let matches;
+
+      // extract command
+      matches = line.match(/^:([a-z_]+) ?/);
+      if (matches) {
+        obj.command = matches[1];
       }
-    });
-    backup.on('finished', function() {
-      try {
-        // Test will time out if the finished event is not emitted
-        done();
-      } catch (err) {
-        done(err);
+
+      // extract batch
+      matches = line.match(/ batch([0-9]+)/);
+      if (matches) {
+        obj.batch = parseInt(matches[1]);
       }
+
+      // if this is one we want
+      if (obj.command === 't' && batches.indexOf(obj.batch) > -1) {
+        const json = line.replace(/^.* batch[0-9]+ /, '').trim();
+        obj.docs = JSON.parse(json);
+        onCommand(obj);
+      }
+    }
+    done();
+  };
+  return change;
+};
+
+module.exports = function(log, batches, callback) {
+  // our sense of state
+  const retval = { };
+
+  // called with each line from the log file
+  const onCommand = function(obj) {
+    retval[obj.batch] = obj;
+  };
+
+  // stream through the previous log file
+  fs.createReadStream(log)
+    .pipe(liner())
+    .pipe(onLine(onCommand, batches))
+    .on('error', function(err) {
+      callback(err);
+    })
+    .on('finish', function() {
+      callback(null, retval);
     });
-  });
-  it('should get a finished event when using file output', function(done) {
-    u.setTimeout(this, 40);
-    // Use the API so we can get events
-    const params = { useApi: true };
-    const actualBackup = `./${this.fileName}`;
-    // Create a file and backup to it
-    const output = fs.createWriteStream(actualBackup);
-    output.on('open', function() {
-      const backup = u.testBackup(params, 'animaldb', output, function(err) {
-        if (err) {
-          done(err);
-        }
-      });
-      backup.on('finished', function() {
-        try {
-          // Test will time out if the finished event is not emitted
-          done();
-        } catch (err) {
-          done(err);
-        }
-      });
-    });
-  });
-});
+};

package/.scannerwork/scanner-report/source-64.txt

@@ -11,121 +11,31 @@
 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 // See the License for the specific language governing permissions and
 // limitations under the License.
-
-/* global describe it */
 'use strict';
 
-const fs = require('fs');
-const u = require('./citestutils.js');
+// stolen from http://strongloop.com/strongblog/practical-examples-of-the-new-node-js-streams-api/
+const stream = require('stream');
+
+module.exports = function(onChange) {
+  const change = new stream.Transform({ objectMode: true });
 
-[{ useApi: true }, { useApi: false }].forEach(function(params) {
-  describe(u.scenario('Basic backup and restore', params), function() {
-    it('should backup animaldb to a file correctly', function(done) {
-      // Allow up to 40 s to backup and compare (it should be much faster)!
-      u.setTimeout(this, 40);
-      const actualBackup = `./${this.fileName}`;
-      // Create a file and backup to it
-      const output = fs.createWriteStream(actualBackup);
-      output.on('open', function() {
-        u.testBackup(params, 'animaldb', output, function(err) {
-          if (err) {
-            done(err);
-          } else {
-            u.readSortAndDeepEqual(actualBackup, './test/fixtures/animaldb_expected.json', done);
-          }
-        });
-      });
-    });
+  change._transform = function(line, encoding, done) {
+    let obj = null;
 
-    it('should restore animaldb to a database correctly', function(done) {
-      // Allow up to 60 s to restore and compare (again it should be faster)!
-      u.setTimeout(this, 60);
-      const input = fs.createReadStream('./test/fixtures/animaldb_expected.json');
-      const dbName = this.dbName;
-      input.on('open', function() {
-        u.testRestore(params, input, dbName, function(err) {
-          if (err) {
-            done(err);
-          } else {
-            u.dbCompare('animaldb', dbName, done);
-          }
-        });
-      });
-    });
+    // one change per line - remove the trailing comma
+    line = line.trim().replace(/,$/, '');
 
-    it('should execute a shallow mode backup successfully', function(done) {
-      // Allow 30 s
-      u.setTimeout(this, 30);
-      const actualBackup = `./${this.fileName}`;
-      const output = fs.createWriteStream(actualBackup);
-      // Add the shallow mode option
-      const p = u.p(params, { opts: { mode: 'shallow' } });
-      output.on('open', function() {
-        u.testBackup(p, 'animaldb', output, function(err) {
-          if (err) {
-            done(err);
-          } else {
-            u.readSortAndDeepEqual(actualBackup, './test/fixtures/animaldb_expected_shallow.json', done);
-          }
-        });
-      });
-    });
+    // extract thee last_seq at the end of the changes feed
+    if (line.match(/^"last_seq":/)) {
+      line = '{' + line;
+    }
+    try {
+      obj = JSON.parse(line);
+    } catch (e) {
+    }
+    onChange(obj);
+    done();
+  };
 
-    describe(u.scenario('Buffer size tests', params), function() {
-      it('should backup/restore animaldb with the same buffer size', function(done) {
-        // Allow up to 60 s for backup and restore of animaldb
-        u.setTimeout(this, 60);
-        const actualBackup = `./${this.fileName}`;
-        const logFile = `./${this.fileName}` + '.log';
-        const p = u.p(params, { opts: { log: logFile, bufferSize: 1 } });
-        u.testBackupAndRestoreViaFile(p, 'animaldb', actualBackup, this.dbName, done);
-      });
-      it('should backup/restore animaldb with backup buffer > restore buffer', function(done) {
-        // Allow up to 60 s for backup and restore of animaldb
-        u.setTimeout(this, 60);
-        const actualBackup = `./${this.fileName}`;
-        const logFile = `./${this.fileName}` + '.log';
-        const dbName = this.dbName;
-        const p = u.p(params, { opts: { log: logFile, bufferSize: 2 } }); // backup
-        const q = u.p(params, { opts: { bufferSize: 1 } }); // restore
-        u.testBackupToFile(p, 'animaldb', actualBackup, function(err) {
-          if (err) {
-            done(err);
-          } else {
-            // restore
-            u.testRestoreFromFile(q, actualBackup, dbName, function(err) {
-              if (!err) {
-                u.dbCompare('animaldb', dbName, done);
-              } else {
-                done(err);
-              }
-            });
-          }
-        });
-      });
-      it('should backup/restore animaldb with backup buffer < restore buffer', function(done) {
-        // Allow up to 60 s for backup and restore of animaldb
-        u.setTimeout(this, 60);
-        const actualBackup = `./${this.fileName}`;
-        const logFile = `./${this.fileName}` + '.log';
-        const dbName = this.dbName;
-        const p = u.p(params, { opts: { log: logFile, bufferSize: 1 } }); // backup
-        const q = u.p(params, { opts: { bufferSize: 2 } }); // restore
-        u.testBackupToFile(p, 'animaldb', actualBackup, function(err) {
-          if (err) {
-            done(err);
-          } else {
-            // restore
-            u.testRestoreFromFile(q, actualBackup, dbName, function(err) {
-              if (!err) {
-                u.dbCompare('animaldb', dbName, done);
-              } else {
-                done(err);
-              }
-            });
-          }
-        });
-      });
-    });
-  });
-});
+  return change;
+};

package/.scannerwork/scanner-report/source-65.txt

@@ -1,4 +1,4 @@
-// Copyright © 2017, 2021 IBM Corp. All rights reserved.
+// Copyright © 2017, 2022 IBM Corp. All rights reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -11,137 +11,70 @@
 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 // See the License for the specific language governing permissions and
 // limitations under the License.
-
-/* global describe it beforeEach */
 'use strict';
 
-const assert = require('assert');
-const fs = require('fs');
-const nock = require('nock');
-const request = require('../includes/request.js');
-const writer = require('../includes/writer.js');
-const noopEmitter = new (require('events')).EventEmitter();
-const liner = require('../includes/liner.js');
-
-const longTestTimeout = 3000;
-
-// The writer expects a line-by-line stream so this utility function does that
-// processing for the tests (which normally happens in the internal restore
-// function).
-function testLinestream(fixture = './test/fixtures/animaldb_expected.json') {
-  return fs.createReadStream(fixture)
-    .pipe(liner());
-}
-
-describe('#unit Check database restore writer', function() {
-  const dbUrl = 'http://localhost:5984/animaldb';
-  const db = request.client(dbUrl, { parallelism: 1 });
-
-  beforeEach('Reset nocks', function() {
-    nock.cleanAll();
-  });
-
-  it('should complete successfully', function(done) {
-    nock(dbUrl)
-      .post('/_bulk_docs')
-      .reply(200, []); // success
-
-    testLinestream()
-      .pipe(writer(db, 500, 1, noopEmitter))
-      .on('error', function(err) {
-        done(err);
-      })
-      .on('finished', function(data) {
-        assert.strictEqual(data.total, 15);
-        assert.ok(nock.isDone());
-        done();
+const async = require('async');
+const error = require('./error.js');
+const events = require('events');
+
+module.exports = function(db, options) {
+  const ee = new events.EventEmitter();
+  const start = new Date().getTime();
+  let batch = 0;
+  let hasErrored = false;
+  let startKey = null;
+  let total = 0;
+
+  async.doUntil(
+    function(callback) {
+      // Note, include_docs: true is set automatically when using the
+      // fetch function.
+      const opts = { db: db.db, limit: options.bufferSize, includeDocs: true };
+
+      // To avoid double fetching a document solely for the purposes of getting
+      // the next ID to use as a startKey for the next page we instead use the
+      // last ID of the current page and append the lowest unicode sort
+      // character.
+      if (startKey) opts.startKey = `${startKey}\0`;
+      db.service.postAllDocs(opts).then(response => {
+        const body = response.result;
+        if (!body.rows) {
+          ee.emit('error', new error.BackupError(
+            'AllDocsError', 'ERROR: Invalid all docs response'));
+          callback();
+        } else {
+          if (body.rows.length < opts.limit) {
+            startKey = null; // last batch
+          } else {
+            startKey = body.rows[opts.limit - 1].id;
+          }
+
+          const docs = [];
+          body.rows.forEach(function(doc) {
+            docs.push(doc.doc);
+          });
+
+          if (docs.length > 0) {
+            ee.emit('received', {
+              batch: batch++,
+              data: docs,
+              length: docs.length,
+              time: (new Date().getTime() - start) / 1000,
+              total: total += docs.length
+            });
+          }
+          callback();
+        }
+      }).catch(err => {
+        err = error.convertResponseError(err);
+        ee.emit('error', err);
+        hasErrored = true;
+        callback();
       });
-  });
-
-  it('should terminate on a fatal error', function(done) {
-    nock(dbUrl)
-      .post('/_bulk_docs')
-      .reply(401, { error: 'Unauthorized' }); // fatal error
-
-    testLinestream()
-      .pipe(writer(db, 500, 1, noopEmitter))
-      .on('error', function(err) {
-        assert.strictEqual(err.name, 'Unauthorized');
-        assert.strictEqual(err.message, 'Access is denied due to invalid credentials.');
-        assert.ok(nock.isDone());
-        done();
-      });
-  });
-
-  it('should retry on transient errors', function(done) {
-    nock(dbUrl)
-      .post('/_bulk_docs')
-      .reply(429, { error: 'Too Many Requests' }) // transient error
-      .post('/_bulk_docs')
-      .reply(500, { error: 'Internal Server Error' }) // transient error
-      .post('/_bulk_docs')
-      .reply(200, { ok: true }); // third time lucky success
+    },
+    function(callback) { callback(null, hasErrored || startKey == null); },
+    function() { ee.emit('finished', { total: total }); }
+  );
 
-    testLinestream()
-      .pipe(writer(db, 500, 1, noopEmitter))
-      .on('error', function(err) {
-        done(err);
-      })
-      .on('finished', function(data) {
-        assert.strictEqual(data.total, 15);
-        assert.ok(nock.isDone());
-        done();
-      });
-  }).timeout(longTestTimeout);
-
-  it('should fail after 3 transient errors', function(done) {
-    nock(dbUrl)
-      .post('/_bulk_docs')
-      .reply(429, { error: 'Too Many Requests' }) // transient error
-      .post('/_bulk_docs')
-      .reply(500, { error: 'Internal Server Error' }) // transient error
-      .post('/_bulk_docs')
-      .reply(503, { error: 'Service Unavailable' }); // Final transient error
-
-    testLinestream()
-      .pipe(writer(db, 500, 1, noopEmitter))
-      .on('error', function(err) {
-        assert.strictEqual(err.name, 'HTTPFatalError');
-        assert.strictEqual(err.message, `503 : post ${dbUrl}/_bulk_docs - Error: Service Unavailable`);
-        assert.ok(nock.isDone());
-        done();
-      });
-  }).timeout(longTestTimeout);
-
-  it('should restore shallow backups without rev info successfully', function(done) {
-    nock(dbUrl)
-      .post('/_bulk_docs')
-      .reply(200, [{ ok: true, id: 'foo', rev: '1-abc' }]); // success
-
-    testLinestream('./test/fixtures/animaldb_old_shallow.json')
-      .pipe(writer(db, 500, 1, noopEmitter))
-      .on('error', function(err) {
-        done(err);
-      })
-      .on('finished', function(data) {
-        assert.strictEqual(data.total, 11);
-        assert.ok(nock.isDone());
-        done();
-      });
-  });
-
-  it('should get a batch error for non-empty array response with new_edits false', function(done) {
-    nock(dbUrl)
-      .post('/_bulk_docs')
-      .reply(200, [{ id: 'foo', error: 'foo', reason: 'bar' }]);
-
-    testLinestream()
-      .pipe(writer(db, 500, 1, noopEmitter))
-      .on('error', function(err) {
-        assert.strictEqual(err.name, 'Error');
-        assert.strictEqual(err.message, 'Error writing batch with new_edits:false and 1 items');
-        assert.ok(nock.isDone());
-        done();
-      });
-  });
-});
+  return ee;
+};