@cloudant/couchbackup 2.9.8 → 2.9.9-SNAPSHOT.100

package/.scannerwork/scanner-report/source-32.txt

@@ -1,4 +1,4 @@
-// Copyright © 2017, 2022 IBM Corp. All rights reserved.
+// Copyright © 2017 IBM Corp. All rights reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -12,134 +12,23 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-/* global describe it beforeEach */
+/* global describe it */
 'use strict';
 
 const assert = require('assert');
-const backup = require('../includes/shallowbackup.js');
-const request = require('../includes/request.js');
-const fs = require('fs');
-const nock = require('nock');
-
-// Function to create a DB object and call the shallow backup function
-// This is normally done by app.js
-function shallowBackup(dbUrl, opts) {
-  const db = request.client(dbUrl, opts);
-  // Disable compression to make body assertions easier
-  db.service.setEnableGzipCompression(false);
-  return backup(db, opts);
-}
-
-// Note all these tests include a body parameter of include_docs and a query
-// string of include_docs because of a quirk of nano that when using the fetch
-// method always adds the include_docs query string.
-describe('#unit Perform backup using shallow backup', function() {
-  const dbUrl = 'http://localhost:5984/animaldb';
-  // Query string keys are stringified by Nano
-  const badgerKey = 'badger\0';
-  const kookaburraKey = 'kookaburra\0';
-  const snipeKey = 'snipe\0';
-
-  beforeEach('Reset nocks', function() {
-    nock.cleanAll();
-  });
-
-  it('should perform a shallow backup', function(done) {
-    const couch = nock(dbUrl)
-      // batch 1
-      .post('/_all_docs', { limit: 3, include_docs: true })
-      .reply(200, JSON.parse(fs.readFileSync('./test/fixtures/animaldb_all_docs_1.json', 'utf8')))
-      // batch 2
-      .post('/_all_docs', { limit: 3, start_key: badgerKey, include_docs: true })
-      .reply(200, JSON.parse(fs.readFileSync('./test/fixtures/animaldb_all_docs_2.json', 'utf8')))
-      // batch 3
-      .post('/_all_docs', { limit: 3, start_key: kookaburraKey, include_docs: true })
-      .reply(200, JSON.parse(fs.readFileSync('./test/fixtures/animaldb_all_docs_3.json', 'utf8')))
-      // batch 4
-      .post('/_all_docs', { limit: 3, start_key: snipeKey, include_docs: true })
-      .reply(200, JSON.parse(fs.readFileSync('./test/fixtures/animaldb_all_docs_4.json', 'utf8')));
-
-    shallowBackup(dbUrl, { bufferSize: 3, parallelism: 1 })
-      .on('error', function(err) {
-        assert.fail(err);
-      })
-      .on('received', function(data) {
-        if (data.batch === 3) {
-          assert.strictEqual(data.length, 2); // smaller last batch
-        } else {
-          assert.strictEqual(data.length, 3);
-        }
-      })
-      .on('finished', function(data) {
-        assert.strictEqual(data.total, 11);
-        assert.ok(couch.isDone());
-        done();
-      });
-  });
-
-  it('should perform a shallow backup with transient error', function(done) {
-    const couch = nock(dbUrl)
-      // batch 1
-      .post('/_all_docs', { limit: 3, include_docs: true })
-      .reply(200, JSON.parse(fs.readFileSync('./test/fixtures/animaldb_all_docs_1.json', 'utf8')))
-      // batch 2
-      .post('/_all_docs', { limit: 3, start_key: badgerKey, include_docs: true })
-      .reply(200, JSON.parse(fs.readFileSync('./test/fixtures/animaldb_all_docs_2.json', 'utf8')))
-      // batch 3 - transient error
-      .post('/_all_docs', { limit: 3, start_key: kookaburraKey, include_docs: true })
-      .reply(500, { error: 'Internal Server Error' })
-      // batch 3 - retry
-      .post('/_all_docs', { limit: 3, start_key: kookaburraKey, include_docs: true })
-      .reply(200, JSON.parse(fs.readFileSync('./test/fixtures/animaldb_all_docs_3.json', 'utf8')))
-      // batch 4
-      .post('/_all_docs', { limit: 3, start_key: snipeKey, include_docs: true })
-      .reply(200, JSON.parse(fs.readFileSync('./test/fixtures/animaldb_all_docs_4.json', 'utf8')));
-
-    shallowBackup(dbUrl, { bufferSize: 3, parallelism: 1 })
-      .on('error', function(err) {
-        assert.strictEqual(err.name, 'HTTPError');
-      })
-      .on('received', function(data) {
-        if (data.batch === 3) {
-          assert.strictEqual(data.length, 2); // smaller last batch
-        } else {
-          assert.strictEqual(data.length, 3);
-        }
-      })
-      .on('finished', function(data) {
-        assert.strictEqual(data.total, 11);
-        assert.ok(couch.isDone());
-        done();
-      });
-  });
-
-  it('should fail to perform a shallow backup on fatal error', function(done) {
-    const couch = nock(dbUrl)
-      // batch 1
-      .post('/_all_docs', { limit: 3, include_docs: true })
-      .reply(200, JSON.parse(fs.readFileSync('./test/fixtures/animaldb_all_docs_1.json', 'utf8')))
-      // batch 2
-      .post('/_all_docs', { limit: 3, start_key: badgerKey, include_docs: true })
-      .reply(200, JSON.parse(fs.readFileSync('./test/fixtures/animaldb_all_docs_2.json', 'utf8')))
-      // batch 3 - fatal error
-      .post('/_all_docs', { limit: 3, start_key: kookaburraKey, include_docs: true })
-      .reply(401, { error: 'Unauthorized' });
-
-    let errCount = 0;
-
-    shallowBackup(dbUrl, { bufferSize: 3, parallelism: 1 })
-      .on('error', function(err) {
-        errCount++;
-        assert.strictEqual(err.name, 'Unauthorized');
-      })
-      .on('received', function(data) {
-        assert.strictEqual(data.length, 3);
-      })
-      .on('finished', function(data) {
-        assert.strictEqual(data.total, 6);
-        assert.ok(couch.isDone());
-        assert.strictEqual(errCount, 1);
-        done();
-      });
+const logfilesummary = require('../includes/logfilesummary.js');
+
+describe('#unit Fetching summary from the log file', function() {
+  it('should fetch a summary correctly', function(done) {
+    logfilesummary('./test/fixtures/test.log', function(err, data) {
+      assert.ok(!err);
+      assert.ok(data);
+      assert.strictEqual(data.changesComplete, true);
+      assert.strictEqual(typeof data.batches, 'object');
+      assert.strictEqual(Object.keys(data.batches).length, 2);
+      assert.deepStrictEqual(data.batches['1'], true);
+      assert.deepStrictEqual(data.batches['4'], true);
+      done();
+    });
   });
 });

package/.scannerwork/scanner-report/source-33.txt

@@ -12,145 +12,136 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-/* global describe it */
+/* global describe it beforeEach */
 'use strict';
 
 const assert = require('assert');
+const fs = require('fs');
 const nock = require('nock');
 const request = require('../includes/request.js');
-const changes = require('../includes/spoolchanges.js');
+const writer = require('../includes/writer.js');
+const noopEmitter = new (require('events')).EventEmitter();
+const liner = require('../includes/liner.js');
 
-const url = 'http://localhost:7777';
-const dbName = 'fakenockdb';
 const longTestTimeout = 3000;
 
-const db = request.client(`${url}/${dbName}`, { parallelism: 1 });
+// The writer expects a line-by-line stream so this utility function does that
+// processing for the tests (which normally happens in the internal restore
+// function).
+function testLinestream(fixture = './test/fixtures/animaldb_expected.json') {
+  return fs.createReadStream(fixture)
+    .pipe(liner());
+}
+
+describe('#unit Check database restore writer', function() {
+  const dbUrl = 'http://localhost:5984/animaldb';
+  const db = request.client(dbUrl, { parallelism: 1 });
 
-const seqSuffix = Buffer.alloc(124, 'abc123').toString('base64');
-function provideChanges(batchSize, totalChanges, fullResponse = false) {
-  let pending = totalChanges;
-  const sparseResultsArray = Array(batchSize).fill({
-    seq: null,
-    id: 'doc',
-    changes: [{ rev: '1-abcdef0123456789abcdef0123456789' }]
+  beforeEach('Reset nocks', function() {
+    nock.cleanAll();
   });
-  nock(url)
-    .post(`/${dbName}/_changes`)
-    .query(true)
-    .times(totalChanges / batchSize + (totalChanges % batchSize > 0 ? 1 : 0))
-    .reply(200, (uri, requestBody) => {
-      pending -= batchSize;
-      const lastSeq = (totalChanges - pending);
-      const seq = lastSeq - batchSize;
-      return {
-        results: fullResponse
-          ? Array.from(Array(batchSize), (_, i) => {
-            return {
-              seq: `${seq + i}-${seqSuffix}`,
-              id: `doc${seq + i}`,
-              changes: [{ rev: '1-abcdef0123456789abcdef0123456789' }]
-            };
-          })
-          : sparseResultsArray,
-        pending: pending,
-        last_seq: `${lastSeq}-abc`
-      };
-    });
-}
 
-describe('#unit Check spool changes', function() {
-  it('should terminate on request error', function(done) {
-    nock(url)
-      .post(`/${dbName}/_changes`)
-      .query(true)
-      .times(3)
-      .replyWithError({ code: 'ECONNRESET', message: 'socket hang up' });
-
-    changes(db, '/dev/null', 500, null, function(err) {
-      assert.strictEqual(err.name, 'SpoolChangesError');
-      assert.strictEqual(err.message, `Failed changes request - socket hang up: post ${url}/${dbName}/_changes`);
-      assert.ok(nock.isDone());
-      done();
-    });
-  }).timeout(longTestTimeout);
+  it('should complete successfully', function(done) {
+    nock(dbUrl)
+      .post('/_bulk_docs')
+      .reply(200, []); // success
 
-  it('should terminate on bad HTTP status code response', function(done) {
-    nock(url)
-      .post(`/${dbName}/_changes`)
-      .query(true)
-      .times(3)
-      .reply(500, function(uri, requestBody) {
-        this.req.response.statusMessage = 'Internal Server Error';
-        return { error: 'foo', reason: 'bar' };
+    testLinestream()
+      .pipe(writer(db, 500, 1, noopEmitter))
+      .on('error', function(err) {
+        done(err);
+      })
+      .on('finished', function(data) {
+        assert.strictEqual(data.total, 15);
+        assert.ok(nock.isDone());
+        done();
       });
+  });
 
-    changes(db, '/dev/null', 500, null, function(err) {
-      assert.strictEqual(err.name, 'HTTPFatalError');
-      assert.strictEqual(err.message, `500 Internal Server Error: post ${url}/${dbName}/_changes - Error: foo, Reason: bar`);
-      assert.ok(nock.isDone());
-      done();
-    });
-  }).timeout(longTestTimeout);
+  it('should terminate on a fatal error', function(done) {
+    nock(dbUrl)
+      .post('/_bulk_docs')
+      .reply(401, { error: 'Unauthorized' }); // fatal error
 
-  it('should keep collecting changes', function(done) {
-    // This test validates that spooling will correctly
-    // continue across multiple requests
-    // (4 batches of 100000 to be precise).
-    // This test might take up to 10 seconds
-    this.timeout(10 * 1000);
-
-    // Use full changes for this test
-    provideChanges(100000, 400000, true);
-    changes(db, '/dev/null', 500, null, function(err) {
-      assert.ok(!err);
-      assert.ok(nock.isDone());
-      done();
-    });
+    testLinestream()
+      .pipe(writer(db, 500, 1, noopEmitter))
+      .on('error', function(err) {
+        assert.strictEqual(err.name, 'Unauthorized');
+        assert.strictEqual(err.message, 'Access is denied due to invalid credentials.');
+        assert.ok(nock.isDone());
+        done();
+      });
   });
 
-  it('should keep collecting sparse changes', function(done) {
-    // This test checks that making thousands of requests doesn't
-    // make anything bad happen.
-    // This test might take up to 25 seconds
-    this.timeout(25 * 1000);
-    // Use sparse changes for this test and a batch size of 1
-    provideChanges(1, 2500);
-    changes(db, '/dev/null', 500, null, function(err) {
-      assert.ok(!err);
-      assert.ok(nock.isDone());
-      done();
-    });
-  });
-});
+  it('should retry on transient errors', function(done) {
+    nock(dbUrl)
+      .post('/_bulk_docs')
+      .reply(429, { error: 'Too Many Requests' }) // transient error
+      .post('/_bulk_docs')
+      .reply(500, { error: 'Internal Server Error' }) // transient error
+      .post('/_bulk_docs')
+      .reply(200, { ok: true }); // third time lucky success
+
+    testLinestream()
+      .pipe(writer(db, 500, 1, noopEmitter))
+      .on('error', function(err) {
+        done(err);
+      })
+      .on('finished', function(data) {
+        assert.strictEqual(data.total, 15);
+        assert.ok(nock.isDone());
+        done();
+      });
+  }).timeout(longTestTimeout);
+
+  it('should fail after 3 transient errors', function(done) {
+    nock(dbUrl)
+      .post('/_bulk_docs')
+      .reply(429, { error: 'Too Many Requests' }) // transient error
+      .post('/_bulk_docs')
+      .reply(500, { error: 'Internal Server Error' }) // transient error
+      .post('/_bulk_docs')
+      .reply(503, { error: 'Service Unavailable' }); // Final transient error
+
+    testLinestream()
+      .pipe(writer(db, 500, 1, noopEmitter))
+      .on('error', function(err) {
+        assert.strictEqual(err.name, 'HTTPFatalError');
+        assert.strictEqual(err.message, `503 : post ${dbUrl}/_bulk_docs - Error: Service Unavailable`);
+        assert.ok(nock.isDone());
+        done();
+      });
+  }).timeout(longTestTimeout);
+
+  it('should restore shallow backups without rev info successfully', function(done) {
+    nock(dbUrl)
+      .post('/_bulk_docs')
+      .reply(200, [{ ok: true, id: 'foo', rev: '1-abc' }]); // success
 
-describe('Longer spool changes checks', function() {
-  it('#slow should keep collecting changes (25M)', function(done) {
-    // This test might take up to 5 minutes
-    this.timeout(5 * 60 * 1000);
-    // Note changes spooling uses a constant batch size, we are setting
-    // a test value here and setting the buffer to match
-    const batch = 100000;
-    // Use sparse changes for this test
-    provideChanges(batch, 25000000);
-    changes(db, '/dev/null', batch, null, function(err) {
-      assert.ok(!err);
-      assert.ok(nock.isDone());
-      done();
-    });
+    testLinestream('./test/fixtures/animaldb_old_shallow.json')
+      .pipe(writer(db, 500, 1, noopEmitter))
+      .on('error', function(err) {
+        done(err);
+      })
+      .on('finished', function(data) {
+        assert.strictEqual(data.total, 11);
+        assert.ok(nock.isDone());
+        done();
+      });
   });
 
-  it('#slower should keep collecting changes (500M)', function(done) {
-    // This test might take up to 90 minutes
-    this.timeout(90 * 60 * 1000);
-    // Note changes spooling uses a constant batch size, we are setting
-    // a test value here and setting the buffer to match
-    const batch = 1000000;
-    // Use full changes for this test to exercise load
-    provideChanges(batch, 500000000, true);
-    changes(db, '/dev/null', batch, null, function(err) {
-      assert.ok(!err);
-      assert.ok(nock.isDone());
-      done();
-    });
+  it('should get a batch error for non-empty array response with new_edits false', function(done) {
+    nock(dbUrl)
+      .post('/_bulk_docs')
+      .reply(200, [{ id: 'foo', error: 'foo', reason: 'bar' }]);
+
+    testLinestream()
+      .pipe(writer(db, 500, 1, noopEmitter))
+      .on('error', function(err) {
+        assert.strictEqual(err.name, 'Error');
+        assert.strictEqual(err.message, 'Error writing batch with new_edits:false and 1 items');
+        assert.ok(nock.isDone());
+        done();
+      });
   });
 });