@cloudant/couchbackup 2.9.16-SNAPSHOT.183 → 2.9.16

package/.scannerwork/scanner-report/source-17.txt

@@ -1,4 +1,4 @@
-// Copyright © 2017, 2023 IBM Corp. All rights reserved.
+// Copyright © 2023 IBM Corp. All rights reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -12,172 +12,89 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-/* global describe it beforeEach */
 'use strict';
 
-const assert = require('assert');
-const backup = require('../includes/shallowbackup.js');
-const request = require('../includes/request.js');
-const fs = require('fs');
-const nock = require('nock');
-
-// Function to create a DB object and call the shallow backup function
-// This is normally done by app.js
-function shallowBackup(dbUrl, opts) {
-  const db = request.client(dbUrl, opts);
-  // Disable compression to make body assertions easier
-  db.service.setEnableGzipCompression(false);
-  return backup(db, opts);
-}
-
-// Note all these tests include a body parameter of include_docs and a query
-// string of include_docs because of a quirk of nano that when using the fetch
-// method always adds the include_docs query string.
-describe('#unit Perform backup using shallow backup', function() {
-  const dbUrl = 'http://localhost:5984/animaldb';
-  // Query string keys are stringified by Nano
-  const badgerKey = 'badger\0';
-  const kookaburraKey = 'kookaburra\0';
-  const snipeKey = 'snipe\0';
-
-  beforeEach('Reset nocks', function() {
-    nock.cleanAll();
-  });
-
-  it('should perform a shallow backup', async function() {
-    const couch = nock(dbUrl)
-      // batch 1
-      .post('/_all_docs', { limit: 3, include_docs: true })
-      .reply(200, JSON.parse(fs.readFileSync('./test/fixtures/animaldb_all_docs_1.json', 'utf8')))
-      // batch 2
-      .post('/_all_docs', { limit: 3, start_key: badgerKey, include_docs: true })
-      .reply(200, JSON.parse(fs.readFileSync('./test/fixtures/animaldb_all_docs_2.json', 'utf8')))
-      // batch 3
-      .post('/_all_docs', { limit: 3, start_key: kookaburraKey, include_docs: true })
-      .reply(200, JSON.parse(fs.readFileSync('./test/fixtures/animaldb_all_docs_3.json', 'utf8')))
-      // batch 4
-      .post('/_all_docs', { limit: 3, start_key: snipeKey, include_docs: true })
-      .reply(200, JSON.parse(fs.readFileSync('./test/fixtures/animaldb_all_docs_4.json', 'utf8')));
-
-    return new Promise((resolve, reject) => {
-      shallowBackup(dbUrl, { bufferSize: 3, parallelism: 1 })
-        .on('error', function(err) {
-          reject(err);
-        })
-        .on('received', function(data) {
-          try {
-            if (data.batch === 3) {
-              assert.strictEqual(data.length, 2); // smaller last batch
-            } else {
-              assert.strictEqual(data.length, 3);
-            }
-          } catch (err) {
-            reject(err);
-          }
-        })
-        .on('finished', function(data) {
-          try {
-            assert.strictEqual(data.total, 11);
-            assert.ok(couch.isDone());
-            resolve();
-          } catch (err) {
-            reject(err);
-          }
-        });
-    });
-  });
-
-  it('should perform a shallow backup with transient error', async function() {
-    const couch = nock(dbUrl)
-      // batch 1
-      .post('/_all_docs', { limit: 3, include_docs: true })
-      .reply(200, JSON.parse(fs.readFileSync('./test/fixtures/animaldb_all_docs_1.json', 'utf8')))
-      // batch 2
-      .post('/_all_docs', { limit: 3, start_key: badgerKey, include_docs: true })
-      .reply(200, JSON.parse(fs.readFileSync('./test/fixtures/animaldb_all_docs_2.json', 'utf8')))
-      // batch 3 - transient error
-      .post('/_all_docs', { limit: 3, start_key: kookaburraKey, include_docs: true })
-      .reply(500, { error: 'Internal Server Error' })
-      // batch 3 - retry
-      .post('/_all_docs', { limit: 3, start_key: kookaburraKey, include_docs: true })
-      .reply(200, JSON.parse(fs.readFileSync('./test/fixtures/animaldb_all_docs_3.json', 'utf8')))
-      // batch 4
-      .post('/_all_docs', { limit: 3, start_key: snipeKey, include_docs: true })
-      .reply(200, JSON.parse(fs.readFileSync('./test/fixtures/animaldb_all_docs_4.json', 'utf8')));
-
-    return new Promise((resolve, reject) => {
-      shallowBackup(dbUrl, { bufferSize: 3, parallelism: 1 })
-        .on('error', function(err) {
-          try {
-            assert.strictEqual(err.name, 'HTTPError');
-          } catch (err) {
-            reject(err);
-          }
-        })
-        .on('received', function(data) {
-          try {
-            if (data.batch === 3) {
-              assert.strictEqual(data.length, 2); // smaller last batch
-            } else {
-              assert.strictEqual(data.length, 3);
-            }
-          } catch (err) {
-            reject(err);
-          }
-        })
-        .on('finished', function(data) {
-          try {
-            assert.strictEqual(data.total, 11);
-            assert.ok(couch.isDone());
-            resolve();
-          } catch (err) {
-            reject(err);
-          }
-        });
-    });
-  });
-
-  it('should fail to perform a shallow backup on fatal error', async function() {
-    const couch = nock(dbUrl)
-      // batch 1
-      .post('/_all_docs', { limit: 3, include_docs: true })
-      .reply(200, JSON.parse(fs.readFileSync('./test/fixtures/animaldb_all_docs_1.json', 'utf8')))
-      // batch 2
-      .post('/_all_docs', { limit: 3, start_key: badgerKey, include_docs: true })
-      .reply(200, JSON.parse(fs.readFileSync('./test/fixtures/animaldb_all_docs_2.json', 'utf8')))
-      // batch 3 - fatal error
-      .post('/_all_docs', { limit: 3, start_key: kookaburraKey, include_docs: true })
-      .reply(401, { error: 'Unauthorized' });
-
-    let errCount = 0;
-
-    return new Promise((resolve, reject) => {
-      shallowBackup(dbUrl, { bufferSize: 3, parallelism: 1 })
-        .on('error', function(err) {
-          try {
-            errCount++;
-            assert.strictEqual(err.name, 'Unauthorized');
-          } catch (err) {
-            reject(err);
-          }
-        })
-        .on('received', function(data) {
-          try {
-            assert.strictEqual(data.length, 3);
-          } catch (err) {
-            reject(err);
-          }
-        })
-        .on('finished', function(data) {
-          try {
-            assert.strictEqual(data.total, 6);
-            assert.ok(couch.isDone());
-            assert.strictEqual(errCount, 1);
-            resolve();
-          } catch (err) {
-            reject(err);
-          }
-        });
+const chunk = require('lodash/chunk');
+const difference = require('lodash/difference');
+const forOwn = require('lodash/forOwn');
+const isEmpty = require('lodash/isEmpty');
+const union = require('lodash/union');
+
+const compare = async function(database1, database2, client) {
+  // check docs same in both dbs
+  const allDocs1 = await getAllDocs(client, database1);
+  const allDocs2 = await getAllDocs(client, database2);
+
+  const onlyInDb1 = (difference(allDocs1, allDocs2));
+  const onlyInDb2 = (difference(allDocs2, allDocs1));
+
+  let databasesSame = isEmpty(onlyInDb1) && isEmpty(onlyInDb2);
+
+  if (!databasesSame) {
+    console.log(onlyInDb1.length + ' documents only in db 1.');
+    console.log('Document IDs only in db 1: ' + onlyInDb1);
+    console.log(onlyInDb2.length + ' documents only in db 2.');
+    console.log('Document IDs only in db 2: ' + onlyInDb2);
+  }
+
+  // check revs same in docs common to both dbs
+  const partitionSize = 500;
+  const batches = chunk(union(allDocs1, allDocs2), partitionSize);
+
+  const missingRevsInDb2 = await getMissingRevs(client, database1, database2, batches);
+  const missingRevsInDb1 = await getMissingRevs(client, database2, database1, batches);
+
+  databasesSame = databasesSame && isEmpty(missingRevsInDb1) && isEmpty(missingRevsInDb2);
+
+  if (!databasesSame) {
+    console.log('Missing revs in db 1:' + JSON.stringify(missingRevsInDb1));
+    console.log('Missing revs in db 2:' + JSON.stringify(missingRevsInDb2));
+  }
+
+  return databasesSame;
+};
+
+const getMissingRevs = async(client, databaseName1, databaseName2, batcheses) => {
+  const fakeRevisionId = '9999-a';
+
+  const missing = {};
+
+  // look in db1 - use a fake revision ID to fetch all leaf revisions
+
+  for (const batches of batcheses) {
+    const documentRevisions = {};
+    batches.forEach(id => (documentRevisions[id] = [fakeRevisionId]));
+
+    const result1 = await client.postRevsDiff({ db: databaseName1, documentRevisions });
+    const revsDiffRequestDb2 = {};
+    forOwn(result1.result, (v, k) => (revsDiffRequestDb2[k] = v.possible_ancestors));
+    // look in db2
+    const result2 = await client.postRevsDiff({ db: databaseName2, documentRevisions: revsDiffRequestDb2 });
+    forOwn(result2.result, (v, k) => {
+      if ('missing' in v) {
+        missing[k] = v.missing;
+      }
     });
-  });
-});
+  }
+  return missing;
+};
+
+const getAllDocs = async function(client, database) {
+  let allDocIds = [];
+  const limit = 2000;
+  let startKey = '\u0000';
+  do {
+    const pageOfDocIds = (await client.postAllDocs({ db: database, startKey, limit })).result.rows.map(r => r.id);
+    allDocIds = allDocIds.concat(pageOfDocIds);
+    if (pageOfDocIds.length < limit) {
+      startKey = null;
+    } else {
+      startKey = pageOfDocIds[limit - 1] + '\u0000';
+    }
+  } while (startKey != null);
+  return allDocIds;
+};
+
+module.exports = {
+  compare
+};

package/.scannerwork/scanner-report/source-18.txt

@@ -12,54 +12,95 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-/* global beforeEach afterEach */
+/* global describe it */
 'use strict';
 
-const { CloudantV1 } = require('@ibm-cloud/cloudant');
-const url = new URL((process.env.COUCH_BACKEND_URL) ? process.env.COUCH_BACKEND_URL : 'https://no-couch-backend-url-set.test');
-const { BasicAuthenticator, NoAuthAuthenticator } = require('ibm-cloud-sdk-core');
-const authenticator = (url.username) ? new BasicAuthenticator({ username: url.username, password: decodeURIComponent(url.password) }) : new NoAuthAuthenticator();
-const serviceOpts = {
-  authenticator: authenticator
-};
-const cloudant = new CloudantV1(serviceOpts);
-// Remove auth from URL before using for service
-cloudant.setServiceUrl(new URL(url.pathname, url.origin).toString());
-const uuid = require('uuid').v4;
 const fs = require('fs');
+const { once } = require('node:events');
+const u = require('./citestutils.js');
 
-// Mocha hooks that will be at the root context so run for all tests
+[{ useApi: true }, { useApi: false }].forEach(function(params) {
+  describe(u.scenario('Basic backup and restore', params), function() {
+    it('should backup animaldb to a file correctly', async function() {
+      // Allow up to 40 s to backup and compare (it should be much faster)!
+      u.setTimeout(this, 40);
+      const actualBackup = `./${this.fileName}`;
+      // Create a file and backup to it
+      const output = fs.createWriteStream(actualBackup);
+      return once(output, 'open')
+        .then(() => {
+          return u.testBackup(params, 'animaldb', output);
+        }).then(() => {
+          return u.readSortAndDeepEqual(actualBackup, './test/fixtures/animaldb_expected.json');
+        });
+    });
 
-beforeEach('Create test database', async function() {
-  // Don't run hook for unit tests, just for CI
-  if (!this.currentTest.fullTitle().includes('#unit')) {
-    // Allow 10 seconds to create the DB
-    this.timeout(10 * 1000);
-    const unique = uuid();
-    this.fileName = `${unique}`;
-    this.dbName = 'couchbackup_test_' + unique;
+    it('should restore animaldb to a database correctly', async function() {
+      // Allow up to 60 s to restore and compare (again it should be faster)!
+      u.setTimeout(this, 60);
+      const input = fs.createReadStream('./test/fixtures/animaldb_expected.json');
+      const dbName = this.dbName;
+      return once(input, 'open').then(() => {
+        return u.testRestore(params, input, dbName);
+      }).then(() => {
+        return u.dbCompare('animaldb', dbName);
+      });
+    });
 
-    return cloudant.putDatabase({ db: this.dbName });
-  }
-});
+    it('should execute a shallow mode backup successfully', async function() {
+      // Allow 30 s
+      u.setTimeout(this, 30);
+      const actualBackup = `./${this.fileName}`;
+      const output = fs.createWriteStream(actualBackup);
+      // Add the shallow mode option
+      const p = u.p(params, { opts: { mode: 'shallow' } });
+      return once(output, 'open')
+        .then(() => {
+          return u.testBackup(p, 'animaldb', output);
+        }).then(() => {
+          return u.readSortAndDeepEqual(actualBackup, './test/fixtures/animaldb_expected_shallow.json');
+        });
+    });
 
-afterEach('Delete test database', async function() {
-  // Don't run hook for unit tests, just for CI
-  if (!this.currentTest.fullTitle().includes('#unit')) {
-    // Allow 10 seconds to delete the DB
-    this.timeout(10 * 1000);
-    deleteIfExists(this.fileName);
-    deleteIfExists(`${this.fileName}.log`);
-    return cloudant.deleteDatabase({ db: this.dbName });
-  }
-});
+    describe(u.scenario('Buffer size tests', params), function() {
+      it('should backup/restore animaldb with the same buffer size', async function() {
+        // Allow up to 60 s for backup and restore of animaldb
+        u.setTimeout(this, 60);
+        const actualBackup = `./${this.fileName}`;
+        const logFile = `./${this.fileName}` + '.log';
+        const p = u.p(params, { opts: { log: logFile, bufferSize: 1 } });
+        return u.testBackupAndRestoreViaFile(p, 'animaldb', actualBackup, this.dbName);
+      });
 
-function deleteIfExists(fileName) {
-  fs.unlink(fileName, function(err) {
-    if (err) {
-      if (err.code !== 'ENOENT') {
-        console.error(`${err.code} ${err.message}`);
-      }
-    }
+      it('should backup/restore animaldb with backup buffer > restore buffer', async function() {
+        // Allow up to 60 s for backup and restore of animaldb
+        u.setTimeout(this, 60);
+        const actualBackup = `./${this.fileName}`;
+        const logFile = `./${this.fileName}` + '.log';
+        const dbName = this.dbName;
+        const p = u.p(params, { opts: { log: logFile, bufferSize: 2 } }); // backup
+        const q = u.p(params, { opts: { bufferSize: 1 } }); // restore
+        return u.testBackupToFile(p, 'animaldb', actualBackup).then(() => {
+          return u.testRestoreFromFile(q, actualBackup, dbName);
+        }).then(() => {
+          return u.dbCompare('animaldb', dbName);
+        });
+      });
+
+      it('should backup/restore animaldb with backup buffer < restore buffer', async function() {
+        // Allow up to 60 s for backup and restore of animaldb
+        u.setTimeout(this, 60);
+        const actualBackup = `./${this.fileName}`;
+        const logFile = `./${this.fileName}` + '.log';
+        const dbName = this.dbName;
+        const p = u.p(params, { opts: { log: logFile, bufferSize: 1 } }); // backup
+        const q = u.p(params, { opts: { bufferSize: 2 } }); // restore
+        return u.testBackupToFile(p, 'animaldb', actualBackup).then(() => {
+          return u.testRestoreFromFile(q, actualBackup, dbName);
+        }).then(() => {
+          return u.dbCompare('animaldb', dbName);
+        });
+      });
+    });
   });
-}
+});

package/.scannerwork/scanner-report/source-19.txt

@@ -12,95 +12,123 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-/* global describe it */
+/* global describe it beforeEach */
 'use strict';
 
+const assert = require('assert');
 const fs = require('fs');
+const nock = require('nock');
+const request = require('../includes/request.js');
+const writer = require('../includes/writer.js');
+const noopEmitter = new (require('events')).EventEmitter();
+const liner = require('../includes/liner.js');
 const { once } = require('node:events');
-const u = require('./citestutils.js');
-
-[{ useApi: true }, { useApi: false }].forEach(function(params) {
-  describe(u.scenario('Basic backup and restore', params), function() {
-    it('should backup animaldb to a file correctly', async function() {
-      // Allow up to 40 s to backup and compare (it should be much faster)!
-      u.setTimeout(this, 40);
-      const actualBackup = `./${this.fileName}`;
-      // Create a file and backup to it
-      const output = fs.createWriteStream(actualBackup);
-      return once(output, 'open')
-        .then(() => {
-          return u.testBackup(params, 'animaldb', output);
-        }).then(() => {
-          return u.readSortAndDeepEqual(actualBackup, './test/fixtures/animaldb_expected.json');
-        });
-    });
-
-    it('should restore animaldb to a database correctly', async function() {
-      // Allow up to 60 s to restore and compare (again it should be faster)!
-      u.setTimeout(this, 60);
-      const input = fs.createReadStream('./test/fixtures/animaldb_expected.json');
-      const dbName = this.dbName;
-      return once(input, 'open').then(() => {
-        return u.testRestore(params, input, dbName);
-      }).then(() => {
-        return u.dbCompare('animaldb', dbName);
-      });
-    });
-
-    it('should execute a shallow mode backup successfully', async function() {
-      // Allow 30 s
-      u.setTimeout(this, 30);
-      const actualBackup = `./${this.fileName}`;
-      const output = fs.createWriteStream(actualBackup);
-      // Add the shallow mode option
-      const p = u.p(params, { opts: { mode: 'shallow' } });
-      return once(output, 'open')
-        .then(() => {
-          return u.testBackup(p, 'animaldb', output);
-        }).then(() => {
-          return u.readSortAndDeepEqual(actualBackup, './test/fixtures/animaldb_expected_shallow.json');
-        });
-    });
-
-    describe(u.scenario('Buffer size tests', params), function() {
-      it('should backup/restore animaldb with the same buffer size', async function() {
-        // Allow up to 60 s for backup and restore of animaldb
-        u.setTimeout(this, 60);
-        const actualBackup = `./${this.fileName}`;
-        const logFile = `./${this.fileName}` + '.log';
-        const p = u.p(params, { opts: { log: logFile, bufferSize: 1 } });
-        return u.testBackupAndRestoreViaFile(p, 'animaldb', actualBackup, this.dbName);
-      });
-
-      it('should backup/restore animaldb with backup buffer > restore buffer', async function() {
-        // Allow up to 60 s for backup and restore of animaldb
-        u.setTimeout(this, 60);
-        const actualBackup = `./${this.fileName}`;
-        const logFile = `./${this.fileName}` + '.log';
-        const dbName = this.dbName;
-        const p = u.p(params, { opts: { log: logFile, bufferSize: 2 } }); // backup
-        const q = u.p(params, { opts: { bufferSize: 1 } }); // restore
-        return u.testBackupToFile(p, 'animaldb', actualBackup).then(() => {
-          return u.testRestoreFromFile(q, actualBackup, dbName);
-        }).then(() => {
-          return u.dbCompare('animaldb', dbName);
-        });
-      });
-
-      it('should backup/restore animaldb with backup buffer < restore buffer', async function() {
-        // Allow up to 60 s for backup and restore of animaldb
-        u.setTimeout(this, 60);
-        const actualBackup = `./${this.fileName}`;
-        const logFile = `./${this.fileName}` + '.log';
-        const dbName = this.dbName;
-        const p = u.p(params, { opts: { log: logFile, bufferSize: 1 } }); // backup
-        const q = u.p(params, { opts: { bufferSize: 2 } }); // restore
-        return u.testBackupToFile(p, 'animaldb', actualBackup).then(() => {
-          return u.testRestoreFromFile(q, actualBackup, dbName);
-        }).then(() => {
-          return u.dbCompare('animaldb', dbName);
-        });
-      });
-    });
+const { pipeline } = require('node:stream/promises');
+const longTestTimeout = 3000;
+
+describe('#unit Check database restore writer', function() {
+  const dbUrl = 'http://localhost:5984/animaldb';
+  const db = request.client(dbUrl, { parallelism: 1 });
+
+  beforeEach('Reset nocks', function() {
+    nock.cleanAll();
+  });
+
+  it('should complete successfully', async function() {
+    nock(dbUrl)
+      .post('/_bulk_docs')
+      .reply(200, []); // success
+
+    const w = writer(db, 500, 1, noopEmitter);
+    return Promise.all([pipeline(fs.createReadStream('./test/fixtures/animaldb_expected.json'), liner(), w),
+      once(w, 'finished').then((data) => {
+        assert.strictEqual(data[0].total, 15);
+        assert.ok(nock.isDone());
+      })]);
+  });
+
+  it('should terminate on a fatal error', async function() {
+    nock(dbUrl)
+      .post('/_bulk_docs')
+      .reply(401, { error: 'Unauthorized' }); // fatal error
+
+    const w = writer(db, 500, 1, noopEmitter);
+    return assert.rejects(
+      pipeline(fs.createReadStream('./test/fixtures/animaldb_expected.json'), liner(), w),
+      (err) => {
+        assert.strictEqual(err.name, 'Unauthorized');
+        assert.strictEqual(err.message, 'Access is denied due to invalid credentials.');
+        assert.ok(nock.isDone());
+        return true;
+      }
+    );
+  });
+
+  it('should retry on transient errors', async function() {
+    nock(dbUrl)
+      .post('/_bulk_docs')
+      .reply(429, { error: 'Too Many Requests' }) // transient error
+      .post('/_bulk_docs')
+      .reply(500, { error: 'Internal Server Error' }) // transient error
+      .post('/_bulk_docs')
+      .reply(200, { ok: true }); // third time lucky success
+
+    const w = writer(db, 500, 1, noopEmitter);
+    return Promise.all([pipeline(fs.createReadStream('./test/fixtures/animaldb_expected.json'), liner(), w),
+      once(w, 'finished').then((data) => {
+        assert.strictEqual(data[0].total, 15);
+        assert.ok(nock.isDone());
+      })]);
+  }).timeout(longTestTimeout);
+
+  it('should fail after 3 transient errors', async function() {
+    nock(dbUrl)
+      .post('/_bulk_docs')
+      .reply(429, { error: 'Too Many Requests' }) // transient error
+      .post('/_bulk_docs')
+      .reply(500, { error: 'Internal Server Error' }) // transient error
+      .post('/_bulk_docs')
+      .reply(503, { error: 'Service Unavailable' }); // Final transient error
+
+    const w = writer(db, 500, 1, noopEmitter);
+    return assert.rejects(
+      pipeline(fs.createReadStream('./test/fixtures/animaldb_expected.json'), liner(), w),
+      (err) => {
+        assert.strictEqual(err.name, 'HTTPFatalError');
+        assert.strictEqual(err.message, `503 : post ${dbUrl}/_bulk_docs - Error: Service Unavailable`);
+        assert.ok(nock.isDone());
+        return true;
+      }
+    );
+  }).timeout(longTestTimeout);
+
+  it('should restore shallow backups without rev info successfully', async function() {
+    nock(dbUrl)
+      .post('/_bulk_docs')
+      .reply(200, [{ ok: true, id: 'foo', rev: '1-abc' }]); // success
+
+    const w = writer(db, 500, 1, noopEmitter);
+    return Promise.all([pipeline(fs.createReadStream('./test/fixtures/animaldb_old_shallow.json'), liner(), w),
+      once(w, 'finished').then((data) => {
+        assert.strictEqual(data[0].total, 11);
+        assert.ok(nock.isDone());
+      })]);
+  });
+
+  it('should get a batch error for non-empty array response with new_edits false', async function() {
+    nock(dbUrl)
+      .post('/_bulk_docs')
+      .reply(200, [{ id: 'foo', error: 'foo', reason: 'bar' }]);
+
+    const w = writer(db, 500, 1, noopEmitter);
+    return assert.rejects(
+      pipeline(fs.createReadStream('./test/fixtures/animaldb_expected.json'), liner(), w),
+      (err) => {
+        assert.strictEqual(err.name, 'Error');
+        assert.strictEqual(err.message, 'Error writing batch with new_edits:false and 1 items');
+        assert.ok(nock.isDone());
+        return true;
+      }
+    );
   });
 });