@cloudant/couchbackup 2.9.15-SNAPSHOT.175 → 2.9.15

package/.scannerwork/scanner-report/source-33.txt

@@ -12,202 +12,54 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-/* global describe it beforeEach */
+/* global beforeEach afterEach */
 'use strict';
 
-const assert = require('assert');
-const nock = require('nock');
-const request = require('../includes/request.js');
-const error = require('../includes/error.js');
-
-const url = 'http://localhost:7777/testdb';
-const db = request.client(url, { parallelism: 1 });
-const timeoutDb = request.client(url, { parallelism: 1, requestTimeout: 500 });
-const longTestTimeout = 3000;
-
-beforeEach('Clean nock', function() {
-  nock.cleanAll();
+const { CloudantV1 } = require('@ibm-cloud/cloudant');
+const url = new URL((process.env.COUCH_BACKEND_URL) ? process.env.COUCH_BACKEND_URL : 'https://no-couch-backend-url-set.test');
+const { BasicAuthenticator, NoAuthAuthenticator } = require('ibm-cloud-sdk-core');
+const authenticator = (url.username) ? new BasicAuthenticator({ username: url.username, password: decodeURIComponent(url.password) }) : new NoAuthAuthenticator();
+const serviceOpts = {
+  authenticator: authenticator
+};
+const cloudant = new CloudantV1(serviceOpts);
+// Remove auth from URL before using for service
+cloudant.setServiceUrl(new URL(url.pathname, url.origin).toString());
+const uuid = require('uuid').v4;
+const fs = require('fs');
+
+// Mocha hooks that will be at the root context so run for all tests
+
+beforeEach('Create test database', async function() {
+  // Don't run hook for unit tests, just for CI
+  if (!this.currentTest.fullTitle().includes('#unit')) {
+    // Allow 10 seconds to create the DB
+    this.timeout(10 * 1000);
+    const unique = uuid();
+    this.fileName = `${unique}`;
+    this.dbName = 'couchbackup_test_' + unique;
+
+    return cloudant.putDatabase({ db: this.dbName });
+  }
 });
 
-describe('#unit Check request headers', function() {
-  it('should have a couchbackup user-agent', async function() {
-    const couch = nock(url)
-      .matchHeader('user-agent', /couchbackup-cloudant\/\d+\.\d+\.\d+(?:-SNAPSHOT)? \(Node.js v\d+\.\d+\.\d+\)/)
-      .head('/good')
-      .reply(200);
-
-    return db.service.headDocument({ db: db.db, docId: 'good' }).then(() => {
-      assert.ok(couch.isDone());
-    });
-  });
+afterEach('Delete test database', async function() {
+  // Don't run hook for unit tests, just for CI
+  if (!this.currentTest.fullTitle().includes('#unit')) {
+    // Allow 10 seconds to delete the DB
+    this.timeout(10 * 1000);
+    deleteIfExists(this.fileName);
+    deleteIfExists(`${this.fileName}.log`);
+    return cloudant.deleteDatabase({ db: this.dbName });
+  }
 });
 
-describe('#unit Check request response error callback', function() {
-  it('should not callback with error for 200 response', async function() {
-    const couch = nock(url)
-      .get('/good')
-      .reply(200, { ok: true });
-
-    return db.service.getDocument({ db: db.db, docId: 'good' }).then(response => {
-      assert.ok(response.result);
-      assert.ok(couch.isDone());
-    });
-  });
-
-  it('should callback with error after 3 500 responses', async function() {
-    const couch = nock(url)
-      .get('/bad')
-      .times(3)
-      .reply(500, function(uri, requestBody) {
-        this.req.response.statusMessage = 'Internal Server Error';
-        return { error: 'foo', reason: 'bar' };
-      });
-
-    return assert.rejects(
-      db.service.getDocument({ db: db.db, docId: 'bad' }),
-      (err) => {
-        err = error.convertResponseError(err);
-        assert.strictEqual(err.name, 'HTTPFatalError');
-        assert.strictEqual(err.message, `500 Internal Server Error: get ${url}/bad - Error: foo, Reason: bar`);
-        assert.ok(couch.isDone());
-        return true;
-      });
-  }).timeout(longTestTimeout);
-
-  it('should callback with error after 3 POST 503 responses', async function() {
-    const couch = nock(url)
-      .post('/_bulk_get')
-      .query(true)
-      .times(3)
-      .reply(503, function(uri, requestBody) {
-        this.req.response.statusMessage = 'Service Unavailable';
-        return { error: 'service_unavailable', reason: 'Service unavailable' };
-      });
-
-    return assert.rejects(
-      db.service.postBulkGet({ db: db.db, revs: true, docs: [] }),
-      (err) => {
-        err = error.convertResponseError(err);
-        assert.strictEqual(err.name, 'HTTPFatalError');
-        assert.strictEqual(err.message, `503 Service Unavailable: post ${url}/_bulk_get - Error: service_unavailable, Reason: Service unavailable`);
-        assert.ok(couch.isDone());
-        return true;
-      });
-  }).timeout(longTestTimeout);
-
-  it('should callback with error after 3 429 responses', async function() {
-    const couch = nock(url)
-      .get('/bad')
-      .times(3)
-      .reply(429, function(uri, requestBody) {
-        this.req.response.statusMessage = 'Too Many Requests';
-        return { error: 'foo', reason: 'bar' };
-      });
-
-    return assert.rejects(
-      db.service.getDocument({ db: db.db, docId: 'bad' }),
-      (err) => {
-        err = error.convertResponseError(err);
-        assert.strictEqual(err.name, 'HTTPFatalError');
-        assert.strictEqual(err.message, `429 Too Many Requests: get ${url}/bad - Error: foo, Reason: bar`);
-        assert.ok(couch.isDone());
-        return true;
-      });
-  }).timeout(longTestTimeout);
-
-  it('should callback with fatal error for 404 response', async function() {
-    const couch = nock(url)
-      .get('/bad')
-      .reply(404, function(uri, requestBody) {
-        this.req.response.statusMessage = 'Not Found';
-        return { error: 'foo', reason: 'bar' };
-      });
-
-    return assert.rejects(
-      db.service.getDocument({ db: db.db, docId: 'bad' }),
-      (err) => {
-        err = error.convertResponseError(err);
-        assert.strictEqual(err.name, 'HTTPFatalError');
-        assert.strictEqual(err.message, `404 Not Found: get ${url}/bad - Error: foo, Reason: bar`);
-        assert.ok(couch.isDone());
-        return true;
-      });
-  });
-
-  it('should callback with same error for no status code error response', async function() {
-    const couch = nock(url)
-      .get('/bad')
-      .times(3)
-      .replyWithError('testing badness');
-
-    return assert.rejects(
-      db.service.getDocument({ db: db.db, docId: 'bad' }),
-      (err) => {
-        const err2 = error.convertResponseError(err);
-        assert.strictEqual(err, err2);
-        assert.ok(couch.isDone());
-        return true;
-      });
-  }).timeout(longTestTimeout);
-
-  it('should retry request if HTTP request gets timed out', async function() {
-    const couch = nock(url)
-      .post('/_bulk_get')
-      .query(true)
-      .delay(1000)
-      .reply(200, { results: { docs: [{ id: '1', ok: { _id: '1' } }] } })
-      .post('/_bulk_get')
-      .query(true)
-      .reply(200, { results: { docs: [{ id: '1', ok: { _id: '1' } }, { id: '2', ok: { _id: '2' } }] } });
-
-    return timeoutDb.service.postBulkGet({ db: db.db, revs: true, docs: [] }).then((response) => {
-      assert.ok(response);
-      assert.ok(response.result);
-      assert.ok(response.result.results);
-      assert.ok(response.result.results.docs);
-      assert.strictEqual(response.result.results.docs.length, 2);
-      assert.ok(couch.isDone());
-    });
-  });
-
-  it('should callback with error code ESOCKETTIMEDOUT if 3 HTTP requests gets timed out', async function() {
-    // Increase the timeout for this test to allow for the delays
-    this.timeout(3000);
-    const couch = nock(url)
-      .post('/_bulk_get')
-      .query(true)
-      .delay(1000)
-      .times(3)
-      .reply(200, { ok: true });
-
-    return assert.rejects(
-      timeoutDb.service.postBulkGet({ db: db.db, revs: true, docs: [] }),
-      (err) => {
-        err = error.convertResponseError(err);
-        // Note axios returns ECONNABORTED rather than ESOCKETTIMEDOUT
-        // See https://github.com/axios/axios/issues/2710 via https://github.com/axios/axios/issues/1543`
-        assert.strictEqual(err.statusText, 'ECONNABORTED');
-        assert.strictEqual(err.message, `timeout of 500ms exceeded: post ${url}/_bulk_get ECONNABORTED`);
-        assert.ok(couch.isDone());
-        return true;
-      });
-  });
-  describe('#unit Check credentials', async function() {
-    it('should properly decode username and password', async function() {
-      const username = 'user%123';
-      const password = 'colon:at@321';
-      const url = `http://${encodeURIComponent(username)}:${encodeURIComponent(password)}@localhost:7777/testdb`;
-      const sessionUrl = 'http://localhost:7777';
-      const couch = nock(sessionUrl)
-        .post('/_session', { username: username, password: password })
-        .reply(200, { ok: true }, { 'Set-Cookie': 'AuthSession=ABC123DEF4356;' })
-        .get('/')
-        .reply(200);
-      const db = request.client(url, { parallelism: 1 });
-      return db.service.getServerInformation().then(response => {
-        assert.ok(response);
-        assert.ok(couch.isDone());
-      });
-    });
+function deleteIfExists(fileName) {
+  fs.unlink(fileName, function(err) {
+    if (err) {
+      if (err.code !== 'ENOENT') {
+        console.error(`${err.code} ${err.message}`);
+      }
+    }
   });
-});
+}

package/.scannerwork/scanner-report/source-34.txt

@@ -15,176 +15,91 @@
 /* global describe it */
 'use strict';
 
-const assert = require('assert');
-const nock = require('nock');
-const request = require('../includes/request.js');
-const changes = require('../includes/spoolchanges.js');
+const fs = require('fs');
+const { once } = require('node:events');
+const u = require('./citestutils.js');
 
-const url = 'http://localhost:7777';
-const dbName = 'fakenockdb';
-const longTestTimeout = 3000;
-
-const db = request.client(`${url}/${dbName}`, { parallelism: 1 });
-
-const seqSuffix = Buffer.alloc(124, 'abc123').toString('base64');
-function provideChanges(batchSize, totalChanges, fullResponse = false) {
-  let pending = totalChanges;
-  const sparseResultsArray = Array(batchSize).fill({
-    seq: null,
-    id: 'doc',
-    changes: [{ rev: '1-abcdef0123456789abcdef0123456789' }]
-  });
-  nock(url)
-    .post(`/${dbName}/_changes`)
-    .query(true)
-    .times(totalChanges / batchSize + (totalChanges % batchSize > 0 ? 1 : 0))
-    .reply(200, (uri, requestBody) => {
-      pending -= batchSize;
-      const lastSeq = (totalChanges - pending);
-      const seq = lastSeq - batchSize;
-      return {
-        results: fullResponse
-          ? Array.from(Array(batchSize), (_, i) => {
-            return {
-              seq: `${seq + i}-${seqSuffix}`,
-              id: `doc${seq + i}`,
-              changes: [{ rev: '1-abcdef0123456789abcdef0123456789' }]
-            };
-          })
-          : sparseResultsArray,
-        pending: pending,
-        last_seq: `${lastSeq}-abc`
-      };
-    });
-}
-
-describe('#unit Check spool changes', function() {
-  it('should terminate on request error', async function() {
-    nock(url)
-      .post(`/${dbName}/_changes`)
-      .query(true)
-      .times(3)
-      .replyWithError({ code: 'ECONNRESET', message: 'socket hang up' });
-
-    return new Promise((resolve, reject) => {
-      changes(db, '/dev/null', 500, null, function(err) {
-        try {
-          assert.strictEqual(err.name, 'SpoolChangesError');
-          assert.strictEqual(err.message, `Failed changes request - socket hang up: post ${url}/${dbName}/_changes`);
-          assert.ok(nock.isDone());
-          resolve();
-        } catch (err) {
-          reject(err);
-        }
-      });
+[{ useApi: true }, { useApi: false }].forEach(function(params) {
+  describe(u.scenario('Basic backup and restore', params), function() {
+    it('should backup animaldb to a file correctly', async function() {
+      // Allow up to 40 s to backup and compare (it should be much faster)!
+      u.setTimeout(this, 40);
+      const actualBackup = `./${this.fileName}`;
+      // Create a file and backup to it
+      const output = fs.createWriteStream(actualBackup);
+      return once(output, 'open')
+        .then(() => {
+          return u.testBackup(params, 'animaldb', output);
+        }).then(() => {
+          return u.readSortAndDeepEqual(actualBackup, './test/fixtures/animaldb_expected.json');
+        });
     });
-  }).timeout(longTestTimeout);
 
-  it('should terminate on bad HTTP status code response', async function() {
-    nock(url)
-      .post(`/${dbName}/_changes`)
-      .query(true)
-      .times(3)
-      .reply(500, function(uri, requestBody) {
-        this.req.response.statusMessage = 'Internal Server Error';
-        return { error: 'foo', reason: 'bar' };
-      });
-    return new Promise((resolve, reject) => {
-      changes(db, '/dev/null', 500, null, function(err) {
-        try {
-          assert.strictEqual(err.name, 'HTTPFatalError');
-          assert.strictEqual(err.message, `500 Internal Server Error: post ${url}/${dbName}/_changes - Error: foo, Reason: bar`);
-          assert.ok(nock.isDone());
-          resolve();
-        } catch (err) {
-          reject(err);
-        }
+    it('should restore animaldb to a database correctly', async function() {
+      // Allow up to 60 s to restore and compare (again it should be faster)!
+      u.setTimeout(this, 60);
+      const input = fs.createReadStream('./test/fixtures/animaldb_expected.json');
+      const dbName = this.dbName;
+      return once(input, 'open').then(() => {
+        return u.testRestore(params, input, dbName);
+      }).then(() => {
+        return u.dbCompare('animaldb', dbName);
       });
     });
-  }).timeout(longTestTimeout);
 
-  it('should keep collecting changes', async function() {
-    // This test validates that spooling will correctly
-    // continue across multiple requests
-    // (4 batches of 100000 to be precise).
-    // This test might take up to 10 seconds
-    this.timeout(10 * 1000);
-
-    // Use full changes for this test
-    provideChanges(100000, 400000, true);
-    return new Promise((resolve, reject) => {
-      changes(db, '/dev/null', 500, null, function(err) {
-        try {
-          assert.ok(!err);
-          assert.ok(nock.isDone());
-          resolve();
-        } catch (err) {
-          reject(err);
-        }
-      });
+    it('should execute a shallow mode backup successfully', async function() {
+      // Allow 30 s
+      u.setTimeout(this, 30);
+      const actualBackup = `./${this.fileName}`;
+      const output = fs.createWriteStream(actualBackup);
+      // Add the shallow mode option
+      const p = u.p(params, { opts: { mode: 'shallow' } });
+      return once(output, 'open')
+        .then(() => {
+          return u.testBackup(p, 'animaldb', output);
+        }).then(() => {
+          return u.readSortAndDeepEqual(actualBackup, './test/fixtures/animaldb_expected_shallow.json');
+        });
     });
-  });
 
-  it('should keep collecting sparse changes', async function() {
-    // This test checks that making thousands of requests doesn't
-    // make anything bad happen.
-    // This test might take up to 25 seconds
-    this.timeout(25 * 1000);
-    // Use sparse changes for this test and a batch size of 1
-    provideChanges(1, 2500);
-    return new Promise((resolve, reject) => {
-      changes(db, '/dev/null', 500, null, function(err) {
-        try {
-          assert.ok(!err);
-          assert.ok(nock.isDone());
-          resolve();
-        } catch (err) {
-          reject(err);
-        }
+    describe(u.scenario('Buffer size tests', params), function() {
+      it('should backup/restore animaldb with the same buffer size', async function() {
+        // Allow up to 60 s for backup and restore of animaldb
+        u.setTimeout(this, 60);
+        const actualBackup = `./${this.fileName}`;
+        const logFile = `./${this.fileName}` + '.log';
+        const p = u.p(params, { opts: { log: logFile, bufferSize: 1 } });
+        return u.testBackupAndRestoreViaFile(p, 'animaldb', actualBackup, this.dbName);
       });
-    });
-  });
-});
 
-describe('Longer spool changes checks', function() {
-  it('#slow should keep collecting changes (25M)', async function() {
-    // This test might take up to 5 minutes
-    this.timeout(5 * 60 * 1000);
-    // Note changes spooling uses a constant batch size, we are setting
-    // a test value here and setting the buffer to match
-    const batch = 100000;
-    // Use sparse changes for this test
-    provideChanges(batch, 25000000);
-    return new Promise((resolve, reject) => {
-      changes(db, '/dev/null', batch, null, function(err) {
-        try {
-          assert.ok(!err);
-          assert.ok(nock.isDone());
-          resolve();
-        } catch (err) {
-          reject(err);
-        }
+      it('should backup/restore animaldb with backup buffer > restore buffer', async function() {
+        // Allow up to 60 s for backup and restore of animaldb
+        u.setTimeout(this, 60);
+        const actualBackup = `./${this.fileName}`;
+        const logFile = `./${this.fileName}` + '.log';
+        const dbName = this.dbName;
+        const p = u.p(params, { opts: { log: logFile, bufferSize: 2 } }); // backup
+        const q = u.p(params, { opts: { bufferSize: 1 } }); // restore
+        return u.testBackupToFile(p, 'animaldb', actualBackup).then(() => {
+          return u.testRestoreFromFile(q, actualBackup, dbName);
+        }).then(() => {
+          return u.dbCompare('animaldb', dbName);
+        });
       });
-    });
-  });
 
-  it('#slower should keep collecting changes (500M)', async function() {
-    // This test might take up to 90 minutes
-    this.timeout(90 * 60 * 1000);
-    // Note changes spooling uses a constant batch size, we are setting
-    // a test value here and setting the buffer to match
-    const batch = 1000000;
-    // Use full changes for this test to exercise load
-    provideChanges(batch, 500000000, true);
-    return new Promise((resolve, reject) => {
-      changes(db, '/dev/null', batch, null, function(err) {
-        try {
-          assert.ok(!err);
-          assert.ok(nock.isDone());
-          resolve();
-        } catch (err) {
-          reject(err);
-        }
+      it('should backup/restore animaldb with backup buffer < restore buffer', async function() {
+        // Allow up to 60 s for backup and restore of animaldb
+        u.setTimeout(this, 60);
+        const actualBackup = `./${this.fileName}`;
+        const logFile = `./${this.fileName}` + '.log';
+        const dbName = this.dbName;
+        const p = u.p(params, { opts: { log: logFile, bufferSize: 1 } }); // backup
+        const q = u.p(params, { opts: { bufferSize: 2 } }); // restore
+        return u.testBackupToFile(p, 'animaldb', actualBackup).then(() => {
+          return u.testRestoreFromFile(q, actualBackup, dbName);
+        }).then(() => {
+          return u.dbCompare('animaldb', dbName);
+        });
       });
     });
   });

package/.scannerwork/scanner-report/source-35.txt

@@ -1,4 +1,4 @@
-// Copyright © 2017, 2023 IBM Corp. All rights reserved.
+// Copyright © 2018, 2023 IBM Corp. All rights reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -15,23 +15,61 @@
 /* global describe it */
 'use strict';
 
+const fs = require('fs');
+const { once } = require('node:events');
+const readline = require('readline');
 const u = require('./citestutils.js');
+const uuid = require('uuid').v4;
 
-describe('Event tests', function() {
-  it('should get a finished event when using stdout', async function() {
-    u.setTimeout(this, 40);
-    // Use the API so we can get events, pass eventEmitter so we get the emitter back
-    const params = { useApi: true, useStdOut: true };
-    // All API backups now set an event listener for finished and it is part of the backup
-    // promise, so if the backup passes the finished event fired.
-    return u.testBackup(params, 'animaldb', process.stdout);
-  });
+const params = { useApi: true };
+
+describe(u.scenario('Concurrent database backups', params), function() {
+  it('should run concurrent API database backups correctly #slower', async function() {
+    // Allow up to 900 s to backup and compare (it should be much faster)!
+    u.setTimeout(this, 900);
+
+    const checkForEmptyBatches = async function(fileName) {
+      let foundEmptyBatch = false;
+
+      const rd = readline.createInterface({
+        input: fs.createReadStream(fileName),
+        output: fs.createWriteStream('/dev/null'),
+        terminal: false
+      });
+
+      rd.on('line', function(line) {
+        if (JSON.parse(line).length === 0) {
+          // Note: Empty batch arrays indicate that the running backup is
+          // incorrectly sharing a log file with another ongoing backup job.
+          foundEmptyBatch = true;
+        }
+      });
+
+      rd.on('close', function() {
+        if (foundEmptyBatch) {
+          return Promise.reject(new Error(`Log file '${fileName}' contains empty batches`));
+        } else {
+          return Promise.resolve();
+        }
+      });
+    };
+
+    const backupPromise = async function() {
+      const actualBackup = `./${uuid()}`;
+      const output = fs.createWriteStream(actualBackup);
+      return once(output, 'open').then(() => {
+        return u.testBackup(params, 'largedb1g', output);
+      }).then(() => {
+        return checkForEmptyBatches(actualBackup);
+      });
+    };
+
+    // [1] Run 'largedb1g' database backup
+    const backup1 = backupPromise();
+
+    // [2] Run 'largedb1g' database backup
+    const backup2 = backupPromise();
 
-  it('should get a finished event when using file output', async function() {
-    u.setTimeout(this, 40);
-    // Use the API so we can get events, pass eventEmitter so we get the emitter back
-    const params = { useApi: true };
-    const actualBackup = `./${this.fileName}`;
-    return u.testBackupToFile(params, 'animaldb', actualBackup);
+    return Promise.all([backup1, backup2]);
   });
 });