@cloudant/couchbackup 2.9.15 → 2.9.16-SNAPSHOT.179

package/.scannerwork/scanner-report/source-36.txt

@@ -1,4 +1,4 @@
-// Copyright © 2023 IBM Corp. All rights reserved.
+// Copyright © 2017, 2023 IBM Corp. All rights reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -12,140 +12,51 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-/* global */
+/* global describe it */
 'use strict';
 
-const { fork, spawn } = require('node:child_process');
+const fs = require('fs');
 const { once } = require('node:events');
-const { Duplex } = require('node:stream');
-const debug = require('debug');
-const logProcess = debug('couchbackup:test:process');
+const u = require('./citestutils.js');
 
-class TestProcess {
-  constructor(cmd, args, mode) {
-    this.cmd = cmd;
-    // Child process stdio [stdin, stdout, stderr, ...extra channels]
-    const childProcessOptions = { stdio: [] };
-    switch (mode) {
-      case 'readable':
-        // Readable only, no writing to stdin so ignore it
-        childProcessOptions.stdio = ['ignore', 'pipe', 'inherit'];
-        break;
-      case 'writable':
-        // Writable only, no reading from stdout so ignore it
-        childProcessOptions.stdio = ['pipe', 'ignore', 'inherit'];
-        break;
-      default:
-        // Default Duplex mode pipe both stdin and stdout
-        childProcessOptions.stdio = ['pipe', 'pipe', 'inherit'];
-        break;
-    }
-    if (cmd.endsWith('.js')) {
-      // Add Node fork ipc channel
-      childProcessOptions.stdio.push('ipc');
-      logProcess(`Forking Node process for ${cmd} with stdio:[${childProcessOptions.stdio}]`);
-      this.childProcess = fork(cmd, args, childProcessOptions);
-    } else {
-      logProcess(`Spawning process for ${cmd} with stdio:[${childProcessOptions.stdio}]`);
-      this.childProcess = spawn(cmd, args, childProcessOptions);
-    }
+[{ useApi: true }, { useApi: false }].forEach(function(params) {
+  describe(u.scenario('Compression tests', params), function() {
+    const p = u.p(params, { compression: true });
 
-    this.childProcessPromise = once(this.childProcess, 'close').then(() => {
-      const code = this.childProcess.exitCode;
-      const signal = this.childProcess.signalCode;
-      logProcess(`Test process ${cmd} closed with code ${code} and signal ${signal}`);
-      if (code === 0) {
-        logProcess(`Resolving process promise for ${cmd}`);
-        return Promise.resolve(code);
-      } else {
-        const e = new Error(`Test child process ${cmd} exited with code ${code} and ${signal}. This may be normal for error case testing.`);
-        e.code = code;
-        e.signal = signal;
-        logProcess(`Will reject process promise for ${cmd} with ${e}`);
-        return Promise.reject(e);
-      }
+    it('should backup animaldb to a compressed file', async function() {
+      // Allow up to 60 s for backup of animaldb
+      u.setTimeout(this, 60);
+      const compressedBackup = `./${this.fileName}`;
+      const output = fs.createWriteStream(compressedBackup);
+      return once(output, 'open')
+        .then(() => {
+          return u.testBackup(p, 'animaldb', output);
+        }).then(() => {
+          return u.assertGzipFile(compressedBackup);
+        });
     });
 
-    switch (mode) {
-      case 'readable':
-        this.duplexFrom = this.childProcess.stdout;
-        break;
-      case 'writable':
-        this.duplexFrom = this.childProcess.stdin;
-        break;
-      default:
-        // Default is duplex
-        this.duplexFrom = { writable: this.childProcess.stdin, readable: this.childProcess.stdout };
-    }
+    it('should backup and restore animaldb via a compressed file', async function() {
+      // Allow up to 60 s for backup and restore of animaldb
+      u.setTimeout(this, 60);
+      const compressedBackup = `./${this.fileName}`;
+      return u.testBackupAndRestoreViaFile(p, 'animaldb', compressedBackup, this.dbName).then(() => {
+        return u.assertGzipFile(compressedBackup);
+      });
+    });
 
-    this.stream = Duplex.from(this.duplexFrom);
-  }
-}
+    it('should backup and restore animaldb via a compressed stream', async function() {
+      // Allow up to 60 s for backup and restore of animaldb
+      u.setTimeout(this, 60);
+      return u.testDirectBackupAndRestore(p, 'animaldb', this.dbName);
+    });
 
-module.exports = {
-  TestProcess,
-  cliBackup: function(databaseName, params = {}) {
-    const args = ['--db', databaseName];
-    if (params.opts) {
-      if (params.opts.mode) {
-        args.push('--mode');
-        args.push(params.opts.mode);
-      }
-      if (params.opts.output) {
-        args.push('--output');
-        args.push(params.opts.output);
-      }
-      if (params.opts.log) {
-        args.push('--log');
-        args.push(params.opts.log);
-      }
-      if (params.opts.resume) {
-        args.push('--resume');
-        args.push(params.opts.resume);
-      }
-      if (params.opts.bufferSize) {
-        args.push('--buffer-size');
-        args.push(params.opts.bufferSize);
-      }
-      if (params.opts.iamApiKey) {
-        args.push('--iam-api-key');
-        args.push(params.opts.iamApiKey);
-      }
-    }
-    return new TestProcess('./bin/couchbackup.bin.js', args, 'readable');
-  },
-  cliRestore: function(databaseName, params) {
-    const args = ['--db', databaseName];
-    if (params.opts) {
-      if (params.opts.bufferSize) {
-        args.push('--buffer-size');
-        args.push(params.opts.bufferSize);
-      }
-      if (params.opts.parallelism) {
-        args.push('--parallelism');
-        args.push(params.opts.parallelism);
-      }
-      if (params.opts.requestTimeout) {
-        args.push('--request-timeout');
-        args.push(params.opts.requestTimeout);
-      }
-      if (params.opts.iamApiKey) {
-        args.push('--iam-api-key');
-        args.push(params.opts.iamApiKey);
-      }
-    }
-    return new TestProcess('./bin/couchrestore.bin.js', args, 'writable');
-  },
-  cliGzip: function() {
-    return new TestProcess('gzip', []);
-  },
-  cliGunzip: function() {
-    return new TestProcess('gunzip', []);
-  },
-  cliEncrypt: function() {
-    return new TestProcess('openssl', ['aes-128-cbc', '-pass', 'pass:12345']);
-  },
-  cliDecrypt: function() {
-    return new TestProcess('openssl', ['aes-128-cbc', '-d', '-pass', 'pass:12345']);
-  }
-};
+    it('should backup and restore largedb2g via a compressed file #slower', async function() {
+      // Takes ~ 25 min using CLI, but sometimes over an hour with API
+      u.setTimeout(this, 180 * 60);
+      const compressedBackup = `./${this.fileName}`;
+      params.compression = true;
+      return u.testBackupAndRestoreViaFile(p, 'largedb2g', compressedBackup, this.dbName);
+    });
+  });
+});

package/.scannerwork/scanner-report/source-37.txt

@@ -12,180 +12,123 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-/* global describe it */
+/* global describe it beforeEach */
 'use strict';
 
 const assert = require('assert');
+const fs = require('fs');
 const nock = require('nock');
 const request = require('../includes/request.js');
-const changes = require('../includes/spoolchanges.js');
-
-const url = 'http://localhost:7777';
-const dbName = 'fakenockdb';
+const writer = require('../includes/writer.js');
+const noopEmitter = new (require('events')).EventEmitter();
+const liner = require('../includes/liner.js');
+const { once } = require('node:events');
+const { pipeline } = require('node:stream/promises');
 const longTestTimeout = 3000;
 
-const db = request.client(`${url}/${dbName}`, { parallelism: 1 });
+describe('#unit Check database restore writer', function() {
+  const dbUrl = 'http://localhost:5984/animaldb';
+  const db = request.client(dbUrl, { parallelism: 1 });
 
-const seqSuffix = Buffer.alloc(124, 'abc123').toString('base64');
-function provideChanges(batchSize, totalChanges, fullResponse = false) {
-  let pending = totalChanges;
-  const sparseResultsArray = Array(batchSize).fill({
-    seq: null,
-    id: 'doc',
-    changes: [{ rev: '1-abcdef0123456789abcdef0123456789' }]
+  beforeEach('Reset nocks', function() {
+    nock.cleanAll();
   });
-  nock(url)
-    .post(`/${dbName}/_changes`)
-    .query(true)
-    .times(totalChanges / batchSize + (totalChanges % batchSize > 0 ? 1 : 0))
-    .reply(200, (uri, requestBody) => {
-      pending -= batchSize;
-      const lastSeq = (totalChanges - pending);
-      const seq = lastSeq - batchSize;
-      return {
-        results: fullResponse
-          ? Array.from(Array(batchSize), (_, i) => {
-            return {
-              seq: `${seq + i}-${seqSuffix}`,
-              id: `doc${seq + i}`,
-              changes: [{ rev: '1-abcdef0123456789abcdef0123456789' }]
-            };
-          })
-          : sparseResultsArray,
-        pending: pending,
-        last_seq: `${lastSeq}-abc`
-      };
-    });
-}
-
-describe('#unit Check spool changes', function() {
-  it('should terminate on request error', async function() {
-    nock(url)
-      .post(`/${dbName}/_changes`)
-      .query(true)
-      .times(3)
-      .replyWithError({ code: 'ECONNRESET', message: 'socket hang up' });
-
-    return new Promise((resolve, reject) => {
-      changes(db, '/dev/null', 500, null, function(err) {
-        try {
-          assert.strictEqual(err.name, 'SpoolChangesError');
-          assert.strictEqual(err.message, `Failed changes request - socket hang up: post ${url}/${dbName}/_changes`);
-          assert.ok(nock.isDone());
-          resolve();
-        } catch (err) {
-          reject(err);
-        }
-      });
-    });
-  }).timeout(longTestTimeout);
 
-  it('should terminate on bad HTTP status code response', async function() {
-    nock(url)
-      .post(`/${dbName}/_changes`)
-      .query(true)
-      .times(3)
-      .reply(500, function(uri, requestBody) {
-        this.req.response.statusMessage = 'Internal Server Error';
-        return { error: 'foo', reason: 'bar' };
-      });
-    return new Promise((resolve, reject) => {
-      changes(db, '/dev/null', 500, null, function(err) {
-        try {
-          assert.strictEqual(err.name, 'HTTPFatalError');
-          assert.strictEqual(err.message, `500 Internal Server Error: post ${url}/${dbName}/_changes - Error: foo, Reason: bar`);
-          assert.ok(nock.isDone());
-          resolve();
-        } catch (err) {
-          reject(err);
-        }
-      });
-    });
-  }).timeout(longTestTimeout);
+  it('should complete successfully', async function() {
+    nock(dbUrl)
+      .post('/_bulk_docs')
+      .reply(200, []); // success
 
-  it('should keep collecting changes', async function() {
-    // This test validates that spooling will correctly
-    // continue across multiple requests
-    // (4 batches of 100000 to be precise).
-    // This test might take up to 10 seconds
-    this.timeout(10 * 1000);
-
-    // Use full changes for this test
-    provideChanges(100000, 400000, true);
-    return new Promise((resolve, reject) => {
-      changes(db, '/dev/null', 500, null, function(err) {
-        try {
-          assert.ok(!err);
-          assert.ok(nock.isDone());
-          resolve();
-        } catch (err) {
-          reject(err);
-        }
-      });
-    });
+    const w = writer(db, 500, 1, noopEmitter);
+    return Promise.all([pipeline(fs.createReadStream('./test/fixtures/animaldb_expected.json'), liner(), w),
+      once(w, 'finished').then((data) => {
+        assert.strictEqual(data[0].total, 15);
+        assert.ok(nock.isDone());
+      })]);
   });
 
-  it('should keep collecting sparse changes', async function() {
-    // This test checks that making thousands of requests doesn't
-    // make anything bad happen.
-    // This test might take up to 25 seconds
-    this.timeout(25 * 1000);
-    // Use sparse changes for this test and a batch size of 1
-    provideChanges(1, 2500);
-    return new Promise((resolve, reject) => {
-      changes(db, '/dev/null', 500, null, function(err) {
-        try {
-          assert.ok(!err);
-          assert.ok(nock.isDone());
-          resolve();
-        } catch (err) {
-          reject(err);
-        }
-      });
-    });
+  it('should terminate on a fatal error', async function() {
+    nock(dbUrl)
+      .post('/_bulk_docs')
+      .reply(401, { error: 'Unauthorized' }); // fatal error
+
+    const w = writer(db, 500, 1, noopEmitter);
+    return assert.rejects(
+      pipeline(fs.createReadStream('./test/fixtures/animaldb_expected.json'), liner(), w),
+      (err) => {
+        assert.strictEqual(err.name, 'Unauthorized');
+        assert.strictEqual(err.message, 'Access is denied due to invalid credentials.');
+        assert.ok(nock.isDone());
+        return true;
+      }
+    );
   });
-});
 
-describe('Longer spool changes checks', function() {
-  it('#slow should keep collecting changes (25M)', async function() {
-    // This test might take up to 5 minutes
-    this.timeout(5 * 60 * 1000);
-    // Note changes spooling uses a constant batch size, we are setting
-    // a test value here and setting the buffer to match
-    const batch = 100000;
-    // Use sparse changes for this test
-    provideChanges(batch, 25000000);
-    return new Promise((resolve, reject) => {
-      changes(db, '/dev/null', batch, null, function(err) {
-        try {
-          assert.ok(!err);
-          assert.ok(nock.isDone());
-          resolve();
-        } catch (err) {
-          reject(err);
-        }
-      });
-    });
+  it('should retry on transient errors', async function() {
+    nock(dbUrl)
+      .post('/_bulk_docs')
+      .reply(429, { error: 'Too Many Requests' }) // transient error
+      .post('/_bulk_docs')
+      .reply(500, { error: 'Internal Server Error' }) // transient error
+      .post('/_bulk_docs')
+      .reply(200, { ok: true }); // third time lucky success
+
+    const w = writer(db, 500, 1, noopEmitter);
+    return Promise.all([pipeline(fs.createReadStream('./test/fixtures/animaldb_expected.json'), liner(), w),
+      once(w, 'finished').then((data) => {
+        assert.strictEqual(data[0].total, 15);
+        assert.ok(nock.isDone());
+      })]);
+  }).timeout(longTestTimeout);
+
+  it('should fail after 3 transient errors', async function() {
+    nock(dbUrl)
+      .post('/_bulk_docs')
+      .reply(429, { error: 'Too Many Requests' }) // transient error
+      .post('/_bulk_docs')
+      .reply(500, { error: 'Internal Server Error' }) // transient error
+      .post('/_bulk_docs')
+      .reply(503, { error: 'Service Unavailable' }); // Final transient error
+
+    const w = writer(db, 500, 1, noopEmitter);
+    return assert.rejects(
+      pipeline(fs.createReadStream('./test/fixtures/animaldb_expected.json'), liner(), w),
+      (err) => {
+        assert.strictEqual(err.name, 'HTTPFatalError');
+        assert.strictEqual(err.message, `503 : post ${dbUrl}/_bulk_docs - Error: Service Unavailable`);
+        assert.ok(nock.isDone());
+        return true;
+      }
+    );
+  }).timeout(longTestTimeout);
+
+  it('should restore shallow backups without rev info successfully', async function() {
+    nock(dbUrl)
+      .post('/_bulk_docs')
+      .reply(200, [{ ok: true, id: 'foo', rev: '1-abc' }]); // success
+
+    const w = writer(db, 500, 1, noopEmitter);
+    return Promise.all([pipeline(fs.createReadStream('./test/fixtures/animaldb_old_shallow.json'), liner(), w),
+      once(w, 'finished').then((data) => {
+        assert.strictEqual(data[0].total, 11);
+        assert.ok(nock.isDone());
+      })]);
   });
 
-  it('#slower should keep collecting changes (500M)', async function() {
-    // This test might take up to 90 minutes
-    this.timeout(90 * 60 * 1000);
-    // Note changes spooling uses a constant batch size, we are setting
-    // a test value here and setting the buffer to match
-    const batch = 1000000;
-    // Use full changes for this test to exercise load
-    provideChanges(batch, 500000000, true);
-    return new Promise((resolve, reject) => {
-      changes(db, '/dev/null', batch, null, function(err) {
-        try {
-          assert.ok(!err);
-          assert.ok(nock.isDone());
-          resolve();
-        } catch (err) {
-          reject(err);
-        }
-      });
-    });
+  it('should get a batch error for non-empty array response with new_edits false', async function() {
+    nock(dbUrl)
+      .post('/_bulk_docs')
+      .reply(200, [{ id: 'foo', error: 'foo', reason: 'bar' }]);
+
+    const w = writer(db, 500, 1, noopEmitter);
+    return assert.rejects(
+      pipeline(fs.createReadStream('./test/fixtures/animaldb_expected.json'), liner(), w),
+      (err) => {
+        assert.strictEqual(err.name, 'Error');
+        assert.strictEqual(err.message, 'Error writing batch with new_edits:false and 1 items');
+        assert.ok(nock.isDone());
+        return true;
+      }
+    );
   });
 });

package/.scannerwork/scanner-report/source-38.txt

@@ -16,20 +16,171 @@
 'use strict';
 
 const assert = require('assert');
-const logfilesummary = require('../includes/logfilesummary.js');
+const nock = require('nock');
+const request = require('../includes/request.js');
+const changes = require('../includes/spoolchanges.js');
 
-describe('#unit Fetching summary from the log file', function() {
-  it('should fetch a summary correctly', function() {
+const url = 'http://localhost:7777';
+const dbName = 'fakenockdb';
+const longTestTimeout = 3000;
+
+const db = request.client(`${url}/${dbName}`, { parallelism: 1 });
+
+const seqSuffix = Buffer.alloc(124, 'abc123').toString('base64');
+function provideChanges(batchSize, totalChanges, fullResponse = false) {
+  let pending = totalChanges;
+  const sparseResultsArray = Array(batchSize).fill({
+    seq: null,
+    id: 'doc',
+    changes: [{ rev: '1-abcdef0123456789abcdef0123456789' }]
+  });
+  nock(url)
+    .post(`/${dbName}/_changes`)
+    .query(true)
+    .times(totalChanges / batchSize + (totalChanges % batchSize > 0 ? 1 : 0))
+    .reply(200, (uri, requestBody) => {
+      pending -= batchSize;
+      const lastSeq = (totalChanges - pending);
+      const seq = lastSeq - batchSize;
+      return {
+        results: fullResponse
+          ? Array.from(Array(batchSize), (_, i) => {
+            return {
+              seq: `${seq + i}-${seqSuffix}`,
+              id: `doc${seq + i}`,
+              changes: [{ rev: '1-abcdef0123456789abcdef0123456789' }]
+            };
+          })
+          : sparseResultsArray,
+        pending: pending,
+        last_seq: `${lastSeq}-abc`
+      };
+    });
+}
+
+describe('#unit Check spool changes', function() {
+  it('should terminate on request error', async function() {
+    nock(url)
+      .post(`/${dbName}/_changes`)
+      .query(true)
+      .times(3)
+      .replyWithError({ code: 'ECONNRESET', message: 'socket hang up' });
+
+    return new Promise((resolve, reject) => {
+      changes(db, '/dev/null', 500, null, function(err) {
+        try {
+          assert.strictEqual(err.name, 'SpoolChangesError');
+          assert.strictEqual(err.message, `Failed changes request - socket hang up: post ${url}/${dbName}/_changes`);
+          assert.ok(nock.isDone());
+          resolve();
+        } catch (err) {
+          reject(err);
+        }
+      });
+    });
+  }).timeout(longTestTimeout);
+
+  it('should terminate on bad HTTP status code response', async function() {
+    nock(url)
+      .post(`/${dbName}/_changes`)
+      .query(true)
+      .times(3)
+      .reply(500, function(uri, requestBody) {
+        this.req.response.statusMessage = 'Internal Server Error';
+        return { error: 'foo', reason: 'bar' };
+      });
+    return new Promise((resolve, reject) => {
+      changes(db, '/dev/null', 500, null, function(err) {
+        try {
+          assert.strictEqual(err.name, 'HTTPFatalError');
+          assert.strictEqual(err.message, `500 Internal Server Error: post ${url}/${dbName}/_changes - Error: foo, Reason: bar`);
+          assert.ok(nock.isDone());
+          resolve();
+        } catch (err) {
+          reject(err);
+        }
+      });
+    });
+  }).timeout(longTestTimeout);
+
+  it('should keep collecting changes', async function() {
+    // This test validates that spooling will correctly
+    // continue across multiple requests
+    // (4 batches of 100000 to be precise).
+    // This test might take up to 10 seconds
+    this.timeout(10 * 1000);
+
+    // Use full changes for this test
+    provideChanges(100000, 400000, true);
+    return new Promise((resolve, reject) => {
+      changes(db, '/dev/null', 500, null, function(err) {
+        try {
+          assert.ok(!err);
+          assert.ok(nock.isDone());
+          resolve();
+        } catch (err) {
+          reject(err);
+        }
+      });
+    });
+  });
+
+  it('should keep collecting sparse changes', async function() {
+    // This test checks that making thousands of requests doesn't
+    // make anything bad happen.
+    // This test might take up to 25 seconds
+    this.timeout(25 * 1000);
+    // Use sparse changes for this test and a batch size of 1
+    provideChanges(1, 2500);
+    return new Promise((resolve, reject) => {
+      changes(db, '/dev/null', 500, null, function(err) {
+        try {
+          assert.ok(!err);
+          assert.ok(nock.isDone());
+          resolve();
+        } catch (err) {
+          reject(err);
+        }
+      });
+    });
+  });
+});
+
+describe('Longer spool changes checks', function() {
+  it('#slow should keep collecting changes (25M)', async function() {
+    // This test might take up to 5 minutes
+    this.timeout(5 * 60 * 1000);
+    // Note changes spooling uses a constant batch size, we are setting
+    // a test value here and setting the buffer to match
+    const batch = 100000;
+    // Use sparse changes for this test
+    provideChanges(batch, 25000000);
+    return new Promise((resolve, reject) => {
+      changes(db, '/dev/null', batch, null, function(err) {
+        try {
+          assert.ok(!err);
+          assert.ok(nock.isDone());
+          resolve();
+        } catch (err) {
+          reject(err);
+        }
+      });
+    });
+  });
+
+  it('#slower should keep collecting changes (500M)', async function() {
+    // This test might take up to 90 minutes
+    this.timeout(90 * 60 * 1000);
+    // Note changes spooling uses a constant batch size, we are setting
+    // a test value here and setting the buffer to match
+    const batch = 1000000;
+    // Use full changes for this test to exercise load
+    provideChanges(batch, 500000000, true);
     return new Promise((resolve, reject) => {
-      logfilesummary('./test/fixtures/test.log', function(err, data) {
+      changes(db, '/dev/null', batch, null, function(err) {
         try {
           assert.ok(!err);
-          assert.ok(data);
-          assert.strictEqual(data.changesComplete, true);
-          assert.strictEqual(typeof data.batches, 'object');
-          assert.strictEqual(Object.keys(data.batches).length, 2);
-          assert.deepStrictEqual(data.batches['1'], true);
-          assert.deepStrictEqual(data.batches['4'], true);
+          assert.ok(nock.isDone());
           resolve();
         } catch (err) {
           reject(err);