@cloudant/couchbackup 2.9.15-SNAPSHOT.175 → 2.9.15

package/.scannerwork/scanner-report/source-36.txt

@@ -1,4 +1,4 @@
-// Copyright © 2017, 2023 IBM Corp. All rights reserved.
+// Copyright © 2023 IBM Corp. All rights reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -12,172 +12,140 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-/* global describe it beforeEach */
+/* global */
 'use strict';
 
-const assert = require('assert');
-const backup = require('../includes/shallowbackup.js');
-const request = require('../includes/request.js');
-const fs = require('fs');
-const nock = require('nock');
+const { fork, spawn } = require('node:child_process');
+const { once } = require('node:events');
+const { Duplex } = require('node:stream');
+const debug = require('debug');
+const logProcess = debug('couchbackup:test:process');
 
-// Function to create a DB object and call the shallow backup function
-// This is normally done by app.js
-function shallowBackup(dbUrl, opts) {
-  const db = request.client(dbUrl, opts);
-  // Disable compression to make body assertions easier
-  db.service.setEnableGzipCompression(false);
-  return backup(db, opts);
-}
-
-// Note all these tests include a body parameter of include_docs and a query
-// string of include_docs because of a quirk of nano that when using the fetch
-// method always adds the include_docs query string.
-describe('#unit Perform backup using shallow backup', function() {
-  const dbUrl = 'http://localhost:5984/animaldb';
-  // Query string keys are stringified by Nano
-  const badgerKey = 'badger\0';
-  const kookaburraKey = 'kookaburra\0';
-  const snipeKey = 'snipe\0';
-
-  beforeEach('Reset nocks', function() {
-    nock.cleanAll();
-  });
+class TestProcess {
+  constructor(cmd, args, mode) {
+    this.cmd = cmd;
+    // Child process stdio [stdin, stdout, stderr, ...extra channels]
+    const childProcessOptions = { stdio: [] };
+    switch (mode) {
+      case 'readable':
+        // Readable only, no writing to stdin so ignore it
+        childProcessOptions.stdio = ['ignore', 'pipe', 'inherit'];
+        break;
+      case 'writable':
+        // Writable only, no reading from stdout so ignore it
+        childProcessOptions.stdio = ['pipe', 'ignore', 'inherit'];
+        break;
+      default:
+        // Default Duplex mode pipe both stdin and stdout
+        childProcessOptions.stdio = ['pipe', 'pipe', 'inherit'];
+        break;
+    }
+    if (cmd.endsWith('.js')) {
+      // Add Node fork ipc channel
+      childProcessOptions.stdio.push('ipc');
+      logProcess(`Forking Node process for ${cmd} with stdio:[${childProcessOptions.stdio}]`);
+      this.childProcess = fork(cmd, args, childProcessOptions);
+    } else {
+      logProcess(`Spawning process for ${cmd} with stdio:[${childProcessOptions.stdio}]`);
+      this.childProcess = spawn(cmd, args, childProcessOptions);
+    }
 
-  it('should perform a shallow backup', async function() {
-    const couch = nock(dbUrl)
-      // batch 1
-      .post('/_all_docs', { limit: 3, include_docs: true })
-      .reply(200, JSON.parse(fs.readFileSync('./test/fixtures/animaldb_all_docs_1.json', 'utf8')))
-      // batch 2
-      .post('/_all_docs', { limit: 3, start_key: badgerKey, include_docs: true })
-      .reply(200, JSON.parse(fs.readFileSync('./test/fixtures/animaldb_all_docs_2.json', 'utf8')))
-      // batch 3
-      .post('/_all_docs', { limit: 3, start_key: kookaburraKey, include_docs: true })
-      .reply(200, JSON.parse(fs.readFileSync('./test/fixtures/animaldb_all_docs_3.json', 'utf8')))
-      // batch 4
-      .post('/_all_docs', { limit: 3, start_key: snipeKey, include_docs: true })
-      .reply(200, JSON.parse(fs.readFileSync('./test/fixtures/animaldb_all_docs_4.json', 'utf8')));
-
-    return new Promise((resolve, reject) => {
-      shallowBackup(dbUrl, { bufferSize: 3, parallelism: 1 })
-        .on('error', function(err) {
-          reject(err);
-        })
-        .on('received', function(data) {
-          try {
-            if (data.batch === 3) {
-              assert.strictEqual(data.length, 2); // smaller last batch
-            } else {
-              assert.strictEqual(data.length, 3);
-            }
-          } catch (err) {
-            reject(err);
-          }
-        })
-        .on('finished', function(data) {
-          try {
-            assert.strictEqual(data.total, 11);
-            assert.ok(couch.isDone());
-            resolve();
-          } catch (err) {
-            reject(err);
-          }
-        });
+    this.childProcessPromise = once(this.childProcess, 'close').then(() => {
+      const code = this.childProcess.exitCode;
+      const signal = this.childProcess.signalCode;
+      logProcess(`Test process ${cmd} closed with code ${code} and signal ${signal}`);
+      if (code === 0) {
+        logProcess(`Resolving process promise for ${cmd}`);
+        return Promise.resolve(code);
+      } else {
+        const e = new Error(`Test child process ${cmd} exited with code ${code} and ${signal}. This may be normal for error case testing.`);
+        e.code = code;
+        e.signal = signal;
+        logProcess(`Will reject process promise for ${cmd} with ${e}`);
+        return Promise.reject(e);
+      }
     });
-  });
 
-  it('should perform a shallow backup with transient error', async function() {
-    const couch = nock(dbUrl)
-      // batch 1
-      .post('/_all_docs', { limit: 3, include_docs: true })
-      .reply(200, JSON.parse(fs.readFileSync('./test/fixtures/animaldb_all_docs_1.json', 'utf8')))
-      // batch 2
-      .post('/_all_docs', { limit: 3, start_key: badgerKey, include_docs: true })
-      .reply(200, JSON.parse(fs.readFileSync('./test/fixtures/animaldb_all_docs_2.json', 'utf8')))
-      // batch 3 - transient error
-      .post('/_all_docs', { limit: 3, start_key: kookaburraKey, include_docs: true })
-      .reply(500, { error: 'Internal Server Error' })
-      // batch 3 - retry
-      .post('/_all_docs', { limit: 3, start_key: kookaburraKey, include_docs: true })
-      .reply(200, JSON.parse(fs.readFileSync('./test/fixtures/animaldb_all_docs_3.json', 'utf8')))
-      // batch 4
-      .post('/_all_docs', { limit: 3, start_key: snipeKey, include_docs: true })
-      .reply(200, JSON.parse(fs.readFileSync('./test/fixtures/animaldb_all_docs_4.json', 'utf8')));
+    switch (mode) {
+      case 'readable':
+        this.duplexFrom = this.childProcess.stdout;
+        break;
+      case 'writable':
+        this.duplexFrom = this.childProcess.stdin;
+        break;
+      default:
+        // Default is duplex
+        this.duplexFrom = { writable: this.childProcess.stdin, readable: this.childProcess.stdout };
+    }
 
-    return new Promise((resolve, reject) => {
-      shallowBackup(dbUrl, { bufferSize: 3, parallelism: 1 })
-        .on('error', function(err) {
-          try {
-            assert.strictEqual(err.name, 'HTTPError');
-          } catch (err) {
-            reject(err);
-          }
-        })
-        .on('received', function(data) {
-          try {
-            if (data.batch === 3) {
-              assert.strictEqual(data.length, 2); // smaller last batch
-            } else {
-              assert.strictEqual(data.length, 3);
-            }
-          } catch (err) {
-            reject(err);
-          }
-        })
-        .on('finished', function(data) {
-          try {
-            assert.strictEqual(data.total, 11);
-            assert.ok(couch.isDone());
-            resolve();
-          } catch (err) {
-            reject(err);
-          }
-        });
-    });
-  });
-
-  it('should fail to perform a shallow backup on fatal error', async function() {
-    const couch = nock(dbUrl)
-      // batch 1
-      .post('/_all_docs', { limit: 3, include_docs: true })
-      .reply(200, JSON.parse(fs.readFileSync('./test/fixtures/animaldb_all_docs_1.json', 'utf8')))
-      // batch 2
-      .post('/_all_docs', { limit: 3, start_key: badgerKey, include_docs: true })
-      .reply(200, JSON.parse(fs.readFileSync('./test/fixtures/animaldb_all_docs_2.json', 'utf8')))
-      // batch 3 - fatal error
-      .post('/_all_docs', { limit: 3, start_key: kookaburraKey, include_docs: true })
-      .reply(401, { error: 'Unauthorized' });
-
-    let errCount = 0;
+    this.stream = Duplex.from(this.duplexFrom);
+  }
+}
 
-    return new Promise((resolve, reject) => {
-      shallowBackup(dbUrl, { bufferSize: 3, parallelism: 1 })
-        .on('error', function(err) {
-          try {
-            errCount++;
-            assert.strictEqual(err.name, 'Unauthorized');
-          } catch (err) {
-            reject(err);
-          }
-        })
-        .on('received', function(data) {
-          try {
-            assert.strictEqual(data.length, 3);
-          } catch (err) {
-            reject(err);
-          }
-        })
-        .on('finished', function(data) {
-          try {
-            assert.strictEqual(data.total, 6);
-            assert.ok(couch.isDone());
-            assert.strictEqual(errCount, 1);
-            resolve();
-          } catch (err) {
-            reject(err);
-          }
-        });
-    });
-  });
-});
+module.exports = {
+  TestProcess,
+  cliBackup: function(databaseName, params = {}) {
+    const args = ['--db', databaseName];
+    if (params.opts) {
+      if (params.opts.mode) {
+        args.push('--mode');
+        args.push(params.opts.mode);
+      }
+      if (params.opts.output) {
+        args.push('--output');
+        args.push(params.opts.output);
+      }
+      if (params.opts.log) {
+        args.push('--log');
+        args.push(params.opts.log);
+      }
+      if (params.opts.resume) {
+        args.push('--resume');
+        args.push(params.opts.resume);
+      }
+      if (params.opts.bufferSize) {
+        args.push('--buffer-size');
+        args.push(params.opts.bufferSize);
+      }
+      if (params.opts.iamApiKey) {
+        args.push('--iam-api-key');
+        args.push(params.opts.iamApiKey);
+      }
+    }
+    return new TestProcess('./bin/couchbackup.bin.js', args, 'readable');
+  },
+  cliRestore: function(databaseName, params) {
+    const args = ['--db', databaseName];
+    if (params.opts) {
+      if (params.opts.bufferSize) {
+        args.push('--buffer-size');
+        args.push(params.opts.bufferSize);
+      }
+      if (params.opts.parallelism) {
+        args.push('--parallelism');
+        args.push(params.opts.parallelism);
+      }
+      if (params.opts.requestTimeout) {
+        args.push('--request-timeout');
+        args.push(params.opts.requestTimeout);
+      }
+      if (params.opts.iamApiKey) {
+        args.push('--iam-api-key');
+        args.push(params.opts.iamApiKey);
+      }
+    }
+    return new TestProcess('./bin/couchrestore.bin.js', args, 'writable');
+  },
+  cliGzip: function() {
+    return new TestProcess('gzip', []);
+  },
+  cliGunzip: function() {
+    return new TestProcess('gunzip', []);
+  },
+  cliEncrypt: function() {
+    return new TestProcess('openssl', ['aes-128-cbc', '-pass', 'pass:12345']);
+  },
+  cliDecrypt: function() {
+    return new TestProcess('openssl', ['aes-128-cbc', '-d', '-pass', 'pass:12345']);
+  }
+};

package/.scannerwork/scanner-report/source-37.txt

@@ -16,86 +16,176 @@
 'use strict';
 
 const assert = require('assert');
-const fs = require('fs');
-const { once } = require('node:events');
-const u = require('./citestutils.js');
+const nock = require('nock');
+const request = require('../includes/request.js');
+const changes = require('../includes/spoolchanges.js');
 
-[{ useApi: true }, { useApi: false }].forEach(function(params) {
-  describe(u.scenario('Resume tests', params), function() {
-    it('should create a log file', async function() {
-      // Allow up to 90 s for this test
-      u.setTimeout(this, 60);
+const url = 'http://localhost:7777';
+const dbName = 'fakenockdb';
+const longTestTimeout = 3000;
 
-      const actualBackup = `./${this.fileName}`;
-      const logFile = `./${this.fileName}` + '.log';
-      const p = u.p(params, { opts: { log: logFile } });
-      return u.testBackupToFile(p, 'animaldb', actualBackup).then(() => {
-        assert.ok(fs.existsSync(logFile), 'The log file should exist.');
-      });
+const db = request.client(`${url}/${dbName}`, { parallelism: 1 });
+
+const seqSuffix = Buffer.alloc(124, 'abc123').toString('base64');
+function provideChanges(batchSize, totalChanges, fullResponse = false) {
+  let pending = totalChanges;
+  const sparseResultsArray = Array(batchSize).fill({
+    seq: null,
+    id: 'doc',
+    changes: [{ rev: '1-abcdef0123456789abcdef0123456789' }]
+  });
+  nock(url)
+    .post(`/${dbName}/_changes`)
+    .query(true)
+    .times(totalChanges / batchSize + (totalChanges % batchSize > 0 ? 1 : 0))
+    .reply(200, (uri, requestBody) => {
+      pending -= batchSize;
+      const lastSeq = (totalChanges - pending);
+      const seq = lastSeq - batchSize;
+      return {
+        results: fullResponse
+          ? Array.from(Array(batchSize), (_, i) => {
+            return {
+              seq: `${seq + i}-${seqSuffix}`,
+              id: `doc${seq + i}`,
+              changes: [{ rev: '1-abcdef0123456789abcdef0123456789' }]
+            };
+          })
+          : sparseResultsArray,
+        pending: pending,
+        last_seq: `${lastSeq}-abc`
+      };
     });
+}
 
-    it('should restore corrupted animaldb to a database correctly', async function() {
-      // Allow up to 60 s to restore and compare (again it should be faster)!
-      u.setTimeout(this, 60);
-      const input = fs.createReadStream('./test/fixtures/animaldb_corrupted.json');
-      const dbName = this.dbName;
-      const p = u.p(params, { expectedRestoreErrorRecoverable: { name: 'BackupFileJsonError' } });
-      return once(input, 'open')
-        .then(() => {
-          return u.testRestore(p, input, dbName);
-        }).then(() => {
-          return u.dbCompare('animaldb', dbName);
-        });
+describe('#unit Check spool changes', function() {
+  it('should terminate on request error', async function() {
+    nock(url)
+      .post(`/${dbName}/_changes`)
+      .query(true)
+      .times(3)
+      .replyWithError({ code: 'ECONNRESET', message: 'socket hang up' });
+
+    return new Promise((resolve, reject) => {
+      changes(db, '/dev/null', 500, null, function(err) {
+        try {
+          assert.strictEqual(err.name, 'SpoolChangesError');
+          assert.strictEqual(err.message, `Failed changes request - socket hang up: post ${url}/${dbName}/_changes`);
+          assert.ok(nock.isDone());
+          resolve();
+        } catch (err) {
+          reject(err);
+        }
+      });
     });
+  }).timeout(longTestTimeout);
 
-    it('should restore resumed animaldb with blank line to a database correctly', async function() {
-      // Allow up to 60 s to restore and compare (again it should be faster)!
-      u.setTimeout(this, 60);
-      const input = fs.createReadStream('./test/fixtures/animaldb_resumed_blank.json');
-      const dbName = this.dbName;
-      return once(input, 'open')
-        .then(() => {
-          return u.testRestore(params, input, dbName);
-        }).then(() => {
-          return u.dbCompare('animaldb', dbName);
-        });
+  it('should terminate on bad HTTP status code response', async function() {
+    nock(url)
+      .post(`/${dbName}/_changes`)
+      .query(true)
+      .times(3)
+      .reply(500, function(uri, requestBody) {
+        this.req.response.statusMessage = 'Internal Server Error';
+        return { error: 'foo', reason: 'bar' };
+      });
+    return new Promise((resolve, reject) => {
+      changes(db, '/dev/null', 500, null, function(err) {
+        try {
+          assert.strictEqual(err.name, 'HTTPFatalError');
+          assert.strictEqual(err.message, `500 Internal Server Error: post ${url}/${dbName}/_changes - Error: foo, Reason: bar`);
+          assert.ok(nock.isDone());
+          resolve();
+        } catch (err) {
+          reject(err);
+        }
+      });
     });
-  });
-});
+  }).timeout(longTestTimeout);
 
-describe('Resume tests', function() {
-  // Currently cannot abort API backups, when we do this test should be run for
-  // both API and CLI
-  it('should correctly backup and restore backup10m', async function() {
-    // Allow up to 90 s for this test
-    u.setTimeout(this, 90);
+  it('should keep collecting changes', async function() {
+    // This test validates that spooling will correctly
+    // continue across multiple requests
+    // (4 batches of 100000 to be precise).
+    // This test might take up to 10 seconds
+    this.timeout(10 * 1000);
 
-    const actualBackup = `./${this.fileName}`;
-    const logFile = `./${this.fileName}` + '.log';
-    // Use abort parameter to terminate the backup
-    const p = u.p(params, { abort: true }, { opts: { log: logFile } });
-    const restoreDb = this.dbName;
-    // Set the database doc count as fewer than this should be written during
-    // resumed backup.
-    p.exclusiveMaxExpected = 5096;
+    // Use full changes for this test
+    provideChanges(100000, 400000, true);
+    return new Promise((resolve, reject) => {
+      changes(db, '/dev/null', 500, null, function(err) {
+        try {
+          assert.ok(!err);
+          assert.ok(nock.isDone());
+          resolve();
+        } catch (err) {
+          reject(err);
+        }
+      });
+    });
+  });
 
-    return u.testBackupAbortResumeRestore(p, 'backup10m', actualBackup, restoreDb);
+  it('should keep collecting sparse changes', async function() {
+    // This test checks that making thousands of requests doesn't
+    // make anything bad happen.
+    // This test might take up to 25 seconds
+    this.timeout(25 * 1000);
+    // Use sparse changes for this test and a batch size of 1
+    provideChanges(1, 2500);
+    return new Promise((resolve, reject) => {
+      changes(db, '/dev/null', 500, null, function(err) {
+        try {
+          assert.ok(!err);
+          assert.ok(nock.isDone());
+          resolve();
+        } catch (err) {
+          reject(err);
+        }
+      });
+    });
   });
-  // Note --output is only valid for CLI usage, this test should only run for CLI
-  const params = { useApi: false };
-  it('should correctly backup and restore backup10m using --output', async function() {
-    // Allow up to 90 s for this test
-    u.setTimeout(this, 90);
+});
 
-    const actualBackup = `./${this.fileName}`;
-    const logFile = `./${this.fileName}` + '.log';
-    // Use abort parameter to terminate the backup
-    const p = u.p(params, { abort: true }, { opts: { output: actualBackup, log: logFile } });
-    const restoreDb = this.dbName;
-    // Set the database doc count as fewer than this should be written during
-    // resumed backup.
-    p.exclusiveMaxExpected = 5096;
+describe('Longer spool changes checks', function() {
+  it('#slow should keep collecting changes (25M)', async function() {
+    // This test might take up to 5 minutes
+    this.timeout(5 * 60 * 1000);
+    // Note changes spooling uses a constant batch size, we are setting
+    // a test value here and setting the buffer to match
+    const batch = 100000;
+    // Use sparse changes for this test
+    provideChanges(batch, 25000000);
+    return new Promise((resolve, reject) => {
+      changes(db, '/dev/null', batch, null, function(err) {
+        try {
+          assert.ok(!err);
+          assert.ok(nock.isDone());
+          resolve();
+        } catch (err) {
+          reject(err);
+        }
+      });
+    });
+  });
 
-    return await u.testBackupAbortResumeRestore(p, 'backup10m', actualBackup, restoreDb);
+  it('#slower should keep collecting changes (500M)', async function() {
+    // This test might take up to 90 minutes
+    this.timeout(90 * 60 * 1000);
+    // Note changes spooling uses a constant batch size, we are setting
+    // a test value here and setting the buffer to match
+    const batch = 1000000;
+    // Use full changes for this test to exercise load
+    provideChanges(batch, 500000000, true);
+    return new Promise((resolve, reject) => {
+      changes(db, '/dev/null', batch, null, function(err) {
+        try {
+          assert.ok(!err);
+          assert.ok(nock.isDone());
+          resolve();
+        } catch (err) {
+          reject(err);
+        }
+      });
+    });
   });
 });