@cloudant/couchbackup 2.9.9-SNAPSHOT.98 → 2.9.9

package/.scannerwork/scanner-report/source-31.txt

@@ -1,4 +1,4 @@
-// Copyright © 2017, 2018 IBM Corp. All rights reserved.
+// Copyright © 2017 IBM Corp. All rights reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -12,164 +12,24 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-// Small script which backs up a Cloudant or CouchDB database to an S3
-// bucket via a stream rather than on-disk file.
-//
-// The script generates the backup object name by combining together the path
-// part of the database URL and the current time.
-
+/* global describe it */
 'use strict';
 
-const stream = require('stream');
-const url = require('url');
-
-const AWS = require('aws-sdk');
-const couchbackup = require('@cloudant/couchbackup');
-const debug = require('debug')('s3-backup');
-const VError = require('verror').VError;
-
-/*
-  Main function, run from base of file.
-*/
-function main() {
-  const argv = require('yargs')
-    .usage('Usage: $0 [options]')
-    .example('$0 -s https://user:pass@host/db -b <bucket>', 'Backup db to bucket')
-    .options({
-      source: { alias: 's', nargs: 1, demandOption: true, describe: 'Source database URL' },
-      bucket: { alias: 'b', nargs: 1, demandOption: true, describe: 'Destination bucket' },
-      prefix: { alias: 'p', nargs: 1, describe: 'Prefix for backup object key', default: 'couchbackup' },
-      s3url: { nargs: 1, describe: 'S3 endpoint URL' },
-      awsprofile: { nargs: 1, describe: 'The profile section to use in the ~/.aws/credentials file', default: 'default' }
-    })
-    .help('h').alias('h', 'help')
-    .epilog('Copyright (C) IBM 2017')
-    .argv;
-
-  const sourceUrl = argv.source;
-  const backupName = new url.URL(sourceUrl).pathname.split('/').filter(function(x) { return x; }).join('-');
-  const backupBucket = argv.bucket;
-  const backupKeyPrefix = `${argv.prefix}-${backupName}`;
-  const shallow = argv.shallow;
-
-  const backupKey = `${backupKeyPrefix}-${new Date().toISOString()}`;
-
-  const s3Endpoint = argv.s3url;
-  const awsProfile = argv.awsprofile;
-
-  // Creds are from ~/.aws/credentials, environment etc. (see S3 docs).
-  const awsOpts = {
-    signatureVersion: 'v4',
-    credentials: new AWS.SharedIniFileCredentials({ profile: awsProfile })
-  };
-  if (typeof s3Endpoint !== 'undefined') {
-    awsOpts.endpoint = new AWS.Endpoint(s3Endpoint);
-  }
-  const s3 = new AWS.S3(awsOpts);
-
-  debug(`Creating a new backup of ${s(sourceUrl)} at ${backupBucket}/${backupKey}...`);
-  bucketAccessible(s3, backupBucket)
-    .then(() => {
-      return backupToS3(sourceUrl, s3, backupBucket, backupKey, shallow);
-    })
-    .then(() => {
-      debug('done.');
-    })
-    .catch((reason) => {
-      debug(`Error: ${reason}`);
-      process.exit(1);
-    });
-}
-
-/**
- * Return a promise that resolves if the bucket is available and
- * rejects if not.
- *
- * @param {any} s3 S3 client object
- * @param {any} bucketName Bucket name
- * @returns Promise
- */
-function bucketAccessible(s3, bucketName) {
-  return new Promise(function(resolve, reject) {
-    const params = {
-      Bucket: bucketName
-    };
-    s3.headBucket(params, function(err, data) {
-      if (err) {
-        reject(new VError(err, 'S3 bucket not accessible'));
-      } else {
-        resolve();
-      }
+const assert = require('assert');
+const logfilegetbatches = require('../includes/logfilegetbatches.js');
+
+describe('#unit Fetching batches from a log file', function() {
+  it('should fetch multiple batches correctly', function(done) {
+    logfilegetbatches('./test/fixtures/test.log', [1, 4], function(err, data) {
+      assert.ok(!err);
+      assert.ok(data);
+      assert.strictEqual(typeof data, 'object');
+      assert.strictEqual(Object.keys(data).length, 2);
+      assert.deepStrictEqual(data['1'].docs, [{ id: '6' }, { id: '7' }, { id: '8' }, { id: '9' }, { id: '10' }]);
+      assert.strictEqual(data['1'].batch, 1);
+      assert.deepStrictEqual(data['4'].docs, [{ id: '21' }, { id: '22' }]);
+      assert.strictEqual(data['4'].batch, 4);
+      done();
     });
   });
-}
-
-/**
- * Backup directly from Cloudant to an object store object via a stream.
- *
- * @param {any} sourceUrl URL of database
- * @param {any} s3Client Object store client
- * @param {any} s3Bucket Backup destination bucket
- * @param {any} s3Key Backup destination key name (shouldn't exist)
- * @param {any} shallow Whether to use the couchbackup `shallow` mode
- * @returns Promise
- */
-function backupToS3(sourceUrl, s3Client, s3Bucket, s3Key, shallow) {
-  return new Promise((resolve, reject) => {
-    debug(`Setting up S3 upload to ${s3Bucket}/${s3Key}`);
-
-    // A pass through stream that has couchbackup's output
-    // written to it and it then read by the S3 upload client.
-    // It has a 64MB highwater mark to allow for fairly
-    // uneven network connectivity.
-    const streamToUpload = new stream.PassThrough({ highWaterMark: 67108864 });
-
-    // Set up S3 upload.
-    const params = {
-      Bucket: s3Bucket,
-      Key: s3Key,
-      Body: streamToUpload
-    };
-    s3Client.upload(params, function(err, data) {
-      debug('Object store upload done');
-      if (err) {
-        debug(err);
-        reject(new VError(err, 'Object store upload failed'));
-        return;
-      }
-      debug('Object store upload succeeded');
-      debug(data);
-      resolve();
-    }).httpUploadProgress = (progress) => {
-      debug(`Object store upload progress: ${progress}`);
-    };
-
-    debug(`Starting streaming data from ${s(sourceUrl)}`);
-    couchbackup.backup(
-      sourceUrl,
-      streamToUpload,
-      (err, obj) => {
-        if (err) {
-          debug(err);
-          reject(new VError(err, 'CouchBackup failed with an error'));
-          return;
-        }
-        debug(`Download from ${s(sourceUrl)} complete.`);
-        streamToUpload.end(); // must call end() to complete upload.
-        // resolve() is called by the upload
-      }
-    );
-  });
-}
-
-/**
- * Remove creds from a URL, e.g., before logging
- *
- * @param {string} url URL to safen
- */
-function s(originalUrl) {
-  const parts = new url.URL(originalUrl);
-  return url.format(parts, { auth: false });
-}
-
-main();
+});

package/.scannerwork/scanner-report/{source-53.txt → source-32.txt}

@@ -17,20 +17,13 @@
 
 const u = require('./citestutils.js');
 
-describe('Encryption tests', function() {
-  // Note CLI only to use openssl command
-  const p = { useApi: false, encryption: true };
-
-  it('should backup and restore animaldb via an encrypted file', function(done) {
-    // Allow up to 60 s for backup and restore of animaldb
-    u.setTimeout(this, 60);
-    const encryptedBackup = `./${this.fileName}`;
-    u.testBackupAndRestoreViaFile(p, 'animaldb', encryptedBackup, this.dbName, function(err) {
-      if (err) {
-        done(err);
-      } else {
-        u.assertEncryptedFile(encryptedBackup, done);
-      }
+[{ useApi: true }, { useApi: false }].forEach(function(params) {
+  describe(u.scenario('#slowest End to end backup and restore', params), function() {
+    // 10 GB is about the largest the CI can handle before getting very upset
+    // about how long things are taking
+    it('should backup and restore largedb10g', function(done) {
+      u.setTimeout(this, 350 * 60);
+      u.testDirectBackupAndRestore(params, 'largedb10g', this.dbName, done);
     });
   });
 });

package/.scannerwork/scanner-report/source-34.txt

@@ -1,4 +1,4 @@
-// Copyright © 2017, 2018 IBM Corp. All rights reserved.
+// Copyright © 2017 IBM Corp. All rights reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -12,179 +12,120 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-// Small script which backs up a Cloudant or CouchDB database to an S3
-// bucket, using an intermediary file on disk.
-//
-// The script generates the backup object name by combining together the path
-// part of the database URL and the current time.
-
+/* global describe it */
 'use strict';
 
-const stream = require('stream');
 const fs = require('fs');
-const url = require('url');
-
-const AWS = require('aws-sdk');
-const couchbackup = require('@cloudant/couchbackup');
-const debug = require('debug')('s3-backup');
-const tmp = require('tmp');
-const VError = require('verror').VError;
-
-/*
-  Main function, run from base of file.
-*/
-function main() {
-  const argv = require('yargs')
-    .usage('Usage: $0 [options]')
-    .example('$0 -s https://user:pass@host/db -b <bucket>', 'Backup db to bucket')
-    .options({
-      source: { alias: 's', nargs: 1, demandOption: true, describe: 'Source database URL' },
-      bucket: { alias: 'b', nargs: 1, demandOption: true, describe: 'Destination bucket' },
-      prefix: { alias: 'p', nargs: 1, describe: 'Prefix for backup object key', default: 'couchbackup' },
-      s3url: { nargs: 1, describe: 'S3 endpoint URL' },
-      awsprofile: { nargs: 1, describe: 'The profile section to use in the ~/.aws/credentials file', default: 'default' }
-    })
-    .help('h').alias('h', 'help')
-    .epilog('Copyright (C) IBM 2017')
-    .argv;
-
-  const sourceUrl = argv.source;
-  const backupBucket = argv.bucket;
-  const backupName = new url.URL(sourceUrl).pathname.split('/').filter(function(x) { return x; }).join('-');
-  const backupKeyPrefix = `${argv.prefix}-${backupName}`;
-
-  const backupKey = `${backupKeyPrefix}-${new Date().toISOString()}`;
-  const backupTmpFile = tmp.fileSync();
-
-  const s3Endpoint = argv.s3url;
-  const awsProfile = argv.awsprofile;
-
-  // Creds are from ~/.aws/credentials, environment etc. (see S3 docs).
-  const awsOpts = {
-    signatureVersion: 'v4',
-    credentials: new AWS.SharedIniFileCredentials({ profile: awsProfile })
-  };
-  if (typeof s3Endpoint !== 'undefined') {
-    awsOpts.endpoint = new AWS.Endpoint(s3Endpoint);
-  }
-  const s3 = new AWS.S3(awsOpts);
-
-  debug(`Creating a new backup of ${s(sourceUrl)} at ${backupBucket}/${backupKey}...`);
-  bucketAccessible(s3, backupBucket)
-    .then(() => {
-      return createBackupFile(sourceUrl, backupTmpFile.name);
-    })
-    .then(() => {
-      return uploadNewBackup(s3, backupTmpFile.name, backupBucket, backupKey);
-    })
-    .then(() => {
-      debug('Backup successful!');
-      backupTmpFile.removeCallback();
-      debug('done.');
-    })
-    .catch((reason) => {
-      debug(`Error: ${reason}`);
+const u = require('./citestutils.js');
+
+[{ useApi: true }, { useApi: false }].forEach(function(params) {
+  describe(u.scenario('Basic backup and restore', params), function() {
+    it('should backup animaldb to a file correctly', function(done) {
+      // Allow up to 40 s to backup and compare (it should be much faster)!
+      u.setTimeout(this, 40);
+      const actualBackup = `./${this.fileName}`;
+      // Create a file and backup to it
+      const output = fs.createWriteStream(actualBackup);
+      output.on('open', function() {
+        u.testBackup(params, 'animaldb', output, function(err) {
+          if (err) {
+            done(err);
+          } else {
+            u.readSortAndDeepEqual(actualBackup, './test/fixtures/animaldb_expected.json', done);
+          }
+        });
+      });
     });
-}
 
-/**
- * Return a promise that resolves if the bucket is available and
- * rejects if not.
- *
- * @param {any} s3 S3 client object
- * @param {any} bucketName Bucket name
- * @returns Promise
- */
-function bucketAccessible(s3, bucketName) {
-  return new Promise(function(resolve, reject) {
-    const params = {
-      Bucket: bucketName
-    };
-    s3.headBucket(params, function(err, data) {
-      if (err) {
-        reject(new VError(err, 'S3 bucket not accessible'));
-      } else {
-        resolve();
-      }
+    it('should restore animaldb to a database correctly', function(done) {
+      // Allow up to 60 s to restore and compare (again it should be faster)!
+      u.setTimeout(this, 60);
+      const input = fs.createReadStream('./test/fixtures/animaldb_expected.json');
+      const dbName = this.dbName;
+      input.on('open', function() {
+        u.testRestore(params, input, dbName, function(err) {
+          if (err) {
+            done(err);
+          } else {
+            u.dbCompare('animaldb', dbName, done);
+          }
+        });
+      });
     });
-  });
-}
 
-/**
- * Use couchbackup to create a backup of the specified database to a file path.
- *
- * @param {any} sourceUrl Database URL
- * @param {any} backupTmpFilePath Path to write file
- * @returns Promise
- */
-function createBackupFile(sourceUrl, backupTmpFilePath) {
-  return new Promise((resolve, reject) => {
-    couchbackup.backup(
-      sourceUrl,
-      fs.createWriteStream(backupTmpFilePath),
-      (err) => {
-        if (err) {
-          return reject(new VError(err, 'CouchBackup process failed'));
-        }
-        debug('couchbackup to file done; uploading to S3');
-        resolve('creating backup file complete');
-      }
-    );
-  });
-}
-
-/**
- * Upload a backup file to an S3 bucket.
- *
- * @param {any} s3 Object store client
- * @param {any} backupTmpFilePath Path of backup file to write.
- * @param {any} bucket Object store bucket name
- * @param {any} key Object store key name
- * @returns Promise
- */
-function uploadNewBackup(s3, backupTmpFilePath, bucket, key) {
-  return new Promise((resolve, reject) => {
-    debug(`Uploading from ${backupTmpFilePath} to ${bucket}/${key}`);
-
-    function uploadFromStream(s3, bucket, key) {
-      const pass = new stream.PassThrough();
-
-      const params = {
-        Bucket: bucket,
-        Key: key,
-        Body: pass
-      };
-      s3.upload(params, function(err, data) {
-        debug('S3 upload done');
-        if (err) {
-          debug(err);
-          reject(new VError(err, 'Upload failed'));
-          return;
-        }
-        debug('Upload succeeded');
-        debug(data);
-        resolve();
-      }).httpUploadProgress = (progress) => {
-        debug(`S3 upload progress: ${progress}`);
-      };
-
-      return pass;
-    }
+    it('should execute a shallow mode backup successfully', function(done) {
+      // Allow 30 s
+      u.setTimeout(this, 30);
+      const actualBackup = `./${this.fileName}`;
+      const output = fs.createWriteStream(actualBackup);
+      // Add the shallow mode option
+      const p = u.p(params, { opts: { mode: 'shallow' } });
+      output.on('open', function() {
+        u.testBackup(p, 'animaldb', output, function(err) {
+          if (err) {
+            done(err);
+          } else {
+            u.readSortAndDeepEqual(actualBackup, './test/fixtures/animaldb_expected_shallow.json', done);
+          }
+        });
+      });
+    });
 
-    const inputStream = fs.createReadStream(backupTmpFilePath);
-    const s3Stream = uploadFromStream(s3, bucket, key);
-    inputStream.pipe(s3Stream);
+    describe(u.scenario('Buffer size tests', params), function() {
+      it('should backup/restore animaldb with the same buffer size', function(done) {
+        // Allow up to 60 s for backup and restore of animaldb
+        u.setTimeout(this, 60);
+        const actualBackup = `./${this.fileName}`;
+        const logFile = `./${this.fileName}` + '.log';
+        const p = u.p(params, { opts: { log: logFile, bufferSize: 1 } });
+        u.testBackupAndRestoreViaFile(p, 'animaldb', actualBackup, this.dbName, done);
+      });
+      it('should backup/restore animaldb with backup buffer > restore buffer', function(done) {
+        // Allow up to 60 s for backup and restore of animaldb
+        u.setTimeout(this, 60);
+        const actualBackup = `./${this.fileName}`;
+        const logFile = `./${this.fileName}` + '.log';
+        const dbName = this.dbName;
+        const p = u.p(params, { opts: { log: logFile, bufferSize: 2 } }); // backup
+        const q = u.p(params, { opts: { bufferSize: 1 } }); // restore
+        u.testBackupToFile(p, 'animaldb', actualBackup, function(err) {
+          if (err) {
+            done(err);
+          } else {
+            // restore
+            u.testRestoreFromFile(q, actualBackup, dbName, function(err) {
+              if (!err) {
+                u.dbCompare('animaldb', dbName, done);
+              } else {
+                done(err);
+              }
+            });
+          }
+        });
+      });
+      it('should backup/restore animaldb with backup buffer < restore buffer', function(done) {
+        // Allow up to 60 s for backup and restore of animaldb
+        u.setTimeout(this, 60);
+        const actualBackup = `./${this.fileName}`;
+        const logFile = `./${this.fileName}` + '.log';
+        const dbName = this.dbName;
+        const p = u.p(params, { opts: { log: logFile, bufferSize: 1 } }); // backup
+        const q = u.p(params, { opts: { bufferSize: 2 } }); // restore
+        u.testBackupToFile(p, 'animaldb', actualBackup, function(err) {
+          if (err) {
+            done(err);
+          } else {
+            // restore
+            u.testRestoreFromFile(q, actualBackup, dbName, function(err) {
+              if (!err) {
+                u.dbCompare('animaldb', dbName, done);
+              } else {
+                done(err);
+              }
+            });
+          }
+        });
+      });
+    });
   });
-}
-
-/**
- * Remove creds from a URL, e.g., before logging
- *
- * @param {string} url URL to safen
- */
-function s(originalUrl) {
-  const parts = new url.URL(originalUrl);
-  return url.format(parts, { auth: false });
-}
-
-main();
+});

package/.scannerwork/scanner-report/source-35.txt

@@ -0,0 +1,68 @@
+// Copyright © 2017, 2018 IBM Corp. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+/* global describe it */
+'use strict';
+
+const fs = require('fs');
+const u = require('./citestutils.js');
+
+[{ useApi: true }, { useApi: false }].forEach(function(params) {
+  describe(u.scenario('Compression tests', params), function() {
+    const p = u.p(params, { compression: true });
+
+    it('should backup animaldb to a compressed file', function(done) {
+      // Allow up to 60 s for backup of animaldb
+      u.setTimeout(this, 60);
+      const compressedBackup = `./${this.fileName}`;
+      const output = fs.createWriteStream(compressedBackup);
+      output.on('open', function() {
+        u.testBackup(p, 'animaldb', output, function(err) {
+          if (err) {
+            done(err);
+          } else {
+            u.assertGzipFile(compressedBackup, done);
+          }
+        });
+      });
+    });
+
+    it('should backup and restore animaldb via a compressed file', function(done) {
+      // Allow up to 60 s for backup and restore of animaldb
+      u.setTimeout(this, 60);
+      const compressedBackup = `./${this.fileName}`;
+      u.testBackupAndRestoreViaFile(p, 'animaldb', compressedBackup, this.dbName, function(err) {
+        if (err) {
+          done(err);
+        } else {
+          u.assertGzipFile(compressedBackup, done);
+        }
+      });
+    });
+
+    it('should backup and restore animaldb via a compressed stream', function(done) {
+      // Allow up to 60 s for backup and restore of animaldb
+      u.setTimeout(this, 60);
+      u.testDirectBackupAndRestore(p, 'animaldb', this.dbName, done);
+    });
+
+    it('should backup and restore largedb2g via a compressed file #slower', function(done) {
+      // Takes ~ 25 min using CLI, but sometimes over an hour with API
+      u.setTimeout(this, 180 * 60);
+      const compressedBackup = `./${this.fileName}`;
+      params.compression = true;
+      u.testBackupAndRestoreViaFile(p, 'largedb2g', compressedBackup, this.dbName, done);
+    });
+  });
+});