@cloudant/couchbackup 2.9.13 → 2.9.14-SNAPSHOT.147

package/.scannerwork/scanner-report/source-41.txt

@@ -1,29 +1,167 @@
-// Copyright © 2017 IBM Corp. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-/* global describe it */
-'use strict';
-
-const u = require('./citestutils.js');
-
-[{ useApi: true }, { useApi: false }].forEach(function(params) {
-  describe(u.scenario('#slowest End to end backup and restore', params), function() {
-    // 10 GB is about the largest the CI can handle before getting very upset
-    // about how long things are taking
-    it('should backup and restore largedb10g', function(done) {
-      u.setTimeout(this, 350 * 60);
-      u.testDirectBackupAndRestore(params, 'largedb10g', this.dbName, done);
-    });
-  });
-});
+<testsuites name="test-iam">
+  <testsuite name="Basic backup and restore using API" tests="3" errors="0" failures="0" skipped="0" timestamp="2023-10-09T12:40:02" time="5.323">
+    <testcase classname="test-iam.Basic backup and restore using API" name="should backup animaldb to a file correctly" time="1.464">
+      <system-out><![CDATA[{ total: 15 }
+]]></system-out>
+    </testcase>
+    <testcase classname="test-iam.Basic backup and restore using API" name="should restore animaldb to a database correctly" time="1.885">
+      <system-out><![CDATA[{ total: 15 }
+]]></system-out>
+    </testcase>
+    <testcase classname="test-iam.Basic backup and restore using API" name="should execute a shallow mode backup successfully" time="0.803">
+      <system-out><![CDATA[{ total: 11 }
+]]></system-out>
+    </testcase>
+  </testsuite>
+  <testsuite name="Basic backup and restore using API Buffer size tests using API" tests="3" errors="0" failures="0" skipped="0" timestamp="2023-10-09T12:40:07" time="12.137">
+    <testcase classname="test-iam.Basic backup and restore using API Buffer size tests using API" name="should backup/restore animaldb with the same buffer size" time="3.833">
+      <system-out><![CDATA[{ total: 15 }
+{ total: 15 }
+]]></system-out>
+    </testcase>
+    <testcase classname="test-iam.Basic backup and restore using API Buffer size tests using API" name="should backup/restore animaldb with backup buffer &gt; restore buffer" time="3.406">
+      <system-out><![CDATA[{ total: 15 }
+{ total: 15 }
+]]></system-out>
+    </testcase>
+    <testcase classname="test-iam.Basic backup and restore using API Buffer size tests using API" name="should backup/restore animaldb with backup buffer &lt; restore buffer" time="4.085">
+      <system-out><![CDATA[{ total: 15 }
+{ total: 15 }
+]]></system-out>
+    </testcase>
+  </testsuite>
+  <testsuite name="Basic backup and restore using CLI" tests="3" errors="0" failures="0" skipped="0" timestamp="2023-10-09T12:40:19" time="10.179">
+    <testcase classname="test-iam.Basic backup and restore using CLI" name="should backup animaldb to a file correctly" time="2.789">
+      <system-out><![CDATA[Backup process close 0 null
+]]></system-out>
+    </testcase>
+    <testcase classname="test-iam.Basic backup and restore using CLI" name="should restore animaldb to a database correctly" time="4.183">
+    </testcase>
+    <testcase classname="test-iam.Basic backup and restore using CLI" name="should execute a shallow mode backup successfully" time="2.392">
+      <system-out><![CDATA[Backup process close 0 null
+]]></system-out>
+    </testcase>
+  </testsuite>
+  <testsuite name="Basic backup and restore using CLI Buffer size tests using CLI" tests="3" errors="0" failures="0" skipped="0" timestamp="2023-10-09T12:40:29" time="23.86">
+    <testcase classname="test-iam.Basic backup and restore using CLI Buffer size tests using CLI" name="should backup/restore animaldb with the same buffer size" time="7.974">
+      <system-out><![CDATA[Backup process close 0 null
+]]></system-out>
+    </testcase>
+    <testcase classname="test-iam.Basic backup and restore using CLI Buffer size tests using CLI" name="should backup/restore animaldb with backup buffer &gt; restore buffer" time="6.922">
+      <system-out><![CDATA[Backup process close 0 null
+]]></system-out>
+    </testcase>
+    <testcase classname="test-iam.Basic backup and restore using CLI Buffer size tests using CLI" name="should backup/restore animaldb with backup buffer &lt; restore buffer" time="8.154">
+      <system-out><![CDATA[Backup process close 0 null
+]]></system-out>
+    </testcase>
+  </testsuite>
+  <testsuite name="Compression tests using API" tests="3" errors="0" failures="0" skipped="0" timestamp="2023-10-09T12:40:53" time="6.602">
+    <testcase classname="test-iam.Compression tests using API" name="should backup animaldb to a compressed file" time="0.873">
+      <system-out><![CDATA[{ total: 15 }
+]]></system-out>
+    </testcase>
+    <testcase classname="test-iam.Compression tests using API" name="should backup and restore animaldb via a compressed file" time="2.735">
+      <system-out><![CDATA[{ total: 15 }
+{ total: 15 }
+]]></system-out>
+    </testcase>
+    <testcase classname="test-iam.Compression tests using API" name="should backup and restore animaldb via a compressed stream" time="2.19">
+      <system-out><![CDATA[{ total: 15 }
+{ total: 15 }
+]]></system-out>
+    </testcase>
+  </testsuite>
+  <testsuite name="Compression tests using CLI" tests="3" errors="0" failures="0" skipped="0" timestamp="2023-10-09T12:41:00" time="14.061">
+    <testcase classname="test-iam.Compression tests using CLI" name="should backup animaldb to a compressed file" time="2.657">
+      <system-out><![CDATA[Backup process close 0 null
+]]></system-out>
+    </testcase>
+    <testcase classname="test-iam.Compression tests using CLI" name="should backup and restore animaldb via a compressed file" time="5.995">
+      <system-out><![CDATA[Backup process close 0 null
+]]></system-out>
+    </testcase>
+    <testcase classname="test-iam.Compression tests using CLI" name="should backup and restore animaldb via a compressed stream" time="4.597">
+      <system-out><![CDATA[Backup process close 0 null
+]]></system-out>
+    </testcase>
+  </testsuite>
+  <testsuite name="End to end backup and restore using API" tests="2" errors="0" failures="0" skipped="0" timestamp="2023-10-09T12:41:14" time="319.803">
+    <testcase classname="test-iam.End to end backup and restore using API" name="should backup and restore animaldb" time="2.198">
+      <system-out><![CDATA[{ total: 15 }
+{ total: 15 }
+]]></system-out>
+    </testcase>
+    <testcase classname="test-iam.End to end backup and restore using API" name="should backup and restore largedb1g #slow" time="316.814">
+      <system-out><![CDATA[{ total: 522948 }
+{ total: 522948 }
+]]></system-out>
+    </testcase>
+  </testsuite>
+  <testsuite name="End to end backup and restore using CLI" tests="2" errors="0" failures="0" skipped="0" timestamp="2023-10-09T12:46:34" time="507.938">
+    <testcase classname="test-iam.End to end backup and restore using CLI" name="should backup and restore animaldb" time="3.293">
+      <system-out><![CDATA[Backup process close 0 null
+]]></system-out>
+    </testcase>
+    <testcase classname="test-iam.End to end backup and restore using CLI" name="should backup and restore largedb1g #slow" time="503.862">
+      <system-out><![CDATA[Backup process close 0 null
+]]></system-out>
+    </testcase>
+  </testsuite>
+  <testsuite name="Encryption tests" tests="1" errors="0" failures="0" skipped="0" timestamp="2023-10-09T12:55:02" time="4.499">
+    <testcase classname="test-iam.Encryption tests" name="should backup and restore animaldb via an encrypted file" time="4.234">
+      <system-out><![CDATA[Backup process close 0 null
+]]></system-out>
+    </testcase>
+  </testsuite>
+  <testsuite name="Write error tests" tests="1" errors="0" failures="0" skipped="0" timestamp="2023-10-09T12:55:06" time="0.274">
+    <testcase classname="test-iam.Write error tests" name="calls callback with error set when stream is not writeable" time="0.01">
+    </testcase>
+  </testsuite>
+  <testsuite name="Event tests" tests="2" errors="0" failures="0" skipped="0" timestamp="2023-10-09T12:55:07" time="2.4">
+    <testcase classname="test-iam.Event tests" name="should get a finished event when using stdout" time="0.897">
+    </testcase>
+    <testcase classname="test-iam.Event tests" name="should get a finished event when using file output" time="0.958">
+    </testcase>
+  </testsuite>
+  <testsuite name="Resume tests using API" tests="3" errors="0" failures="0" skipped="0" timestamp="2023-10-09T12:55:09" time="5.09">
+    <testcase classname="test-iam.Resume tests using API" name="should create a log file" time="0.897">
+      <system-out><![CDATA[{ total: 15 }
+]]></system-out>
+    </testcase>
+    <testcase classname="test-iam.Resume tests using API" name="should restore corrupted animaldb to a database correctly" time="1.685">
+      <system-out><![CDATA[{ total: 15 }
+]]></system-out>
+    </testcase>
+    <testcase classname="test-iam.Resume tests using API" name="should restore resumed animaldb with blank line to a database correctly" time="1.704">
+      <system-out><![CDATA[{ total: 15 }
+]]></system-out>
+    </testcase>
+  </testsuite>
+  <testsuite name="Resume tests using CLI" tests="3" errors="0" failures="0" skipped="0" timestamp="2023-10-09T12:55:14" time="7.358">
+    <testcase classname="test-iam.Resume tests using CLI" name="should create a log file" time="1.507">
+      <system-out><![CDATA[Backup process close 0 null
+]]></system-out>
+    </testcase>
+    <testcase classname="test-iam.Resume tests using CLI" name="should restore corrupted animaldb to a database correctly" time="2.489">
+    </testcase>
+    <testcase classname="test-iam.Resume tests using CLI" name="should restore resumed animaldb with blank line to a database correctly" time="2.554">
+    </testcase>
+  </testsuite>
+  <testsuite name="Resume tests" tests="2" errors="0" failures="0" skipped="0" timestamp="2023-10-09T12:55:21" time="36.519">
+    <testcase classname="test-iam.Resume tests" name="should correctly backup and restore backup10m" time="17.816">
+      <system-out><![CDATA[Backup process close null SIGTERM
+Backup process close 0 null
+]]></system-out>
+    </testcase>
+    <testcase classname="test-iam.Resume tests" name="should correctly backup and restore backup10m using --output" time="17.751">
+      <system-out><![CDATA[Backup process close null SIGTERM
+Backup process close 0 null
+]]></system-out>
+    </testcase>
+  </testsuite>
+  <testsuite name="Longer spool changes checks" tests="1" errors="0" failures="0" skipped="0" timestamp="2023-10-09T12:55:58" time="44.72">
+    <testcase classname="test-iam.Longer spool changes checks" name="#slow should keep collecting changes (25M)" time="44.448">
+    </testcase>
+  </testsuite>
+</testsuites>

package/.scannerwork/scanner-report/source-46.txt

@@ -1,4 +1,4 @@
-// Copyright © 2017, 2021 IBM Corp. All rights reserved.
+// Copyright © 2017, 2018 IBM Corp. All rights reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -12,136 +12,164 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-/* global describe it beforeEach */
+// Small script which backs up a Cloudant or CouchDB database to an S3
+// bucket via a stream rather than on-disk file.
+//
+// The script generates the backup object name by combining together the path
+// part of the database URL and the current time.
+
 'use strict';
 
-const assert = require('assert');
-const fs = require('fs');
-const nock = require('nock');
-const request = require('../includes/request.js');
-const writer = require('../includes/writer.js');
-const noopEmitter = new (require('events')).EventEmitter();
-const liner = require('../includes/liner.js');
-
-const longTestTimeout = 3000;
-
-// The writer expects a line-by-line stream so this utility function does that
-// processing for the tests (which normally happens in the internal restore
-// function).
-function testLinestream(fixture = './test/fixtures/animaldb_expected.json') {
-  return fs.createReadStream(fixture)
-    .pipe(liner());
+const stream = require('stream');
+const url = require('url');
+
+const AWS = require('aws-sdk');
+const couchbackup = require('@cloudant/couchbackup');
+const debug = require('debug')('s3-backup');
+const VError = require('verror').VError;
+
+/*
+  Main function, run from base of file.
+*/
+function main() {
+  const argv = require('yargs')
+    .usage('Usage: $0 [options]')
+    .example('$0 -s https://user:pass@host/db -b <bucket>', 'Backup db to bucket')
+    .options({
+      source: { alias: 's', nargs: 1, demandOption: true, describe: 'Source database URL' },
+      bucket: { alias: 'b', nargs: 1, demandOption: true, describe: 'Destination bucket' },
+      prefix: { alias: 'p', nargs: 1, describe: 'Prefix for backup object key', default: 'couchbackup' },
+      s3url: { nargs: 1, describe: 'S3 endpoint URL' },
+      awsprofile: { nargs: 1, describe: 'The profile section to use in the ~/.aws/credentials file', default: 'default' }
+    })
+    .help('h').alias('h', 'help')
+    .epilog('Copyright (C) IBM 2017')
+    .argv;
+
+  const sourceUrl = argv.source;
+  const backupName = new url.URL(sourceUrl).pathname.split('/').filter(function(x) { return x; }).join('-');
+  const backupBucket = argv.bucket;
+  const backupKeyPrefix = `${argv.prefix}-${backupName}`;
+  const shallow = argv.shallow;
+
+  const backupKey = `${backupKeyPrefix}-${new Date().toISOString()}`;
+
+  const s3Endpoint = argv.s3url;
+  const awsProfile = argv.awsprofile;
+
+  // Creds are from ~/.aws/credentials, environment etc. (see S3 docs).
+  const awsOpts = {
+    signatureVersion: 'v4',
+    credentials: new AWS.SharedIniFileCredentials({ profile: awsProfile })
+  };
+  if (typeof s3Endpoint !== 'undefined') {
+    awsOpts.endpoint = new AWS.Endpoint(s3Endpoint);
+  }
+  const s3 = new AWS.S3(awsOpts);
+
+  debug(`Creating a new backup of ${s(sourceUrl)} at ${backupBucket}/${backupKey}...`);
+  bucketAccessible(s3, backupBucket)
+    .then(() => {
+      return backupToS3(sourceUrl, s3, backupBucket, backupKey, shallow);
+    })
+    .then(() => {
+      debug('done.');
+    })
+    .catch((reason) => {
+      debug(`Error: ${reason}`);
+      process.exit(1);
+    });
 }
 
-describe('#unit Check database restore writer', function() {
-  const dbUrl = 'http://localhost:5984/animaldb';
-  const db = request.client(dbUrl, { parallelism: 1 });
-
-  beforeEach('Reset nocks', function() {
-    nock.cleanAll();
-  });
-
-  it('should complete successfully', function(done) {
-    nock(dbUrl)
-      .post('/_bulk_docs')
-      .reply(200, []); // success
-
-    testLinestream()
-      .pipe(writer(db, 500, 1, noopEmitter))
-      .on('error', function(err) {
-        done(err);
-      })
-      .on('finished', function(data) {
-        assert.strictEqual(data.total, 15);
-        assert.ok(nock.isDone());
-        done();
-      });
+/**
+ * Return a promise that resolves if the bucket is available and
+ * rejects if not.
+ *
+ * @param {any} s3 S3 client object
+ * @param {any} bucketName Bucket name
+ * @returns Promise
+ */
+function bucketAccessible(s3, bucketName) {
+  return new Promise(function(resolve, reject) {
+    const params = {
+      Bucket: bucketName
+    };
+    s3.headBucket(params, function(err, data) {
+      if (err) {
+        reject(new VError(err, 'S3 bucket not accessible'));
+      } else {
+        resolve();
+      }
+    });
   });
+}
 
-  it('should terminate on a fatal error', function(done) {
-    nock(dbUrl)
-      .post('/_bulk_docs')
-      .reply(401, { error: 'Unauthorized' }); // fatal error
-
-    testLinestream()
-      .pipe(writer(db, 500, 1, noopEmitter))
-      .on('error', function(err) {
-        assert.strictEqual(err.name, 'Unauthorized');
-        assert.strictEqual(err.message, 'Access is denied due to invalid credentials.');
-        assert.ok(nock.isDone());
-        done();
-      });
+/**
+ * Backup directly from Cloudant to an object store object via a stream.
+ *
+ * @param {any} sourceUrl URL of database
+ * @param {any} s3Client Object store client
+ * @param {any} s3Bucket Backup destination bucket
+ * @param {any} s3Key Backup destination key name (shouldn't exist)
+ * @param {any} shallow Whether to use the couchbackup `shallow` mode
+ * @returns Promise
+ */
+function backupToS3(sourceUrl, s3Client, s3Bucket, s3Key, shallow) {
+  return new Promise((resolve, reject) => {
+    debug(`Setting up S3 upload to ${s3Bucket}/${s3Key}`);
+
+    // A pass through stream that has couchbackup's output
+    // written to it and it then read by the S3 upload client.
+    // It has a 64MB highwater mark to allow for fairly
+    // uneven network connectivity.
+    const streamToUpload = new stream.PassThrough({ highWaterMark: 67108864 });
+
+    // Set up S3 upload.
+    const params = {
+      Bucket: s3Bucket,
+      Key: s3Key,
+      Body: streamToUpload
+    };
+    s3Client.upload(params, function(err, data) {
+      debug('Object store upload done');
+      if (err) {
+        debug(err);
+        reject(new VError(err, 'Object store upload failed'));
+        return;
+      }
+      debug('Object store upload succeeded');
+      debug(data);
+      resolve();
+    }).httpUploadProgress = (progress) => {
+      debug(`Object store upload progress: ${progress}`);
+    };
+
+    debug(`Starting streaming data from ${s(sourceUrl)}`);
+    couchbackup.backup(
+      sourceUrl,
+      streamToUpload,
+      (err, obj) => {
+        if (err) {
+          debug(err);
+          reject(new VError(err, 'CouchBackup failed with an error'));
+          return;
+        }
+        debug(`Download from ${s(sourceUrl)} complete.`);
+        streamToUpload.end(); // must call end() to complete upload.
+        // resolve() is called by the upload
+      }
+    );
   });
+}
 
-  it('should retry on transient errors', function(done) {
-    nock(dbUrl)
-      .post('/_bulk_docs')
-      .reply(429, { error: 'Too Many Requests' }) // transient error
-      .post('/_bulk_docs')
-      .reply(500, { error: 'Internal Server Error' }) // transient error
-      .post('/_bulk_docs')
-      .reply(200, { ok: true }); // third time lucky success
-
-    testLinestream()
-      .pipe(writer(db, 500, 1, noopEmitter))
-      .on('error', function(err) {
-        done(err);
-      })
-      .on('finished', function(data) {
-        assert.strictEqual(data.total, 15);
-        assert.ok(nock.isDone());
-        done();
-      });
-  }).timeout(longTestTimeout);
-
-  it('should fail after 3 transient errors', function(done) {
-    nock(dbUrl)
-      .post('/_bulk_docs')
-      .reply(429, { error: 'Too Many Requests' }) // transient error
-      .post('/_bulk_docs')
-      .reply(500, { error: 'Internal Server Error' }) // transient error
-      .post('/_bulk_docs')
-      .reply(503, { error: 'Service Unavailable' }); // Final transient error
-
-    testLinestream()
-      .pipe(writer(db, 500, 1, noopEmitter))
-      .on('error', function(err) {
-        assert.strictEqual(err.name, 'HTTPFatalError');
-        assert.strictEqual(err.message, `503 : post ${dbUrl}/_bulk_docs - Error: Service Unavailable`);
-        assert.ok(nock.isDone());
-        done();
-      });
-  }).timeout(longTestTimeout);
-
-  it('should restore shallow backups without rev info successfully', function(done) {
-    nock(dbUrl)
-      .post('/_bulk_docs')
-      .reply(200, [{ ok: true, id: 'foo', rev: '1-abc' }]); // success
-
-    testLinestream('./test/fixtures/animaldb_old_shallow.json')
-      .pipe(writer(db, 500, 1, noopEmitter))
-      .on('error', function(err) {
-        done(err);
-      })
-      .on('finished', function(data) {
-        assert.strictEqual(data.total, 11);
-        assert.ok(nock.isDone());
-        done();
-      });
-  });
+/**
+ * Remove creds from a URL, e.g., before logging
+ *
+ * @param {string} url URL to safen
+ */
+function s(originalUrl) {
+  const parts = new url.URL(originalUrl);
+  return url.format(parts, { auth: false });
+}
 
-  it('should get a batch error for non-empty array response with new_edits false', function(done) {
-    nock(dbUrl)
-      .post('/_bulk_docs')
-      .reply(200, [{ id: 'foo', error: 'foo', reason: 'bar' }]);
-
-    testLinestream()
-      .pipe(writer(db, 500, 1, noopEmitter))
-      .on('error', function(err) {
-        assert.strictEqual(err.name, 'Error');
-        assert.strictEqual(err.message, 'Error writing batch with new_edits:false and 1 items');
-        assert.ok(nock.isDone());
-        done();
-      });
-  });
-});
+main();