@cloudant/couchbackup 2.9.13 → 2.9.14-SNAPSHOT.146

package/.scannerwork/scanner-report/source-6.txt

@@ -0,0 +1,100 @@
+// Copyright © 2023 IBM Corp. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+'use strict';
+
+const chunk = require('lodash/chunk');
+const difference = require('lodash/difference');
+const forOwn = require('lodash/forOwn');
+const isEmpty = require('lodash/isEmpty');
+const union = require('lodash/union');
+
+const compare = async(database1, database2, client) => {
+  // check docs same in both dbs
+  const allDocs1 = await getAllDocs(client, database1);
+  const allDocs2 = await getAllDocs(client, database2);
+
+  const onlyInDb1 = (difference(allDocs1, allDocs2));
+  const onlyInDb2 = (difference(allDocs2, allDocs1));
+
+  let databasesSame = isEmpty(onlyInDb1) && isEmpty(onlyInDb2);
+
+  if (!databasesSame) {
+    console.log(onlyInDb1.length + ' documents only in db 1.');
+    console.log('Document IDs only in db 1: ' + onlyInDb1);
+    console.log(onlyInDb2.length + ' documents only in db 2.');
+    console.log('Document IDs only in db 2: ' + onlyInDb2);
+  }
+
+  // check revs same in docs common to both dbs
+  const partitionSize = 500;
+  const batches = chunk(union(allDocs1, allDocs2), partitionSize);
+
+  const missingRevsInDb2 = await getMissingRevs(client, database1, database2, batches);
+  const missingRevsInDb1 = await getMissingRevs(client, database2, database1, batches);
+
+  databasesSame = databasesSame && isEmpty(missingRevsInDb1) && isEmpty(missingRevsInDb2);
+
+  if (!databasesSame) {
+    console.log('Missing revs in db 1:' + JSON.stringify(missingRevsInDb1));
+    console.log('Missing revs in db 2:' + JSON.stringify(missingRevsInDb2));
+  }
+
+  return databasesSame;
+};
+
+const getMissingRevs = async(client, databaseName1, databaseName2, batcheses) => {
+  const fakeRevisionId = '9999-a';
+
+  const missing = {};
+
+  // look in db1 - use a fake revision ID to fetch all leaf revisions
+
+  for (const batches of batcheses) {
+    const documentRevisions = {};
+    batches.forEach(id => (documentRevisions[id] = [fakeRevisionId]));
+
+    const result1 = await client.postRevsDiff({ db: databaseName1, documentRevisions });
+    const revsDiffRequestDb2 = {};
+    forOwn(result1.result, (v, k) => (revsDiffRequestDb2[k] = v.possible_ancestors));
+    // look in db2
+    const result2 = await client.postRevsDiff({ db: databaseName2, documentRevisions: revsDiffRequestDb2 });
+    forOwn(result2.result, (v, k) => {
+      if ('missing' in v) {
+        missing[k] = v.missing;
+      }
+    });
+  }
+  return missing;
+};
+
+const getAllDocs = async(client, database) => {
+  let allDocIds = [];
+  const limit = 2000;
+  let startKey = '\u0000';
+  do {
+    const pageOfDocIds = (await client.postAllDocs({ db: database, startKey, limit })).result.rows.map(r => r.id);
+    allDocIds = allDocIds.concat(pageOfDocIds);
+    if (pageOfDocIds.length < limit) {
+      startKey = null;
+    } else {
+      startKey = pageOfDocIds[limit - 1] + '\u0000';
+    }
+  } while (startKey != null);
+  return allDocIds;
+};
+
+module.exports = {
+  compare
+};

package/.scannerwork/scanner-report/source-60.txt

@@ -1,4 +1,4 @@
-// Copyright © 2017 IBM Corp. All rights reserved.
+// Copyright © 2017, 2018 IBM Corp. All rights reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -11,65 +11,165 @@
 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 // See the License for the specific language governing permissions and
 // limitations under the License.
+
+// Small script which backs up a Cloudant or CouchDB database to an S3
+// bucket via a stream rather than on-disk file.
+//
+// The script generates the backup object name by combining together the path
+// part of the database URL and the current time.
+
 'use strict';
 
-const fs = require('fs');
 const stream = require('stream');
-const liner = require('./liner.js');
-
-const onLine = function(onCommand, batches) {
-  const change = new stream.Transform({ objectMode: true });
-  change._transform = function(line, encoding, done) {
-    if (line && line[0] === ':') {
-      const obj = {
-        command: null,
-        batch: null,
-        docs: []
-      };
-
-      let matches;
-
-      // extract command
-      matches = line.match(/^:([a-z_]+) ?/);
-      if (matches) {
-        obj.command = matches[1];
-      }
+const url = require('url');
 
-      // extract batch
-      matches = line.match(/ batch([0-9]+)/);
-      if (matches) {
-        obj.batch = parseInt(matches[1]);
-      }
+const AWS = require('aws-sdk');
+const couchbackup = require('@cloudant/couchbackup');
+const debug = require('debug')('s3-backup');
+const VError = require('verror').VError;
 
-      // if this is one we want
-      if (obj.command === 't' && batches.indexOf(obj.batch) > -1) {
-        const json = line.replace(/^.* batch[0-9]+ /, '').trim();
-        obj.docs = JSON.parse(json);
-        onCommand(obj);
-      }
-    }
-    done();
-  };
-  return change;
-};
+/*
+  Main function, run from base of file.
+*/
+function main() {
+  const argv = require('yargs')
+    .usage('Usage: $0 [options]')
+    .example('$0 -s https://user:pass@host/db -b <bucket>', 'Backup db to bucket')
+    .options({
+      source: { alias: 's', nargs: 1, demandOption: true, describe: 'Source database URL' },
+      bucket: { alias: 'b', nargs: 1, demandOption: true, describe: 'Destination bucket' },
+      prefix: { alias: 'p', nargs: 1, describe: 'Prefix for backup object key', default: 'couchbackup' },
+      s3url: { nargs: 1, describe: 'S3 endpoint URL' },
+      awsprofile: { nargs: 1, describe: 'The profile section to use in the ~/.aws/credentials file', default: 'default' }
+    })
+    .help('h').alias('h', 'help')
+    .epilog('Copyright (C) IBM 2017')
+    .argv;
+
+  const sourceUrl = argv.source;
+  const backupName = new url.URL(sourceUrl).pathname.split('/').filter(function(x) { return x; }).join('-');
+  const backupBucket = argv.bucket;
+  const backupKeyPrefix = `${argv.prefix}-${backupName}`;
+  const shallow = argv.shallow;
 
-module.exports = function(log, batches, callback) {
-  // our sense of state
-  const retval = { };
+  const backupKey = `${backupKeyPrefix}-${new Date().toISOString()}`;
 
-  // called with each line from the log file
-  const onCommand = function(obj) {
-    retval[obj.batch] = obj;
+  const s3Endpoint = argv.s3url;
+  const awsProfile = argv.awsprofile;
+
+  // Creds are from ~/.aws/credentials, environment etc. (see S3 docs).
+  const awsOpts = {
+    signatureVersion: 'v4',
+    credentials: new AWS.SharedIniFileCredentials({ profile: awsProfile })
   };
+  if (typeof s3Endpoint !== 'undefined') {
+    awsOpts.endpoint = new AWS.Endpoint(s3Endpoint);
+  }
+  const s3 = new AWS.S3(awsOpts);
 
-  // stream through the previous log file
-  fs.createReadStream(log)
-    .pipe(liner())
-    .pipe(onLine(onCommand, batches))
-    .on('error', function(err) {
-      callback(err);
+  debug(`Creating a new backup of ${s(sourceUrl)} at ${backupBucket}/${backupKey}...`);
+  bucketAccessible(s3, backupBucket)
+    .then(() => {
+      return backupToS3(sourceUrl, s3, backupBucket, backupKey, shallow);
+    })
+    .then(() => {
+      debug('done.');
     })
-    .on('finish', function() {
-      callback(null, retval);
+    .catch((reason) => {
+      debug(`Error: ${reason}`);
+      process.exit(1);
     });
-};
+}
+
+/**
+ * Return a promise that resolves if the bucket is available and
+ * rejects if not.
+ *
+ * @param {any} s3 S3 client object
+ * @param {any} bucketName Bucket name
+ * @returns Promise
+ */
+function bucketAccessible(s3, bucketName) {
+  return new Promise(function(resolve, reject) {
+    const params = {
+      Bucket: bucketName
+    };
+    s3.headBucket(params, function(err, data) {
+      if (err) {
+        reject(new VError(err, 'S3 bucket not accessible'));
+      } else {
+        resolve();
+      }
+    });
+  });
+}
+
+/**
+ * Backup directly from Cloudant to an object store object via a stream.
+ *
+ * @param {any} sourceUrl URL of database
+ * @param {any} s3Client Object store client
+ * @param {any} s3Bucket Backup destination bucket
+ * @param {any} s3Key Backup destination key name (shouldn't exist)
+ * @param {any} shallow Whether to use the couchbackup `shallow` mode
+ * @returns Promise
+ */
+function backupToS3(sourceUrl, s3Client, s3Bucket, s3Key, shallow) {
+  return new Promise((resolve, reject) => {
+    debug(`Setting up S3 upload to ${s3Bucket}/${s3Key}`);
+
+    // A pass through stream that has couchbackup's output
+    // written to it and it then read by the S3 upload client.
+    // It has a 64MB highwater mark to allow for fairly
+    // uneven network connectivity.
+    const streamToUpload = new stream.PassThrough({ highWaterMark: 67108864 });
+
+    // Set up S3 upload.
+    const params = {
+      Bucket: s3Bucket,
+      Key: s3Key,
+      Body: streamToUpload
+    };
+    s3Client.upload(params, function(err, data) {
+      debug('Object store upload done');
+      if (err) {
+        debug(err);
+        reject(new VError(err, 'Object store upload failed'));
+        return;
+      }
+      debug('Object store upload succeeded');
+      debug(data);
+      resolve();
+    }).httpUploadProgress = (progress) => {
+      debug(`Object store upload progress: ${progress}`);
+    };
+
+    debug(`Starting streaming data from ${s(sourceUrl)}`);
+    couchbackup.backup(
+      sourceUrl,
+      streamToUpload,
+      (err, obj) => {
+        if (err) {
+          debug(err);
+          reject(new VError(err, 'CouchBackup failed with an error'));
+          return;
+        }
+        debug(`Download from ${s(sourceUrl)} complete.`);
+        streamToUpload.end(); // must call end() to complete upload.
+        // resolve() is called by the upload
+      }
+    );
+  });
+}
+
+/**
+ * Remove creds from a URL, e.g., before logging
+ *
+ * @param {string} url URL to safen
+ */
+function s(originalUrl) {
+  const parts = new url.URL(originalUrl);
+  return url.format(parts, { auth: false });
+}
+
+main();

package/.scannerwork/scanner-report/source-62.txt

@@ -1,54 +1,167 @@
-// Copyright © 2017, 2018 IBM Corp. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-'use strict';
-
-/**
- * Utility methods for the command line interface.
- * @module cliutils
- * @see module:cliutils
- */
-
-const url = require('url');
-
-module.exports = {
-
-  /**
-   * Combine a base URL and a database name, ensuring at least single slash
-   * between root and database name. This allows users to have Couch behind
-   * proxies that mount Couch's / endpoint at some other mount point.
-   * @param {string} root - root URL
-   * @param {string} databaseName - database name
-   * @return concatenated URL.
-   *
-   * @private
-   */
-  databaseUrl: function databaseUrl(root, databaseName) {
-    if (!root.endsWith('/')) {
-      root = root + '/';
-    }
-    return new url.URL(encodeURIComponent(databaseName), root).toString();
-  },
-
-  /**
-   * Generate CLI argument usage text.
-   *
-   * @param {string} description - argument description.
-   * @param {string} defaultValue - default argument value.
-   *
-   * @private
-   */
-  getUsage: function getUsage(description, defaultValue) {
-    return `${description} ${defaultValue !== undefined ? ` (default: ${defaultValue})` : ''}`;
-  }
-};
+<testsuites name="test-iam">
+  <testsuite name="Basic backup and restore using API" tests="3" errors="0" failures="0" skipped="0" timestamp="2023-09-28T17:48:23" time="4.516">
+    <testcase classname="test-iam.Basic backup and restore using API" name="should backup animaldb to a file correctly" time="1.004">
+      <system-out><![CDATA[{ total: 15 }
+]]></system-out>
+    </testcase>
+    <testcase classname="test-iam.Basic backup and restore using API" name="should restore animaldb to a database correctly" time="1.768">
+      <system-out><![CDATA[{ total: 15 }
+]]></system-out>
+    </testcase>
+    <testcase classname="test-iam.Basic backup and restore using API" name="should execute a shallow mode backup successfully" time="0.562">
+      <system-out><![CDATA[{ total: 11 }
+]]></system-out>
+    </testcase>
+  </testsuite>
+  <testsuite name="Basic backup and restore using API Buffer size tests using API" tests="3" errors="0" failures="0" skipped="0" timestamp="2023-09-28T17:48:27" time="11.381">
+    <testcase classname="test-iam.Basic backup and restore using API Buffer size tests using API" name="should backup/restore animaldb with the same buffer size" time="3.718">
+      <system-out><![CDATA[{ total: 15 }
+{ total: 15 }
+]]></system-out>
+    </testcase>
+    <testcase classname="test-iam.Basic backup and restore using API Buffer size tests using API" name="should backup/restore animaldb with backup buffer &gt; restore buffer" time="3.307">
+      <system-out><![CDATA[{ total: 15 }
+{ total: 15 }
+]]></system-out>
+    </testcase>
+    <testcase classname="test-iam.Basic backup and restore using API Buffer size tests using API" name="should backup/restore animaldb with backup buffer &lt; restore buffer" time="3.544">
+      <system-out><![CDATA[{ total: 15 }
+{ total: 15 }
+]]></system-out>
+    </testcase>
+  </testsuite>
+  <testsuite name="Basic backup and restore using CLI" tests="3" errors="0" failures="0" skipped="0" timestamp="2023-09-28T17:48:39" time="6.778">
+    <testcase classname="test-iam.Basic backup and restore using CLI" name="should backup animaldb to a file correctly" time="1.637">
+      <system-out><![CDATA[Backup process close 0 null
+]]></system-out>
+    </testcase>
+    <testcase classname="test-iam.Basic backup and restore using CLI" name="should restore animaldb to a database correctly" time="2.759">
+    </testcase>
+    <testcase classname="test-iam.Basic backup and restore using CLI" name="should execute a shallow mode backup successfully" time="1.575">
+      <system-out><![CDATA[Backup process close 0 null
+]]></system-out>
+    </testcase>
+  </testsuite>
+  <testsuite name="Basic backup and restore using CLI Buffer size tests using CLI" tests="3" errors="0" failures="0" skipped="0" timestamp="2023-09-28T17:48:45" time="19.568">
+    <testcase classname="test-iam.Basic backup and restore using CLI Buffer size tests using CLI" name="should backup/restore animaldb with the same buffer size" time="5.473">
+      <system-out><![CDATA[Backup process close 0 null
+]]></system-out>
+    </testcase>
+    <testcase classname="test-iam.Basic backup and restore using CLI Buffer size tests using CLI" name="should backup/restore animaldb with backup buffer &gt; restore buffer" time="6.086">
+      <system-out><![CDATA[Backup process close 0 null
+]]></system-out>
+    </testcase>
+    <testcase classname="test-iam.Basic backup and restore using CLI Buffer size tests using CLI" name="should backup/restore animaldb with backup buffer &lt; restore buffer" time="7.196">
+      <system-out><![CDATA[Backup process close 0 null
+]]></system-out>
+    </testcase>
+  </testsuite>
+  <testsuite name="Compression tests using API" tests="3" errors="0" failures="0" skipped="0" timestamp="2023-09-28T17:49:05" time="6.471">
+    <testcase classname="test-iam.Compression tests using API" name="should backup animaldb to a compressed file" time="0.902">
+      <system-out><![CDATA[{ total: 15 }
+]]></system-out>
+    </testcase>
+    <testcase classname="test-iam.Compression tests using API" name="should backup and restore animaldb via a compressed file" time="2.654">
+      <system-out><![CDATA[{ total: 15 }
+{ total: 15 }
+]]></system-out>
+    </testcase>
+    <testcase classname="test-iam.Compression tests using API" name="should backup and restore animaldb via a compressed stream" time="2.102">
+      <system-out><![CDATA[{ total: 15 }
+{ total: 15 }
+]]></system-out>
+    </testcase>
+  </testsuite>
+  <testsuite name="Compression tests using CLI" tests="3" errors="0" failures="0" skipped="0" timestamp="2023-09-28T17:49:11" time="11.064">
+    <testcase classname="test-iam.Compression tests using CLI" name="should backup animaldb to a compressed file" time="2.549">
+      <system-out><![CDATA[Backup process close 0 null
+]]></system-out>
+    </testcase>
+    <testcase classname="test-iam.Compression tests using CLI" name="should backup and restore animaldb via a compressed file" time="4.413">
+      <system-out><![CDATA[Backup process close 0 null
+]]></system-out>
+    </testcase>
+    <testcase classname="test-iam.Compression tests using CLI" name="should backup and restore animaldb via a compressed stream" time="3.295">
+      <system-out><![CDATA[Backup process close 0 null
+]]></system-out>
+    </testcase>
+  </testsuite>
+  <testsuite name="End to end backup and restore using API" tests="2" errors="0" failures="0" skipped="0" timestamp="2023-09-28T17:49:22" time="336.733">
+    <testcase classname="test-iam.End to end backup and restore using API" name="should backup and restore animaldb" time="2.15">
+      <system-out><![CDATA[{ total: 15 }
+{ total: 15 }
+]]></system-out>
+    </testcase>
+    <testcase classname="test-iam.End to end backup and restore using API" name="should backup and restore largedb1g #slow" time="333.805">
+      <system-out><![CDATA[{ total: 522948 }
+{ total: 522948 }
+]]></system-out>
+    </testcase>
+  </testsuite>
+  <testsuite name="End to end backup and restore using CLI" tests="2" errors="0" failures="0" skipped="0" timestamp="2023-09-28T17:54:59" time="552.076">
+    <testcase classname="test-iam.End to end backup and restore using CLI" name="should backup and restore animaldb" time="2.998">
+      <system-out><![CDATA[Backup process close 0 null
+]]></system-out>
+    </testcase>
+    <testcase classname="test-iam.End to end backup and restore using CLI" name="should backup and restore largedb1g #slow" time="548.301">
+      <system-out><![CDATA[Backup process close 0 null
+]]></system-out>
+    </testcase>
+  </testsuite>
+  <testsuite name="Encryption tests" tests="1" errors="0" failures="0" skipped="0" timestamp="2023-09-28T18:04:11" time="4.542">
+    <testcase classname="test-iam.Encryption tests" name="should backup and restore animaldb via an encrypted file" time="4.279">
+      <system-out><![CDATA[Backup process close 0 null
+]]></system-out>
+    </testcase>
+  </testsuite>
+  <testsuite name="Write error tests" tests="1" errors="0" failures="0" skipped="0" timestamp="2023-09-28T18:04:16" time="0.264">
+    <testcase classname="test-iam.Write error tests" name="calls callback with error set when stream is not writeable" time="0.007">
+    </testcase>
+  </testsuite>
+  <testsuite name="Event tests" tests="2" errors="0" failures="0" skipped="0" timestamp="2023-09-28T18:04:16" time="2.255">
+    <testcase classname="test-iam.Event tests" name="should get a finished event when using stdout" time="0.811">
+    </testcase>
+    <testcase classname="test-iam.Event tests" name="should get a finished event when using file output" time="0.912">
+    </testcase>
+  </testsuite>
+  <testsuite name="Resume tests using API" tests="3" errors="0" failures="0" skipped="0" timestamp="2023-09-28T18:04:18" time="5.213">
+    <testcase classname="test-iam.Resume tests using API" name="should create a log file" time="0.973">
+      <system-out><![CDATA[{ total: 15 }
+]]></system-out>
+    </testcase>
+    <testcase classname="test-iam.Resume tests using API" name="should restore corrupted animaldb to a database correctly" time="1.735">
+      <system-out><![CDATA[{ total: 15 }
+]]></system-out>
+    </testcase>
+    <testcase classname="test-iam.Resume tests using API" name="should restore resumed animaldb with blank line to a database correctly" time="1.716">
+      <system-out><![CDATA[{ total: 15 }
+]]></system-out>
+    </testcase>
+  </testsuite>
+  <testsuite name="Resume tests using CLI" tests="3" errors="0" failures="0" skipped="0" timestamp="2023-09-28T18:04:24" time="8.3">
+    <testcase classname="test-iam.Resume tests using CLI" name="should create a log file" time="1.782">
+      <system-out><![CDATA[Backup process close 0 null
+]]></system-out>
+    </testcase>
+    <testcase classname="test-iam.Resume tests using CLI" name="should restore corrupted animaldb to a database correctly" time="3.187">
+    </testcase>
+    <testcase classname="test-iam.Resume tests using CLI" name="should restore resumed animaldb with blank line to a database correctly" time="2.537">
+    </testcase>
+  </testsuite>
+  <testsuite name="Resume tests" tests="2" errors="0" failures="0" skipped="0" timestamp="2023-09-28T18:04:32" time="36.201">
+    <testcase classname="test-iam.Resume tests" name="should correctly backup and restore backup10m" time="17.333">
+      <system-out><![CDATA[Backup process close null SIGTERM
+Backup process close 0 null
+]]></system-out>
+    </testcase>
+    <testcase classname="test-iam.Resume tests" name="should correctly backup and restore backup10m using --output" time="18.352">
+      <system-out><![CDATA[Backup process close null SIGTERM
+Backup process close 0 null
+]]></system-out>
+    </testcase>
+  </testsuite>
+  <testsuite name="Longer spool changes checks" tests="1" errors="0" failures="0" skipped="0" timestamp="2023-09-28T18:05:08" time="54.231">
+    <testcase classname="test-iam.Longer spool changes checks" name="#slow should keep collecting changes (25M)" time="53.965">
+    </testcase>
+  </testsuite>
+</testsuites>