@cloudant/couchbackup 2.9.14-SNAPSHOT.157 → 2.9.14

package/.scannerwork/scanner-report/source-41.txt

@@ -0,0 +1,130 @@
+// Copyright © 2017, 2023 IBM Corp. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+/* global describe it before after */
+'use strict';
+
+const assert = require('assert');
+const applyEnvVars = require('../includes/config.js').applyEnvironmentVariables;
+
+describe('#unit Configuration', function() {
+  let processEnvCopy;
+
+  before('Save env', function() {
+    // Copy env so we can reset it after the tests
+    processEnvCopy = JSON.parse(JSON.stringify(process.env));
+  });
+
+  after('Reset env', function() {
+    process.env = processEnvCopy;
+  });
+
+  it('respects the COUCH_URL env variable', function() {
+    process.env.COUCH_URL = 'http://user:pass@myurl.com';
+    const config = {};
+    applyEnvVars(config);
+    assert.strictEqual(typeof config.url, 'string');
+    assert.strictEqual(config.url, process.env.COUCH_URL);
+  });
+
+  it('respects the COUCH_DATABASE env variable', function() {
+    process.env.COUCH_DATABASE = 'mydb';
+    const config = {};
+    applyEnvVars(config);
+    assert.strictEqual(typeof config.db, 'string');
+    assert.strictEqual(config.db, process.env.COUCH_DATABASE);
+  });
+
+  it('respects the COUCH_BUFFER_SIZE env variable', function() {
+    process.env.COUCH_BUFFER_SIZE = '1000';
+    const config = {};
+    applyEnvVars(config);
+    assert.strictEqual(typeof config.bufferSize, 'number');
+    assert.strictEqual(config.bufferSize, 1000);
+  });
+
+  it('respects the COUCH_PARALLELISM env variable', function() {
+    process.env.COUCH_PARALLELISM = '20';
+    const config = {};
+    applyEnvVars(config);
+    assert.strictEqual(typeof config.parallelism, 'number');
+    assert.strictEqual(config.parallelism, 20);
+  });
+
+  it('respects the COUCH_REQUEST_TIMEOUT env variable', function() {
+    process.env.COUCH_REQUEST_TIMEOUT = '10000';
+    const config = {};
+    applyEnvVars(config);
+    assert.strictEqual(typeof config.requestTimeout, 'number');
+    assert.strictEqual(config.requestTimeout, 10000);
+  });
+
+  it('respects the CLOUDANT_IAM_API_KEY env variable', function() {
+    const key = 'ABC123-ZYX987_cba789-xyz321';
+    process.env.CLOUDANT_IAM_API_KEY = key;
+    const config = {};
+    applyEnvVars(config);
+    assert.strictEqual(typeof config.iamApiKey, 'string');
+    assert.strictEqual(config.iamApiKey, key);
+  });
+
+  it('respects the CLOUDANT_IAM_TOKEN_URL env variable', function() {
+    const u = 'https://testhost.example:1234/identity/token';
+    process.env.CLOUDANT_IAM_TOKEN_URL = u;
+    const config = {};
+    applyEnvVars(config);
+    assert.strictEqual(typeof config.iamTokenUrl, 'string');
+    assert.strictEqual(config.iamTokenUrl, u);
+  });
+
+  it('respects the COUCH_LOG env variable', function() {
+    process.env.COUCH_LOG = 'my.log';
+    const config = {};
+    applyEnvVars(config);
+    assert.strictEqual(typeof config.log, 'string');
+    assert.strictEqual(config.log, process.env.COUCH_LOG);
+  });
+
+  it('respects the COUCH_RESUME env variable', function() {
+    process.env.COUCH_RESUME = 'true';
+    const config = {};
+    applyEnvVars(config);
+    assert.strictEqual(typeof config.resume, 'boolean');
+    assert.strictEqual(config.resume, true);
+  });
+
+  it('respects the COUCH_OUTPUT env variable', function() {
+    process.env.COUCH_OUTPUT = 'myfile.txt';
+    const config = {};
+    applyEnvVars(config);
+    assert.strictEqual(typeof config.output, 'string');
+    assert.strictEqual(config.output, process.env.COUCH_OUTPUT);
+  });
+
+  it('respects the COUCH_MODE env variable', function() {
+    process.env.COUCH_MODE = 'shallow';
+    const config = {};
+    applyEnvVars(config);
+    assert.strictEqual(typeof config.mode, 'string');
+    assert.strictEqual(config.mode, 'shallow');
+  });
+
+  it('respects the COUCH_QUIET env variable', function() {
+    process.env.COUCH_QUIET = 'true';
+    const config = {};
+    applyEnvVars(config);
+    assert.strictEqual(typeof config.quiet, 'boolean');
+    assert.strictEqual(config.quiet, true);
+  });
+});

package/.scannerwork/scanner-report/source-42.txt

@@ -0,0 +1,183 @@
+// Copyright © 2017, 2023 IBM Corp. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+/* global describe it beforeEach */
+'use strict';
+
+const assert = require('assert');
+const backup = require('../includes/shallowbackup.js');
+const request = require('../includes/request.js');
+const fs = require('fs');
+const nock = require('nock');
+
+// Function to create a DB object and call the shallow backup function
+// This is normally done by app.js
+function shallowBackup(dbUrl, opts) {
+  const db = request.client(dbUrl, opts);
+  // Disable compression to make body assertions easier
+  db.service.setEnableGzipCompression(false);
+  return backup(db, opts);
+}
+
+// Note all these tests include a body parameter of include_docs and a query
+// string of include_docs because of a quirk of nano that when using the fetch
+// method always adds the include_docs query string.
+describe('#unit Perform backup using shallow backup', function() {
+  const dbUrl = 'http://localhost:5984/animaldb';
+  // Query string keys are stringified by Nano
+  const badgerKey = 'badger\0';
+  const kookaburraKey = 'kookaburra\0';
+  const snipeKey = 'snipe\0';
+
+  beforeEach('Reset nocks', function() {
+    nock.cleanAll();
+  });
+
+  it('should perform a shallow backup', async function() {
+    const couch = nock(dbUrl)
+      // batch 1
+      .post('/_all_docs', { limit: 3, include_docs: true })
+      .reply(200, JSON.parse(fs.readFileSync('./test/fixtures/animaldb_all_docs_1.json', 'utf8')))
+      // batch 2
+      .post('/_all_docs', { limit: 3, start_key: badgerKey, include_docs: true })
+      .reply(200, JSON.parse(fs.readFileSync('./test/fixtures/animaldb_all_docs_2.json', 'utf8')))
+      // batch 3
+      .post('/_all_docs', { limit: 3, start_key: kookaburraKey, include_docs: true })
+      .reply(200, JSON.parse(fs.readFileSync('./test/fixtures/animaldb_all_docs_3.json', 'utf8')))
+      // batch 4
+      .post('/_all_docs', { limit: 3, start_key: snipeKey, include_docs: true })
+      .reply(200, JSON.parse(fs.readFileSync('./test/fixtures/animaldb_all_docs_4.json', 'utf8')));
+
+    return new Promise((resolve, reject) => {
+      shallowBackup(dbUrl, { bufferSize: 3, parallelism: 1 })
+        .on('error', function(err) {
+          reject(err);
+        })
+        .on('received', function(data) {
+          try {
+            if (data.batch === 3) {
+              assert.strictEqual(data.length, 2); // smaller last batch
+            } else {
+              assert.strictEqual(data.length, 3);
+            }
+          } catch (err) {
+            reject(err);
+          }
+        })
+        .on('finished', function(data) {
+          try {
+            assert.strictEqual(data.total, 11);
+            assert.ok(couch.isDone());
+            resolve();
+          } catch (err) {
+            reject(err);
+          }
+        });
+    });
+  });
+
+  it('should perform a shallow backup with transient error', async function() {
+    const couch = nock(dbUrl)
+      // batch 1
+      .post('/_all_docs', { limit: 3, include_docs: true })
+      .reply(200, JSON.parse(fs.readFileSync('./test/fixtures/animaldb_all_docs_1.json', 'utf8')))
+      // batch 2
+      .post('/_all_docs', { limit: 3, start_key: badgerKey, include_docs: true })
+      .reply(200, JSON.parse(fs.readFileSync('./test/fixtures/animaldb_all_docs_2.json', 'utf8')))
+      // batch 3 - transient error
+      .post('/_all_docs', { limit: 3, start_key: kookaburraKey, include_docs: true })
+      .reply(500, { error: 'Internal Server Error' })
+      // batch 3 - retry
+      .post('/_all_docs', { limit: 3, start_key: kookaburraKey, include_docs: true })
+      .reply(200, JSON.parse(fs.readFileSync('./test/fixtures/animaldb_all_docs_3.json', 'utf8')))
+      // batch 4
+      .post('/_all_docs', { limit: 3, start_key: snipeKey, include_docs: true })
+      .reply(200, JSON.parse(fs.readFileSync('./test/fixtures/animaldb_all_docs_4.json', 'utf8')));
+
+    return new Promise((resolve, reject) => {
+      shallowBackup(dbUrl, { bufferSize: 3, parallelism: 1 })
+        .on('error', function(err) {
+          try {
+            assert.strictEqual(err.name, 'HTTPError');
+          } catch (err) {
+            reject(err);
+          }
+        })
+        .on('received', function(data) {
+          try {
+            if (data.batch === 3) {
+              assert.strictEqual(data.length, 2); // smaller last batch
+            } else {
+              assert.strictEqual(data.length, 3);
+            }
+          } catch (err) {
+            reject(err);
+          }
+        })
+        .on('finished', function(data) {
+          try {
+            assert.strictEqual(data.total, 11);
+            assert.ok(couch.isDone());
+            resolve();
+          } catch (err) {
+            reject(err);
+          }
+        });
+    });
+  });
+
+  it('should fail to perform a shallow backup on fatal error', async function() {
+    const couch = nock(dbUrl)
+      // batch 1
+      .post('/_all_docs', { limit: 3, include_docs: true })
+      .reply(200, JSON.parse(fs.readFileSync('./test/fixtures/animaldb_all_docs_1.json', 'utf8')))
+      // batch 2
+      .post('/_all_docs', { limit: 3, start_key: badgerKey, include_docs: true })
+      .reply(200, JSON.parse(fs.readFileSync('./test/fixtures/animaldb_all_docs_2.json', 'utf8')))
+      // batch 3 - fatal error
+      .post('/_all_docs', { limit: 3, start_key: kookaburraKey, include_docs: true })
+      .reply(401, { error: 'Unauthorized' });
+
+    let errCount = 0;
+
+    return new Promise((resolve, reject) => {
+      shallowBackup(dbUrl, { bufferSize: 3, parallelism: 1 })
+        .on('error', function(err) {
+          try {
+            errCount++;
+            assert.strictEqual(err.name, 'Unauthorized');
+          } catch (err) {
+            reject(err);
+          }
+        })
+        .on('received', function(data) {
+          try {
+            assert.strictEqual(data.length, 3);
+          } catch (err) {
+            reject(err);
+          }
+        })
+        .on('finished', function(data) {
+          try {
+            assert.strictEqual(data.total, 6);
+            assert.ok(couch.isDone());
+            assert.strictEqual(errCount, 1);
+            resolve();
+          } catch (err) {
+            reject(err);
+          }
+        });
+    });
+  });
+});

package/.scannerwork/scanner-report/source-44.txt

@@ -0,0 +1,190 @@
+// Copyright © 2017, 2018 IBM Corp. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Small script which backs up a Cloudant or CouchDB database to an S3
+// bucket, using an intermediary file on disk.
+//
+// The script generates the backup object name by combining together the path
+// part of the database URL and the current time.
+
+'use strict';
+
+const stream = require('stream');
+const fs = require('fs');
+const url = require('url');
+
+const AWS = require('aws-sdk');
+const couchbackup = require('@cloudant/couchbackup');
+const debug = require('debug')('s3-backup');
+const tmp = require('tmp');
+const VError = require('verror').VError;
+
+/*
+  Main function, run from base of file.
+*/
+function main() {
+  const argv = require('yargs')
+    .usage('Usage: $0 [options]')
+    .example('$0 -s https://user:pass@host/db -b <bucket>', 'Backup db to bucket')
+    .options({
+      source: { alias: 's', nargs: 1, demandOption: true, describe: 'Source database URL' },
+      bucket: { alias: 'b', nargs: 1, demandOption: true, describe: 'Destination bucket' },
+      prefix: { alias: 'p', nargs: 1, describe: 'Prefix for backup object key', default: 'couchbackup' },
+      s3url: { nargs: 1, describe: 'S3 endpoint URL' },
+      awsprofile: { nargs: 1, describe: 'The profile section to use in the ~/.aws/credentials file', default: 'default' }
+    })
+    .help('h').alias('h', 'help')
+    .epilog('Copyright (C) IBM 2017')
+    .argv;
+
+  const sourceUrl = argv.source;
+  const backupBucket = argv.bucket;
+  const backupName = new url.URL(sourceUrl).pathname.split('/').filter(function(x) { return x; }).join('-');
+  const backupKeyPrefix = `${argv.prefix}-${backupName}`;
+
+  const backupKey = `${backupKeyPrefix}-${new Date().toISOString()}`;
+  const backupTmpFile = tmp.fileSync();
+
+  const s3Endpoint = argv.s3url;
+  const awsProfile = argv.awsprofile;
+
+  // Creds are from ~/.aws/credentials, environment etc. (see S3 docs).
+  const awsOpts = {
+    signatureVersion: 'v4',
+    credentials: new AWS.SharedIniFileCredentials({ profile: awsProfile })
+  };
+  if (typeof s3Endpoint !== 'undefined') {
+    awsOpts.endpoint = new AWS.Endpoint(s3Endpoint);
+  }
+  const s3 = new AWS.S3(awsOpts);
+
+  debug(`Creating a new backup of ${s(sourceUrl)} at ${backupBucket}/${backupKey}...`);
+  bucketAccessible(s3, backupBucket)
+    .then(() => {
+      return createBackupFile(sourceUrl, backupTmpFile.name);
+    })
+    .then(() => {
+      return uploadNewBackup(s3, backupTmpFile.name, backupBucket, backupKey);
+    })
+    .then(() => {
+      debug('Backup successful!');
+      backupTmpFile.removeCallback();
+      debug('done.');
+    })
+    .catch((reason) => {
+      debug(`Error: ${reason}`);
+    });
+}
+
+/**
+ * Return a promise that resolves if the bucket is available and
+ * rejects if not.
+ *
+ * @param {any} s3 S3 client object
+ * @param {any} bucketName Bucket name
+ * @returns Promise
+ */
+function bucketAccessible(s3, bucketName) {
+  return new Promise(function(resolve, reject) {
+    const params = {
+      Bucket: bucketName
+    };
+    s3.headBucket(params, function(err, data) {
+      if (err) {
+        reject(new VError(err, 'S3 bucket not accessible'));
+      } else {
+        resolve();
+      }
+    });
+  });
+}
+
+/**
+ * Use couchbackup to create a backup of the specified database to a file path.
+ *
+ * @param {any} sourceUrl Database URL
+ * @param {any} backupTmpFilePath Path to write file
+ * @returns Promise
+ */
+function createBackupFile(sourceUrl, backupTmpFilePath) {
+  return new Promise((resolve, reject) => {
+    couchbackup.backup(
+      sourceUrl,
+      fs.createWriteStream(backupTmpFilePath),
+      (err) => {
+        if (err) {
+          return reject(new VError(err, 'CouchBackup process failed'));
+        }
+        debug('couchbackup to file done; uploading to S3');
+        resolve('creating backup file complete');
+      }
+    );
+  });
+}
+
+/**
+ * Upload a backup file to an S3 bucket.
+ *
+ * @param {any} s3 Object store client
+ * @param {any} backupTmpFilePath Path of backup file to write.
+ * @param {any} bucket Object store bucket name
+ * @param {any} key Object store key name
+ * @returns Promise
+ */
+function uploadNewBackup(s3, backupTmpFilePath, bucket, key) {
+  return new Promise((resolve, reject) => {
+    debug(`Uploading from ${backupTmpFilePath} to ${bucket}/${key}`);
+
+    function uploadFromStream(s3, bucket, key) {
+      const pass = new stream.PassThrough();
+
+      const params = {
+        Bucket: bucket,
+        Key: key,
+        Body: pass
+      };
+      s3.upload(params, function(err, data) {
+        debug('S3 upload done');
+        if (err) {
+          debug(err);
+          reject(new VError(err, 'Upload failed'));
+          return;
+        }
+        debug('Upload succeeded');
+        debug(data);
+        resolve();
+      }).httpUploadProgress = (progress) => {
+        debug(`S3 upload progress: ${progress}`);
+      };
+
+      return pass;
+    }
+
+    const inputStream = fs.createReadStream(backupTmpFilePath);
+    const s3Stream = uploadFromStream(s3, bucket, key);
+    inputStream.pipe(s3Stream);
+  });
+}
+
+/**
+ * Remove creds from a URL, e.g., before logging
+ *
+ * @param {string} url URL to safen
+ */
+function s(originalUrl) {
+  const parts = new url.URL(originalUrl);
+  return url.format(parts, { auth: false });
+}
+
+main();

package/.scannerwork/scanner-report/source-49.txt

@@ -0,0 +1,59 @@
+// Copyright © 2017, 2018 IBM Corp. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+'use strict';
+
+/**
+ * Utility methods for the command line interface.
+ * @module cliutils
+ * @see module:cliutils
+ */
+
+const url = require('url');
+const error = require('./error.js');
+
+module.exports = {
+
+  /**
+   * Combine a base URL and a database name, ensuring at least single slash
+   * between root and database name. This allows users to have Couch behind
+   * proxies that mount Couch's / endpoint at some other mount point.
+   * @param {string} root - root URL
+   * @param {string} databaseName - database name
+   * @return concatenated URL.
+   *
+   * @private
+   */
+  databaseUrl: function databaseUrl(root, databaseName) {
+    if (!root.endsWith('/')) {
+      root = root + '/';
+    }
+    try {
+      return new url.URL(encodeURIComponent(databaseName), root).toString();
+    } catch (err) {
+      throw error.wrapPossibleInvalidUrlError(err);
+    }
+  },
+
+  /**
+   * Generate CLI argument usage text.
+   *
+   * @param {string} description - argument description.
+   * @param {string} defaultValue - default argument value.
+   *
+   * @private
+   */
+  getUsage: function getUsage(description, defaultValue) {
+    return `${description} ${defaultValue !== undefined ? ` (default: ${defaultValue})` : ''}`;
+  }
+};