@cloudant/couchbackup 2.10.3-SNAPSHOT-239 → 2.10.3-SNAPSHOT-241

package/README.md

@@ -22,7 +22,7 @@ It comes with a companion command-line utility that can restore the backed up da
 CouchBackup has some restrictions in the data it's able to backup:
 
 * **`couchbackup` does not do CouchDB replication as such, it simply streams through a database's `_changes` feed, and uses `POST /db/_bulk_get` to fetch the documents, storing the documents it finds on disk.**
-* **`couchbackup` does not support backing up or restoring databases containing documents with attachments. The recommendation is to store attachments directly in an object store. DO NOT USE THIS TOOL FOR DATABASES CONTAINING ATTACHMENTS.** [Note](#note-on-attachments)
+* **`couchbackup` does not support backing up or restoring databases containing documents with attachments. The recommendation is to store attachments directly in an object store. The "attachments" option is provided as-is and is not supported. This option is for Apache CouchDB only and is experimental. DO NOT USE THIS OPTION WITH IBM Cloudant backups.** [Note](#note-on-attachments)
 
 ## Installation
 
@@ -228,6 +228,7 @@ This tool can script the backup of your databases. Move the backup and log files
 * `CLOUDANT_IAM_API_KEY` - optional [IAM API key](https://console.bluemix.net/docs/services/Cloudant/guides/iam.html#ibm-cloud-identity-and-access-management)
  to use to access the Cloudant database instead of user information credentials in the URL. The endpoint used to retrieve the token defaults to
  `https://iam.cloud.ibm.com/identity/token`, but can be overridden if necessary using the `CLOUDANT_IAM_TOKEN_URL` environment variable.
+* `COUCH_ATTACHMENTS` - _EXPERIMENTAL & UNSUPPORTED_ (see [Note](#note-on-attachments)) if `true` will include attachments as part of the backup or restore process.
 * `DEBUG` - if set to `couchbackup`, all debug messages print on `stderr` during a backup or restore process
 
 _Note:_ Environment variables are only used with the CLI. When
@@ -246,6 +247,7 @@ _Note:_ Environment variables are only used with the CLI. When
 * `--mode` - same as `COUCH_MODE`
 * `--iam-api-key` - same as `CLOUDANT_IAM_API_KEY`
 * `--quiet` - same as `COUCH_QUIET`
+* `--attachments` - _EXPERIMENTAL & UNSUPPORTED_ (see [Note](#note-on-attachments)) same as `COUCH_ATTACHMENTS`
 
 ## Using programmatically
 
@@ -289,6 +291,7 @@ target locations are not required.
 * `iamApiKey`: see `CLOUDANT_IAM_API_KEY`.
 * `iamTokenUrl`: optionally used with `iamApiKey` to override the default URL for
  retrieving IAM tokens.
+* `attachments`: _EXPERIMENTAL & UNSUPPORTED_ (see [Note](#note-on-attachments)), see `CLOUDANT_ATTACHMENTS`.
 
 When the backup completes or fails the callback functions gets called with
 the standard `err, data` parameters.
@@ -352,6 +355,7 @@ target locations are not required.
 * `iamApiKey`: see `CLOUDANT_IAM_API_KEY`.
 * `iamTokenUrl`: optionally used with `iamApiKey` to override the default URL for
  retrieving IAM tokens.
+* `attachments`: _EXPERIMENTAL & UNSUPPORTED_ (see [Note](#note-on-attachments)), see `CLOUDANT_ATTACHMENTS`.
 
 When the restore completes or fails the callback functions gets called with
 the standard `err, data` parameters.
@@ -436,16 +440,25 @@ details them.
 ### `couchrestore`
 
 * `13`: restore target database is not new and empty.
+* `60`: `attachments` option used for backup, but wasn't used for restore.
+* `61`: `attachments` option used for restore, but wasn't used for backup.
 
 ## Note on attachments
 
-TLDR; If you backup a database that has attachments `couchbackup` cannot restore it.
+TLDR; If you backup a database that has attachments without using the `attachments` option `couchbackup` can't restore it.
 
 As documented above `couchbackup` does not support backing up or restoring databases containing documents with attachments.
+
+The recommendation is to store attachments directly in an object store with a link in the JSON document instead of using the
+native attachment API.
+
+### With experimental `attachments` option
+
+The `attachments` option is provided as-is and is not supported. This option is for Apache CouchDB only and is experimental. Do not use this option with IBM Cloudant backups.
+
+### Without experimental `attachments` option
+
 Backing up a database that includes documents with attachments appears to complete successfully. However, the attachment
 content is not downloaded and the backup file contains attachment metadata. So attempts to
 restore the backup result in errors because the attachment metadata references attachments that are not present
 in the restored database.
-
-The recommendation is to store attachments directly in an object store with a link in the JSON document instead of using the
-native attachment API.

package/app.js

@@ -97,7 +97,9 @@ async function validateOptions(opts) {
     { key: 'parallelism', type: 'number' },
     { key: 'requestTimeout', type: 'number' },
     { key: 'mode', type: 'enum', values: ['full', 'shallow'] },
-    { key: 'resume', type: 'boolean' }
+    { key: 'resume', type: 'boolean' },
+    { key: 'quiet', type: 'boolean' },
+    { key: 'attachments', type: 'boolean' }
   ];
 
   for (const rule of rules) {
@@ -189,6 +191,14 @@ async function validateLogOnResume(opts) {
   return true;
 }
 
+async function attachmentWarnings(opts) {
+  if (opts && opts.attachments) {
+    console.warn('WARNING: The "attachments" option is provided as-is and is not supported. ' +
+      'This option is for Apache CouchDB only and is experimental. ' +
+      'Do not use this option with IBM Cloudant.');
+  }
+}
+
 /**
  * Validate arguments.
  *
@@ -201,7 +211,8 @@ async function validateArgs(url, opts, isBackup = true) {
   const isIAM = opts && typeof opts.iamApiKey === 'string';
   const validations = [
     validateURL(url, isIAM),
-    validateOptions(opts)
+    validateOptions(opts),
+    attachmentWarnings(opts)
   ];
   if (isBackup) {
     validations.push(
@@ -326,7 +337,7 @@ module.exports = {
         } else {
           // Write a file header including the name, version and mode
           debug('Will write backup file header.');
-          metadataToWrite = `${JSON.stringify({ name: pkg.name, version: pkg.version, mode: opts.mode })}\n`;
+          metadataToWrite = `${JSON.stringify({ name: pkg.name, version: pkg.version, mode: opts.mode, attachments: opts.attachments })}\n`;
         }
         return new Promise((resolve, reject) => {
           targetStream.write(metadataToWrite, 'utf-8', (err) => {

package/bin/couchbackup.bin.js

@@ -37,7 +37,8 @@ try {
     requestTimeout: program.requestTimeout,
     resume: program.resume,
     iamApiKey: program.iamApiKey,
-    iamTokenUrl: program.iamTokenUrl
+    iamTokenUrl: program.iamTokenUrl,
+    attachments: program.attachments
   };
 
   // log configuration to console

package/bin/couchrestore.bin.js

@@ -32,7 +32,8 @@ try {
     parallelism: program.parallelism,
     requestTimeout: program.requestTimeout,
     iamApiKey: program.iamApiKey,
-    iamTokenUrl: program.iamTokenUrl
+    iamTokenUrl: program.iamTokenUrl,
+    attachments: program.attachments
   };
 
   // log configuration to console

package/includes/allDocsGenerator.js

@@ -1,4 +1,4 @@
-// Copyright © 2023 IBM Corp. All rights reserved.
+// Copyright © 2023, 2024 IBM Corp. All rights reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -28,6 +28,9 @@ module.exports = async function * (dbClient, options = {}) {
   let lastPage = false;
   let startKey = null;
   const opts = { db: dbClient.dbName, limit: options.bufferSize, includeDocs: true };
+  if (options.attachments === true) {
+    opts.attachments = true;
+  }
   do {
     if (startKey) opts.startKey = startKey;
     yield dbClient.service.postAllDocs(opts).then(response => {

package/includes/attachmentMappings.js

@@ -0,0 +1,63 @@
+// Copyright © 2024 IBM Corp. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+'use strict';
+
+const debug = require('debug');
+const mappingDebug = debug('couchbackup:mappings');
+
+/**
+ * The cloudant-node-sdk helpfully automatically converts the base64 encoded
+ * inline attachments into Buffer so the binary data can be used by consuming
+ * applicaitons without the need to decode b64.
+ * However, in the case of couchbackup we actually want the b64 data so that
+ * we can write it in the inline attachment format to the backup file.
+ * This class provides the mappings between Buffer and Base64 binary data.
+ */
+class Attachments {
+  encode(backupBatch) {
+    backupBatch.docs.map(doc => {
+      if (doc._attachments) {
+        Object.entries(doc._attachments).forEach(([k, attachment]) => {
+          mappingDebug(`Preparing attachment ${k} for backup.`);
+          // Attachment data is a Buffer
+          // Base64 encode the attachment data for the backup file
+          attachment.data = attachment.data.toString('base64');
+          return [k, attachment];
+        });
+      }
+      return doc;
+    });
+    return backupBatch;
+  }
+
+  decode(restoreBatch) {
+    restoreBatch.docs.map(doc => {
+      if (doc._attachments) {
+        Object.entries(doc._attachments).forEach(([k, attachment]) => {
+          mappingDebug(`Preparing attachment ${k} for restore.`);
+          // Attachment data is a Base64 string
+          // Base64 decode the attachment data into a Buffer
+          attachment.data = Buffer.from(attachment.data, 'base64');
+          return [k, attachment];
+        });
+      }
+      return doc;
+    });
+    return restoreBatch;
+  }
+}
+
+module.exports = {
+  Attachments
+};

package/includes/backup.js

@@ -15,6 +15,7 @@
 
 const { createWriteStream } = require('node:fs');
 const { pipeline } = require('node:stream/promises');
+const { Attachments } = require('./attachmentMappings.js');
 const { Backup } = require('./backupMappings.js');
 const { BackupError } = require('./error.js');
 const logFileSummary = require('./logfilesummary.js');
@@ -87,13 +88,14 @@ module.exports = function(dbClient, options, targetStream, ee) {
     })
     // Create a pipeline of the source streams and the backup mappings
     .then((srcStreams) => {
-      const backup = new Backup(dbClient);
+      const backup = new Backup(dbClient, options);
       const postWrite = (backupBatch) => {
         total += backupBatch.docs.length;
         const totalRunningTimeSec = (new Date().getTime() - start) / 1000;
         ee.emit('written', { total, time: totalRunningTimeSec, batch: backupBatch.batch });
       };
 
+      const mappingStreams = [];
       const destinationStreams = [];
       if (options.mode === 'shallow') {
         // shallow mode writes only to backup file
@@ -108,8 +110,10 @@ module.exports = function(dbClient, options, targetStream, ee) {
         );
       } else {
         // full mode needs to fetch spooled changes and writes a backup file then finally a log file
+        mappingStreams.push(...[
+          new MappingStream(backup.pendingToFetched, options.parallelism) // fetch the batches at the configured concurrency
+        ]);
         destinationStreams.push(...[
-          new MappingStream(backup.pendingToFetched, options.parallelism), // fetch the batches at the configured concurrency
           new WritableWithPassThrough(
             'backup', // name for logging
             targetStream, // backup file
@@ -126,8 +130,15 @@ module.exports = function(dbClient, options, targetStream, ee) {
         ]);
       }
 
+      if (options.attachments) {
+        mappingStreams.push(
+          new MappingStream(new Attachments().encode, options.parallelism)
+        );
+      }
+
       return pipeline(
         ...srcStreams, // the source streams from the previous block (all docs async generator for shallow or for full either spool changes or resumed log)
+        ...mappingStreams, // map from source to destination content
         ...destinationStreams // the appropriate destination streams for the mode
       );
     })

package/includes/backupMappings.js

@@ -1,4 +1,4 @@
-// Copyright © 2017, 2023 IBM Corp. All rights reserved.
+// Copyright © 2017, 2024 IBM Corp. All rights reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -171,8 +171,9 @@ class LogMapper {
 }
 
 class Backup {
-  constructor(dbClient) {
+  constructor(dbClient, options) {
     this.dbClient = dbClient;
+    this.options = options;
   }
 
   /**
@@ -207,11 +208,15 @@ class Backup {
   pendingToFetched = async(backupBatch) => {
     mappingDebug(`Fetching batch ${backupBatch.batch}.`);
     try {
-      const response = await this.dbClient.service.postBulkGet({
+      const bulkGetOpts = {
         db: this.dbClient.dbName,
         revs: true,
         docs: backupBatch.docs
-      });
+      };
+      if (this.options.attachments) {
+        bulkGetOpts.attachments = true;
+      }
+      const response = await this.dbClient.service.postBulkGet(bulkGetOpts);
 
       mappingDebug(`Good server response for batch ${backupBatch.batch}.`);
       // create an output array with the docs returned

package/includes/config.js

@@ -1,4 +1,4 @@
-// Copyright © 2017, 2023 IBM Corp. All rights reserved.
+// Copyright © 2017, 2024 IBM Corp. All rights reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -22,6 +22,7 @@ const { join, normalize } = require('node:path');
 */
 function apiDefaults() {
   return {
+    attachments: false,
     parallelism: 5,
     bufferSize: 500,
     requestTimeout: 120000,
@@ -110,6 +111,11 @@ function applyEnvironmentVariables(opts) {
   if (typeof process.env.CLOUDANT_IAM_TOKEN_URL !== 'undefined') {
     opts.iamTokenUrl = process.env.CLOUDANT_IAM_TOKEN_URL;
   }
+
+  // if we are instructed to be quiet
+  if (typeof process.env.COUCH_ATTACHMENTS !== 'undefined' && process.env.COUCH_ATTACHMENTS === 'true') {
+    opts.attachments = true;
+  }
 }
 
 module.exports = {

package/includes/error.js

@@ -1,4 +1,4 @@
-// Copyright © 2017, 2023 IBM Corp. All rights reserved.
+// Copyright © 2017, 2024 IBM Corp. All rights reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -27,7 +27,9 @@ const codes = {
   LogFileExists: 23,
   SpoolChangesError: 30,
   HTTPFatalError: 40,
-  BulkGetError: 50
+  BulkGetError: 50,
+  AttachmentsNotEnabledError: 60,
+  AttachmentsMetadataAbsent: 61
 };
 
 class BackupError extends Error {

package/includes/parser.js

@@ -33,6 +33,8 @@ function parseBackupArgs() {
     .version(pkg.version)
     .description('Backup a CouchDB/Cloudant database to a backup text file.')
     .usage('[options...]')
+    .option('-a, --attachments',
+      cliutils.getUsage('*EXPERIMENTAL/UNSUPPORTED*: enable backup of attachments', defaults.attachments))
     .option('-b, --buffer-size <n>',
       cliutils.getUsage('number of documents fetched at once', defaults.bufferSize),
       Number)
@@ -97,6 +99,8 @@ function parseRestoreArgs() {
     .version(pkg.version)
     .description('Restore a CouchDB/Cloudant database from a backup text file.')
     .usage('[options...]')
+    .option('-a, --attachments',
+      cliutils.getUsage('*EXPERIMENTAL/UNSUPPORTED*: enable restore of attachments', defaults.attachments))
     .option('-b, --buffer-size <n>',
       cliutils.getUsage('number of documents restored at once', defaults.bufferSize),
       Number)

package/includes/restore.js

@@ -14,8 +14,9 @@
 'use strict';
 
 const debug = require('debug')('couchbackup:restore');
-const { Liner } = require('../includes/liner.js');
-const { Restore } = require('../includes/restoreMappings.js');
+const { Attachments } = require('./attachmentMappings.js');
+const { Liner } = require('./liner.js');
+const { Restore } = require('./restoreMappings.js');
 const { BatchingStream, MappingStream } = require('./transforms.js');
 const { Writable } = require('node:stream');
 const { pipeline } = require('node:stream/promises');
@@ -30,7 +31,7 @@ const { pipeline } = require('node:stream/promises');
  * @returns a promise that resolves when the restore is complete or rejects if it errors
  */
 module.exports = function(dbClient, options, readstream, ee) {
-  const restore = new Restore(dbClient);
+  const restore = new Restore(dbClient, options);
   const start = new Date().getTime(); // restore start time
   let total = 0; // the total restored
 
@@ -48,14 +49,29 @@ module.exports = function(dbClient, options, readstream, ee) {
     }
   });
 
-  return pipeline(
+  const batchPreparationStreams = [
     readstream, // the backup file
     new Liner(), // line by line
     new MappingStream(restore.backupLineToDocsArray), // convert line to a docs array
     new BatchingStream(options.bufferSize, true), // make new arrays of the correct buffer size
-    new MappingStream(restore.docsToRestoreBatch), // make a restore batch
+    new MappingStream(restore.docsToRestoreBatch) // make a restore batch
+  ];
+  const mappingStreams = [];
+  const restoreStreams = [
     new MappingStream(restore.pendingToRestored, options.parallelism), // do the restore at the desired level of concurrency
     output // emit restored events
+  ];
+
+  if (options.attachments) {
+    mappingStreams.push(
+      new MappingStream(new Attachments().decode, options.parallelism)
+    );
+  }
+
+  return pipeline(
+    ...batchPreparationStreams,
+    ...mappingStreams,
+    ...restoreStreams
   ).then(() => {
     return { total };
   });

package/includes/restoreMappings.js

@@ -1,4 +1,4 @@
-// Copyright © 2017, 2023 IBM Corp. All rights reserved.
+// Copyright © 2017, 2024 IBM Corp. All rights reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -28,8 +28,9 @@ class Restore {
   suppressAllBrokenJSONErrors = true;
   backupMode;
 
-  constructor(dbClient) {
+  constructor(dbClient, options) {
     this.dbClient = dbClient;
+    this.options = options;
     this.batchCounter = 0;
   }
 
@@ -65,7 +66,7 @@ class Restore {
         return lineAsJson;
       } else if (backupLine.lineNumber === 1 && lineAsJson.name && lineAsJson.version && lineAsJson.mode) {
         // First line is metadata.
-        mappingDebug(`Parsed backup file metadata ${lineAsJson.name} ${lineAsJson.version} ${lineAsJson.mode}.`);
+        mappingDebug(`Parsed backup file metadata ${lineAsJson.name} ${lineAsJson.version} ${lineAsJson.mode} ${lineAsJson.attachments}.`);
         // This identifies a version of 2.10.0 or newer that wrote the backup file.
         // Set the mode that was used for the backup file.
         this.backupMode = lineAsJson.mode;
@@ -73,6 +74,16 @@ class Restore {
         // were associated wiht a resume, so unset the ignore flag.
         this.suppressAllBrokenJSONErrors = false;
         // Later we may add other version/feature specific toggles here.
+        if (lineAsJson.attachments === true) {
+          if (!this.options.attachments) {
+            // Error out if trying to restore attachments without the option
+            throw new BackupError('AttachmentsNotEnabledError', 'To restore a backup file with attachments, enable the attachments option.');
+          }
+        } else {
+          if (this.options.attachments) {
+            throw new BackupError('AttachmentsMetadataAbsent', 'Cannot restore with attachments because the backup file was not created with the attachments option.');
+          }
+        }
       } else if (lineAsJson.marker && lineAsJson.marker === marker) {
         mappingDebug(`Resume marker on line  ${backupLine.lineNumber} of backup file.`);
       } else {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@cloudant/couchbackup",
-  "version": "2.10.3-SNAPSHOT-239",
+  "version": "2.10.3-SNAPSHOT-241",
   "description": "CouchBackup - command-line backup utility for Cloudant/CouchDB",
   "homepage": "https://github.com/IBM/couchbackup",
   "repository": {
@@ -48,7 +48,7 @@
     "eslint-plugin-promise": "6.6.0",
     "http-proxy": "1.18.1",
     "mocha": "10.7.3",
-    "nock": "13.5.4",
+    "nock": "13.5.5",
     "tail": "2.2.6",
     "uuid": "10.0.0"
   },