@cloudant/couchbackup 2.9.17 → 2.10.0-206

package/includes/restoreMappings.js

@@ -0,0 +1,141 @@
+// Copyright © 2017, 2023 IBM Corp. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+'use strict';
+
+const { BackupError } = require('./error.js');
+const debug = require('debug');
+
+const mappingDebug = debug('couchbackup:mappings');
+const marker = '@cloudant/couchbackup:resume';
+const RESUME_COMMENT = `${JSON.stringify({ marker })}`; // Special marker for resumes
+
+class Restore {
+  // For compatibility with old versions ignore all broken JSON by default.
+  // (Old versions did not have a distinguishable resume marker).
+  // If we are restoring a backup file from a newer version we'll read the metadata
+  // and change the flag.
+  suppressAllBrokenJSONErrors = true;
+  backupMode;
+
+  constructor(dbClient) {
+    this.dbClient = dbClient;
+    this.batchCounter = 0;
+  }
+
+  /**
+   * Mapper for converting a backup file line to an array of documents pending restoration.
+   *
+   * @param {object} backupLine object representation of a backup file line {lineNumber: #, line: '...'}
+   * @returns {array} array of documents parsed from the line or an empty array for invalid lines
+   */
+  backupLineToDocsArray = (backupLine) => {
+    if (backupLine && backupLine.line !== '' && backupLine.line !== RESUME_COMMENT) {
+      // see if it parses as JSON
+      let lineAsJson;
+      try {
+        lineAsJson = JSON.parse(backupLine.line);
+      } catch (err) {
+        mappingDebug(`Invalid JSON on line ${backupLine.lineNumber} of backup file.`);
+        if (this.suppressAllBrokenJSONErrors) {
+          // The backup file comes from an older version of couchbackup that predates RESUME_COMMENT.
+          // For compatibility ignore the broken JSON line assuming it was part of a resume.
+          mappingDebug(`Ignoring invalid JSON on line ${backupLine.lineNumber} of backup file as it was written by couchbackup version < 2.10.0 and could be a valid resume point.`);
+          return [];
+        } else if (this.backupMode === 'full' && backupLine.line.slice(-RESUME_COMMENT.length) === RESUME_COMMENT) {
+          mappingDebug(`Ignoring invalid JSON on line ${backupLine.lineNumber} of full mode backup file as it was resumed.`);
+          return [];
+        } else {
+          // If the backup wasn't resumed and we aren't ignoring errors then it is invalid and we should error
+          throw new BackupError('BackupFileJsonError', `Error on line ${backupLine.lineNumber} of backup file - cannot parse as JSON`);
+        }
+      }
+      // if it's an array
+      if (lineAsJson && Array.isArray(lineAsJson)) {
+        return lineAsJson;
+      } else if (backupLine.lineNumber === 1 && lineAsJson.name && lineAsJson.version && lineAsJson.mode) {
+        // First line is metadata.
+        mappingDebug(`Parsed backup file metadata ${lineAsJson.name} ${lineAsJson.version} ${lineAsJson.mode}.`);
+        // This identifies a version of 2.10.0 or newer that wrote the backup file.
+        // Set the mode that was used for the backup file.
+        this.backupMode = lineAsJson.mode;
+        // For newer versions we don't need to ignore all broken JSON, only ones that
+        // were associated wiht a resume, so unset the ignore flag.
+        this.suppressAllBrokenJSONErrors = false;
+        // Later we may add other version/feature specific toggles here.
+      } else if (lineAsJson.marker && lineAsJson.marker === marker) {
+        mappingDebug(`Resume marker on line  ${backupLine.lineNumber} of backup file.`);
+      } else {
+        throw new BackupError('BackupFileJsonError', `Error on line ${backupLine.lineNumber} of backup file - not an array or expected metadata`);
+      }
+    }
+    // Return an empty array if there was a blank line (including a line of only the resume marker)
+    return [];
+  };
+
+  /**
+   * Mapper to wrap an array of docs in batch metadata
+   * @param {array} docs an array of documents to be restored
+   * @returns {object} a pending restore batch {batch: #, docs: [...]}
+   */
+  docsToRestoreBatch = (docs) => {
+    return { batch: this.batchCounter++, docs };
+  };
+
+  /**
+   * Mapper for converting a pending restore batch to a _bulk_docs request
+   * and awaiting the response and finally returing a "restored" object
+   * with the batch number and number of restored docs.
+   *
+   * @param {object} restoreBatch a pending restore batch {batch: #, docs: [{_id: id, ...}, ...]}
+   * @returns {object} a restored batch object { batch: #, documents: #}
+   */
+  pendingToRestored = async(restoreBatch) => {
+    // Save the batch number
+    const batch = restoreBatch.batch;
+    mappingDebug(`Preparing to restore ${batch}`);
+    // Remove it from the restoreBatch since we'll use that as our payload
+    delete restoreBatch.batch;
+    if (!restoreBatch.docs || restoreBatch.docs.length === 0) {
+      mappingDebug(`Nothing to restore in batch ${batch}.`);
+      return { batch, documents: 0 };
+    }
+    mappingDebug(`Restoring batch ${batch} with ${restoreBatch.docs.length} docs.`);
+    // if we are restoring known revisions, we need to supply new_edits=false
+    if (restoreBatch.docs[0] && restoreBatch.docs[0]._rev) {
+      restoreBatch.new_edits = false;
+      mappingDebug('Using new_edits false mode.');
+    }
+    try {
+      const response = await this.dbClient.service.postBulkDocs({
+        db: this.dbClient.dbName,
+        bulkDocs: restoreBatch
+      });
+      if (!response.result || (restoreBatch.new_edits === false && response.result.length > 0)) {
+        mappingDebug(`Some errors restoring batch ${batch}.`);
+        throw new Error(`Error writing batch ${batch} with new_edits:${restoreBatch.new_edits !== false}` +
+          ` and ${response.result ? response.result.length : 'unavailable'} items`);
+      }
+      mappingDebug(`Successfully restored batch ${batch}.`);
+      return { batch, documents: restoreBatch.docs.length };
+    } catch (err) {
+      mappingDebug(`Error writing docs when restoring batch ${batch}`);
+      throw err;
+    }
+  };
+}
+
+module.exports = {
+  Restore,
+  RESUME_COMMENT
+};

package/includes/spoolchanges.js

@@ -1,4 +1,4 @@
-// Copyright © 2017, 2022 IBM Corp. All rights reserved.
+// Copyright © 2017, 2024 IBM Corp. All rights reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -13,99 +13,77 @@
 // limitations under the License.
 'use strict';
 
-const fs = require('fs');
-const liner = require('./liner.js');
-const change = require('./change.js');
-const error = require('./error.js');
+const { createWriteStream } = require('node:fs');
+const { pipeline } = require('node:stream/promises');
+const { BackupError } = require('./error.js');
+const { BatchingStream, DelegateWritable, MappingStream } = require('./transforms.js');
 const debug = require('debug')('couchbackup:spoolchanges');
+const { ChangesFollower } = require('@ibm-cloud/cloudant');
 
 /**
  * Write log file for all changes from a database, ready for downloading
  * in batches.
  *
- * @param {string} dbUrl - URL of database
+ * @param {object} dbClient - object for connection to source database containing name, service and url
  * @param {string} log - path to log file to use
+ * @param {function} eeFn - event emitter function to call after each write
  * @param {number} bufferSize - the number of changes per batch/log line
- * @param {function(err)} callback - a callback to run on completion
+ * @param {number} tolerance - changes follower error tolerance
  */
-module.exports = function(db, log, bufferSize, ee, callback) {
-  // list of document ids to process
-  const buffer = [];
+module.exports = function(dbClient, log, eeFn, bufferSize = 500, tolerance = 600000) {
+  let lastSeq;
   let batch = 0;
-  let lastSeq = null;
-  const logStream = fs.createWriteStream(log);
-  let pending = 0;
-  // The number of changes to fetch per request
-  const limit = 100000;
+  let totalBuffer = 0;
 
-  // send documents ids to the queue in batches of bufferSize + the last batch
-  const processBuffer = function(lastOne) {
-    if (buffer.length >= bufferSize || (lastOne && buffer.length > 0)) {
-      debug('writing', buffer.length, 'changes to the backup file');
-      const b = { docs: buffer.splice(0, bufferSize), batch: batch };
-      logStream.write(':t batch' + batch + ' ' + JSON.stringify(b.docs) + '\n');
-      ee.emit('changes', batch);
-      batch++;
+  class LogWriter extends DelegateWritable {
+    constructor(log) {
+      super('logFileChangesWriter', // name for debug
+        createWriteStream(log, { flags: 'a' }), // log file write stream (append mode)
+        () => {
+          debug('finished streaming database changes');
+          return ':changes_complete ' + lastSeq + '\n';
+        }, // Function to write complete last chunk
+        mapBackupBatchToPendingLogLine, // map the changes batch to a log line
+        eeFn // postWrite function to emit the 'batch' event
+      );
     }
-  };
+  }
 
-  // called once per received change
-  const onChange = function(c) {
-    if (c) {
-      if (c.error) {
-        ee.emit('error', new error.BackupError('InvalidChange', `Received invalid change: ${c}`));
-      } else if (c.changes) {
-        const obj = { id: c.id };
-        buffer.push(obj);
-        processBuffer(false);
-      } else if (c.last_seq) {
-        lastSeq = c.last_seq;
-        pending = c.pending;
+  // Map a batch of changes to document IDs, checking for errors
+  const mapChangesToIds = function(changesBatch) {
+    return changesBatch.map((changeResultItem) => {
+      if (changeResultItem.changes && changeResultItem.changes.length > 0) {
+        if (changeResultItem.seq) {
+          lastSeq = changeResultItem.seq;
+        }
+        // Extract the document ID from the change
+        return { id: changeResultItem.id };
+      } else {
+        throw new BackupError('SpoolChangesError', `Received invalid change: ${JSON.stringify(changeResultItem)}`);
       }
-    }
+    });
   };
 
-  function getChanges(since = 0) {
-    debug('making changes request since ' + since);
-    return db.service.postChangesAsStream({ db: db.db, since: since, limit: limit, seqInterval: limit })
-      .then(response => {
-        response.result.pipe(liner())
-          .on('error', function(err) {
-            logStream.end();
-            callback(err);
-          })
-          .pipe(change(onChange))
-          .on('error', function(err) {
-            logStream.end();
-            callback(err);
-          })
-          .on('finish', function() {
-            processBuffer(true);
-            if (!lastSeq) {
-              logStream.end();
-              debug('changes request terminated before last_seq was sent');
-              callback(new error.BackupError('SpoolChangesError', 'Changes request terminated before last_seq was sent'));
-            } else {
-              debug(`changes request completed with last_seq: ${lastSeq} and ${pending} changes pending.`);
-              if (pending > 0) {
-                // Return the next promise
-                return getChanges(lastSeq);
-              } else {
-                debug('finished streaming database changes');
-                logStream.end(':changes_complete ' + lastSeq + '\n', 'utf8', callback);
-              }
-            }
-          });
-      })
-      .catch(err => {
-        logStream.end();
-        if (err.status && err.status >= 400) {
-          callback(error.convertResponseError(err));
-        } else if (err.name !== 'SpoolChangesError') {
-          callback(new error.BackupError('SpoolChangesError', `Failed changes request - ${err.message}`));
-        }
-      });
-  }
+  const mapChangesBatchToBackupBatch = function(changesBatch) {
+    return { command: 't', batch: batch++, docs: mapChangesToIds(changesBatch) };
+  };
+
+  const mapBackupBatchToPendingLogLine = function(backupBatch) {
+    totalBuffer += backupBatch.docs.length;
+    debug('writing', backupBatch.docs.length, 'changes to the backup log file with total of', totalBuffer);
+    return `:t batch${backupBatch.batch} ${JSON.stringify(backupBatch.docs)}\n`;
+  };
+
+  const changesParams = {
+    db: dbClient.dbName,
+    seqInterval: bufferSize
+  };
 
-  getChanges();
+  const changesFollower = new ChangesFollower(dbClient.service, changesParams, tolerance);
+  return pipeline(
+    changesFollower.startOneOff(), // stream of changes from the DB
+    new BatchingStream(bufferSize), // group changes into bufferSize batches for mapping
+    new MappingStream(mapChangesBatchToBackupBatch), // map a batch of ChangesResultItem to doc IDs
+    new LogWriter(log)
+  );
 };

package/includes/transforms.js

@@ -0,0 +1,378 @@
+// Copyright © 2023, 2024 IBM Corp. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+'use strict';
+
+const { Duplex, PassThrough, Writable, getDefaultHighWaterMark, setDefaultHighWaterMark } = require('node:stream');
+const debug = require('debug');
+
+/**
+ * Input: stream of elements
+ * Output: stream of arrays of batchSize elements each
+ */
+class BatchingStream extends Duplex {
+  // The buffer of elements to batch
+  elementsToBatch = [];
+  // The current batch ID
+  batchId = 0;
+  // Flag whether the Readable side is currently draining
+  isReadableDraining = true;
+  // Flag whether the writable side is complete
+  isWritableComplete = false;
+
+  /**
+   * Make a new BatchingStream with the given output batch
+   * size and whether it is accepting arrays for rebatching
+   * or single elements and buffering the set number of batches.
+   *
+   * @param {number} batchSize output batch (array) size
+   * @param {boolean} rebatch true to accept arrays and resize them (defaults to false to accept single items)
+   * @param {number} batchHighWaterMark the number of batches to buffer before applying upstream back-pressure
+   */
+  constructor(batchSize, rebatch = false, batchHighWaterMark = 1) {
+    // This Duplex stream is always objectMode and doesn't use the stream
+    // buffers. It does use an internal buffer of elements for batching, which
+    // holds up to 1 batch in memory.
+    // The Writable side of this Duplex is written to by the element supplier.
+    // The Readable side of this Duplex is read by the batch consumer.
+    super({ objectMode: true, readableHighWaterMark: 0, writableHighWaterMark: 0 });
+    // Logging config
+    this.log = debug((`couchbackup:transform:${rebatch ? 're' : ''}batch`));
+    this.log(`Batching to size ${batchSize}`);
+    this.batchSize = batchSize;
+    this.rebatch = rebatch;
+    this.elementHighWaterMark = batchHighWaterMark * this.batchSize;
+  }
+
+  /**
+   * Check the available elementsToBatch and if the downstream consumer is
+   * accepting make and push as many batches as possible.
+   *
+   * This will not push if the Readable is not draining (downstream back-pressure).
+   * Batches will be pushed if:
+   * 1. The Readable is draining and there are at least batch size elements
+   * 2. The Readable is draining and there will be no new elements (the Writable is complete)
+   *    and there are any elements available.
+   * Condition 2 allows for a smaller sized partial final batch.
+   */
+  tryPushingBatches() {
+    this.log('Try to push batches.',
+     `Available elements:${this.elementsToBatch.length}`,
+     `Readable draining:${this.isReadableDraining}`,
+     `Writable complete:${this.isWritableComplete}`);
+    while (this.isReadableDraining &&
+      (this.elementsToBatch.length >= this.batchSize ||
+        (this.isWritableComplete && this.elementsToBatch.length > 0))) {
+      // Splice up to batchSize elements from the available elements
+      const batch = this.elementsToBatch.splice(0, this.batchSize);
+      this.log(`Writing batch ${this.batchId} with ${batch.length} elements.`);
+      // Increment the batch ID ready for the next batch
+      this.batchId++;
+      // push the batch downstream
+      if (!this.push(batch)) {
+        // There was back-pressure from downstream.
+        // Unset the draining flag and break the loop.
+        this.isReadableDraining = false;
+        break;
+      }
+    }
+    if (this.elementsToBatch.length < this.batchSize) {
+      // We've drained the buffer, release upstream.
+      if (this.pendingCallback) {
+        this.log('Unblocking after downstream reads.');
+        this.pendingCallback();
+        this.pendingCallback = null;
+      }
+    }
+    if (this.elementsToBatch.length === 0 && this.isWritableComplete) {
+      this.log('No further elements, signalling EOF.');
+      this.push(null);
+    }
+  }
+
+  /**
+   * Implementation of _read.
+   * The Duplex.read is called when downstream can accept more data.
+   * That in turn calls this _read implementation.
+   *
+   * @param {number} size ignored for objectMode
+   */
+  _read(size) {
+    // Downstream asked for data set the draining flag.
+    this.isReadableDraining = true;
+    // Push any available batches.
+    this.tryPushingBatches();
+  }
+
+  /**
+   * Implementation of _write that accepts elements and
+   * adds them to an array of elementsToBatch.
+   * If the size of elementsToBatch exceeds the configured
+   * high water mark then the element supplier receives back-pressure
+   * via a delayed callback.
+   *
+   * @param {*} element the element to add to a batch
+   * @param {string} encoding ignored (objects are passed as-is)
+   * @param {function} callback called back when elementsToBatch is not too full
+   */
+  _write(element, encoding, callback) {
+    if (!this.rebatch) {
+      // If we're not rebatching we're dealing with a single element
+      // but the push is cleaner if we can spread, so wrap in an array.
+      element = [element];
+    }
+    if (this.elementsToBatch.push(...element) >= this.elementHighWaterMark) {
+      // Delay callback as we have more than 1 batch buffered
+      // Downstream cannot accept more batches, delay the
+      // callback to back-pressure our element supplier until
+      // after the next downstream read.
+      this.log(`Back pressure after batch ${this.batchId}`);
+      this.pendingCallback = callback;
+      // If there are enough elements we must try to push batches
+      // to satisfy the Readable contract which will not call read
+      // again until after a push in the event that no data was available.
+      this.tryPushingBatches();
+    } else {
+      // Callback immediately if there are fewer elements
+      callback();
+    }
+  }
+
+  /**
+   * Implementation of _final to ensure that any partial batches
+   * remaining when the supplying stream ends are pushed downstream.
+   *
+   * @param {function} callback
+   */
+  _final(callback) {
+    this.log('Flushing batch transform.');
+    // Set the writable complete flag
+    this.isWritableComplete = true;
+    // Try to push batches
+    this.tryPushingBatches();
+    callback();
+  }
+}
+
+class DelegateWritable extends Writable {
+  /**
+   * A Writable that delegates to another writable wrapping it in some
+   * helpful operations and handling "ending" of special streams like
+   * process.stdout.
+   *
+   * @param {string} name - the name of this DelegateWritable for logging
+   * @param {Writable} targetWritable - the Writable stream to write to
+   * @param {function} lastChunkFn - a no-args function to call to get a final chunk to write
+   * @param {function} chunkMapFn - a function(chunk) that can transform/map a chunk before writing
+   * @param {function} postWriteFn - a function(chunk) that can perform an action after a write completes
+   */
+  constructor(name, targetWritable, lastChunkFn, chunkMapFn, postWriteFn) {
+    super({ objectMode: true });
+    this.name = name;
+    this.targetWritable = targetWritable;
+    this.lastChunkFn = lastChunkFn;
+    this.chunkMapFn = chunkMapFn;
+    this.postWriteFn = postWriteFn;
+    this.log = debug((`couchbackup:transform:delegate:${name}`));
+  }
+
+  _write(chunk, encoding, callback) {
+    const toWrite = (this.chunkMapFn) ? this.chunkMapFn(chunk) : chunk;
+    this.targetWritable.write(toWrite, encoding, (err) => {
+      if (!err) {
+        this.log('completed target chunk write');
+        if (this.postWriteFn) {
+          this.postWriteFn(chunk);
+        }
+      }
+      callback(err);
+    });
+  }
+
+  _final(callback) {
+    this.log('Finalizing');
+    const lastChunk = (this.lastChunkFn && this.lastChunkFn()) || null;
+    // We can't 'end' stdout, so use a final write instead for that case
+    if (this.targetWritable === process.stdout) {
+      // we can't 'write' null, so don't do anything if there is no last chunk
+      if (lastChunk) {
+        this.targetWritable.write(lastChunk, 'utf-8', (err) => {
+          if (!err) {
+            this.log('wrote last chunk to stdout');
+          } else {
+            this.log('error writing last chunk to stdout');
+          }
+          callback(err);
+        });
+      } else {
+        this.log('no last chunk to write to stdout');
+        callback();
+      }
+    } else {
+      this.targetWritable.end(lastChunk, 'utf-8', (err) => {
+        if (!err) {
+          this.log('wrote last chunk and ended target writable');
+        } else {
+          this.log('error ending target writable');
+        }
+        callback(err);
+      });
+    }
+  }
+}
+
+/**
+ * A helper PassThrough class that is used in our custom
+ * Duplex streams.
+ */
+class DuplexPassThrough extends PassThrough {
+  constructor(opts) {
+    super({ objectMode: true, readableHighWaterMark: 0, writableHighWaterMark: 0, ...opts });
+  }
+
+  // The destroy call on this PassThrough stream
+  // gets ahead of real errors reaching the
+  // callback/promise at the end of the pipeline.
+  // This allows an AbortError to get propagated
+  // from the micro-task queue instead because the
+  // real error is blocked behind a callback somewhere.
+  // That in turn masks the real cause of the failure,
+  // so we defer the _destroy in a setImmediate.
+  _destroy(err, cb) {
+    setImmediate(() => {
+      cb(err);
+    });
+  }
+}
+
+class DuplexMapper extends Duplex {
+  constructor(fn, style, concurrency = 1) {
+    const operatorOpts = { concurrency, highWaterMark: concurrency };
+    const inputStream = new DuplexPassThrough();
+    let outputStream;
+    switch (style) {
+      case 'map':
+        outputStream = inputStream.map(fn, operatorOpts);
+        break;
+      case 'filter':
+        outputStream = inputStream.filter(fn, operatorOpts);
+        break;
+      default:
+        throw new Error('Invalid style.');
+    }
+    // Workaround the fact that Duplex.from doesn't allow customizing the HWM
+    // Set a new objectMode default value while we create the stream, then reset it.
+    const originalHWM = getDefaultHighWaterMark(true);
+    // Use concurrency as the highWaterMark to allow one item on deck for each "thread"
+    setDefaultHighWaterMark(true, concurrency);
+    try {
+      return Duplex.from({ readable: outputStream, writable: inputStream });
+    } finally {
+      setDefaultHighWaterMark(true, originalHWM);
+    }
+  }
+}
+
+/**
+ * Input: stream of x
+ * Output: stream of x with elements not passing the filter removed
+ */
+class FilterStream extends DuplexMapper {
+  constructor(filterFunction) {
+    super(filterFunction, 'filter');
+  }
+}
+
+/**
+ * Input: stream of x
+ * Output: stream of mappingFunction(x)
+ */
+class MappingStream extends DuplexMapper {
+  constructor(mappingFunction, concurrency = 1) {
+    super(mappingFunction, 'map', concurrency);
+  }
+}
+
+/**
+ * PassThrough stream that calls another function
+ * to perform a side effect.
+ */
+class SideEffect extends DuplexPassThrough {
+  constructor(fn, options) {
+    super(options);
+    this.fn = fn;
+    this.log = debug(('couchbackup:transform:sideeffect'));
+  }
+
+  async doSideEffect(chunk, encoding) {
+    return await this.fn(chunk, encoding);
+  }
+
+  _transform(chunk, encoding, callback) {
+    this.log('Performing side effect');
+    this.doSideEffect(chunk, encoding)
+      .then(() => {
+        this.log('Passing through');
+        super._transform(chunk, encoding, callback);
+      }).catch((err) => {
+        this.log(`Caught error ${err}`);
+        callback(err);
+      });
+  }
+}
+
+class WritableWithPassThrough extends SideEffect {
+  /**
+   * A Writable that passes through the original chunk.
+   * The chunk is also passed to a SideEffect which behaves as DelegateWritable does.
+   *
+   * @param {string} name - the name of the DelegateWritable for logging
+   * @param {Writable} targetWritable - the Writable stream to write to
+   * @param {function} lastChunkFn - a no-args function to call to get a final chunk to write
+   * @param {function} chunkMapFn - a function(chunk) that can transform/map a chunk before writing
+   * @param {function} postWriteFn - a function(chunk) that can perform an action after a write completes
+   */
+  constructor(name, targetWritable, lastChunkFn, chunkMapFn, postWriteFn) {
+    // Initialize super without a side effect fn because we need to set some
+    // properties before we can define it.
+    super(null, { objectMode: true });
+    this.log = debug(`couchbackup:transform:writablepassthrough:${name}`);
+    this.delegateWritable = new DelegateWritable(name, targetWritable, lastChunkFn, chunkMapFn, postWriteFn);
+    // Now set the side effect fn we omitted earlier
+    this.fn = (chunk, encoding) => {
+      return new Promise((resolve, reject) => {
+        this.delegateWritable.write(chunk, encoding, (err) => {
+          if (err) {
+            reject(err);
+          } else {
+            resolve();
+          }
+        });
+      });
+    };
+  }
+
+  _flush(callback) {
+    this.log('Flushing writable passthrough');
+    this.delegateWritable.end(callback);
+  }
+}
+
+module.exports = {
+  BatchingStream,
+  DelegateWritable,
+  FilterStream,
+  MappingStream,
+  SideEffect,
+  WritableWithPassThrough
+};