@sanity/export 2.33.4-performance-opts.16 → 3.0.0-canary.186

package/package.json

@@ -1,17 +1,17 @@
 {
   "name": "@sanity/export",
-  "version": "2.33.4-performance-opts.16+6acb5be7ec",
+  "version": "3.0.0-canary.186+7aff92534",
   "description": "Export Sanity documents and assets",
-  "main": "lib/export.js",
+  "main": "./src/export.js",
+  "files": [
+    "src"
+  ],
   "engines": {
-    "node": ">=12.0.0"
+    "node": ">=14.0.0"
   },
   "author": "Sanity.io <hello@sanity.io>",
   "license": "MIT",
   "scripts": {
-    "build": "babel src --copy-files --out-dir lib",
-    "clean": "rimraf lib dest",
-    "prebuild": "npm run clean",
     "test": "jest"
   },
   "keywords": [
@@ -26,15 +26,14 @@
   "dependencies": {
     "archiver": "^5.0.0",
     "debug": "^3.2.7",
-    "fs-extra": "^7.0.0",
     "get-it": "^5.2.1",
-    "lodash": "^4.17.15",
+    "lodash": "^4.17.21",
     "mississippi": "^4.0.0",
     "p-queue": "^2.3.0",
+    "rimraf": "^3.0.2",
     "split2": "^3.2.2"
   },
   "devDependencies": {
-    "rimraf": "^2.7.1",
     "string-to-stream": "^1.1.0"
   },
   "publishConfig": {
@@ -49,5 +48,5 @@
     "url": "https://github.com/sanity-io/sanity/issues"
   },
   "homepage": "https://www.sanity.io/",
-  "gitHead": "6acb5be7ec060806773c41c9b015f318e2fbfbd9"
+  "gitHead": "7aff92534da0e74236f647102792b896001346aa"
 }

package/src/AssetHandler.js

@@ -1,13 +1,14 @@
 const path = require('path')
 const crypto = require('crypto')
 const {parse: parseUrl, format: formatUrl} = require('url')
-const fse = require('fs-extra')
+const {mkdirSync, createWriteStream} = require('fs')
 const miss = require('mississippi')
 const PQueue = require('p-queue')
 const {omit, noop} = require('lodash')
 const pkg = require('../package.json')
 const requestStream = require('./requestStream')
 const debug = require('./debug')
+const rimraf = require('./util/rimraf')
 
 const EXCLUDE_PROPS = ['_id', '_type', 'assetId', 'extension', 'mimeType', 'path', 'url']
 const ACTION_REMOVE = 'remove'
@@ -112,8 +113,8 @@ class AssetHandler {
     }
 
     /* eslint-disable no-sync */
-    fse.ensureDirSync(path.join(this.tmpDir, 'files'))
-    fse.ensureDirSync(path.join(this.tmpDir, 'images'))
+    mkdirSync(path.join(this.tmpDir, 'files'), {recursive: true})
+    mkdirSync(path.join(this.tmpDir, 'images'), {recursive: true})
     /* eslint-enable no-sync */
     this.assetDirsCreated = true
   }
@@ -195,7 +196,7 @@ class AssetHandler {
 
       const detailsString = `Details:\n - ${details.filter(Boolean).join('\n - ')}`
 
-      await fse.unlink(tmpPath)
+      await rimraf(tmpPath)
       this.queue.clear()
 
       const error = new Error(
@@ -306,7 +307,7 @@ function writeHashedStream(filePath, stream) {
   })
 
   return new Promise((resolve, reject) =>
-    miss.pipe(stream, hasher, fse.createWriteStream(filePath), (err) => {
+    miss.pipe(stream, hasher, createWriteStream(filePath), (err) => {
       if (err) {
         reject(err)
         return

package/src/export.js

@@ -1,10 +1,11 @@
 const os = require('os')
 const path = require('path')
 const zlib = require('zlib')
-const fse = require('fs-extra')
+const fs = require('fs')
 const miss = require('mississippi')
 const split = require('split2')
 const archiver = require('archiver')
+const rimraf = require('./util/rimraf')
 const debug = require('./debug')
 const AssetHandler = require('./AssetHandler')
 const stringifyStream = require('./stringifyStream')
@@ -35,7 +36,7 @@ function exportDataset(opts) {
   const prefix = `${opts.dataset}-export-${slugDate}`
   const tmpDir = path.join(os.tmpdir(), prefix)
   const cleanup = () =>
-    fse.remove(tmpDir).catch((err) => {
+    rimraf(tmpDir).catch((err) => {
       debug(`Error while cleaning up temporary files: ${err.message}`)
     })
 
@@ -54,7 +55,7 @@ function exportDataset(opts) {
     outputStream = options.outputPath
   } else {
     outputStream =
-      options.outputPath === '-' ? process.stdout : fse.createWriteStream(options.outputPath)
+      options.outputPath === '-' ? process.stdout : fs.createWriteStream(options.outputPath)
   }
 
   let assetStreamHandler = assetHandler.noop

package/src/util/rimraf.js

@@ -0,0 +1,4 @@
+const {promisify} = require('util')
+const rimrafCb = require('rimraf')
+
+module.exports = promisify(rimrafCb)

package/jest.config.js

@@ -1,6 +0,0 @@
-const createConfig = require('../../../createJestConfig')
-
-module.exports = createConfig({
-  displayName: require('./package.json').name,
-  testEnvironment: 'node',
-})

package/lib/AssetHandler.js

@@ -1,407 +0,0 @@
-"use strict";
-
-var _excluded = ["asset"];
-
-function _slicedToArray(arr, i) { return _arrayWithHoles(arr) || _iterableToArrayLimit(arr, i) || _unsupportedIterableToArray(arr, i) || _nonIterableRest(); }
-
-function _nonIterableRest() { throw new TypeError("Invalid attempt to destructure non-iterable instance.\nIn order to be iterable, non-array objects must have a [Symbol.iterator]() method."); }
-
-function _unsupportedIterableToArray(o, minLen) { if (!o) return; if (typeof o === "string") return _arrayLikeToArray(o, minLen); var n = Object.prototype.toString.call(o).slice(8, -1); if (n === "Object" && o.constructor) n = o.constructor.name; if (n === "Map" || n === "Set") return Array.from(o); if (n === "Arguments" || /^(?:Ui|I)nt(?:8|16|32)(?:Clamped)?Array$/.test(n)) return _arrayLikeToArray(o, minLen); }
-
-function _arrayLikeToArray(arr, len) { if (len == null || len > arr.length) len = arr.length; for (var i = 0, arr2 = new Array(len); i < len; i++) { arr2[i] = arr[i]; } return arr2; }
-
-function _iterableToArrayLimit(arr, i) { var _i = arr == null ? null : typeof Symbol !== "undefined" && arr[Symbol.iterator] || arr["@@iterator"]; if (_i == null) return; var _arr = []; var _n = true; var _d = false; var _s, _e; try { for (_i = _i.call(arr); !(_n = (_s = _i.next()).done); _n = true) { _arr.push(_s.value); if (i && _arr.length === i) break; } } catch (err) { _d = true; _e = err; } finally { try { if (!_n && _i["return"] != null) _i["return"](); } finally { if (_d) throw _e; } } return _arr; }
-
-function _arrayWithHoles(arr) { if (Array.isArray(arr)) return arr; }
-
-function ownKeys(object, enumerableOnly) { var keys = Object.keys(object); if (Object.getOwnPropertySymbols) { var symbols = Object.getOwnPropertySymbols(object); enumerableOnly && (symbols = symbols.filter(function (sym) { return Object.getOwnPropertyDescriptor(object, sym).enumerable; })), keys.push.apply(keys, symbols); } return keys; }
-
-function _objectSpread(target) { for (var i = 1; i < arguments.length; i++) { var source = null != arguments[i] ? arguments[i] : {}; i % 2 ? ownKeys(Object(source), !0).forEach(function (key) { _defineProperty(target, key, source[key]); }) : Object.getOwnPropertyDescriptors ? Object.defineProperties(target, Object.getOwnPropertyDescriptors(source)) : ownKeys(Object(source)).forEach(function (key) { Object.defineProperty(target, key, Object.getOwnPropertyDescriptor(source, key)); }); } return target; }
-
-function _objectWithoutProperties(source, excluded) { if (source == null) return {}; var target = _objectWithoutPropertiesLoose(source, excluded); var key, i; if (Object.getOwnPropertySymbols) { var sourceSymbolKeys = Object.getOwnPropertySymbols(source); for (i = 0; i < sourceSymbolKeys.length; i++) { key = sourceSymbolKeys[i]; if (excluded.indexOf(key) >= 0) continue; if (!Object.prototype.propertyIsEnumerable.call(source, key)) continue; target[key] = source[key]; } } return target; }
-
-function _objectWithoutPropertiesLoose(source, excluded) { if (source == null) return {}; var target = {}; var sourceKeys = Object.keys(source); var key, i; for (i = 0; i < sourceKeys.length; i++) { key = sourceKeys[i]; if (excluded.indexOf(key) >= 0) continue; target[key] = source[key]; } return target; }
-
-function asyncGeneratorStep(gen, resolve, reject, _next, _throw, key, arg) { try { var info = gen[key](arg); var value = info.value; } catch (error) { reject(error); return; } if (info.done) { resolve(value); } else { Promise.resolve(value).then(_next, _throw); } }
-
-function _asyncToGenerator(fn) { return function () { var self = this, args = arguments; return new Promise(function (resolve, reject) { var gen = fn.apply(self, args); function _next(value) { asyncGeneratorStep(gen, resolve, reject, _next, _throw, "next", value); } function _throw(err) { asyncGeneratorStep(gen, resolve, reject, _next, _throw, "throw", err); } _next(undefined); }); }; }
-
-function _defineProperty(obj, key, value) { if (key in obj) { Object.defineProperty(obj, key, { value: value, enumerable: true, configurable: true, writable: true }); } else { obj[key] = value; } return obj; }
-
-var path = require('path');
-
-var crypto = require('crypto');
-
-var _require = require('url'),
-    parseUrl = _require.parse,
-    formatUrl = _require.format;
-
-var fse = require('fs-extra');
-
-var miss = require('mississippi');
-
-var PQueue = require('p-queue');
-
-var _require2 = require('lodash'),
-    omit = _require2.omit,
-    noop = _require2.noop;
-
-var pkg = require('../package.json');
-
-var requestStream = require('./requestStream');
-
-var debug = require('./debug');
-
-var EXCLUDE_PROPS = ['_id', '_type', 'assetId', 'extension', 'mimeType', 'path', 'url'];
-var ACTION_REMOVE = 'remove';
-var ACTION_REWRITE = 'rewrite';
-var ASSET_DOWNLOAD_CONCURRENCY = 8;
-
-class AssetHandler {
-  constructor(options) {
-    var _this = this;
-
-    _defineProperty(this, "rewriteAssets", miss.through.obj( /*#__PURE__*/function () {
-      var _ref = _asyncToGenerator(function* (doc, enc, callback) {
-        if (['sanity.imageAsset', 'sanity.fileAsset'].includes(doc._type)) {
-          var type = doc._type === 'sanity.imageAsset' ? 'image' : 'file';
-          var filePath = "".concat(type, "s/").concat(generateFilename(doc._id));
-
-          _this.assetsSeen.set(doc._id, type);
-
-          _this.queueAssetDownload(doc, filePath, type);
-
-          callback();
-          return;
-        }
-
-        callback(null, _this.findAndModify(doc, ACTION_REWRITE));
-      });
-
-      return function (_x, _x2, _x3) {
-        return _ref.apply(this, arguments);
-      };
-    }()));
-
-    _defineProperty(this, "stripAssets", miss.through.obj( /*#__PURE__*/function () {
-      var _ref2 = _asyncToGenerator(function* (doc, enc, callback) {
-        if (['sanity.imageAsset', 'sanity.fileAsset'].includes(doc._type)) {
-          callback();
-          return;
-        }
-
-        callback(null, _this.findAndModify(doc, ACTION_REMOVE));
-      });
-
-      return function (_x4, _x5, _x6) {
-        return _ref2.apply(this, arguments);
-      };
-    }()));
-
-    _defineProperty(this, "skipAssets", miss.through.obj((doc, enc, callback) => {
-      var isAsset = ['sanity.imageAsset', 'sanity.fileAsset'].includes(doc._type);
-
-      if (isAsset) {
-        callback();
-        return;
-      }
-
-      callback(null, doc);
-    }));
-
-    _defineProperty(this, "noop", miss.through.obj((doc, enc, callback) => callback(null, doc)));
-
-    _defineProperty(this, "findAndModify", (item, action) => {
-      if (Array.isArray(item)) {
-        var children = item.map(child => this.findAndModify(child, action));
-        return children.filter(Boolean);
-      }
-
-      if (!item || typeof item !== 'object') {
-        return item;
-      }
-
-      var isAsset = isAssetField(item);
-
-      if (isAsset && action === ACTION_REMOVE) {
-        return undefined;
-      }
-
-      if (isAsset && action === ACTION_REWRITE) {
-        var asset = item.asset,
-            other = _objectWithoutProperties(item, _excluded);
-
-        var assetId = asset._ref;
-        var assetType = getAssetType(item);
-        var filePath = "".concat(assetType, "s/").concat(generateFilename(assetId));
-        return _objectSpread({
-          _sanityAsset: "".concat(assetType, "@file://./").concat(filePath)
-        }, this.findAndModify(other, action));
-      }
-
-      var newItem = {};
-      var keys = Object.keys(item);
-
-      for (var i = 0; i < keys.length; i++) {
-        var key = keys[i];
-        var value = item[key];
-        newItem[key] = this.findAndModify(value, action);
-
-        if (typeof newItem[key] === 'undefined') {
-          delete newItem[key];
-        }
-      }
-
-      return newItem;
-    });
-
-    var concurrency = options.concurrency || ASSET_DOWNLOAD_CONCURRENCY;
-    debug('Using asset download concurrency of %d', concurrency);
-    this.client = options.client;
-    this.tmpDir = options.tmpDir;
-    this.assetDirsCreated = false;
-    this.downloading = [];
-    this.assetsSeen = new Map();
-    this.assetMap = {};
-    this.filesWritten = 0;
-    this.queueSize = 0;
-    this.queue = options.queue || new PQueue({
-      concurrency
-    });
-    this.rejectedError = null;
-
-    this.reject = err => {
-      this.rejectedError = err;
-    };
-  }
-
-  clear() {
-    this.assetsSeen.clear();
-    this.queue.clear();
-    this.queueSize = 0;
-  }
-
-  finish() {
-    return new Promise((resolve, reject) => {
-      if (this.rejectedError) {
-        reject(this.rejectedError);
-        return;
-      }
-
-      this.reject = reject;
-      this.queue.onIdle().then(() => resolve(this.assetMap));
-    });
-  } // Called when we want to download all assets to local filesystem and rewrite documents to hold
-  // placeholder asset references (_sanityAsset: 'image@file:///local/path')
-
-
-  queueAssetDownload(assetDoc, dstPath, type) {
-    if (!assetDoc.url) {
-      debug('Asset document "%s" does not have a URL property, skipping', assetDoc._id);
-      return;
-    }
-
-    debug('Adding download task for %s (destination: %s)', assetDoc._id, dstPath);
-    this.queueSize++;
-    this.downloading.push(assetDoc.url);
-    this.queue.add(() => this.downloadAsset(assetDoc, dstPath));
-  }
-
-  maybeCreateAssetDirs() {
-    if (this.assetDirsCreated) {
-      return;
-    }
-    /* eslint-disable no-sync */
-
-
-    fse.ensureDirSync(path.join(this.tmpDir, 'files'));
-    fse.ensureDirSync(path.join(this.tmpDir, 'images'));
-    /* eslint-enable no-sync */
-
-    this.assetDirsCreated = true;
-  }
-
-  getAssetRequestOptions(assetDoc) {
-    var token = this.client.config().token;
-    var headers = {
-      'User-Agent': "".concat(pkg.name, "@").concat(pkg.version)
-    };
-    var isImage = assetDoc._type === 'sanity.imageAsset';
-    var url = parseUrl(assetDoc.url, true);
-
-    if (isImage && ['cdn.sanity.io', 'cdn.sanity.work'].includes(url.hostname) && token) {
-      headers.Authorization = "Bearer ".concat(token);
-      url.query = _objectSpread(_objectSpread({}, url.query || {}), {}, {
-        dlRaw: 'true'
-      });
-    }
-
-    return {
-      url: formatUrl(url),
-      headers
-    };
-  }
-
-  downloadAsset(assetDoc, dstPath) {
-    var _arguments = arguments,
-        _this2 = this;
-
-    return _asyncToGenerator(function* () {
-      var attemptNum = _arguments.length > 2 && _arguments[2] !== undefined ? _arguments[2] : 0;
-      var url = assetDoc.url;
-
-      var options = _this2.getAssetRequestOptions(assetDoc);
-
-      var stream;
-
-      try {
-        stream = yield requestStream(options);
-      } catch (err) {
-        _this2.reject(err);
-
-        return false;
-      }
-
-      if (stream.statusCode !== 200) {
-        _this2.queue.clear();
-
-        var err = yield tryGetErrorFromStream(stream);
-        var errMsg = "Referenced asset URL \"".concat(url, "\" returned HTTP ").concat(stream.statusCode);
-
-        if (err) {
-          errMsg = "".concat(errMsg, ":\n\n").concat(err);
-        }
-
-        _this2.reject(new Error(errMsg));
-
-        return false;
-      }
-
-      _this2.maybeCreateAssetDirs();
-
-      debug('Asset stream ready, writing to filesystem at %s', dstPath);
-      var tmpPath = path.join(_this2.tmpDir, dstPath);
-
-      var _yield$writeHashedStr = yield writeHashedStream(tmpPath, stream),
-          sha1 = _yield$writeHashedStr.sha1,
-          md5 = _yield$writeHashedStr.md5,
-          size = _yield$writeHashedStr.size; // Verify it against our downloaded stream to make sure we have the same copy
-
-
-      var contentLength = stream.headers['content-length'];
-      var remoteSha1 = stream.headers['x-sanity-sha1'];
-      var remoteMd5 = stream.headers['x-sanity-md5'];
-      var hasHash = Boolean(remoteSha1 || remoteMd5);
-      var method = sha1 ? 'sha1' : 'md5'; // Asset validity is primarily determined by the sha1 hash. However, the sha1 hash is computed
-      // before certain processes (i.e. svg sanitization) which can result in a different hash.
-      // When the sha1 hashes don't match, fallback to using the md5 hash.
-
-      var sha1Differs = remoteSha1 && sha1 !== remoteSha1;
-      var md5Differs = remoteMd5 && md5 !== remoteMd5;
-      var differs = sha1Differs && md5Differs;
-
-      if (differs && attemptNum < 3) {
-        debug('%s does not match downloaded asset, retrying (#%d) [%s]', method, attemptNum + 1, url);
-        return _this2.downloadAsset(assetDoc, dstPath, attemptNum + 1);
-      } else if (differs) {
-        var details = [hasHash && (method === 'md5' ? "md5 should be ".concat(remoteMd5, ", got ").concat(md5) : "sha1 should be ".concat(remoteSha1, ", got ").concat(sha1)), contentLength && parseInt(contentLength, 10) !== size && "Asset should be ".concat(contentLength, " bytes, got ").concat(size), "Did not succeed after ".concat(attemptNum, " attempts.")];
-        var detailsString = "Details:\n - ".concat(details.filter(Boolean).join('\n - '));
-        yield fse.unlink(tmpPath);
-
-        _this2.queue.clear();
-
-        var error = new Error("Failed to download asset at ".concat(assetDoc.url, ", giving up. ").concat(detailsString));
-
-        _this2.reject(error);
-
-        return false;
-      }
-
-      var isImage = assetDoc._type === 'sanity.imageAsset';
-      var type = isImage ? 'image' : 'file';
-      var id = "".concat(type, "-").concat(sha1);
-      var metaProps = omit(assetDoc, EXCLUDE_PROPS);
-
-      if (Object.keys(metaProps).length > 0) {
-        _this2.assetMap[id] = metaProps;
-      }
-
-      _this2.downloading.splice(_this2.downloading.findIndex(datUrl => datUrl === url), 1);
-
-      _this2.filesWritten++;
-      return true;
-    })();
-  }
-
-}
-
-function isAssetField(item) {
-  return item.asset && item.asset._ref && isSanityAsset(item.asset._ref);
-}
-
-function getAssetType(item) {
-  if (!item.asset || typeof item.asset._ref !== 'string') {
-    return null;
-  }
-
-  var _ref3 = item.asset._ref.match(/^(image|file)-/) || [],
-      _ref4 = _slicedToArray(_ref3, 2),
-      type = _ref4[1];
-
-  return type || null;
-}
-
-function isSanityAsset(assetId) {
-  return /^image-[a-f0-9]{40}-\d+x\d+-[a-z]+$/.test(assetId) || /^file-[a-f0-9]{40}-[a-z0-9]+$/.test(assetId);
-}
-
-function generateFilename(assetId) {
-  var _ref5 = assetId.match(/^(image|file)-(.*?)(-[a-z]+)?$/) || [],
-      _ref6 = _slicedToArray(_ref5, 4),
-      asset = _ref6[2],
-      ext = _ref6[3];
-
-  var extension = (ext || 'bin').replace(/^-/, '');
-  return asset ? "".concat(asset, ".").concat(extension) : "".concat(assetId, ".bin");
-}
-
-function writeHashedStream(filePath, stream) {
-  var size = 0;
-  var md5 = crypto.createHash('md5');
-  var sha1 = crypto.createHash('sha1');
-  var hasher = miss.through((chunk, enc, cb) => {
-    size += chunk.length;
-    md5.update(chunk);
-    sha1.update(chunk);
-    cb(null, chunk);
-  });
-  return new Promise((resolve, reject) => miss.pipe(stream, hasher, fse.createWriteStream(filePath), err => {
-    if (err) {
-      reject(err);
-      return;
-    }
-
-    resolve({
-      size,
-      sha1: sha1.digest('hex'),
-      md5: md5.digest('hex')
-    });
-  }));
-}
-
-function tryGetErrorFromStream(stream) {
-  return new Promise((resolve, reject) => {
-    miss.pipe(stream, miss.concat(parse), err => err ? reject(err) : noop);
-
-    function parse(body) {
-      try {
-        var parsed = JSON.parse(body.toString('utf8'));
-        resolve(parsed.message || parsed.error || null);
-      } catch (err) {
-        resolve(body.toString('utf8').slice(0, 16000));
-      }
-    }
-  });
-}
-
-module.exports = AssetHandler;

package/lib/debug.js

@@ -1,3 +0,0 @@
-"use strict";
-
-module.exports = require('debug')('sanity:export');

package/lib/export.js

@@ -1,249 +0,0 @@
-"use strict";
-
-function asyncGeneratorStep(gen, resolve, reject, _next, _throw, key, arg) { try { var info = gen[key](arg); var value = info.value; } catch (error) { reject(error); return; } if (info.done) { resolve(value); } else { Promise.resolve(value).then(_next, _throw); } }
-
-function _asyncToGenerator(fn) { return function () { var self = this, args = arguments; return new Promise(function (resolve, reject) { var gen = fn.apply(self, args); function _next(value) { asyncGeneratorStep(gen, resolve, reject, _next, _throw, "next", value); } function _throw(err) { asyncGeneratorStep(gen, resolve, reject, _next, _throw, "throw", err); } _next(undefined); }); }; }
-
-var os = require('os');
-
-var path = require('path');
-
-var zlib = require('zlib');
-
-var fse = require('fs-extra');
-
-var miss = require('mississippi');
-
-var split = require('split2');
-
-var archiver = require('archiver');
-
-var debug = require('./debug');
-
-var AssetHandler = require('./AssetHandler');
-
-var stringifyStream = require('./stringifyStream');
-
-var validateOptions = require('./validateOptions');
-
-var rejectOnApiError = require('./rejectOnApiError');
-
-var getDocumentsStream = require('./getDocumentsStream');
-
-var filterSystemDocuments = require('./filterSystemDocuments');
-
-var filterDocumentTypes = require('./filterDocumentTypes');
-
-var filterDrafts = require('./filterDrafts');
-
-var logFirstChunk = require('./logFirstChunk');
-
-var tryParseJson = require('./tryParseJson');
-
-var noop = () => null;
-
-function exportDataset(opts) {
-  var options = validateOptions(opts);
-  var onProgress = options.onProgress || noop;
-  var archive = archiver('tar', {
-    gzip: true,
-    gzipOptions: {
-      level: options.compress ? zlib.Z_DEFAULT_COMPRESSION : zlib.Z_NO_COMPRESSION
-    }
-  });
-  var slugDate = new Date().toISOString().replace(/[^a-z0-9]/gi, '-').toLowerCase();
-  var prefix = "".concat(opts.dataset, "-export-").concat(slugDate);
-  var tmpDir = path.join(os.tmpdir(), prefix);
-
-  var cleanup = () => fse.remove(tmpDir).catch(err => {
-    debug("Error while cleaning up temporary files: ".concat(err.message));
-  });
-
-  var assetHandler = new AssetHandler({
-    client: options.client,
-    tmpDir,
-    prefix,
-    concurrency: options.assetConcurrency
-  });
-  debug('Outputting assets (temporarily) to %s', tmpDir);
-  debug('Outputting to %s', options.outputPath === '-' ? 'stdout' : options.outputPath);
-  var outputStream;
-
-  if (isWritableStream(options.outputPath)) {
-    outputStream = options.outputPath;
-  } else {
-    outputStream = options.outputPath === '-' ? process.stdout : fse.createWriteStream(options.outputPath);
-  }
-
-  var assetStreamHandler = assetHandler.noop;
-
-  if (!options.raw) {
-    assetStreamHandler = options.assets ? assetHandler.rewriteAssets : assetHandler.stripAssets;
-  }
-
-  return new Promise( /*#__PURE__*/function () {
-    var _ref = _asyncToGenerator(function* (resolve, reject) {
-      miss.finished(archive, /*#__PURE__*/function () {
-        var _ref2 = _asyncToGenerator(function* (archiveErr) {
-          if (archiveErr) {
-            debug('Archiving errored! %s', archiveErr.stack);
-            yield cleanup();
-            reject(archiveErr);
-            return;
-          }
-
-          debug('Archive finished!');
-        });
-
-        return function (_x3) {
-          return _ref2.apply(this, arguments);
-        };
-      }());
-      debug('Getting dataset export stream');
-      onProgress({
-        step: 'Exporting documents...'
-      });
-      var documentCount = 0;
-      var lastReported = Date.now();
-
-      var reportDocumentCount = (chunk, enc, cb) => {
-        ++documentCount;
-        var now = Date.now();
-
-        if (now - lastReported > 50) {
-          onProgress({
-            step: 'Exporting documents...',
-            current: documentCount,
-            total: '?',
-            update: true
-          });
-          lastReported = now;
-        }
-
-        cb(null, chunk);
-      };
-
-      var inputStream = yield getDocumentsStream(options.client, options.dataset);
-      debug('Got HTTP %d', inputStream.statusCode);
-      debug('Response headers: %o', inputStream.headers);
-      var jsonStream = miss.pipeline(inputStream, logFirstChunk(), split(tryParseJson), rejectOnApiError(), filterSystemDocuments(), assetStreamHandler, filterDocumentTypes(options.types), options.drafts ? miss.through.obj() : filterDrafts(), stringifyStream(), miss.through(reportDocumentCount));
-      miss.finished(jsonStream, /*#__PURE__*/function () {
-        var _ref3 = _asyncToGenerator(function* (err) {
-          if (err) {
-            return;
-          }
-
-          onProgress({
-            step: 'Exporting documents...',
-            current: documentCount,
-            total: documentCount,
-            update: true
-          });
-
-          if (!options.raw && options.assets) {
-            onProgress({
-              step: 'Downloading assets...'
-            });
-          }
-
-          var prevCompleted = 0;
-          var progressInterval = setInterval(() => {
-            var completed = assetHandler.queueSize - assetHandler.queue.size - assetHandler.queue.pending;
-
-            if (prevCompleted === completed) {
-              return;
-            }
-
-            prevCompleted = completed;
-            onProgress({
-              step: 'Downloading assets...',
-              current: completed,
-              total: assetHandler.queueSize,
-              update: true
-            });
-          }, 500);
-          debug('Waiting for asset handler to complete downloads');
-
-          try {
-            var assetMap = yield assetHandler.finish(); // Make sure we mark the progress as done (eg 100/100 instead of 99/100)
-
-            onProgress({
-              step: 'Downloading assets...',
-              current: assetHandler.queueSize,
-              total: assetHandler.queueSize,
-              update: true
-            });
-            archive.append(JSON.stringify(assetMap), {
-              name: 'assets.json',
-              prefix
-            });
-            clearInterval(progressInterval);
-          } catch (assetErr) {
-            clearInterval(progressInterval);
-            yield cleanup();
-            reject(assetErr);
-            return;
-          } // Add all downloaded assets to archive
-
-
-          archive.directory(path.join(tmpDir, 'files'), "".concat(prefix, "/files"), {
-            store: true
-          });
-          archive.directory(path.join(tmpDir, 'images'), "".concat(prefix, "/images"), {
-            store: true
-          });
-          debug('Finalizing archive, flushing streams');
-          onProgress({
-            step: 'Adding assets to archive...'
-          });
-          archive.finalize();
-        });
-
-        return function (_x4) {
-          return _ref3.apply(this, arguments);
-        };
-      }());
-      archive.on('warning', err => {
-        debug('Archive warning: %s', err.message);
-      });
-      archive.append(jsonStream, {
-        name: 'data.ndjson',
-        prefix
-      });
-      miss.pipe(archive, outputStream, onComplete);
-
-      function onComplete(_x5) {
-        return _onComplete.apply(this, arguments);
-      }
-
-      function _onComplete() {
-        _onComplete = _asyncToGenerator(function* (err) {
-          onProgress({
-            step: 'Clearing temporary files...'
-          });
-          yield cleanup();
-
-          if (!err) {
-            resolve();
-            return;
-          }
-
-          debug('Error during streaming: %s', err.stack);
-          assetHandler.clear();
-          reject(err);
-        });
-        return _onComplete.apply(this, arguments);
-      }
-    });
-
-    return function (_x, _x2) {
-      return _ref.apply(this, arguments);
-    };
-  }());
-}
-
-function isWritableStream(val) {
-  return val !== null && typeof val === 'object' && typeof val.pipe === 'function' && typeof val._write === 'function' && typeof val._writableState === 'object';
-}
-
-module.exports = exportDataset;

package/lib/filterDocumentTypes.js

@@ -1,14 +0,0 @@
-"use strict";
-
-var miss = require('mississippi');
-
-module.exports = allowedTypes => allowedTypes ? miss.through.obj((doc, enc, callback) => {
-  var type = doc && doc._type;
-
-  if (allowedTypes.includes(type)) {
-    callback(null, doc);
-    return;
-  }
-
-  callback();
-}) : miss.through.obj();

package/lib/filterDrafts.js

@@ -1,13 +0,0 @@
-"use strict";
-
-var miss = require('mississippi');
-
-var isDraft = doc => doc && doc._id && doc._id.indexOf('drafts.') === 0;
-
-module.exports = () => miss.through.obj((doc, enc, callback) => {
-  if (isDraft(doc)) {
-    return callback();
-  }
-
-  return callback(null, doc);
-});

package/lib/filterSystemDocuments.js

@@ -1,16 +0,0 @@
-"use strict";
-
-var miss = require('mississippi');
-
-var debug = require('./debug');
-
-var isSystemDocument = doc => doc && doc._id && doc._id.indexOf('_.') === 0;
-
-module.exports = () => miss.through.obj((doc, enc, callback) => {
-  if (isSystemDocument(doc)) {
-    debug('%s is a system document, skipping', doc && doc._id);
-    return callback();
-  }
-
-  return callback(null, doc);
-});

package/lib/getDocumentsStream.js

@@ -1,29 +0,0 @@
-"use strict";
-
-function ownKeys(object, enumerableOnly) { var keys = Object.keys(object); if (Object.getOwnPropertySymbols) { var symbols = Object.getOwnPropertySymbols(object); enumerableOnly && (symbols = symbols.filter(function (sym) { return Object.getOwnPropertyDescriptor(object, sym).enumerable; })), keys.push.apply(keys, symbols); } return keys; }
-
-function _objectSpread(target) { for (var i = 1; i < arguments.length; i++) { var source = null != arguments[i] ? arguments[i] : {}; i % 2 ? ownKeys(Object(source), !0).forEach(function (key) { _defineProperty(target, key, source[key]); }) : Object.getOwnPropertyDescriptors ? Object.defineProperties(target, Object.getOwnPropertyDescriptors(source)) : ownKeys(Object(source)).forEach(function (key) { Object.defineProperty(target, key, Object.getOwnPropertyDescriptor(source, key)); }); } return target; }
-
-function _defineProperty(obj, key, value) { if (key in obj) { Object.defineProperty(obj, key, { value: value, enumerable: true, configurable: true, writable: true }); } else { obj[key] = value; } return obj; }
-
-var pkg = require('../package.json');
-
-var requestStream = require('./requestStream');
-
-module.exports = (client, dataset) => {
-  // Sanity client doesn't handle streams natively since we want to support node/browser
-  // with same API. We're just using it here to get hold of URLs and tokens.
-  var url = client.getUrl("/data/export/".concat(dataset));
-  var token = client.config().token;
-
-  var headers = _objectSpread({
-    'User-Agent': "".concat(pkg.name, "@").concat(pkg.version)
-  }, token ? {
-    Authorization: "Bearer ".concat(token)
-  } : {});
-
-  return requestStream({
-    url,
-    headers
-  });
-};

package/lib/logFirstChunk.js

@@ -1,18 +0,0 @@
-"use strict";
-
-var miss = require('mississippi');
-
-var debug = require('./debug');
-
-module.exports = () => {
-  var firstChunk = true;
-  return miss.through((chunk, enc, callback) => {
-    if (firstChunk) {
-      var string = chunk.toString('utf8').split('\n')[0];
-      debug('First chunk received: %s', string.slice(0, 300));
-      firstChunk = false;
-    }
-
-    callback(null, chunk);
-  });
-};

package/lib/rejectOnApiError.js

@@ -1,17 +0,0 @@
-"use strict";
-
-var miss = require('mississippi');
-
-module.exports = () => miss.through.obj((doc, enc, callback) => {
-  if (doc.error && doc.statusCode) {
-    callback(new Error([doc.statusCode, doc.error].join(': ')));
-    return;
-  }
-
-  if (!doc._id && doc.error) {
-    callback(new Error(doc.error.description || doc.error.message || JSON.stringify(doc)));
-    return;
-  }
-
-  callback(null, doc);
-});

package/lib/requestStream.js

@@ -1,84 +0,0 @@
-"use strict";
-
-function ownKeys(object, enumerableOnly) { var keys = Object.keys(object); if (Object.getOwnPropertySymbols) { var symbols = Object.getOwnPropertySymbols(object); enumerableOnly && (symbols = symbols.filter(function (sym) { return Object.getOwnPropertyDescriptor(object, sym).enumerable; })), keys.push.apply(keys, symbols); } return keys; }
-
-function _objectSpread(target) { for (var i = 1; i < arguments.length; i++) { var source = null != arguments[i] ? arguments[i] : {}; i % 2 ? ownKeys(Object(source), !0).forEach(function (key) { _defineProperty(target, key, source[key]); }) : Object.getOwnPropertyDescriptors ? Object.defineProperties(target, Object.getOwnPropertyDescriptors(source)) : ownKeys(Object(source)).forEach(function (key) { Object.defineProperty(target, key, Object.getOwnPropertyDescriptor(source, key)); }); } return target; }
-
-function _defineProperty(obj, key, value) { if (key in obj) { Object.defineProperty(obj, key, { value: value, enumerable: true, configurable: true, writable: true }); } else { obj[key] = value; } return obj; }
-
-function asyncGeneratorStep(gen, resolve, reject, _next, _throw, key, arg) { try { var info = gen[key](arg); var value = info.value; } catch (error) { reject(error); return; } if (info.done) { resolve(value); } else { Promise.resolve(value).then(_next, _throw); } }
-
-function _asyncToGenerator(fn) { return function () { var self = this, args = arguments; return new Promise(function (resolve, reject) { var gen = fn.apply(self, args); function _next(value) { asyncGeneratorStep(gen, resolve, reject, _next, _throw, "next", value); } function _throw(err) { asyncGeneratorStep(gen, resolve, reject, _next, _throw, "throw", err); } _next(undefined); }); }; }
-
-var getIt = require('get-it');
-
-var _require = require('get-it/middleware'),
-    keepAlive = _require.keepAlive,
-    promise = _require.promise;
-
-var debug = require('./debug');
-
-var request = getIt([keepAlive(), promise({
-  onlyBody: true
-})]);
-var socketsWithTimeout = new WeakSet();
-var CONNECTION_TIMEOUT = 15 * 1000; // 15 seconds
-
-var READ_TIMEOUT = 3 * 60 * 1000; // 3 minutes
-
-var MAX_RETRIES = 5;
-
-function delay(ms) {
-  return new Promise(resolve => setTimeout(resolve, ms));
-}
-/* eslint-disable no-await-in-loop, max-depth */
-
-
-module.exports = /*#__PURE__*/function () {
-  var _ref = _asyncToGenerator(function* (options) {
-    var error;
-
-    for (var i = 0; i < MAX_RETRIES; i++) {
-      try {
-        var _ret = yield* function* () {
-          var response = yield request(_objectSpread(_objectSpread({}, options), {}, {
-            stream: true,
-            maxRedirects: 0,
-            timeout: {
-              connect: CONNECTION_TIMEOUT,
-              socket: READ_TIMEOUT
-            }
-          }));
-
-          if (response.connection && typeof response.connection.setTimeout === 'function' && !socketsWithTimeout.has(response.connection)) {
-            socketsWithTimeout.add(response.connection);
-            response.connection.setTimeout(READ_TIMEOUT, () => {
-              response.destroy(new Error("Read timeout: No data received on socket for ".concat(READ_TIMEOUT, " ms")));
-            });
-          }
-
-          return {
-            v: response
-          };
-        }();
-
-        if (typeof _ret === "object") return _ret.v;
-      } catch (err) {
-        error = err;
-
-        if (err.response && err.response.statusCode && err.response.statusCode < 500) {
-          break;
-        }
-
-        debug('Error, retrying after 1500ms: %s', err.message);
-        yield delay(1500);
-      }
-    }
-
-    throw error;
-  });
-
-  return function (_x) {
-    return _ref.apply(this, arguments);
-  };
-}();

package/lib/stringifyStream.js

@@ -1,5 +0,0 @@
-"use strict";
-
-var miss = require('mississippi');
-
-module.exports = () => miss.through.obj((doc, enc, callback) => callback(null, "".concat(JSON.stringify(doc), "\n")));

package/lib/tryParseJson.js

@@ -1,25 +0,0 @@
-"use strict";
-
-module.exports = line => {
-  try {
-    return JSON.parse(line);
-  } catch (err) {
-    // Catch half-done lines with an error at the end
-    var errorPosition = line.lastIndexOf('{"error":');
-
-    if (errorPosition === -1) {
-      err.message = "".concat(err.message, " (").concat(line, ")");
-      throw err;
-    }
-
-    var errorJson = line.slice(errorPosition);
-    var errorLine = JSON.parse(errorJson);
-    var error = errorLine && errorLine.error;
-
-    if (error && error.description) {
-      throw new Error("Error streaming dataset: ".concat(error.description, "\n\n").concat(errorJson, "\n"));
-    }
-
-    throw err;
-  }
-};

package/lib/validateOptions.js

@@ -1,58 +0,0 @@
-"use strict";
-
-var defaults = require('lodash/defaults');
-
-var clientMethods = ['getUrl', 'config'];
-var booleanFlags = ['assets', 'raw', 'compress', 'drafts'];
-var exportDefaults = {
-  compress: true,
-  drafts: true,
-  assets: true,
-  raw: false
-};
-
-function validateOptions(opts) {
-  var options = defaults({}, opts, exportDefaults);
-
-  if (typeof options.dataset !== 'string' || options.dataset.length < 1) {
-    throw new Error("options.dataset must be a valid dataset name");
-  }
-
-  if (options.onProgress && typeof options.onProgress !== 'function') {
-    throw new Error("options.onProgress must be a function");
-  }
-
-  if (!options.client) {
-    throw new Error('`options.client` must be set to an instance of @sanity/client');
-  }
-
-  var missing = clientMethods.find(key => typeof options.client[key] !== 'function');
-
-  if (missing) {
-    throw new Error("`options.client` is not a valid @sanity/client instance - no \"".concat(missing, "\" method found"));
-  }
-
-  var clientConfig = options.client.config();
-
-  if (!clientConfig.token) {
-    throw new Error('Client is not instantiated with a `token`');
-  }
-
-  booleanFlags.forEach(flag => {
-    if (typeof options[flag] !== 'boolean') {
-      throw new Error("Flag ".concat(flag, " must be a boolean (true/false)"));
-    }
-  });
-
-  if (!options.outputPath) {
-    throw new Error('outputPath must be specified (- for stdout)');
-  }
-
-  if (options.assetConcurrency && (options.assetConcurrency < 1 || options.assetConcurrency > 24)) {
-    throw new Error('`assetConcurrency` must be between 1 and 24');
-  }
-
-  return options;
-}
-
-module.exports = validateOptions;