@sanity/export 3.0.1-dev-preview.0 → 3.0.1-findability.147

package/package.json

@@ -1,19 +1,18 @@
 {
   "name": "@sanity/export",
-  "version": "3.0.1-dev-preview.0",
+  "version": "3.0.1-findability.147+8b2f6c2afe",
   "description": "Export Sanity documents and assets",
-  "main": "./lib/export.js",
-  "types": "./lib/dts/src/export.d.ts",
+  "main": "./src/export.js",
+  "files": [
+    "src"
+  ],
   "engines": {
     "node": ">=14.0.0"
   },
   "author": "Sanity.io <hello@sanity.io>",
   "license": "MIT",
   "scripts": {
-    "build": "../../../bin/pkg-utils transpile --target node",
-    "clean": "rimraf lib",
-    "test": "jest",
-    "watch": "../../../bin/pkg-utils transpile --target node --watch"
+    "test": "jest"
   },
   "keywords": [
     "sanity",
@@ -49,5 +48,5 @@
     "url": "https://github.com/sanity-io/sanity/issues"
   },
   "homepage": "https://www.sanity.io/",
-  "gitHead": "174eea8e59f8d7f4f1d9661de99f76aa683601f3"
+  "gitHead": "8b2f6c2afe0fbb21c7557c90057745732d21b877"
 }

package/src/AssetHandler.js

@@ -125,7 +125,7 @@ class AssetHandler {
     const isImage = assetDoc._type === 'sanity.imageAsset'
 
     const url = parseUrl(assetDoc.url, true)
-    if (isImage && ['cdn.sanity.io', 'cdn.sanity.work'].includes(url.hostname)) {
+    if (isImage && ['cdn.sanity.io', 'cdn.sanity.work'].includes(url.hostname) && token) {
       headers.Authorization = `Bearer ${token}`
       url.query = {...(url.query || {}), dlRaw: 'true'}
     }
@@ -168,14 +168,14 @@ class AssetHandler {
     const remoteSha1 = stream.headers['x-sanity-sha1']
     const remoteMd5 = stream.headers['x-sanity-md5']
     const hasHash = Boolean(remoteSha1 || remoteMd5)
-    const method = md5 ? 'md5' : 'sha1'
-
-    let differs = false
-    if (remoteMd5 && md5) {
-      differs = remoteMd5 !== md5
-    } else if (remoteSha1 && sha1) {
-      differs = remoteSha1 !== sha1
-    }
+    const method = sha1 ? 'sha1' : 'md5'
+
+    // Asset validity is primarily determined by the sha1 hash. However, the sha1 hash is computed
+    // before certain processes (i.e. svg sanitization) which can result in a different hash.
+    // When the sha1 hashes don't match, fallback to using the md5 hash.
+    const sha1Differs = remoteSha1 && sha1 !== remoteSha1
+    const md5Differs = remoteMd5 && md5 !== remoteMd5
+    const differs = sha1Differs && md5Differs
 
     if (differs && attemptNum < 3) {
       debug('%s does not match downloaded asset, retrying (#%d) [%s]', method, attemptNum + 1, url)

package/lib/AssetHandler.js

@@ -1,356 +0,0 @@
-"use strict";
-
-function _defineProperty(obj, key, value) { if (key in obj) { Object.defineProperty(obj, key, { value: value, enumerable: true, configurable: true, writable: true }); } else { obj[key] = value; } return obj; }
-
-const path = require('path');
-
-const crypto = require('crypto');
-
-const {
-  parse: parseUrl,
-  format: formatUrl
-} = require('url');
-
-const {
-  mkdirSync,
-  createWriteStream
-} = require('fs');
-
-const miss = require('mississippi');
-
-const PQueue = require('p-queue');
-
-const {
-  omit,
-  noop
-} = require('lodash');
-
-const pkg = require('../package.json');
-
-const requestStream = require('./requestStream');
-
-const debug = require('./debug');
-
-const rimraf = require('./util/rimraf');
-
-const EXCLUDE_PROPS = ['_id', '_type', 'assetId', 'extension', 'mimeType', 'path', 'url'];
-const ACTION_REMOVE = 'remove';
-const ACTION_REWRITE = 'rewrite';
-const ASSET_DOWNLOAD_CONCURRENCY = 8;
-
-class AssetHandler {
-  constructor(options) {
-    _defineProperty(this, "rewriteAssets", miss.through.obj(async (doc, enc, callback) => {
-      if (['sanity.imageAsset', 'sanity.fileAsset'].includes(doc._type)) {
-        const type = doc._type === 'sanity.imageAsset' ? 'image' : 'file';
-        const filePath = "".concat(type, "s/").concat(generateFilename(doc._id));
-        this.assetsSeen.set(doc._id, type);
-        this.queueAssetDownload(doc, filePath, type);
-        callback();
-        return;
-      }
-
-      callback(null, this.findAndModify(doc, ACTION_REWRITE));
-    }));
-
-    _defineProperty(this, "stripAssets", miss.through.obj(async (doc, enc, callback) => {
-      if (['sanity.imageAsset', 'sanity.fileAsset'].includes(doc._type)) {
-        callback();
-        return;
-      }
-
-      callback(null, this.findAndModify(doc, ACTION_REMOVE));
-    }));
-
-    _defineProperty(this, "skipAssets", miss.through.obj((doc, enc, callback) => {
-      const isAsset = ['sanity.imageAsset', 'sanity.fileAsset'].includes(doc._type);
-
-      if (isAsset) {
-        callback();
-        return;
-      }
-
-      callback(null, doc);
-    }));
-
-    _defineProperty(this, "noop", miss.through.obj((doc, enc, callback) => callback(null, doc)));
-
-    _defineProperty(this, "findAndModify", (item, action) => {
-      if (Array.isArray(item)) {
-        const children = item.map(child => this.findAndModify(child, action));
-        return children.filter(Boolean);
-      }
-
-      if (!item || typeof item !== 'object') {
-        return item;
-      }
-
-      const isAsset = isAssetField(item);
-
-      if (isAsset && action === ACTION_REMOVE) {
-        return undefined;
-      }
-
-      if (isAsset && action === ACTION_REWRITE) {
-        const {
-          asset,
-          ...other
-        } = item;
-        const assetId = asset._ref;
-        const assetType = getAssetType(item);
-        const filePath = "".concat(assetType, "s/").concat(generateFilename(assetId));
-        return {
-          _sanityAsset: "".concat(assetType, "@file://./").concat(filePath),
-          ...this.findAndModify(other, action)
-        };
-      }
-
-      const newItem = {};
-      const keys = Object.keys(item);
-
-      for (let i = 0; i < keys.length; i++) {
-        const key = keys[i];
-        const value = item[key];
-        newItem[key] = this.findAndModify(value, action);
-
-        if (typeof newItem[key] === 'undefined') {
-          delete newItem[key];
-        }
-      }
-
-      return newItem;
-    });
-
-    const concurrency = options.concurrency || ASSET_DOWNLOAD_CONCURRENCY;
-    debug('Using asset download concurrency of %d', concurrency);
-    this.client = options.client;
-    this.tmpDir = options.tmpDir;
-    this.assetDirsCreated = false;
-    this.downloading = [];
-    this.assetsSeen = new Map();
-    this.assetMap = {};
-    this.filesWritten = 0;
-    this.queueSize = 0;
-    this.queue = options.queue || new PQueue({
-      concurrency
-    });
-    this.rejectedError = null;
-
-    this.reject = err => {
-      this.rejectedError = err;
-    };
-  }
-
-  clear() {
-    this.assetsSeen.clear();
-    this.queue.clear();
-    this.queueSize = 0;
-  }
-
-  finish() {
-    return new Promise((resolve, reject) => {
-      if (this.rejectedError) {
-        reject(this.rejectedError);
-        return;
-      }
-
-      this.reject = reject;
-      this.queue.onIdle().then(() => resolve(this.assetMap));
-    });
-  } // Called when we want to download all assets to local filesystem and rewrite documents to hold
-  // placeholder asset references (_sanityAsset: 'image@file:///local/path')
-
-
-  queueAssetDownload(assetDoc, dstPath, type) {
-    if (!assetDoc.url) {
-      debug('Asset document "%s" does not have a URL property, skipping', assetDoc._id);
-      return;
-    }
-
-    debug('Adding download task for %s (destination: %s)', assetDoc._id, dstPath);
-    this.queueSize++;
-    this.downloading.push(assetDoc.url);
-    this.queue.add(() => this.downloadAsset(assetDoc, dstPath));
-  }
-
-  maybeCreateAssetDirs() {
-    if (this.assetDirsCreated) {
-      return;
-    }
-    /* eslint-disable no-sync */
-
-
-    mkdirSync(path.join(this.tmpDir, 'files'), {
-      recursive: true
-    });
-    mkdirSync(path.join(this.tmpDir, 'images'), {
-      recursive: true
-    });
-    /* eslint-enable no-sync */
-
-    this.assetDirsCreated = true;
-  }
-
-  getAssetRequestOptions(assetDoc) {
-    const token = this.client.config().token;
-    const headers = {
-      'User-Agent': "".concat(pkg.name, "@").concat(pkg.version)
-    };
-    const isImage = assetDoc._type === 'sanity.imageAsset';
-    const url = parseUrl(assetDoc.url, true);
-
-    if (isImage && ['cdn.sanity.io', 'cdn.sanity.work'].includes(url.hostname)) {
-      headers.Authorization = "Bearer ".concat(token);
-      url.query = { ...(url.query || {}),
-        dlRaw: 'true'
-      };
-    }
-
-    return {
-      url: formatUrl(url),
-      headers
-    };
-  }
-
-  async downloadAsset(assetDoc, dstPath) {
-    let attemptNum = arguments.length > 2 && arguments[2] !== undefined ? arguments[2] : 0;
-    const {
-      url
-    } = assetDoc;
-    const options = this.getAssetRequestOptions(assetDoc);
-    let stream;
-
-    try {
-      stream = await requestStream(options);
-    } catch (err) {
-      this.reject(err);
-      return false;
-    }
-
-    if (stream.statusCode !== 200) {
-      this.queue.clear();
-      const err = await tryGetErrorFromStream(stream);
-      let errMsg = "Referenced asset URL \"".concat(url, "\" returned HTTP ").concat(stream.statusCode);
-
-      if (err) {
-        errMsg = "".concat(errMsg, ":\n\n").concat(err);
-      }
-
-      this.reject(new Error(errMsg));
-      return false;
-    }
-
-    this.maybeCreateAssetDirs();
-    debug('Asset stream ready, writing to filesystem at %s', dstPath);
-    const tmpPath = path.join(this.tmpDir, dstPath);
-    const {
-      sha1,
-      md5,
-      size
-    } = await writeHashedStream(tmpPath, stream); // Verify it against our downloaded stream to make sure we have the same copy
-
-    const contentLength = stream.headers['content-length'];
-    const remoteSha1 = stream.headers['x-sanity-sha1'];
-    const remoteMd5 = stream.headers['x-sanity-md5'];
-    const hasHash = Boolean(remoteSha1 || remoteMd5);
-    const method = md5 ? 'md5' : 'sha1';
-    let differs = false;
-
-    if (remoteMd5 && md5) {
-      differs = remoteMd5 !== md5;
-    } else if (remoteSha1 && sha1) {
-      differs = remoteSha1 !== sha1;
-    }
-
-    if (differs && attemptNum < 3) {
-      debug('%s does not match downloaded asset, retrying (#%d) [%s]', method, attemptNum + 1, url);
-      return this.downloadAsset(assetDoc, dstPath, attemptNum + 1);
-    } else if (differs) {
-      const details = [hasHash && (method === 'md5' ? "md5 should be ".concat(remoteMd5, ", got ").concat(md5) : "sha1 should be ".concat(remoteSha1, ", got ").concat(sha1)), contentLength && parseInt(contentLength, 10) !== size && "Asset should be ".concat(contentLength, " bytes, got ").concat(size), "Did not succeed after ".concat(attemptNum, " attempts.")];
-      const detailsString = "Details:\n - ".concat(details.filter(Boolean).join('\n - '));
-      await rimraf(tmpPath);
-      this.queue.clear();
-      const error = new Error("Failed to download asset at ".concat(assetDoc.url, ", giving up. ").concat(detailsString));
-      this.reject(error);
-      return false;
-    }
-
-    const isImage = assetDoc._type === 'sanity.imageAsset';
-    const type = isImage ? 'image' : 'file';
-    const id = "".concat(type, "-").concat(sha1);
-    const metaProps = omit(assetDoc, EXCLUDE_PROPS);
-
-    if (Object.keys(metaProps).length > 0) {
-      this.assetMap[id] = metaProps;
-    }
-
-    this.downloading.splice(this.downloading.findIndex(datUrl => datUrl === url), 1);
-    this.filesWritten++;
-    return true;
-  }
-
-}
-
-function isAssetField(item) {
-  return item.asset && item.asset._ref && isSanityAsset(item.asset._ref);
-}
-
-function getAssetType(item) {
-  if (!item.asset || typeof item.asset._ref !== 'string') {
-    return null;
-  }
-
-  const [, type] = item.asset._ref.match(/^(image|file)-/) || [];
-  return type || null;
-}
-
-function isSanityAsset(assetId) {
-  return /^image-[a-f0-9]{40}-\d+x\d+-[a-z]+$/.test(assetId) || /^file-[a-f0-9]{40}-[a-z0-9]+$/.test(assetId);
-}
-
-function generateFilename(assetId) {
-  const [,, asset, ext] = assetId.match(/^(image|file)-(.*?)(-[a-z]+)?$/) || [];
-  const extension = (ext || 'bin').replace(/^-/, '');
-  return asset ? "".concat(asset, ".").concat(extension) : "".concat(assetId, ".bin");
-}
-
-function writeHashedStream(filePath, stream) {
-  let size = 0;
-  const md5 = crypto.createHash('md5');
-  const sha1 = crypto.createHash('sha1');
-  const hasher = miss.through((chunk, enc, cb) => {
-    size += chunk.length;
-    md5.update(chunk);
-    sha1.update(chunk);
-    cb(null, chunk);
-  });
-  return new Promise((resolve, reject) => miss.pipe(stream, hasher, createWriteStream(filePath), err => {
-    if (err) {
-      reject(err);
-      return;
-    }
-
-    resolve({
-      size,
-      sha1: sha1.digest('hex'),
-      md5: md5.digest('hex')
-    });
-  }));
-}
-
-function tryGetErrorFromStream(stream) {
-  return new Promise((resolve, reject) => {
-    miss.pipe(stream, miss.concat(parse), err => err ? reject(err) : noop);
-
-    function parse(body) {
-      try {
-        const parsed = JSON.parse(body.toString('utf8'));
-        resolve(parsed.message || parsed.error || null);
-      } catch (err) {
-        resolve(body.toString('utf8').slice(0, 16000));
-      }
-    }
-  });
-}
-
-module.exports = AssetHandler;

package/lib/debug.js

@@ -1,3 +0,0 @@
-"use strict";
-
-module.exports = require('debug')('sanity:export');

package/lib/export.js

@@ -1,222 +0,0 @@
-"use strict";
-
-const os = require('os');
-
-const path = require('path');
-
-const zlib = require('zlib');
-
-const fs = require('fs');
-
-const miss = require('mississippi');
-
-const split = require('split2');
-
-const archiver = require('archiver');
-
-const rimraf = require('./util/rimraf');
-
-const debug = require('./debug');
-
-const AssetHandler = require('./AssetHandler');
-
-const stringifyStream = require('./stringifyStream');
-
-const validateOptions = require('./validateOptions');
-
-const rejectOnApiError = require('./rejectOnApiError');
-
-const getDocumentsStream = require('./getDocumentsStream');
-
-const filterSystemDocuments = require('./filterSystemDocuments');
-
-const filterDocumentTypes = require('./filterDocumentTypes');
-
-const filterDrafts = require('./filterDrafts');
-
-const logFirstChunk = require('./logFirstChunk');
-
-const tryParseJson = require('./tryParseJson');
-
-const noop = () => null;
-
-function exportDataset(opts) {
-  const options = validateOptions(opts);
-  const onProgress = options.onProgress || noop;
-  const archive = archiver('tar', {
-    gzip: true,
-    gzipOptions: {
-      level: options.compress ? zlib.Z_DEFAULT_COMPRESSION : zlib.Z_NO_COMPRESSION
-    }
-  });
-  const slugDate = new Date().toISOString().replace(/[^a-z0-9]/gi, '-').toLowerCase();
-  const prefix = "".concat(opts.dataset, "-export-").concat(slugDate);
-  const tmpDir = path.join(os.tmpdir(), prefix);
-
-  const cleanup = () => rimraf(tmpDir).catch(err => {
-    debug("Error while cleaning up temporary files: ".concat(err.message));
-  });
-
-  const assetHandler = new AssetHandler({
-    client: options.client,
-    tmpDir,
-    prefix,
-    concurrency: options.assetConcurrency
-  });
-  debug('Outputting assets (temporarily) to %s', tmpDir);
-  debug('Outputting to %s', options.outputPath === '-' ? 'stdout' : options.outputPath);
-  let outputStream;
-
-  if (isWritableStream(options.outputPath)) {
-    outputStream = options.outputPath;
-  } else {
-    outputStream = options.outputPath === '-' ? process.stdout : fs.createWriteStream(options.outputPath);
-  }
-
-  let assetStreamHandler = assetHandler.noop;
-
-  if (!options.raw) {
-    assetStreamHandler = options.assets ? assetHandler.rewriteAssets : assetHandler.stripAssets;
-  }
-
-  return new Promise(async (resolve, reject) => {
-    miss.finished(archive, async archiveErr => {
-      if (archiveErr) {
-        debug('Archiving errored! %s', archiveErr.stack);
-        await cleanup();
-        reject(archiveErr);
-        return;
-      }
-
-      debug('Archive finished!');
-    });
-    debug('Getting dataset export stream');
-    onProgress({
-      step: 'Exporting documents...'
-    });
-    let documentCount = 0;
-    let lastReported = Date.now();
-
-    const reportDocumentCount = (chunk, enc, cb) => {
-      ++documentCount;
-      const now = Date.now();
-
-      if (now - lastReported > 50) {
-        onProgress({
-          step: 'Exporting documents...',
-          current: documentCount,
-          total: '?',
-          update: true
-        });
-        lastReported = now;
-      }
-
-      cb(null, chunk);
-    };
-
-    const inputStream = await getDocumentsStream(options.client, options.dataset);
-    debug('Got HTTP %d', inputStream.statusCode);
-    debug('Response headers: %o', inputStream.headers);
-    const jsonStream = miss.pipeline(inputStream, logFirstChunk(), split(tryParseJson), rejectOnApiError(), filterSystemDocuments(), assetStreamHandler, filterDocumentTypes(options.types), options.drafts ? miss.through.obj() : filterDrafts(), stringifyStream(), miss.through(reportDocumentCount));
-    miss.finished(jsonStream, async err => {
-      if (err) {
-        return;
-      }
-
-      onProgress({
-        step: 'Exporting documents...',
-        current: documentCount,
-        total: documentCount,
-        update: true
-      });
-
-      if (!options.raw && options.assets) {
-        onProgress({
-          step: 'Downloading assets...'
-        });
-      }
-
-      let prevCompleted = 0;
-      const progressInterval = setInterval(() => {
-        const completed = assetHandler.queueSize - assetHandler.queue.size - assetHandler.queue.pending;
-
-        if (prevCompleted === completed) {
-          return;
-        }
-
-        prevCompleted = completed;
-        onProgress({
-          step: 'Downloading assets...',
-          current: completed,
-          total: assetHandler.queueSize,
-          update: true
-        });
-      }, 500);
-      debug('Waiting for asset handler to complete downloads');
-
-      try {
-        const assetMap = await assetHandler.finish(); // Make sure we mark the progress as done (eg 100/100 instead of 99/100)
-
-        onProgress({
-          step: 'Downloading assets...',
-          current: assetHandler.queueSize,
-          total: assetHandler.queueSize,
-          update: true
-        });
-        archive.append(JSON.stringify(assetMap), {
-          name: 'assets.json',
-          prefix
-        });
-        clearInterval(progressInterval);
-      } catch (assetErr) {
-        clearInterval(progressInterval);
-        await cleanup();
-        reject(assetErr);
-        return;
-      } // Add all downloaded assets to archive
-
-
-      archive.directory(path.join(tmpDir, 'files'), "".concat(prefix, "/files"), {
-        store: true
-      });
-      archive.directory(path.join(tmpDir, 'images'), "".concat(prefix, "/images"), {
-        store: true
-      });
-      debug('Finalizing archive, flushing streams');
-      onProgress({
-        step: 'Adding assets to archive...'
-      });
-      archive.finalize();
-    });
-    archive.on('warning', err => {
-      debug('Archive warning: %s', err.message);
-    });
-    archive.append(jsonStream, {
-      name: 'data.ndjson',
-      prefix
-    });
-    miss.pipe(archive, outputStream, onComplete);
-
-    async function onComplete(err) {
-      onProgress({
-        step: 'Clearing temporary files...'
-      });
-      await cleanup();
-
-      if (!err) {
-        resolve();
-        return;
-      }
-
-      debug('Error during streaming: %s', err.stack);
-      assetHandler.clear();
-      reject(err);
-    }
-  });
-}
-
-function isWritableStream(val) {
-  return val !== null && typeof val === 'object' && typeof val.pipe === 'function' && typeof val._write === 'function' && typeof val._writableState === 'object';
-}
-
-module.exports = exportDataset;

package/lib/filterDocumentTypes.js

@@ -1,14 +0,0 @@
-"use strict";
-
-const miss = require('mississippi');
-
-module.exports = allowedTypes => allowedTypes ? miss.through.obj((doc, enc, callback) => {
-  const type = doc && doc._type;
-
-  if (allowedTypes.includes(type)) {
-    callback(null, doc);
-    return;
-  }
-
-  callback();
-}) : miss.through.obj();

package/lib/filterDrafts.js

@@ -1,13 +0,0 @@
-"use strict";
-
-const miss = require('mississippi');
-
-const isDraft = doc => doc && doc._id && doc._id.indexOf('drafts.') === 0;
-
-module.exports = () => miss.through.obj((doc, enc, callback) => {
-  if (isDraft(doc)) {
-    return callback();
-  }
-
-  return callback(null, doc);
-});

package/lib/filterSystemDocuments.js

@@ -1,16 +0,0 @@
-"use strict";
-
-const miss = require('mississippi');
-
-const debug = require('./debug');
-
-const isSystemDocument = doc => doc && doc._id && doc._id.indexOf('_.') === 0;
-
-module.exports = () => miss.through.obj((doc, enc, callback) => {
-  if (isSystemDocument(doc)) {
-    debug('%s is a system document, skipping', doc && doc._id);
-    return callback();
-  }
-
-  return callback(null, doc);
-});

package/lib/getDocumentsStream.js

@@ -1,22 +0,0 @@
-"use strict";
-
-const pkg = require('../package.json');
-
-const requestStream = require('./requestStream');
-
-module.exports = (client, dataset) => {
-  // Sanity client doesn't handle streams natively since we want to support node/browser
-  // with same API. We're just using it here to get hold of URLs and tokens.
-  const url = client.getUrl("/data/export/".concat(dataset));
-  const token = client.config().token;
-  const headers = {
-    'User-Agent': "".concat(pkg.name, "@").concat(pkg.version),
-    ...(token ? {
-      Authorization: "Bearer ".concat(token)
-    } : {})
-  };
-  return requestStream({
-    url,
-    headers
-  });
-};

package/lib/logFirstChunk.js

@@ -1,18 +0,0 @@
-"use strict";
-
-const miss = require('mississippi');
-
-const debug = require('./debug');
-
-module.exports = () => {
-  let firstChunk = true;
-  return miss.through((chunk, enc, callback) => {
-    if (firstChunk) {
-      const string = chunk.toString('utf8').split('\n')[0];
-      debug('First chunk received: %s', string.slice(0, 300));
-      firstChunk = false;
-    }
-
-    callback(null, chunk);
-  });
-};

package/lib/rejectOnApiError.js

@@ -1,17 +0,0 @@
-"use strict";
-
-const miss = require('mississippi');
-
-module.exports = () => miss.through.obj((doc, enc, callback) => {
-  if (doc.error && doc.statusCode) {
-    callback(new Error([doc.statusCode, doc.error].join(': ')));
-    return;
-  }
-
-  if (!doc._id && doc.error) {
-    callback(new Error(doc.error.description || doc.error.message || JSON.stringify(doc)));
-    return;
-  }
-
-  callback(null, doc);
-});

package/lib/requestStream.js

@@ -1,63 +0,0 @@
-"use strict";
-
-const getIt = require('get-it');
-
-const {
-  keepAlive,
-  promise
-} = require('get-it/middleware');
-
-const debug = require('./debug');
-
-const request = getIt([keepAlive(), promise({
-  onlyBody: true
-})]);
-const socketsWithTimeout = new WeakSet();
-const CONNECTION_TIMEOUT = 15 * 1000; // 15 seconds
-
-const READ_TIMEOUT = 3 * 60 * 1000; // 3 minutes
-
-const MAX_RETRIES = 5;
-
-function delay(ms) {
-  return new Promise(resolve => setTimeout(resolve, ms));
-}
-/* eslint-disable no-await-in-loop, max-depth */
-
-
-module.exports = async options => {
-  let error;
-
-  for (let i = 0; i < MAX_RETRIES; i++) {
-    try {
-      const response = await request({ ...options,
-        stream: true,
-        maxRedirects: 0,
-        timeout: {
-          connect: CONNECTION_TIMEOUT,
-          socket: READ_TIMEOUT
-        }
-      });
-
-      if (response.connection && typeof response.connection.setTimeout === 'function' && !socketsWithTimeout.has(response.connection)) {
-        socketsWithTimeout.add(response.connection);
-        response.connection.setTimeout(READ_TIMEOUT, () => {
-          response.destroy(new Error("Read timeout: No data received on socket for ".concat(READ_TIMEOUT, " ms")));
-        });
-      }
-
-      return response;
-    } catch (err) {
-      error = err;
-
-      if (err.response && err.response.statusCode && err.response.statusCode < 500) {
-        break;
-      }
-
-      debug('Error, retrying after 1500ms: %s', err.message);
-      await delay(1500);
-    }
-  }
-
-  throw error;
-};

package/lib/stringifyStream.js

@@ -1,5 +0,0 @@
-"use strict";
-
-const miss = require('mississippi');
-
-module.exports = () => miss.through.obj((doc, enc, callback) => callback(null, "".concat(JSON.stringify(doc), "\n")));

package/lib/tryParseJson.js

@@ -1,25 +0,0 @@
-"use strict";
-
-module.exports = line => {
-  try {
-    return JSON.parse(line);
-  } catch (err) {
-    // Catch half-done lines with an error at the end
-    const errorPosition = line.lastIndexOf('{"error":');
-
-    if (errorPosition === -1) {
-      err.message = "".concat(err.message, " (").concat(line, ")");
-      throw err;
-    }
-
-    const errorJson = line.slice(errorPosition);
-    const errorLine = JSON.parse(errorJson);
-    const error = errorLine && errorLine.error;
-
-    if (error && error.description) {
-      throw new Error("Error streaming dataset: ".concat(error.description, "\n\n").concat(errorJson, "\n"));
-    }
-
-    throw err;
-  }
-};

package/lib/util/rimraf.js

@@ -1,9 +0,0 @@
-"use strict";
-
-const {
-  promisify
-} = require('util');
-
-const rimrafCb = require('rimraf');
-
-module.exports = promisify(rimrafCb);

package/lib/validateOptions.js

@@ -1,58 +0,0 @@
-"use strict";
-
-const defaults = require('lodash/defaults');
-
-const clientMethods = ['getUrl', 'config'];
-const booleanFlags = ['assets', 'raw', 'compress', 'drafts'];
-const exportDefaults = {
-  compress: true,
-  drafts: true,
-  assets: true,
-  raw: false
-};
-
-function validateOptions(opts) {
-  const options = defaults({}, opts, exportDefaults);
-
-  if (typeof options.dataset !== 'string' || options.dataset.length < 1) {
-    throw new Error("options.dataset must be a valid dataset name");
-  }
-
-  if (options.onProgress && typeof options.onProgress !== 'function') {
-    throw new Error("options.onProgress must be a function");
-  }
-
-  if (!options.client) {
-    throw new Error('`options.client` must be set to an instance of @sanity/client');
-  }
-
-  const missing = clientMethods.find(key => typeof options.client[key] !== 'function');
-
-  if (missing) {
-    throw new Error("`options.client` is not a valid @sanity/client instance - no \"".concat(missing, "\" method found"));
-  }
-
-  const clientConfig = options.client.config();
-
-  if (!clientConfig.token) {
-    throw new Error('Client is not instantiated with a `token`');
-  }
-
-  booleanFlags.forEach(flag => {
-    if (typeof options[flag] !== 'boolean') {
-      throw new Error("Flag ".concat(flag, " must be a boolean (true/false)"));
-    }
-  });
-
-  if (!options.outputPath) {
-    throw new Error('outputPath must be specified (- for stdout)');
-  }
-
-  if (options.assetConcurrency && (options.assetConcurrency < 1 || options.assetConcurrency > 24)) {
-    throw new Error('`assetConcurrency` must be between 1 and 24');
-  }
-
-  return options;
-}
-
-module.exports = validateOptions;