@sanity/export 2.21.10-shopify.5 → 2.21.12-purple-unicorn.1302

package/lib/export.js

@@ -1,65 +1,61 @@
 "use strict";
 
-function asyncGeneratorStep(gen, resolve, reject, _next, _throw, key, arg) { try { var info = gen[key](arg); var value = info.value; } catch (error) { reject(error); return; } if (info.done) { resolve(value); } else { Promise.resolve(value).then(_next, _throw); } }
+const os = require('os');
 
-function _asyncToGenerator(fn) { return function () { var self = this, args = arguments; return new Promise(function (resolve, reject) { var gen = fn.apply(self, args); function _next(value) { asyncGeneratorStep(gen, resolve, reject, _next, _throw, "next", value); } function _throw(err) { asyncGeneratorStep(gen, resolve, reject, _next, _throw, "throw", err); } _next(undefined); }); }; }
+const path = require('path');
 
-var os = require('os');
+const zlib = require('zlib');
 
-var path = require('path');
+const fse = require('fs-extra');
 
-var zlib = require('zlib');
+const miss = require('mississippi');
 
-var fse = require('fs-extra');
+const split = require('split2');
 
-var miss = require('mississippi');
+const archiver = require('archiver');
 
-var split = require('split2');
+const debug = require('./debug');
 
-var archiver = require('archiver');
+const AssetHandler = require('./AssetHandler');
 
-var debug = require('./debug');
+const stringifyStream = require('./stringifyStream');
 
-var AssetHandler = require('./AssetHandler');
+const validateOptions = require('./validateOptions');
 
-var stringifyStream = require('./stringifyStream');
+const rejectOnApiError = require('./rejectOnApiError');
 
-var validateOptions = require('./validateOptions');
+const getDocumentsStream = require('./getDocumentsStream');
 
-var rejectOnApiError = require('./rejectOnApiError');
+const filterSystemDocuments = require('./filterSystemDocuments');
 
-var getDocumentsStream = require('./getDocumentsStream');
+const filterDocumentTypes = require('./filterDocumentTypes');
 
-var filterSystemDocuments = require('./filterSystemDocuments');
+const filterDrafts = require('./filterDrafts');
 
-var filterDocumentTypes = require('./filterDocumentTypes');
+const logFirstChunk = require('./logFirstChunk');
 
-var filterDrafts = require('./filterDrafts');
+const tryParseJson = require('./tryParseJson');
 
-var logFirstChunk = require('./logFirstChunk');
-
-var tryParseJson = require('./tryParseJson');
-
-var noop = () => null;
+const noop = () => null;
 
 function exportDataset(opts) {
-  var options = validateOptions(opts);
-  var onProgress = options.onProgress || noop;
-  var archive = archiver('tar', {
+  const options = validateOptions(opts);
+  const onProgress = options.onProgress || noop;
+  const archive = archiver('tar', {
     gzip: true,
     gzipOptions: {
       level: options.compress ? zlib.Z_DEFAULT_COMPRESSION : zlib.Z_NO_COMPRESSION
     }
   });
-  var slugDate = new Date().toISOString().replace(/[^a-z0-9]/gi, '-').toLowerCase();
-  var prefix = "".concat(opts.dataset, "-export-").concat(slugDate);
-  var tmpDir = path.join(os.tmpdir(), prefix);
+  const slugDate = new Date().toISOString().replace(/[^a-z0-9]/gi, '-').toLowerCase();
+  const prefix = "".concat(opts.dataset, "-export-").concat(slugDate);
+  const tmpDir = path.join(os.tmpdir(), prefix);
 
-  var cleanup = () => fse.remove(tmpDir).catch(err => {
+  const cleanup = () => fse.remove(tmpDir).catch(err => {
     debug("Error while cleaning up temporary files: ".concat(err.message));
   });
 
-  var assetHandler = new AssetHandler({
+  const assetHandler = new AssetHandler({
     client: options.client,
     tmpDir,
     prefix,
@@ -67,172 +63,158 @@ function exportDataset(opts) {
   });
   debug('Outputting assets (temporarily) to %s', tmpDir);
   debug('Outputting to %s', options.outputPath === '-' ? 'stdout' : options.outputPath);
-  var outputStream = options.outputPath === '-' ? process.stdout : fse.createWriteStream(options.outputPath);
-  var assetStreamHandler = assetHandler.noop;
+  let outputStream;
+
+  if (isWritableStream(options.outputPath)) {
+    outputStream = options.outputPath;
+  } else {
+    outputStream = options.outputPath === '-' ? process.stdout : fse.createWriteStream(options.outputPath);
+  }
+
+  let assetStreamHandler = assetHandler.noop;
 
   if (!options.raw) {
     assetStreamHandler = options.assets ? assetHandler.rewriteAssets : assetHandler.stripAssets;
   }
 
-  return new Promise( /*#__PURE__*/function () {
-    var _ref = _asyncToGenerator(function* (resolve, reject) {
-      miss.finished(archive, /*#__PURE__*/function () {
-        var _ref2 = _asyncToGenerator(function* (archiveErr) {
-          if (archiveErr) {
-            debug('Archiving errored! %s', archiveErr.stack);
-            yield cleanup();
-            reject(archiveErr);
-            return;
-          }
-
-          debug('Archive finished!');
+  return new Promise(async (resolve, reject) => {
+    miss.finished(archive, async archiveErr => {
+      if (archiveErr) {
+        debug('Archiving errored! %s', archiveErr.stack);
+        await cleanup();
+        reject(archiveErr);
+        return;
+      }
+
+      debug('Archive finished!');
+    });
+    debug('Getting dataset export stream');
+    onProgress({
+      step: 'Exporting documents...'
+    });
+    let documentCount = 0;
+    let lastReported = Date.now();
+
+    const reportDocumentCount = (chunk, enc, cb) => {
+      ++documentCount;
+      const now = Date.now();
+
+      if (now - lastReported > 50) {
+        onProgress({
+          step: 'Exporting documents...',
+          current: documentCount,
+          total: '?',
+          update: true
         });
+        lastReported = now;
+      }
+
+      cb(null, chunk);
+    };
+
+    const inputStream = await getDocumentsStream(options.client, options.dataset);
+    debug('Got HTTP %d', inputStream.statusCode);
+    debug('Response headers: %o', inputStream.headers);
+    const jsonStream = miss.pipeline(inputStream, logFirstChunk(), split(tryParseJson), rejectOnApiError(), filterSystemDocuments(), assetStreamHandler, filterDocumentTypes(options.types), options.drafts ? miss.through.obj() : filterDrafts(), stringifyStream(), miss.through(reportDocumentCount));
+    miss.finished(jsonStream, async err => {
+      if (err) {
+        return;
+      }
 
-        return function (_x3) {
-          return _ref2.apply(this, arguments);
-        };
-      }());
-      debug('Getting dataset export stream');
       onProgress({
-        step: 'Exporting documents...'
+        step: 'Exporting documents...',
+        current: documentCount,
+        total: documentCount,
+        update: true
       });
-      var documentCount = 0;
-      var lastReported = Date.now();
-
-      var reportDocumentCount = (chunk, enc, cb) => {
-        ++documentCount;
-        var now = Date.now();
-
-        if (now - lastReported > 50) {
-          onProgress({
-            step: 'Exporting documents...',
-            current: documentCount,
-            total: '?',
-            update: true
-          });
-          lastReported = now;
+
+      if (!options.raw && options.assets) {
+        onProgress({
+          step: 'Downloading assets...'
+        });
+      }
+
+      let prevCompleted = 0;
+      const progressInterval = setInterval(() => {
+        const completed = assetHandler.queueSize - assetHandler.queue.size - assetHandler.queue.pending;
+
+        if (prevCompleted === completed) {
+          return;
         }
 
-        cb(null, chunk);
-      };
-
-      var inputStream = yield getDocumentsStream(options.client, options.dataset);
-      debug('Got HTTP %d', inputStream.statusCode);
-      debug('Response headers: %o', inputStream.headers);
-      var jsonStream = miss.pipeline(inputStream, logFirstChunk(), split(tryParseJson), rejectOnApiError(), filterSystemDocuments(), assetStreamHandler, filterDocumentTypes(options.types), options.drafts ? miss.through.obj() : filterDrafts(), stringifyStream(), miss.through(reportDocumentCount));
-      miss.finished(jsonStream, /*#__PURE__*/function () {
-        var _ref3 = _asyncToGenerator(function* (err) {
-          if (err) {
-            return;
-          }
-
-          onProgress({
-            step: 'Exporting documents...',
-            current: documentCount,
-            total: documentCount,
-            update: true
-          });
-
-          if (!options.raw && options.assets) {
-            onProgress({
-              step: 'Downloading assets...'
-            });
-          }
-
-          var prevCompleted = 0;
-          var progressInterval = setInterval(() => {
-            var completed = assetHandler.queueSize - assetHandler.queue.size - assetHandler.queue.pending;
-
-            if (prevCompleted === completed) {
-              return;
-            }
-
-            prevCompleted = completed;
-            onProgress({
-              step: 'Downloading assets...',
-              current: completed,
-              total: assetHandler.queueSize,
-              update: true
-            });
-          }, 500);
-          debug('Waiting for asset handler to complete downloads');
-
-          try {
-            var assetMap = yield assetHandler.finish(); // Make sure we mark the progress as done (eg 100/100 instead of 99/100)
-
-            onProgress({
-              step: 'Downloading assets...',
-              current: assetHandler.queueSize,
-              total: assetHandler.queueSize,
-              update: true
-            });
-            archive.append(JSON.stringify(assetMap), {
-              name: 'assets.json',
-              prefix
-            });
-            clearInterval(progressInterval);
-          } catch (assetErr) {
-            clearInterval(progressInterval);
-            yield cleanup();
-            reject(assetErr);
-            return;
-          } // Add all downloaded assets to archive
-
-
-          archive.directory(path.join(tmpDir, 'files'), "".concat(prefix, "/files"), {
-            store: true
-          });
-          archive.directory(path.join(tmpDir, 'images'), "".concat(prefix, "/images"), {
-            store: true
-          });
-          debug('Finalizing archive, flushing streams');
-          onProgress({
-            step: 'Adding assets to archive...'
-          });
-          archive.finalize();
+        prevCompleted = completed;
+        onProgress({
+          step: 'Downloading assets...',
+          current: completed,
+          total: assetHandler.queueSize,
+          update: true
         });
+      }, 500);
+      debug('Waiting for asset handler to complete downloads');
+
+      try {
+        const assetMap = await assetHandler.finish(); // Make sure we mark the progress as done (eg 100/100 instead of 99/100)
 
-        return function (_x4) {
-          return _ref3.apply(this, arguments);
-        };
-      }());
-      archive.on('warning', err => {
-        debug('Archive warning: %s', err.message);
+        onProgress({
+          step: 'Downloading assets...',
+          current: assetHandler.queueSize,
+          total: assetHandler.queueSize,
+          update: true
+        });
+        archive.append(JSON.stringify(assetMap), {
+          name: 'assets.json',
+          prefix
+        });
+        clearInterval(progressInterval);
+      } catch (assetErr) {
+        clearInterval(progressInterval);
+        await cleanup();
+        reject(assetErr);
+        return;
+      } // Add all downloaded assets to archive
+
+
+      archive.directory(path.join(tmpDir, 'files'), "".concat(prefix, "/files"), {
+        store: true
+      });
+      archive.directory(path.join(tmpDir, 'images'), "".concat(prefix, "/images"), {
+        store: true
       });
-      archive.append(jsonStream, {
-        name: 'data.ndjson',
-        prefix
+      debug('Finalizing archive, flushing streams');
+      onProgress({
+        step: 'Adding assets to archive...'
       });
-      miss.pipe(archive, outputStream, onComplete);
+      archive.finalize();
+    });
+    archive.on('warning', err => {
+      debug('Archive warning: %s', err.message);
+    });
+    archive.append(jsonStream, {
+      name: 'data.ndjson',
+      prefix
+    });
+    miss.pipe(archive, outputStream, onComplete);
 
-      function onComplete(_x5) {
-        return _onComplete.apply(this, arguments);
-      }
+    async function onComplete(err) {
+      onProgress({
+        step: 'Clearing temporary files...'
+      });
+      await cleanup();
 
-      function _onComplete() {
-        _onComplete = _asyncToGenerator(function* (err) {
-          onProgress({
-            step: 'Clearing temporary files...'
-          });
-          yield cleanup();
-
-          if (!err) {
-            resolve();
-            return;
-          }
-
-          debug('Error during streaming: %s', err.stack);
-          assetHandler.clear();
-          reject(err);
-        });
-        return _onComplete.apply(this, arguments);
+      if (!err) {
+        resolve();
+        return;
       }
-    });
 
-    return function (_x, _x2) {
-      return _ref.apply(this, arguments);
-    };
-  }());
+      debug('Error during streaming: %s', err.stack);
+      assetHandler.clear();
+      reject(err);
+    }
+  });
+}
+
+function isWritableStream(val) {
+  return val !== null && typeof val === 'object' && typeof val.pipe === 'function' && typeof val._write === 'function' && typeof val._writableState === 'object';
 }
 
 module.exports = exportDataset;

package/lib/filterDocumentTypes.js

@@ -1,9 +1,9 @@
 "use strict";
 
-var miss = require('mississippi');
+const miss = require('mississippi');
 
 module.exports = allowedTypes => allowedTypes ? miss.through.obj((doc, enc, callback) => {
-  var type = doc && doc._type;
+  const type = doc && doc._type;
 
   if (allowedTypes.includes(type)) {
     callback(null, doc);

package/lib/filterDrafts.js

@@ -1,8 +1,8 @@
 "use strict";
 
-var miss = require('mississippi');
+const miss = require('mississippi');
 
-var isDraft = doc => doc && doc._id && doc._id.indexOf('drafts.') === 0;
+const isDraft = doc => doc && doc._id && doc._id.indexOf('drafts.') === 0;
 
 module.exports = () => miss.through.obj((doc, enc, callback) => {
   if (isDraft(doc)) {

package/lib/filterSystemDocuments.js

@@ -1,10 +1,10 @@
 "use strict";
 
-var miss = require('mississippi');
+const miss = require('mississippi');
 
-var debug = require('./debug');
+const debug = require('./debug');
 
-var isSystemDocument = doc => doc && doc._id && doc._id.indexOf('_.') === 0;
+const isSystemDocument = doc => doc && doc._id && doc._id.indexOf('_.') === 0;
 
 module.exports = () => miss.through.obj((doc, enc, callback) => {
   if (isSystemDocument(doc)) {

package/lib/getDocumentsStream.js

@@ -1,27 +1,20 @@
 "use strict";
 
-function ownKeys(object, enumerableOnly) { var keys = Object.keys(object); if (Object.getOwnPropertySymbols) { var symbols = Object.getOwnPropertySymbols(object); if (enumerableOnly) { symbols = symbols.filter(function (sym) { return Object.getOwnPropertyDescriptor(object, sym).enumerable; }); } keys.push.apply(keys, symbols); } return keys; }
+const pkg = require('../package.json');
 
-function _objectSpread(target) { for (var i = 1; i < arguments.length; i++) { var source = arguments[i] != null ? arguments[i] : {}; if (i % 2) { ownKeys(Object(source), true).forEach(function (key) { _defineProperty(target, key, source[key]); }); } else if (Object.getOwnPropertyDescriptors) { Object.defineProperties(target, Object.getOwnPropertyDescriptors(source)); } else { ownKeys(Object(source)).forEach(function (key) { Object.defineProperty(target, key, Object.getOwnPropertyDescriptor(source, key)); }); } } return target; }
-
-function _defineProperty(obj, key, value) { if (key in obj) { Object.defineProperty(obj, key, { value: value, enumerable: true, configurable: true, writable: true }); } else { obj[key] = value; } return obj; }
-
-var pkg = require('../package.json');
-
-var requestStream = require('./requestStream');
+const requestStream = require('./requestStream');
 
 module.exports = (client, dataset) => {
   // Sanity client doesn't handle streams natively since we want to support node/browser
   // with same API. We're just using it here to get hold of URLs and tokens.
-  var url = client.getUrl("/data/export/".concat(dataset));
-  var token = client.config().token;
-
-  var headers = _objectSpread({
-    'User-Agent': "".concat(pkg.name, "@").concat(pkg.version)
-  }, token ? {
-    Authorization: "Bearer ".concat(token)
-  } : {});
-
+  const url = client.getUrl("/data/export/".concat(dataset));
+  const token = client.config().token;
+  const headers = {
+    'User-Agent': "".concat(pkg.name, "@").concat(pkg.version),
+    ...(token ? {
+      Authorization: "Bearer ".concat(token)
+    } : {})
+  };
   return requestStream({
     url,
     headers

package/lib/logFirstChunk.js

@@ -1,14 +1,14 @@
 "use strict";
 
-var miss = require('mississippi');
+const miss = require('mississippi');
 
-var debug = require('./debug');
+const debug = require('./debug');
 
 module.exports = () => {
-  var firstChunk = true;
+  let firstChunk = true;
   return miss.through((chunk, enc, callback) => {
     if (firstChunk) {
-      var string = chunk.toString('utf8').split('\n')[0];
+      const string = chunk.toString('utf8').split('\n')[0];
       debug('First chunk received: %s', string.slice(0, 300));
       firstChunk = false;
     }

package/lib/rejectOnApiError.js

@@ -1,6 +1,6 @@
 "use strict";
 
-var miss = require('mississippi');
+const miss = require('mississippi');
 
 module.exports = () => miss.through.obj((doc, enc, callback) => {
   if (doc.error && doc.statusCode) {

package/lib/requestStream.js

@@ -1,54 +1,23 @@
 "use strict";
 
-function ownKeys(object, enumerableOnly) { var keys = Object.keys(object); if (Object.getOwnPropertySymbols) { var symbols = Object.getOwnPropertySymbols(object); if (enumerableOnly) { symbols = symbols.filter(function (sym) { return Object.getOwnPropertyDescriptor(object, sym).enumerable; }); } keys.push.apply(keys, symbols); } return keys; }
+const getIt = require('get-it');
 
-function _objectSpread(target) { for (var i = 1; i < arguments.length; i++) { var source = arguments[i] != null ? arguments[i] : {}; if (i % 2) { ownKeys(Object(source), true).forEach(function (key) { _defineProperty(target, key, source[key]); }); } else if (Object.getOwnPropertyDescriptors) { Object.defineProperties(target, Object.getOwnPropertyDescriptors(source)); } else { ownKeys(Object(source)).forEach(function (key) { Object.defineProperty(target, key, Object.getOwnPropertyDescriptor(source, key)); }); } } return target; }
+const {
+  keepAlive,
+  promise
+} = require('get-it/middleware');
 
-function _defineProperty(obj, key, value) { if (key in obj) { Object.defineProperty(obj, key, { value: value, enumerable: true, configurable: true, writable: true }); } else { obj[key] = value; } return obj; }
+const debug = require('./debug');
 
-function asyncGeneratorStep(gen, resolve, reject, _next, _throw, key, arg) { try { var info = gen[key](arg); var value = info.value; } catch (error) { reject(error); return; } if (info.done) { resolve(value); } else { Promise.resolve(value).then(_next, _throw); } }
+const request = getIt([keepAlive(), promise({
+  onlyBody: true
+})]);
+const socketsWithTimeout = new WeakSet();
+const CONNECTION_TIMEOUT = 15 * 1000; // 15 seconds
 
-function _asyncToGenerator(fn) { return function () { var self = this, args = arguments; return new Promise(function (resolve, reject) { var gen = fn.apply(self, args); function _next(value) { asyncGeneratorStep(gen, resolve, reject, _next, _throw, "next", value); } function _throw(err) { asyncGeneratorStep(gen, resolve, reject, _next, _throw, "throw", err); } _next(undefined); }); }; }
+const READ_TIMEOUT = 3 * 60 * 1000; // 3 minutes
 
-var simpleGet = require('simple-get');
-
-var HttpAgent = require('agentkeepalive');
-
-var debug = require('./debug');
-
-var HttpsAgent = HttpAgent.HttpsAgent;
-var httpAgent = new HttpAgent();
-var httpsAgent = new HttpsAgent();
-var socketsWithTimeout = new WeakSet();
-var CONNECTION_TIMEOUT = 15 * 1000; // 15 seconds
-
-var READ_TIMEOUT = 3 * 60 * 1000; // 3 minutes
-
-var MAX_RETRIES = 5; // Just a promisified simpleGet
-
-function getStream(options) {
-  return new Promise((resolve, reject) => {
-    var rejected = false;
-    var openTimeout = setTimeout(() => {
-      rejected = true;
-      reject(new Error("Connection timed out after ".concat(CONNECTION_TIMEOUT, " ms")));
-    }, CONNECTION_TIMEOUT);
-    simpleGet(options, (err, res) => {
-      clearTimeout(openTimeout);
-
-      if (rejected) {
-        return;
-      }
-
-      if (err) {
-        reject(err);
-        return;
-      }
-
-      resolve(res);
-    });
-  });
-}
+const MAX_RETRIES = 5;
 
 function delay(ms) {
   return new Promise(resolve => setTimeout(resolve, ms));
@@ -56,51 +25,39 @@ function delay(ms) {
 /* eslint-disable no-await-in-loop, max-depth */
 
 
-module.exports = /*#__PURE__*/function () {
-  var _ref = _asyncToGenerator(function* (options) {
-    var agent = options.url.startsWith('https:') ? httpsAgent : httpAgent;
-
-    var reqOptions = _objectSpread(_objectSpread({}, options), {}, {
-      followRedirects: false,
-      agent
-    });
-
-    var error;
-
-    for (var i = 0; i < MAX_RETRIES; i++) {
-      try {
-        var _ret = yield* function* () {
-          var response = yield getStream(reqOptions);
-
-          if (response.connection && typeof response.connection.setTimeout === 'function' && !socketsWithTimeout.has(response.connection)) {
-            socketsWithTimeout.add(response.connection);
-            response.connection.setTimeout(READ_TIMEOUT, () => {
-              response.destroy(new Error("Read timeout: No data received on socket for ".concat(READ_TIMEOUT, " ms")));
-            });
-          }
+module.exports = async options => {
+  let error;
 
-          return {
-            v: response
-          };
-        }();
+  for (let i = 0; i < MAX_RETRIES; i++) {
+    try {
+      const response = await request({ ...options,
+        stream: true,
+        maxRedirects: 0,
+        timeout: {
+          connect: CONNECTION_TIMEOUT,
+          socket: READ_TIMEOUT
+        }
+      });
 
-        if (typeof _ret === "object") return _ret.v;
-      } catch (err) {
-        error = err;
+      if (response.connection && typeof response.connection.setTimeout === 'function' && !socketsWithTimeout.has(response.connection)) {
+        socketsWithTimeout.add(response.connection);
+        response.connection.setTimeout(READ_TIMEOUT, () => {
+          response.destroy(new Error("Read timeout: No data received on socket for ".concat(READ_TIMEOUT, " ms")));
+        });
+      }
 
-        if (err.statusCode && err.statusCode < 500) {
-          break;
-        }
+      return response;
+    } catch (err) {
+      error = err;
 
-        debug('Error, retrying after 1500ms: %s', err.message);
-        yield delay(1500);
+      if (err.response && err.response.statusCode && err.response.statusCode < 500) {
+        break;
       }
-    }
 
-    throw error;
-  });
+      debug('Error, retrying after 1500ms: %s', err.message);
+      await delay(1500);
+    }
+  }
 
-  return function (_x) {
-    return _ref.apply(this, arguments);
-  };
-}();
+  throw error;
+};

package/lib/stringifyStream.js

@@ -1,5 +1,5 @@
 "use strict";
 
-var miss = require('mississippi');
+const miss = require('mississippi');
 
 module.exports = () => miss.through.obj((doc, enc, callback) => callback(null, "".concat(JSON.stringify(doc), "\n")));