@sanity/export 0.136.3-gql-rtb.375 → 0.136.3-purple-unicorn-patch.5629

package/LICENSE

@@ -1,6 +1,6 @@
 MIT License
 
-Copyright (c) 2017 Sanity
+Copyright (c) 2016 - 2022 Sanity.io
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

package/README.md

@@ -20,8 +20,8 @@ exportDataset({
   // Name of dataset to export
   dataset: 'myDataset',
 
-  // Path to write zip-file to, or `-` for stdout
-  outputPath: '/home/your-user/myDataset.zip',
+  // Path to write tar.gz-archive file to, or `-` for stdout
+  outputPath: '/home/your-user/myDataset.tar.gz',
 
   // Whether or not to export assets. Note that this operation is currently slightly lossy;
   // metadata stored on the asset document itself (original filename, for instance) might be lost
@@ -38,14 +38,17 @@ exportDataset({
 
   // Export only given document types (`_type`)
   // Optional, default: all types
-  types: ['products', 'shops']
+  types: ['products', 'shops'],
+
+  // Run 12 concurrent asset downloads
+  assetConcurrency: 12
 })
 ```
 
 ## Future improvements
 
-* Restore original filenames, keep track of duplicates, increase counter (`filename (<num>).ext`)
-* Skip archiving on raw/no-asset mode?
+- Restore original filenames, keep track of duplicates, increase counter (`filename (<num>).ext`)
+- Skip archiving on raw/no-asset mode?
 
 ## CLI-tool
 

package/lib/AssetHandler.js

@@ -1,37 +1,26 @@
 "use strict";
 
-function _slicedToArray(arr, i) { return _arrayWithHoles(arr) || _iterableToArrayLimit(arr, i) || _nonIterableRest(); }
-
-function _nonIterableRest() { throw new TypeError("Invalid attempt to destructure non-iterable instance"); }
-
-function _iterableToArrayLimit(arr, i) { var _arr = []; var _n = true; var _d = false; var _e = undefined; try { for (var _i = arr[Symbol.iterator](), _s; !(_n = (_s = _i.next()).done); _n = true) { _arr.push(_s.value); if (i && _arr.length === i) break; } } catch (err) { _d = true; _e = err; } finally { try { if (!_n && _i["return"] != null) _i["return"](); } finally { if (_d) throw _e; } } return _arr; }
-
-function _arrayWithHoles(arr) { if (Array.isArray(arr)) return arr; }
-
-function _objectSpread(target) { for (var i = 1; i < arguments.length; i++) { var source = arguments[i] != null ? arguments[i] : {}; var ownKeys = Object.keys(source); if (typeof Object.getOwnPropertySymbols === 'function') { ownKeys = ownKeys.concat(Object.getOwnPropertySymbols(source).filter(function (sym) { return Object.getOwnPropertyDescriptor(source, sym).enumerable; })); } ownKeys.forEach(function (key) { _defineProperty(target, key, source[key]); }); } return target; }
-
-function _objectWithoutProperties(source, excluded) { if (source == null) return {}; var target = _objectWithoutPropertiesLoose(source, excluded); var key, i; if (Object.getOwnPropertySymbols) { var sourceSymbolKeys = Object.getOwnPropertySymbols(source); for (i = 0; i < sourceSymbolKeys.length; i++) { key = sourceSymbolKeys[i]; if (excluded.indexOf(key) >= 0) continue; if (!Object.prototype.propertyIsEnumerable.call(source, key)) continue; target[key] = source[key]; } } return target; }
-
-function _objectWithoutPropertiesLoose(source, excluded) { if (source == null) return {}; var target = {}; var sourceKeys = Object.keys(source); var key, i; for (i = 0; i < sourceKeys.length; i++) { key = sourceKeys[i]; if (excluded.indexOf(key) >= 0) continue; target[key] = source[key]; } return target; }
-
-function asyncGeneratorStep(gen, resolve, reject, _next, _throw, key, arg) { try { var info = gen[key](arg); var value = info.value; } catch (error) { reject(error); return; } if (info.done) { resolve(value); } else { Promise.resolve(value).then(_next, _throw); } }
-
-function _asyncToGenerator(fn) { return function () { var self = this, args = arguments; return new Promise(function (resolve, reject) { var gen = fn.apply(self, args); function _next(value) { asyncGeneratorStep(gen, resolve, reject, _next, _throw, "next", value); } function _throw(err) { asyncGeneratorStep(gen, resolve, reject, _next, _throw, "throw", err); } _next(undefined); }); }; }
-
 function _defineProperty(obj, key, value) { if (key in obj) { Object.defineProperty(obj, key, { value: value, enumerable: true, configurable: true, writable: true }); } else { obj[key] = value; } return obj; }
 
 const path = require('path');
 
 const crypto = require('crypto');
 
+const {
+  parse: parseUrl,
+  format: formatUrl
+} = require('url');
+
 const fse = require('fs-extra');
 
 const miss = require('mississippi');
 
 const PQueue = require('p-queue');
 
-const _require = require('lodash'),
-      omit = _require.omit;
+const {
+  omit,
+  noop
+} = require('lodash');
 
 const pkg = require('../package.json');
 
@@ -42,51 +31,31 @@ const debug = require('./debug');
 const EXCLUDE_PROPS = ['_id', '_type', 'assetId', 'extension', 'mimeType', 'path', 'url'];
 const ACTION_REMOVE = 'remove';
 const ACTION_REWRITE = 'rewrite';
+const ASSET_DOWNLOAD_CONCURRENCY = 8;
 
 class AssetHandler {
   constructor(options) {
-    var _this = this;
-
-    _defineProperty(this, "rewriteAssets", miss.through.obj(
-    /*#__PURE__*/
-    function () {
-      var _ref = _asyncToGenerator(function* (doc, enc, callback) {
-        if (['sanity.imageAsset', 'sanity.fileAsset'].includes(doc._type)) {
-          const type = doc._type === 'sanity.imageAsset' ? 'image' : 'file';
-          const filePath = `${type}s/${generateFilename(doc._id)}`;
-
-          _this.assetsSeen.set(doc._id, type);
-
-          _this.queueAssetDownload(doc, filePath, type);
-
-          callback();
-          return;
-        }
-
-        callback(null, (yield _this.findAndModify(doc, ACTION_REWRITE)));
-      });
+    _defineProperty(this, "rewriteAssets", miss.through.obj(async (doc, enc, callback) => {
+      if (['sanity.imageAsset', 'sanity.fileAsset'].includes(doc._type)) {
+        const type = doc._type === 'sanity.imageAsset' ? 'image' : 'file';
+        const filePath = "".concat(type, "s/").concat(generateFilename(doc._id));
+        this.assetsSeen.set(doc._id, type);
+        this.queueAssetDownload(doc, filePath, type);
+        callback();
+        return;
+      }
 
-      return function (_x, _x2, _x3) {
-        return _ref.apply(this, arguments);
-      };
-    }()));
-
-    _defineProperty(this, "stripAssets", miss.through.obj(
-    /*#__PURE__*/
-    function () {
-      var _ref2 = _asyncToGenerator(function* (doc, enc, callback) {
-        if (['sanity.imageAsset', 'sanity.fileAsset'].includes(doc._type)) {
-          callback();
-          return;
-        }
+      callback(null, this.findAndModify(doc, ACTION_REWRITE));
+    }));
 
-        callback(null, (yield _this.findAndModify(doc, ACTION_REMOVE)));
-      });
+    _defineProperty(this, "stripAssets", miss.through.obj(async (doc, enc, callback) => {
+      if (['sanity.imageAsset', 'sanity.fileAsset'].includes(doc._type)) {
+        callback();
+        return;
+      }
 
-      return function (_x4, _x5, _x6) {
-        return _ref2.apply(this, arguments);
-      };
-    }()));
+      callback(null, this.findAndModify(doc, ACTION_REMOVE));
+    }));
 
     _defineProperty(this, "skipAssets", miss.through.obj((doc, enc, callback) => {
       const isAsset = ['sanity.imageAsset', 'sanity.fileAsset'].includes(doc._type);
@@ -101,97 +70,69 @@ class AssetHandler {
 
     _defineProperty(this, "noop", miss.through.obj((doc, enc, callback) => callback(null, doc)));
 
-    _defineProperty(this, "findAndModify",
-    /*#__PURE__*/
-    function () {
-      var _ref3 = _asyncToGenerator(function* (item, action) {
-        if (Array.isArray(item)) {
-          const children = yield Promise.all(item.map(child => _this.findAndModify(child, action)));
-          return children.filter(Boolean);
-        }
-
-        if (!item || typeof item !== 'object') {
-          return item;
-        }
-
-        const isAsset = isAssetField(item);
-
-        if (isAsset && action === ACTION_REMOVE) {
-          return undefined;
-        }
-
-        if (isAsset && action === ACTION_REWRITE) {
-          const asset = item.asset,
-                other = _objectWithoutProperties(item, ["asset"]);
-
-          const assetId = asset._ref;
+    _defineProperty(this, "findAndModify", (item, action) => {
+      if (Array.isArray(item)) {
+        const children = item.map(child => this.findAndModify(child, action));
+        return children.filter(Boolean);
+      }
 
-          if (isModernAsset(assetId)) {
-            const assetType = getAssetType(item);
-            const filePath = `${assetType}s/${generateFilename(assetId)}`;
-            return _objectSpread({
-              _sanityAsset: `${assetType}@file://./${filePath}`
-            }, other);
-          } // Legacy asset
+      if (!item || typeof item !== 'object') {
+        return item;
+      }
 
+      const isAsset = isAssetField(item);
 
-          const type = _this.assetsSeen.get(assetId) || (yield _this.lookupAssetType(assetId));
-          const filePath = `${type}s/${generateFilename(assetId)}`;
-          return _objectSpread({
-            _sanityAsset: `${type}@file://./${filePath}`
-          }, other);
-        }
+      if (isAsset && action === ACTION_REMOVE) {
+        return undefined;
+      }
 
-        const newItem = {};
-        const keys = Object.keys(item);
+      if (isAsset && action === ACTION_REWRITE) {
+        const {
+          asset,
+          ...other
+        } = item;
+        const assetId = asset._ref;
+        const assetType = getAssetType(item);
+        const filePath = "".concat(assetType, "s/").concat(generateFilename(assetId));
+        return {
+          _sanityAsset: "".concat(assetType, "@file://./").concat(filePath),
+          ...this.findAndModify(other, action)
+        };
+      }
 
-        for (let i = 0; i < keys.length; i++) {
-          const key = keys[i];
-          const value = item[key]; // eslint-disable-next-line no-await-in-loop
+      const newItem = {};
+      const keys = Object.keys(item);
 
-          newItem[key] = yield _this.findAndModify(value, action);
+      for (let i = 0; i < keys.length; i++) {
+        const key = keys[i];
+        const value = item[key];
+        newItem[key] = this.findAndModify(value, action);
 
-          if (typeof newItem[key] === 'undefined') {
-            delete newItem[key];
-          }
+        if (typeof newItem[key] === 'undefined') {
+          delete newItem[key];
         }
+      }
 
-        return newItem;
-      });
-
-      return function (_x7, _x8) {
-        return _ref3.apply(this, arguments);
-      };
-    }());
-
-    _defineProperty(this, "lookupAssetType",
-    /*#__PURE__*/
-    function () {
-      var _ref4 = _asyncToGenerator(function* (assetId) {
-        const docType = yield _this.client.fetch('*[_id == $id][0]._type', {
-          id: assetId
-        });
-        return docType === 'sanity.imageAsset' ? 'image' : 'file';
-      });
-
-      return function (_x9) {
-        return _ref4.apply(this, arguments);
-      };
-    }());
+      return newItem;
+    });
 
+    const concurrency = options.concurrency || ASSET_DOWNLOAD_CONCURRENCY;
+    debug('Using asset download concurrency of %d', concurrency);
     this.client = options.client;
     this.tmpDir = options.tmpDir;
     this.assetDirsCreated = false;
+    this.downloading = [];
     this.assetsSeen = new Map();
     this.assetMap = {};
     this.filesWritten = 0;
     this.queueSize = 0;
     this.queue = options.queue || new PQueue({
-      concurrency: 3
+      concurrency
     });
+    this.rejectedError = null;
 
-    this.reject = () => {
-      throw new Error('Asset handler errored before `finish()` was called');
+    this.reject = err => {
+      this.rejectedError = err;
     };
   }
 
@@ -203,6 +144,11 @@ class AssetHandler {
 
   finish() {
     return new Promise((resolve, reject) => {
+      if (this.rejectedError) {
+        reject(this.rejectedError);
+        return;
+      }
+
       this.reject = reject;
       this.queue.onIdle().then(() => resolve(this.assetMap));
     });
@@ -218,58 +164,126 @@ class AssetHandler {
 
     debug('Adding download task for %s (destination: %s)', assetDoc._id, dstPath);
     this.queueSize++;
+    this.downloading.push(assetDoc.url);
     this.queue.add(() => this.downloadAsset(assetDoc, dstPath));
   }
 
-  downloadAsset(assetDoc, dstPath) {
-    var _this2 = this;
+  maybeCreateAssetDirs() {
+    if (this.assetDirsCreated) {
+      return;
+    }
+    /* eslint-disable no-sync */
+
+
+    fse.ensureDirSync(path.join(this.tmpDir, 'files'));
+    fse.ensureDirSync(path.join(this.tmpDir, 'images'));
+    /* eslint-enable no-sync */
+
+    this.assetDirsCreated = true;
+  }
+
+  getAssetRequestOptions(assetDoc) {
+    const token = this.client.config().token;
+    const headers = {
+      'User-Agent': "".concat(pkg.name, "@").concat(pkg.version)
+    };
+    const isImage = assetDoc._type === 'sanity.imageAsset';
+    const url = parseUrl(assetDoc.url, true);
 
-    return _asyncToGenerator(function* () {
-      const url = assetDoc.url;
-      const headers = {
-        'User-Agent': `${pkg.name}@${pkg.version}`
+    if (isImage && ['cdn.sanity.io', 'cdn.sanity.work'].includes(url.hostname)) {
+      headers.Authorization = "Bearer ".concat(token);
+      url.query = { ...(url.query || {}),
+        dlRaw: 'true'
       };
-      const stream = yield requestStream({
-        url,
-        headers
-      });
+    }
 
-      if (stream.statusCode !== 200) {
-        _this2.queue.clear();
+    return {
+      url: formatUrl(url),
+      headers
+    };
+  }
 
-        _this2.reject(new Error(`Referenced asset URL "${url}" returned HTTP ${stream.statusCode}`));
+  async downloadAsset(assetDoc, dstPath) {
+    let attemptNum = arguments.length > 2 && arguments[2] !== undefined ? arguments[2] : 0;
+    const {
+      url
+    } = assetDoc;
+    const options = this.getAssetRequestOptions(assetDoc);
+    let stream;
+
+    try {
+      stream = await requestStream(options);
+    } catch (err) {
+      this.reject(err);
+      return false;
+    }
 
-        return;
+    if (stream.statusCode !== 200) {
+      this.queue.clear();
+      const err = await tryGetErrorFromStream(stream);
+      let errMsg = "Referenced asset URL \"".concat(url, "\" returned HTTP ").concat(stream.statusCode);
+
+      if (err) {
+        errMsg = "".concat(errMsg, ":\n\n").concat(err);
       }
 
-      if (!_this2.assetDirsCreated) {
-        /* eslint-disable no-sync */
-        fse.ensureDirSync(path.join(_this2.tmpDir, 'files'));
-        fse.ensureDirSync(path.join(_this2.tmpDir, 'images'));
-        /* eslint-enable no-sync */
+      this.reject(new Error(errMsg));
+      return false;
+    }
 
-        _this2.assetDirsCreated = true;
-      }
+    this.maybeCreateAssetDirs();
+    debug('Asset stream ready, writing to filesystem at %s', dstPath);
+    const tmpPath = path.join(this.tmpDir, dstPath);
+    const {
+      sha1,
+      md5,
+      size
+    } = await writeHashedStream(tmpPath, stream); // Verify it against our downloaded stream to make sure we have the same copy
+
+    const contentLength = stream.headers['content-length'];
+    const remoteSha1 = stream.headers['x-sanity-sha1'];
+    const remoteMd5 = stream.headers['x-sanity-md5'];
+    const hasHash = Boolean(remoteSha1 || remoteMd5);
+    const method = md5 ? 'md5' : 'sha1';
+    let differs = false;
+
+    if (remoteMd5 && md5) {
+      differs = remoteMd5 !== md5;
+    } else if (remoteSha1 && sha1) {
+      differs = remoteSha1 !== sha1;
+    }
 
-      debug('Asset stream ready, writing to filesystem at %s', dstPath);
-      const hash = yield writeHashedStream(path.join(_this2.tmpDir, dstPath), stream);
-      const type = assetDoc._type === 'sanity.imageAsset' ? 'image' : 'file';
-      const id = `${type}-${hash}`;
-      const metaProps = omit(assetDoc, EXCLUDE_PROPS);
+    if (differs && attemptNum < 3) {
+      debug('%s does not match downloaded asset, retrying (#%d) [%s]', method, attemptNum + 1, url);
+      return this.downloadAsset(assetDoc, dstPath, attemptNum + 1);
+    } else if (differs) {
+      const details = [hasHash && (method === 'md5' ? "md5 should be ".concat(remoteMd5, ", got ").concat(md5) : "sha1 should be ".concat(remoteSha1, ", got ").concat(sha1)), contentLength && parseInt(contentLength, 10) !== size && "Asset should be ".concat(contentLength, " bytes, got ").concat(size), "Did not succeed after ".concat(attemptNum, " attempts.")];
+      const detailsString = "Details:\n - ".concat(details.filter(Boolean).join('\n - '));
+      await fse.unlink(tmpPath);
+      this.queue.clear();
+      const error = new Error("Failed to download asset at ".concat(assetDoc.url, ", giving up. ").concat(detailsString));
+      this.reject(error);
+      return false;
+    }
 
-      if (Object.keys(metaProps).length > 0) {
-        _this2.assetMap[id] = metaProps;
-      }
+    const isImage = assetDoc._type === 'sanity.imageAsset';
+    const type = isImage ? 'image' : 'file';
+    const id = "".concat(type, "-").concat(sha1);
+    const metaProps = omit(assetDoc, EXCLUDE_PROPS);
 
-      _this2.filesWritten++;
-    })();
-  } // eslint-disable-next-line complexity
+    if (Object.keys(metaProps).length > 0) {
+      this.assetMap[id] = metaProps;
+    }
 
+    this.downloading.splice(this.downloading.findIndex(datUrl => datUrl === url), 1);
+    this.filesWritten++;
+    return true;
+  }
 
 }
 
 function isAssetField(item) {
-  return item.asset && item.asset._ref;
+  return item.asset && item.asset._ref && isSanityAsset(item.asset._ref);
 }
 
 function getAssetType(item) {
@@ -277,36 +291,57 @@ function getAssetType(item) {
     return null;
   }
 
-  const _ref5 = item.asset._ref.match(/^(image|file)-/) || [],
-        _ref6 = _slicedToArray(_ref5, 2),
-        type = _ref6[1];
-
+  const [, type] = item.asset._ref.match(/^(image|file)-/) || [];
   return type || null;
 }
 
-function isModernAsset(assetId) {
-  return /^(image|file)/.test(assetId);
+function isSanityAsset(assetId) {
+  return /^image-[a-f0-9]{40}-\d+x\d+-[a-z]+$/.test(assetId) || /^file-[a-f0-9]{40}-[a-z0-9]+$/.test(assetId);
 }
 
 function generateFilename(assetId) {
-  const _ref7 = assetId.match(/^(image|file)-(.*?)(-[a-z]+)?$/) || [],
-        _ref8 = _slicedToArray(_ref7, 4),
-        asset = _ref8[2],
-        ext = _ref8[3];
-
+  const [,, asset, ext] = assetId.match(/^(image|file)-(.*?)(-[a-z]+)?$/) || [];
   const extension = (ext || 'bin').replace(/^-/, '');
-  return asset ? `${asset}.${extension}` : `${assetId}.bin`;
+  return asset ? "".concat(asset, ".").concat(extension) : "".concat(assetId, ".bin");
 }
 
 function writeHashedStream(filePath, stream) {
-  const hash = crypto.createHash('sha1');
+  let size = 0;
+  const md5 = crypto.createHash('md5');
+  const sha1 = crypto.createHash('sha1');
   const hasher = miss.through((chunk, enc, cb) => {
-    hash.update(chunk);
+    size += chunk.length;
+    md5.update(chunk);
+    sha1.update(chunk);
     cb(null, chunk);
   });
   return new Promise((resolve, reject) => miss.pipe(stream, hasher, fse.createWriteStream(filePath), err => {
-    return err ? reject(err) : resolve(hash.digest('hex'));
+    if (err) {
+      reject(err);
+      return;
+    }
+
+    resolve({
+      size,
+      sha1: sha1.digest('hex'),
+      md5: md5.digest('hex')
+    });
   }));
 }
 
+function tryGetErrorFromStream(stream) {
+  return new Promise((resolve, reject) => {
+    miss.pipe(stream, miss.concat(parse), err => err ? reject(err) : noop);
+
+    function parse(body) {
+      try {
+        const parsed = JSON.parse(body.toString('utf8'));
+        resolve(parsed.message || parsed.error || null);
+      } catch (err) {
+        resolve(body.toString('utf8').slice(0, 16000));
+      }
+    }
+  });
+}
+
 module.exports = AssetHandler;