@sanity/export 2.34.1-canary.0 → 2.34.2-empty-template-cli.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,92 +1,136 @@
1
1
  "use strict";
2
2
 
3
3
  var _excluded = ["asset"];
4
+
4
5
  function _slicedToArray(arr, i) { return _arrayWithHoles(arr) || _iterableToArrayLimit(arr, i) || _unsupportedIterableToArray(arr, i) || _nonIterableRest(); }
6
+
5
7
  function _nonIterableRest() { throw new TypeError("Invalid attempt to destructure non-iterable instance.\nIn order to be iterable, non-array objects must have a [Symbol.iterator]() method."); }
8
+
6
9
  function _unsupportedIterableToArray(o, minLen) { if (!o) return; if (typeof o === "string") return _arrayLikeToArray(o, minLen); var n = Object.prototype.toString.call(o).slice(8, -1); if (n === "Object" && o.constructor) n = o.constructor.name; if (n === "Map" || n === "Set") return Array.from(o); if (n === "Arguments" || /^(?:Ui|I)nt(?:8|16|32)(?:Clamped)?Array$/.test(n)) return _arrayLikeToArray(o, minLen); }
10
+
7
11
  function _arrayLikeToArray(arr, len) { if (len == null || len > arr.length) len = arr.length; for (var i = 0, arr2 = new Array(len); i < len; i++) { arr2[i] = arr[i]; } return arr2; }
12
+
8
13
  function _iterableToArrayLimit(arr, i) { var _i = arr == null ? null : typeof Symbol !== "undefined" && arr[Symbol.iterator] || arr["@@iterator"]; if (_i == null) return; var _arr = []; var _n = true; var _d = false; var _s, _e; try { for (_i = _i.call(arr); !(_n = (_s = _i.next()).done); _n = true) { _arr.push(_s.value); if (i && _arr.length === i) break; } } catch (err) { _d = true; _e = err; } finally { try { if (!_n && _i["return"] != null) _i["return"](); } finally { if (_d) throw _e; } } return _arr; }
14
+
9
15
  function _arrayWithHoles(arr) { if (Array.isArray(arr)) return arr; }
16
+
10
17
  function ownKeys(object, enumerableOnly) { var keys = Object.keys(object); if (Object.getOwnPropertySymbols) { var symbols = Object.getOwnPropertySymbols(object); enumerableOnly && (symbols = symbols.filter(function (sym) { return Object.getOwnPropertyDescriptor(object, sym).enumerable; })), keys.push.apply(keys, symbols); } return keys; }
18
+
11
19
  function _objectSpread(target) { for (var i = 1; i < arguments.length; i++) { var source = null != arguments[i] ? arguments[i] : {}; i % 2 ? ownKeys(Object(source), !0).forEach(function (key) { _defineProperty(target, key, source[key]); }) : Object.getOwnPropertyDescriptors ? Object.defineProperties(target, Object.getOwnPropertyDescriptors(source)) : ownKeys(Object(source)).forEach(function (key) { Object.defineProperty(target, key, Object.getOwnPropertyDescriptor(source, key)); }); } return target; }
20
+
12
21
  function _objectWithoutProperties(source, excluded) { if (source == null) return {}; var target = _objectWithoutPropertiesLoose(source, excluded); var key, i; if (Object.getOwnPropertySymbols) { var sourceSymbolKeys = Object.getOwnPropertySymbols(source); for (i = 0; i < sourceSymbolKeys.length; i++) { key = sourceSymbolKeys[i]; if (excluded.indexOf(key) >= 0) continue; if (!Object.prototype.propertyIsEnumerable.call(source, key)) continue; target[key] = source[key]; } } return target; }
22
+
13
23
  function _objectWithoutPropertiesLoose(source, excluded) { if (source == null) return {}; var target = {}; var sourceKeys = Object.keys(source); var key, i; for (i = 0; i < sourceKeys.length; i++) { key = sourceKeys[i]; if (excluded.indexOf(key) >= 0) continue; target[key] = source[key]; } return target; }
24
+
14
25
  function asyncGeneratorStep(gen, resolve, reject, _next, _throw, key, arg) { try { var info = gen[key](arg); var value = info.value; } catch (error) { reject(error); return; } if (info.done) { resolve(value); } else { Promise.resolve(value).then(_next, _throw); } }
26
+
15
27
  function _asyncToGenerator(fn) { return function () { var self = this, args = arguments; return new Promise(function (resolve, reject) { var gen = fn.apply(self, args); function _next(value) { asyncGeneratorStep(gen, resolve, reject, _next, _throw, "next", value); } function _throw(err) { asyncGeneratorStep(gen, resolve, reject, _next, _throw, "throw", err); } _next(undefined); }); }; }
28
+
16
29
  function _defineProperty(obj, key, value) { if (key in obj) { Object.defineProperty(obj, key, { value: value, enumerable: true, configurable: true, writable: true }); } else { obj[key] = value; } return obj; }
30
+
17
31
  var path = require('path');
32
+
18
33
  var crypto = require('crypto');
34
+
19
35
  var _require = require('url'),
20
- parseUrl = _require.parse,
21
- formatUrl = _require.format;
36
+ parseUrl = _require.parse,
37
+ formatUrl = _require.format;
38
+
22
39
  var fse = require('fs-extra');
40
+
23
41
  var miss = require('mississippi');
42
+
24
43
  var PQueue = require('p-queue');
44
+
25
45
  var _require2 = require('lodash'),
26
- omit = _require2.omit,
27
- noop = _require2.noop;
46
+ omit = _require2.omit,
47
+ noop = _require2.noop;
48
+
28
49
  var pkg = require('../package.json');
50
+
29
51
  var requestStream = require('./requestStream');
52
+
30
53
  var debug = require('./debug');
54
+
31
55
  var EXCLUDE_PROPS = ['_id', '_type', 'assetId', 'extension', 'mimeType', 'path', 'url'];
32
56
  var ACTION_REMOVE = 'remove';
33
57
  var ACTION_REWRITE = 'rewrite';
34
58
  var ASSET_DOWNLOAD_CONCURRENCY = 8;
59
+
35
60
  class AssetHandler {
36
61
  constructor(options) {
37
62
  var _this = this;
63
+
38
64
  _defineProperty(this, "rewriteAssets", miss.through.obj( /*#__PURE__*/function () {
39
65
  var _ref = _asyncToGenerator(function* (doc, enc, callback) {
40
66
  if (['sanity.imageAsset', 'sanity.fileAsset'].includes(doc._type)) {
41
67
  var type = doc._type === 'sanity.imageAsset' ? 'image' : 'file';
42
68
  var filePath = "".concat(type, "s/").concat(generateFilename(doc._id));
69
+
43
70
  _this.assetsSeen.set(doc._id, type);
71
+
44
72
  _this.queueAssetDownload(doc, filePath, type);
73
+
45
74
  callback();
46
75
  return;
47
76
  }
77
+
48
78
  callback(null, _this.findAndModify(doc, ACTION_REWRITE));
49
79
  });
80
+
50
81
  return function (_x, _x2, _x3) {
51
82
  return _ref.apply(this, arguments);
52
83
  };
53
84
  }()));
85
+
54
86
  _defineProperty(this, "stripAssets", miss.through.obj( /*#__PURE__*/function () {
55
87
  var _ref2 = _asyncToGenerator(function* (doc, enc, callback) {
56
88
  if (['sanity.imageAsset', 'sanity.fileAsset'].includes(doc._type)) {
57
89
  callback();
58
90
  return;
59
91
  }
92
+
60
93
  callback(null, _this.findAndModify(doc, ACTION_REMOVE));
61
94
  });
95
+
62
96
  return function (_x4, _x5, _x6) {
63
97
  return _ref2.apply(this, arguments);
64
98
  };
65
99
  }()));
100
+
66
101
  _defineProperty(this, "skipAssets", miss.through.obj((doc, enc, callback) => {
67
102
  var isAsset = ['sanity.imageAsset', 'sanity.fileAsset'].includes(doc._type);
103
+
68
104
  if (isAsset) {
69
105
  callback();
70
106
  return;
71
107
  }
108
+
72
109
  callback(null, doc);
73
110
  }));
111
+
74
112
  _defineProperty(this, "noop", miss.through.obj((doc, enc, callback) => callback(null, doc)));
113
+
75
114
  _defineProperty(this, "findAndModify", (item, action) => {
76
115
  if (Array.isArray(item)) {
77
116
  var children = item.map(child => this.findAndModify(child, action));
78
117
  return children.filter(Boolean);
79
118
  }
119
+
80
120
  if (!item || typeof item !== 'object') {
81
121
  return item;
82
122
  }
123
+
83
124
  var isAsset = isAssetField(item);
125
+
84
126
  if (isAsset && action === ACTION_REMOVE) {
85
127
  return undefined;
86
128
  }
129
+
87
130
  if (isAsset && action === ACTION_REWRITE) {
88
131
  var asset = item.asset,
89
- other = _objectWithoutProperties(item, _excluded);
132
+ other = _objectWithoutProperties(item, _excluded);
133
+
90
134
  var assetId = asset._ref;
91
135
  var assetType = getAssetType(item);
92
136
  var filePath = "".concat(assetType, "s/").concat(generateFilename(assetId));
@@ -94,18 +138,23 @@ class AssetHandler {
94
138
  _sanityAsset: "".concat(assetType, "@file://./").concat(filePath)
95
139
  }, this.findAndModify(other, action));
96
140
  }
141
+
97
142
  var newItem = {};
98
143
  var keys = Object.keys(item);
144
+
99
145
  for (var i = 0; i < keys.length; i++) {
100
146
  var key = keys[i];
101
147
  var value = item[key];
102
148
  newItem[key] = this.findAndModify(value, action);
149
+
103
150
  if (typeof newItem[key] === 'undefined') {
104
151
  delete newItem[key];
105
152
  }
106
153
  }
154
+
107
155
  return newItem;
108
156
  });
157
+
109
158
  var concurrency = options.concurrency || ASSET_DOWNLOAD_CONCURRENCY;
110
159
  debug('Using asset download concurrency of %d', concurrency);
111
160
  this.client = options.client;
@@ -120,50 +169,58 @@ class AssetHandler {
120
169
  concurrency
121
170
  });
122
171
  this.rejectedError = null;
172
+
123
173
  this.reject = err => {
124
174
  this.rejectedError = err;
125
175
  };
126
176
  }
177
+
127
178
  clear() {
128
179
  this.assetsSeen.clear();
129
180
  this.queue.clear();
130
181
  this.queueSize = 0;
131
182
  }
183
+
132
184
  finish() {
133
185
  return new Promise((resolve, reject) => {
134
186
  if (this.rejectedError) {
135
187
  reject(this.rejectedError);
136
188
  return;
137
189
  }
190
+
138
191
  this.reject = reject;
139
192
  this.queue.onIdle().then(() => resolve(this.assetMap));
140
193
  });
141
- }
142
-
143
- // Called when we want to download all assets to local filesystem and rewrite documents to hold
194
+ } // Called when we want to download all assets to local filesystem and rewrite documents to hold
144
195
  // placeholder asset references (_sanityAsset: 'image@file:///local/path')
145
196
 
197
+
146
198
  queueAssetDownload(assetDoc, dstPath, type) {
147
199
  if (!assetDoc.url) {
148
200
  debug('Asset document "%s" does not have a URL property, skipping', assetDoc._id);
149
201
  return;
150
202
  }
203
+
151
204
  debug('Adding download task for %s (destination: %s)', assetDoc._id, dstPath);
152
205
  this.queueSize++;
153
206
  this.downloading.push(assetDoc.url);
154
207
  this.queue.add(() => this.downloadAsset(assetDoc, dstPath));
155
208
  }
209
+
156
210
  maybeCreateAssetDirs() {
157
211
  if (this.assetDirsCreated) {
158
212
  return;
159
213
  }
160
-
161
214
  /* eslint-disable no-sync */
215
+
216
+
162
217
  fse.ensureDirSync(path.join(this.tmpDir, 'files'));
163
218
  fse.ensureDirSync(path.join(this.tmpDir, 'images'));
164
219
  /* eslint-enable no-sync */
220
+
165
221
  this.assetDirsCreated = true;
166
222
  }
223
+
167
224
  getAssetRequestOptions(assetDoc) {
168
225
  var token = this.client.config().token;
169
226
  var headers = {
@@ -171,62 +228,78 @@ class AssetHandler {
171
228
  };
172
229
  var isImage = assetDoc._type === 'sanity.imageAsset';
173
230
  var url = parseUrl(assetDoc.url, true);
231
+
174
232
  if (isImage && ['cdn.sanity.io', 'cdn.sanity.work'].includes(url.hostname) && token) {
175
233
  headers.Authorization = "Bearer ".concat(token);
176
234
  url.query = _objectSpread(_objectSpread({}, url.query || {}), {}, {
177
235
  dlRaw: 'true'
178
236
  });
179
237
  }
238
+
180
239
  return {
181
240
  url: formatUrl(url),
182
241
  headers
183
242
  };
184
243
  }
244
+
185
245
  downloadAsset(assetDoc, dstPath) {
186
246
  var _arguments = arguments,
187
- _this2 = this;
247
+ _this2 = this;
248
+
188
249
  return _asyncToGenerator(function* () {
189
250
  var attemptNum = _arguments.length > 2 && _arguments[2] !== undefined ? _arguments[2] : 0;
190
251
  var url = assetDoc.url;
252
+
191
253
  var options = _this2.getAssetRequestOptions(assetDoc);
254
+
192
255
  var stream;
256
+
193
257
  try {
194
258
  stream = yield requestStream(options);
195
259
  } catch (err) {
196
260
  _this2.reject(err);
261
+
197
262
  return false;
198
263
  }
264
+
199
265
  if (stream.statusCode !== 200) {
200
266
  _this2.queue.clear();
267
+
201
268
  var err = yield tryGetErrorFromStream(stream);
202
269
  var errMsg = "Referenced asset URL \"".concat(url, "\" returned HTTP ").concat(stream.statusCode);
270
+
203
271
  if (err) {
204
272
  errMsg = "".concat(errMsg, ":\n\n").concat(err);
205
273
  }
274
+
206
275
  _this2.reject(new Error(errMsg));
276
+
207
277
  return false;
208
278
  }
279
+
209
280
  _this2.maybeCreateAssetDirs();
281
+
210
282
  debug('Asset stream ready, writing to filesystem at %s', dstPath);
211
283
  var tmpPath = path.join(_this2.tmpDir, dstPath);
284
+
212
285
  var _yield$writeHashedStr = yield writeHashedStream(tmpPath, stream),
213
- sha1 = _yield$writeHashedStr.sha1,
214
- md5 = _yield$writeHashedStr.md5,
215
- size = _yield$writeHashedStr.size;
286
+ sha1 = _yield$writeHashedStr.sha1,
287
+ md5 = _yield$writeHashedStr.md5,
288
+ size = _yield$writeHashedStr.size; // Verify it against our downloaded stream to make sure we have the same copy
289
+
216
290
 
217
- // Verify it against our downloaded stream to make sure we have the same copy
218
291
  var contentLength = stream.headers['content-length'];
219
292
  var remoteSha1 = stream.headers['x-sanity-sha1'];
220
293
  var remoteMd5 = stream.headers['x-sanity-md5'];
221
294
  var hasHash = Boolean(remoteSha1 || remoteMd5);
222
- var method = sha1 ? 'sha1' : 'md5';
223
-
224
- // Asset validity is primarily determined by the sha1 hash. However, the sha1 hash is computed
295
+ var method = sha1 ? 'sha1' : 'md5'; // Asset validity is primarily determined by the sha1 hash. However, the sha1 hash is computed
225
296
  // before certain processes (i.e. svg sanitization) which can result in a different hash.
226
297
  // When the sha1 hashes don't match, fallback to using the md5 hash.
298
+
227
299
  var sha1Differs = remoteSha1 && sha1 !== remoteSha1;
228
300
  var md5Differs = remoteMd5 && md5 !== remoteMd5;
229
301
  var differs = sha1Differs && md5Differs;
302
+
230
303
  if (differs && attemptNum < 3) {
231
304
  debug('%s does not match downloaded asset, retrying (#%d) [%s]', method, attemptNum + 1, url);
232
305
  return _this2.downloadAsset(assetDoc, dstPath, attemptNum + 1);
@@ -234,47 +307,64 @@ class AssetHandler {
234
307
  var details = [hasHash && (method === 'md5' ? "md5 should be ".concat(remoteMd5, ", got ").concat(md5) : "sha1 should be ".concat(remoteSha1, ", got ").concat(sha1)), contentLength && parseInt(contentLength, 10) !== size && "Asset should be ".concat(contentLength, " bytes, got ").concat(size), "Did not succeed after ".concat(attemptNum, " attempts.")];
235
308
  var detailsString = "Details:\n - ".concat(details.filter(Boolean).join('\n - '));
236
309
  yield fse.unlink(tmpPath);
310
+
237
311
  _this2.queue.clear();
312
+
238
313
  var error = new Error("Failed to download asset at ".concat(assetDoc.url, ", giving up. ").concat(detailsString));
314
+
239
315
  _this2.reject(error);
316
+
240
317
  return false;
241
318
  }
319
+
242
320
  var isImage = assetDoc._type === 'sanity.imageAsset';
243
321
  var type = isImage ? 'image' : 'file';
244
322
  var id = "".concat(type, "-").concat(sha1);
245
323
  var metaProps = omit(assetDoc, EXCLUDE_PROPS);
324
+
246
325
  if (Object.keys(metaProps).length > 0) {
247
326
  _this2.assetMap[id] = metaProps;
248
327
  }
328
+
249
329
  _this2.downloading.splice(_this2.downloading.findIndex(datUrl => datUrl === url), 1);
330
+
250
331
  _this2.filesWritten++;
251
332
  return true;
252
333
  })();
253
334
  }
335
+
254
336
  }
337
+
255
338
  function isAssetField(item) {
256
339
  return item.asset && item.asset._ref && isSanityAsset(item.asset._ref);
257
340
  }
341
+
258
342
  function getAssetType(item) {
259
343
  if (!item.asset || typeof item.asset._ref !== 'string') {
260
344
  return null;
261
345
  }
346
+
262
347
  var _ref3 = item.asset._ref.match(/^(image|file)-/) || [],
263
- _ref4 = _slicedToArray(_ref3, 2),
264
- type = _ref4[1];
348
+ _ref4 = _slicedToArray(_ref3, 2),
349
+ type = _ref4[1];
350
+
265
351
  return type || null;
266
352
  }
353
+
267
354
  function isSanityAsset(assetId) {
268
355
  return /^image-[a-f0-9]{40}-\d+x\d+-[a-z]+$/.test(assetId) || /^file-[a-f0-9]{40}-[a-z0-9]+$/.test(assetId);
269
356
  }
357
+
270
358
  function generateFilename(assetId) {
271
359
  var _ref5 = assetId.match(/^(image|file)-(.*?)(-[a-z]+)?$/) || [],
272
- _ref6 = _slicedToArray(_ref5, 4),
273
- asset = _ref6[2],
274
- ext = _ref6[3];
360
+ _ref6 = _slicedToArray(_ref5, 4),
361
+ asset = _ref6[2],
362
+ ext = _ref6[3];
363
+
275
364
  var extension = (ext || 'bin').replace(/^-/, '');
276
365
  return asset ? "".concat(asset, ".").concat(extension) : "".concat(assetId, ".bin");
277
366
  }
367
+
278
368
  function writeHashedStream(filePath, stream) {
279
369
  var size = 0;
280
370
  var md5 = crypto.createHash('md5');
@@ -290,6 +380,7 @@ function writeHashedStream(filePath, stream) {
290
380
  reject(err);
291
381
  return;
292
382
  }
383
+
293
384
  resolve({
294
385
  size,
295
386
  sha1: sha1.digest('hex'),
@@ -297,9 +388,11 @@ function writeHashedStream(filePath, stream) {
297
388
  });
298
389
  }));
299
390
  }
391
+
300
392
  function tryGetErrorFromStream(stream) {
301
393
  return new Promise((resolve, reject) => {
302
394
  miss.pipe(stream, miss.concat(parse), err => err ? reject(err) : noop);
395
+
303
396
  function parse(body) {
304
397
  try {
305
398
  var parsed = JSON.parse(body.toString('utf8'));
@@ -310,4 +403,5 @@ function tryGetErrorFromStream(stream) {
310
403
  }
311
404
  });
312
405
  }
406
+
313
407
  module.exports = AssetHandler;
package/lib/export.js CHANGED
@@ -1,26 +1,47 @@
1
1
  "use strict";
2
2
 
3
3
  function asyncGeneratorStep(gen, resolve, reject, _next, _throw, key, arg) { try { var info = gen[key](arg); var value = info.value; } catch (error) { reject(error); return; } if (info.done) { resolve(value); } else { Promise.resolve(value).then(_next, _throw); } }
4
+
4
5
  function _asyncToGenerator(fn) { return function () { var self = this, args = arguments; return new Promise(function (resolve, reject) { var gen = fn.apply(self, args); function _next(value) { asyncGeneratorStep(gen, resolve, reject, _next, _throw, "next", value); } function _throw(err) { asyncGeneratorStep(gen, resolve, reject, _next, _throw, "throw", err); } _next(undefined); }); }; }
6
+
5
7
  var os = require('os');
8
+
6
9
  var path = require('path');
10
+
7
11
  var zlib = require('zlib');
12
+
8
13
  var fse = require('fs-extra');
14
+
9
15
  var miss = require('mississippi');
16
+
10
17
  var split = require('split2');
18
+
11
19
  var archiver = require('archiver');
20
+
12
21
  var debug = require('./debug');
22
+
13
23
  var AssetHandler = require('./AssetHandler');
24
+
14
25
  var stringifyStream = require('./stringifyStream');
26
+
15
27
  var validateOptions = require('./validateOptions');
28
+
16
29
  var rejectOnApiError = require('./rejectOnApiError');
30
+
17
31
  var getDocumentsStream = require('./getDocumentsStream');
32
+
18
33
  var filterSystemDocuments = require('./filterSystemDocuments');
34
+
19
35
  var filterDocumentTypes = require('./filterDocumentTypes');
36
+
20
37
  var filterDrafts = require('./filterDrafts');
38
+
21
39
  var logFirstChunk = require('./logFirstChunk');
40
+
22
41
  var tryParseJson = require('./tryParseJson');
42
+
23
43
  var noop = () => null;
44
+
24
45
  function exportDataset(opts) {
25
46
  var options = validateOptions(opts);
26
47
  var onProgress = options.onProgress || noop;
@@ -33,9 +54,11 @@ function exportDataset(opts) {
33
54
  var slugDate = new Date().toISOString().replace(/[^a-z0-9]/gi, '-').toLowerCase();
34
55
  var prefix = "".concat(opts.dataset, "-export-").concat(slugDate);
35
56
  var tmpDir = path.join(os.tmpdir(), prefix);
57
+
36
58
  var cleanup = () => fse.remove(tmpDir).catch(err => {
37
59
  debug("Error while cleaning up temporary files: ".concat(err.message));
38
60
  });
61
+
39
62
  var assetHandler = new AssetHandler({
40
63
  client: options.client,
41
64
  tmpDir,
@@ -45,15 +68,19 @@ function exportDataset(opts) {
45
68
  debug('Outputting assets (temporarily) to %s', tmpDir);
46
69
  debug('Outputting to %s', options.outputPath === '-' ? 'stdout' : options.outputPath);
47
70
  var outputStream;
71
+
48
72
  if (isWritableStream(options.outputPath)) {
49
73
  outputStream = options.outputPath;
50
74
  } else {
51
75
  outputStream = options.outputPath === '-' ? process.stdout : fse.createWriteStream(options.outputPath);
52
76
  }
77
+
53
78
  var assetStreamHandler = assetHandler.noop;
79
+
54
80
  if (!options.raw) {
55
81
  assetStreamHandler = options.assets ? assetHandler.rewriteAssets : assetHandler.stripAssets;
56
82
  }
83
+
57
84
  return new Promise( /*#__PURE__*/function () {
58
85
  var _ref = _asyncToGenerator(function* (resolve, reject) {
59
86
  miss.finished(archive, /*#__PURE__*/function () {
@@ -64,8 +91,10 @@ function exportDataset(opts) {
64
91
  reject(archiveErr);
65
92
  return;
66
93
  }
94
+
67
95
  debug('Archive finished!');
68
96
  });
97
+
69
98
  return function (_x3) {
70
99
  return _ref2.apply(this, arguments);
71
100
  };
@@ -76,9 +105,11 @@ function exportDataset(opts) {
76
105
  });
77
106
  var documentCount = 0;
78
107
  var lastReported = Date.now();
108
+
79
109
  var reportDocumentCount = (chunk, enc, cb) => {
80
110
  ++documentCount;
81
111
  var now = Date.now();
112
+
82
113
  if (now - lastReported > 50) {
83
114
  onProgress({
84
115
  step: 'Exporting documents...',
@@ -88,8 +119,10 @@ function exportDataset(opts) {
88
119
  });
89
120
  lastReported = now;
90
121
  }
122
+
91
123
  cb(null, chunk);
92
124
  };
125
+
93
126
  var inputStream = yield getDocumentsStream(options.client, options.dataset);
94
127
  debug('Got HTTP %d', inputStream.statusCode);
95
128
  debug('Response headers: %o', inputStream.headers);
@@ -99,23 +132,28 @@ function exportDataset(opts) {
99
132
  if (err) {
100
133
  return;
101
134
  }
135
+
102
136
  onProgress({
103
137
  step: 'Exporting documents...',
104
138
  current: documentCount,
105
139
  total: documentCount,
106
140
  update: true
107
141
  });
142
+
108
143
  if (!options.raw && options.assets) {
109
144
  onProgress({
110
145
  step: 'Downloading assets...'
111
146
  });
112
147
  }
148
+
113
149
  var prevCompleted = 0;
114
150
  var progressInterval = setInterval(() => {
115
151
  var completed = assetHandler.queueSize - assetHandler.queue.size - assetHandler.queue.pending;
152
+
116
153
  if (prevCompleted === completed) {
117
154
  return;
118
155
  }
156
+
119
157
  prevCompleted = completed;
120
158
  onProgress({
121
159
  step: 'Downloading assets...',
@@ -125,10 +163,10 @@ function exportDataset(opts) {
125
163
  });
126
164
  }, 500);
127
165
  debug('Waiting for asset handler to complete downloads');
166
+
128
167
  try {
129
- var assetMap = yield assetHandler.finish();
168
+ var assetMap = yield assetHandler.finish(); // Make sure we mark the progress as done (eg 100/100 instead of 99/100)
130
169
 
131
- // Make sure we mark the progress as done (eg 100/100 instead of 99/100)
132
170
  onProgress({
133
171
  step: 'Downloading assets...',
134
172
  current: assetHandler.queueSize,
@@ -145,9 +183,9 @@ function exportDataset(opts) {
145
183
  yield cleanup();
146
184
  reject(assetErr);
147
185
  return;
148
- }
186
+ } // Add all downloaded assets to archive
187
+
149
188
 
150
- // Add all downloaded assets to archive
151
189
  archive.directory(path.join(tmpDir, 'files'), "".concat(prefix, "/files"), {
152
190
  store: true
153
191
  });
@@ -160,6 +198,7 @@ function exportDataset(opts) {
160
198
  });
161
199
  archive.finalize();
162
200
  });
201
+
163
202
  return function (_x4) {
164
203
  return _ref3.apply(this, arguments);
165
204
  };
@@ -172,19 +211,23 @@ function exportDataset(opts) {
172
211
  prefix
173
212
  });
174
213
  miss.pipe(archive, outputStream, onComplete);
214
+
175
215
  function onComplete(_x5) {
176
216
  return _onComplete.apply(this, arguments);
177
217
  }
218
+
178
219
  function _onComplete() {
179
220
  _onComplete = _asyncToGenerator(function* (err) {
180
221
  onProgress({
181
222
  step: 'Clearing temporary files...'
182
223
  });
183
224
  yield cleanup();
225
+
184
226
  if (!err) {
185
227
  resolve();
186
228
  return;
187
229
  }
230
+
188
231
  debug('Error during streaming: %s', err.stack);
189
232
  assetHandler.clear();
190
233
  reject(err);
@@ -192,12 +235,15 @@ function exportDataset(opts) {
192
235
  return _onComplete.apply(this, arguments);
193
236
  }
194
237
  });
238
+
195
239
  return function (_x, _x2) {
196
240
  return _ref.apply(this, arguments);
197
241
  };
198
242
  }());
199
243
  }
244
+
200
245
  function isWritableStream(val) {
201
246
  return val !== null && typeof val === 'object' && typeof val.pipe === 'function' && typeof val._write === 'function' && typeof val._writableState === 'object';
202
247
  }
248
+
203
249
  module.exports = exportDataset;
@@ -1,11 +1,14 @@
1
1
  "use strict";
2
2
 
3
3
  var miss = require('mississippi');
4
+
4
5
  module.exports = allowedTypes => allowedTypes ? miss.through.obj((doc, enc, callback) => {
5
6
  var type = doc && doc._type;
7
+
6
8
  if (allowedTypes.includes(type)) {
7
9
  callback(null, doc);
8
10
  return;
9
11
  }
12
+
10
13
  callback();
11
14
  }) : miss.through.obj();
@@ -1,10 +1,13 @@
1
1
  "use strict";
2
2
 
3
3
  var miss = require('mississippi');
4
+
4
5
  var isDraft = doc => doc && doc._id && doc._id.indexOf('drafts.') === 0;
6
+
5
7
  module.exports = () => miss.through.obj((doc, enc, callback) => {
6
8
  if (isDraft(doc)) {
7
9
  return callback();
8
10
  }
11
+
9
12
  return callback(null, doc);
10
13
  });
@@ -1,12 +1,16 @@
1
1
  "use strict";
2
2
 
3
3
  var miss = require('mississippi');
4
+
4
5
  var debug = require('./debug');
6
+
5
7
  var isSystemDocument = doc => doc && doc._id && doc._id.indexOf('_.') === 0;
8
+
6
9
  module.exports = () => miss.through.obj((doc, enc, callback) => {
7
10
  if (isSystemDocument(doc)) {
8
11
  debug('%s is a system document, skipping', doc && doc._id);
9
12
  return callback();
10
13
  }
14
+
11
15
  return callback(null, doc);
12
16
  });
@@ -1,20 +1,27 @@
1
1
  "use strict";
2
2
 
3
3
  function ownKeys(object, enumerableOnly) { var keys = Object.keys(object); if (Object.getOwnPropertySymbols) { var symbols = Object.getOwnPropertySymbols(object); enumerableOnly && (symbols = symbols.filter(function (sym) { return Object.getOwnPropertyDescriptor(object, sym).enumerable; })), keys.push.apply(keys, symbols); } return keys; }
4
+
4
5
  function _objectSpread(target) { for (var i = 1; i < arguments.length; i++) { var source = null != arguments[i] ? arguments[i] : {}; i % 2 ? ownKeys(Object(source), !0).forEach(function (key) { _defineProperty(target, key, source[key]); }) : Object.getOwnPropertyDescriptors ? Object.defineProperties(target, Object.getOwnPropertyDescriptors(source)) : ownKeys(Object(source)).forEach(function (key) { Object.defineProperty(target, key, Object.getOwnPropertyDescriptor(source, key)); }); } return target; }
6
+
5
7
  function _defineProperty(obj, key, value) { if (key in obj) { Object.defineProperty(obj, key, { value: value, enumerable: true, configurable: true, writable: true }); } else { obj[key] = value; } return obj; }
8
+
6
9
  var pkg = require('../package.json');
10
+
7
11
  var requestStream = require('./requestStream');
12
+
8
13
  module.exports = (client, dataset) => {
9
14
  // Sanity client doesn't handle streams natively since we want to support node/browser
10
15
  // with same API. We're just using it here to get hold of URLs and tokens.
11
16
  var url = client.getUrl("/data/export/".concat(dataset));
12
17
  var token = client.config().token;
18
+
13
19
  var headers = _objectSpread({
14
20
  'User-Agent': "".concat(pkg.name, "@").concat(pkg.version)
15
21
  }, token ? {
16
22
  Authorization: "Bearer ".concat(token)
17
23
  } : {});
24
+
18
25
  return requestStream({
19
26
  url,
20
27
  headers
@@ -1,7 +1,9 @@
1
1
  "use strict";
2
2
 
3
3
  var miss = require('mississippi');
4
+
4
5
  var debug = require('./debug');
6
+
5
7
  module.exports = () => {
6
8
  var firstChunk = true;
7
9
  return miss.through((chunk, enc, callback) => {
@@ -10,6 +12,7 @@ module.exports = () => {
10
12
  debug('First chunk received: %s', string.slice(0, 300));
11
13
  firstChunk = false;
12
14
  }
15
+
13
16
  callback(null, chunk);
14
17
  });
15
18
  };
@@ -1,14 +1,17 @@
1
1
  "use strict";
2
2
 
3
3
  var miss = require('mississippi');
4
+
4
5
  module.exports = () => miss.through.obj((doc, enc, callback) => {
5
6
  if (doc.error && doc.statusCode) {
6
7
  callback(new Error([doc.statusCode, doc.error].join(': ')));
7
8
  return;
8
9
  }
10
+
9
11
  if (!doc._id && doc.error) {
10
12
  callback(new Error(doc.error.description || doc.error.message || JSON.stringify(doc)));
11
13
  return;
12
14
  }
15
+
13
16
  callback(null, doc);
14
17
  });
@@ -1,30 +1,43 @@
1
1
  "use strict";
2
2
 
3
3
  function ownKeys(object, enumerableOnly) { var keys = Object.keys(object); if (Object.getOwnPropertySymbols) { var symbols = Object.getOwnPropertySymbols(object); enumerableOnly && (symbols = symbols.filter(function (sym) { return Object.getOwnPropertyDescriptor(object, sym).enumerable; })), keys.push.apply(keys, symbols); } return keys; }
4
+
4
5
  function _objectSpread(target) { for (var i = 1; i < arguments.length; i++) { var source = null != arguments[i] ? arguments[i] : {}; i % 2 ? ownKeys(Object(source), !0).forEach(function (key) { _defineProperty(target, key, source[key]); }) : Object.getOwnPropertyDescriptors ? Object.defineProperties(target, Object.getOwnPropertyDescriptors(source)) : ownKeys(Object(source)).forEach(function (key) { Object.defineProperty(target, key, Object.getOwnPropertyDescriptor(source, key)); }); } return target; }
6
+
5
7
  function _defineProperty(obj, key, value) { if (key in obj) { Object.defineProperty(obj, key, { value: value, enumerable: true, configurable: true, writable: true }); } else { obj[key] = value; } return obj; }
8
+
6
9
  function asyncGeneratorStep(gen, resolve, reject, _next, _throw, key, arg) { try { var info = gen[key](arg); var value = info.value; } catch (error) { reject(error); return; } if (info.done) { resolve(value); } else { Promise.resolve(value).then(_next, _throw); } }
10
+
7
11
  function _asyncToGenerator(fn) { return function () { var self = this, args = arguments; return new Promise(function (resolve, reject) { var gen = fn.apply(self, args); function _next(value) { asyncGeneratorStep(gen, resolve, reject, _next, _throw, "next", value); } function _throw(err) { asyncGeneratorStep(gen, resolve, reject, _next, _throw, "throw", err); } _next(undefined); }); }; }
12
+
8
13
  var getIt = require('get-it');
14
+
9
15
  var _require = require('get-it/middleware'),
10
- keepAlive = _require.keepAlive,
11
- promise = _require.promise;
16
+ keepAlive = _require.keepAlive,
17
+ promise = _require.promise;
18
+
12
19
  var debug = require('./debug');
20
+
13
21
  var request = getIt([keepAlive(), promise({
14
22
  onlyBody: true
15
23
  })]);
16
24
  var socketsWithTimeout = new WeakSet();
17
25
  var CONNECTION_TIMEOUT = 15 * 1000; // 15 seconds
26
+
18
27
  var READ_TIMEOUT = 3 * 60 * 1000; // 3 minutes
28
+
19
29
  var MAX_RETRIES = 5;
30
+
20
31
  function delay(ms) {
21
32
  return new Promise(resolve => setTimeout(resolve, ms));
22
33
  }
23
-
24
34
  /* eslint-disable no-await-in-loop, max-depth */
35
+
36
+
25
37
  module.exports = /*#__PURE__*/function () {
26
38
  var _ref = _asyncToGenerator(function* (options) {
27
39
  var error;
40
+
28
41
  for (var i = 0; i < MAX_RETRIES; i++) {
29
42
  try {
30
43
  var _ret = yield* function* () {
@@ -36,28 +49,35 @@ module.exports = /*#__PURE__*/function () {
36
49
  socket: READ_TIMEOUT
37
50
  }
38
51
  }));
52
+
39
53
  if (response.connection && typeof response.connection.setTimeout === 'function' && !socketsWithTimeout.has(response.connection)) {
40
54
  socketsWithTimeout.add(response.connection);
41
55
  response.connection.setTimeout(READ_TIMEOUT, () => {
42
56
  response.destroy(new Error("Read timeout: No data received on socket for ".concat(READ_TIMEOUT, " ms")));
43
57
  });
44
58
  }
59
+
45
60
  return {
46
61
  v: response
47
62
  };
48
63
  }();
64
+
49
65
  if (typeof _ret === "object") return _ret.v;
50
66
  } catch (err) {
51
67
  error = err;
68
+
52
69
  if (err.response && err.response.statusCode && err.response.statusCode < 500) {
53
70
  break;
54
71
  }
72
+
55
73
  debug('Error, retrying after 1500ms: %s', err.message);
56
74
  yield delay(1500);
57
75
  }
58
76
  }
77
+
59
78
  throw error;
60
79
  });
80
+
61
81
  return function (_x) {
62
82
  return _ref.apply(this, arguments);
63
83
  };
@@ -1,4 +1,5 @@
1
1
  "use strict";
2
2
 
3
3
  var miss = require('mississippi');
4
+
4
5
  module.exports = () => miss.through.obj((doc, enc, callback) => callback(null, "".concat(JSON.stringify(doc), "\n")));
@@ -6,16 +6,20 @@ module.exports = line => {
6
6
  } catch (err) {
7
7
  // Catch half-done lines with an error at the end
8
8
  var errorPosition = line.lastIndexOf('{"error":');
9
+
9
10
  if (errorPosition === -1) {
10
11
  err.message = "".concat(err.message, " (").concat(line, ")");
11
12
  throw err;
12
13
  }
14
+
13
15
  var errorJson = line.slice(errorPosition);
14
16
  var errorLine = JSON.parse(errorJson);
15
17
  var error = errorLine && errorLine.error;
18
+
16
19
  if (error && error.description) {
17
20
  throw new Error("Error streaming dataset: ".concat(error.description, "\n\n").concat(errorJson, "\n"));
18
21
  }
22
+
19
23
  throw err;
20
24
  }
21
25
  };
@@ -1,6 +1,7 @@
1
1
  "use strict";
2
2
 
3
3
  var defaults = require('lodash/defaults');
4
+
4
5
  var clientMethods = ['getUrl', 'config'];
5
6
  var booleanFlags = ['assets', 'raw', 'compress', 'drafts'];
6
7
  var exportDefaults = {
@@ -9,36 +10,49 @@ var exportDefaults = {
9
10
  assets: true,
10
11
  raw: false
11
12
  };
13
+
12
14
  function validateOptions(opts) {
13
15
  var options = defaults({}, opts, exportDefaults);
16
+
14
17
  if (typeof options.dataset !== 'string' || options.dataset.length < 1) {
15
18
  throw new Error("options.dataset must be a valid dataset name");
16
19
  }
20
+
17
21
  if (options.onProgress && typeof options.onProgress !== 'function') {
18
22
  throw new Error("options.onProgress must be a function");
19
23
  }
24
+
20
25
  if (!options.client) {
21
26
  throw new Error('`options.client` must be set to an instance of @sanity/client');
22
27
  }
28
+
23
29
  var missing = clientMethods.find(key => typeof options.client[key] !== 'function');
30
+
24
31
  if (missing) {
25
32
  throw new Error("`options.client` is not a valid @sanity/client instance - no \"".concat(missing, "\" method found"));
26
33
  }
34
+
27
35
  var clientConfig = options.client.config();
36
+
28
37
  if (!clientConfig.token) {
29
38
  throw new Error('Client is not instantiated with a `token`');
30
39
  }
40
+
31
41
  booleanFlags.forEach(flag => {
32
42
  if (typeof options[flag] !== 'boolean') {
33
43
  throw new Error("Flag ".concat(flag, " must be a boolean (true/false)"));
34
44
  }
35
45
  });
46
+
36
47
  if (!options.outputPath) {
37
48
  throw new Error('outputPath must be specified (- for stdout)');
38
49
  }
50
+
39
51
  if (options.assetConcurrency && (options.assetConcurrency < 1 || options.assetConcurrency > 24)) {
40
52
  throw new Error('`assetConcurrency` must be between 1 and 24');
41
53
  }
54
+
42
55
  return options;
43
56
  }
57
+
44
58
  module.exports = validateOptions;
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@sanity/export",
3
- "version": "2.34.1-canary.0+b3e58b8893",
3
+ "version": "2.34.2-empty-template-cli.0+184a9350e5",
4
4
  "description": "Export Sanity documents and assets",
5
5
  "main": "lib/export.js",
6
6
  "engines": {
@@ -49,5 +49,5 @@
49
49
  "url": "https://github.com/sanity-io/sanity/issues"
50
50
  },
51
51
  "homepage": "https://www.sanity.io/",
52
- "gitHead": "b3e58b8893c9573844279488896df2313df914f1"
52
+ "gitHead": "184a9350e50aa05877a3b10e052ce4e2ca743ef0"
53
53
  }