@sanity/export 2.34.2-empty-template-cli.0 → 2.34.3-cdr-preview.14

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,136 +1,92 @@
1
1
  "use strict";
2
2
 
3
3
  var _excluded = ["asset"];
4
-
5
4
  function _slicedToArray(arr, i) { return _arrayWithHoles(arr) || _iterableToArrayLimit(arr, i) || _unsupportedIterableToArray(arr, i) || _nonIterableRest(); }
6
-
7
5
  function _nonIterableRest() { throw new TypeError("Invalid attempt to destructure non-iterable instance.\nIn order to be iterable, non-array objects must have a [Symbol.iterator]() method."); }
8
-
9
6
  function _unsupportedIterableToArray(o, minLen) { if (!o) return; if (typeof o === "string") return _arrayLikeToArray(o, minLen); var n = Object.prototype.toString.call(o).slice(8, -1); if (n === "Object" && o.constructor) n = o.constructor.name; if (n === "Map" || n === "Set") return Array.from(o); if (n === "Arguments" || /^(?:Ui|I)nt(?:8|16|32)(?:Clamped)?Array$/.test(n)) return _arrayLikeToArray(o, minLen); }
10
-
11
7
  function _arrayLikeToArray(arr, len) { if (len == null || len > arr.length) len = arr.length; for (var i = 0, arr2 = new Array(len); i < len; i++) { arr2[i] = arr[i]; } return arr2; }
12
-
13
8
  function _iterableToArrayLimit(arr, i) { var _i = arr == null ? null : typeof Symbol !== "undefined" && arr[Symbol.iterator] || arr["@@iterator"]; if (_i == null) return; var _arr = []; var _n = true; var _d = false; var _s, _e; try { for (_i = _i.call(arr); !(_n = (_s = _i.next()).done); _n = true) { _arr.push(_s.value); if (i && _arr.length === i) break; } } catch (err) { _d = true; _e = err; } finally { try { if (!_n && _i["return"] != null) _i["return"](); } finally { if (_d) throw _e; } } return _arr; }
14
-
15
9
  function _arrayWithHoles(arr) { if (Array.isArray(arr)) return arr; }
16
-
17
10
  function ownKeys(object, enumerableOnly) { var keys = Object.keys(object); if (Object.getOwnPropertySymbols) { var symbols = Object.getOwnPropertySymbols(object); enumerableOnly && (symbols = symbols.filter(function (sym) { return Object.getOwnPropertyDescriptor(object, sym).enumerable; })), keys.push.apply(keys, symbols); } return keys; }
18
-
19
11
  function _objectSpread(target) { for (var i = 1; i < arguments.length; i++) { var source = null != arguments[i] ? arguments[i] : {}; i % 2 ? ownKeys(Object(source), !0).forEach(function (key) { _defineProperty(target, key, source[key]); }) : Object.getOwnPropertyDescriptors ? Object.defineProperties(target, Object.getOwnPropertyDescriptors(source)) : ownKeys(Object(source)).forEach(function (key) { Object.defineProperty(target, key, Object.getOwnPropertyDescriptor(source, key)); }); } return target; }
20
-
21
12
  function _objectWithoutProperties(source, excluded) { if (source == null) return {}; var target = _objectWithoutPropertiesLoose(source, excluded); var key, i; if (Object.getOwnPropertySymbols) { var sourceSymbolKeys = Object.getOwnPropertySymbols(source); for (i = 0; i < sourceSymbolKeys.length; i++) { key = sourceSymbolKeys[i]; if (excluded.indexOf(key) >= 0) continue; if (!Object.prototype.propertyIsEnumerable.call(source, key)) continue; target[key] = source[key]; } } return target; }
22
-
23
13
  function _objectWithoutPropertiesLoose(source, excluded) { if (source == null) return {}; var target = {}; var sourceKeys = Object.keys(source); var key, i; for (i = 0; i < sourceKeys.length; i++) { key = sourceKeys[i]; if (excluded.indexOf(key) >= 0) continue; target[key] = source[key]; } return target; }
24
-
25
14
  function asyncGeneratorStep(gen, resolve, reject, _next, _throw, key, arg) { try { var info = gen[key](arg); var value = info.value; } catch (error) { reject(error); return; } if (info.done) { resolve(value); } else { Promise.resolve(value).then(_next, _throw); } }
26
-
27
15
  function _asyncToGenerator(fn) { return function () { var self = this, args = arguments; return new Promise(function (resolve, reject) { var gen = fn.apply(self, args); function _next(value) { asyncGeneratorStep(gen, resolve, reject, _next, _throw, "next", value); } function _throw(err) { asyncGeneratorStep(gen, resolve, reject, _next, _throw, "throw", err); } _next(undefined); }); }; }
28
-
29
16
  function _defineProperty(obj, key, value) { if (key in obj) { Object.defineProperty(obj, key, { value: value, enumerable: true, configurable: true, writable: true }); } else { obj[key] = value; } return obj; }
30
-
31
17
  var path = require('path');
32
-
33
18
  var crypto = require('crypto');
34
-
35
19
  var _require = require('url'),
36
- parseUrl = _require.parse,
37
- formatUrl = _require.format;
38
-
20
+ parseUrl = _require.parse,
21
+ formatUrl = _require.format;
39
22
  var fse = require('fs-extra');
40
-
41
23
  var miss = require('mississippi');
42
-
43
24
  var PQueue = require('p-queue');
44
-
45
25
  var _require2 = require('lodash'),
46
- omit = _require2.omit,
47
- noop = _require2.noop;
48
-
26
+ omit = _require2.omit,
27
+ noop = _require2.noop;
49
28
  var pkg = require('../package.json');
50
-
51
29
  var requestStream = require('./requestStream');
52
-
53
30
  var debug = require('./debug');
54
-
55
31
  var EXCLUDE_PROPS = ['_id', '_type', 'assetId', 'extension', 'mimeType', 'path', 'url'];
56
32
  var ACTION_REMOVE = 'remove';
57
33
  var ACTION_REWRITE = 'rewrite';
58
34
  var ASSET_DOWNLOAD_CONCURRENCY = 8;
59
-
60
35
  class AssetHandler {
61
36
  constructor(options) {
62
37
  var _this = this;
63
-
64
38
  _defineProperty(this, "rewriteAssets", miss.through.obj( /*#__PURE__*/function () {
65
39
  var _ref = _asyncToGenerator(function* (doc, enc, callback) {
66
40
  if (['sanity.imageAsset', 'sanity.fileAsset'].includes(doc._type)) {
67
41
  var type = doc._type === 'sanity.imageAsset' ? 'image' : 'file';
68
42
  var filePath = "".concat(type, "s/").concat(generateFilename(doc._id));
69
-
70
43
  _this.assetsSeen.set(doc._id, type);
71
-
72
44
  _this.queueAssetDownload(doc, filePath, type);
73
-
74
45
  callback();
75
46
  return;
76
47
  }
77
-
78
48
  callback(null, _this.findAndModify(doc, ACTION_REWRITE));
79
49
  });
80
-
81
50
  return function (_x, _x2, _x3) {
82
51
  return _ref.apply(this, arguments);
83
52
  };
84
53
  }()));
85
-
86
54
  _defineProperty(this, "stripAssets", miss.through.obj( /*#__PURE__*/function () {
87
55
  var _ref2 = _asyncToGenerator(function* (doc, enc, callback) {
88
56
  if (['sanity.imageAsset', 'sanity.fileAsset'].includes(doc._type)) {
89
57
  callback();
90
58
  return;
91
59
  }
92
-
93
60
  callback(null, _this.findAndModify(doc, ACTION_REMOVE));
94
61
  });
95
-
96
62
  return function (_x4, _x5, _x6) {
97
63
  return _ref2.apply(this, arguments);
98
64
  };
99
65
  }()));
100
-
101
66
  _defineProperty(this, "skipAssets", miss.through.obj((doc, enc, callback) => {
102
67
  var isAsset = ['sanity.imageAsset', 'sanity.fileAsset'].includes(doc._type);
103
-
104
68
  if (isAsset) {
105
69
  callback();
106
70
  return;
107
71
  }
108
-
109
72
  callback(null, doc);
110
73
  }));
111
-
112
74
  _defineProperty(this, "noop", miss.through.obj((doc, enc, callback) => callback(null, doc)));
113
-
114
75
  _defineProperty(this, "findAndModify", (item, action) => {
115
76
  if (Array.isArray(item)) {
116
77
  var children = item.map(child => this.findAndModify(child, action));
117
78
  return children.filter(Boolean);
118
79
  }
119
-
120
80
  if (!item || typeof item !== 'object') {
121
81
  return item;
122
82
  }
123
-
124
83
  var isAsset = isAssetField(item);
125
-
126
84
  if (isAsset && action === ACTION_REMOVE) {
127
85
  return undefined;
128
86
  }
129
-
130
87
  if (isAsset && action === ACTION_REWRITE) {
131
88
  var asset = item.asset,
132
- other = _objectWithoutProperties(item, _excluded);
133
-
89
+ other = _objectWithoutProperties(item, _excluded);
134
90
  var assetId = asset._ref;
135
91
  var assetType = getAssetType(item);
136
92
  var filePath = "".concat(assetType, "s/").concat(generateFilename(assetId));
@@ -138,23 +94,18 @@ class AssetHandler {
138
94
  _sanityAsset: "".concat(assetType, "@file://./").concat(filePath)
139
95
  }, this.findAndModify(other, action));
140
96
  }
141
-
142
97
  var newItem = {};
143
98
  var keys = Object.keys(item);
144
-
145
99
  for (var i = 0; i < keys.length; i++) {
146
100
  var key = keys[i];
147
101
  var value = item[key];
148
102
  newItem[key] = this.findAndModify(value, action);
149
-
150
103
  if (typeof newItem[key] === 'undefined') {
151
104
  delete newItem[key];
152
105
  }
153
106
  }
154
-
155
107
  return newItem;
156
108
  });
157
-
158
109
  var concurrency = options.concurrency || ASSET_DOWNLOAD_CONCURRENCY;
159
110
  debug('Using asset download concurrency of %d', concurrency);
160
111
  this.client = options.client;
@@ -169,58 +120,50 @@ class AssetHandler {
169
120
  concurrency
170
121
  });
171
122
  this.rejectedError = null;
172
-
173
123
  this.reject = err => {
174
124
  this.rejectedError = err;
175
125
  };
176
126
  }
177
-
178
127
  clear() {
179
128
  this.assetsSeen.clear();
180
129
  this.queue.clear();
181
130
  this.queueSize = 0;
182
131
  }
183
-
184
132
  finish() {
185
133
  return new Promise((resolve, reject) => {
186
134
  if (this.rejectedError) {
187
135
  reject(this.rejectedError);
188
136
  return;
189
137
  }
190
-
191
138
  this.reject = reject;
192
139
  this.queue.onIdle().then(() => resolve(this.assetMap));
193
140
  });
194
- } // Called when we want to download all assets to local filesystem and rewrite documents to hold
195
- // placeholder asset references (_sanityAsset: 'image@file:///local/path')
141
+ }
196
142
 
143
+ // Called when we want to download all assets to local filesystem and rewrite documents to hold
144
+ // placeholder asset references (_sanityAsset: 'image@file:///local/path')
197
145
 
198
146
  queueAssetDownload(assetDoc, dstPath, type) {
199
147
  if (!assetDoc.url) {
200
148
  debug('Asset document "%s" does not have a URL property, skipping', assetDoc._id);
201
149
  return;
202
150
  }
203
-
204
151
  debug('Adding download task for %s (destination: %s)', assetDoc._id, dstPath);
205
152
  this.queueSize++;
206
153
  this.downloading.push(assetDoc.url);
207
154
  this.queue.add(() => this.downloadAsset(assetDoc, dstPath));
208
155
  }
209
-
210
156
  maybeCreateAssetDirs() {
211
157
  if (this.assetDirsCreated) {
212
158
  return;
213
159
  }
214
- /* eslint-disable no-sync */
215
-
216
160
 
161
+ /* eslint-disable no-sync */
217
162
  fse.ensureDirSync(path.join(this.tmpDir, 'files'));
218
163
  fse.ensureDirSync(path.join(this.tmpDir, 'images'));
219
164
  /* eslint-enable no-sync */
220
-
221
165
  this.assetDirsCreated = true;
222
166
  }
223
-
224
167
  getAssetRequestOptions(assetDoc) {
225
168
  var token = this.client.config().token;
226
169
  var headers = {
@@ -228,78 +171,62 @@ class AssetHandler {
228
171
  };
229
172
  var isImage = assetDoc._type === 'sanity.imageAsset';
230
173
  var url = parseUrl(assetDoc.url, true);
231
-
232
174
  if (isImage && ['cdn.sanity.io', 'cdn.sanity.work'].includes(url.hostname) && token) {
233
175
  headers.Authorization = "Bearer ".concat(token);
234
176
  url.query = _objectSpread(_objectSpread({}, url.query || {}), {}, {
235
177
  dlRaw: 'true'
236
178
  });
237
179
  }
238
-
239
180
  return {
240
181
  url: formatUrl(url),
241
182
  headers
242
183
  };
243
184
  }
244
-
245
185
  downloadAsset(assetDoc, dstPath) {
246
186
  var _arguments = arguments,
247
- _this2 = this;
248
-
187
+ _this2 = this;
249
188
  return _asyncToGenerator(function* () {
250
189
  var attemptNum = _arguments.length > 2 && _arguments[2] !== undefined ? _arguments[2] : 0;
251
190
  var url = assetDoc.url;
252
-
253
191
  var options = _this2.getAssetRequestOptions(assetDoc);
254
-
255
192
  var stream;
256
-
257
193
  try {
258
194
  stream = yield requestStream(options);
259
195
  } catch (err) {
260
196
  _this2.reject(err);
261
-
262
197
  return false;
263
198
  }
264
-
265
199
  if (stream.statusCode !== 200) {
266
200
  _this2.queue.clear();
267
-
268
201
  var err = yield tryGetErrorFromStream(stream);
269
202
  var errMsg = "Referenced asset URL \"".concat(url, "\" returned HTTP ").concat(stream.statusCode);
270
-
271
203
  if (err) {
272
204
  errMsg = "".concat(errMsg, ":\n\n").concat(err);
273
205
  }
274
-
275
206
  _this2.reject(new Error(errMsg));
276
-
277
207
  return false;
278
208
  }
279
-
280
209
  _this2.maybeCreateAssetDirs();
281
-
282
210
  debug('Asset stream ready, writing to filesystem at %s', dstPath);
283
211
  var tmpPath = path.join(_this2.tmpDir, dstPath);
284
-
285
212
  var _yield$writeHashedStr = yield writeHashedStream(tmpPath, stream),
286
- sha1 = _yield$writeHashedStr.sha1,
287
- md5 = _yield$writeHashedStr.md5,
288
- size = _yield$writeHashedStr.size; // Verify it against our downloaded stream to make sure we have the same copy
289
-
213
+ sha1 = _yield$writeHashedStr.sha1,
214
+ md5 = _yield$writeHashedStr.md5,
215
+ size = _yield$writeHashedStr.size;
290
216
 
217
+ // Verify it against our downloaded stream to make sure we have the same copy
291
218
  var contentLength = stream.headers['content-length'];
292
219
  var remoteSha1 = stream.headers['x-sanity-sha1'];
293
220
  var remoteMd5 = stream.headers['x-sanity-md5'];
294
221
  var hasHash = Boolean(remoteSha1 || remoteMd5);
295
- var method = sha1 ? 'sha1' : 'md5'; // Asset validity is primarily determined by the sha1 hash. However, the sha1 hash is computed
222
+ var method = sha1 ? 'sha1' : 'md5';
223
+
224
+ // Asset validity is primarily determined by the sha1 hash. However, the sha1 hash is computed
296
225
  // before certain processes (i.e. svg sanitization) which can result in a different hash.
297
226
  // When the sha1 hashes don't match, fallback to using the md5 hash.
298
-
299
227
  var sha1Differs = remoteSha1 && sha1 !== remoteSha1;
300
228
  var md5Differs = remoteMd5 && md5 !== remoteMd5;
301
229
  var differs = sha1Differs && md5Differs;
302
-
303
230
  if (differs && attemptNum < 3) {
304
231
  debug('%s does not match downloaded asset, retrying (#%d) [%s]', method, attemptNum + 1, url);
305
232
  return _this2.downloadAsset(assetDoc, dstPath, attemptNum + 1);
@@ -307,64 +234,47 @@ class AssetHandler {
307
234
  var details = [hasHash && (method === 'md5' ? "md5 should be ".concat(remoteMd5, ", got ").concat(md5) : "sha1 should be ".concat(remoteSha1, ", got ").concat(sha1)), contentLength && parseInt(contentLength, 10) !== size && "Asset should be ".concat(contentLength, " bytes, got ").concat(size), "Did not succeed after ".concat(attemptNum, " attempts.")];
308
235
  var detailsString = "Details:\n - ".concat(details.filter(Boolean).join('\n - '));
309
236
  yield fse.unlink(tmpPath);
310
-
311
237
  _this2.queue.clear();
312
-
313
238
  var error = new Error("Failed to download asset at ".concat(assetDoc.url, ", giving up. ").concat(detailsString));
314
-
315
239
  _this2.reject(error);
316
-
317
240
  return false;
318
241
  }
319
-
320
242
  var isImage = assetDoc._type === 'sanity.imageAsset';
321
243
  var type = isImage ? 'image' : 'file';
322
244
  var id = "".concat(type, "-").concat(sha1);
323
245
  var metaProps = omit(assetDoc, EXCLUDE_PROPS);
324
-
325
246
  if (Object.keys(metaProps).length > 0) {
326
247
  _this2.assetMap[id] = metaProps;
327
248
  }
328
-
329
249
  _this2.downloading.splice(_this2.downloading.findIndex(datUrl => datUrl === url), 1);
330
-
331
250
  _this2.filesWritten++;
332
251
  return true;
333
252
  })();
334
253
  }
335
-
336
254
  }
337
-
338
255
  function isAssetField(item) {
339
256
  return item.asset && item.asset._ref && isSanityAsset(item.asset._ref);
340
257
  }
341
-
342
258
  function getAssetType(item) {
343
259
  if (!item.asset || typeof item.asset._ref !== 'string') {
344
260
  return null;
345
261
  }
346
-
347
262
  var _ref3 = item.asset._ref.match(/^(image|file)-/) || [],
348
- _ref4 = _slicedToArray(_ref3, 2),
349
- type = _ref4[1];
350
-
263
+ _ref4 = _slicedToArray(_ref3, 2),
264
+ type = _ref4[1];
351
265
  return type || null;
352
266
  }
353
-
354
267
  function isSanityAsset(assetId) {
355
268
  return /^image-[a-f0-9]{40}-\d+x\d+-[a-z]+$/.test(assetId) || /^file-[a-f0-9]{40}-[a-z0-9]+$/.test(assetId);
356
269
  }
357
-
358
270
  function generateFilename(assetId) {
359
271
  var _ref5 = assetId.match(/^(image|file)-(.*?)(-[a-z]+)?$/) || [],
360
- _ref6 = _slicedToArray(_ref5, 4),
361
- asset = _ref6[2],
362
- ext = _ref6[3];
363
-
272
+ _ref6 = _slicedToArray(_ref5, 4),
273
+ asset = _ref6[2],
274
+ ext = _ref6[3];
364
275
  var extension = (ext || 'bin').replace(/^-/, '');
365
276
  return asset ? "".concat(asset, ".").concat(extension) : "".concat(assetId, ".bin");
366
277
  }
367
-
368
278
  function writeHashedStream(filePath, stream) {
369
279
  var size = 0;
370
280
  var md5 = crypto.createHash('md5');
@@ -380,7 +290,6 @@ function writeHashedStream(filePath, stream) {
380
290
  reject(err);
381
291
  return;
382
292
  }
383
-
384
293
  resolve({
385
294
  size,
386
295
  sha1: sha1.digest('hex'),
@@ -388,11 +297,9 @@ function writeHashedStream(filePath, stream) {
388
297
  });
389
298
  }));
390
299
  }
391
-
392
300
  function tryGetErrorFromStream(stream) {
393
301
  return new Promise((resolve, reject) => {
394
302
  miss.pipe(stream, miss.concat(parse), err => err ? reject(err) : noop);
395
-
396
303
  function parse(body) {
397
304
  try {
398
305
  var parsed = JSON.parse(body.toString('utf8'));
@@ -403,5 +310,4 @@ function tryGetErrorFromStream(stream) {
403
310
  }
404
311
  });
405
312
  }
406
-
407
313
  module.exports = AssetHandler;
package/lib/export.js CHANGED
@@ -1,47 +1,26 @@
1
1
  "use strict";
2
2
 
3
3
  function asyncGeneratorStep(gen, resolve, reject, _next, _throw, key, arg) { try { var info = gen[key](arg); var value = info.value; } catch (error) { reject(error); return; } if (info.done) { resolve(value); } else { Promise.resolve(value).then(_next, _throw); } }
4
-
5
4
  function _asyncToGenerator(fn) { return function () { var self = this, args = arguments; return new Promise(function (resolve, reject) { var gen = fn.apply(self, args); function _next(value) { asyncGeneratorStep(gen, resolve, reject, _next, _throw, "next", value); } function _throw(err) { asyncGeneratorStep(gen, resolve, reject, _next, _throw, "throw", err); } _next(undefined); }); }; }
6
-
7
5
  var os = require('os');
8
-
9
6
  var path = require('path');
10
-
11
7
  var zlib = require('zlib');
12
-
13
8
  var fse = require('fs-extra');
14
-
15
9
  var miss = require('mississippi');
16
-
17
10
  var split = require('split2');
18
-
19
11
  var archiver = require('archiver');
20
-
21
12
  var debug = require('./debug');
22
-
23
13
  var AssetHandler = require('./AssetHandler');
24
-
25
14
  var stringifyStream = require('./stringifyStream');
26
-
27
15
  var validateOptions = require('./validateOptions');
28
-
29
16
  var rejectOnApiError = require('./rejectOnApiError');
30
-
31
17
  var getDocumentsStream = require('./getDocumentsStream');
32
-
33
18
  var filterSystemDocuments = require('./filterSystemDocuments');
34
-
35
19
  var filterDocumentTypes = require('./filterDocumentTypes');
36
-
37
20
  var filterDrafts = require('./filterDrafts');
38
-
39
21
  var logFirstChunk = require('./logFirstChunk');
40
-
41
22
  var tryParseJson = require('./tryParseJson');
42
-
43
23
  var noop = () => null;
44
-
45
24
  function exportDataset(opts) {
46
25
  var options = validateOptions(opts);
47
26
  var onProgress = options.onProgress || noop;
@@ -54,11 +33,9 @@ function exportDataset(opts) {
54
33
  var slugDate = new Date().toISOString().replace(/[^a-z0-9]/gi, '-').toLowerCase();
55
34
  var prefix = "".concat(opts.dataset, "-export-").concat(slugDate);
56
35
  var tmpDir = path.join(os.tmpdir(), prefix);
57
-
58
36
  var cleanup = () => fse.remove(tmpDir).catch(err => {
59
37
  debug("Error while cleaning up temporary files: ".concat(err.message));
60
38
  });
61
-
62
39
  var assetHandler = new AssetHandler({
63
40
  client: options.client,
64
41
  tmpDir,
@@ -68,19 +45,15 @@ function exportDataset(opts) {
68
45
  debug('Outputting assets (temporarily) to %s', tmpDir);
69
46
  debug('Outputting to %s', options.outputPath === '-' ? 'stdout' : options.outputPath);
70
47
  var outputStream;
71
-
72
48
  if (isWritableStream(options.outputPath)) {
73
49
  outputStream = options.outputPath;
74
50
  } else {
75
51
  outputStream = options.outputPath === '-' ? process.stdout : fse.createWriteStream(options.outputPath);
76
52
  }
77
-
78
53
  var assetStreamHandler = assetHandler.noop;
79
-
80
54
  if (!options.raw) {
81
55
  assetStreamHandler = options.assets ? assetHandler.rewriteAssets : assetHandler.stripAssets;
82
56
  }
83
-
84
57
  return new Promise( /*#__PURE__*/function () {
85
58
  var _ref = _asyncToGenerator(function* (resolve, reject) {
86
59
  miss.finished(archive, /*#__PURE__*/function () {
@@ -91,10 +64,8 @@ function exportDataset(opts) {
91
64
  reject(archiveErr);
92
65
  return;
93
66
  }
94
-
95
67
  debug('Archive finished!');
96
68
  });
97
-
98
69
  return function (_x3) {
99
70
  return _ref2.apply(this, arguments);
100
71
  };
@@ -105,11 +76,9 @@ function exportDataset(opts) {
105
76
  });
106
77
  var documentCount = 0;
107
78
  var lastReported = Date.now();
108
-
109
79
  var reportDocumentCount = (chunk, enc, cb) => {
110
80
  ++documentCount;
111
81
  var now = Date.now();
112
-
113
82
  if (now - lastReported > 50) {
114
83
  onProgress({
115
84
  step: 'Exporting documents...',
@@ -119,10 +88,8 @@ function exportDataset(opts) {
119
88
  });
120
89
  lastReported = now;
121
90
  }
122
-
123
91
  cb(null, chunk);
124
92
  };
125
-
126
93
  var inputStream = yield getDocumentsStream(options.client, options.dataset);
127
94
  debug('Got HTTP %d', inputStream.statusCode);
128
95
  debug('Response headers: %o', inputStream.headers);
@@ -132,28 +99,23 @@ function exportDataset(opts) {
132
99
  if (err) {
133
100
  return;
134
101
  }
135
-
136
102
  onProgress({
137
103
  step: 'Exporting documents...',
138
104
  current: documentCount,
139
105
  total: documentCount,
140
106
  update: true
141
107
  });
142
-
143
108
  if (!options.raw && options.assets) {
144
109
  onProgress({
145
110
  step: 'Downloading assets...'
146
111
  });
147
112
  }
148
-
149
113
  var prevCompleted = 0;
150
114
  var progressInterval = setInterval(() => {
151
115
  var completed = assetHandler.queueSize - assetHandler.queue.size - assetHandler.queue.pending;
152
-
153
116
  if (prevCompleted === completed) {
154
117
  return;
155
118
  }
156
-
157
119
  prevCompleted = completed;
158
120
  onProgress({
159
121
  step: 'Downloading assets...',
@@ -163,10 +125,10 @@ function exportDataset(opts) {
163
125
  });
164
126
  }, 500);
165
127
  debug('Waiting for asset handler to complete downloads');
166
-
167
128
  try {
168
- var assetMap = yield assetHandler.finish(); // Make sure we mark the progress as done (eg 100/100 instead of 99/100)
129
+ var assetMap = yield assetHandler.finish();
169
130
 
131
+ // Make sure we mark the progress as done (eg 100/100 instead of 99/100)
170
132
  onProgress({
171
133
  step: 'Downloading assets...',
172
134
  current: assetHandler.queueSize,
@@ -183,9 +145,9 @@ function exportDataset(opts) {
183
145
  yield cleanup();
184
146
  reject(assetErr);
185
147
  return;
186
- } // Add all downloaded assets to archive
187
-
148
+ }
188
149
 
150
+ // Add all downloaded assets to archive
189
151
  archive.directory(path.join(tmpDir, 'files'), "".concat(prefix, "/files"), {
190
152
  store: true
191
153
  });
@@ -198,7 +160,6 @@ function exportDataset(opts) {
198
160
  });
199
161
  archive.finalize();
200
162
  });
201
-
202
163
  return function (_x4) {
203
164
  return _ref3.apply(this, arguments);
204
165
  };
@@ -211,23 +172,19 @@ function exportDataset(opts) {
211
172
  prefix
212
173
  });
213
174
  miss.pipe(archive, outputStream, onComplete);
214
-
215
175
  function onComplete(_x5) {
216
176
  return _onComplete.apply(this, arguments);
217
177
  }
218
-
219
178
  function _onComplete() {
220
179
  _onComplete = _asyncToGenerator(function* (err) {
221
180
  onProgress({
222
181
  step: 'Clearing temporary files...'
223
182
  });
224
183
  yield cleanup();
225
-
226
184
  if (!err) {
227
185
  resolve();
228
186
  return;
229
187
  }
230
-
231
188
  debug('Error during streaming: %s', err.stack);
232
189
  assetHandler.clear();
233
190
  reject(err);
@@ -235,15 +192,12 @@ function exportDataset(opts) {
235
192
  return _onComplete.apply(this, arguments);
236
193
  }
237
194
  });
238
-
239
195
  return function (_x, _x2) {
240
196
  return _ref.apply(this, arguments);
241
197
  };
242
198
  }());
243
199
  }
244
-
245
200
  function isWritableStream(val) {
246
201
  return val !== null && typeof val === 'object' && typeof val.pipe === 'function' && typeof val._write === 'function' && typeof val._writableState === 'object';
247
202
  }
248
-
249
203
  module.exports = exportDataset;
@@ -1,14 +1,11 @@
1
1
  "use strict";
2
2
 
3
3
  var miss = require('mississippi');
4
-
5
4
  module.exports = allowedTypes => allowedTypes ? miss.through.obj((doc, enc, callback) => {
6
5
  var type = doc && doc._type;
7
-
8
6
  if (allowedTypes.includes(type)) {
9
7
  callback(null, doc);
10
8
  return;
11
9
  }
12
-
13
10
  callback();
14
11
  }) : miss.through.obj();
@@ -1,13 +1,10 @@
1
1
  "use strict";
2
2
 
3
3
  var miss = require('mississippi');
4
-
5
4
  var isDraft = doc => doc && doc._id && doc._id.indexOf('drafts.') === 0;
6
-
7
5
  module.exports = () => miss.through.obj((doc, enc, callback) => {
8
6
  if (isDraft(doc)) {
9
7
  return callback();
10
8
  }
11
-
12
9
  return callback(null, doc);
13
10
  });
@@ -1,16 +1,12 @@
1
1
  "use strict";
2
2
 
3
3
  var miss = require('mississippi');
4
-
5
4
  var debug = require('./debug');
6
-
7
5
  var isSystemDocument = doc => doc && doc._id && doc._id.indexOf('_.') === 0;
8
-
9
6
  module.exports = () => miss.through.obj((doc, enc, callback) => {
10
7
  if (isSystemDocument(doc)) {
11
8
  debug('%s is a system document, skipping', doc && doc._id);
12
9
  return callback();
13
10
  }
14
-
15
11
  return callback(null, doc);
16
12
  });
@@ -1,27 +1,20 @@
1
1
  "use strict";
2
2
 
3
3
  function ownKeys(object, enumerableOnly) { var keys = Object.keys(object); if (Object.getOwnPropertySymbols) { var symbols = Object.getOwnPropertySymbols(object); enumerableOnly && (symbols = symbols.filter(function (sym) { return Object.getOwnPropertyDescriptor(object, sym).enumerable; })), keys.push.apply(keys, symbols); } return keys; }
4
-
5
4
  function _objectSpread(target) { for (var i = 1; i < arguments.length; i++) { var source = null != arguments[i] ? arguments[i] : {}; i % 2 ? ownKeys(Object(source), !0).forEach(function (key) { _defineProperty(target, key, source[key]); }) : Object.getOwnPropertyDescriptors ? Object.defineProperties(target, Object.getOwnPropertyDescriptors(source)) : ownKeys(Object(source)).forEach(function (key) { Object.defineProperty(target, key, Object.getOwnPropertyDescriptor(source, key)); }); } return target; }
6
-
7
5
  function _defineProperty(obj, key, value) { if (key in obj) { Object.defineProperty(obj, key, { value: value, enumerable: true, configurable: true, writable: true }); } else { obj[key] = value; } return obj; }
8
-
9
6
  var pkg = require('../package.json');
10
-
11
7
  var requestStream = require('./requestStream');
12
-
13
8
  module.exports = (client, dataset) => {
14
9
  // Sanity client doesn't handle streams natively since we want to support node/browser
15
10
  // with same API. We're just using it here to get hold of URLs and tokens.
16
11
  var url = client.getUrl("/data/export/".concat(dataset));
17
12
  var token = client.config().token;
18
-
19
13
  var headers = _objectSpread({
20
14
  'User-Agent': "".concat(pkg.name, "@").concat(pkg.version)
21
15
  }, token ? {
22
16
  Authorization: "Bearer ".concat(token)
23
17
  } : {});
24
-
25
18
  return requestStream({
26
19
  url,
27
20
  headers
@@ -1,9 +1,7 @@
1
1
  "use strict";
2
2
 
3
3
  var miss = require('mississippi');
4
-
5
4
  var debug = require('./debug');
6
-
7
5
  module.exports = () => {
8
6
  var firstChunk = true;
9
7
  return miss.through((chunk, enc, callback) => {
@@ -12,7 +10,6 @@ module.exports = () => {
12
10
  debug('First chunk received: %s', string.slice(0, 300));
13
11
  firstChunk = false;
14
12
  }
15
-
16
13
  callback(null, chunk);
17
14
  });
18
15
  };
@@ -1,17 +1,14 @@
1
1
  "use strict";
2
2
 
3
3
  var miss = require('mississippi');
4
-
5
4
  module.exports = () => miss.through.obj((doc, enc, callback) => {
6
5
  if (doc.error && doc.statusCode) {
7
6
  callback(new Error([doc.statusCode, doc.error].join(': ')));
8
7
  return;
9
8
  }
10
-
11
9
  if (!doc._id && doc.error) {
12
10
  callback(new Error(doc.error.description || doc.error.message || JSON.stringify(doc)));
13
11
  return;
14
12
  }
15
-
16
13
  callback(null, doc);
17
14
  });
@@ -1,43 +1,30 @@
1
1
  "use strict";
2
2
 
3
3
  function ownKeys(object, enumerableOnly) { var keys = Object.keys(object); if (Object.getOwnPropertySymbols) { var symbols = Object.getOwnPropertySymbols(object); enumerableOnly && (symbols = symbols.filter(function (sym) { return Object.getOwnPropertyDescriptor(object, sym).enumerable; })), keys.push.apply(keys, symbols); } return keys; }
4
-
5
4
  function _objectSpread(target) { for (var i = 1; i < arguments.length; i++) { var source = null != arguments[i] ? arguments[i] : {}; i % 2 ? ownKeys(Object(source), !0).forEach(function (key) { _defineProperty(target, key, source[key]); }) : Object.getOwnPropertyDescriptors ? Object.defineProperties(target, Object.getOwnPropertyDescriptors(source)) : ownKeys(Object(source)).forEach(function (key) { Object.defineProperty(target, key, Object.getOwnPropertyDescriptor(source, key)); }); } return target; }
6
-
7
5
  function _defineProperty(obj, key, value) { if (key in obj) { Object.defineProperty(obj, key, { value: value, enumerable: true, configurable: true, writable: true }); } else { obj[key] = value; } return obj; }
8
-
9
6
  function asyncGeneratorStep(gen, resolve, reject, _next, _throw, key, arg) { try { var info = gen[key](arg); var value = info.value; } catch (error) { reject(error); return; } if (info.done) { resolve(value); } else { Promise.resolve(value).then(_next, _throw); } }
10
-
11
7
  function _asyncToGenerator(fn) { return function () { var self = this, args = arguments; return new Promise(function (resolve, reject) { var gen = fn.apply(self, args); function _next(value) { asyncGeneratorStep(gen, resolve, reject, _next, _throw, "next", value); } function _throw(err) { asyncGeneratorStep(gen, resolve, reject, _next, _throw, "throw", err); } _next(undefined); }); }; }
12
-
13
8
  var getIt = require('get-it');
14
-
15
9
  var _require = require('get-it/middleware'),
16
- keepAlive = _require.keepAlive,
17
- promise = _require.promise;
18
-
10
+ keepAlive = _require.keepAlive,
11
+ promise = _require.promise;
19
12
  var debug = require('./debug');
20
-
21
13
  var request = getIt([keepAlive(), promise({
22
14
  onlyBody: true
23
15
  })]);
24
16
  var socketsWithTimeout = new WeakSet();
25
17
  var CONNECTION_TIMEOUT = 15 * 1000; // 15 seconds
26
-
27
18
  var READ_TIMEOUT = 3 * 60 * 1000; // 3 minutes
28
-
29
19
  var MAX_RETRIES = 5;
30
-
31
20
  function delay(ms) {
32
21
  return new Promise(resolve => setTimeout(resolve, ms));
33
22
  }
34
- /* eslint-disable no-await-in-loop, max-depth */
35
-
36
23
 
24
+ /* eslint-disable no-await-in-loop, max-depth */
37
25
  module.exports = /*#__PURE__*/function () {
38
26
  var _ref = _asyncToGenerator(function* (options) {
39
27
  var error;
40
-
41
28
  for (var i = 0; i < MAX_RETRIES; i++) {
42
29
  try {
43
30
  var _ret = yield* function* () {
@@ -49,35 +36,28 @@ module.exports = /*#__PURE__*/function () {
49
36
  socket: READ_TIMEOUT
50
37
  }
51
38
  }));
52
-
53
39
  if (response.connection && typeof response.connection.setTimeout === 'function' && !socketsWithTimeout.has(response.connection)) {
54
40
  socketsWithTimeout.add(response.connection);
55
41
  response.connection.setTimeout(READ_TIMEOUT, () => {
56
42
  response.destroy(new Error("Read timeout: No data received on socket for ".concat(READ_TIMEOUT, " ms")));
57
43
  });
58
44
  }
59
-
60
45
  return {
61
46
  v: response
62
47
  };
63
48
  }();
64
-
65
49
  if (typeof _ret === "object") return _ret.v;
66
50
  } catch (err) {
67
51
  error = err;
68
-
69
52
  if (err.response && err.response.statusCode && err.response.statusCode < 500) {
70
53
  break;
71
54
  }
72
-
73
55
  debug('Error, retrying after 1500ms: %s', err.message);
74
56
  yield delay(1500);
75
57
  }
76
58
  }
77
-
78
59
  throw error;
79
60
  });
80
-
81
61
  return function (_x) {
82
62
  return _ref.apply(this, arguments);
83
63
  };
@@ -1,5 +1,4 @@
1
1
  "use strict";
2
2
 
3
3
  var miss = require('mississippi');
4
-
5
4
  module.exports = () => miss.through.obj((doc, enc, callback) => callback(null, "".concat(JSON.stringify(doc), "\n")));
@@ -6,20 +6,16 @@ module.exports = line => {
6
6
  } catch (err) {
7
7
  // Catch half-done lines with an error at the end
8
8
  var errorPosition = line.lastIndexOf('{"error":');
9
-
10
9
  if (errorPosition === -1) {
11
10
  err.message = "".concat(err.message, " (").concat(line, ")");
12
11
  throw err;
13
12
  }
14
-
15
13
  var errorJson = line.slice(errorPosition);
16
14
  var errorLine = JSON.parse(errorJson);
17
15
  var error = errorLine && errorLine.error;
18
-
19
16
  if (error && error.description) {
20
17
  throw new Error("Error streaming dataset: ".concat(error.description, "\n\n").concat(errorJson, "\n"));
21
18
  }
22
-
23
19
  throw err;
24
20
  }
25
21
  };
@@ -1,7 +1,6 @@
1
1
  "use strict";
2
2
 
3
3
  var defaults = require('lodash/defaults');
4
-
5
4
  var clientMethods = ['getUrl', 'config'];
6
5
  var booleanFlags = ['assets', 'raw', 'compress', 'drafts'];
7
6
  var exportDefaults = {
@@ -10,49 +9,36 @@ var exportDefaults = {
10
9
  assets: true,
11
10
  raw: false
12
11
  };
13
-
14
12
  function validateOptions(opts) {
15
13
  var options = defaults({}, opts, exportDefaults);
16
-
17
14
  if (typeof options.dataset !== 'string' || options.dataset.length < 1) {
18
15
  throw new Error("options.dataset must be a valid dataset name");
19
16
  }
20
-
21
17
  if (options.onProgress && typeof options.onProgress !== 'function') {
22
18
  throw new Error("options.onProgress must be a function");
23
19
  }
24
-
25
20
  if (!options.client) {
26
21
  throw new Error('`options.client` must be set to an instance of @sanity/client');
27
22
  }
28
-
29
23
  var missing = clientMethods.find(key => typeof options.client[key] !== 'function');
30
-
31
24
  if (missing) {
32
25
  throw new Error("`options.client` is not a valid @sanity/client instance - no \"".concat(missing, "\" method found"));
33
26
  }
34
-
35
27
  var clientConfig = options.client.config();
36
-
37
28
  if (!clientConfig.token) {
38
29
  throw new Error('Client is not instantiated with a `token`');
39
30
  }
40
-
41
31
  booleanFlags.forEach(flag => {
42
32
  if (typeof options[flag] !== 'boolean') {
43
33
  throw new Error("Flag ".concat(flag, " must be a boolean (true/false)"));
44
34
  }
45
35
  });
46
-
47
36
  if (!options.outputPath) {
48
37
  throw new Error('outputPath must be specified (- for stdout)');
49
38
  }
50
-
51
39
  if (options.assetConcurrency && (options.assetConcurrency < 1 || options.assetConcurrency > 24)) {
52
40
  throw new Error('`assetConcurrency` must be between 1 and 24');
53
41
  }
54
-
55
42
  return options;
56
43
  }
57
-
58
44
  module.exports = validateOptions;
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@sanity/export",
3
- "version": "2.34.2-empty-template-cli.0+184a9350e5",
3
+ "version": "2.34.3-cdr-preview.14+94349f5e1d",
4
4
  "description": "Export Sanity documents and assets",
5
5
  "main": "lib/export.js",
6
6
  "engines": {
@@ -49,5 +49,5 @@
49
49
  "url": "https://github.com/sanity-io/sanity/issues"
50
50
  },
51
51
  "homepage": "https://www.sanity.io/",
52
- "gitHead": "184a9350e50aa05877a3b10e052ce4e2ca743ef0"
52
+ "gitHead": "94349f5e1d9ff65fb08e14b7b1f42dd69c508c47"
53
53
  }