@sanity/export 2.33.4-shopify.8 → 2.34.1-canary.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/AssetHandler.js +22 -116
- package/lib/export.js +4 -50
- package/lib/filterDocumentTypes.js +0 -3
- package/lib/filterDrafts.js +0 -3
- package/lib/filterSystemDocuments.js +0 -4
- package/lib/getDocumentsStream.js +0 -7
- package/lib/logFirstChunk.js +0 -3
- package/lib/rejectOnApiError.js +0 -3
- package/lib/requestStream.js +3 -23
- package/lib/stringifyStream.js +0 -1
- package/lib/tryParseJson.js +0 -4
- package/lib/validateOptions.js +0 -14
- package/package.json +2 -2
package/lib/AssetHandler.js
CHANGED
|
@@ -1,136 +1,92 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
|
|
3
3
|
var _excluded = ["asset"];
|
|
4
|
-
|
|
5
4
|
function _slicedToArray(arr, i) { return _arrayWithHoles(arr) || _iterableToArrayLimit(arr, i) || _unsupportedIterableToArray(arr, i) || _nonIterableRest(); }
|
|
6
|
-
|
|
7
5
|
function _nonIterableRest() { throw new TypeError("Invalid attempt to destructure non-iterable instance.\nIn order to be iterable, non-array objects must have a [Symbol.iterator]() method."); }
|
|
8
|
-
|
|
9
6
|
function _unsupportedIterableToArray(o, minLen) { if (!o) return; if (typeof o === "string") return _arrayLikeToArray(o, minLen); var n = Object.prototype.toString.call(o).slice(8, -1); if (n === "Object" && o.constructor) n = o.constructor.name; if (n === "Map" || n === "Set") return Array.from(o); if (n === "Arguments" || /^(?:Ui|I)nt(?:8|16|32)(?:Clamped)?Array$/.test(n)) return _arrayLikeToArray(o, minLen); }
|
|
10
|
-
|
|
11
7
|
function _arrayLikeToArray(arr, len) { if (len == null || len > arr.length) len = arr.length; for (var i = 0, arr2 = new Array(len); i < len; i++) { arr2[i] = arr[i]; } return arr2; }
|
|
12
|
-
|
|
13
8
|
function _iterableToArrayLimit(arr, i) { var _i = arr == null ? null : typeof Symbol !== "undefined" && arr[Symbol.iterator] || arr["@@iterator"]; if (_i == null) return; var _arr = []; var _n = true; var _d = false; var _s, _e; try { for (_i = _i.call(arr); !(_n = (_s = _i.next()).done); _n = true) { _arr.push(_s.value); if (i && _arr.length === i) break; } } catch (err) { _d = true; _e = err; } finally { try { if (!_n && _i["return"] != null) _i["return"](); } finally { if (_d) throw _e; } } return _arr; }
|
|
14
|
-
|
|
15
9
|
function _arrayWithHoles(arr) { if (Array.isArray(arr)) return arr; }
|
|
16
|
-
|
|
17
10
|
function ownKeys(object, enumerableOnly) { var keys = Object.keys(object); if (Object.getOwnPropertySymbols) { var symbols = Object.getOwnPropertySymbols(object); enumerableOnly && (symbols = symbols.filter(function (sym) { return Object.getOwnPropertyDescriptor(object, sym).enumerable; })), keys.push.apply(keys, symbols); } return keys; }
|
|
18
|
-
|
|
19
11
|
function _objectSpread(target) { for (var i = 1; i < arguments.length; i++) { var source = null != arguments[i] ? arguments[i] : {}; i % 2 ? ownKeys(Object(source), !0).forEach(function (key) { _defineProperty(target, key, source[key]); }) : Object.getOwnPropertyDescriptors ? Object.defineProperties(target, Object.getOwnPropertyDescriptors(source)) : ownKeys(Object(source)).forEach(function (key) { Object.defineProperty(target, key, Object.getOwnPropertyDescriptor(source, key)); }); } return target; }
|
|
20
|
-
|
|
21
12
|
function _objectWithoutProperties(source, excluded) { if (source == null) return {}; var target = _objectWithoutPropertiesLoose(source, excluded); var key, i; if (Object.getOwnPropertySymbols) { var sourceSymbolKeys = Object.getOwnPropertySymbols(source); for (i = 0; i < sourceSymbolKeys.length; i++) { key = sourceSymbolKeys[i]; if (excluded.indexOf(key) >= 0) continue; if (!Object.prototype.propertyIsEnumerable.call(source, key)) continue; target[key] = source[key]; } } return target; }
|
|
22
|
-
|
|
23
13
|
function _objectWithoutPropertiesLoose(source, excluded) { if (source == null) return {}; var target = {}; var sourceKeys = Object.keys(source); var key, i; for (i = 0; i < sourceKeys.length; i++) { key = sourceKeys[i]; if (excluded.indexOf(key) >= 0) continue; target[key] = source[key]; } return target; }
|
|
24
|
-
|
|
25
14
|
function asyncGeneratorStep(gen, resolve, reject, _next, _throw, key, arg) { try { var info = gen[key](arg); var value = info.value; } catch (error) { reject(error); return; } if (info.done) { resolve(value); } else { Promise.resolve(value).then(_next, _throw); } }
|
|
26
|
-
|
|
27
15
|
function _asyncToGenerator(fn) { return function () { var self = this, args = arguments; return new Promise(function (resolve, reject) { var gen = fn.apply(self, args); function _next(value) { asyncGeneratorStep(gen, resolve, reject, _next, _throw, "next", value); } function _throw(err) { asyncGeneratorStep(gen, resolve, reject, _next, _throw, "throw", err); } _next(undefined); }); }; }
|
|
28
|
-
|
|
29
16
|
function _defineProperty(obj, key, value) { if (key in obj) { Object.defineProperty(obj, key, { value: value, enumerable: true, configurable: true, writable: true }); } else { obj[key] = value; } return obj; }
|
|
30
|
-
|
|
31
17
|
var path = require('path');
|
|
32
|
-
|
|
33
18
|
var crypto = require('crypto');
|
|
34
|
-
|
|
35
19
|
var _require = require('url'),
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
20
|
+
parseUrl = _require.parse,
|
|
21
|
+
formatUrl = _require.format;
|
|
39
22
|
var fse = require('fs-extra');
|
|
40
|
-
|
|
41
23
|
var miss = require('mississippi');
|
|
42
|
-
|
|
43
24
|
var PQueue = require('p-queue');
|
|
44
|
-
|
|
45
25
|
var _require2 = require('lodash'),
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
26
|
+
omit = _require2.omit,
|
|
27
|
+
noop = _require2.noop;
|
|
49
28
|
var pkg = require('../package.json');
|
|
50
|
-
|
|
51
29
|
var requestStream = require('./requestStream');
|
|
52
|
-
|
|
53
30
|
var debug = require('./debug');
|
|
54
|
-
|
|
55
31
|
var EXCLUDE_PROPS = ['_id', '_type', 'assetId', 'extension', 'mimeType', 'path', 'url'];
|
|
56
32
|
var ACTION_REMOVE = 'remove';
|
|
57
33
|
var ACTION_REWRITE = 'rewrite';
|
|
58
34
|
var ASSET_DOWNLOAD_CONCURRENCY = 8;
|
|
59
|
-
|
|
60
35
|
class AssetHandler {
|
|
61
36
|
constructor(options) {
|
|
62
37
|
var _this = this;
|
|
63
|
-
|
|
64
38
|
_defineProperty(this, "rewriteAssets", miss.through.obj( /*#__PURE__*/function () {
|
|
65
39
|
var _ref = _asyncToGenerator(function* (doc, enc, callback) {
|
|
66
40
|
if (['sanity.imageAsset', 'sanity.fileAsset'].includes(doc._type)) {
|
|
67
41
|
var type = doc._type === 'sanity.imageAsset' ? 'image' : 'file';
|
|
68
42
|
var filePath = "".concat(type, "s/").concat(generateFilename(doc._id));
|
|
69
|
-
|
|
70
43
|
_this.assetsSeen.set(doc._id, type);
|
|
71
|
-
|
|
72
44
|
_this.queueAssetDownload(doc, filePath, type);
|
|
73
|
-
|
|
74
45
|
callback();
|
|
75
46
|
return;
|
|
76
47
|
}
|
|
77
|
-
|
|
78
48
|
callback(null, _this.findAndModify(doc, ACTION_REWRITE));
|
|
79
49
|
});
|
|
80
|
-
|
|
81
50
|
return function (_x, _x2, _x3) {
|
|
82
51
|
return _ref.apply(this, arguments);
|
|
83
52
|
};
|
|
84
53
|
}()));
|
|
85
|
-
|
|
86
54
|
_defineProperty(this, "stripAssets", miss.through.obj( /*#__PURE__*/function () {
|
|
87
55
|
var _ref2 = _asyncToGenerator(function* (doc, enc, callback) {
|
|
88
56
|
if (['sanity.imageAsset', 'sanity.fileAsset'].includes(doc._type)) {
|
|
89
57
|
callback();
|
|
90
58
|
return;
|
|
91
59
|
}
|
|
92
|
-
|
|
93
60
|
callback(null, _this.findAndModify(doc, ACTION_REMOVE));
|
|
94
61
|
});
|
|
95
|
-
|
|
96
62
|
return function (_x4, _x5, _x6) {
|
|
97
63
|
return _ref2.apply(this, arguments);
|
|
98
64
|
};
|
|
99
65
|
}()));
|
|
100
|
-
|
|
101
66
|
_defineProperty(this, "skipAssets", miss.through.obj((doc, enc, callback) => {
|
|
102
67
|
var isAsset = ['sanity.imageAsset', 'sanity.fileAsset'].includes(doc._type);
|
|
103
|
-
|
|
104
68
|
if (isAsset) {
|
|
105
69
|
callback();
|
|
106
70
|
return;
|
|
107
71
|
}
|
|
108
|
-
|
|
109
72
|
callback(null, doc);
|
|
110
73
|
}));
|
|
111
|
-
|
|
112
74
|
_defineProperty(this, "noop", miss.through.obj((doc, enc, callback) => callback(null, doc)));
|
|
113
|
-
|
|
114
75
|
_defineProperty(this, "findAndModify", (item, action) => {
|
|
115
76
|
if (Array.isArray(item)) {
|
|
116
77
|
var children = item.map(child => this.findAndModify(child, action));
|
|
117
78
|
return children.filter(Boolean);
|
|
118
79
|
}
|
|
119
|
-
|
|
120
80
|
if (!item || typeof item !== 'object') {
|
|
121
81
|
return item;
|
|
122
82
|
}
|
|
123
|
-
|
|
124
83
|
var isAsset = isAssetField(item);
|
|
125
|
-
|
|
126
84
|
if (isAsset && action === ACTION_REMOVE) {
|
|
127
85
|
return undefined;
|
|
128
86
|
}
|
|
129
|
-
|
|
130
87
|
if (isAsset && action === ACTION_REWRITE) {
|
|
131
88
|
var asset = item.asset,
|
|
132
|
-
|
|
133
|
-
|
|
89
|
+
other = _objectWithoutProperties(item, _excluded);
|
|
134
90
|
var assetId = asset._ref;
|
|
135
91
|
var assetType = getAssetType(item);
|
|
136
92
|
var filePath = "".concat(assetType, "s/").concat(generateFilename(assetId));
|
|
@@ -138,23 +94,18 @@ class AssetHandler {
|
|
|
138
94
|
_sanityAsset: "".concat(assetType, "@file://./").concat(filePath)
|
|
139
95
|
}, this.findAndModify(other, action));
|
|
140
96
|
}
|
|
141
|
-
|
|
142
97
|
var newItem = {};
|
|
143
98
|
var keys = Object.keys(item);
|
|
144
|
-
|
|
145
99
|
for (var i = 0; i < keys.length; i++) {
|
|
146
100
|
var key = keys[i];
|
|
147
101
|
var value = item[key];
|
|
148
102
|
newItem[key] = this.findAndModify(value, action);
|
|
149
|
-
|
|
150
103
|
if (typeof newItem[key] === 'undefined') {
|
|
151
104
|
delete newItem[key];
|
|
152
105
|
}
|
|
153
106
|
}
|
|
154
|
-
|
|
155
107
|
return newItem;
|
|
156
108
|
});
|
|
157
|
-
|
|
158
109
|
var concurrency = options.concurrency || ASSET_DOWNLOAD_CONCURRENCY;
|
|
159
110
|
debug('Using asset download concurrency of %d', concurrency);
|
|
160
111
|
this.client = options.client;
|
|
@@ -169,58 +120,50 @@ class AssetHandler {
|
|
|
169
120
|
concurrency
|
|
170
121
|
});
|
|
171
122
|
this.rejectedError = null;
|
|
172
|
-
|
|
173
123
|
this.reject = err => {
|
|
174
124
|
this.rejectedError = err;
|
|
175
125
|
};
|
|
176
126
|
}
|
|
177
|
-
|
|
178
127
|
clear() {
|
|
179
128
|
this.assetsSeen.clear();
|
|
180
129
|
this.queue.clear();
|
|
181
130
|
this.queueSize = 0;
|
|
182
131
|
}
|
|
183
|
-
|
|
184
132
|
finish() {
|
|
185
133
|
return new Promise((resolve, reject) => {
|
|
186
134
|
if (this.rejectedError) {
|
|
187
135
|
reject(this.rejectedError);
|
|
188
136
|
return;
|
|
189
137
|
}
|
|
190
|
-
|
|
191
138
|
this.reject = reject;
|
|
192
139
|
this.queue.onIdle().then(() => resolve(this.assetMap));
|
|
193
140
|
});
|
|
194
|
-
}
|
|
195
|
-
// placeholder asset references (_sanityAsset: 'image@file:///local/path')
|
|
141
|
+
}
|
|
196
142
|
|
|
143
|
+
// Called when we want to download all assets to local filesystem and rewrite documents to hold
|
|
144
|
+
// placeholder asset references (_sanityAsset: 'image@file:///local/path')
|
|
197
145
|
|
|
198
146
|
queueAssetDownload(assetDoc, dstPath, type) {
|
|
199
147
|
if (!assetDoc.url) {
|
|
200
148
|
debug('Asset document "%s" does not have a URL property, skipping', assetDoc._id);
|
|
201
149
|
return;
|
|
202
150
|
}
|
|
203
|
-
|
|
204
151
|
debug('Adding download task for %s (destination: %s)', assetDoc._id, dstPath);
|
|
205
152
|
this.queueSize++;
|
|
206
153
|
this.downloading.push(assetDoc.url);
|
|
207
154
|
this.queue.add(() => this.downloadAsset(assetDoc, dstPath));
|
|
208
155
|
}
|
|
209
|
-
|
|
210
156
|
maybeCreateAssetDirs() {
|
|
211
157
|
if (this.assetDirsCreated) {
|
|
212
158
|
return;
|
|
213
159
|
}
|
|
214
|
-
/* eslint-disable no-sync */
|
|
215
|
-
|
|
216
160
|
|
|
161
|
+
/* eslint-disable no-sync */
|
|
217
162
|
fse.ensureDirSync(path.join(this.tmpDir, 'files'));
|
|
218
163
|
fse.ensureDirSync(path.join(this.tmpDir, 'images'));
|
|
219
164
|
/* eslint-enable no-sync */
|
|
220
|
-
|
|
221
165
|
this.assetDirsCreated = true;
|
|
222
166
|
}
|
|
223
|
-
|
|
224
167
|
getAssetRequestOptions(assetDoc) {
|
|
225
168
|
var token = this.client.config().token;
|
|
226
169
|
var headers = {
|
|
@@ -228,78 +171,62 @@ class AssetHandler {
|
|
|
228
171
|
};
|
|
229
172
|
var isImage = assetDoc._type === 'sanity.imageAsset';
|
|
230
173
|
var url = parseUrl(assetDoc.url, true);
|
|
231
|
-
|
|
232
174
|
if (isImage && ['cdn.sanity.io', 'cdn.sanity.work'].includes(url.hostname) && token) {
|
|
233
175
|
headers.Authorization = "Bearer ".concat(token);
|
|
234
176
|
url.query = _objectSpread(_objectSpread({}, url.query || {}), {}, {
|
|
235
177
|
dlRaw: 'true'
|
|
236
178
|
});
|
|
237
179
|
}
|
|
238
|
-
|
|
239
180
|
return {
|
|
240
181
|
url: formatUrl(url),
|
|
241
182
|
headers
|
|
242
183
|
};
|
|
243
184
|
}
|
|
244
|
-
|
|
245
185
|
downloadAsset(assetDoc, dstPath) {
|
|
246
186
|
var _arguments = arguments,
|
|
247
|
-
|
|
248
|
-
|
|
187
|
+
_this2 = this;
|
|
249
188
|
return _asyncToGenerator(function* () {
|
|
250
189
|
var attemptNum = _arguments.length > 2 && _arguments[2] !== undefined ? _arguments[2] : 0;
|
|
251
190
|
var url = assetDoc.url;
|
|
252
|
-
|
|
253
191
|
var options = _this2.getAssetRequestOptions(assetDoc);
|
|
254
|
-
|
|
255
192
|
var stream;
|
|
256
|
-
|
|
257
193
|
try {
|
|
258
194
|
stream = yield requestStream(options);
|
|
259
195
|
} catch (err) {
|
|
260
196
|
_this2.reject(err);
|
|
261
|
-
|
|
262
197
|
return false;
|
|
263
198
|
}
|
|
264
|
-
|
|
265
199
|
if (stream.statusCode !== 200) {
|
|
266
200
|
_this2.queue.clear();
|
|
267
|
-
|
|
268
201
|
var err = yield tryGetErrorFromStream(stream);
|
|
269
202
|
var errMsg = "Referenced asset URL \"".concat(url, "\" returned HTTP ").concat(stream.statusCode);
|
|
270
|
-
|
|
271
203
|
if (err) {
|
|
272
204
|
errMsg = "".concat(errMsg, ":\n\n").concat(err);
|
|
273
205
|
}
|
|
274
|
-
|
|
275
206
|
_this2.reject(new Error(errMsg));
|
|
276
|
-
|
|
277
207
|
return false;
|
|
278
208
|
}
|
|
279
|
-
|
|
280
209
|
_this2.maybeCreateAssetDirs();
|
|
281
|
-
|
|
282
210
|
debug('Asset stream ready, writing to filesystem at %s', dstPath);
|
|
283
211
|
var tmpPath = path.join(_this2.tmpDir, dstPath);
|
|
284
|
-
|
|
285
212
|
var _yield$writeHashedStr = yield writeHashedStream(tmpPath, stream),
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
213
|
+
sha1 = _yield$writeHashedStr.sha1,
|
|
214
|
+
md5 = _yield$writeHashedStr.md5,
|
|
215
|
+
size = _yield$writeHashedStr.size;
|
|
290
216
|
|
|
217
|
+
// Verify it against our downloaded stream to make sure we have the same copy
|
|
291
218
|
var contentLength = stream.headers['content-length'];
|
|
292
219
|
var remoteSha1 = stream.headers['x-sanity-sha1'];
|
|
293
220
|
var remoteMd5 = stream.headers['x-sanity-md5'];
|
|
294
221
|
var hasHash = Boolean(remoteSha1 || remoteMd5);
|
|
295
|
-
var method = sha1 ? 'sha1' : 'md5';
|
|
222
|
+
var method = sha1 ? 'sha1' : 'md5';
|
|
223
|
+
|
|
224
|
+
// Asset validity is primarily determined by the sha1 hash. However, the sha1 hash is computed
|
|
296
225
|
// before certain processes (i.e. svg sanitization) which can result in a different hash.
|
|
297
226
|
// When the sha1 hashes don't match, fallback to using the md5 hash.
|
|
298
|
-
|
|
299
227
|
var sha1Differs = remoteSha1 && sha1 !== remoteSha1;
|
|
300
228
|
var md5Differs = remoteMd5 && md5 !== remoteMd5;
|
|
301
229
|
var differs = sha1Differs && md5Differs;
|
|
302
|
-
|
|
303
230
|
if (differs && attemptNum < 3) {
|
|
304
231
|
debug('%s does not match downloaded asset, retrying (#%d) [%s]', method, attemptNum + 1, url);
|
|
305
232
|
return _this2.downloadAsset(assetDoc, dstPath, attemptNum + 1);
|
|
@@ -307,64 +234,47 @@ class AssetHandler {
|
|
|
307
234
|
var details = [hasHash && (method === 'md5' ? "md5 should be ".concat(remoteMd5, ", got ").concat(md5) : "sha1 should be ".concat(remoteSha1, ", got ").concat(sha1)), contentLength && parseInt(contentLength, 10) !== size && "Asset should be ".concat(contentLength, " bytes, got ").concat(size), "Did not succeed after ".concat(attemptNum, " attempts.")];
|
|
308
235
|
var detailsString = "Details:\n - ".concat(details.filter(Boolean).join('\n - '));
|
|
309
236
|
yield fse.unlink(tmpPath);
|
|
310
|
-
|
|
311
237
|
_this2.queue.clear();
|
|
312
|
-
|
|
313
238
|
var error = new Error("Failed to download asset at ".concat(assetDoc.url, ", giving up. ").concat(detailsString));
|
|
314
|
-
|
|
315
239
|
_this2.reject(error);
|
|
316
|
-
|
|
317
240
|
return false;
|
|
318
241
|
}
|
|
319
|
-
|
|
320
242
|
var isImage = assetDoc._type === 'sanity.imageAsset';
|
|
321
243
|
var type = isImage ? 'image' : 'file';
|
|
322
244
|
var id = "".concat(type, "-").concat(sha1);
|
|
323
245
|
var metaProps = omit(assetDoc, EXCLUDE_PROPS);
|
|
324
|
-
|
|
325
246
|
if (Object.keys(metaProps).length > 0) {
|
|
326
247
|
_this2.assetMap[id] = metaProps;
|
|
327
248
|
}
|
|
328
|
-
|
|
329
249
|
_this2.downloading.splice(_this2.downloading.findIndex(datUrl => datUrl === url), 1);
|
|
330
|
-
|
|
331
250
|
_this2.filesWritten++;
|
|
332
251
|
return true;
|
|
333
252
|
})();
|
|
334
253
|
}
|
|
335
|
-
|
|
336
254
|
}
|
|
337
|
-
|
|
338
255
|
function isAssetField(item) {
|
|
339
256
|
return item.asset && item.asset._ref && isSanityAsset(item.asset._ref);
|
|
340
257
|
}
|
|
341
|
-
|
|
342
258
|
function getAssetType(item) {
|
|
343
259
|
if (!item.asset || typeof item.asset._ref !== 'string') {
|
|
344
260
|
return null;
|
|
345
261
|
}
|
|
346
|
-
|
|
347
262
|
var _ref3 = item.asset._ref.match(/^(image|file)-/) || [],
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
|
|
263
|
+
_ref4 = _slicedToArray(_ref3, 2),
|
|
264
|
+
type = _ref4[1];
|
|
351
265
|
return type || null;
|
|
352
266
|
}
|
|
353
|
-
|
|
354
267
|
function isSanityAsset(assetId) {
|
|
355
268
|
return /^image-[a-f0-9]{40}-\d+x\d+-[a-z]+$/.test(assetId) || /^file-[a-f0-9]{40}-[a-z0-9]+$/.test(assetId);
|
|
356
269
|
}
|
|
357
|
-
|
|
358
270
|
function generateFilename(assetId) {
|
|
359
271
|
var _ref5 = assetId.match(/^(image|file)-(.*?)(-[a-z]+)?$/) || [],
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
272
|
+
_ref6 = _slicedToArray(_ref5, 4),
|
|
273
|
+
asset = _ref6[2],
|
|
274
|
+
ext = _ref6[3];
|
|
364
275
|
var extension = (ext || 'bin').replace(/^-/, '');
|
|
365
276
|
return asset ? "".concat(asset, ".").concat(extension) : "".concat(assetId, ".bin");
|
|
366
277
|
}
|
|
367
|
-
|
|
368
278
|
function writeHashedStream(filePath, stream) {
|
|
369
279
|
var size = 0;
|
|
370
280
|
var md5 = crypto.createHash('md5');
|
|
@@ -380,7 +290,6 @@ function writeHashedStream(filePath, stream) {
|
|
|
380
290
|
reject(err);
|
|
381
291
|
return;
|
|
382
292
|
}
|
|
383
|
-
|
|
384
293
|
resolve({
|
|
385
294
|
size,
|
|
386
295
|
sha1: sha1.digest('hex'),
|
|
@@ -388,11 +297,9 @@ function writeHashedStream(filePath, stream) {
|
|
|
388
297
|
});
|
|
389
298
|
}));
|
|
390
299
|
}
|
|
391
|
-
|
|
392
300
|
function tryGetErrorFromStream(stream) {
|
|
393
301
|
return new Promise((resolve, reject) => {
|
|
394
302
|
miss.pipe(stream, miss.concat(parse), err => err ? reject(err) : noop);
|
|
395
|
-
|
|
396
303
|
function parse(body) {
|
|
397
304
|
try {
|
|
398
305
|
var parsed = JSON.parse(body.toString('utf8'));
|
|
@@ -403,5 +310,4 @@ function tryGetErrorFromStream(stream) {
|
|
|
403
310
|
}
|
|
404
311
|
});
|
|
405
312
|
}
|
|
406
|
-
|
|
407
313
|
module.exports = AssetHandler;
|
package/lib/export.js
CHANGED
|
@@ -1,47 +1,26 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
|
|
3
3
|
function asyncGeneratorStep(gen, resolve, reject, _next, _throw, key, arg) { try { var info = gen[key](arg); var value = info.value; } catch (error) { reject(error); return; } if (info.done) { resolve(value); } else { Promise.resolve(value).then(_next, _throw); } }
|
|
4
|
-
|
|
5
4
|
function _asyncToGenerator(fn) { return function () { var self = this, args = arguments; return new Promise(function (resolve, reject) { var gen = fn.apply(self, args); function _next(value) { asyncGeneratorStep(gen, resolve, reject, _next, _throw, "next", value); } function _throw(err) { asyncGeneratorStep(gen, resolve, reject, _next, _throw, "throw", err); } _next(undefined); }); }; }
|
|
6
|
-
|
|
7
5
|
var os = require('os');
|
|
8
|
-
|
|
9
6
|
var path = require('path');
|
|
10
|
-
|
|
11
7
|
var zlib = require('zlib');
|
|
12
|
-
|
|
13
8
|
var fse = require('fs-extra');
|
|
14
|
-
|
|
15
9
|
var miss = require('mississippi');
|
|
16
|
-
|
|
17
10
|
var split = require('split2');
|
|
18
|
-
|
|
19
11
|
var archiver = require('archiver');
|
|
20
|
-
|
|
21
12
|
var debug = require('./debug');
|
|
22
|
-
|
|
23
13
|
var AssetHandler = require('./AssetHandler');
|
|
24
|
-
|
|
25
14
|
var stringifyStream = require('./stringifyStream');
|
|
26
|
-
|
|
27
15
|
var validateOptions = require('./validateOptions');
|
|
28
|
-
|
|
29
16
|
var rejectOnApiError = require('./rejectOnApiError');
|
|
30
|
-
|
|
31
17
|
var getDocumentsStream = require('./getDocumentsStream');
|
|
32
|
-
|
|
33
18
|
var filterSystemDocuments = require('./filterSystemDocuments');
|
|
34
|
-
|
|
35
19
|
var filterDocumentTypes = require('./filterDocumentTypes');
|
|
36
|
-
|
|
37
20
|
var filterDrafts = require('./filterDrafts');
|
|
38
|
-
|
|
39
21
|
var logFirstChunk = require('./logFirstChunk');
|
|
40
|
-
|
|
41
22
|
var tryParseJson = require('./tryParseJson');
|
|
42
|
-
|
|
43
23
|
var noop = () => null;
|
|
44
|
-
|
|
45
24
|
function exportDataset(opts) {
|
|
46
25
|
var options = validateOptions(opts);
|
|
47
26
|
var onProgress = options.onProgress || noop;
|
|
@@ -54,11 +33,9 @@ function exportDataset(opts) {
|
|
|
54
33
|
var slugDate = new Date().toISOString().replace(/[^a-z0-9]/gi, '-').toLowerCase();
|
|
55
34
|
var prefix = "".concat(opts.dataset, "-export-").concat(slugDate);
|
|
56
35
|
var tmpDir = path.join(os.tmpdir(), prefix);
|
|
57
|
-
|
|
58
36
|
var cleanup = () => fse.remove(tmpDir).catch(err => {
|
|
59
37
|
debug("Error while cleaning up temporary files: ".concat(err.message));
|
|
60
38
|
});
|
|
61
|
-
|
|
62
39
|
var assetHandler = new AssetHandler({
|
|
63
40
|
client: options.client,
|
|
64
41
|
tmpDir,
|
|
@@ -68,19 +45,15 @@ function exportDataset(opts) {
|
|
|
68
45
|
debug('Outputting assets (temporarily) to %s', tmpDir);
|
|
69
46
|
debug('Outputting to %s', options.outputPath === '-' ? 'stdout' : options.outputPath);
|
|
70
47
|
var outputStream;
|
|
71
|
-
|
|
72
48
|
if (isWritableStream(options.outputPath)) {
|
|
73
49
|
outputStream = options.outputPath;
|
|
74
50
|
} else {
|
|
75
51
|
outputStream = options.outputPath === '-' ? process.stdout : fse.createWriteStream(options.outputPath);
|
|
76
52
|
}
|
|
77
|
-
|
|
78
53
|
var assetStreamHandler = assetHandler.noop;
|
|
79
|
-
|
|
80
54
|
if (!options.raw) {
|
|
81
55
|
assetStreamHandler = options.assets ? assetHandler.rewriteAssets : assetHandler.stripAssets;
|
|
82
56
|
}
|
|
83
|
-
|
|
84
57
|
return new Promise( /*#__PURE__*/function () {
|
|
85
58
|
var _ref = _asyncToGenerator(function* (resolve, reject) {
|
|
86
59
|
miss.finished(archive, /*#__PURE__*/function () {
|
|
@@ -91,10 +64,8 @@ function exportDataset(opts) {
|
|
|
91
64
|
reject(archiveErr);
|
|
92
65
|
return;
|
|
93
66
|
}
|
|
94
|
-
|
|
95
67
|
debug('Archive finished!');
|
|
96
68
|
});
|
|
97
|
-
|
|
98
69
|
return function (_x3) {
|
|
99
70
|
return _ref2.apply(this, arguments);
|
|
100
71
|
};
|
|
@@ -105,11 +76,9 @@ function exportDataset(opts) {
|
|
|
105
76
|
});
|
|
106
77
|
var documentCount = 0;
|
|
107
78
|
var lastReported = Date.now();
|
|
108
|
-
|
|
109
79
|
var reportDocumentCount = (chunk, enc, cb) => {
|
|
110
80
|
++documentCount;
|
|
111
81
|
var now = Date.now();
|
|
112
|
-
|
|
113
82
|
if (now - lastReported > 50) {
|
|
114
83
|
onProgress({
|
|
115
84
|
step: 'Exporting documents...',
|
|
@@ -119,10 +88,8 @@ function exportDataset(opts) {
|
|
|
119
88
|
});
|
|
120
89
|
lastReported = now;
|
|
121
90
|
}
|
|
122
|
-
|
|
123
91
|
cb(null, chunk);
|
|
124
92
|
};
|
|
125
|
-
|
|
126
93
|
var inputStream = yield getDocumentsStream(options.client, options.dataset);
|
|
127
94
|
debug('Got HTTP %d', inputStream.statusCode);
|
|
128
95
|
debug('Response headers: %o', inputStream.headers);
|
|
@@ -132,28 +99,23 @@ function exportDataset(opts) {
|
|
|
132
99
|
if (err) {
|
|
133
100
|
return;
|
|
134
101
|
}
|
|
135
|
-
|
|
136
102
|
onProgress({
|
|
137
103
|
step: 'Exporting documents...',
|
|
138
104
|
current: documentCount,
|
|
139
105
|
total: documentCount,
|
|
140
106
|
update: true
|
|
141
107
|
});
|
|
142
|
-
|
|
143
108
|
if (!options.raw && options.assets) {
|
|
144
109
|
onProgress({
|
|
145
110
|
step: 'Downloading assets...'
|
|
146
111
|
});
|
|
147
112
|
}
|
|
148
|
-
|
|
149
113
|
var prevCompleted = 0;
|
|
150
114
|
var progressInterval = setInterval(() => {
|
|
151
115
|
var completed = assetHandler.queueSize - assetHandler.queue.size - assetHandler.queue.pending;
|
|
152
|
-
|
|
153
116
|
if (prevCompleted === completed) {
|
|
154
117
|
return;
|
|
155
118
|
}
|
|
156
|
-
|
|
157
119
|
prevCompleted = completed;
|
|
158
120
|
onProgress({
|
|
159
121
|
step: 'Downloading assets...',
|
|
@@ -163,10 +125,10 @@ function exportDataset(opts) {
|
|
|
163
125
|
});
|
|
164
126
|
}, 500);
|
|
165
127
|
debug('Waiting for asset handler to complete downloads');
|
|
166
|
-
|
|
167
128
|
try {
|
|
168
|
-
var assetMap = yield assetHandler.finish();
|
|
129
|
+
var assetMap = yield assetHandler.finish();
|
|
169
130
|
|
|
131
|
+
// Make sure we mark the progress as done (eg 100/100 instead of 99/100)
|
|
170
132
|
onProgress({
|
|
171
133
|
step: 'Downloading assets...',
|
|
172
134
|
current: assetHandler.queueSize,
|
|
@@ -183,9 +145,9 @@ function exportDataset(opts) {
|
|
|
183
145
|
yield cleanup();
|
|
184
146
|
reject(assetErr);
|
|
185
147
|
return;
|
|
186
|
-
}
|
|
187
|
-
|
|
148
|
+
}
|
|
188
149
|
|
|
150
|
+
// Add all downloaded assets to archive
|
|
189
151
|
archive.directory(path.join(tmpDir, 'files'), "".concat(prefix, "/files"), {
|
|
190
152
|
store: true
|
|
191
153
|
});
|
|
@@ -198,7 +160,6 @@ function exportDataset(opts) {
|
|
|
198
160
|
});
|
|
199
161
|
archive.finalize();
|
|
200
162
|
});
|
|
201
|
-
|
|
202
163
|
return function (_x4) {
|
|
203
164
|
return _ref3.apply(this, arguments);
|
|
204
165
|
};
|
|
@@ -211,23 +172,19 @@ function exportDataset(opts) {
|
|
|
211
172
|
prefix
|
|
212
173
|
});
|
|
213
174
|
miss.pipe(archive, outputStream, onComplete);
|
|
214
|
-
|
|
215
175
|
function onComplete(_x5) {
|
|
216
176
|
return _onComplete.apply(this, arguments);
|
|
217
177
|
}
|
|
218
|
-
|
|
219
178
|
function _onComplete() {
|
|
220
179
|
_onComplete = _asyncToGenerator(function* (err) {
|
|
221
180
|
onProgress({
|
|
222
181
|
step: 'Clearing temporary files...'
|
|
223
182
|
});
|
|
224
183
|
yield cleanup();
|
|
225
|
-
|
|
226
184
|
if (!err) {
|
|
227
185
|
resolve();
|
|
228
186
|
return;
|
|
229
187
|
}
|
|
230
|
-
|
|
231
188
|
debug('Error during streaming: %s', err.stack);
|
|
232
189
|
assetHandler.clear();
|
|
233
190
|
reject(err);
|
|
@@ -235,15 +192,12 @@ function exportDataset(opts) {
|
|
|
235
192
|
return _onComplete.apply(this, arguments);
|
|
236
193
|
}
|
|
237
194
|
});
|
|
238
|
-
|
|
239
195
|
return function (_x, _x2) {
|
|
240
196
|
return _ref.apply(this, arguments);
|
|
241
197
|
};
|
|
242
198
|
}());
|
|
243
199
|
}
|
|
244
|
-
|
|
245
200
|
function isWritableStream(val) {
|
|
246
201
|
return val !== null && typeof val === 'object' && typeof val.pipe === 'function' && typeof val._write === 'function' && typeof val._writableState === 'object';
|
|
247
202
|
}
|
|
248
|
-
|
|
249
203
|
module.exports = exportDataset;
|
|
@@ -1,14 +1,11 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
|
|
3
3
|
var miss = require('mississippi');
|
|
4
|
-
|
|
5
4
|
module.exports = allowedTypes => allowedTypes ? miss.through.obj((doc, enc, callback) => {
|
|
6
5
|
var type = doc && doc._type;
|
|
7
|
-
|
|
8
6
|
if (allowedTypes.includes(type)) {
|
|
9
7
|
callback(null, doc);
|
|
10
8
|
return;
|
|
11
9
|
}
|
|
12
|
-
|
|
13
10
|
callback();
|
|
14
11
|
}) : miss.through.obj();
|
package/lib/filterDrafts.js
CHANGED
|
@@ -1,13 +1,10 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
|
|
3
3
|
var miss = require('mississippi');
|
|
4
|
-
|
|
5
4
|
var isDraft = doc => doc && doc._id && doc._id.indexOf('drafts.') === 0;
|
|
6
|
-
|
|
7
5
|
module.exports = () => miss.through.obj((doc, enc, callback) => {
|
|
8
6
|
if (isDraft(doc)) {
|
|
9
7
|
return callback();
|
|
10
8
|
}
|
|
11
|
-
|
|
12
9
|
return callback(null, doc);
|
|
13
10
|
});
|
|
@@ -1,16 +1,12 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
|
|
3
3
|
var miss = require('mississippi');
|
|
4
|
-
|
|
5
4
|
var debug = require('./debug');
|
|
6
|
-
|
|
7
5
|
var isSystemDocument = doc => doc && doc._id && doc._id.indexOf('_.') === 0;
|
|
8
|
-
|
|
9
6
|
module.exports = () => miss.through.obj((doc, enc, callback) => {
|
|
10
7
|
if (isSystemDocument(doc)) {
|
|
11
8
|
debug('%s is a system document, skipping', doc && doc._id);
|
|
12
9
|
return callback();
|
|
13
10
|
}
|
|
14
|
-
|
|
15
11
|
return callback(null, doc);
|
|
16
12
|
});
|
|
@@ -1,27 +1,20 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
|
|
3
3
|
function ownKeys(object, enumerableOnly) { var keys = Object.keys(object); if (Object.getOwnPropertySymbols) { var symbols = Object.getOwnPropertySymbols(object); enumerableOnly && (symbols = symbols.filter(function (sym) { return Object.getOwnPropertyDescriptor(object, sym).enumerable; })), keys.push.apply(keys, symbols); } return keys; }
|
|
4
|
-
|
|
5
4
|
function _objectSpread(target) { for (var i = 1; i < arguments.length; i++) { var source = null != arguments[i] ? arguments[i] : {}; i % 2 ? ownKeys(Object(source), !0).forEach(function (key) { _defineProperty(target, key, source[key]); }) : Object.getOwnPropertyDescriptors ? Object.defineProperties(target, Object.getOwnPropertyDescriptors(source)) : ownKeys(Object(source)).forEach(function (key) { Object.defineProperty(target, key, Object.getOwnPropertyDescriptor(source, key)); }); } return target; }
|
|
6
|
-
|
|
7
5
|
function _defineProperty(obj, key, value) { if (key in obj) { Object.defineProperty(obj, key, { value: value, enumerable: true, configurable: true, writable: true }); } else { obj[key] = value; } return obj; }
|
|
8
|
-
|
|
9
6
|
var pkg = require('../package.json');
|
|
10
|
-
|
|
11
7
|
var requestStream = require('./requestStream');
|
|
12
|
-
|
|
13
8
|
module.exports = (client, dataset) => {
|
|
14
9
|
// Sanity client doesn't handle streams natively since we want to support node/browser
|
|
15
10
|
// with same API. We're just using it here to get hold of URLs and tokens.
|
|
16
11
|
var url = client.getUrl("/data/export/".concat(dataset));
|
|
17
12
|
var token = client.config().token;
|
|
18
|
-
|
|
19
13
|
var headers = _objectSpread({
|
|
20
14
|
'User-Agent': "".concat(pkg.name, "@").concat(pkg.version)
|
|
21
15
|
}, token ? {
|
|
22
16
|
Authorization: "Bearer ".concat(token)
|
|
23
17
|
} : {});
|
|
24
|
-
|
|
25
18
|
return requestStream({
|
|
26
19
|
url,
|
|
27
20
|
headers
|
package/lib/logFirstChunk.js
CHANGED
|
@@ -1,9 +1,7 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
|
|
3
3
|
var miss = require('mississippi');
|
|
4
|
-
|
|
5
4
|
var debug = require('./debug');
|
|
6
|
-
|
|
7
5
|
module.exports = () => {
|
|
8
6
|
var firstChunk = true;
|
|
9
7
|
return miss.through((chunk, enc, callback) => {
|
|
@@ -12,7 +10,6 @@ module.exports = () => {
|
|
|
12
10
|
debug('First chunk received: %s', string.slice(0, 300));
|
|
13
11
|
firstChunk = false;
|
|
14
12
|
}
|
|
15
|
-
|
|
16
13
|
callback(null, chunk);
|
|
17
14
|
});
|
|
18
15
|
};
|
package/lib/rejectOnApiError.js
CHANGED
|
@@ -1,17 +1,14 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
|
|
3
3
|
var miss = require('mississippi');
|
|
4
|
-
|
|
5
4
|
module.exports = () => miss.through.obj((doc, enc, callback) => {
|
|
6
5
|
if (doc.error && doc.statusCode) {
|
|
7
6
|
callback(new Error([doc.statusCode, doc.error].join(': ')));
|
|
8
7
|
return;
|
|
9
8
|
}
|
|
10
|
-
|
|
11
9
|
if (!doc._id && doc.error) {
|
|
12
10
|
callback(new Error(doc.error.description || doc.error.message || JSON.stringify(doc)));
|
|
13
11
|
return;
|
|
14
12
|
}
|
|
15
|
-
|
|
16
13
|
callback(null, doc);
|
|
17
14
|
});
|
package/lib/requestStream.js
CHANGED
|
@@ -1,43 +1,30 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
|
|
3
3
|
function ownKeys(object, enumerableOnly) { var keys = Object.keys(object); if (Object.getOwnPropertySymbols) { var symbols = Object.getOwnPropertySymbols(object); enumerableOnly && (symbols = symbols.filter(function (sym) { return Object.getOwnPropertyDescriptor(object, sym).enumerable; })), keys.push.apply(keys, symbols); } return keys; }
|
|
4
|
-
|
|
5
4
|
function _objectSpread(target) { for (var i = 1; i < arguments.length; i++) { var source = null != arguments[i] ? arguments[i] : {}; i % 2 ? ownKeys(Object(source), !0).forEach(function (key) { _defineProperty(target, key, source[key]); }) : Object.getOwnPropertyDescriptors ? Object.defineProperties(target, Object.getOwnPropertyDescriptors(source)) : ownKeys(Object(source)).forEach(function (key) { Object.defineProperty(target, key, Object.getOwnPropertyDescriptor(source, key)); }); } return target; }
|
|
6
|
-
|
|
7
5
|
function _defineProperty(obj, key, value) { if (key in obj) { Object.defineProperty(obj, key, { value: value, enumerable: true, configurable: true, writable: true }); } else { obj[key] = value; } return obj; }
|
|
8
|
-
|
|
9
6
|
function asyncGeneratorStep(gen, resolve, reject, _next, _throw, key, arg) { try { var info = gen[key](arg); var value = info.value; } catch (error) { reject(error); return; } if (info.done) { resolve(value); } else { Promise.resolve(value).then(_next, _throw); } }
|
|
10
|
-
|
|
11
7
|
function _asyncToGenerator(fn) { return function () { var self = this, args = arguments; return new Promise(function (resolve, reject) { var gen = fn.apply(self, args); function _next(value) { asyncGeneratorStep(gen, resolve, reject, _next, _throw, "next", value); } function _throw(err) { asyncGeneratorStep(gen, resolve, reject, _next, _throw, "throw", err); } _next(undefined); }); }; }
|
|
12
|
-
|
|
13
8
|
var getIt = require('get-it');
|
|
14
|
-
|
|
15
9
|
var _require = require('get-it/middleware'),
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
10
|
+
keepAlive = _require.keepAlive,
|
|
11
|
+
promise = _require.promise;
|
|
19
12
|
var debug = require('./debug');
|
|
20
|
-
|
|
21
13
|
var request = getIt([keepAlive(), promise({
|
|
22
14
|
onlyBody: true
|
|
23
15
|
})]);
|
|
24
16
|
var socketsWithTimeout = new WeakSet();
|
|
25
17
|
var CONNECTION_TIMEOUT = 15 * 1000; // 15 seconds
|
|
26
|
-
|
|
27
18
|
var READ_TIMEOUT = 3 * 60 * 1000; // 3 minutes
|
|
28
|
-
|
|
29
19
|
var MAX_RETRIES = 5;
|
|
30
|
-
|
|
31
20
|
function delay(ms) {
|
|
32
21
|
return new Promise(resolve => setTimeout(resolve, ms));
|
|
33
22
|
}
|
|
34
|
-
/* eslint-disable no-await-in-loop, max-depth */
|
|
35
|
-
|
|
36
23
|
|
|
24
|
+
/* eslint-disable no-await-in-loop, max-depth */
|
|
37
25
|
module.exports = /*#__PURE__*/function () {
|
|
38
26
|
var _ref = _asyncToGenerator(function* (options) {
|
|
39
27
|
var error;
|
|
40
|
-
|
|
41
28
|
for (var i = 0; i < MAX_RETRIES; i++) {
|
|
42
29
|
try {
|
|
43
30
|
var _ret = yield* function* () {
|
|
@@ -49,35 +36,28 @@ module.exports = /*#__PURE__*/function () {
|
|
|
49
36
|
socket: READ_TIMEOUT
|
|
50
37
|
}
|
|
51
38
|
}));
|
|
52
|
-
|
|
53
39
|
if (response.connection && typeof response.connection.setTimeout === 'function' && !socketsWithTimeout.has(response.connection)) {
|
|
54
40
|
socketsWithTimeout.add(response.connection);
|
|
55
41
|
response.connection.setTimeout(READ_TIMEOUT, () => {
|
|
56
42
|
response.destroy(new Error("Read timeout: No data received on socket for ".concat(READ_TIMEOUT, " ms")));
|
|
57
43
|
});
|
|
58
44
|
}
|
|
59
|
-
|
|
60
45
|
return {
|
|
61
46
|
v: response
|
|
62
47
|
};
|
|
63
48
|
}();
|
|
64
|
-
|
|
65
49
|
if (typeof _ret === "object") return _ret.v;
|
|
66
50
|
} catch (err) {
|
|
67
51
|
error = err;
|
|
68
|
-
|
|
69
52
|
if (err.response && err.response.statusCode && err.response.statusCode < 500) {
|
|
70
53
|
break;
|
|
71
54
|
}
|
|
72
|
-
|
|
73
55
|
debug('Error, retrying after 1500ms: %s', err.message);
|
|
74
56
|
yield delay(1500);
|
|
75
57
|
}
|
|
76
58
|
}
|
|
77
|
-
|
|
78
59
|
throw error;
|
|
79
60
|
});
|
|
80
|
-
|
|
81
61
|
return function (_x) {
|
|
82
62
|
return _ref.apply(this, arguments);
|
|
83
63
|
};
|
package/lib/stringifyStream.js
CHANGED
package/lib/tryParseJson.js
CHANGED
|
@@ -6,20 +6,16 @@ module.exports = line => {
|
|
|
6
6
|
} catch (err) {
|
|
7
7
|
// Catch half-done lines with an error at the end
|
|
8
8
|
var errorPosition = line.lastIndexOf('{"error":');
|
|
9
|
-
|
|
10
9
|
if (errorPosition === -1) {
|
|
11
10
|
err.message = "".concat(err.message, " (").concat(line, ")");
|
|
12
11
|
throw err;
|
|
13
12
|
}
|
|
14
|
-
|
|
15
13
|
var errorJson = line.slice(errorPosition);
|
|
16
14
|
var errorLine = JSON.parse(errorJson);
|
|
17
15
|
var error = errorLine && errorLine.error;
|
|
18
|
-
|
|
19
16
|
if (error && error.description) {
|
|
20
17
|
throw new Error("Error streaming dataset: ".concat(error.description, "\n\n").concat(errorJson, "\n"));
|
|
21
18
|
}
|
|
22
|
-
|
|
23
19
|
throw err;
|
|
24
20
|
}
|
|
25
21
|
};
|
package/lib/validateOptions.js
CHANGED
|
@@ -1,7 +1,6 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
|
|
3
3
|
var defaults = require('lodash/defaults');
|
|
4
|
-
|
|
5
4
|
var clientMethods = ['getUrl', 'config'];
|
|
6
5
|
var booleanFlags = ['assets', 'raw', 'compress', 'drafts'];
|
|
7
6
|
var exportDefaults = {
|
|
@@ -10,49 +9,36 @@ var exportDefaults = {
|
|
|
10
9
|
assets: true,
|
|
11
10
|
raw: false
|
|
12
11
|
};
|
|
13
|
-
|
|
14
12
|
function validateOptions(opts) {
|
|
15
13
|
var options = defaults({}, opts, exportDefaults);
|
|
16
|
-
|
|
17
14
|
if (typeof options.dataset !== 'string' || options.dataset.length < 1) {
|
|
18
15
|
throw new Error("options.dataset must be a valid dataset name");
|
|
19
16
|
}
|
|
20
|
-
|
|
21
17
|
if (options.onProgress && typeof options.onProgress !== 'function') {
|
|
22
18
|
throw new Error("options.onProgress must be a function");
|
|
23
19
|
}
|
|
24
|
-
|
|
25
20
|
if (!options.client) {
|
|
26
21
|
throw new Error('`options.client` must be set to an instance of @sanity/client');
|
|
27
22
|
}
|
|
28
|
-
|
|
29
23
|
var missing = clientMethods.find(key => typeof options.client[key] !== 'function');
|
|
30
|
-
|
|
31
24
|
if (missing) {
|
|
32
25
|
throw new Error("`options.client` is not a valid @sanity/client instance - no \"".concat(missing, "\" method found"));
|
|
33
26
|
}
|
|
34
|
-
|
|
35
27
|
var clientConfig = options.client.config();
|
|
36
|
-
|
|
37
28
|
if (!clientConfig.token) {
|
|
38
29
|
throw new Error('Client is not instantiated with a `token`');
|
|
39
30
|
}
|
|
40
|
-
|
|
41
31
|
booleanFlags.forEach(flag => {
|
|
42
32
|
if (typeof options[flag] !== 'boolean') {
|
|
43
33
|
throw new Error("Flag ".concat(flag, " must be a boolean (true/false)"));
|
|
44
34
|
}
|
|
45
35
|
});
|
|
46
|
-
|
|
47
36
|
if (!options.outputPath) {
|
|
48
37
|
throw new Error('outputPath must be specified (- for stdout)');
|
|
49
38
|
}
|
|
50
|
-
|
|
51
39
|
if (options.assetConcurrency && (options.assetConcurrency < 1 || options.assetConcurrency > 24)) {
|
|
52
40
|
throw new Error('`assetConcurrency` must be between 1 and 24');
|
|
53
41
|
}
|
|
54
|
-
|
|
55
42
|
return options;
|
|
56
43
|
}
|
|
57
|
-
|
|
58
44
|
module.exports = validateOptions;
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@sanity/export",
|
|
3
|
-
"version": "2.
|
|
3
|
+
"version": "2.34.1-canary.0+b3e58b8893",
|
|
4
4
|
"description": "Export Sanity documents and assets",
|
|
5
5
|
"main": "lib/export.js",
|
|
6
6
|
"engines": {
|
|
@@ -49,5 +49,5 @@
|
|
|
49
49
|
"url": "https://github.com/sanity-io/sanity/issues"
|
|
50
50
|
},
|
|
51
51
|
"homepage": "https://www.sanity.io/",
|
|
52
|
-
"gitHead": "
|
|
52
|
+
"gitHead": "b3e58b8893c9573844279488896df2313df914f1"
|
|
53
53
|
}
|