@sanity/export 2.34.1-canary.0 → 2.34.2-empty-template-cli.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/AssetHandler.js +116 -22
- package/lib/export.js +50 -4
- package/lib/filterDocumentTypes.js +3 -0
- package/lib/filterDrafts.js +3 -0
- package/lib/filterSystemDocuments.js +4 -0
- package/lib/getDocumentsStream.js +7 -0
- package/lib/logFirstChunk.js +3 -0
- package/lib/rejectOnApiError.js +3 -0
- package/lib/requestStream.js +23 -3
- package/lib/stringifyStream.js +1 -0
- package/lib/tryParseJson.js +4 -0
- package/lib/validateOptions.js +14 -0
- package/package.json +2 -2
package/lib/AssetHandler.js
CHANGED
|
@@ -1,92 +1,136 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
|
|
3
3
|
var _excluded = ["asset"];
|
|
4
|
+
|
|
4
5
|
function _slicedToArray(arr, i) { return _arrayWithHoles(arr) || _iterableToArrayLimit(arr, i) || _unsupportedIterableToArray(arr, i) || _nonIterableRest(); }
|
|
6
|
+
|
|
5
7
|
function _nonIterableRest() { throw new TypeError("Invalid attempt to destructure non-iterable instance.\nIn order to be iterable, non-array objects must have a [Symbol.iterator]() method."); }
|
|
8
|
+
|
|
6
9
|
function _unsupportedIterableToArray(o, minLen) { if (!o) return; if (typeof o === "string") return _arrayLikeToArray(o, minLen); var n = Object.prototype.toString.call(o).slice(8, -1); if (n === "Object" && o.constructor) n = o.constructor.name; if (n === "Map" || n === "Set") return Array.from(o); if (n === "Arguments" || /^(?:Ui|I)nt(?:8|16|32)(?:Clamped)?Array$/.test(n)) return _arrayLikeToArray(o, minLen); }
|
|
10
|
+
|
|
7
11
|
function _arrayLikeToArray(arr, len) { if (len == null || len > arr.length) len = arr.length; for (var i = 0, arr2 = new Array(len); i < len; i++) { arr2[i] = arr[i]; } return arr2; }
|
|
12
|
+
|
|
8
13
|
function _iterableToArrayLimit(arr, i) { var _i = arr == null ? null : typeof Symbol !== "undefined" && arr[Symbol.iterator] || arr["@@iterator"]; if (_i == null) return; var _arr = []; var _n = true; var _d = false; var _s, _e; try { for (_i = _i.call(arr); !(_n = (_s = _i.next()).done); _n = true) { _arr.push(_s.value); if (i && _arr.length === i) break; } } catch (err) { _d = true; _e = err; } finally { try { if (!_n && _i["return"] != null) _i["return"](); } finally { if (_d) throw _e; } } return _arr; }
|
|
14
|
+
|
|
9
15
|
function _arrayWithHoles(arr) { if (Array.isArray(arr)) return arr; }
|
|
16
|
+
|
|
10
17
|
function ownKeys(object, enumerableOnly) { var keys = Object.keys(object); if (Object.getOwnPropertySymbols) { var symbols = Object.getOwnPropertySymbols(object); enumerableOnly && (symbols = symbols.filter(function (sym) { return Object.getOwnPropertyDescriptor(object, sym).enumerable; })), keys.push.apply(keys, symbols); } return keys; }
|
|
18
|
+
|
|
11
19
|
function _objectSpread(target) { for (var i = 1; i < arguments.length; i++) { var source = null != arguments[i] ? arguments[i] : {}; i % 2 ? ownKeys(Object(source), !0).forEach(function (key) { _defineProperty(target, key, source[key]); }) : Object.getOwnPropertyDescriptors ? Object.defineProperties(target, Object.getOwnPropertyDescriptors(source)) : ownKeys(Object(source)).forEach(function (key) { Object.defineProperty(target, key, Object.getOwnPropertyDescriptor(source, key)); }); } return target; }
|
|
20
|
+
|
|
12
21
|
function _objectWithoutProperties(source, excluded) { if (source == null) return {}; var target = _objectWithoutPropertiesLoose(source, excluded); var key, i; if (Object.getOwnPropertySymbols) { var sourceSymbolKeys = Object.getOwnPropertySymbols(source); for (i = 0; i < sourceSymbolKeys.length; i++) { key = sourceSymbolKeys[i]; if (excluded.indexOf(key) >= 0) continue; if (!Object.prototype.propertyIsEnumerable.call(source, key)) continue; target[key] = source[key]; } } return target; }
|
|
22
|
+
|
|
13
23
|
function _objectWithoutPropertiesLoose(source, excluded) { if (source == null) return {}; var target = {}; var sourceKeys = Object.keys(source); var key, i; for (i = 0; i < sourceKeys.length; i++) { key = sourceKeys[i]; if (excluded.indexOf(key) >= 0) continue; target[key] = source[key]; } return target; }
|
|
24
|
+
|
|
14
25
|
function asyncGeneratorStep(gen, resolve, reject, _next, _throw, key, arg) { try { var info = gen[key](arg); var value = info.value; } catch (error) { reject(error); return; } if (info.done) { resolve(value); } else { Promise.resolve(value).then(_next, _throw); } }
|
|
26
|
+
|
|
15
27
|
function _asyncToGenerator(fn) { return function () { var self = this, args = arguments; return new Promise(function (resolve, reject) { var gen = fn.apply(self, args); function _next(value) { asyncGeneratorStep(gen, resolve, reject, _next, _throw, "next", value); } function _throw(err) { asyncGeneratorStep(gen, resolve, reject, _next, _throw, "throw", err); } _next(undefined); }); }; }
|
|
28
|
+
|
|
16
29
|
function _defineProperty(obj, key, value) { if (key in obj) { Object.defineProperty(obj, key, { value: value, enumerable: true, configurable: true, writable: true }); } else { obj[key] = value; } return obj; }
|
|
30
|
+
|
|
17
31
|
var path = require('path');
|
|
32
|
+
|
|
18
33
|
var crypto = require('crypto');
|
|
34
|
+
|
|
19
35
|
var _require = require('url'),
|
|
20
|
-
|
|
21
|
-
|
|
36
|
+
parseUrl = _require.parse,
|
|
37
|
+
formatUrl = _require.format;
|
|
38
|
+
|
|
22
39
|
var fse = require('fs-extra');
|
|
40
|
+
|
|
23
41
|
var miss = require('mississippi');
|
|
42
|
+
|
|
24
43
|
var PQueue = require('p-queue');
|
|
44
|
+
|
|
25
45
|
var _require2 = require('lodash'),
|
|
26
|
-
|
|
27
|
-
|
|
46
|
+
omit = _require2.omit,
|
|
47
|
+
noop = _require2.noop;
|
|
48
|
+
|
|
28
49
|
var pkg = require('../package.json');
|
|
50
|
+
|
|
29
51
|
var requestStream = require('./requestStream');
|
|
52
|
+
|
|
30
53
|
var debug = require('./debug');
|
|
54
|
+
|
|
31
55
|
var EXCLUDE_PROPS = ['_id', '_type', 'assetId', 'extension', 'mimeType', 'path', 'url'];
|
|
32
56
|
var ACTION_REMOVE = 'remove';
|
|
33
57
|
var ACTION_REWRITE = 'rewrite';
|
|
34
58
|
var ASSET_DOWNLOAD_CONCURRENCY = 8;
|
|
59
|
+
|
|
35
60
|
class AssetHandler {
|
|
36
61
|
constructor(options) {
|
|
37
62
|
var _this = this;
|
|
63
|
+
|
|
38
64
|
_defineProperty(this, "rewriteAssets", miss.through.obj( /*#__PURE__*/function () {
|
|
39
65
|
var _ref = _asyncToGenerator(function* (doc, enc, callback) {
|
|
40
66
|
if (['sanity.imageAsset', 'sanity.fileAsset'].includes(doc._type)) {
|
|
41
67
|
var type = doc._type === 'sanity.imageAsset' ? 'image' : 'file';
|
|
42
68
|
var filePath = "".concat(type, "s/").concat(generateFilename(doc._id));
|
|
69
|
+
|
|
43
70
|
_this.assetsSeen.set(doc._id, type);
|
|
71
|
+
|
|
44
72
|
_this.queueAssetDownload(doc, filePath, type);
|
|
73
|
+
|
|
45
74
|
callback();
|
|
46
75
|
return;
|
|
47
76
|
}
|
|
77
|
+
|
|
48
78
|
callback(null, _this.findAndModify(doc, ACTION_REWRITE));
|
|
49
79
|
});
|
|
80
|
+
|
|
50
81
|
return function (_x, _x2, _x3) {
|
|
51
82
|
return _ref.apply(this, arguments);
|
|
52
83
|
};
|
|
53
84
|
}()));
|
|
85
|
+
|
|
54
86
|
_defineProperty(this, "stripAssets", miss.through.obj( /*#__PURE__*/function () {
|
|
55
87
|
var _ref2 = _asyncToGenerator(function* (doc, enc, callback) {
|
|
56
88
|
if (['sanity.imageAsset', 'sanity.fileAsset'].includes(doc._type)) {
|
|
57
89
|
callback();
|
|
58
90
|
return;
|
|
59
91
|
}
|
|
92
|
+
|
|
60
93
|
callback(null, _this.findAndModify(doc, ACTION_REMOVE));
|
|
61
94
|
});
|
|
95
|
+
|
|
62
96
|
return function (_x4, _x5, _x6) {
|
|
63
97
|
return _ref2.apply(this, arguments);
|
|
64
98
|
};
|
|
65
99
|
}()));
|
|
100
|
+
|
|
66
101
|
_defineProperty(this, "skipAssets", miss.through.obj((doc, enc, callback) => {
|
|
67
102
|
var isAsset = ['sanity.imageAsset', 'sanity.fileAsset'].includes(doc._type);
|
|
103
|
+
|
|
68
104
|
if (isAsset) {
|
|
69
105
|
callback();
|
|
70
106
|
return;
|
|
71
107
|
}
|
|
108
|
+
|
|
72
109
|
callback(null, doc);
|
|
73
110
|
}));
|
|
111
|
+
|
|
74
112
|
_defineProperty(this, "noop", miss.through.obj((doc, enc, callback) => callback(null, doc)));
|
|
113
|
+
|
|
75
114
|
_defineProperty(this, "findAndModify", (item, action) => {
|
|
76
115
|
if (Array.isArray(item)) {
|
|
77
116
|
var children = item.map(child => this.findAndModify(child, action));
|
|
78
117
|
return children.filter(Boolean);
|
|
79
118
|
}
|
|
119
|
+
|
|
80
120
|
if (!item || typeof item !== 'object') {
|
|
81
121
|
return item;
|
|
82
122
|
}
|
|
123
|
+
|
|
83
124
|
var isAsset = isAssetField(item);
|
|
125
|
+
|
|
84
126
|
if (isAsset && action === ACTION_REMOVE) {
|
|
85
127
|
return undefined;
|
|
86
128
|
}
|
|
129
|
+
|
|
87
130
|
if (isAsset && action === ACTION_REWRITE) {
|
|
88
131
|
var asset = item.asset,
|
|
89
|
-
|
|
132
|
+
other = _objectWithoutProperties(item, _excluded);
|
|
133
|
+
|
|
90
134
|
var assetId = asset._ref;
|
|
91
135
|
var assetType = getAssetType(item);
|
|
92
136
|
var filePath = "".concat(assetType, "s/").concat(generateFilename(assetId));
|
|
@@ -94,18 +138,23 @@ class AssetHandler {
|
|
|
94
138
|
_sanityAsset: "".concat(assetType, "@file://./").concat(filePath)
|
|
95
139
|
}, this.findAndModify(other, action));
|
|
96
140
|
}
|
|
141
|
+
|
|
97
142
|
var newItem = {};
|
|
98
143
|
var keys = Object.keys(item);
|
|
144
|
+
|
|
99
145
|
for (var i = 0; i < keys.length; i++) {
|
|
100
146
|
var key = keys[i];
|
|
101
147
|
var value = item[key];
|
|
102
148
|
newItem[key] = this.findAndModify(value, action);
|
|
149
|
+
|
|
103
150
|
if (typeof newItem[key] === 'undefined') {
|
|
104
151
|
delete newItem[key];
|
|
105
152
|
}
|
|
106
153
|
}
|
|
154
|
+
|
|
107
155
|
return newItem;
|
|
108
156
|
});
|
|
157
|
+
|
|
109
158
|
var concurrency = options.concurrency || ASSET_DOWNLOAD_CONCURRENCY;
|
|
110
159
|
debug('Using asset download concurrency of %d', concurrency);
|
|
111
160
|
this.client = options.client;
|
|
@@ -120,50 +169,58 @@ class AssetHandler {
|
|
|
120
169
|
concurrency
|
|
121
170
|
});
|
|
122
171
|
this.rejectedError = null;
|
|
172
|
+
|
|
123
173
|
this.reject = err => {
|
|
124
174
|
this.rejectedError = err;
|
|
125
175
|
};
|
|
126
176
|
}
|
|
177
|
+
|
|
127
178
|
clear() {
|
|
128
179
|
this.assetsSeen.clear();
|
|
129
180
|
this.queue.clear();
|
|
130
181
|
this.queueSize = 0;
|
|
131
182
|
}
|
|
183
|
+
|
|
132
184
|
finish() {
|
|
133
185
|
return new Promise((resolve, reject) => {
|
|
134
186
|
if (this.rejectedError) {
|
|
135
187
|
reject(this.rejectedError);
|
|
136
188
|
return;
|
|
137
189
|
}
|
|
190
|
+
|
|
138
191
|
this.reject = reject;
|
|
139
192
|
this.queue.onIdle().then(() => resolve(this.assetMap));
|
|
140
193
|
});
|
|
141
|
-
}
|
|
142
|
-
|
|
143
|
-
// Called when we want to download all assets to local filesystem and rewrite documents to hold
|
|
194
|
+
} // Called when we want to download all assets to local filesystem and rewrite documents to hold
|
|
144
195
|
// placeholder asset references (_sanityAsset: 'image@file:///local/path')
|
|
145
196
|
|
|
197
|
+
|
|
146
198
|
queueAssetDownload(assetDoc, dstPath, type) {
|
|
147
199
|
if (!assetDoc.url) {
|
|
148
200
|
debug('Asset document "%s" does not have a URL property, skipping', assetDoc._id);
|
|
149
201
|
return;
|
|
150
202
|
}
|
|
203
|
+
|
|
151
204
|
debug('Adding download task for %s (destination: %s)', assetDoc._id, dstPath);
|
|
152
205
|
this.queueSize++;
|
|
153
206
|
this.downloading.push(assetDoc.url);
|
|
154
207
|
this.queue.add(() => this.downloadAsset(assetDoc, dstPath));
|
|
155
208
|
}
|
|
209
|
+
|
|
156
210
|
maybeCreateAssetDirs() {
|
|
157
211
|
if (this.assetDirsCreated) {
|
|
158
212
|
return;
|
|
159
213
|
}
|
|
160
|
-
|
|
161
214
|
/* eslint-disable no-sync */
|
|
215
|
+
|
|
216
|
+
|
|
162
217
|
fse.ensureDirSync(path.join(this.tmpDir, 'files'));
|
|
163
218
|
fse.ensureDirSync(path.join(this.tmpDir, 'images'));
|
|
164
219
|
/* eslint-enable no-sync */
|
|
220
|
+
|
|
165
221
|
this.assetDirsCreated = true;
|
|
166
222
|
}
|
|
223
|
+
|
|
167
224
|
getAssetRequestOptions(assetDoc) {
|
|
168
225
|
var token = this.client.config().token;
|
|
169
226
|
var headers = {
|
|
@@ -171,62 +228,78 @@ class AssetHandler {
|
|
|
171
228
|
};
|
|
172
229
|
var isImage = assetDoc._type === 'sanity.imageAsset';
|
|
173
230
|
var url = parseUrl(assetDoc.url, true);
|
|
231
|
+
|
|
174
232
|
if (isImage && ['cdn.sanity.io', 'cdn.sanity.work'].includes(url.hostname) && token) {
|
|
175
233
|
headers.Authorization = "Bearer ".concat(token);
|
|
176
234
|
url.query = _objectSpread(_objectSpread({}, url.query || {}), {}, {
|
|
177
235
|
dlRaw: 'true'
|
|
178
236
|
});
|
|
179
237
|
}
|
|
238
|
+
|
|
180
239
|
return {
|
|
181
240
|
url: formatUrl(url),
|
|
182
241
|
headers
|
|
183
242
|
};
|
|
184
243
|
}
|
|
244
|
+
|
|
185
245
|
downloadAsset(assetDoc, dstPath) {
|
|
186
246
|
var _arguments = arguments,
|
|
187
|
-
|
|
247
|
+
_this2 = this;
|
|
248
|
+
|
|
188
249
|
return _asyncToGenerator(function* () {
|
|
189
250
|
var attemptNum = _arguments.length > 2 && _arguments[2] !== undefined ? _arguments[2] : 0;
|
|
190
251
|
var url = assetDoc.url;
|
|
252
|
+
|
|
191
253
|
var options = _this2.getAssetRequestOptions(assetDoc);
|
|
254
|
+
|
|
192
255
|
var stream;
|
|
256
|
+
|
|
193
257
|
try {
|
|
194
258
|
stream = yield requestStream(options);
|
|
195
259
|
} catch (err) {
|
|
196
260
|
_this2.reject(err);
|
|
261
|
+
|
|
197
262
|
return false;
|
|
198
263
|
}
|
|
264
|
+
|
|
199
265
|
if (stream.statusCode !== 200) {
|
|
200
266
|
_this2.queue.clear();
|
|
267
|
+
|
|
201
268
|
var err = yield tryGetErrorFromStream(stream);
|
|
202
269
|
var errMsg = "Referenced asset URL \"".concat(url, "\" returned HTTP ").concat(stream.statusCode);
|
|
270
|
+
|
|
203
271
|
if (err) {
|
|
204
272
|
errMsg = "".concat(errMsg, ":\n\n").concat(err);
|
|
205
273
|
}
|
|
274
|
+
|
|
206
275
|
_this2.reject(new Error(errMsg));
|
|
276
|
+
|
|
207
277
|
return false;
|
|
208
278
|
}
|
|
279
|
+
|
|
209
280
|
_this2.maybeCreateAssetDirs();
|
|
281
|
+
|
|
210
282
|
debug('Asset stream ready, writing to filesystem at %s', dstPath);
|
|
211
283
|
var tmpPath = path.join(_this2.tmpDir, dstPath);
|
|
284
|
+
|
|
212
285
|
var _yield$writeHashedStr = yield writeHashedStream(tmpPath, stream),
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
286
|
+
sha1 = _yield$writeHashedStr.sha1,
|
|
287
|
+
md5 = _yield$writeHashedStr.md5,
|
|
288
|
+
size = _yield$writeHashedStr.size; // Verify it against our downloaded stream to make sure we have the same copy
|
|
289
|
+
|
|
216
290
|
|
|
217
|
-
// Verify it against our downloaded stream to make sure we have the same copy
|
|
218
291
|
var contentLength = stream.headers['content-length'];
|
|
219
292
|
var remoteSha1 = stream.headers['x-sanity-sha1'];
|
|
220
293
|
var remoteMd5 = stream.headers['x-sanity-md5'];
|
|
221
294
|
var hasHash = Boolean(remoteSha1 || remoteMd5);
|
|
222
|
-
var method = sha1 ? 'sha1' : 'md5';
|
|
223
|
-
|
|
224
|
-
// Asset validity is primarily determined by the sha1 hash. However, the sha1 hash is computed
|
|
295
|
+
var method = sha1 ? 'sha1' : 'md5'; // Asset validity is primarily determined by the sha1 hash. However, the sha1 hash is computed
|
|
225
296
|
// before certain processes (i.e. svg sanitization) which can result in a different hash.
|
|
226
297
|
// When the sha1 hashes don't match, fallback to using the md5 hash.
|
|
298
|
+
|
|
227
299
|
var sha1Differs = remoteSha1 && sha1 !== remoteSha1;
|
|
228
300
|
var md5Differs = remoteMd5 && md5 !== remoteMd5;
|
|
229
301
|
var differs = sha1Differs && md5Differs;
|
|
302
|
+
|
|
230
303
|
if (differs && attemptNum < 3) {
|
|
231
304
|
debug('%s does not match downloaded asset, retrying (#%d) [%s]', method, attemptNum + 1, url);
|
|
232
305
|
return _this2.downloadAsset(assetDoc, dstPath, attemptNum + 1);
|
|
@@ -234,47 +307,64 @@ class AssetHandler {
|
|
|
234
307
|
var details = [hasHash && (method === 'md5' ? "md5 should be ".concat(remoteMd5, ", got ").concat(md5) : "sha1 should be ".concat(remoteSha1, ", got ").concat(sha1)), contentLength && parseInt(contentLength, 10) !== size && "Asset should be ".concat(contentLength, " bytes, got ").concat(size), "Did not succeed after ".concat(attemptNum, " attempts.")];
|
|
235
308
|
var detailsString = "Details:\n - ".concat(details.filter(Boolean).join('\n - '));
|
|
236
309
|
yield fse.unlink(tmpPath);
|
|
310
|
+
|
|
237
311
|
_this2.queue.clear();
|
|
312
|
+
|
|
238
313
|
var error = new Error("Failed to download asset at ".concat(assetDoc.url, ", giving up. ").concat(detailsString));
|
|
314
|
+
|
|
239
315
|
_this2.reject(error);
|
|
316
|
+
|
|
240
317
|
return false;
|
|
241
318
|
}
|
|
319
|
+
|
|
242
320
|
var isImage = assetDoc._type === 'sanity.imageAsset';
|
|
243
321
|
var type = isImage ? 'image' : 'file';
|
|
244
322
|
var id = "".concat(type, "-").concat(sha1);
|
|
245
323
|
var metaProps = omit(assetDoc, EXCLUDE_PROPS);
|
|
324
|
+
|
|
246
325
|
if (Object.keys(metaProps).length > 0) {
|
|
247
326
|
_this2.assetMap[id] = metaProps;
|
|
248
327
|
}
|
|
328
|
+
|
|
249
329
|
_this2.downloading.splice(_this2.downloading.findIndex(datUrl => datUrl === url), 1);
|
|
330
|
+
|
|
250
331
|
_this2.filesWritten++;
|
|
251
332
|
return true;
|
|
252
333
|
})();
|
|
253
334
|
}
|
|
335
|
+
|
|
254
336
|
}
|
|
337
|
+
|
|
255
338
|
function isAssetField(item) {
|
|
256
339
|
return item.asset && item.asset._ref && isSanityAsset(item.asset._ref);
|
|
257
340
|
}
|
|
341
|
+
|
|
258
342
|
function getAssetType(item) {
|
|
259
343
|
if (!item.asset || typeof item.asset._ref !== 'string') {
|
|
260
344
|
return null;
|
|
261
345
|
}
|
|
346
|
+
|
|
262
347
|
var _ref3 = item.asset._ref.match(/^(image|file)-/) || [],
|
|
263
|
-
|
|
264
|
-
|
|
348
|
+
_ref4 = _slicedToArray(_ref3, 2),
|
|
349
|
+
type = _ref4[1];
|
|
350
|
+
|
|
265
351
|
return type || null;
|
|
266
352
|
}
|
|
353
|
+
|
|
267
354
|
function isSanityAsset(assetId) {
|
|
268
355
|
return /^image-[a-f0-9]{40}-\d+x\d+-[a-z]+$/.test(assetId) || /^file-[a-f0-9]{40}-[a-z0-9]+$/.test(assetId);
|
|
269
356
|
}
|
|
357
|
+
|
|
270
358
|
function generateFilename(assetId) {
|
|
271
359
|
var _ref5 = assetId.match(/^(image|file)-(.*?)(-[a-z]+)?$/) || [],
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
360
|
+
_ref6 = _slicedToArray(_ref5, 4),
|
|
361
|
+
asset = _ref6[2],
|
|
362
|
+
ext = _ref6[3];
|
|
363
|
+
|
|
275
364
|
var extension = (ext || 'bin').replace(/^-/, '');
|
|
276
365
|
return asset ? "".concat(asset, ".").concat(extension) : "".concat(assetId, ".bin");
|
|
277
366
|
}
|
|
367
|
+
|
|
278
368
|
function writeHashedStream(filePath, stream) {
|
|
279
369
|
var size = 0;
|
|
280
370
|
var md5 = crypto.createHash('md5');
|
|
@@ -290,6 +380,7 @@ function writeHashedStream(filePath, stream) {
|
|
|
290
380
|
reject(err);
|
|
291
381
|
return;
|
|
292
382
|
}
|
|
383
|
+
|
|
293
384
|
resolve({
|
|
294
385
|
size,
|
|
295
386
|
sha1: sha1.digest('hex'),
|
|
@@ -297,9 +388,11 @@ function writeHashedStream(filePath, stream) {
|
|
|
297
388
|
});
|
|
298
389
|
}));
|
|
299
390
|
}
|
|
391
|
+
|
|
300
392
|
function tryGetErrorFromStream(stream) {
|
|
301
393
|
return new Promise((resolve, reject) => {
|
|
302
394
|
miss.pipe(stream, miss.concat(parse), err => err ? reject(err) : noop);
|
|
395
|
+
|
|
303
396
|
function parse(body) {
|
|
304
397
|
try {
|
|
305
398
|
var parsed = JSON.parse(body.toString('utf8'));
|
|
@@ -310,4 +403,5 @@ function tryGetErrorFromStream(stream) {
|
|
|
310
403
|
}
|
|
311
404
|
});
|
|
312
405
|
}
|
|
406
|
+
|
|
313
407
|
module.exports = AssetHandler;
|
package/lib/export.js
CHANGED
|
@@ -1,26 +1,47 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
|
|
3
3
|
function asyncGeneratorStep(gen, resolve, reject, _next, _throw, key, arg) { try { var info = gen[key](arg); var value = info.value; } catch (error) { reject(error); return; } if (info.done) { resolve(value); } else { Promise.resolve(value).then(_next, _throw); } }
|
|
4
|
+
|
|
4
5
|
function _asyncToGenerator(fn) { return function () { var self = this, args = arguments; return new Promise(function (resolve, reject) { var gen = fn.apply(self, args); function _next(value) { asyncGeneratorStep(gen, resolve, reject, _next, _throw, "next", value); } function _throw(err) { asyncGeneratorStep(gen, resolve, reject, _next, _throw, "throw", err); } _next(undefined); }); }; }
|
|
6
|
+
|
|
5
7
|
var os = require('os');
|
|
8
|
+
|
|
6
9
|
var path = require('path');
|
|
10
|
+
|
|
7
11
|
var zlib = require('zlib');
|
|
12
|
+
|
|
8
13
|
var fse = require('fs-extra');
|
|
14
|
+
|
|
9
15
|
var miss = require('mississippi');
|
|
16
|
+
|
|
10
17
|
var split = require('split2');
|
|
18
|
+
|
|
11
19
|
var archiver = require('archiver');
|
|
20
|
+
|
|
12
21
|
var debug = require('./debug');
|
|
22
|
+
|
|
13
23
|
var AssetHandler = require('./AssetHandler');
|
|
24
|
+
|
|
14
25
|
var stringifyStream = require('./stringifyStream');
|
|
26
|
+
|
|
15
27
|
var validateOptions = require('./validateOptions');
|
|
28
|
+
|
|
16
29
|
var rejectOnApiError = require('./rejectOnApiError');
|
|
30
|
+
|
|
17
31
|
var getDocumentsStream = require('./getDocumentsStream');
|
|
32
|
+
|
|
18
33
|
var filterSystemDocuments = require('./filterSystemDocuments');
|
|
34
|
+
|
|
19
35
|
var filterDocumentTypes = require('./filterDocumentTypes');
|
|
36
|
+
|
|
20
37
|
var filterDrafts = require('./filterDrafts');
|
|
38
|
+
|
|
21
39
|
var logFirstChunk = require('./logFirstChunk');
|
|
40
|
+
|
|
22
41
|
var tryParseJson = require('./tryParseJson');
|
|
42
|
+
|
|
23
43
|
var noop = () => null;
|
|
44
|
+
|
|
24
45
|
function exportDataset(opts) {
|
|
25
46
|
var options = validateOptions(opts);
|
|
26
47
|
var onProgress = options.onProgress || noop;
|
|
@@ -33,9 +54,11 @@ function exportDataset(opts) {
|
|
|
33
54
|
var slugDate = new Date().toISOString().replace(/[^a-z0-9]/gi, '-').toLowerCase();
|
|
34
55
|
var prefix = "".concat(opts.dataset, "-export-").concat(slugDate);
|
|
35
56
|
var tmpDir = path.join(os.tmpdir(), prefix);
|
|
57
|
+
|
|
36
58
|
var cleanup = () => fse.remove(tmpDir).catch(err => {
|
|
37
59
|
debug("Error while cleaning up temporary files: ".concat(err.message));
|
|
38
60
|
});
|
|
61
|
+
|
|
39
62
|
var assetHandler = new AssetHandler({
|
|
40
63
|
client: options.client,
|
|
41
64
|
tmpDir,
|
|
@@ -45,15 +68,19 @@ function exportDataset(opts) {
|
|
|
45
68
|
debug('Outputting assets (temporarily) to %s', tmpDir);
|
|
46
69
|
debug('Outputting to %s', options.outputPath === '-' ? 'stdout' : options.outputPath);
|
|
47
70
|
var outputStream;
|
|
71
|
+
|
|
48
72
|
if (isWritableStream(options.outputPath)) {
|
|
49
73
|
outputStream = options.outputPath;
|
|
50
74
|
} else {
|
|
51
75
|
outputStream = options.outputPath === '-' ? process.stdout : fse.createWriteStream(options.outputPath);
|
|
52
76
|
}
|
|
77
|
+
|
|
53
78
|
var assetStreamHandler = assetHandler.noop;
|
|
79
|
+
|
|
54
80
|
if (!options.raw) {
|
|
55
81
|
assetStreamHandler = options.assets ? assetHandler.rewriteAssets : assetHandler.stripAssets;
|
|
56
82
|
}
|
|
83
|
+
|
|
57
84
|
return new Promise( /*#__PURE__*/function () {
|
|
58
85
|
var _ref = _asyncToGenerator(function* (resolve, reject) {
|
|
59
86
|
miss.finished(archive, /*#__PURE__*/function () {
|
|
@@ -64,8 +91,10 @@ function exportDataset(opts) {
|
|
|
64
91
|
reject(archiveErr);
|
|
65
92
|
return;
|
|
66
93
|
}
|
|
94
|
+
|
|
67
95
|
debug('Archive finished!');
|
|
68
96
|
});
|
|
97
|
+
|
|
69
98
|
return function (_x3) {
|
|
70
99
|
return _ref2.apply(this, arguments);
|
|
71
100
|
};
|
|
@@ -76,9 +105,11 @@ function exportDataset(opts) {
|
|
|
76
105
|
});
|
|
77
106
|
var documentCount = 0;
|
|
78
107
|
var lastReported = Date.now();
|
|
108
|
+
|
|
79
109
|
var reportDocumentCount = (chunk, enc, cb) => {
|
|
80
110
|
++documentCount;
|
|
81
111
|
var now = Date.now();
|
|
112
|
+
|
|
82
113
|
if (now - lastReported > 50) {
|
|
83
114
|
onProgress({
|
|
84
115
|
step: 'Exporting documents...',
|
|
@@ -88,8 +119,10 @@ function exportDataset(opts) {
|
|
|
88
119
|
});
|
|
89
120
|
lastReported = now;
|
|
90
121
|
}
|
|
122
|
+
|
|
91
123
|
cb(null, chunk);
|
|
92
124
|
};
|
|
125
|
+
|
|
93
126
|
var inputStream = yield getDocumentsStream(options.client, options.dataset);
|
|
94
127
|
debug('Got HTTP %d', inputStream.statusCode);
|
|
95
128
|
debug('Response headers: %o', inputStream.headers);
|
|
@@ -99,23 +132,28 @@ function exportDataset(opts) {
|
|
|
99
132
|
if (err) {
|
|
100
133
|
return;
|
|
101
134
|
}
|
|
135
|
+
|
|
102
136
|
onProgress({
|
|
103
137
|
step: 'Exporting documents...',
|
|
104
138
|
current: documentCount,
|
|
105
139
|
total: documentCount,
|
|
106
140
|
update: true
|
|
107
141
|
});
|
|
142
|
+
|
|
108
143
|
if (!options.raw && options.assets) {
|
|
109
144
|
onProgress({
|
|
110
145
|
step: 'Downloading assets...'
|
|
111
146
|
});
|
|
112
147
|
}
|
|
148
|
+
|
|
113
149
|
var prevCompleted = 0;
|
|
114
150
|
var progressInterval = setInterval(() => {
|
|
115
151
|
var completed = assetHandler.queueSize - assetHandler.queue.size - assetHandler.queue.pending;
|
|
152
|
+
|
|
116
153
|
if (prevCompleted === completed) {
|
|
117
154
|
return;
|
|
118
155
|
}
|
|
156
|
+
|
|
119
157
|
prevCompleted = completed;
|
|
120
158
|
onProgress({
|
|
121
159
|
step: 'Downloading assets...',
|
|
@@ -125,10 +163,10 @@ function exportDataset(opts) {
|
|
|
125
163
|
});
|
|
126
164
|
}, 500);
|
|
127
165
|
debug('Waiting for asset handler to complete downloads');
|
|
166
|
+
|
|
128
167
|
try {
|
|
129
|
-
var assetMap = yield assetHandler.finish();
|
|
168
|
+
var assetMap = yield assetHandler.finish(); // Make sure we mark the progress as done (eg 100/100 instead of 99/100)
|
|
130
169
|
|
|
131
|
-
// Make sure we mark the progress as done (eg 100/100 instead of 99/100)
|
|
132
170
|
onProgress({
|
|
133
171
|
step: 'Downloading assets...',
|
|
134
172
|
current: assetHandler.queueSize,
|
|
@@ -145,9 +183,9 @@ function exportDataset(opts) {
|
|
|
145
183
|
yield cleanup();
|
|
146
184
|
reject(assetErr);
|
|
147
185
|
return;
|
|
148
|
-
}
|
|
186
|
+
} // Add all downloaded assets to archive
|
|
187
|
+
|
|
149
188
|
|
|
150
|
-
// Add all downloaded assets to archive
|
|
151
189
|
archive.directory(path.join(tmpDir, 'files'), "".concat(prefix, "/files"), {
|
|
152
190
|
store: true
|
|
153
191
|
});
|
|
@@ -160,6 +198,7 @@ function exportDataset(opts) {
|
|
|
160
198
|
});
|
|
161
199
|
archive.finalize();
|
|
162
200
|
});
|
|
201
|
+
|
|
163
202
|
return function (_x4) {
|
|
164
203
|
return _ref3.apply(this, arguments);
|
|
165
204
|
};
|
|
@@ -172,19 +211,23 @@ function exportDataset(opts) {
|
|
|
172
211
|
prefix
|
|
173
212
|
});
|
|
174
213
|
miss.pipe(archive, outputStream, onComplete);
|
|
214
|
+
|
|
175
215
|
function onComplete(_x5) {
|
|
176
216
|
return _onComplete.apply(this, arguments);
|
|
177
217
|
}
|
|
218
|
+
|
|
178
219
|
function _onComplete() {
|
|
179
220
|
_onComplete = _asyncToGenerator(function* (err) {
|
|
180
221
|
onProgress({
|
|
181
222
|
step: 'Clearing temporary files...'
|
|
182
223
|
});
|
|
183
224
|
yield cleanup();
|
|
225
|
+
|
|
184
226
|
if (!err) {
|
|
185
227
|
resolve();
|
|
186
228
|
return;
|
|
187
229
|
}
|
|
230
|
+
|
|
188
231
|
debug('Error during streaming: %s', err.stack);
|
|
189
232
|
assetHandler.clear();
|
|
190
233
|
reject(err);
|
|
@@ -192,12 +235,15 @@ function exportDataset(opts) {
|
|
|
192
235
|
return _onComplete.apply(this, arguments);
|
|
193
236
|
}
|
|
194
237
|
});
|
|
238
|
+
|
|
195
239
|
return function (_x, _x2) {
|
|
196
240
|
return _ref.apply(this, arguments);
|
|
197
241
|
};
|
|
198
242
|
}());
|
|
199
243
|
}
|
|
244
|
+
|
|
200
245
|
function isWritableStream(val) {
|
|
201
246
|
return val !== null && typeof val === 'object' && typeof val.pipe === 'function' && typeof val._write === 'function' && typeof val._writableState === 'object';
|
|
202
247
|
}
|
|
248
|
+
|
|
203
249
|
module.exports = exportDataset;
|
|
@@ -1,11 +1,14 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
|
|
3
3
|
var miss = require('mississippi');
|
|
4
|
+
|
|
4
5
|
module.exports = allowedTypes => allowedTypes ? miss.through.obj((doc, enc, callback) => {
|
|
5
6
|
var type = doc && doc._type;
|
|
7
|
+
|
|
6
8
|
if (allowedTypes.includes(type)) {
|
|
7
9
|
callback(null, doc);
|
|
8
10
|
return;
|
|
9
11
|
}
|
|
12
|
+
|
|
10
13
|
callback();
|
|
11
14
|
}) : miss.through.obj();
|
package/lib/filterDrafts.js
CHANGED
|
@@ -1,10 +1,13 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
|
|
3
3
|
var miss = require('mississippi');
|
|
4
|
+
|
|
4
5
|
var isDraft = doc => doc && doc._id && doc._id.indexOf('drafts.') === 0;
|
|
6
|
+
|
|
5
7
|
module.exports = () => miss.through.obj((doc, enc, callback) => {
|
|
6
8
|
if (isDraft(doc)) {
|
|
7
9
|
return callback();
|
|
8
10
|
}
|
|
11
|
+
|
|
9
12
|
return callback(null, doc);
|
|
10
13
|
});
|
|
@@ -1,12 +1,16 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
|
|
3
3
|
var miss = require('mississippi');
|
|
4
|
+
|
|
4
5
|
var debug = require('./debug');
|
|
6
|
+
|
|
5
7
|
var isSystemDocument = doc => doc && doc._id && doc._id.indexOf('_.') === 0;
|
|
8
|
+
|
|
6
9
|
module.exports = () => miss.through.obj((doc, enc, callback) => {
|
|
7
10
|
if (isSystemDocument(doc)) {
|
|
8
11
|
debug('%s is a system document, skipping', doc && doc._id);
|
|
9
12
|
return callback();
|
|
10
13
|
}
|
|
14
|
+
|
|
11
15
|
return callback(null, doc);
|
|
12
16
|
});
|
|
@@ -1,20 +1,27 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
|
|
3
3
|
function ownKeys(object, enumerableOnly) { var keys = Object.keys(object); if (Object.getOwnPropertySymbols) { var symbols = Object.getOwnPropertySymbols(object); enumerableOnly && (symbols = symbols.filter(function (sym) { return Object.getOwnPropertyDescriptor(object, sym).enumerable; })), keys.push.apply(keys, symbols); } return keys; }
|
|
4
|
+
|
|
4
5
|
function _objectSpread(target) { for (var i = 1; i < arguments.length; i++) { var source = null != arguments[i] ? arguments[i] : {}; i % 2 ? ownKeys(Object(source), !0).forEach(function (key) { _defineProperty(target, key, source[key]); }) : Object.getOwnPropertyDescriptors ? Object.defineProperties(target, Object.getOwnPropertyDescriptors(source)) : ownKeys(Object(source)).forEach(function (key) { Object.defineProperty(target, key, Object.getOwnPropertyDescriptor(source, key)); }); } return target; }
|
|
6
|
+
|
|
5
7
|
function _defineProperty(obj, key, value) { if (key in obj) { Object.defineProperty(obj, key, { value: value, enumerable: true, configurable: true, writable: true }); } else { obj[key] = value; } return obj; }
|
|
8
|
+
|
|
6
9
|
var pkg = require('../package.json');
|
|
10
|
+
|
|
7
11
|
var requestStream = require('./requestStream');
|
|
12
|
+
|
|
8
13
|
module.exports = (client, dataset) => {
|
|
9
14
|
// Sanity client doesn't handle streams natively since we want to support node/browser
|
|
10
15
|
// with same API. We're just using it here to get hold of URLs and tokens.
|
|
11
16
|
var url = client.getUrl("/data/export/".concat(dataset));
|
|
12
17
|
var token = client.config().token;
|
|
18
|
+
|
|
13
19
|
var headers = _objectSpread({
|
|
14
20
|
'User-Agent': "".concat(pkg.name, "@").concat(pkg.version)
|
|
15
21
|
}, token ? {
|
|
16
22
|
Authorization: "Bearer ".concat(token)
|
|
17
23
|
} : {});
|
|
24
|
+
|
|
18
25
|
return requestStream({
|
|
19
26
|
url,
|
|
20
27
|
headers
|
package/lib/logFirstChunk.js
CHANGED
|
@@ -1,7 +1,9 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
|
|
3
3
|
var miss = require('mississippi');
|
|
4
|
+
|
|
4
5
|
var debug = require('./debug');
|
|
6
|
+
|
|
5
7
|
module.exports = () => {
|
|
6
8
|
var firstChunk = true;
|
|
7
9
|
return miss.through((chunk, enc, callback) => {
|
|
@@ -10,6 +12,7 @@ module.exports = () => {
|
|
|
10
12
|
debug('First chunk received: %s', string.slice(0, 300));
|
|
11
13
|
firstChunk = false;
|
|
12
14
|
}
|
|
15
|
+
|
|
13
16
|
callback(null, chunk);
|
|
14
17
|
});
|
|
15
18
|
};
|
package/lib/rejectOnApiError.js
CHANGED
|
@@ -1,14 +1,17 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
|
|
3
3
|
var miss = require('mississippi');
|
|
4
|
+
|
|
4
5
|
module.exports = () => miss.through.obj((doc, enc, callback) => {
|
|
5
6
|
if (doc.error && doc.statusCode) {
|
|
6
7
|
callback(new Error([doc.statusCode, doc.error].join(': ')));
|
|
7
8
|
return;
|
|
8
9
|
}
|
|
10
|
+
|
|
9
11
|
if (!doc._id && doc.error) {
|
|
10
12
|
callback(new Error(doc.error.description || doc.error.message || JSON.stringify(doc)));
|
|
11
13
|
return;
|
|
12
14
|
}
|
|
15
|
+
|
|
13
16
|
callback(null, doc);
|
|
14
17
|
});
|
package/lib/requestStream.js
CHANGED
|
@@ -1,30 +1,43 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
|
|
3
3
|
function ownKeys(object, enumerableOnly) { var keys = Object.keys(object); if (Object.getOwnPropertySymbols) { var symbols = Object.getOwnPropertySymbols(object); enumerableOnly && (symbols = symbols.filter(function (sym) { return Object.getOwnPropertyDescriptor(object, sym).enumerable; })), keys.push.apply(keys, symbols); } return keys; }
|
|
4
|
+
|
|
4
5
|
function _objectSpread(target) { for (var i = 1; i < arguments.length; i++) { var source = null != arguments[i] ? arguments[i] : {}; i % 2 ? ownKeys(Object(source), !0).forEach(function (key) { _defineProperty(target, key, source[key]); }) : Object.getOwnPropertyDescriptors ? Object.defineProperties(target, Object.getOwnPropertyDescriptors(source)) : ownKeys(Object(source)).forEach(function (key) { Object.defineProperty(target, key, Object.getOwnPropertyDescriptor(source, key)); }); } return target; }
|
|
6
|
+
|
|
5
7
|
function _defineProperty(obj, key, value) { if (key in obj) { Object.defineProperty(obj, key, { value: value, enumerable: true, configurable: true, writable: true }); } else { obj[key] = value; } return obj; }
|
|
8
|
+
|
|
6
9
|
function asyncGeneratorStep(gen, resolve, reject, _next, _throw, key, arg) { try { var info = gen[key](arg); var value = info.value; } catch (error) { reject(error); return; } if (info.done) { resolve(value); } else { Promise.resolve(value).then(_next, _throw); } }
|
|
10
|
+
|
|
7
11
|
function _asyncToGenerator(fn) { return function () { var self = this, args = arguments; return new Promise(function (resolve, reject) { var gen = fn.apply(self, args); function _next(value) { asyncGeneratorStep(gen, resolve, reject, _next, _throw, "next", value); } function _throw(err) { asyncGeneratorStep(gen, resolve, reject, _next, _throw, "throw", err); } _next(undefined); }); }; }
|
|
12
|
+
|
|
8
13
|
var getIt = require('get-it');
|
|
14
|
+
|
|
9
15
|
var _require = require('get-it/middleware'),
|
|
10
|
-
|
|
11
|
-
|
|
16
|
+
keepAlive = _require.keepAlive,
|
|
17
|
+
promise = _require.promise;
|
|
18
|
+
|
|
12
19
|
var debug = require('./debug');
|
|
20
|
+
|
|
13
21
|
var request = getIt([keepAlive(), promise({
|
|
14
22
|
onlyBody: true
|
|
15
23
|
})]);
|
|
16
24
|
var socketsWithTimeout = new WeakSet();
|
|
17
25
|
var CONNECTION_TIMEOUT = 15 * 1000; // 15 seconds
|
|
26
|
+
|
|
18
27
|
var READ_TIMEOUT = 3 * 60 * 1000; // 3 minutes
|
|
28
|
+
|
|
19
29
|
var MAX_RETRIES = 5;
|
|
30
|
+
|
|
20
31
|
function delay(ms) {
|
|
21
32
|
return new Promise(resolve => setTimeout(resolve, ms));
|
|
22
33
|
}
|
|
23
|
-
|
|
24
34
|
/* eslint-disable no-await-in-loop, max-depth */
|
|
35
|
+
|
|
36
|
+
|
|
25
37
|
module.exports = /*#__PURE__*/function () {
|
|
26
38
|
var _ref = _asyncToGenerator(function* (options) {
|
|
27
39
|
var error;
|
|
40
|
+
|
|
28
41
|
for (var i = 0; i < MAX_RETRIES; i++) {
|
|
29
42
|
try {
|
|
30
43
|
var _ret = yield* function* () {
|
|
@@ -36,28 +49,35 @@ module.exports = /*#__PURE__*/function () {
|
|
|
36
49
|
socket: READ_TIMEOUT
|
|
37
50
|
}
|
|
38
51
|
}));
|
|
52
|
+
|
|
39
53
|
if (response.connection && typeof response.connection.setTimeout === 'function' && !socketsWithTimeout.has(response.connection)) {
|
|
40
54
|
socketsWithTimeout.add(response.connection);
|
|
41
55
|
response.connection.setTimeout(READ_TIMEOUT, () => {
|
|
42
56
|
response.destroy(new Error("Read timeout: No data received on socket for ".concat(READ_TIMEOUT, " ms")));
|
|
43
57
|
});
|
|
44
58
|
}
|
|
59
|
+
|
|
45
60
|
return {
|
|
46
61
|
v: response
|
|
47
62
|
};
|
|
48
63
|
}();
|
|
64
|
+
|
|
49
65
|
if (typeof _ret === "object") return _ret.v;
|
|
50
66
|
} catch (err) {
|
|
51
67
|
error = err;
|
|
68
|
+
|
|
52
69
|
if (err.response && err.response.statusCode && err.response.statusCode < 500) {
|
|
53
70
|
break;
|
|
54
71
|
}
|
|
72
|
+
|
|
55
73
|
debug('Error, retrying after 1500ms: %s', err.message);
|
|
56
74
|
yield delay(1500);
|
|
57
75
|
}
|
|
58
76
|
}
|
|
77
|
+
|
|
59
78
|
throw error;
|
|
60
79
|
});
|
|
80
|
+
|
|
61
81
|
return function (_x) {
|
|
62
82
|
return _ref.apply(this, arguments);
|
|
63
83
|
};
|
package/lib/stringifyStream.js
CHANGED
package/lib/tryParseJson.js
CHANGED
|
@@ -6,16 +6,20 @@ module.exports = line => {
|
|
|
6
6
|
} catch (err) {
|
|
7
7
|
// Catch half-done lines with an error at the end
|
|
8
8
|
var errorPosition = line.lastIndexOf('{"error":');
|
|
9
|
+
|
|
9
10
|
if (errorPosition === -1) {
|
|
10
11
|
err.message = "".concat(err.message, " (").concat(line, ")");
|
|
11
12
|
throw err;
|
|
12
13
|
}
|
|
14
|
+
|
|
13
15
|
var errorJson = line.slice(errorPosition);
|
|
14
16
|
var errorLine = JSON.parse(errorJson);
|
|
15
17
|
var error = errorLine && errorLine.error;
|
|
18
|
+
|
|
16
19
|
if (error && error.description) {
|
|
17
20
|
throw new Error("Error streaming dataset: ".concat(error.description, "\n\n").concat(errorJson, "\n"));
|
|
18
21
|
}
|
|
22
|
+
|
|
19
23
|
throw err;
|
|
20
24
|
}
|
|
21
25
|
};
|
package/lib/validateOptions.js
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
|
|
3
3
|
var defaults = require('lodash/defaults');
|
|
4
|
+
|
|
4
5
|
var clientMethods = ['getUrl', 'config'];
|
|
5
6
|
var booleanFlags = ['assets', 'raw', 'compress', 'drafts'];
|
|
6
7
|
var exportDefaults = {
|
|
@@ -9,36 +10,49 @@ var exportDefaults = {
|
|
|
9
10
|
assets: true,
|
|
10
11
|
raw: false
|
|
11
12
|
};
|
|
13
|
+
|
|
12
14
|
function validateOptions(opts) {
|
|
13
15
|
var options = defaults({}, opts, exportDefaults);
|
|
16
|
+
|
|
14
17
|
if (typeof options.dataset !== 'string' || options.dataset.length < 1) {
|
|
15
18
|
throw new Error("options.dataset must be a valid dataset name");
|
|
16
19
|
}
|
|
20
|
+
|
|
17
21
|
if (options.onProgress && typeof options.onProgress !== 'function') {
|
|
18
22
|
throw new Error("options.onProgress must be a function");
|
|
19
23
|
}
|
|
24
|
+
|
|
20
25
|
if (!options.client) {
|
|
21
26
|
throw new Error('`options.client` must be set to an instance of @sanity/client');
|
|
22
27
|
}
|
|
28
|
+
|
|
23
29
|
var missing = clientMethods.find(key => typeof options.client[key] !== 'function');
|
|
30
|
+
|
|
24
31
|
if (missing) {
|
|
25
32
|
throw new Error("`options.client` is not a valid @sanity/client instance - no \"".concat(missing, "\" method found"));
|
|
26
33
|
}
|
|
34
|
+
|
|
27
35
|
var clientConfig = options.client.config();
|
|
36
|
+
|
|
28
37
|
if (!clientConfig.token) {
|
|
29
38
|
throw new Error('Client is not instantiated with a `token`');
|
|
30
39
|
}
|
|
40
|
+
|
|
31
41
|
booleanFlags.forEach(flag => {
|
|
32
42
|
if (typeof options[flag] !== 'boolean') {
|
|
33
43
|
throw new Error("Flag ".concat(flag, " must be a boolean (true/false)"));
|
|
34
44
|
}
|
|
35
45
|
});
|
|
46
|
+
|
|
36
47
|
if (!options.outputPath) {
|
|
37
48
|
throw new Error('outputPath must be specified (- for stdout)');
|
|
38
49
|
}
|
|
50
|
+
|
|
39
51
|
if (options.assetConcurrency && (options.assetConcurrency < 1 || options.assetConcurrency > 24)) {
|
|
40
52
|
throw new Error('`assetConcurrency` must be between 1 and 24');
|
|
41
53
|
}
|
|
54
|
+
|
|
42
55
|
return options;
|
|
43
56
|
}
|
|
57
|
+
|
|
44
58
|
module.exports = validateOptions;
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@sanity/export",
|
|
3
|
-
"version": "2.34.
|
|
3
|
+
"version": "2.34.2-empty-template-cli.0+184a9350e5",
|
|
4
4
|
"description": "Export Sanity documents and assets",
|
|
5
5
|
"main": "lib/export.js",
|
|
6
6
|
"engines": {
|
|
@@ -49,5 +49,5 @@
|
|
|
49
49
|
"url": "https://github.com/sanity-io/sanity/issues"
|
|
50
50
|
},
|
|
51
51
|
"homepage": "https://www.sanity.io/",
|
|
52
|
-
"gitHead": "
|
|
52
|
+
"gitHead": "184a9350e50aa05877a3b10e052ce4e2ca743ef0"
|
|
53
53
|
}
|