contentful-import 8.2.22 → 8.2.26
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.js +49 -75
- package/dist/parseOptions.js +27 -20
- package/dist/tasks/get-destination-data.js +116 -122
- package/dist/tasks/init-client.js +12 -6
- package/dist/tasks/push-to-space/assets.js +55 -65
- package/dist/tasks/push-to-space/creation.js +154 -157
- package/dist/tasks/push-to-space/publishing.js +100 -128
- package/dist/tasks/push-to-space/push-to-space.js +242 -399
- package/dist/transform/transform-space.js +9 -10
- package/dist/transform/transformers.js +12 -7
- package/dist/usageParams.js +6 -4
- package/dist/utils/headers.js +9 -9
- package/dist/utils/schema.js +26 -21
- package/dist/utils/sort-entries.js +13 -14
- package/dist/utils/sort-locales.js +2 -4
- package/dist/utils/validations.js +11 -8
- package/package.json +9 -9
package/dist/index.js
CHANGED
|
@@ -1,66 +1,64 @@
|
|
|
1
|
-
|
|
1
|
+
"use strict";
|
|
2
2
|
|
|
3
3
|
Object.defineProperty(exports, "__esModule", {
|
|
4
4
|
value: true
|
|
5
5
|
});
|
|
6
6
|
exports.default = runContentfulImport;
|
|
7
7
|
|
|
8
|
-
var _cliTable = require(
|
|
8
|
+
var _cliTable = require("cli-table3");
|
|
9
9
|
|
|
10
10
|
var _cliTable2 = _interopRequireDefault(_cliTable);
|
|
11
11
|
|
|
12
|
-
var _listr = require(
|
|
12
|
+
var _listr = require("listr");
|
|
13
13
|
|
|
14
14
|
var _listr2 = _interopRequireDefault(_listr);
|
|
15
15
|
|
|
16
|
-
var _listrUpdateRenderer = require(
|
|
16
|
+
var _listrUpdateRenderer = require("listr-update-renderer");
|
|
17
17
|
|
|
18
18
|
var _listrUpdateRenderer2 = _interopRequireDefault(_listrUpdateRenderer);
|
|
19
19
|
|
|
20
|
-
var _listrVerboseRenderer = require(
|
|
20
|
+
var _listrVerboseRenderer = require("listr-verbose-renderer");
|
|
21
21
|
|
|
22
22
|
var _listrVerboseRenderer2 = _interopRequireDefault(_listrVerboseRenderer);
|
|
23
23
|
|
|
24
|
-
var _lodash = require(
|
|
24
|
+
var _lodash = require("lodash");
|
|
25
25
|
|
|
26
|
-
var _moment = require(
|
|
26
|
+
var _moment = require("moment");
|
|
27
27
|
|
|
28
28
|
var _moment2 = _interopRequireDefault(_moment);
|
|
29
29
|
|
|
30
|
-
var _pQueue = require(
|
|
30
|
+
var _pQueue = require("p-queue");
|
|
31
31
|
|
|
32
32
|
var _pQueue2 = _interopRequireDefault(_pQueue);
|
|
33
33
|
|
|
34
|
-
var _logging = require(
|
|
34
|
+
var _logging = require("contentful-batch-libs/dist/logging");
|
|
35
35
|
|
|
36
|
-
var _listr3 = require(
|
|
36
|
+
var _listr3 = require("contentful-batch-libs/dist/listr");
|
|
37
37
|
|
|
38
|
-
var _initClient = require(
|
|
38
|
+
var _initClient = require("./tasks/init-client");
|
|
39
39
|
|
|
40
40
|
var _initClient2 = _interopRequireDefault(_initClient);
|
|
41
41
|
|
|
42
|
-
var _getDestinationData = require(
|
|
42
|
+
var _getDestinationData = require("./tasks/get-destination-data");
|
|
43
43
|
|
|
44
44
|
var _getDestinationData2 = _interopRequireDefault(_getDestinationData);
|
|
45
45
|
|
|
46
|
-
var _pushToSpace = require(
|
|
46
|
+
var _pushToSpace = require("./tasks/push-to-space/push-to-space");
|
|
47
47
|
|
|
48
48
|
var _pushToSpace2 = _interopRequireDefault(_pushToSpace);
|
|
49
49
|
|
|
50
|
-
var _transformSpace = require(
|
|
50
|
+
var _transformSpace = require("./transform/transform-space");
|
|
51
51
|
|
|
52
52
|
var _transformSpace2 = _interopRequireDefault(_transformSpace);
|
|
53
53
|
|
|
54
|
-
var _validations = require(
|
|
54
|
+
var _validations = require("./utils/validations");
|
|
55
55
|
|
|
56
|
-
var _parseOptions = require(
|
|
56
|
+
var _parseOptions = require("./parseOptions");
|
|
57
57
|
|
|
58
58
|
var _parseOptions2 = _interopRequireDefault(_parseOptions);
|
|
59
59
|
|
|
60
60
|
function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; }
|
|
61
61
|
|
|
62
|
-
function _asyncToGenerator(fn) { return function () { var gen = fn.apply(this, arguments); return new Promise(function (resolve, reject) { function step(key, arg) { try { var info = gen[key](arg); var value = info.value; } catch (error) { reject(error); return; } if (info.done) { resolve(value); } else { return Promise.resolve(value).then(function (value) { step("next", value); }, function (err) { step("throw", err); }); } } return step("next"); }); }; }
|
|
63
|
-
|
|
64
62
|
const ONE_SECOND = 1000;
|
|
65
63
|
|
|
66
64
|
function createListrOptions(options) {
|
|
@@ -69,6 +67,7 @@ function createListrOptions(options) {
|
|
|
69
67
|
renderer: _listrVerboseRenderer2.default
|
|
70
68
|
};
|
|
71
69
|
}
|
|
70
|
+
|
|
72
71
|
return {
|
|
73
72
|
renderer: _listrUpdateRenderer2.default,
|
|
74
73
|
collapse: false
|
|
@@ -83,15 +82,14 @@ function runContentfulImport(params) {
|
|
|
83
82
|
interval: ONE_SECOND,
|
|
84
83
|
intervalCap: options.rateLimit,
|
|
85
84
|
carryoverConcurrencyCount: true
|
|
86
|
-
});
|
|
85
|
+
}); // Setup custom log listener to store log messages for later
|
|
87
86
|
|
|
88
|
-
// Setup custom log listener to store log messages for later
|
|
89
87
|
(0, _logging.setupLogging)(log);
|
|
90
|
-
|
|
91
88
|
const infoTable = new _cliTable2.default();
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
89
|
+
infoTable.push([{
|
|
90
|
+
colSpan: 2,
|
|
91
|
+
content: 'The following entities are going to be imported:'
|
|
92
|
+
}]);
|
|
95
93
|
Object.keys(options.content).forEach(type => {
|
|
96
94
|
if (options.skipLocales && type === 'locales') {
|
|
97
95
|
return;
|
|
@@ -107,9 +105,7 @@ function runContentfulImport(params) {
|
|
|
107
105
|
|
|
108
106
|
infoTable.push([(0, _lodash.startCase)(type), options.content[type].length]);
|
|
109
107
|
});
|
|
110
|
-
|
|
111
108
|
console.log(infoTable.toString());
|
|
112
|
-
|
|
113
109
|
const tasks = new _listr2.default([{
|
|
114
110
|
title: 'Validating content-file',
|
|
115
111
|
task: ctx => {
|
|
@@ -117,53 +113,34 @@ function runContentfulImport(params) {
|
|
|
117
113
|
}
|
|
118
114
|
}, {
|
|
119
115
|
title: 'Initialize client',
|
|
120
|
-
task: (0, _listr3.wrapTask)(
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
});
|
|
124
|
-
|
|
125
|
-
return function (_x) {
|
|
126
|
-
return _ref.apply(this, arguments);
|
|
127
|
-
};
|
|
128
|
-
})())
|
|
116
|
+
task: (0, _listr3.wrapTask)(async ctx => {
|
|
117
|
+
ctx.client = (0, _initClient2.default)(options);
|
|
118
|
+
})
|
|
129
119
|
}, {
|
|
130
120
|
title: 'Checking if destination space already has any content and retrieving it',
|
|
131
|
-
task: (0, _listr3.wrapTask)((
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
requestQueue
|
|
141
|
-
});
|
|
142
|
-
|
|
143
|
-
ctx.sourceDataUntransformed = options.content;
|
|
144
|
-
ctx.destinationData = destinationData;
|
|
145
|
-
(0, _validations.assertDefaultLocale)(ctx.sourceDataUntransformed, ctx.destinationData);
|
|
121
|
+
task: (0, _listr3.wrapTask)(async (ctx, task) => {
|
|
122
|
+
const destinationData = await (0, _getDestinationData2.default)({
|
|
123
|
+
client: ctx.client,
|
|
124
|
+
spaceId: options.spaceId,
|
|
125
|
+
environmentId: options.environmentId,
|
|
126
|
+
sourceData: options.content,
|
|
127
|
+
skipLocales: options.skipLocales,
|
|
128
|
+
skipContentModel: options.skipContentModel,
|
|
129
|
+
requestQueue
|
|
146
130
|
});
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
})())
|
|
131
|
+
ctx.sourceDataUntransformed = options.content;
|
|
132
|
+
ctx.destinationData = destinationData;
|
|
133
|
+
(0, _validations.assertDefaultLocale)(ctx.sourceDataUntransformed, ctx.destinationData);
|
|
134
|
+
})
|
|
152
135
|
}, {
|
|
153
136
|
title: 'Apply transformations to source data',
|
|
154
|
-
task: (0, _listr3.wrapTask)(
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
});
|
|
159
|
-
|
|
160
|
-
return function (_x4) {
|
|
161
|
-
return _ref3.apply(this, arguments);
|
|
162
|
-
};
|
|
163
|
-
})())
|
|
137
|
+
task: (0, _listr3.wrapTask)(async ctx => {
|
|
138
|
+
const transformedSourceData = (0, _transformSpace2.default)(ctx.sourceDataUntransformed, ctx.destinationData);
|
|
139
|
+
ctx.sourceData = transformedSourceData;
|
|
140
|
+
})
|
|
164
141
|
}, {
|
|
165
142
|
title: 'Push content to destination space',
|
|
166
|
-
task: (ctx,
|
|
143
|
+
task: (ctx, task) => {
|
|
167
144
|
return (0, _pushToSpace2.default)({
|
|
168
145
|
sourceData: ctx.sourceData,
|
|
169
146
|
destinationData: ctx.destinationData,
|
|
@@ -183,22 +160,21 @@ function runContentfulImport(params) {
|
|
|
183
160
|
});
|
|
184
161
|
}
|
|
185
162
|
}], listrOptions);
|
|
186
|
-
|
|
187
163
|
return tasks.run({
|
|
188
164
|
data: {}
|
|
189
165
|
}).then(ctx => {
|
|
190
166
|
console.log('Finished importing all data');
|
|
191
|
-
|
|
192
167
|
const resultTypes = Object.keys(ctx.data);
|
|
168
|
+
|
|
193
169
|
if (resultTypes.length) {
|
|
194
170
|
const resultTable = new _cliTable2.default();
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
171
|
+
resultTable.push([{
|
|
172
|
+
colSpan: 2,
|
|
173
|
+
content: 'Imported entities'
|
|
174
|
+
}]);
|
|
198
175
|
resultTypes.forEach(type => {
|
|
199
176
|
resultTable.push([(0, _lodash.startCase)(type), ctx.data[type].length]);
|
|
200
177
|
});
|
|
201
|
-
|
|
202
178
|
console.log(resultTable.toString());
|
|
203
179
|
} else {
|
|
204
180
|
console.log('No data was imported');
|
|
@@ -206,9 +182,7 @@ function runContentfulImport(params) {
|
|
|
206
182
|
|
|
207
183
|
const durationHuman = options.startTime.fromNow(true);
|
|
208
184
|
const durationSeconds = (0, _moment2.default)().diff(options.startTime, 'seconds');
|
|
209
|
-
|
|
210
185
|
console.log(`The import took ${durationHuman} (${durationSeconds}s)`);
|
|
211
|
-
|
|
212
186
|
return ctx.data;
|
|
213
187
|
}).catch(err => {
|
|
214
188
|
log.push({
|
|
@@ -231,8 +205,8 @@ function runContentfulImport(params) {
|
|
|
231
205
|
}
|
|
232
206
|
|
|
233
207
|
console.log('The import was successful.');
|
|
234
|
-
|
|
235
208
|
return data;
|
|
236
209
|
});
|
|
237
210
|
}
|
|
211
|
+
|
|
238
212
|
module.exports = exports.default;
|
package/dist/parseOptions.js
CHANGED
|
@@ -1,31 +1,38 @@
|
|
|
1
|
-
|
|
1
|
+
"use strict";
|
|
2
2
|
|
|
3
3
|
Object.defineProperty(exports, "__esModule", {
|
|
4
4
|
value: true
|
|
5
5
|
});
|
|
6
|
-
|
|
7
|
-
var _extends = Object.assign || function (target) { for (var i = 1; i < arguments.length; i++) { var source = arguments[i]; for (var key in source) { if (Object.prototype.hasOwnProperty.call(source, key)) { target[key] = source[key]; } } } return target; };
|
|
8
|
-
|
|
9
6
|
exports.default = parseOptions;
|
|
10
7
|
|
|
11
|
-
var _fs = require(
|
|
8
|
+
var _fs = require("fs");
|
|
12
9
|
|
|
13
10
|
var _fs2 = _interopRequireDefault(_fs);
|
|
14
11
|
|
|
15
|
-
var _path = require(
|
|
12
|
+
var _path = require("path");
|
|
16
13
|
|
|
17
|
-
var _moment = require(
|
|
14
|
+
var _moment = require("moment");
|
|
18
15
|
|
|
19
16
|
var _moment2 = _interopRequireDefault(_moment);
|
|
20
17
|
|
|
21
|
-
var _package = require(
|
|
18
|
+
var _package = require("../package");
|
|
19
|
+
|
|
20
|
+
var _headers = require("./utils/headers");
|
|
21
|
+
|
|
22
|
+
var _proxy = require("contentful-batch-libs/dist/proxy");
|
|
22
23
|
|
|
23
|
-
var
|
|
24
|
+
var _addSequenceHeader = require("contentful-batch-libs/dist/add-sequence-header");
|
|
24
25
|
|
|
25
|
-
var
|
|
26
|
+
var _addSequenceHeader2 = _interopRequireDefault(_addSequenceHeader);
|
|
26
27
|
|
|
27
28
|
function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; }
|
|
28
29
|
|
|
30
|
+
function ownKeys(object, enumerableOnly) { var keys = Object.keys(object); if (Object.getOwnPropertySymbols) { var symbols = Object.getOwnPropertySymbols(object); if (enumerableOnly) { symbols = symbols.filter(function (sym) { return Object.getOwnPropertyDescriptor(object, sym).enumerable; }); } keys.push.apply(keys, symbols); } return keys; }
|
|
31
|
+
|
|
32
|
+
function _objectSpread(target) { for (var i = 1; i < arguments.length; i++) { var source = arguments[i] != null ? arguments[i] : {}; if (i % 2) { ownKeys(Object(source), true).forEach(function (key) { _defineProperty(target, key, source[key]); }); } else if (Object.getOwnPropertyDescriptors) { Object.defineProperties(target, Object.getOwnPropertyDescriptors(source)); } else { ownKeys(Object(source)).forEach(function (key) { Object.defineProperty(target, key, Object.getOwnPropertyDescriptor(source, key)); }); } } return target; }
|
|
33
|
+
|
|
34
|
+
function _defineProperty(obj, key, value) { if (key in obj) { Object.defineProperty(obj, key, { value: value, enumerable: true, configurable: true, writable: true }); } else { obj[key] = value; } return obj; }
|
|
35
|
+
|
|
29
36
|
const SUPPORTED_ENTITY_TYPES = ['contentTypes', 'tags', 'entries', 'assets', 'locales', 'webhooks', 'editorInterfaces'];
|
|
30
37
|
|
|
31
38
|
function parseOptions(params) {
|
|
@@ -39,14 +46,14 @@ function parseOptions(params) {
|
|
|
39
46
|
uploadAssets: false,
|
|
40
47
|
rateLimit: 7
|
|
41
48
|
};
|
|
42
|
-
|
|
43
49
|
const configFile = params.config ? require((0, _path.resolve)(process.cwd(), params.config)) : {};
|
|
44
50
|
|
|
45
|
-
const options =
|
|
46
|
-
headers: params.headers || (0, _headers.getHeadersConfig)(params.header)
|
|
51
|
+
const options = _objectSpread(_objectSpread(_objectSpread(_objectSpread({}, defaultOptions), configFile), params), {}, {
|
|
52
|
+
headers: (0, _addSequenceHeader2.default)(params.headers || (0, _headers.getHeadersConfig)(params.header))
|
|
53
|
+
}); // Validation
|
|
54
|
+
|
|
47
55
|
|
|
48
|
-
|
|
49
|
-
});if (!options.spaceId) {
|
|
56
|
+
if (!options.spaceId) {
|
|
50
57
|
throw new Error('The `spaceId` option is required.');
|
|
51
58
|
}
|
|
52
59
|
|
|
@@ -68,6 +75,7 @@ function parseOptions(params) {
|
|
|
68
75
|
|
|
69
76
|
const proxySimpleExp = /.+:\d+/;
|
|
70
77
|
const proxyAuthExp = /.+:.+@.+:\d+/;
|
|
78
|
+
|
|
71
79
|
if (typeof options.proxy === 'string' && options.proxy && !(proxySimpleExp.test(options.proxy) || proxyAuthExp.test(options.proxy))) {
|
|
72
80
|
throw new Error('Please provide the proxy config in the following format:\nhost:port or user:password@host:port');
|
|
73
81
|
}
|
|
@@ -78,19 +86,17 @@ function parseOptions(params) {
|
|
|
78
86
|
options.errorLogFile = (0, _path.resolve)(process.cwd(), `contentful-import-error-log-${options.spaceId}-${options.startTime.format('YYYY-MM-DDTHH-mm-SS')}.json`);
|
|
79
87
|
} else {
|
|
80
88
|
options.errorLogFile = (0, _path.resolve)(process.cwd(), options.errorLogFile);
|
|
81
|
-
}
|
|
89
|
+
} // Further processing
|
|
90
|
+
|
|
82
91
|
|
|
83
|
-
// Further processing
|
|
84
92
|
options.accessToken = options.managementToken;
|
|
85
|
-
options.content = options.content || JSON.parse(_fs2.default.readFileSync(options.contentFile));
|
|
93
|
+
options.content = options.content || JSON.parse(_fs2.default.readFileSync(options.contentFile)); // Clean up content to only include supported entity types
|
|
86
94
|
|
|
87
|
-
// Clean up content to only include supported entity types
|
|
88
95
|
Object.keys(options.content).forEach(type => {
|
|
89
96
|
if (SUPPORTED_ENTITY_TYPES.indexOf(type) === -1) {
|
|
90
97
|
delete options.content[type];
|
|
91
98
|
}
|
|
92
99
|
});
|
|
93
|
-
|
|
94
100
|
SUPPORTED_ENTITY_TYPES.forEach(type => {
|
|
95
101
|
options.content[type] = options.content[type] || [];
|
|
96
102
|
});
|
|
@@ -108,4 +114,5 @@ function parseOptions(params) {
|
|
|
108
114
|
options.feature = options.managementFeature || 'library-import';
|
|
109
115
|
return options;
|
|
110
116
|
}
|
|
117
|
+
|
|
111
118
|
module.exports = exports.default;
|
|
@@ -1,70 +1,82 @@
|
|
|
1
|
-
|
|
1
|
+
"use strict";
|
|
2
2
|
|
|
3
3
|
Object.defineProperty(exports, "__esModule", {
|
|
4
4
|
value: true
|
|
5
5
|
});
|
|
6
|
+
exports.default = getDestinationData;
|
|
6
7
|
|
|
7
|
-
var
|
|
8
|
-
|
|
9
|
-
let batchedIdQuery = (() => {
|
|
10
|
-
var _ref = _asyncToGenerator(function* ({ environment, type, ids, requestQueue }) {
|
|
11
|
-
const method = METHODS[type].method;
|
|
12
|
-
const entityTypeName = METHODS[type].name;
|
|
13
|
-
const batches = getIdBatches(ids);
|
|
14
|
-
|
|
15
|
-
let totalFetched = 0;
|
|
16
|
-
|
|
17
|
-
const allPendingResponses = batches.map(function (idBatch) {
|
|
18
|
-
return requestQueue.add(_asyncToGenerator(function* () {
|
|
19
|
-
const response = yield environment[method]({
|
|
20
|
-
'sys.id[in]': idBatch,
|
|
21
|
-
limit: idBatch.split(',').length
|
|
22
|
-
});
|
|
23
|
-
totalFetched = totalFetched + response.items.length;
|
|
24
|
-
_logging.logEmitter.emit('info', `Fetched ${totalFetched} of ${response.total} ${entityTypeName}`);
|
|
25
|
-
|
|
26
|
-
return response.items;
|
|
27
|
-
}));
|
|
28
|
-
});
|
|
29
|
-
|
|
30
|
-
const responses = yield _bluebird2.default.all(allPendingResponses);
|
|
31
|
-
|
|
32
|
-
return responses.flat();
|
|
33
|
-
});
|
|
34
|
-
|
|
35
|
-
return function batchedIdQuery(_x) {
|
|
36
|
-
return _ref.apply(this, arguments);
|
|
37
|
-
};
|
|
38
|
-
})();
|
|
39
|
-
|
|
40
|
-
var _bluebird = require('bluebird');
|
|
8
|
+
var _bluebird = require("bluebird");
|
|
41
9
|
|
|
42
10
|
var _bluebird2 = _interopRequireDefault(_bluebird);
|
|
43
11
|
|
|
44
|
-
var _logging = require(
|
|
12
|
+
var _logging = require("contentful-batch-libs/dist/logging");
|
|
45
13
|
|
|
46
14
|
function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; }
|
|
47
15
|
|
|
48
|
-
function
|
|
16
|
+
function ownKeys(object, enumerableOnly) { var keys = Object.keys(object); if (Object.getOwnPropertySymbols) { var symbols = Object.getOwnPropertySymbols(object); if (enumerableOnly) { symbols = symbols.filter(function (sym) { return Object.getOwnPropertyDescriptor(object, sym).enumerable; }); } keys.push.apply(keys, symbols); } return keys; }
|
|
17
|
+
|
|
18
|
+
function _objectSpread(target) { for (var i = 1; i < arguments.length; i++) { var source = arguments[i] != null ? arguments[i] : {}; if (i % 2) { ownKeys(Object(source), true).forEach(function (key) { _defineProperty(target, key, source[key]); }); } else if (Object.getOwnPropertyDescriptors) { Object.defineProperties(target, Object.getOwnPropertyDescriptors(source)); } else { ownKeys(Object(source)).forEach(function (key) { Object.defineProperty(target, key, Object.getOwnPropertyDescriptor(source, key)); }); } } return target; }
|
|
19
|
+
|
|
20
|
+
function _defineProperty(obj, key, value) { if (key in obj) { Object.defineProperty(obj, key, { value: value, enumerable: true, configurable: true, writable: true }); } else { obj[key] = value; } return obj; }
|
|
49
21
|
|
|
50
22
|
const BATCH_CHAR_LIMIT = 1990;
|
|
51
23
|
const BATCH_SIZE_LIMIT = 100;
|
|
52
|
-
|
|
53
24
|
const METHODS = {
|
|
54
|
-
contentTypes: {
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
25
|
+
contentTypes: {
|
|
26
|
+
name: 'content types',
|
|
27
|
+
method: 'getContentTypes'
|
|
28
|
+
},
|
|
29
|
+
locales: {
|
|
30
|
+
name: 'locales',
|
|
31
|
+
method: 'getLocales'
|
|
32
|
+
},
|
|
33
|
+
entries: {
|
|
34
|
+
name: 'entries',
|
|
35
|
+
method: 'getEntries'
|
|
36
|
+
},
|
|
37
|
+
assets: {
|
|
38
|
+
name: 'assets',
|
|
39
|
+
method: 'getAssets'
|
|
40
|
+
}
|
|
58
41
|
};
|
|
59
42
|
|
|
43
|
+
async function batchedIdQuery({
|
|
44
|
+
environment,
|
|
45
|
+
type,
|
|
46
|
+
ids,
|
|
47
|
+
requestQueue
|
|
48
|
+
}) {
|
|
49
|
+
const method = METHODS[type].method;
|
|
50
|
+
const entityTypeName = METHODS[type].name;
|
|
51
|
+
const batches = getIdBatches(ids);
|
|
52
|
+
let totalFetched = 0;
|
|
53
|
+
const allPendingResponses = batches.map(idBatch => {
|
|
54
|
+
return requestQueue.add(async () => {
|
|
55
|
+
const response = await environment[method]({
|
|
56
|
+
'sys.id[in]': idBatch,
|
|
57
|
+
limit: idBatch.split(',').length
|
|
58
|
+
});
|
|
59
|
+
totalFetched = totalFetched + response.items.length;
|
|
60
|
+
|
|
61
|
+
_logging.logEmitter.emit('info', `Fetched ${totalFetched} of ${response.total} ${entityTypeName}`);
|
|
62
|
+
|
|
63
|
+
return response.items;
|
|
64
|
+
});
|
|
65
|
+
});
|
|
66
|
+
const responses = await _bluebird2.default.all(allPendingResponses);
|
|
67
|
+
return responses.flat();
|
|
68
|
+
}
|
|
69
|
+
|
|
60
70
|
function getIdBatches(ids) {
|
|
61
71
|
const batches = [];
|
|
62
72
|
let currentBatch = '';
|
|
63
73
|
let currentSize = 0;
|
|
74
|
+
|
|
64
75
|
while (ids.length > 0) {
|
|
65
76
|
const id = ids.splice(0, 1);
|
|
66
77
|
currentBatch += id;
|
|
67
78
|
currentSize = currentSize + 1;
|
|
79
|
+
|
|
68
80
|
if (currentSize === BATCH_SIZE_LIMIT || currentBatch.length > BATCH_CHAR_LIMIT || ids.length === 0) {
|
|
69
81
|
batches.push(currentBatch);
|
|
70
82
|
currentBatch = '';
|
|
@@ -73,9 +85,9 @@ function getIdBatches(ids) {
|
|
|
73
85
|
currentBatch += ',';
|
|
74
86
|
}
|
|
75
87
|
}
|
|
88
|
+
|
|
76
89
|
return batches;
|
|
77
90
|
}
|
|
78
|
-
|
|
79
91
|
/**
|
|
80
92
|
* Gets content from a space which will have content copied to it, based on a
|
|
81
93
|
* collection of existing content.
|
|
@@ -85,94 +97,76 @@ function getIdBatches(ids) {
|
|
|
85
97
|
*
|
|
86
98
|
*/
|
|
87
99
|
|
|
88
|
-
exports.default = (() => {
|
|
89
|
-
var _ref3 = _asyncToGenerator(function* ({
|
|
90
|
-
client,
|
|
91
|
-
spaceId,
|
|
92
|
-
environmentId,
|
|
93
|
-
sourceData,
|
|
94
|
-
contentModelOnly,
|
|
95
|
-
skipLocales,
|
|
96
|
-
skipContentModel,
|
|
97
|
-
requestQueue
|
|
98
|
-
}) {
|
|
99
|
-
const space = yield client.getSpace(spaceId);
|
|
100
|
-
const environment = yield space.getEnvironment(environmentId);
|
|
101
|
-
const result = {
|
|
102
|
-
contentTypes: [],
|
|
103
|
-
tags: [],
|
|
104
|
-
locales: [],
|
|
105
|
-
entries: [],
|
|
106
|
-
assets: []
|
|
107
|
-
|
|
108
|
-
// Make sure all required properties are available and at least an empty array
|
|
109
|
-
};sourceData = _extends({}, result, sourceData);
|
|
110
|
-
|
|
111
|
-
if (!skipContentModel) {
|
|
112
|
-
const contentTypeIds = sourceData.contentTypes.map(function (e) {
|
|
113
|
-
return e.sys.id;
|
|
114
|
-
});
|
|
115
|
-
result.contentTypes = batchedIdQuery({
|
|
116
|
-
environment,
|
|
117
|
-
type: 'contentTypes',
|
|
118
|
-
ids: contentTypeIds,
|
|
119
|
-
requestQueue
|
|
120
|
-
});
|
|
121
|
-
|
|
122
|
-
if (!skipLocales) {
|
|
123
|
-
const localeIds = sourceData.locales.map(function (e) {
|
|
124
|
-
return e.sys.id;
|
|
125
|
-
});
|
|
126
|
-
result.locales = batchedIdQuery({
|
|
127
|
-
environment,
|
|
128
|
-
type: 'locales',
|
|
129
|
-
ids: localeIds,
|
|
130
|
-
requestQueue
|
|
131
|
-
});
|
|
132
|
-
}
|
|
133
|
-
}
|
|
134
100
|
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
101
|
+
async function getDestinationData({
|
|
102
|
+
client,
|
|
103
|
+
spaceId,
|
|
104
|
+
environmentId,
|
|
105
|
+
sourceData,
|
|
106
|
+
contentModelOnly,
|
|
107
|
+
skipLocales,
|
|
108
|
+
skipContentModel,
|
|
109
|
+
requestQueue
|
|
110
|
+
}) {
|
|
111
|
+
const space = await client.getSpace(spaceId);
|
|
112
|
+
const environment = await space.getEnvironment(environmentId);
|
|
113
|
+
const result = {
|
|
114
|
+
contentTypes: [],
|
|
115
|
+
tags: [],
|
|
116
|
+
locales: [],
|
|
117
|
+
entries: [],
|
|
118
|
+
assets: []
|
|
119
|
+
}; // Make sure all required properties are available and at least an empty array
|
|
120
|
+
|
|
121
|
+
sourceData = _objectSpread(_objectSpread({}, result), sourceData);
|
|
122
|
+
|
|
123
|
+
if (!skipContentModel) {
|
|
124
|
+
const contentTypeIds = sourceData.contentTypes.map(e => e.sys.id);
|
|
125
|
+
result.contentTypes = batchedIdQuery({
|
|
126
|
+
environment,
|
|
127
|
+
type: 'contentTypes',
|
|
128
|
+
ids: contentTypeIds,
|
|
129
|
+
requestQueue
|
|
142
130
|
});
|
|
143
131
|
|
|
144
|
-
if (
|
|
145
|
-
|
|
132
|
+
if (!skipLocales) {
|
|
133
|
+
const localeIds = sourceData.locales.map(e => e.sys.id);
|
|
134
|
+
result.locales = batchedIdQuery({
|
|
135
|
+
environment,
|
|
136
|
+
type: 'locales',
|
|
137
|
+
ids: localeIds,
|
|
138
|
+
requestQueue
|
|
139
|
+
});
|
|
146
140
|
}
|
|
141
|
+
} // include tags even if contentModelOnly = true
|
|
147
142
|
|
|
148
|
-
const entryIds = sourceData.entries.map(function (e) {
|
|
149
|
-
return e.sys.id;
|
|
150
|
-
});
|
|
151
|
-
const assetIds = sourceData.assets.map(function (e) {
|
|
152
|
-
return e.sys.id;
|
|
153
|
-
});
|
|
154
|
-
result.entries = batchedIdQuery({
|
|
155
|
-
environment,
|
|
156
|
-
type: 'entries',
|
|
157
|
-
ids: entryIds,
|
|
158
|
-
requestQueue
|
|
159
|
-
});
|
|
160
|
-
result.assets = batchedIdQuery({
|
|
161
|
-
environment,
|
|
162
|
-
type: 'assets',
|
|
163
|
-
ids: assetIds,
|
|
164
|
-
requestQueue
|
|
165
|
-
});
|
|
166
|
-
result.webhooks = [];
|
|
167
143
|
|
|
168
|
-
|
|
144
|
+
result.tags = environment.getTags().then(response => response.items).catch(e => {
|
|
145
|
+
// users without access to Tags will get 404
|
|
146
|
+
// if they dont have access, remove tags array so they're not handled in future steps
|
|
147
|
+
delete result.tags;
|
|
169
148
|
});
|
|
170
149
|
|
|
171
|
-
|
|
172
|
-
return
|
|
150
|
+
if (contentModelOnly) {
|
|
151
|
+
return _bluebird2.default.props(result);
|
|
173
152
|
}
|
|
174
153
|
|
|
175
|
-
|
|
176
|
-
|
|
154
|
+
const entryIds = sourceData.entries.map(e => e.sys.id);
|
|
155
|
+
const assetIds = sourceData.assets.map(e => e.sys.id);
|
|
156
|
+
result.entries = batchedIdQuery({
|
|
157
|
+
environment,
|
|
158
|
+
type: 'entries',
|
|
159
|
+
ids: entryIds,
|
|
160
|
+
requestQueue
|
|
161
|
+
});
|
|
162
|
+
result.assets = batchedIdQuery({
|
|
163
|
+
environment,
|
|
164
|
+
type: 'assets',
|
|
165
|
+
ids: assetIds,
|
|
166
|
+
requestQueue
|
|
167
|
+
});
|
|
168
|
+
result.webhooks = [];
|
|
169
|
+
return _bluebird2.default.props(result);
|
|
170
|
+
}
|
|
177
171
|
|
|
178
172
|
module.exports = exports.default;
|
|
@@ -1,16 +1,19 @@
|
|
|
1
|
-
|
|
1
|
+
"use strict";
|
|
2
2
|
|
|
3
3
|
Object.defineProperty(exports, "__esModule", {
|
|
4
4
|
value: true
|
|
5
5
|
});
|
|
6
|
+
exports.default = initClient;
|
|
6
7
|
|
|
7
|
-
var
|
|
8
|
+
var _contentfulManagement = require("contentful-management");
|
|
8
9
|
|
|
9
|
-
|
|
10
|
+
var _logging = require("contentful-batch-libs/dist/logging");
|
|
11
|
+
|
|
12
|
+
function ownKeys(object, enumerableOnly) { var keys = Object.keys(object); if (Object.getOwnPropertySymbols) { var symbols = Object.getOwnPropertySymbols(object); if (enumerableOnly) { symbols = symbols.filter(function (sym) { return Object.getOwnPropertyDescriptor(object, sym).enumerable; }); } keys.push.apply(keys, symbols); } return keys; }
|
|
10
13
|
|
|
11
|
-
var
|
|
14
|
+
function _objectSpread(target) { for (var i = 1; i < arguments.length; i++) { var source = arguments[i] != null ? arguments[i] : {}; if (i % 2) { ownKeys(Object(source), true).forEach(function (key) { _defineProperty(target, key, source[key]); }); } else if (Object.getOwnPropertyDescriptors) { Object.defineProperties(target, Object.getOwnPropertyDescriptors(source)); } else { ownKeys(Object(source)).forEach(function (key) { Object.defineProperty(target, key, Object.getOwnPropertyDescriptor(source, key)); }); } } return target; }
|
|
12
15
|
|
|
13
|
-
|
|
16
|
+
function _defineProperty(obj, key, value) { if (key in obj) { Object.defineProperty(obj, key, { value: value, enumerable: true, configurable: true, writable: true }); } else { obj[key] = value; } return obj; }
|
|
14
17
|
|
|
15
18
|
function logHandler(level, data) {
|
|
16
19
|
_logging.logEmitter.emit(level, data);
|
|
@@ -21,7 +24,10 @@ function initClient(opts) {
|
|
|
21
24
|
timeout: 30000,
|
|
22
25
|
logHandler
|
|
23
26
|
};
|
|
24
|
-
|
|
27
|
+
|
|
28
|
+
const config = _objectSpread(_objectSpread({}, defaultOpts), opts);
|
|
29
|
+
|
|
25
30
|
return (0, _contentfulManagement.createClient)(config);
|
|
26
31
|
}
|
|
32
|
+
|
|
27
33
|
module.exports = exports.default;
|