contentful-export 8.2.1 → 8.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.js +1 -4
- package/dist/tasks/download-assets.js +36 -5
- package/dist/tasks/get-space-data.js +162 -105
- package/dist/tasks/init-client.js +5 -9
- package/package.json +2 -1
package/dist/index.js
CHANGED
|
@@ -18,9 +18,8 @@ var _formatDistance = require("date-fns/formatDistance");
|
|
|
18
18
|
var _contentfulBatchLibs = require("contentful-batch-libs");
|
|
19
19
|
var _downloadAssets = _interopRequireDefault(require("./tasks/download-assets"));
|
|
20
20
|
var _getSpaceData = _interopRequireDefault(require("./tasks/get-space-data"));
|
|
21
|
-
var _initClient =
|
|
21
|
+
var _initClient = _interopRequireDefault(require("./tasks/init-client"));
|
|
22
22
|
var _parseOptions = _interopRequireDefault(require("./parseOptions"));
|
|
23
|
-
function _interopRequireWildcard(e, t) { if ("function" == typeof WeakMap) var r = new WeakMap(), n = new WeakMap(); return (_interopRequireWildcard = function (e, t) { if (!t && e && e.__esModule) return e; var o, i, f = { __proto__: null, default: e }; if (null === e || "object" != typeof e && "function" != typeof e) return f; if (o = t ? n : r) { if (o.has(e)) return o.get(e); o.set(e, f); } for (const t in e) "default" !== t && {}.hasOwnProperty.call(e, t) && ((i = (o = Object.defineProperty) && Object.getOwnPropertyDescriptor(e, t)) && (i.get || i.set) ? o(f, t, i) : f[t] = e[t]); return f; })(e, t); }
|
|
24
23
|
function _interopRequireDefault(e) { return e && e.__esModule ? e : { default: e }; }
|
|
25
24
|
const accessP = _bluebird.default.promisify(_fs.access);
|
|
26
25
|
const tableOptions = {
|
|
@@ -54,7 +53,6 @@ function runContentfulExport(params) {
|
|
|
54
53
|
try {
|
|
55
54
|
// CMA client
|
|
56
55
|
ctx.client = (0, _initClient.default)(options);
|
|
57
|
-
ctx.plainClient = (0, _initClient.initPlainClient)(options);
|
|
58
56
|
if (options.deliveryToken && !options.includeDrafts) {
|
|
59
57
|
// CDA client for fetching only public entries
|
|
60
58
|
ctx.cdaClient = (0, _initClient.default)(options, true);
|
|
@@ -69,7 +67,6 @@ function runContentfulExport(params) {
|
|
|
69
67
|
task: ctx => {
|
|
70
68
|
return (0, _getSpaceData.default)({
|
|
71
69
|
client: ctx.client,
|
|
72
|
-
plainClient: ctx.plainClient,
|
|
73
70
|
cdaClient: ctx.cdaClient,
|
|
74
71
|
spaceId: options.spaceId,
|
|
75
72
|
environmentId: options.environmentId,
|
|
@@ -13,19 +13,26 @@ var _stream = require("stream");
|
|
|
13
13
|
var _util = require("util");
|
|
14
14
|
var _embargoedAssets = require("../utils/embargoedAssets");
|
|
15
15
|
var _axios = _interopRequireDefault(require("axios"));
|
|
16
|
+
var _axiosRetry = _interopRequireDefault(require("axios-retry"));
|
|
16
17
|
function _interopRequireDefault(e) { return e && e.__esModule ? e : { default: e }; }
|
|
17
18
|
const streamPipeline = (0, _util.promisify)(_stream.pipeline);
|
|
18
19
|
|
|
20
|
+
// For streaming responses, axios timeout applies to time-to-first-byte, not total transfer duration
|
|
21
|
+
const DEFAULT_DOWNLOAD_TIMEOUT_MS = 30000;
|
|
22
|
+
const DEFAULT_RETRY_LIMIT = 3;
|
|
23
|
+
|
|
19
24
|
/**
|
|
20
25
|
* @param {Object} options - The options for downloading the asset.
|
|
21
26
|
* @param {string} options.url - The URL of the asset to download.
|
|
22
27
|
* @param {string} options.directory - The directory where the asset should be saved.
|
|
23
28
|
* @param {import('axios').AxiosInstance} options.httpClient - The HTTP client to use for downloading the asset.
|
|
29
|
+
* @param {string} options.assetId - The ID of the asset being downloaded, used in error messages.
|
|
24
30
|
*/
|
|
25
31
|
async function downloadAsset({
|
|
26
32
|
url,
|
|
27
33
|
directory,
|
|
28
|
-
httpClient
|
|
34
|
+
httpClient,
|
|
35
|
+
assetId
|
|
29
36
|
}) {
|
|
30
37
|
// handle urls without protocol
|
|
31
38
|
if (url.startsWith('//')) {
|
|
@@ -57,7 +64,14 @@ async function downloadAsset({
|
|
|
57
64
|
* @type {import('axios').AxiosError}
|
|
58
65
|
*/
|
|
59
66
|
const axiosError = e;
|
|
60
|
-
|
|
67
|
+
if (axiosError.response) {
|
|
68
|
+
throw new Error(`error downloading asset ${assetId} (${url}): HTTP ${axiosError.response.status} ${axiosError.response.statusText}`, {
|
|
69
|
+
cause: axiosError
|
|
70
|
+
});
|
|
71
|
+
}
|
|
72
|
+
throw new Error(`error downloading asset ${assetId} (${url}): ${e.message}`, {
|
|
73
|
+
cause: e
|
|
74
|
+
});
|
|
61
75
|
}
|
|
62
76
|
}
|
|
63
77
|
function downloadAssets(options) {
|
|
@@ -67,11 +81,24 @@ function downloadAssets(options) {
|
|
|
67
81
|
let errorCount = 0;
|
|
68
82
|
const httpClient = _axios.default.create({
|
|
69
83
|
headers: options.headers,
|
|
70
|
-
timeout:
|
|
84
|
+
timeout: DEFAULT_DOWNLOAD_TIMEOUT_MS,
|
|
71
85
|
httpAgent: options.httpAgent,
|
|
72
86
|
httpsAgent: options.httpsAgent,
|
|
73
87
|
proxy: options.proxy
|
|
74
88
|
});
|
|
89
|
+
(0, _axiosRetry.default)(httpClient, {
|
|
90
|
+
retries: DEFAULT_RETRY_LIMIT,
|
|
91
|
+
retryDelay: _axiosRetry.default.exponentialDelay,
|
|
92
|
+
shouldResetTimeout: true,
|
|
93
|
+
retryCondition: error => {
|
|
94
|
+
var _error$response;
|
|
95
|
+
return _axiosRetry.default.isNetworkOrIdempotentRequestError(error) || ((_error$response = error.response) === null || _error$response === void 0 ? void 0 : _error$response.status) === 429;
|
|
96
|
+
},
|
|
97
|
+
onRetry: (retryCount, error) => {
|
|
98
|
+
const status = error.response ? `HTTP ${error.response.status}` : error.code || error.message;
|
|
99
|
+
_contentfulBatchLibs.logEmitter.emit('warning', `Asset download failed (${status}), retrying (attempt ${retryCount})...`);
|
|
100
|
+
}
|
|
101
|
+
});
|
|
75
102
|
return _bluebird.default.map(ctx.data.assets, asset => {
|
|
76
103
|
const entityName = (0, _contentfulBatchLibs.getEntityName)(asset);
|
|
77
104
|
if (!asset.fields.file) {
|
|
@@ -87,10 +114,12 @@ function downloadAssets(options) {
|
|
|
87
114
|
errorCount++;
|
|
88
115
|
return _bluebird.default.resolve();
|
|
89
116
|
}
|
|
117
|
+
const assetId = asset.sys.id;
|
|
90
118
|
let startingPromise = _bluebird.default.resolve({
|
|
91
119
|
url,
|
|
92
120
|
directory: options.exportDir,
|
|
93
|
-
httpClient
|
|
121
|
+
httpClient,
|
|
122
|
+
assetId
|
|
94
123
|
});
|
|
95
124
|
if ((0, _embargoedAssets.isEmbargoedAsset)(url)) {
|
|
96
125
|
const {
|
|
@@ -103,13 +132,15 @@ function downloadAssets(options) {
|
|
|
103
132
|
startingPromise = (0, _embargoedAssets.signUrl)(host, accessToken, spaceId, environmentId, url, expiresAtMs, httpClient).then(signedUrl => ({
|
|
104
133
|
url: signedUrl,
|
|
105
134
|
directory: options.exportDir,
|
|
106
|
-
httpClient
|
|
135
|
+
httpClient,
|
|
136
|
+
assetId
|
|
107
137
|
}));
|
|
108
138
|
}
|
|
109
139
|
return startingPromise.then(downloadAsset).then(() => {
|
|
110
140
|
task.output = `${_figures.default.tick} downloaded ${entityName} (${url})`;
|
|
111
141
|
successCount++;
|
|
112
142
|
}).catch(error => {
|
|
143
|
+
_contentfulBatchLibs.logEmitter.emit('warning', error.message);
|
|
113
144
|
task.output = `${_figures.default.cross} error downloading ${url}: ${error.message}`;
|
|
114
145
|
errorCount++;
|
|
115
146
|
});
|
|
@@ -9,6 +9,11 @@ var _contentfulBatchLibs = require("contentful-batch-libs");
|
|
|
9
9
|
var _listr = _interopRequireDefault(require("listr"));
|
|
10
10
|
var _listrVerboseRenderer = _interopRequireDefault(require("listr-verbose-renderer"));
|
|
11
11
|
function _interopRequireDefault(e) { return e && e.__esModule ? e : { default: e }; }
|
|
12
|
+
function ownKeys(e, r) { var t = Object.keys(e); if (Object.getOwnPropertySymbols) { var o = Object.getOwnPropertySymbols(e); r && (o = o.filter(function (r) { return Object.getOwnPropertyDescriptor(e, r).enumerable; })), t.push.apply(t, o); } return t; }
|
|
13
|
+
function _objectSpread(e) { for (var r = 1; r < arguments.length; r++) { var t = null != arguments[r] ? arguments[r] : {}; r % 2 ? ownKeys(Object(t), !0).forEach(function (r) { _defineProperty(e, r, t[r]); }) : Object.getOwnPropertyDescriptors ? Object.defineProperties(e, Object.getOwnPropertyDescriptors(t)) : ownKeys(Object(t)).forEach(function (r) { Object.defineProperty(e, r, Object.getOwnPropertyDescriptor(t, r)); }); } return e; }
|
|
14
|
+
function _defineProperty(e, r, t) { return (r = _toPropertyKey(r)) in e ? Object.defineProperty(e, r, { value: t, enumerable: !0, configurable: !0, writable: !0 }) : e[r] = t, e; }
|
|
15
|
+
function _toPropertyKey(t) { var i = _toPrimitive(t, "string"); return "symbol" == typeof i ? i : i + ""; }
|
|
16
|
+
function _toPrimitive(t, r) { if ("object" != typeof t || !t) return t; var e = t[Symbol.toPrimitive]; if (void 0 !== e) { var i = e.call(t, r || "default"); if ("object" != typeof i) return i; throw new TypeError("@@toPrimitive must return a primitive value."); } return ("string" === r ? String : Number)(t); }
|
|
12
17
|
const MAX_ALLOWED_LIMIT = 1000;
|
|
13
18
|
let pageLimit = MAX_ALLOWED_LIMIT;
|
|
14
19
|
|
|
@@ -18,7 +23,6 @@ let pageLimit = MAX_ALLOWED_LIMIT;
|
|
|
18
23
|
*/
|
|
19
24
|
function getFullSourceSpace({
|
|
20
25
|
client,
|
|
21
|
-
plainClient,
|
|
22
26
|
cdaClient,
|
|
23
27
|
spaceId,
|
|
24
28
|
environmentId = 'master',
|
|
@@ -43,22 +47,32 @@ function getFullSourceSpace({
|
|
|
43
47
|
renderer: _listrVerboseRenderer.default
|
|
44
48
|
};
|
|
45
49
|
return new _listr.default([{
|
|
46
|
-
title: 'Connecting to space',
|
|
47
|
-
task: (0, _contentfulBatchLibs.wrapTask)(
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
50
|
+
title: 'Connecting to space/environment',
|
|
51
|
+
task: (0, _contentfulBatchLibs.wrapTask)(async () => {
|
|
52
|
+
try {
|
|
53
|
+
await client.space.get({
|
|
54
|
+
spaceId
|
|
55
|
+
});
|
|
56
|
+
} catch (err) {
|
|
57
|
+
throw new Error(`Unable to retrieve space ${spaceId}, please ensure the space exists and the token is valid. (${err.message})`);
|
|
58
|
+
}
|
|
59
|
+
try {
|
|
60
|
+
await client.environment.get({
|
|
61
|
+
spaceId,
|
|
62
|
+
environmentId
|
|
63
|
+
});
|
|
64
|
+
} catch (err) {
|
|
65
|
+
throw new Error(`Unable to retrieve environment ${environmentId}, please ensure the environment exists and the token is valid. (${err.message})`);
|
|
66
|
+
}
|
|
54
67
|
})
|
|
55
68
|
}, {
|
|
56
69
|
title: 'Fetching content types data',
|
|
57
70
|
task: (0, _contentfulBatchLibs.wrapTask)(ctx => {
|
|
58
|
-
return pagedGet({
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
71
|
+
return pagedGet(query => client.contentType.getMany({
|
|
72
|
+
spaceId,
|
|
73
|
+
environmentId,
|
|
74
|
+
query
|
|
75
|
+
})).then(extractItems).then(items => {
|
|
62
76
|
ctx.data.contentTypes = items;
|
|
63
77
|
});
|
|
64
78
|
}),
|
|
@@ -66,20 +80,24 @@ function getFullSourceSpace({
|
|
|
66
80
|
}, {
|
|
67
81
|
title: 'Fetching tags data',
|
|
68
82
|
task: (0, _contentfulBatchLibs.wrapTask)(ctx => {
|
|
69
|
-
return pagedGet({
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
83
|
+
return pagedGet(query => client.tag.getMany({
|
|
84
|
+
spaceId,
|
|
85
|
+
environmentId,
|
|
86
|
+
query
|
|
87
|
+
})).then(extractItems).then(items => {
|
|
73
88
|
ctx.data.tags = items;
|
|
74
|
-
}).catch(
|
|
75
|
-
|
|
89
|
+
}).catch(err => {
|
|
90
|
+
_contentfulBatchLibs.logEmitter.emit('error', new Error(`Fetching tags failed: ${err.message}`, {
|
|
91
|
+
cause: err
|
|
92
|
+
}));
|
|
93
|
+
throw err;
|
|
76
94
|
});
|
|
77
95
|
}),
|
|
78
96
|
skip: () => skipTags
|
|
79
97
|
}, {
|
|
80
98
|
title: 'Fetching editor interfaces data',
|
|
81
99
|
task: (0, _contentfulBatchLibs.wrapTask)(ctx => {
|
|
82
|
-
return getEditorInterfaces(ctx.data.contentTypes).then(editorInterfaces => {
|
|
100
|
+
return getEditorInterfaces(client, spaceId, environmentId, ctx.data.contentTypes).then(editorInterfaces => {
|
|
83
101
|
ctx.data.editorInterfaces = editorInterfaces.filter(editorInterface => {
|
|
84
102
|
return editorInterface !== null;
|
|
85
103
|
});
|
|
@@ -89,17 +107,17 @@ function getFullSourceSpace({
|
|
|
89
107
|
}, {
|
|
90
108
|
title: 'Fetching content entries data',
|
|
91
109
|
task: (0, _contentfulBatchLibs.wrapTask)(ctx => {
|
|
92
|
-
const source = (cdaClient === null || cdaClient === void 0 ? void 0 : cdaClient.withAllLocales) || ctx.environment;
|
|
93
110
|
if (cdaClient) {
|
|
94
111
|
// let's not fetch children when using Content Delivery API
|
|
95
112
|
queryEntries = queryEntries || {};
|
|
96
113
|
queryEntries.include = 0;
|
|
97
114
|
}
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
query
|
|
102
|
-
})
|
|
115
|
+
const fetchFn = cdaClient ? query => cdaClient.withAllLocales.getEntries(query) : query => client.entry.getMany({
|
|
116
|
+
spaceId,
|
|
117
|
+
environmentId,
|
|
118
|
+
query
|
|
119
|
+
});
|
|
120
|
+
return pagedGet(fetchFn, queryEntries).then(extractItems).then(items => filterDrafts(items, includeDrafts, cdaClient)).then(items => filterArchived(items, includeArchived)).then(items => removeTags(items, stripTags)).then(items => {
|
|
103
121
|
ctx.data.entries = items;
|
|
104
122
|
});
|
|
105
123
|
}),
|
|
@@ -107,13 +125,13 @@ function getFullSourceSpace({
|
|
|
107
125
|
}, {
|
|
108
126
|
title: 'Fetching assets data',
|
|
109
127
|
task: (0, _contentfulBatchLibs.wrapTask)(ctx => {
|
|
110
|
-
const source = (cdaClient === null || cdaClient === void 0 ? void 0 : cdaClient.withAllLocales) || ctx.environment;
|
|
111
128
|
queryAssets = queryAssets || {};
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
query
|
|
116
|
-
})
|
|
129
|
+
const fetchFn = cdaClient ? query => cdaClient.withAllLocales.getAssets(query) : query => client.asset.getMany({
|
|
130
|
+
spaceId,
|
|
131
|
+
environmentId,
|
|
132
|
+
query
|
|
133
|
+
});
|
|
134
|
+
return pagedGet(fetchFn, queryAssets).then(extractItems).then(items => filterDrafts(items, includeDrafts, cdaClient)).then(items => filterArchived(items, includeArchived)).then(items => removeTags(items, stripTags)).then(items => {
|
|
117
135
|
ctx.data.assets = items;
|
|
118
136
|
});
|
|
119
137
|
}),
|
|
@@ -121,33 +139,34 @@ function getFullSourceSpace({
|
|
|
121
139
|
}, {
|
|
122
140
|
title: 'Fetching locales data',
|
|
123
141
|
task: (0, _contentfulBatchLibs.wrapTask)(ctx => {
|
|
124
|
-
return pagedGet({
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
142
|
+
return pagedGet(query => client.locale.getMany({
|
|
143
|
+
spaceId,
|
|
144
|
+
environmentId,
|
|
145
|
+
query
|
|
146
|
+
})).then(extractItems).then(items => {
|
|
128
147
|
ctx.data.locales = items;
|
|
129
148
|
});
|
|
130
149
|
}),
|
|
131
150
|
skip: () => skipContentModel
|
|
132
151
|
}, {
|
|
133
152
|
title: 'Fetching webhooks data',
|
|
134
|
-
task: (0, _contentfulBatchLibs.wrapTask)(ctx => {
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
153
|
+
task: (0, _contentfulBatchLibs.wrapTask)(async ctx => {
|
|
154
|
+
const webhooksResponse = await client.webhook.getMany({
|
|
155
|
+
query: {
|
|
156
|
+
// webhooks are capped to 100 per space
|
|
157
|
+
limit: 100
|
|
158
|
+
},
|
|
159
|
+
spaceId
|
|
140
160
|
});
|
|
161
|
+
ctx.data.webhooks = webhooksResponse.items;
|
|
141
162
|
}),
|
|
142
163
|
skip: () => skipWebhooks || environmentId !== 'master' && 'Webhooks can only be exported from master environment'
|
|
143
164
|
}, {
|
|
144
165
|
title: 'Fetching roles data',
|
|
145
|
-
task: (0, _contentfulBatchLibs.wrapTask)(ctx => {
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
}).then(extractItems).then(items => {
|
|
150
|
-
ctx.data.roles = items;
|
|
166
|
+
task: (0, _contentfulBatchLibs.wrapTask)(async ctx => {
|
|
167
|
+
ctx.data.roles = await pagedRolesGet({
|
|
168
|
+
client,
|
|
169
|
+
spaceId
|
|
151
170
|
});
|
|
152
171
|
}),
|
|
153
172
|
skip: () => skipRoles || environmentId !== 'master' && 'Roles can only be exported from master environment'
|
|
@@ -155,12 +174,11 @@ function getFullSourceSpace({
|
|
|
155
174
|
title: 'Fetching Design Tokens data',
|
|
156
175
|
task: (0, _contentfulBatchLibs.wrapTask)(async ctx => {
|
|
157
176
|
try {
|
|
158
|
-
ctx.data.designTokens = await cursorPagedGet({
|
|
159
|
-
client: plainClient,
|
|
177
|
+
ctx.data.designTokens = await cursorPagedGet(query => client.designToken.getMany({
|
|
160
178
|
spaceId,
|
|
161
179
|
environmentId,
|
|
162
|
-
|
|
163
|
-
});
|
|
180
|
+
query
|
|
181
|
+
}), 'designToken');
|
|
164
182
|
} catch (err) {
|
|
165
183
|
_contentfulBatchLibs.logEmitter.emit('warning', `Skipping Design Tokens export: ${err.message}`);
|
|
166
184
|
ctx.data.designTokens = [];
|
|
@@ -171,12 +189,11 @@ function getFullSourceSpace({
|
|
|
171
189
|
title: 'Fetching Components data',
|
|
172
190
|
task: (0, _contentfulBatchLibs.wrapTask)(async ctx => {
|
|
173
191
|
try {
|
|
174
|
-
ctx.data.components = await cursorPagedGet({
|
|
175
|
-
client: plainClient,
|
|
192
|
+
ctx.data.components = await cursorPagedGet(query => client.component.getMany({
|
|
176
193
|
spaceId,
|
|
177
194
|
environmentId,
|
|
178
|
-
|
|
179
|
-
});
|
|
195
|
+
query
|
|
196
|
+
}), 'component');
|
|
180
197
|
} catch (err) {
|
|
181
198
|
_contentfulBatchLibs.logEmitter.emit('warning', `Skipping Components export: ${err.message}`);
|
|
182
199
|
ctx.data.components = [];
|
|
@@ -187,12 +204,11 @@ function getFullSourceSpace({
|
|
|
187
204
|
title: 'Fetching Experience Templates data',
|
|
188
205
|
task: (0, _contentfulBatchLibs.wrapTask)(async ctx => {
|
|
189
206
|
try {
|
|
190
|
-
ctx.data.experienceTemplates = await cursorPagedGet({
|
|
191
|
-
client: plainClient,
|
|
207
|
+
ctx.data.experienceTemplates = await cursorPagedGet(query => client.experienceTemplate.getMany({
|
|
192
208
|
spaceId,
|
|
193
209
|
environmentId,
|
|
194
|
-
|
|
195
|
-
});
|
|
210
|
+
query
|
|
211
|
+
}), 'experienceTemplate');
|
|
196
212
|
} catch (err) {
|
|
197
213
|
_contentfulBatchLibs.logEmitter.emit('warning', `Skipping Experience Templates export: ${err.message}`);
|
|
198
214
|
ctx.data.experienceTemplates = [];
|
|
@@ -203,12 +219,11 @@ function getFullSourceSpace({
|
|
|
203
219
|
title: 'Fetching Data Assemblies data',
|
|
204
220
|
task: (0, _contentfulBatchLibs.wrapTask)(async ctx => {
|
|
205
221
|
try {
|
|
206
|
-
ctx.data.dataAssemblies = await cursorPagedGet({
|
|
207
|
-
client: plainClient,
|
|
222
|
+
ctx.data.dataAssemblies = await cursorPagedGet(query => client.dataAssembly.getMany({
|
|
208
223
|
spaceId,
|
|
209
224
|
environmentId,
|
|
210
|
-
|
|
211
|
-
});
|
|
225
|
+
query
|
|
226
|
+
}), 'dataAssembly');
|
|
212
227
|
} catch (err) {
|
|
213
228
|
_contentfulBatchLibs.logEmitter.emit('warning', `Skipping Data Assemblies export: ${err.message}`);
|
|
214
229
|
ctx.data.dataAssemblies = [];
|
|
@@ -219,12 +234,11 @@ function getFullSourceSpace({
|
|
|
219
234
|
title: 'Fetching Experience Fragments data',
|
|
220
235
|
task: (0, _contentfulBatchLibs.wrapTask)(async ctx => {
|
|
221
236
|
try {
|
|
222
|
-
ctx.data.experienceFragments = await cursorPagedGet({
|
|
223
|
-
client: plainClient,
|
|
237
|
+
ctx.data.experienceFragments = await cursorPagedGet(query => client.experienceFragment.getMany({
|
|
224
238
|
spaceId,
|
|
225
239
|
environmentId,
|
|
226
|
-
|
|
227
|
-
});
|
|
240
|
+
query
|
|
241
|
+
}), 'experienceFragment');
|
|
228
242
|
} catch (err) {
|
|
229
243
|
_contentfulBatchLibs.logEmitter.emit('warning', `Skipping Experience Fragments export: ${err.message}`);
|
|
230
244
|
ctx.data.experienceFragments = [];
|
|
@@ -235,12 +249,11 @@ function getFullSourceSpace({
|
|
|
235
249
|
title: 'Fetching Experiences data',
|
|
236
250
|
task: (0, _contentfulBatchLibs.wrapTask)(async ctx => {
|
|
237
251
|
try {
|
|
238
|
-
ctx.data.experiences = await cursorPagedGet({
|
|
239
|
-
client: plainClient,
|
|
252
|
+
ctx.data.experiences = await cursorPagedGet(query => client.experience.getMany({
|
|
240
253
|
spaceId,
|
|
241
254
|
environmentId,
|
|
242
|
-
|
|
243
|
-
});
|
|
255
|
+
query
|
|
256
|
+
}), 'experience');
|
|
244
257
|
} catch (err) {
|
|
245
258
|
_contentfulBatchLibs.logEmitter.emit('warning', `Skipping Experiences export: ${err.message}`);
|
|
246
259
|
ctx.data.experiences = [];
|
|
@@ -249,15 +262,19 @@ function getFullSourceSpace({
|
|
|
249
262
|
skip: () => !includeExperienceOrchestration
|
|
250
263
|
}], listrOptions);
|
|
251
264
|
}
|
|
252
|
-
function getEditorInterfaces(contentTypes) {
|
|
265
|
+
function getEditorInterfaces(client, spaceId, environmentId, contentTypes) {
|
|
253
266
|
return _bluebird.default.map(contentTypes, contentType => {
|
|
254
|
-
return
|
|
267
|
+
return client.editorInterface.get({
|
|
268
|
+
spaceId,
|
|
269
|
+
environmentId,
|
|
270
|
+
contentTypeId: contentType.sys.id
|
|
271
|
+
}).then(editorInterface => {
|
|
255
272
|
_contentfulBatchLibs.logEmitter.emit('info', `Fetched editor interface for ${contentType.name}`);
|
|
256
273
|
return editorInterface;
|
|
257
274
|
}).catch(() => {
|
|
258
275
|
// old contentTypes may not have an editor interface but we'll handle in a later stage
|
|
259
276
|
// but it should not stop getting the data process
|
|
260
|
-
_contentfulBatchLibs.logEmitter.emit('warning', `No editor interface found for ${contentType}`);
|
|
277
|
+
_contentfulBatchLibs.logEmitter.emit('warning', `No editor interface found for ${contentType.name || contentType.sys.id}`);
|
|
261
278
|
return _bluebird.default.resolve(null);
|
|
262
279
|
});
|
|
263
280
|
}, {
|
|
@@ -269,55 +286,101 @@ function getEditorInterfaces(contentTypes) {
|
|
|
269
286
|
* Gets all ExO entities using cursor-based pagination (pageNext/pagePrev tokens).
|
|
270
287
|
* ExO list endpoints do not support skip-based pagination or the order param.
|
|
271
288
|
*/
|
|
272
|
-
async function cursorPagedGet({
|
|
273
|
-
client,
|
|
274
|
-
spaceId,
|
|
275
|
-
environmentId,
|
|
276
|
-
method
|
|
277
|
-
}) {
|
|
278
|
-
const [entity, operation] = method.split('.');
|
|
289
|
+
async function cursorPagedGet(fetchFn, entityLabel) {
|
|
279
290
|
const allItems = [];
|
|
280
291
|
let pageNext = null;
|
|
281
292
|
do {
|
|
282
293
|
var _response$pages$next, _response$pages;
|
|
283
294
|
const query = {
|
|
284
|
-
spaceId,
|
|
285
|
-
environmentId,
|
|
286
295
|
limit: pageLimit
|
|
287
296
|
};
|
|
288
297
|
if (pageNext) {
|
|
289
298
|
query.pageNext = pageNext;
|
|
290
299
|
}
|
|
291
|
-
const response = await
|
|
300
|
+
const response = await fetchFn(query);
|
|
292
301
|
allItems.push(...response.items);
|
|
293
|
-
_contentfulBatchLibs.logEmitter.emit('info', `Fetched ${allItems.length} ${
|
|
302
|
+
_contentfulBatchLibs.logEmitter.emit('info', `Fetched ${allItems.length} ${entityLabel} items`);
|
|
294
303
|
pageNext = (_response$pages$next = (_response$pages = response.pages) === null || _response$pages === void 0 ? void 0 : _response$pages.next) !== null && _response$pages$next !== void 0 ? _response$pages$next : null;
|
|
295
304
|
} while (pageNext);
|
|
296
305
|
return allItems;
|
|
297
306
|
}
|
|
298
307
|
|
|
299
308
|
/**
|
|
309
|
+
* Gets all roles. Roles are scheduled to switch from skip/limit to
|
|
310
|
+
* cursor-based (pages.next/pagePrev) pagination on Feb 15, 2027:
|
|
311
|
+
* https://www.contentful.com/developers/api-changes/space-roles-collection-endpoints-update/
|
|
312
|
+
* Handles both shapes so this keeps working across that migration.
|
|
313
|
+
*/
|
|
314
|
+
async function pagedRolesGet({
|
|
315
|
+
client,
|
|
316
|
+
spaceId
|
|
317
|
+
}) {
|
|
318
|
+
const allItems = [];
|
|
319
|
+
const order = 'sys.createdAt,sys.id';
|
|
320
|
+
let response = await client.role.getMany({
|
|
321
|
+
spaceId,
|
|
322
|
+
query: {
|
|
323
|
+
limit: pageLimit,
|
|
324
|
+
order
|
|
325
|
+
}
|
|
326
|
+
});
|
|
327
|
+
allItems.push(...response.items);
|
|
328
|
+
_contentfulBatchLibs.logEmitter.emit('info', `Fetched ${allItems.length} roles`);
|
|
329
|
+
if (response.pages !== undefined) {
|
|
330
|
+
var _response$pages$next2, _response$pages2;
|
|
331
|
+
let pageNext = (_response$pages$next2 = (_response$pages2 = response.pages) === null || _response$pages2 === void 0 ? void 0 : _response$pages2.next) !== null && _response$pages$next2 !== void 0 ? _response$pages$next2 : null;
|
|
332
|
+
while (pageNext) {
|
|
333
|
+
var _response$pages$next3, _response$pages3;
|
|
334
|
+
response = await client.role.getMany({
|
|
335
|
+
spaceId,
|
|
336
|
+
query: {
|
|
337
|
+
limit: pageLimit,
|
|
338
|
+
order,
|
|
339
|
+
pageNext
|
|
340
|
+
}
|
|
341
|
+
});
|
|
342
|
+
allItems.push(...response.items);
|
|
343
|
+
_contentfulBatchLibs.logEmitter.emit('info', `Fetched ${allItems.length} roles`);
|
|
344
|
+
pageNext = (_response$pages$next3 = (_response$pages3 = response.pages) === null || _response$pages3 === void 0 ? void 0 : _response$pages3.next) !== null && _response$pages$next3 !== void 0 ? _response$pages$next3 : null;
|
|
345
|
+
}
|
|
346
|
+
} else {
|
|
347
|
+
let skip = allItems.length;
|
|
348
|
+
while (allItems.length < response.total && response.items.length > 0) {
|
|
349
|
+
response = await client.role.getMany({
|
|
350
|
+
spaceId,
|
|
351
|
+
query: {
|
|
352
|
+
limit: pageLimit,
|
|
353
|
+
order,
|
|
354
|
+
skip
|
|
355
|
+
}
|
|
356
|
+
});
|
|
357
|
+
allItems.push(...response.items);
|
|
358
|
+
_contentfulBatchLibs.logEmitter.emit('info', `Fetched ${allItems.length} roles`);
|
|
359
|
+
skip += response.items.length;
|
|
360
|
+
}
|
|
361
|
+
}
|
|
362
|
+
return allItems;
|
|
363
|
+
}
|
|
364
|
+
|
|
365
|
+
/**
|
|
366
|
+
* Recursively fetches all pages of a offset-based paginated CMA collection and returns
|
|
367
|
+
* them merged into a single response shaped like `{ items: [...], total, ... }`
|
|
368
|
+
*
|
|
300
369
|
* Gets all the existing entities based on pagination parameters.
|
|
301
370
|
* The first call will have no aggregated response. Subsequent calls will
|
|
302
371
|
* concatenate the new responses to the original one.
|
|
303
372
|
*/
|
|
304
|
-
function pagedGet({
|
|
305
|
-
|
|
306
|
-
method,
|
|
307
|
-
skip = 0,
|
|
308
|
-
aggregatedResponse = null,
|
|
309
|
-
query = null
|
|
310
|
-
}) {
|
|
311
|
-
const userQueryLimit = query && query.limit;
|
|
373
|
+
function pagedGet(fetchFn, userQuery = null, skip = 0, aggregatedResponse = null) {
|
|
374
|
+
const userQueryLimit = userQuery && userQuery.limit;
|
|
312
375
|
const fetchedTotal = aggregatedResponse && aggregatedResponse.items.length;
|
|
313
376
|
const limit = userQueryLimit ? Math.min(pageLimit, userQueryLimit - fetchedTotal) : pageLimit;
|
|
314
|
-
const
|
|
377
|
+
const query = _objectSpread(_objectSpread({
|
|
315
378
|
skip,
|
|
316
379
|
order: 'sys.createdAt,sys.id'
|
|
317
|
-
},
|
|
380
|
+
}, userQuery), {}, {
|
|
318
381
|
limit
|
|
319
382
|
});
|
|
320
|
-
return
|
|
383
|
+
return fetchFn(query).then(response => {
|
|
321
384
|
if (!aggregatedResponse) {
|
|
322
385
|
aggregatedResponse = response;
|
|
323
386
|
} else {
|
|
@@ -325,20 +388,14 @@ function pagedGet({
|
|
|
325
388
|
}
|
|
326
389
|
const totalItemsLength = aggregatedResponse.items.length;
|
|
327
390
|
const total = response.total;
|
|
328
|
-
logPagingStatus(response,
|
|
391
|
+
logPagingStatus(response, query, userQueryLimit);
|
|
329
392
|
const gotAllQueryLimitedItems = userQueryLimit && totalItemsLength >= userQueryLimit;
|
|
330
393
|
const gotAllItems = totalItemsLength >= total;
|
|
331
394
|
const gotNoItems = totalItemsLength <= 0;
|
|
332
395
|
if (gotAllQueryLimitedItems || gotAllItems || gotNoItems) {
|
|
333
396
|
return aggregatedResponse;
|
|
334
397
|
}
|
|
335
|
-
return pagedGet(
|
|
336
|
-
source,
|
|
337
|
-
method,
|
|
338
|
-
skip: skip + response.items.length,
|
|
339
|
-
aggregatedResponse,
|
|
340
|
-
query
|
|
341
|
-
});
|
|
398
|
+
return pagedGet(fetchFn, userQuery, skip + response.items.length, aggregatedResponse);
|
|
342
399
|
});
|
|
343
400
|
}
|
|
344
401
|
function logPagingStatus(response, requestQuery, userLimit) {
|
|
@@ -4,7 +4,6 @@ Object.defineProperty(exports, "__esModule", {
|
|
|
4
4
|
value: true
|
|
5
5
|
});
|
|
6
6
|
exports.default = initClient;
|
|
7
|
-
exports.initPlainClient = initPlainClient;
|
|
8
7
|
var _contentful = require("contentful");
|
|
9
8
|
var _contentfulBatchLibs = require("contentful-batch-libs");
|
|
10
9
|
var _contentfulManagement = require("contentful-management");
|
|
@@ -32,13 +31,10 @@ function initClient(opts, useCda = false) {
|
|
|
32
31
|
return (0, _contentful.createClient)(cdaConfig).withoutLinkResolution;
|
|
33
32
|
}
|
|
34
33
|
return (0, _contentfulManagement.createClient)(config, {
|
|
35
|
-
|
|
34
|
+
defaults: {
|
|
35
|
+
spaceId: config.spaceId,
|
|
36
|
+
environmentId: config.environmentId
|
|
37
|
+
}
|
|
36
38
|
});
|
|
37
39
|
}
|
|
38
|
-
|
|
39
|
-
return (0, _contentfulManagement.createClient)({
|
|
40
|
-
accessToken: opts.managementToken,
|
|
41
|
-
host: opts.host,
|
|
42
|
-
logHandler
|
|
43
|
-
});
|
|
44
|
-
}
|
|
40
|
+
module.exports = exports.default;
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "contentful-export",
|
|
3
|
-
"version": "8.
|
|
3
|
+
"version": "8.3.0",
|
|
4
4
|
"description": "this tool allows you to export a space to a JSON dump",
|
|
5
5
|
"main": "dist/index.js",
|
|
6
6
|
"types": "types.d.ts",
|
|
@@ -46,6 +46,7 @@
|
|
|
46
46
|
},
|
|
47
47
|
"dependencies": {
|
|
48
48
|
"axios": "^1.13.5",
|
|
49
|
+
"axios-retry": "^4.5.0",
|
|
49
50
|
"bfj": "^9.1.3",
|
|
50
51
|
"bluebird": "^3.3.3",
|
|
51
52
|
"cli-table3": "^0.6.0",
|