contentful-export 8.2.1 → 8.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.js CHANGED
@@ -18,9 +18,8 @@ var _formatDistance = require("date-fns/formatDistance");
18
18
  var _contentfulBatchLibs = require("contentful-batch-libs");
19
19
  var _downloadAssets = _interopRequireDefault(require("./tasks/download-assets"));
20
20
  var _getSpaceData = _interopRequireDefault(require("./tasks/get-space-data"));
21
- var _initClient = _interopRequireWildcard(require("./tasks/init-client"));
21
+ var _initClient = _interopRequireDefault(require("./tasks/init-client"));
22
22
  var _parseOptions = _interopRequireDefault(require("./parseOptions"));
23
- function _interopRequireWildcard(e, t) { if ("function" == typeof WeakMap) var r = new WeakMap(), n = new WeakMap(); return (_interopRequireWildcard = function (e, t) { if (!t && e && e.__esModule) return e; var o, i, f = { __proto__: null, default: e }; if (null === e || "object" != typeof e && "function" != typeof e) return f; if (o = t ? n : r) { if (o.has(e)) return o.get(e); o.set(e, f); } for (const t in e) "default" !== t && {}.hasOwnProperty.call(e, t) && ((i = (o = Object.defineProperty) && Object.getOwnPropertyDescriptor(e, t)) && (i.get || i.set) ? o(f, t, i) : f[t] = e[t]); return f; })(e, t); }
24
23
  function _interopRequireDefault(e) { return e && e.__esModule ? e : { default: e }; }
25
24
  const accessP = _bluebird.default.promisify(_fs.access);
26
25
  const tableOptions = {
@@ -54,7 +53,6 @@ function runContentfulExport(params) {
54
53
  try {
55
54
  // CMA client
56
55
  ctx.client = (0, _initClient.default)(options);
57
- ctx.plainClient = (0, _initClient.initPlainClient)(options);
58
56
  if (options.deliveryToken && !options.includeDrafts) {
59
57
  // CDA client for fetching only public entries
60
58
  ctx.cdaClient = (0, _initClient.default)(options, true);
@@ -69,7 +67,6 @@ function runContentfulExport(params) {
69
67
  task: ctx => {
70
68
  return (0, _getSpaceData.default)({
71
69
  client: ctx.client,
72
- plainClient: ctx.plainClient,
73
70
  cdaClient: ctx.cdaClient,
74
71
  spaceId: options.spaceId,
75
72
  environmentId: options.environmentId,
@@ -13,19 +13,26 @@ var _stream = require("stream");
13
13
  var _util = require("util");
14
14
  var _embargoedAssets = require("../utils/embargoedAssets");
15
15
  var _axios = _interopRequireDefault(require("axios"));
16
+ var _axiosRetry = _interopRequireDefault(require("axios-retry"));
16
17
  function _interopRequireDefault(e) { return e && e.__esModule ? e : { default: e }; }
17
18
  const streamPipeline = (0, _util.promisify)(_stream.pipeline);
18
19
 
20
+ // For streaming responses, axios timeout applies to time-to-first-byte, not total transfer duration
21
+ const DEFAULT_DOWNLOAD_TIMEOUT_MS = 30000;
22
+ const DEFAULT_RETRY_LIMIT = 3;
23
+
19
24
  /**
20
25
  * @param {Object} options - The options for downloading the asset.
21
26
  * @param {string} options.url - The URL of the asset to download.
22
27
  * @param {string} options.directory - The directory where the asset should be saved.
23
28
  * @param {import('axios').AxiosInstance} options.httpClient - The HTTP client to use for downloading the asset.
29
+ * @param {string} options.assetId - The ID of the asset being downloaded, used in error messages.
24
30
  */
25
31
  async function downloadAsset({
26
32
  url,
27
33
  directory,
28
- httpClient
34
+ httpClient,
35
+ assetId
29
36
  }) {
30
37
  // handle urls without protocol
31
38
  if (url.startsWith('//')) {
@@ -57,7 +64,14 @@ async function downloadAsset({
57
64
  * @type {import('axios').AxiosError}
58
65
  */
59
66
  const axiosError = e;
60
- throw new Error(`error response status: ${axiosError.response.status}`);
67
+ if (axiosError.response) {
68
+ throw new Error(`error downloading asset ${assetId} (${url}): HTTP ${axiosError.response.status} ${axiosError.response.statusText}`, {
69
+ cause: axiosError
70
+ });
71
+ }
72
+ throw new Error(`error downloading asset ${assetId} (${url}): ${e.message}`, {
73
+ cause: e
74
+ });
61
75
  }
62
76
  }
63
77
  function downloadAssets(options) {
@@ -67,11 +81,24 @@ function downloadAssets(options) {
67
81
  let errorCount = 0;
68
82
  const httpClient = _axios.default.create({
69
83
  headers: options.headers,
70
- timeout: options.timeout,
84
+ timeout: DEFAULT_DOWNLOAD_TIMEOUT_MS,
71
85
  httpAgent: options.httpAgent,
72
86
  httpsAgent: options.httpsAgent,
73
87
  proxy: options.proxy
74
88
  });
89
+ (0, _axiosRetry.default)(httpClient, {
90
+ retries: DEFAULT_RETRY_LIMIT,
91
+ retryDelay: _axiosRetry.default.exponentialDelay,
92
+ shouldResetTimeout: true,
93
+ retryCondition: error => {
94
+ var _error$response;
95
+ return _axiosRetry.default.isNetworkOrIdempotentRequestError(error) || ((_error$response = error.response) === null || _error$response === void 0 ? void 0 : _error$response.status) === 429;
96
+ },
97
+ onRetry: (retryCount, error) => {
98
+ const status = error.response ? `HTTP ${error.response.status}` : error.code || error.message;
99
+ _contentfulBatchLibs.logEmitter.emit('warning', `Asset download failed (${status}), retrying (attempt ${retryCount})...`);
100
+ }
101
+ });
75
102
  return _bluebird.default.map(ctx.data.assets, asset => {
76
103
  const entityName = (0, _contentfulBatchLibs.getEntityName)(asset);
77
104
  if (!asset.fields.file) {
@@ -87,10 +114,12 @@ function downloadAssets(options) {
87
114
  errorCount++;
88
115
  return _bluebird.default.resolve();
89
116
  }
117
+ const assetId = asset.sys.id;
90
118
  let startingPromise = _bluebird.default.resolve({
91
119
  url,
92
120
  directory: options.exportDir,
93
- httpClient
121
+ httpClient,
122
+ assetId
94
123
  });
95
124
  if ((0, _embargoedAssets.isEmbargoedAsset)(url)) {
96
125
  const {
@@ -103,13 +132,15 @@ function downloadAssets(options) {
103
132
  startingPromise = (0, _embargoedAssets.signUrl)(host, accessToken, spaceId, environmentId, url, expiresAtMs, httpClient).then(signedUrl => ({
104
133
  url: signedUrl,
105
134
  directory: options.exportDir,
106
- httpClient
135
+ httpClient,
136
+ assetId
107
137
  }));
108
138
  }
109
139
  return startingPromise.then(downloadAsset).then(() => {
110
140
  task.output = `${_figures.default.tick} downloaded ${entityName} (${url})`;
111
141
  successCount++;
112
142
  }).catch(error => {
143
+ _contentfulBatchLibs.logEmitter.emit('warning', error.message);
113
144
  task.output = `${_figures.default.cross} error downloading ${url}: ${error.message}`;
114
145
  errorCount++;
115
146
  });
@@ -9,6 +9,11 @@ var _contentfulBatchLibs = require("contentful-batch-libs");
9
9
  var _listr = _interopRequireDefault(require("listr"));
10
10
  var _listrVerboseRenderer = _interopRequireDefault(require("listr-verbose-renderer"));
11
11
  function _interopRequireDefault(e) { return e && e.__esModule ? e : { default: e }; }
12
+ function ownKeys(e, r) { var t = Object.keys(e); if (Object.getOwnPropertySymbols) { var o = Object.getOwnPropertySymbols(e); r && (o = o.filter(function (r) { return Object.getOwnPropertyDescriptor(e, r).enumerable; })), t.push.apply(t, o); } return t; }
13
+ function _objectSpread(e) { for (var r = 1; r < arguments.length; r++) { var t = null != arguments[r] ? arguments[r] : {}; r % 2 ? ownKeys(Object(t), !0).forEach(function (r) { _defineProperty(e, r, t[r]); }) : Object.getOwnPropertyDescriptors ? Object.defineProperties(e, Object.getOwnPropertyDescriptors(t)) : ownKeys(Object(t)).forEach(function (r) { Object.defineProperty(e, r, Object.getOwnPropertyDescriptor(t, r)); }); } return e; }
14
+ function _defineProperty(e, r, t) { return (r = _toPropertyKey(r)) in e ? Object.defineProperty(e, r, { value: t, enumerable: !0, configurable: !0, writable: !0 }) : e[r] = t, e; }
15
+ function _toPropertyKey(t) { var i = _toPrimitive(t, "string"); return "symbol" == typeof i ? i : i + ""; }
16
+ function _toPrimitive(t, r) { if ("object" != typeof t || !t) return t; var e = t[Symbol.toPrimitive]; if (void 0 !== e) { var i = e.call(t, r || "default"); if ("object" != typeof i) return i; throw new TypeError("@@toPrimitive must return a primitive value."); } return ("string" === r ? String : Number)(t); }
12
17
  const MAX_ALLOWED_LIMIT = 1000;
13
18
  let pageLimit = MAX_ALLOWED_LIMIT;
14
19
 
@@ -18,7 +23,6 @@ let pageLimit = MAX_ALLOWED_LIMIT;
18
23
  */
19
24
  function getFullSourceSpace({
20
25
  client,
21
- plainClient,
22
26
  cdaClient,
23
27
  spaceId,
24
28
  environmentId = 'master',
@@ -43,22 +47,32 @@ function getFullSourceSpace({
43
47
  renderer: _listrVerboseRenderer.default
44
48
  };
45
49
  return new _listr.default([{
46
- title: 'Connecting to space',
47
- task: (0, _contentfulBatchLibs.wrapTask)(ctx => {
48
- return client.getSpace(spaceId).then(space => {
49
- ctx.space = space;
50
- return space.getEnvironment(environmentId);
51
- }).then(environment => {
52
- ctx.environment = environment;
53
- });
50
+ title: 'Connecting to space/environment',
51
+ task: (0, _contentfulBatchLibs.wrapTask)(async () => {
52
+ try {
53
+ await client.space.get({
54
+ spaceId
55
+ });
56
+ } catch (err) {
57
+ throw new Error(`Unable to retrieve space ${spaceId}, please ensure the space exists and the token is valid. (${err.message})`);
58
+ }
59
+ try {
60
+ await client.environment.get({
61
+ spaceId,
62
+ environmentId
63
+ });
64
+ } catch (err) {
65
+ throw new Error(`Unable to retrieve environment ${environmentId}, please ensure the environment exists and the token is valid. (${err.message})`);
66
+ }
54
67
  })
55
68
  }, {
56
69
  title: 'Fetching content types data',
57
70
  task: (0, _contentfulBatchLibs.wrapTask)(ctx => {
58
- return pagedGet({
59
- source: ctx.environment,
60
- method: 'getContentTypes'
61
- }).then(extractItems).then(items => {
71
+ return pagedGet(query => client.contentType.getMany({
72
+ spaceId,
73
+ environmentId,
74
+ query
75
+ })).then(extractItems).then(items => {
62
76
  ctx.data.contentTypes = items;
63
77
  });
64
78
  }),
@@ -66,20 +80,24 @@ function getFullSourceSpace({
66
80
  }, {
67
81
  title: 'Fetching tags data',
68
82
  task: (0, _contentfulBatchLibs.wrapTask)(ctx => {
69
- return pagedGet({
70
- source: ctx.environment,
71
- method: 'getTags'
72
- }).then(extractItems).then(items => {
83
+ return pagedGet(query => client.tag.getMany({
84
+ spaceId,
85
+ environmentId,
86
+ query
87
+ })).then(extractItems).then(items => {
73
88
  ctx.data.tags = items;
74
- }).catch(() => {
75
- ctx.data.tags = [];
89
+ }).catch(err => {
90
+ _contentfulBatchLibs.logEmitter.emit('error', new Error(`Fetching tags failed: ${err.message}`, {
91
+ cause: err
92
+ }));
93
+ throw err;
76
94
  });
77
95
  }),
78
96
  skip: () => skipTags
79
97
  }, {
80
98
  title: 'Fetching editor interfaces data',
81
99
  task: (0, _contentfulBatchLibs.wrapTask)(ctx => {
82
- return getEditorInterfaces(ctx.data.contentTypes).then(editorInterfaces => {
100
+ return getEditorInterfaces(client, spaceId, environmentId, ctx.data.contentTypes).then(editorInterfaces => {
83
101
  ctx.data.editorInterfaces = editorInterfaces.filter(editorInterface => {
84
102
  return editorInterface !== null;
85
103
  });
@@ -89,17 +107,17 @@ function getFullSourceSpace({
89
107
  }, {
90
108
  title: 'Fetching content entries data',
91
109
  task: (0, _contentfulBatchLibs.wrapTask)(ctx => {
92
- const source = (cdaClient === null || cdaClient === void 0 ? void 0 : cdaClient.withAllLocales) || ctx.environment;
93
110
  if (cdaClient) {
94
111
  // let's not fetch children when using Content Delivery API
95
112
  queryEntries = queryEntries || {};
96
113
  queryEntries.include = 0;
97
114
  }
98
- return pagedGet({
99
- source,
100
- method: 'getEntries',
101
- query: queryEntries
102
- }).then(extractItems).then(items => filterDrafts(items, includeDrafts, cdaClient)).then(items => filterArchived(items, includeArchived)).then(items => removeTags(items, stripTags)).then(items => {
115
+ const fetchFn = cdaClient ? query => cdaClient.withAllLocales.getEntries(query) : query => client.entry.getMany({
116
+ spaceId,
117
+ environmentId,
118
+ query
119
+ });
120
+ return pagedGet(fetchFn, queryEntries).then(extractItems).then(items => filterDrafts(items, includeDrafts, cdaClient)).then(items => filterArchived(items, includeArchived)).then(items => removeTags(items, stripTags)).then(items => {
103
121
  ctx.data.entries = items;
104
122
  });
105
123
  }),
@@ -107,13 +125,13 @@ function getFullSourceSpace({
107
125
  }, {
108
126
  title: 'Fetching assets data',
109
127
  task: (0, _contentfulBatchLibs.wrapTask)(ctx => {
110
- const source = (cdaClient === null || cdaClient === void 0 ? void 0 : cdaClient.withAllLocales) || ctx.environment;
111
128
  queryAssets = queryAssets || {};
112
- return pagedGet({
113
- source,
114
- method: 'getAssets',
115
- query: queryAssets
116
- }).then(extractItems).then(items => filterDrafts(items, includeDrafts, cdaClient)).then(items => filterArchived(items, includeArchived)).then(items => removeTags(items, stripTags)).then(items => {
129
+ const fetchFn = cdaClient ? query => cdaClient.withAllLocales.getAssets(query) : query => client.asset.getMany({
130
+ spaceId,
131
+ environmentId,
132
+ query
133
+ });
134
+ return pagedGet(fetchFn, queryAssets).then(extractItems).then(items => filterDrafts(items, includeDrafts, cdaClient)).then(items => filterArchived(items, includeArchived)).then(items => removeTags(items, stripTags)).then(items => {
117
135
  ctx.data.assets = items;
118
136
  });
119
137
  }),
@@ -121,33 +139,34 @@ function getFullSourceSpace({
121
139
  }, {
122
140
  title: 'Fetching locales data',
123
141
  task: (0, _contentfulBatchLibs.wrapTask)(ctx => {
124
- return pagedGet({
125
- source: ctx.environment,
126
- method: 'getLocales'
127
- }).then(extractItems).then(items => {
142
+ return pagedGet(query => client.locale.getMany({
143
+ spaceId,
144
+ environmentId,
145
+ query
146
+ })).then(extractItems).then(items => {
128
147
  ctx.data.locales = items;
129
148
  });
130
149
  }),
131
150
  skip: () => skipContentModel
132
151
  }, {
133
152
  title: 'Fetching webhooks data',
134
- task: (0, _contentfulBatchLibs.wrapTask)(ctx => {
135
- return pagedGet({
136
- source: ctx.space,
137
- method: 'getWebhooks'
138
- }).then(extractItems).then(items => {
139
- ctx.data.webhooks = items;
153
+ task: (0, _contentfulBatchLibs.wrapTask)(async ctx => {
154
+ const webhooksResponse = await client.webhook.getMany({
155
+ query: {
156
+ // webhooks are capped to 100 per space
157
+ limit: 100
158
+ },
159
+ spaceId
140
160
  });
161
+ ctx.data.webhooks = webhooksResponse.items;
141
162
  }),
142
163
  skip: () => skipWebhooks || environmentId !== 'master' && 'Webhooks can only be exported from master environment'
143
164
  }, {
144
165
  title: 'Fetching roles data',
145
- task: (0, _contentfulBatchLibs.wrapTask)(ctx => {
146
- return pagedGet({
147
- source: ctx.space,
148
- method: 'getRoles'
149
- }).then(extractItems).then(items => {
150
- ctx.data.roles = items;
166
+ task: (0, _contentfulBatchLibs.wrapTask)(async ctx => {
167
+ ctx.data.roles = await pagedRolesGet({
168
+ client,
169
+ spaceId
151
170
  });
152
171
  }),
153
172
  skip: () => skipRoles || environmentId !== 'master' && 'Roles can only be exported from master environment'
@@ -155,12 +174,11 @@ function getFullSourceSpace({
155
174
  title: 'Fetching Design Tokens data',
156
175
  task: (0, _contentfulBatchLibs.wrapTask)(async ctx => {
157
176
  try {
158
- ctx.data.designTokens = await cursorPagedGet({
159
- client: plainClient,
177
+ ctx.data.designTokens = await cursorPagedGet(query => client.designToken.getMany({
160
178
  spaceId,
161
179
  environmentId,
162
- method: 'designToken.getMany'
163
- });
180
+ query
181
+ }), 'designToken');
164
182
  } catch (err) {
165
183
  _contentfulBatchLibs.logEmitter.emit('warning', `Skipping Design Tokens export: ${err.message}`);
166
184
  ctx.data.designTokens = [];
@@ -171,12 +189,11 @@ function getFullSourceSpace({
171
189
  title: 'Fetching Components data',
172
190
  task: (0, _contentfulBatchLibs.wrapTask)(async ctx => {
173
191
  try {
174
- ctx.data.components = await cursorPagedGet({
175
- client: plainClient,
192
+ ctx.data.components = await cursorPagedGet(query => client.component.getMany({
176
193
  spaceId,
177
194
  environmentId,
178
- method: 'component.getMany'
179
- });
195
+ query
196
+ }), 'component');
180
197
  } catch (err) {
181
198
  _contentfulBatchLibs.logEmitter.emit('warning', `Skipping Components export: ${err.message}`);
182
199
  ctx.data.components = [];
@@ -187,12 +204,11 @@ function getFullSourceSpace({
187
204
  title: 'Fetching Experience Templates data',
188
205
  task: (0, _contentfulBatchLibs.wrapTask)(async ctx => {
189
206
  try {
190
- ctx.data.experienceTemplates = await cursorPagedGet({
191
- client: plainClient,
207
+ ctx.data.experienceTemplates = await cursorPagedGet(query => client.experienceTemplate.getMany({
192
208
  spaceId,
193
209
  environmentId,
194
- method: 'experienceTemplate.getMany'
195
- });
210
+ query
211
+ }), 'experienceTemplate');
196
212
  } catch (err) {
197
213
  _contentfulBatchLibs.logEmitter.emit('warning', `Skipping Experience Templates export: ${err.message}`);
198
214
  ctx.data.experienceTemplates = [];
@@ -203,12 +219,11 @@ function getFullSourceSpace({
203
219
  title: 'Fetching Data Assemblies data',
204
220
  task: (0, _contentfulBatchLibs.wrapTask)(async ctx => {
205
221
  try {
206
- ctx.data.dataAssemblies = await cursorPagedGet({
207
- client: plainClient,
222
+ ctx.data.dataAssemblies = await cursorPagedGet(query => client.dataAssembly.getMany({
208
223
  spaceId,
209
224
  environmentId,
210
- method: 'dataAssembly.getMany'
211
- });
225
+ query
226
+ }), 'dataAssembly');
212
227
  } catch (err) {
213
228
  _contentfulBatchLibs.logEmitter.emit('warning', `Skipping Data Assemblies export: ${err.message}`);
214
229
  ctx.data.dataAssemblies = [];
@@ -219,12 +234,11 @@ function getFullSourceSpace({
219
234
  title: 'Fetching Experience Fragments data',
220
235
  task: (0, _contentfulBatchLibs.wrapTask)(async ctx => {
221
236
  try {
222
- ctx.data.experienceFragments = await cursorPagedGet({
223
- client: plainClient,
237
+ ctx.data.experienceFragments = await cursorPagedGet(query => client.experienceFragment.getMany({
224
238
  spaceId,
225
239
  environmentId,
226
- method: 'experienceFragment.getMany'
227
- });
240
+ query
241
+ }), 'experienceFragment');
228
242
  } catch (err) {
229
243
  _contentfulBatchLibs.logEmitter.emit('warning', `Skipping Experience Fragments export: ${err.message}`);
230
244
  ctx.data.experienceFragments = [];
@@ -235,12 +249,11 @@ function getFullSourceSpace({
235
249
  title: 'Fetching Experiences data',
236
250
  task: (0, _contentfulBatchLibs.wrapTask)(async ctx => {
237
251
  try {
238
- ctx.data.experiences = await cursorPagedGet({
239
- client: plainClient,
252
+ ctx.data.experiences = await cursorPagedGet(query => client.experience.getMany({
240
253
  spaceId,
241
254
  environmentId,
242
- method: 'experience.getMany'
243
- });
255
+ query
256
+ }), 'experience');
244
257
  } catch (err) {
245
258
  _contentfulBatchLibs.logEmitter.emit('warning', `Skipping Experiences export: ${err.message}`);
246
259
  ctx.data.experiences = [];
@@ -249,15 +262,19 @@ function getFullSourceSpace({
249
262
  skip: () => !includeExperienceOrchestration
250
263
  }], listrOptions);
251
264
  }
252
- function getEditorInterfaces(contentTypes) {
265
+ function getEditorInterfaces(client, spaceId, environmentId, contentTypes) {
253
266
  return _bluebird.default.map(contentTypes, contentType => {
254
- return contentType.getEditorInterface().then(editorInterface => {
267
+ return client.editorInterface.get({
268
+ spaceId,
269
+ environmentId,
270
+ contentTypeId: contentType.sys.id
271
+ }).then(editorInterface => {
255
272
  _contentfulBatchLibs.logEmitter.emit('info', `Fetched editor interface for ${contentType.name}`);
256
273
  return editorInterface;
257
274
  }).catch(() => {
258
275
  // old contentTypes may not have an editor interface but we'll handle in a later stage
259
276
  // but it should not stop getting the data process
260
- _contentfulBatchLibs.logEmitter.emit('warning', `No editor interface found for ${contentType}`);
277
+ _contentfulBatchLibs.logEmitter.emit('warning', `No editor interface found for ${contentType.name || contentType.sys.id}`);
261
278
  return _bluebird.default.resolve(null);
262
279
  });
263
280
  }, {
@@ -269,55 +286,101 @@ function getEditorInterfaces(contentTypes) {
269
286
  * Gets all ExO entities using cursor-based pagination (pageNext/pagePrev tokens).
270
287
  * ExO list endpoints do not support skip-based pagination or the order param.
271
288
  */
272
- async function cursorPagedGet({
273
- client,
274
- spaceId,
275
- environmentId,
276
- method
277
- }) {
278
- const [entity, operation] = method.split('.');
289
+ async function cursorPagedGet(fetchFn, entityLabel) {
279
290
  const allItems = [];
280
291
  let pageNext = null;
281
292
  do {
282
293
  var _response$pages$next, _response$pages;
283
294
  const query = {
284
- spaceId,
285
- environmentId,
286
295
  limit: pageLimit
287
296
  };
288
297
  if (pageNext) {
289
298
  query.pageNext = pageNext;
290
299
  }
291
- const response = await client[entity][operation](query);
300
+ const response = await fetchFn(query);
292
301
  allItems.push(...response.items);
293
- _contentfulBatchLibs.logEmitter.emit('info', `Fetched ${allItems.length} ${entity} items`);
302
+ _contentfulBatchLibs.logEmitter.emit('info', `Fetched ${allItems.length} ${entityLabel} items`);
294
303
  pageNext = (_response$pages$next = (_response$pages = response.pages) === null || _response$pages === void 0 ? void 0 : _response$pages.next) !== null && _response$pages$next !== void 0 ? _response$pages$next : null;
295
304
  } while (pageNext);
296
305
  return allItems;
297
306
  }
298
307
 
299
308
  /**
309
+ * Gets all roles. Roles are scheduled to switch from skip/limit to
310
+ * cursor-based (pages.next/pagePrev) pagination on Feb 15, 2027:
311
+ * https://www.contentful.com/developers/api-changes/space-roles-collection-endpoints-update/
312
+ * Handles both shapes so this keeps working across that migration.
313
+ */
314
+ async function pagedRolesGet({
315
+ client,
316
+ spaceId
317
+ }) {
318
+ const allItems = [];
319
+ const order = 'sys.createdAt,sys.id';
320
+ let response = await client.role.getMany({
321
+ spaceId,
322
+ query: {
323
+ limit: pageLimit,
324
+ order
325
+ }
326
+ });
327
+ allItems.push(...response.items);
328
+ _contentfulBatchLibs.logEmitter.emit('info', `Fetched ${allItems.length} roles`);
329
+ if (response.pages !== undefined) {
330
+ var _response$pages$next2, _response$pages2;
331
+ let pageNext = (_response$pages$next2 = (_response$pages2 = response.pages) === null || _response$pages2 === void 0 ? void 0 : _response$pages2.next) !== null && _response$pages$next2 !== void 0 ? _response$pages$next2 : null;
332
+ while (pageNext) {
333
+ var _response$pages$next3, _response$pages3;
334
+ response = await client.role.getMany({
335
+ spaceId,
336
+ query: {
337
+ limit: pageLimit,
338
+ order,
339
+ pageNext
340
+ }
341
+ });
342
+ allItems.push(...response.items);
343
+ _contentfulBatchLibs.logEmitter.emit('info', `Fetched ${allItems.length} roles`);
344
+ pageNext = (_response$pages$next3 = (_response$pages3 = response.pages) === null || _response$pages3 === void 0 ? void 0 : _response$pages3.next) !== null && _response$pages$next3 !== void 0 ? _response$pages$next3 : null;
345
+ }
346
+ } else {
347
+ let skip = allItems.length;
348
+ while (allItems.length < response.total && response.items.length > 0) {
349
+ response = await client.role.getMany({
350
+ spaceId,
351
+ query: {
352
+ limit: pageLimit,
353
+ order,
354
+ skip
355
+ }
356
+ });
357
+ allItems.push(...response.items);
358
+ _contentfulBatchLibs.logEmitter.emit('info', `Fetched ${allItems.length} roles`);
359
+ skip += response.items.length;
360
+ }
361
+ }
362
+ return allItems;
363
+ }
364
+
365
+ /**
366
+ * Recursively fetches all pages of a offset-based paginated CMA collection and returns
367
+ * them merged into a single response shaped like `{ items: [...], total, ... }`
368
+ *
300
369
  * Gets all the existing entities based on pagination parameters.
301
370
  * The first call will have no aggregated response. Subsequent calls will
302
371
  * concatenate the new responses to the original one.
303
372
  */
304
- function pagedGet({
305
- source,
306
- method,
307
- skip = 0,
308
- aggregatedResponse = null,
309
- query = null
310
- }) {
311
- const userQueryLimit = query && query.limit;
373
+ function pagedGet(fetchFn, userQuery = null, skip = 0, aggregatedResponse = null) {
374
+ const userQueryLimit = userQuery && userQuery.limit;
312
375
  const fetchedTotal = aggregatedResponse && aggregatedResponse.items.length;
313
376
  const limit = userQueryLimit ? Math.min(pageLimit, userQueryLimit - fetchedTotal) : pageLimit;
314
- const requestQuery = Object.assign({}, {
377
+ const query = _objectSpread(_objectSpread({
315
378
  skip,
316
379
  order: 'sys.createdAt,sys.id'
317
- }, query, {
380
+ }, userQuery), {}, {
318
381
  limit
319
382
  });
320
- return source[method](requestQuery).then(response => {
383
+ return fetchFn(query).then(response => {
321
384
  if (!aggregatedResponse) {
322
385
  aggregatedResponse = response;
323
386
  } else {
@@ -325,20 +388,14 @@ function pagedGet({
325
388
  }
326
389
  const totalItemsLength = aggregatedResponse.items.length;
327
390
  const total = response.total;
328
- logPagingStatus(response, requestQuery, userQueryLimit);
391
+ logPagingStatus(response, query, userQueryLimit);
329
392
  const gotAllQueryLimitedItems = userQueryLimit && totalItemsLength >= userQueryLimit;
330
393
  const gotAllItems = totalItemsLength >= total;
331
394
  const gotNoItems = totalItemsLength <= 0;
332
395
  if (gotAllQueryLimitedItems || gotAllItems || gotNoItems) {
333
396
  return aggregatedResponse;
334
397
  }
335
- return pagedGet({
336
- source,
337
- method,
338
- skip: skip + response.items.length,
339
- aggregatedResponse,
340
- query
341
- });
398
+ return pagedGet(fetchFn, userQuery, skip + response.items.length, aggregatedResponse);
342
399
  });
343
400
  }
344
401
  function logPagingStatus(response, requestQuery, userLimit) {
@@ -4,7 +4,6 @@ Object.defineProperty(exports, "__esModule", {
4
4
  value: true
5
5
  });
6
6
  exports.default = initClient;
7
- exports.initPlainClient = initPlainClient;
8
7
  var _contentful = require("contentful");
9
8
  var _contentfulBatchLibs = require("contentful-batch-libs");
10
9
  var _contentfulManagement = require("contentful-management");
@@ -32,13 +31,10 @@ function initClient(opts, useCda = false) {
32
31
  return (0, _contentful.createClient)(cdaConfig).withoutLinkResolution;
33
32
  }
34
33
  return (0, _contentfulManagement.createClient)(config, {
35
- type: 'legacy'
34
+ defaults: {
35
+ spaceId: config.spaceId,
36
+ environmentId: config.environmentId
37
+ }
36
38
  });
37
39
  }
38
- function initPlainClient(opts) {
39
- return (0, _contentfulManagement.createClient)({
40
- accessToken: opts.managementToken,
41
- host: opts.host,
42
- logHandler
43
- });
44
- }
40
+ module.exports = exports.default;
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "contentful-export",
3
- "version": "8.2.1",
3
+ "version": "8.3.0",
4
4
  "description": "this tool allows you to export a space to a JSON dump",
5
5
  "main": "dist/index.js",
6
6
  "types": "types.d.ts",
@@ -46,6 +46,7 @@
46
46
  },
47
47
  "dependencies": {
48
48
  "axios": "^1.13.5",
49
+ "axios-retry": "^4.5.0",
49
50
  "bfj": "^9.1.3",
50
51
  "bluebird": "^3.3.3",
51
52
  "cli-table3": "^0.6.0",