contentful-export 7.19.158 → 7.19.159

package/dist/tasks/get-space-data.js

@@ -178,41 +178,54 @@ function pagedGet({
   aggregatedResponse = null,
   query = null
 }) {
-  let queryLimit = query && query.limit;
-  if (queryLimit && aggregatedResponse) {
-    if (skip + pageLimit > queryLimit) {
-      // this means the total we are about to fetch is > our limit
-      queryLimit = queryLimit % pageLimit;
-    }
-  }
-  const fullQuery = Object.assign({}, {
+  const userQueryLimit = query && query.limit;
+  const fetchedTotal = aggregatedResponse && aggregatedResponse.items.length;
+  const limit = userQueryLimit ? Math.min(pageLimit, userQueryLimit - fetchedTotal) : pageLimit;
+  const requestQuery = Object.assign({}, {
     skip,
     order: 'sys.createdAt,sys.id'
   }, query, {
-    limit: queryLimit && queryLimit < pageLimit ? queryLimit : pageLimit
+    limit
   });
-  return source[method](fullQuery).then(response => {
+  return source[method](requestQuery).then(response => {
     if (!aggregatedResponse) {
       aggregatedResponse = response;
     } else {
       aggregatedResponse.items = aggregatedResponse.items.concat(response.items);
     }
-    const total = queryLimit && queryLimit < response.total ? queryLimit : response.total;
-    const page = Math.ceil(skip / pageLimit) + 1;
-    const pages = Math.ceil(total / pageLimit);
-    _contentfulBatchLibs.logEmitter.emit('info', `Fetched ${aggregatedResponse.items.length} of ${total} items (Page ${page}/${pages})`);
-    if (skip + pageLimit < total) {
-      return pagedGet({
-        source,
-        method,
-        skip: skip + pageLimit,
-        aggregatedResponse,
-        query
-      });
+    const totalItemsLength = aggregatedResponse.items.length;
+    const total = response.total;
+    logPagingStatus(response, requestQuery, userQueryLimit);
+    const gotAllQueryLimitedItems = userQueryLimit && totalItemsLength >= userQueryLimit;
+    const gotAllItems = totalItemsLength >= total;
+    const gotNoItems = totalItemsLength <= 0;
+    if (gotAllQueryLimitedItems || gotAllItems || gotNoItems) {
+      return aggregatedResponse;
     }
-    return aggregatedResponse;
+    return pagedGet({
+      source,
+      method,
+      skip: skip + response.items.length,
+      aggregatedResponse,
+      query
+    });
   });
 }
+function logPagingStatus(response, requestQuery, userLimit) {
+  const {
+    total,
+    limit,
+    items
+  } = response;
+  const pagedItemsLength = items.length;
+
+  // sometimes our pageLimit or queryLimit of 1000 is overridden by the API (like in locales)
+  const imposedLimit = limit || requestQuery.limit;
+  const limitedTotal = userLimit ? Math.min(userLimit, total) : total;
+  const page = Math.ceil(requestQuery.skip / imposedLimit) + 1;
+  const pages = Math.ceil(limitedTotal / imposedLimit);
+  _contentfulBatchLibs.logEmitter.emit('info', `Fetched ${pagedItemsLength} of ${total} items (Page ${page}/${pages})`);
+}
 function extractItems(response) {
   return response.items;
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "contentful-export",
-  "version": "7.19.158",
+  "version": "7.19.159",
   "description": "this tool allows you to export a space to a JSON dump",
   "main": "dist/index.js",
   "types": "types.d.ts",
@@ -46,9 +46,9 @@
     "bfj": "^8.0.0",
     "bluebird": "^3.3.3",
     "cli-table3": "^0.6.0",
-    "contentful": "^11.0.3",
+    "contentful": "^11.1.4",
     "contentful-batch-libs": "^9.4.1",
-    "contentful-management": "^11.0.1",
+    "contentful-management": "^11.36.0",
     "date-fns": "^4.1.0",
     "figures": "^3.2.0",
     "jsonwebtoken": "^9.0.0",