contentful-import 8.5.21 → 8.5.23

package/dist/index.js

@@ -21,6 +21,11 @@ var _transformSpace = _interopRequireDefault(require("./transform/transform-spac
 var _validations = require("./utils/validations");
 var _parseOptions = _interopRequireDefault(require("./parseOptions"));
 function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; }
+function ownKeys(object, enumerableOnly) { var keys = Object.keys(object); if (Object.getOwnPropertySymbols) { var symbols = Object.getOwnPropertySymbols(object); enumerableOnly && (symbols = symbols.filter(function (sym) { return Object.getOwnPropertyDescriptor(object, sym).enumerable; })), keys.push.apply(keys, symbols); } return keys; }
+function _objectSpread(target) { for (var i = 1; i < arguments.length; i++) { var source = null != arguments[i] ? arguments[i] : {}; i % 2 ? ownKeys(Object(source), !0).forEach(function (key) { _defineProperty(target, key, source[key]); }) : Object.getOwnPropertyDescriptors ? Object.defineProperties(target, Object.getOwnPropertyDescriptors(source)) : ownKeys(Object(source)).forEach(function (key) { Object.defineProperty(target, key, Object.getOwnPropertyDescriptor(source, key)); }); } return target; }
+function _defineProperty(obj, key, value) { key = _toPropertyKey(key); if (key in obj) { Object.defineProperty(obj, key, { value: value, enumerable: true, configurable: true, writable: true }); } else { obj[key] = value; } return obj; }
+function _toPropertyKey(arg) { var key = _toPrimitive(arg, "string"); return typeof key === "symbol" ? key : String(key); }
+function _toPrimitive(input, hint) { if (typeof input !== "object" || input === null) return input; var prim = input[Symbol.toPrimitive]; if (prim !== undefined) { var res = prim.call(input, hint || "default"); if (typeof res !== "object") return res; throw new TypeError("@@toPrimitive must return a primitive value."); } return (hint === "string" ? String : Number)(input); }
 const ONE_SECOND = 1000;
 function createListrOptions(options) {
   if (options.useVerboseRenderer) {
@@ -33,9 +38,9 @@ function createListrOptions(options) {
     collapse: false
   };
 }
-function runContentfulImport(params) {
+async function runContentfulImport(params) {
   const log = [];
-  const options = (0, _parseOptions.default)(params);
+  const options = await (0, _parseOptions.default)(params);
   const listrOptions = createListrOptions(options);
   const requestQueue = new _pQueue.default({
     interval: ONE_SECOND,
@@ -71,7 +76,9 @@ function runContentfulImport(params) {
   }, {
     title: 'Initialize client',
     task: (0, _listr2.wrapTask)(async ctx => {
-      ctx.client = (0, _initClient.default)(options);
+      ctx.client = (0, _initClient.default)(_objectSpread(_objectSpread({}, options), {}, {
+        content: undefined
+      }));
     })
   }, {
     title: 'Checking if destination space already has any content and retrieving it',

package/dist/parseOptions.js

@@ -11,6 +11,7 @@ var _package = require("../package");
 var _headers = require("./utils/headers");
 var _proxy = require("contentful-batch-libs/dist/proxy");
 var _addSequenceHeader = _interopRequireDefault(require("contentful-batch-libs/dist/add-sequence-header"));
+var _jsonExt = require("@discoveryjs/json-ext");
 function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; }
 function ownKeys(object, enumerableOnly) { var keys = Object.keys(object); if (Object.getOwnPropertySymbols) { var symbols = Object.getOwnPropertySymbols(object); enumerableOnly && (symbols = symbols.filter(function (sym) { return Object.getOwnPropertyDescriptor(object, sym).enumerable; })), keys.push.apply(keys, symbols); } return keys; }
 function _objectSpread(target) { for (var i = 1; i < arguments.length; i++) { var source = null != arguments[i] ? arguments[i] : {}; i % 2 ? ownKeys(Object(source), !0).forEach(function (key) { _defineProperty(target, key, source[key]); }) : Object.getOwnPropertyDescriptors ? Object.defineProperties(target, Object.getOwnPropertyDescriptors(source)) : ownKeys(Object(source)).forEach(function (key) { Object.defineProperty(target, key, Object.getOwnPropertyDescriptor(source, key)); }); } return target; }
@@ -18,7 +19,7 @@ function _defineProperty(obj, key, value) { key = _toPropertyKey(key); if (key i
 function _toPropertyKey(arg) { var key = _toPrimitive(arg, "string"); return typeof key === "symbol" ? key : String(key); }
 function _toPrimitive(input, hint) { if (typeof input !== "object" || input === null) return input; var prim = input[Symbol.toPrimitive]; if (prim !== undefined) { var res = prim.call(input, hint || "default"); if (typeof res !== "object") return res; throw new TypeError("@@toPrimitive must return a primitive value."); } return (hint === "string" ? String : Number)(input); }
 const SUPPORTED_ENTITY_TYPES = ['contentTypes', 'tags', 'entries', 'assets', 'locales', 'webhooks', 'editorInterfaces'];
-function parseOptions(params) {
+async function parseOptions(params) {
   const defaultOptions = {
     skipContentModel: false,
     skipLocales: false,
@@ -61,10 +62,14 @@ function parseOptions(params) {
   } else {
     options.errorLogFile = (0, _path.resolve)(process.cwd(), options.errorLogFile);
   }
-
-  // Further processing
   options.accessToken = options.managementToken;
-  options.content = options.content || JSON.parse(_fs.default.readFileSync(options.contentFile));
+  if (!options.content) {
+    // using a stream parser allows input files > 512 MB
+    const fileStream = _fs.default.createReadStream(options.contentFile, {
+      encoding: 'utf8'
+    });
+    options.content = await (0, _jsonExt.parseChunked)(fileStream);
+  }
 
   // Clean up content to only include supported entity types
   Object.keys(options.content).forEach(type => {

package/dist/tasks/get-destination-data.js

@@ -43,6 +43,7 @@ async function batchedIdQuery({
   const batches = getIdBatches(ids);
   let totalFetched = 0;
   const allPendingResponses = batches.map(idBatch => {
+    // TODO: add batch count to indicate that it's running
     return requestQueue.add(async () => {
       const response = await environment[method]({
         'sys.id[in]': idBatch,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "contentful-import",
-  "version": "8.5.21",
+  "version": "8.5.23",
   "description": "this tool allows you to import JSON dump exported by contentful-export",
   "main": "dist/index.js",
   "engines": {
@@ -50,10 +50,11 @@
     "url": "https://github.com/contentful/contentful-import/issues"
   },
   "dependencies": {
+    "@discoveryjs/json-ext": "^0.5.7",
     "bluebird": "^3.5.1",
     "cli-table3": "^0.6.0",
     "contentful-batch-libs": "^9.4.2",
-    "contentful-management": "^10.26.0",
+    "contentful-management": "^10.27.0",
     "date-fns": "^2.29.3",
     "joi": "^17.7.0",
     "listr": "^0.14.1",
@@ -65,7 +66,7 @@
   },
   "devDependencies": {
     "@babel/cli": "^7.20.7",
-    "@babel/core": "^7.20.7",
+    "@babel/core": "^7.20.12",
     "@babel/plugin-proposal-object-rest-spread": "^7.20.7",
     "@babel/preset-env": "^7.20.2",
     "babel-eslint": "^10.1.0",
@@ -75,14 +76,14 @@
     "cz-conventional-changelog": "^3.1.0",
     "eslint": "^7.2.0",
     "eslint-config-standard": "^16.0.0",
-    "eslint-plugin-import": "^2.26.0",
-    "eslint-plugin-jest": "^27.2.0",
+    "eslint-plugin-import": "^2.27.5",
+    "eslint-plugin-jest": "^27.2.1",
     "eslint-plugin-node": "^11.1.0",
     "eslint-plugin-promise": "^6.1.1",
     "eslint-plugin-standard": "^5.0.0",
     "husky": "^8.0.3",
     "jest": "^29.3.1",
-    "rimraf": "^3.0.2",
+    "rimraf": "^4.1.1",
     "semantic-release": "^19.0.5"
   },
   "files": [