contentful-import 8.5.1 → 8.5.3

package/dist/index.js

@@ -4,56 +4,35 @@ Object.defineProperty(exports, "__esModule", {
   value: true
 });
 exports.default = runContentfulImport;
-
 var _cliTable = _interopRequireDefault(require("cli-table3"));
-
 var _differenceInSeconds = _interopRequireDefault(require("date-fns/differenceInSeconds"));
-
 var _formatDistance = _interopRequireDefault(require("date-fns/formatDistance"));
-
 var _listr = _interopRequireDefault(require("listr"));
-
 var _listrUpdateRenderer = _interopRequireDefault(require("listr-update-renderer"));
-
 var _listrVerboseRenderer = _interopRequireDefault(require("listr-verbose-renderer"));
-
 var _lodash = require("lodash");
-
 var _pQueue = _interopRequireDefault(require("p-queue"));
-
 var _logging = require("contentful-batch-libs/dist/logging");
-
 var _listr2 = require("contentful-batch-libs/dist/listr");
-
 var _initClient = _interopRequireDefault(require("./tasks/init-client"));
-
 var _getDestinationData = _interopRequireDefault(require("./tasks/get-destination-data"));
-
 var _pushToSpace = _interopRequireDefault(require("./tasks/push-to-space/push-to-space"));
-
 var _transformSpace = _interopRequireDefault(require("./transform/transform-space"));
-
 var _validations = require("./utils/validations");
-
 var _parseOptions = _interopRequireDefault(require("./parseOptions"));
-
 function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; }
-
 const ONE_SECOND = 1000;
-
 function createListrOptions(options) {
   if (options.useVerboseRenderer) {
     return {
       renderer: _listrVerboseRenderer.default
     };
   }
-
   return {
     renderer: _listrUpdateRenderer.default,
     collapse: false
   };
 }
-
 function runContentfulImport(params) {
   const log = [];
   const options = (0, _parseOptions.default)(params);
@@ -62,8 +41,9 @@ function runContentfulImport(params) {
     interval: ONE_SECOND,
     intervalCap: options.rateLimit,
     carryoverConcurrencyCount: true
-  }); // Setup custom log listener to store log messages for later
+  });
 
+  // Setup custom log listener to store log messages for later
   (0, _logging.setupLogging)(log);
   const infoTable = new _cliTable.default();
   infoTable.push([{
@@ -74,15 +54,12 @@ function runContentfulImport(params) {
     if (options.skipLocales && type === 'locales') {
       return;
     }
-
     if (options.skipContentModel && ['contentTypes', 'editorInterfaces'].indexOf(type) >= 0) {
       return;
     }
-
     if (options.contentModelOnly && !(['contentTypes', 'editorInterfaces', 'locales'].indexOf(type) >= 0)) {
       return;
     }
-
     infoTable.push([(0, _lodash.startCase)(type), options.content[type].length]);
   });
   console.log(infoTable.toString());
@@ -145,7 +122,6 @@ function runContentfulImport(params) {
   }).then(ctx => {
     console.log('Finished importing all data');
     const resultTypes = Object.keys(ctx.data);
-
     if (resultTypes.length) {
       const resultTable = new _cliTable.default();
       resultTable.push([{
@@ -159,7 +135,6 @@ function runContentfulImport(params) {
     } else {
       console.log('No data was imported');
     }
-
     const endTime = new Date();
     const durationHuman = (0, _formatDistance.default)(endTime, options.startTime);
     const durationSeconds = (0, _differenceInSeconds.default)(endTime, options.startTime);
@@ -175,7 +150,6 @@ function runContentfulImport(params) {
     const errorLog = log.filter(logMessage => logMessage.level !== 'info' && logMessage.level !== 'warning');
     const displayLog = log.filter(logMessage => logMessage.level !== 'info');
     (0, _logging.displayErrorLog)(displayLog);
-
     if (errorLog.length) {
       return (0, _logging.writeErrorLogFile)(options.errorLogFile, errorLog).then(() => {
         const multiError = new Error('Errors occurred');
@@ -184,10 +158,8 @@ function runContentfulImport(params) {
         throw multiError;
       });
     }
-
     console.log('The import was successful.');
     return data;
   });
 }
-
 module.exports = exports.default;

package/dist/parseOptions.js

@@ -4,31 +4,18 @@ Object.defineProperty(exports, "__esModule", {
   value: true
 });
 exports.default = parseOptions;
-
 var _fs = _interopRequireDefault(require("fs"));
-
 var _path = require("path");
-
 var _format = _interopRequireDefault(require("date-fns/format"));
-
 var _package = require("../package");
-
 var _headers = require("./utils/headers");
-
 var _proxy = require("contentful-batch-libs/dist/proxy");
-
 var _addSequenceHeader = _interopRequireDefault(require("contentful-batch-libs/dist/add-sequence-header"));
-
 function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; }
-
 function ownKeys(object, enumerableOnly) { var keys = Object.keys(object); if (Object.getOwnPropertySymbols) { var symbols = Object.getOwnPropertySymbols(object); enumerableOnly && (symbols = symbols.filter(function (sym) { return Object.getOwnPropertyDescriptor(object, sym).enumerable; })), keys.push.apply(keys, symbols); } return keys; }
-
 function _objectSpread(target) { for (var i = 1; i < arguments.length; i++) { var source = null != arguments[i] ? arguments[i] : {}; i % 2 ? ownKeys(Object(source), !0).forEach(function (key) { _defineProperty(target, key, source[key]); }) : Object.getOwnPropertyDescriptors ? Object.defineProperties(target, Object.getOwnPropertyDescriptors(source)) : ownKeys(Object(source)).forEach(function (key) { Object.defineProperty(target, key, Object.getOwnPropertyDescriptor(source, key)); }); } return target; }
-
 function _defineProperty(obj, key, value) { if (key in obj) { Object.defineProperty(obj, key, { value: value, enumerable: true, configurable: true, writable: true }); } else { obj[key] = value; } return obj; }
-
 const SUPPORTED_ENTITY_TYPES = ['contentTypes', 'tags', 'entries', 'assets', 'locales', 'webhooks', 'editorInterfaces'];
-
 function parseOptions(params) {
   const defaultOptions = {
     skipContentModel: false,
@@ -41,51 +28,43 @@ function parseOptions(params) {
     rateLimit: 7
   };
   const configFile = params.config ? require((0, _path.resolve)(process.cwd(), params.config)) : {};
-
   const options = _objectSpread(_objectSpread(_objectSpread(_objectSpread({}, defaultOptions), configFile), params), {}, {
     headers: (0, _addSequenceHeader.default)(params.headers || (0, _headers.getHeadersConfig)(params.header))
-  }); // Validation
-
+  });
 
+  // Validation
   if (!options.spaceId) {
     throw new Error('The `spaceId` option is required.');
   }
-
   if (!options.managementToken) {
     throw new Error('The `managementToken` option is required.');
   }
-
   if (!options.contentFile && !options.content) {
     throw new Error('Either the `contentFile` or `content` option are required.');
   }
-
   if (options.contentModelOnly && options.skipContentModel) {
     throw new Error('`contentModelOnly` and `skipContentModel` cannot be used together');
   }
-
   if (options.skipLocales && !options.contentModelOnly) {
     throw new Error('`skipLocales` can only be used together with `contentModelOnly`');
   }
-
   const proxySimpleExp = /.+:\d+/;
   const proxyAuthExp = /.+:.+@.+:\d+/;
-
   if (typeof options.proxy === 'string' && options.proxy && !(proxySimpleExp.test(options.proxy) || proxyAuthExp.test(options.proxy))) {
     throw new Error('Please provide the proxy config in the following format:\nhost:port or user:password@host:port');
   }
-
   options.startTime = new Date();
-
   if (!options.errorLogFile) {
     options.errorLogFile = (0, _path.resolve)(process.cwd(), `contentful-import-error-log-${options.spaceId}-${(0, _format.default)(options.startTime, "yyyy-MM-dd'T'HH-mm-ss")}.json`);
   } else {
     options.errorLogFile = (0, _path.resolve)(process.cwd(), options.errorLogFile);
-  } // Further processing
-
+  }
 
+  // Further processing
   options.accessToken = options.managementToken;
-  options.content = options.content || JSON.parse(_fs.default.readFileSync(options.contentFile)); // Clean up content to only include supported entity types
+  options.content = options.content || JSON.parse(_fs.default.readFileSync(options.contentFile));
 
+  // Clean up content to only include supported entity types
   Object.keys(options.content).forEach(type => {
     if (SUPPORTED_ENTITY_TYPES.indexOf(type) === -1) {
       delete options.content[type];
@@ -94,19 +73,15 @@ function parseOptions(params) {
   SUPPORTED_ENTITY_TYPES.forEach(type => {
     options.content[type] = options.content[type] || [];
   });
-
   if (typeof options.proxy === 'string') {
     options.proxy = (0, _proxy.proxyStringToObject)(options.proxy);
   }
-
   if (!options.rawProxy && options.proxy) {
     options.httpsAgent = (0, _proxy.agentFromProxy)(options.proxy);
     delete options.proxy;
   }
-
   options.application = options.managementApplication || `contentful.import/${_package.version}`;
   options.feature = options.managementFeature || 'library-import';
   return options;
 }
-
 module.exports = exports.default;

package/dist/tasks/get-destination-data.js

@@ -4,19 +4,12 @@ Object.defineProperty(exports, "__esModule", {
   value: true
 });
 exports.default = getDestinationData;
-
 var _bluebird = _interopRequireDefault(require("bluebird"));
-
 var _logging = require("contentful-batch-libs/dist/logging");
-
 function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; }
-
 function ownKeys(object, enumerableOnly) { var keys = Object.keys(object); if (Object.getOwnPropertySymbols) { var symbols = Object.getOwnPropertySymbols(object); enumerableOnly && (symbols = symbols.filter(function (sym) { return Object.getOwnPropertyDescriptor(object, sym).enumerable; })), keys.push.apply(keys, symbols); } return keys; }
-
 function _objectSpread(target) { for (var i = 1; i < arguments.length; i++) { var source = null != arguments[i] ? arguments[i] : {}; i % 2 ? ownKeys(Object(source), !0).forEach(function (key) { _defineProperty(target, key, source[key]); }) : Object.getOwnPropertyDescriptors ? Object.defineProperties(target, Object.getOwnPropertyDescriptors(source)) : ownKeys(Object(source)).forEach(function (key) { Object.defineProperty(target, key, Object.getOwnPropertyDescriptor(source, key)); }); } return target; }
-
 function _defineProperty(obj, key, value) { if (key in obj) { Object.defineProperty(obj, key, { value: value, enumerable: true, configurable: true, writable: true }); } else { obj[key] = value; } return obj; }
-
 const BATCH_CHAR_LIMIT = 1990;
 const BATCH_SIZE_LIMIT = 100;
 const METHODS = {
@@ -37,7 +30,6 @@ const METHODS = {
     method: 'getAssets'
   }
 };
-
 async function batchedIdQuery({
   environment,
   type,
@@ -55,26 +47,21 @@ async function batchedIdQuery({
         limit: idBatch.split(',').length
       });
       totalFetched = totalFetched + response.items.length;
-
       _logging.logEmitter.emit('info', `Fetched ${totalFetched} of ${response.total} ${entityTypeName}`);
-
       return response.items;
     });
   });
   const responses = await _bluebird.default.all(allPendingResponses);
   return responses.flat();
 }
-
 function getIdBatches(ids) {
   const batches = [];
   let currentBatch = '';
   let currentSize = 0;
-
   while (ids.length > 0) {
     const id = ids.splice(0, 1);
     currentBatch += id;
     currentSize = currentSize + 1;
-
     if (currentSize === BATCH_SIZE_LIMIT || currentBatch.length > BATCH_CHAR_LIMIT || ids.length === 0) {
       batches.push(currentBatch);
       currentBatch = '';
@@ -83,9 +70,9 @@ function getIdBatches(ids) {
       currentBatch += ',';
     }
   }
-
   return batches;
 }
+
 /**
  * Gets content from a space which will have content copied to it, based on a
  * collection of existing content.
@@ -95,7 +82,6 @@ function getIdBatches(ids) {
  *
  */
 
-
 async function getDestinationData({
   client,
   spaceId,
@@ -114,10 +100,10 @@ async function getDestinationData({
     locales: [],
     entries: [],
     assets: []
-  }; // Make sure all required properties are available and at least an empty array
+  };
 
+  // Make sure all required properties are available and at least an empty array
   sourceData = _objectSpread(_objectSpread({}, result), sourceData);
-
   if (!skipContentModel) {
     const contentTypeIds = sourceData.contentTypes.map(e => e.sys.id);
     result.contentTypes = batchedIdQuery({
@@ -126,7 +112,6 @@ async function getDestinationData({
       ids: contentTypeIds,
       requestQueue
     });
-
     if (!skipLocales) {
       const localeIds = sourceData.locales.map(e => e.sys.id);
       result.locales = batchedIdQuery({
@@ -136,19 +121,17 @@ async function getDestinationData({
         requestQueue
       });
     }
-  } // include tags even if contentModelOnly = true
-
+  }
 
+  // include tags even if contentModelOnly = true
   result.tags = environment.getTags().then(response => response.items).catch(e => {
     // users without access to Tags will get 404
     // if they dont have access, remove tags array so they're not handled in future steps
     delete result.tags;
   });
-
   if (contentModelOnly) {
     return _bluebird.default.props(result);
   }
-
   const entryIds = sourceData.entries.map(e => e.sys.id);
   const assetIds = sourceData.assets.map(e => e.sys.id);
   result.entries = batchedIdQuery({
@@ -166,5 +149,4 @@ async function getDestinationData({
   result.webhooks = [];
   return _bluebird.default.props(result);
 }
-
 module.exports = exports.default;

package/dist/tasks/init-client.js

@@ -4,30 +4,20 @@ Object.defineProperty(exports, "__esModule", {
   value: true
 });
 exports.default = initClient;
-
 var _contentfulManagement = require("contentful-management");
-
 var _logging = require("contentful-batch-libs/dist/logging");
-
 function ownKeys(object, enumerableOnly) { var keys = Object.keys(object); if (Object.getOwnPropertySymbols) { var symbols = Object.getOwnPropertySymbols(object); enumerableOnly && (symbols = symbols.filter(function (sym) { return Object.getOwnPropertyDescriptor(object, sym).enumerable; })), keys.push.apply(keys, symbols); } return keys; }
-
 function _objectSpread(target) { for (var i = 1; i < arguments.length; i++) { var source = null != arguments[i] ? arguments[i] : {}; i % 2 ? ownKeys(Object(source), !0).forEach(function (key) { _defineProperty(target, key, source[key]); }) : Object.getOwnPropertyDescriptors ? Object.defineProperties(target, Object.getOwnPropertyDescriptors(source)) : ownKeys(Object(source)).forEach(function (key) { Object.defineProperty(target, key, Object.getOwnPropertyDescriptor(source, key)); }); } return target; }
-
 function _defineProperty(obj, key, value) { if (key in obj) { Object.defineProperty(obj, key, { value: value, enumerable: true, configurable: true, writable: true }); } else { obj[key] = value; } return obj; }
-
 function logHandler(level, data) {
   _logging.logEmitter.emit(level, data);
 }
-
 function initClient(opts) {
   const defaultOpts = {
     timeout: 30000,
     logHandler
   };
-
   const config = _objectSpread(_objectSpread({}, defaultOpts), opts);
-
   return (0, _contentfulManagement.createClient)(config);
 }
-
 module.exports = exports.default;

package/dist/tasks/push-to-space/assets.js

@@ -5,25 +5,16 @@ Object.defineProperty(exports, "__esModule", {
 });
 exports.getAssetStreamForURL = getAssetStreamForURL;
 exports.processAssets = processAssets;
-
 var _fs = _interopRequireDefault(require("fs"));
-
 var _path = require("path");
-
 var _util = require("util");
-
 var _getEntityName = _interopRequireDefault(require("contentful-batch-libs/dist/get-entity-name"));
-
 var _logging = require("contentful-batch-libs/dist/logging");
-
 function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; }
-
 const stat = (0, _util.promisify)(_fs.default.stat);
-
 async function getAssetStreamForURL(url, assetsDirectory) {
   const [, assetPath] = url.split('//');
   const filePath = (0, _path.join)(assetsDirectory, assetPath);
-
   try {
     await stat(filePath);
     return _fs.default.createReadStream(filePath);
@@ -33,7 +24,6 @@ async function getAssetStreamForURL(url, assetsDirectory) {
     throw error;
   }
 }
-
 async function processAssets({
   assets,
   timeout,
@@ -43,21 +33,17 @@ async function processAssets({
   const pendingProcessingAssets = assets.map(asset => {
     return requestQueue.add(async () => {
       _logging.logEmitter.emit('info', `Processing Asset ${(0, _getEntityName.default)(asset)}`);
-
       const processingOptions = Object.assign({}, timeout && {
         processingCheckWait: timeout
       }, retryLimit && {
         processingCheckRetry: retryLimit
       });
-
       try {
         const processedAsset = await asset.processForAllLocales(processingOptions);
         return processedAsset;
       } catch (err) {
         err.entity = asset;
-
         _logging.logEmitter.emit('error', err);
-
         return null;
       }
     });

package/dist/tasks/push-to-space/creation.js

@@ -6,17 +6,11 @@ Object.defineProperty(exports, "__esModule", {
 exports.createEntities = createEntities;
 exports.createEntries = createEntries;
 exports.createLocales = createLocales;
-
 var _collection = require("lodash/collection");
-
 var _object = require("lodash/object");
-
 var _getEntityName = _interopRequireDefault(require("contentful-batch-libs/dist/get-entity-name"));
-
 var _logging = require("contentful-batch-libs/dist/logging");
-
 function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; }
-
 /**
  * Creates a list of entities
  * Applies to all entities except Entries, as the CMA API for those is slightly different
@@ -34,10 +28,10 @@ function createEntities({
     destinationEntitiesById,
     requestQueue
   });
-} // TODO
-// Locales need to be created in series
-
+}
 
+// TODO
+// Locales need to be created in series
 function createLocales({
   context,
   entities,
@@ -51,7 +45,6 @@ function createLocales({
     requestQueue
   });
 }
-
 async function createEntitiesWithConcurrency({
   context,
   entities,
@@ -71,11 +64,11 @@ async function createEntitiesWithConcurrency({
       }
     });
   });
-  const createdEntities = await Promise.all(pendingCreatedEntities); // Filter null values in case of errors
+  const createdEntities = await Promise.all(pendingCreatedEntities);
 
+  // Filter null values in case of errors
   return createdEntities.filter(entity => entity);
 }
-
 async function createEntitiesInSequence({
   context,
   entities,
@@ -83,11 +76,9 @@ async function createEntitiesInSequence({
   requestQueue
 }) {
   const createdEntities = [];
-
   for (const entity of entities) {
     const destinationEntity = getDestinationEntityForSourceEntity(destinationEntitiesById, entity.transformed);
     const operation = destinationEntity ? 'update' : 'create';
-
     try {
       // Even though we run things in sequence here,
       // we still want to go through the normal rate limiting queue
@@ -99,20 +90,17 @@ async function createEntitiesInSequence({
       createdEntities.push(createdEntity);
     } catch (err) {
       const maybeSubstituteEntity = handleCreationErrors(entity, err);
-
       if (maybeSubstituteEntity) {
         createdEntities.push(maybeSubstituteEntity);
       }
     }
   }
-
   return createdEntities;
 }
+
 /**
  * Creates a list of entries
  */
-
-
 async function createEntries({
   context,
   entities,
@@ -130,7 +118,6 @@ async function createEntries({
   }));
   return createdEntries.filter(entry => entry);
 }
-
 async function createEntry({
   entry,
   target,
@@ -141,7 +128,6 @@ async function createEntry({
   const contentTypeId = entry.original.sys.contentType.sys.id;
   const destinationEntry = getDestinationEntityForSourceEntity(destinationEntitiesById, entry.transformed);
   const operation = destinationEntry ? 'update' : 'create';
-
   try {
     const createdOrUpdatedEntry = await requestQueue.add(() => {
       return destinationEntry ? updateDestinationWithSourceData(destinationEntry, entry.transformed) : createEntryInDestination(target, contentTypeId, entry.transformed);
@@ -163,75 +149,62 @@ async function createEntry({
         requestQueue
       });
     }
-
     err.entity = entry;
+    _logging.logEmitter.emit('error', err);
 
-    _logging.logEmitter.emit('error', err); // No need to pass this entry down to publishing if it wasn't created
-
-
+    // No need to pass this entry down to publishing if it wasn't created
     return null;
   }
 }
-
 function updateDestinationWithSourceData(destinationEntity, sourceEntity) {
   const plainData = getPlainData(sourceEntity);
   (0, _object.assign)(destinationEntity, plainData);
   return destinationEntity.update();
 }
-
 function createInDestination(context, sourceEntity) {
   const {
     type,
     target
   } = context;
-
   if (type === 'Tag') {
     // tags are created with a different signature
     return createTagInDestination(context, sourceEntity);
   }
-
   const id = (0, _object.get)(sourceEntity, 'sys.id');
   const plainData = getPlainData(sourceEntity);
   return id ? target[`create${type}WithId`](id, plainData) : target[`create${type}`](plainData);
 }
-
 function createEntryInDestination(space, contentTypeId, sourceEntity) {
   const id = sourceEntity.sys.id;
   const plainData = getPlainData(sourceEntity);
   return id ? space.createEntryWithId(contentTypeId, id, plainData) : space.createEntry(contentTypeId, plainData);
 }
-
 function createTagInDestination(context, sourceEntity) {
   const id = sourceEntity.sys.id;
   const visibility = sourceEntity.sys.visibility || 'private';
   const name = sourceEntity.name;
   return context.target.createTag(id, name, visibility);
 }
+
 /**
  * Handles entity creation errors.
  * If the error is a VersionMismatch the error is thrown and a message is returned
  * instructing the user on what this situation probably means.
  */
-
-
 function handleCreationErrors(entity, err) {
   // Handle the case where a locale already exists and skip it
   if ((0, _object.get)(err, 'error.sys.id') === 'ValidationFailed') {
     const errors = (0, _object.get)(err, 'error.details.errors');
-
     if (errors && errors.length > 0 && errors[0].name === 'taken') {
       return entity;
     }
   }
-
   err.entity = entity.original;
+  _logging.logEmitter.emit('error', err);
 
-  _logging.logEmitter.emit('error', err); // No need to pass this entity down to publishing if it wasn't created
-
-
+  // No need to pass this entity down to publishing if it wasn't created
   return null;
 }
-
 function cleanupUnknownFields(fields, errors) {
   return (0, _object.omitBy)(fields, (field, fieldId) => {
     return (0, _collection.find)(errors, error => {
@@ -240,19 +213,14 @@ function cleanupUnknownFields(fields, errors) {
     });
   });
 }
-
 function getDestinationEntityForSourceEntity(destinationEntitiesById, sourceEntity) {
   return destinationEntitiesById.get((0, _object.get)(sourceEntity, 'sys.id')) || null;
 }
-
 function creationSuccessNotifier(method, createdEntity) {
   const verb = method[0].toUpperCase() + method.substr(1, method.length) + 'd';
-
   _logging.logEmitter.emit('info', `${verb} ${createdEntity.sys.type} ${(0, _getEntityName.default)(createdEntity)}`);
-
   return createdEntity;
 }
-
 function getPlainData(entity) {
   const data = entity.toPlainObject ? entity.toPlainObject() : entity;
   return (0, _object.omit)(data, 'sys');