@contentstack/cli-cm-import 0.1.1-beta.9 → 1.0.1

package/src/lib/util/log.js

@@ -9,54 +9,91 @@ var path = require('path');
 var mkdirp = require('mkdirp');
 var slice = Array.prototype.slice;
 
-function returnString (args) {
+
+
+const ansiRegexPattern = [
+  '[\\u001B\\u009B][[\\]()#;?]*(?:(?:(?:(?:;[-a-zA-Z\\d\\/#&.:=?%@~_]+)*|[a-zA-Z\\d]+(?:;[-a-zA-Z\\d\\/#&.:=?%@~_]*)*)?\\u0007)',
+'(?:(?:\\d{1,4}(?:;\\d{0,4})*)?[\\dA-PR-TZcf-ntqry=><~]))'
+].join('|');
+
+function returnString(args) {
   var returnStr = '';
   if (args && args.length) {
-    returnStr = args.map(function (item) {
-      if (item && typeof (item) === 'object') {
-        return JSON.stringify(item);
-      }
-      return item;
-    }).join('  ').trim();
+    returnStr = args
+      .map(function (item) {
+        if (item && typeof item === 'object') {
+          return JSON.stringify(item).replace(/authtoken\":\"blt................/g, 'authtoken":"blt....');
+        }
+        return item;
+      })
+      .join('  ')
+      .trim();
   }
+  returnStr = returnStr.replace(new RegExp(ansiRegexPattern, 'g'), "").trim();
   return returnStr;
 }
 
 var myCustomLevels = {
   levels: {
-    error: 0,
     warn: 1,
     info: 2,
-    debug: 3
+    debug: 3,
   },
   colors: {
     info: 'blue',
     debug: 'green',
     warn: 'yellow',
-    error: 'red'
-  }
+    error: 'red',
+  },
 };
 
-function init (_logPath, logfileName) {
-  var logsDir = path.resolve(_logPath, 'logs', 'import')  
-  // Create dir if doesn't already exist
-  mkdirp.sync(logsDir)
-  var logPath = path.join(logsDir, logfileName + '.log');
+let logger;
+let errorLogger;
+
+let successTransport;
+let errorTransport;
 
-  var transports = [new(winston.transports.File)({
-    filename: logPath,
-    maxFiles: 20,
-    maxsize: 1000000,
-    tailable: true,
-    json: true
-  })];
+// removed logfileName from arguments
+function init(_logPath) {
+  if (!logger || !errorLogger) {
+    var logsDir = path.resolve(_logPath, 'logs', 'import');
+    // Create dir if doesn't already exist
+    mkdirp.sync(logsDir);
 
-  transports.push(new(winston.transports.Console)());
+    successTransport = {
+      filename: path.join(logsDir, 'success.log'),
+      maxFiles: 20,
+      maxsize: 1000000,
+      tailable: true,
+      json: true,
+      level: 'info',
+    };
 
-  var logger = new(winston.Logger)({
-    transports: transports,
-    levels: myCustomLevels.levels
-  });
+    errorTransport = {
+      filename: path.join(logsDir, 'error.log'),
+      maxFiles: 20,
+      maxsize: 1000000,
+      tailable: true,
+      json: true,
+      level: 'error',
+    };
+
+    logger = winston.createLogger({
+      transports: [
+        new winston.transports.File(successTransport),
+        new winston.transports.Console({ format: winston.format.simple() }),
+      ],
+      levels: myCustomLevels.levels,
+    });
+
+    errorLogger = winston.createLogger({
+      transports: [
+        new winston.transports.File(errorTransport),
+        new winston.transports.Console({ level: 'error', format: winston.format.simple() }),
+      ],
+      levels: { error: 0 },
+    });
+  }
 
   return {
     log: function () {
@@ -77,7 +114,7 @@ function init (_logPath, logfileName) {
       var args = slice.call(arguments);
       var logString = returnString(args);
       if (logString) {
-        logger.log('error', logString);
+        errorLogger.log('error', logString);
       }
     },
     debug: function () {
@@ -86,16 +123,17 @@ function init (_logPath, logfileName) {
       if (logString) {
         logger.log('debug', logString);
       }
-    }
+    },
   };
 }
 
 exports.addlogs = async (config, message, type) => {
-  var configLogPath
-  config.source_stack && config.target_stack ? configLogPath = config.data : configLogPath = config.oldPath
+  var configLogPath = config.source_stack && config.target_stack ? config.data : config.oldPath;
+  // ignoring the type argument, as we are not using it to create a logfile anymore
   if (type !== 'error') {
-    init(configLogPath, type).log(message)
+    // removed type argument from init method
+    init(configLogPath).log(message);
   } else {
-    init(configLogPath, type).error(message)
+    init(configLogPath).error(message);
   }
-}
+};

package/src/lib/util/login.js

@@ -7,48 +7,49 @@
 
 const chalk = require('chalk');
 
-const {addlogs} = require('../util/log')
-// var pkg = require('../../package');
-let stack = require('../util/contentstack-management-sdk')
-let client
-
+const { addlogs } = require('../util/log');
+let stack = require('../util/contentstack-management-sdk');
+let client;
 
 module.exports = function (config) {
-  client = stack.Client(config)
-  
+  client = stack.Client(config);
+
   return new Promise(function (resolve, reject) {
     // eslint-disable-next-line no-console
     if (config.email && config.password) {
-      console.log('Logging into Contentstack')
-      return client.login({ email: config.email, password: config.password})
-      .then(function (response) {
-      // eslint-disable-next-line no-console
-        console.log(chalk.green('Contentstack account authenticated successfully!'));
-        config.authtoken = response.user.authtoken
-        config.headers = {
-          api_key: config.target_stack,
-          authtoken: config.authtoken,
-          'X-User-Agent': 'contentstack-import/v'
-        };        
-        return resolve(config);
-      }).catch(reject);
+      console.log('Logging into Contentstack');
+      return client
+        .login({ email: config.email, password: config.password })
+        .then(function (response) {
+          // eslint-disable-next-line no-console
+          console.log(chalk.green('Contentstack account authenticated successfully!'));
+          config.authtoken = response.user.authtoken;
+          config.headers = {
+            api_key: config.target_stack,
+            authtoken: config.authtoken,
+            'X-User-Agent': 'contentstack-import/v',
+          };
+          return resolve(config);
+        })
+        .catch(reject);
     } else if (config.management_token) {
-      return resolve()
-    } else if (config.auth_token) {                 
-      //  return resolve()
-      client.stack({api_key: config.target_stack, management_token: config.management_token}).fetch()
-      .then(function () {
-          return resolve()
-      })
-      .catch(error => {   
-        let errorstack_key = error.errors.api_key       
-        if(error.errors.api_key) {
-          addlogs(config, chalk.red("Stack Api key " + errorstack_key[0], "Please enter valid Key"), 'error')
-          return reject()
-        }
-        addlogs(config, error.errorMessage, 'error')
-        return reject()
-      })
+      return resolve();
+    } else if (config.auth_token) {
+      client
+        .stack({ api_key: config.target_stack, management_token: config.management_token })
+        .fetch()
+        .then(function () {
+          return resolve();
+        })
+        .catch((error) => {
+          let errorstack_key = error.errors.api_key;
+          if (error.errors.api_key) {
+            addlogs(config, chalk.red('Stack Api key ' + errorstack_key[0], 'Please enter valid Key'), 'error');
+            return reject(error);
+          }
+          addlogs(config, error.errorMessage, 'error');
+          return reject(error);
+        });
     }
   });
 };

package/src/lib/util/lookupReplaceAssets.js

@@ -4,62 +4,173 @@
  * MIT Licensed
  */
 
-var url = require('url');
-var path = require('path');
-var _ = require('lodash');
-var marked = require('marked');
+let url = require('url');
+let path = require('path');
+let _ = require('lodash');
+let { marked } = require('marked');
 
-var helper = require('./fs');
+let helper = require('./fs');
 
 // get assets object
 module.exports = function (data, mappedAssetUids, mappedAssetUrls, assetUidMapperPath) {
-  if (!_.has(data, 'entry') || !_.has(data, 'content_type') || !_.isPlainObject(mappedAssetUids) || !_.isPlainObject(
-    mappedAssetUrls) || typeof assetUidMapperPath !== 'string') {
+  if (
+    !_.has(data, 'entry') ||
+    !_.has(data, 'content_type') ||
+    !_.isPlainObject(mappedAssetUids) ||
+    !_.isPlainObject(mappedAssetUrls) ||
+    typeof assetUidMapperPath !== 'string'
+  ) {
     throw new Error('Invalid inputs for lookupAssets!');
   }
-  var parent = [];
-  var assetUids = [];
-  var assetUrls = [];
-  var unmatchedUids = [];
-  var unmatchedUrls = [];
-  var matchedUids = [];
-  var matchedUrls = [];
-
-  var find = function (schema, entry) {
-    for (var i = 0, _i = schema.length; i < _i; i++) {
-      if ((schema[i].data_type === 'text' && schema[i].field_metadata && (schema[i].field_metadata.markdown ||
-          schema[i].field_metadata.rich_text_type))) {
+  let parent = [];
+  let assetUids = [];
+  let assetUrls = [];
+  let unmatchedUids = [];
+  let unmatchedUrls = [];
+  let matchedUids = [];
+  let matchedUrls = [];
+
+  let find = function (schema, entryToFind) {
+    for (let i = 0, _i = schema.length; i < _i; i++) {
+      if (
+        schema[i].data_type === 'text' &&
+        schema[i].field_metadata &&
+        (schema[i].field_metadata.markdown || schema[i].field_metadata.rich_text_type)
+      ) {
         parent.push(schema[i].uid);
-        findFileUrls(parent, schema[i], entry, assetUrls);
+        findFileUrls(schema[i], entryToFind, assetUrls);
         parent.pop();
       }
       if (schema[i].data_type === 'group' || schema[i].data_type === 'global_field') {
         parent.push(schema[i].uid);
-        find(schema[i].schema, entry);
+        find(schema[i].schema, entryToFind);
         parent.pop();
       }
       if (schema[i].data_type === 'blocks') {
-        for (var j = 0, _j = schema[i].blocks.length; j < _j; j++) {
-          if(schema[i].blocks[j].schema) {
+        for (let j = 0, _j = schema[i].blocks.length; j < _j; j++) {
+          if (schema[i].blocks[j].schema) {
             parent.push(schema[i].uid);
             parent.push(schema[i].blocks[j].uid);
-            find(schema[i].blocks[j].schema, entry);
+            find(schema[i].blocks[j].schema, entryToFind);
             parent.pop();
             parent.pop();
           }
         }
       }
+      // added rich_text_type field check because some marketplace extensions also
+      // have data_type has json
+      if (schema[i].data_type === 'json' && schema[i].field_metadata.rich_text_type) {
+        parent.push(schema[i].uid);
+        // findFileUrls(schema[i], entry, assetUrls)
+        findAssetIdsFromJsonRte(data.entry, data.content_type.schema);
+        // maybe only one of these checks would be enough
+        parent.pop();
+      }
     }
   };
-  
-  
+
+  function findAssetIdsFromJsonRte(entryObj, ctSchema) {
+    for (const element of ctSchema) {
+      switch (element.data_type) {
+        case 'blocks': {
+          if (entryObj[element.uid]) {
+            if (element.multiple) {
+              entryObj[element.uid].forEach((e) => {
+                let key = Object.keys(e).pop();
+                let subBlock = element.blocks.filter((block) => block.uid === key).pop();
+                findAssetIdsFromJsonRte(e[key], subBlock.schema);
+              });
+            }
+          }
+          break;
+        }
+        case 'global_field':
+        case 'group': {
+          if (entryObj[element.uid]) {
+            if (element.multiple) {
+              entryObj[element.uid].forEach((e) => {
+                findAssetIdsFromJsonRte(e, element.schema);
+              });
+            } else {
+              findAssetIdsFromJsonRte(entryObj[element.uid], element.schema);
+            }
+          }
+          break;
+        }
+        case 'json': {
+          if (entryObj[element.uid] && element.field_metadata.rich_text_type) {
+            if (element.multiple) {
+              entryObj[element.uid].forEach((jsonRteData) => {
+                gatherJsonRteAssetIds(jsonRteData);
+              });
+            } else {
+              gatherJsonRteAssetIds(entryObj[element.uid]);
+            }
+          }
+          break;
+        }
+      }
+    }
+  }
+
+  function gatherJsonRteAssetIds(jsonRteData) {
+    jsonRteData.children.forEach((element) => {
+      if (element.type) {
+        switch (element.type) {
+          case 'a':
+          case 'p': {
+            if (element.children && element.children.length > 0) {
+              gatherJsonRteAssetIds(element);
+            }
+            break;
+          }
+          case 'reference': {
+            if (Object.keys(element.attrs).length > 0 && element.attrs.type === 'asset') {
+              if (assetUids.indexOf(element.attrs['asset-uid']) === -1) {
+                assetUids.push(element.attrs['asset-uid']);
+              }
+              // assets references inserted as link inside entry reference inserted as link did not have asset-link property
+              // instead it had an 'href' property. I haven't seen 'asset-link' and 'href' together yet
+              // writing this condition assuming that this never occurs, need to confirm
+              // (element.attrs['asset-link']) ? assetUrls.push(element.attrs['asset-link']) : assetUrls.push(element.attrs['asset-link'])
+              if (element.attrs['asset-link']) {
+                if (assetUrls.indexOf(element.attrs['asset-link']) === -1) {
+                  assetUrls.push(element.attrs['asset-link']);
+                }
+              } else if (element.attrs['href']) {
+                if (assetUrls.indexOf(element.attrs['href']) === -1) {
+                  assetUrls.push(element.attrs['href']);
+                }
+              }
+            }
+            if (element.children && element.children.length > 0) {
+              gatherJsonRteAssetIds(element);
+            }
+            break;
+          }
+        }
+      }
+    });
+  }
+
   find(data.content_type.schema, data.entry);
   updateFileFields(data.entry, data, null, mappedAssetUids, matchedUids, unmatchedUids);
   assetUids = _.uniq(assetUids);
   assetUrls = _.uniq(assetUrls);
-  var entry = JSON.stringify(data.entry);
+  let entry = JSON.stringify(data.entry);
+
+  assetUrls.forEach(function (assetUrl) {
+    let mappedAssetUrl = mappedAssetUrls[assetUrl];
+    if (typeof mappedAssetUrl !== 'undefined') {
+      entry = entry.replace(new RegExp(assetUrl, 'img'), mappedAssetUrl);
+      matchedUrls.push(mappedAssetUrl);
+    } else {
+      unmatchedUrls.push(assetUrl);
+    }
+  });
+
   assetUids.forEach(function (assetUid) {
-    var uid = mappedAssetUids[assetUid];
+    let uid = mappedAssetUids[assetUid];
     if (typeof uid !== 'undefined') {
       entry = entry.replace(new RegExp(assetUid, 'img'), uid);
       matchedUids.push(assetUid);
@@ -68,63 +179,53 @@ module.exports = function (data, mappedAssetUids, mappedAssetUrls, assetUidMappe
     }
   });
 
-  assetUrls.forEach(function (assetUrl) {
-    var url = mappedAssetUrls[assetUrl];
-    if (typeof url !== 'undefined') {
-      entry = entry.replace(new RegExp(assetUrl, 'img'), url);
-      unmatchedUrls.push(url);
-    } else {
-      unmatchedUrls.push(assetUrl);
-    }
-  });
-
   if (matchedUids.length) {
-    var matchedAssetUids = helper.readFile(path.join(assetUidMapperPath, 'matched-asset-uids.json'));
+    let matchedAssetUids = helper.readFile(path.join(assetUidMapperPath, 'matched-asset-uids.json'));
     matchedAssetUids = matchedAssetUids || {};
     if (matchedAssetUids.hasOwnProperty(data.content_type.uid)) {
       matchedAssetUids[data.content_type.uid][data.entry.uid] = matchedUids;
     } else {
       matchedAssetUids[data.content_type.uid] = {
-        [data.entry.uid]: matchedUids
+        [data.entry.uid]: matchedUids,
       };
     }
     helper.writeFile(path.join(assetUidMapperPath, 'matched-asset-uids.json'));
   }
 
   if (unmatchedUids.length) {
-    var unmatchedAssetUids = helper.readFile(path.join(assetUidMapperPath, 'unmatched-asset-uids.json'));
+    let unmatchedAssetUids = helper.readFile(path.join(assetUidMapperPath, 'unmatched-asset-uids.json'));
     unmatchedAssetUids = unmatchedAssetUids || {};
     if (unmatchedAssetUids.hasOwnProperty(data.content_type.uid)) {
       unmatchedAssetUids[data.content_type.uid][data.entry.uid] = unmatchedUids;
     } else {
       unmatchedAssetUids[data.content_type.uid] = {
-        [data.entry.uid]: unmatchedUids
+        [data.entry.uid]: unmatchedUids,
       };
     }
     helper.writeFile(path.join(assetUidMapperPath, 'unmatched-asset-uids.json'));
   }
 
   if (unmatchedUrls.length) {
-    var unmatchedAssetUrls = helper.readFile(path.join(assetUidMapperPath, 'unmatched-asset-urls.json'));
+    let unmatchedAssetUrls = helper.readFile(path.join(assetUidMapperPath, 'unmatched-asset-urls.json'));
     unmatchedAssetUrls = unmatchedAssetUrls || {};
     if (unmatchedAssetUrls.hasOwnProperty(data.content_type.uid)) {
       unmatchedAssetUrls[data.content_type.uid][data.entry.uid] = unmatchedUrls;
     } else {
       unmatchedAssetUrls[data.content_type.uid] = {
-        [data.entry.uid]: unmatchedUrls
+        [data.entry.uid]: unmatchedUrls,
       };
     }
     helper.writeFile(path.join(assetUidMapperPath, 'unmatched-asset-urls.json'));
   }
 
   if (matchedUrls.length) {
-    var matchedAssetUrls = helper.readFile(path.join(assetUidMapperPath, 'matched-asset-urls.json'));
+    let matchedAssetUrls = helper.readFile(path.join(assetUidMapperPath, 'matched-asset-urls.json'));
     matchedAssetUrls = matchedAssetUrls || {};
     if (matchedAssetUrls.hasOwnProperty(data.content_type.uid)) {
       matchedAssetUrls[data.content_type.uid][data.entry.uid] = matchedUrls;
     } else {
       matchedAssetUrls[data.content_type.uid] = {
-        [data.entry.uid]: matchedUrls
+        [data.entry.uid]: matchedUrls,
       };
     }
     helper.writeFile(path.join(assetUidMapperPath, 'matched-asset-urls.json'));
@@ -133,20 +234,23 @@ module.exports = function (data, mappedAssetUids, mappedAssetUrls, assetUidMappe
   return JSON.parse(entry);
 };
 
-function findFileUrls (schema, _entry, assetUrls) {
-  var markdownRegEx;
-  var markdownMatch;
+function findFileUrls(schema, _entry, assetUrls) {
+  let markdownRegEx;
+  let markdownMatch;
 
   // Regex to detect v2 asset uri patterns
-  var _matches, regex;
+  let _matches, regex;
   if (schema && schema.field_metadata && schema.field_metadata.markdown) {
     regex = new RegExp(
       // eslint-disable-next-line no-control-regex
-      'https://(contentstack-|)api.(built|contentstack).io/(.*?)/download(.*?)uid=([a-z0-9]+[^\?&\s\n])((.*)[\n\s]?)',
-      'g');
+      'https://(contentstack-|)api.(built|contentstack).io/(.*?)/download(.*?)uid=([a-z0-9]+[^?&s\n])((.*)[\ns]?)',
+      'g',
+    );
   } else {
     regex = new RegExp(
-      'https://(contentstack-|)api.(built|contentstack).io/(.*?)/download(.*?)uid=([a-z0-9]+[^\?&\'"])(.*?)', 'g');
+      'https://(contentstack-|)api.(built|contentstack).io/(.*?)/download(.*?)uid=([a-z0-9]+[^?&\'"])(.*?)',
+      'g',
+    );
   }
   while ((_matches = regex.exec(_entry)) !== null) {
     if (_matches && _matches.length) {
@@ -155,14 +259,17 @@ function findFileUrls (schema, _entry, assetUrls) {
       }
     }
   }
-  var text;
+  let text;
   // Regex to detect v3 asset uri patterns
   if (schema && schema.field_metadata && schema.field_metadata.markdown) {
-    text = marked(_entry);
+    text = marked(JSON.stringify(_entry));
   } else {
-    text = _entry;
+    text = JSON.stringify(_entry);
   }
-  markdownRegEx = new RegExp('(https://(assets|images).contentstack.io/v3/assets/(.*?)/(.*?)/(.*?)/(.*)(?="))', 'g');
+  markdownRegEx = new RegExp(
+    '(https://(assets|(eu-|azure-na-)?images).contentstack.(io|com)/v3/assets/(.*?)/(.*?)/(.*?)/(.*?)(?="))',
+    'g',
+  );
   while ((markdownMatch = markdownRegEx.exec(text)) !== null) {
     if (markdownMatch && typeof markdownMatch[0] === 'string') {
       assetUrls.push(markdownMatch[0]);
@@ -170,7 +277,7 @@ function findFileUrls (schema, _entry, assetUrls) {
   }
 }
 
-function updateFileFields (objekt, parent, pos, mappedAssetUids, matchedUids, unmatchedUids) {
+function updateFileFields(objekt, parent, pos, mappedAssetUids, matchedUids, unmatchedUids) {
   if (_.isPlainObject(objekt) && _.has(objekt, 'filename') && _.has(objekt, 'uid')) {
     if (typeof pos !== 'undefined') {
       if (typeof pos === 'number' || typeof pos === 'string') {
@@ -184,10 +291,9 @@ function updateFileFields (objekt, parent, pos, mappedAssetUids, matchedUids, un
       }
     }
   } else if (_.isPlainObject(objekt)) {
-    for (var key in objekt)
-      updateFileFields(objekt[key], objekt, key, mappedAssetUids, matchedUids, unmatchedUids);
+    for (let key in objekt) updateFileFields(objekt[key], objekt, key, mappedAssetUids, matchedUids, unmatchedUids);
   } else if (_.isArray(objekt) && objekt.length) {
-    for (var i = 0; i <= objekt.length; i++)
+    for (let i = 0; i <= objekt.length; i++)
       updateFileFields(objekt[i], objekt, i, mappedAssetUids, matchedUids, unmatchedUids);
 
     parent[pos] = _.compact(objekt);