@contentstack/cli-migration 0.1.1-beta.1

package/src/utils/constants.js

@@ -0,0 +1,219 @@
+'use strict'
+
+exports.mapObject = new Map()
+
+exports.version = 3 // TODO: Fetch it from CMA
+
+exports.defaultDataType = 'text'
+
+exports.MANAGEMENT_SDK = 'MANAGEMENT_SDK'
+exports.MANAGEMENT_SDK = 'MANAGEMENT_TOKEN'
+exports.AUTH_TOKEN = 'AUTH_TOKEN'
+exports.API_KEY = 'API_KEY'
+exports.BRANCH = 'BRANCH'
+
+exports.data_type = 'data_type'
+exports.mandatory = 'mandatory'
+exports._default = 'default'
+exports.unique = 'unique'
+exports.display_name = 'display_name'
+exports.reference_to = 'reference_to'
+exports.field_metadata = 'field_metadata'
+
+exports.actions = {
+  CUSTOM_TASK: 'CUSTOM_TASK',
+  CREATE_CT: 'CREATE_CT',
+  DELETE_CT: 'DELETE_CT',
+  EDIT_CT: 'EDIT_CT',
+  LOCALES: 'LOCALES',
+  EDIT_FIELD: 'EDIT_FIELD',
+  DELETE_FIELD: 'DELETE_FIELD',
+  MOVE_FIELD: 'MOVE_FIELD',
+}
+
+// Http call max retry
+exports.MAX_RETRY = 3
+
+// This key holds the value for http objects in map
+exports.requests = 'REQUESTS'
+
+exports.limit = 1 // Limit for concurrent tasks executed parallely
+
+exports.nonWritableMethods = ['GET', 'DELETE']
+
+exports.ContentType = 'Content type'
+exports.Entry = 'Entry'
+
+exports.errorMessageHandler = {
+  POST: 'saving',
+  GET: 'fetching',
+  PUT: 'updating',
+  DELETE: 'deleting',
+}
+
+exports.successMessageHandler = {
+  POST: 'saved',
+  GET: 'fetched',
+  PUT: 'updated',
+  DELETE: 'deleted',
+}
+// map key
+exports.actionMapper = 'actions'
+
+exports.batchLimit = 20
+
+exports.contentTypeProperties = ['description', 'title', 'uid', 'options', 'force', 'schema']
+
+exports.validationAction = {
+  create: 'create',
+  edit: 'edit',
+  customTask: 'customTask',
+  transformEntries: 'transformEntries',
+  deriveLinkedEntries: 'deriveLinkedEntries',
+  transformEntriesToType: 'transformEntriesToType',
+  typeError: 'typeError',
+  apiError: 'apiError',
+  schema: 'schema',
+  __migrationError: 'migrationError',
+  field: 'field',
+}
+
+exports.transformEntriesProperties = [
+  {
+    name: 'contentType',
+    type: 'string',
+    mandatory: true,
+  },
+  {
+    name: 'from',
+    type: 'array',
+    mandatory: true,
+  },
+  {
+    name: 'to',
+    type: 'array',
+    mandatory: true,
+  },
+  {
+    name: 'shouldPublish',
+    type: 'boolean',
+    mandatory: false,
+    dependsOn: 'environments',
+  },
+  {
+    name: 'environments',
+    type: 'array',
+    mandatory: false,
+  },
+  {
+    name: 'transformEntryForLocale',
+    type: 'function',
+    mandatory: true,
+  },
+]
+
+exports.deriveLinkedEntriesProperties = [
+  {
+    name: 'contentType',
+    type: 'string',
+    mandatory: true,
+  },
+  {
+    name: 'derivedContentType',
+    type: 'string',
+    mandatory: true,
+  },
+  {
+    name: 'from',
+    type: 'array',
+    mandatory: true,
+  },
+  {
+    name: 'toReferenceField',
+    type: 'string',
+    mandatory: true,
+  },
+  {
+    name: 'derivedFields',
+    type: 'array',
+    mandatory: true,
+  },
+  {
+    name: 'identityKey',
+    type: 'function',
+    mandatory: true,
+  },
+  {
+    name: 'shouldPublish',
+    type: 'boolean',
+    mandatory: false,
+    dependsOn: 'environments',
+  },
+  {
+    name: 'environments',
+    type: 'array',
+    mandatory: false,
+  },
+  {
+    name: 'deriveEntryForLocale',
+    type: 'function',
+    mandatory: true,
+  },
+]
+
+exports.transformEntriesToTypeProperties = [
+  {
+    name: 'sourceContentType',
+    type: 'string',
+    mandatory: true,
+  },
+  {
+    name: 'targetContentType',
+    type: 'string',
+    mandatory: true,
+  },
+  {
+    name: 'from',
+    type: 'array',
+    mandatory: true,
+  },
+  {
+    name: 'shouldPublish',
+    type: 'boolean',
+    mandatory: false,
+    dependsOn: 'environments',
+  },
+  {
+    name: 'environments',
+    type: 'array',
+    mandatory: false,
+  },
+  {
+    name: 'removeOldEntries',
+    type: 'boolean',
+    mandatory: false,
+  },
+  {
+    name: 'identityKey',
+    type: 'function',
+    mandatory: true,
+  },
+  {
+    name: 'transformEntryForLocale',
+    type: 'function',
+    mandatory: true,
+  },
+]
+
+exports.SDK_ACTIONS = {
+  CONTENTTYPE_GET: 'CONTENTTYPE_GET',
+  CONTENTTYPE_POST: 'CONTENTTYPE_POST',
+  CONTENTTYPE_DELETE: 'CONTENTTYPE_GET',
+  CONTENTTYPE_PUT: 'CONTENTTYPE_PUT',
+  LOCALES_GET: 'LOCALES_GET',
+  ENTRY_GET: 'ENTRY_GET',
+  ENTRY_POST: 'ENTRY_POST',
+  ENTRY_PUT: 'ENTRY_PUT',
+  ENTRY_DELETE: 'ENTRY_DELETE',
+  ENTRY_PUBLISH: 'ENTRY_PUBLISH',
+}

package/src/utils/contentstack-sdk.js

@@ -0,0 +1,71 @@
+'use strict'
+/** Dependencies */
+
+// Map helper
+const {getMapInstance, getDataWithAction, get} = require('./map')
+// Constants
+const {MANAGEMENT_SDK, SDK_ACTIONS} = require('./constants')
+// List of actions
+const {
+  CONTENTTYPE_DELETE,
+  CONTENTTYPE_GET,
+  CONTENTTYPE_POST,
+  CONTENTTYPE_PUT,
+  LOCALES_GET,
+  ENTRY_DELETE,
+  ENTRY_GET,
+  ENTRY_POST,
+  ENTRY_PUBLISH,
+  ENTRY_PUT,
+} = SDK_ACTIONS
+
+module.exports = ({action, id, sdkAction}) => {
+  return async _data => {
+    _data = getData(_data, id, action)
+
+    const mapInstance = getMapInstance()
+    const managementSdk = get(MANAGEMENT_SDK, mapInstance)
+    const {stack} = managementSdk
+
+    let response
+
+    switch (sdkAction) {
+    case CONTENTTYPE_GET:
+      response = await stack.contentType(id).fetch()
+      return response
+    case CONTENTTYPE_POST:
+      response = await stack.contentType().create(_data)
+      return response
+    case CONTENTTYPE_PUT:
+      // const contentType = await stack.contentType(id).fetch();
+      response = await stack.contentType(_data).update()
+      return response
+    case CONTENTTYPE_DELETE:
+      response = await stack.contentType(id).delete()
+      return response
+    case LOCALES_GET:
+      return response
+    case ENTRY_GET:
+      return response
+    case ENTRY_POST:
+      return response
+    case ENTRY_PUBLISH:
+      return response
+    case ENTRY_DELETE:
+      return response
+    case ENTRY_PUT:
+      return response
+    default:
+    }
+  }
+}
+
+function getData(_data, id, action) {
+  let mapInstance = getMapInstance()
+
+  let data = _data ? _data : getDataWithAction(id, mapInstance, action)
+
+  // return stringify(data);
+  return data
+}
+

package/src/utils/error-handler.js

@@ -0,0 +1,21 @@
+'use strict'
+
+const {error} = require('./logger')
+const {errorMessageHandler} = require('./constants')
+
+module.exports = (data, type, method, err) => {
+  if (data && type && method) {
+    error(`Error occurred while ${errorMessageHandler[method]} ${type}: ${data}.`)
+  }
+
+  if (err.errorMessage) {
+    error(err.errorMessage)
+  }
+  if (err instanceof Error && err && err.message && err.stack) {
+    error(err.message)
+    // error(err.stack)
+  } else {
+    error(err)
+  }
+  // throw new Error(err);
+}

package/src/utils/error-helper.js

@@ -0,0 +1,58 @@
+
+const {highlight} = require('cardinal')
+const {keys} = Object
+const chalk = require('chalk')
+
+const {readFile} = require('./fs-helper')
+const groupBy = require('./group-by')
+
+const getLineWithContext = (lines, lineNumber, context) => {
+  const line = (lineNumber - 1)
+
+  const firstLine = (line > context) ? (line - context) : 0
+  const lastLine = (line + context) < lines.length ? line + context : lines.length
+
+  return {
+    before: lines.slice(firstLine, line),
+    line: lines[line],
+    after: lines.slice(line + 1, lastLine + 1),
+  }
+}
+
+module.exports = errors => {
+  const errorsByFile = groupBy(errors, 'file')
+  const messages = []
+  for (const file of keys(errorsByFile)) {
+    const fileContents = readFile(file)
+    const highlightedCode = highlight(fileContents, {linenos: true})
+    const lines = highlightedCode.split('\n')
+
+    const fileErrorsMessage = chalk`{red Errors in ${file}}\n\n`
+    const errorMessages = errorsByFile[file].map(error => {
+      const callsite = error.meta.callsite
+      const context = 2
+      const {before, line, after} = getLineWithContext(lines, callsite.line, context)
+
+      const beforeLines = before.map(line => chalk`${line}\n`)
+      const afterLines = after.map(line => chalk`${line}\n`)
+      const highlightedLine = chalk`{bold ${line}}\n`
+
+      const formattedCode = beforeLines + highlightedLine + afterLines
+      if (error.payload.apiError) {
+        return chalk`{red Line ${String(callsite.line)}:} {bold ${error.payload.apiError.message}}\n${formattedCode}`
+      }
+      if (error.message) {
+        return chalk`{red Line ${String(callsite.line)}:} {bold ${error.message}}\n${formattedCode}`
+      }
+      return chalk`{red Line ${String(callsite.line)}:} {bold something went wrong here.}\n${formattedCode}`
+    }).join('\n')
+
+    messages.push(`${fileErrorsMessage}${errorMessages}`)
+  }
+  // eslint-disable-next-line
+  // console.error(chalk`{red.bold Validation failed}\n\n`);
+  // eslint-disable-next-line
+  console.log(messages.join('\n'));
+  // eslint-disable-next-line
+  console.log(chalk`{bold.red Migration unsuccessful}`);
+}

package/src/utils/fs-helper.js

@@ -0,0 +1,14 @@
+'use strict'
+
+const {existsSync, mkdirSync, readFileSync} = require('fs')
+
+exports.makeDir = dirname => {
+  !this.existsSync(dirname) && mkdirSync(dirname)
+}
+
+exports.existsSync = filePath => existsSync(filePath)
+
+exports.readFile = filePath => {
+  if (!existsSync(filePath)) throw new Error('File does not exist')
+  return readFileSync(filePath, 'utf-8')
+}

package/src/utils/get-batches.js

@@ -0,0 +1,7 @@
+'use strict'
+
+module.exports = (count, batchLimit) => {
+  const partitions = Math.ceil(count / batchLimit)
+  // Returns array filled with indexes
+  return new Array(partitions).fill(null).map((_, i) => i)
+}

package/src/utils/get-config.js

@@ -0,0 +1,13 @@
+'use strict'
+
+const {apiConfig} = require('../config')
+
+module.exports = ({method, path, sdkAction}) => {
+  return {
+    ...apiConfig,
+    path: path ? `${apiConfig.version}${path}` : apiConfig.version,
+    method,
+    headers: {...apiConfig.headers},
+    sdkAction,
+  }
+}

package/src/utils/group-by.js

@@ -0,0 +1,38 @@
+'use strict'
+
+module.exports = function groupBy(data, field, i = 0, finalObj = {}, originalArray = []) {
+  if (!data) return finalObj
+
+  if (Array.isArray(data)) {
+    groupBy(data[i], field, 0, finalObj, data)
+  } else if (field in data) {
+    let dataField = data[field]
+    if (dataField in finalObj) {
+      finalObj[dataField].push(originalArray[i])
+    } else {
+      finalObj[dataField] = []
+      finalObj[dataField].push(originalArray[i])
+    }
+
+    i++
+
+    // Breaks when i has been incremented more than length of original array
+    if (i !== 0 && i >= originalArray.length) return finalObj
+    /**
+       * After the field is found only then increment i and inspect next item
+       * This will restrict iterating through array just to the length of array
+      */
+    groupBy(originalArray[i], field, i, finalObj, originalArray)
+  } else {
+    for (let key in data) {
+      if (key) {
+        let dataKey = data[key]
+        if (!Array.isArray(dataKey) && typeof dataKey === 'object') {
+          groupBy(dataKey, field, i, finalObj, originalArray)
+        }
+      }
+    }
+  }
+
+  return finalObj
+}

package/src/utils/index.js

@@ -0,0 +1,21 @@
+'use strict'
+
+module.exports = {
+  map: require('./map'),
+  constants: require('./constants'),
+  schemaHelper: require('./schema-helper'),
+  objectHelper: require('./object-helper'),
+  fsHelper: require('./fs-helper'),
+  logger: require('./logger'),
+  https: require('./request'),
+  safePromise: require('./safe-promise'),
+  getConfig: require('./get-config'),
+  successHandler: require('./success-handler'),
+  errorHandler: require('./error-handler'),
+  getCallsite: require('./callsite'),
+  errorHelper: require('./error-helper'),
+  groupBy: require('./group-by'),
+  getBatches: require('./get-batches'),
+  autoRetry: require('./auto-retry'),
+  contentstackSdk: require('./contentstack-sdk'),
+}

package/src/utils/logger.js

@@ -0,0 +1,84 @@
+'use strict'
+
+const {createLogger, format, transports} = require('winston')
+const {resolve, join} = require('path')
+const {slice} = Array.prototype
+const {stringify} = JSON
+
+const {combine, label, printf, colorize} = format
+
+// FS helper
+const {makeDir} = require('./fs-helper')
+
+const {NODE_ENV} = process.env
+
+function getString(args) {
+  let str = ''
+  if (args && args.length > 0) {
+    str = args.map(item =>
+      item && typeof item === 'object' ?
+        stringify(item) :
+        item
+    )
+    .join(' ')
+    .trim()
+  }
+  return str
+}
+
+const customFormat = printf(({level, message}) => {
+  return `${level}: ${message}`
+})
+
+function init(logFileName) {
+  const logsDir = resolve('logs')
+  // Create dir if does not exist
+  makeDir(logsDir)
+
+  const logPath = join(logsDir, logFileName + '.log')
+  const logger = createLogger({
+    format: combine(
+      colorize(),
+      label({label: 'Migration'}),
+      customFormat
+    ),
+    transports: [
+      new transports.File({filename: logPath}),
+      new transports.Console(),
+    ],
+  })
+
+  let args
+  let logString
+
+  return {
+    log: function () {
+      args = slice.call(arguments)
+      logString = getString(args)
+      logString && logger.log('info', logString)
+    },
+    warn: function () {
+      args = slice.call(arguments)
+      logString = getString(args)
+      logString && logger.log('warn', logString)
+    },
+    error: function () {
+      args = slice.call(arguments)
+      logString = getString(args)
+      logString && logger.log('error', logString)
+    },
+    debug: function () {
+      args = slice.call(arguments)
+      logString = getString(args)
+      logString && logger.log('debug', logString)
+    },
+  }
+}
+
+exports.success = init('success').log
+if (NODE_ENV === 'test') {
+  exports.error = init('warn').warn
+} else {
+  exports.error = init('error').error
+}
+exports.warn = init('warn').warn

package/src/utils/map.js

@@ -0,0 +1,40 @@
+'use strict'
+
+const {mapObject, actionMapper, requests} = require('./constants')
+
+exports.getMapInstance = () => {
+  return mapObject
+}
+
+exports.get = (id, mapInstance, data = []) => {
+  // Create key if does not exist
+  let __data = mapInstance.get(id)
+
+  !__data && (
+    mapInstance.set(id, data),
+    __data = mapInstance.get(id)
+  )
+
+  return __data
+}
+
+exports.set = (id, mapInstance, data) => {
+  return mapInstance.set(id, data)
+}
+
+exports.remove = (id, mapInstance) => {
+  return mapInstance.delete(id)
+}
+
+exports.getDataWithAction = (id, mapInstance, action) => {
+  let data = this.get(id, mapInstance)
+  data = data[action]
+  return data
+}
+
+exports.resetMapInstance = mapInstance => {
+  this.set(actionMapper, mapInstance, [])
+  this.set(requests, mapInstance, [])
+}
+
+exports.delete = () => { }

package/src/utils/object-helper.js

@@ -0,0 +1,9 @@
+'use strict'
+
+exports.getEntryObj = (fields, obj) => {
+  let entryObj = {}
+  fields.forEach(field => {
+    entryObj[field] = obj[field]
+  })
+  return entryObj
+}

package/src/utils/request.js

@@ -0,0 +1,95 @@
+'use strict'
+
+// Dependencies
+const {request} = require('https')
+const {stringify, parse} = JSON
+
+// Map helper
+const {getMapInstance, getDataWithAction} = require('./map')
+
+// constants
+const {actions, nonWritableMethods} = require('./constants')
+
+// Properties
+const {DELETE_CT} = actions
+
+module.exports = ({
+  hostname,
+  path,
+  headers,
+  method,
+  id,
+  action,
+}) => {
+  let options = {
+    hostname,
+    path,
+    headers,
+    method,
+    id,
+    action,
+  }
+  return _data => {
+    // get data here using id and action
+    let data = getData(_data, id, action, method)
+    // Special handling for non writable methods
+    options = getNewOptions(options, data, action, method)
+
+    return new Promise((resolve, reject) => {
+      const req = request(options, res => {
+        let response = ''
+
+        res.on('data', _res => {
+          response += _res.toString()
+        })
+
+        res.on('end', () => {
+          try {
+            response = parse(response)
+            resolve(response)
+          } catch (err) {
+            reject('Error while parsing response!')
+            // throw new Error('Error while parsing response!');
+          }
+        })
+      })
+
+      req.on('error', err => {
+        reject(err)
+      })
+
+      !nonWritableMethods.includes(method) && req.write(data)
+      req.end()
+    })
+  }
+}
+
+function getData(_data, id, action, method) {
+  if (method === 'GET') return
+  // if (!nonWritableMethods.includes(method)) {
+  let mapInstance = getMapInstance()
+
+  let data = _data ? _data : getDataWithAction(id, mapInstance, action)
+  return stringify(data)
+}
+
+function getNewOptions(options, data, action, method) {
+  // Special handling for delete method
+  if (action === DELETE_CT) {
+    try {
+      data = parse(data)
+    } catch (err) {
+      throw 'Error while parsing data for delete operation'
+    }
+    options.path = `${options.path}?force=${data.content_type.force}`
+  }
+
+  if (!nonWritableMethods.includes(method)) {
+    options.headers['Content-Length'] = data.length
+  } else {
+    delete options.headers['Content-Type']
+    delete options.headers['Content-Length']
+  }
+
+  return options
+}

package/src/utils/safe-promise.js

@@ -0,0 +1,3 @@
+'use strict'
+
+module.exports = promise => promise.then(res => [null, res]).catch(err => [err])