@contentstack/cli-migration 0.1.1-beta.1

package/README.md

@@ -0,0 +1,69 @@
+@contentstack/cli-migration
+===========================
+
+The Contentstack CLI’s “Migration” plugin allows developers to automate the content migration process and easily migrate your content from your system to Contentstack. 
+
+
+[![oclif](https://img.shields.io/badge/cli-oclif-brightgreen.svg)](https://oclif.io)
+[![Version](https://img.shields.io/npm/v/@contentstack/cli-migration.svg)](https://npmjs.org/package/@contentstack/cli-migration)
+[![Downloads/week](https://img.shields.io/npm/dw/@contentstack/cli-migration.svg)](https://npmjs.org/package/@contentstack/cli-migration)
+[![License](https://img.shields.io/npm/l/@contentstack/cli-migration.svg)](https://github.com/ninadhatkar/cli-migration/blob/master/package.json)
+
+<!-- toc -->
+* [Usage](#usage)
+* [Commands](#commands)
+<!-- tocstop -->
+# Usage
+<!-- usage -->
+```sh-session
+$ npm install -g @contentstack/cli-migration
+$ csdx COMMAND
+running command...
+$ csdx (-v|--version|version)
+@contentstack/cli-migration/0.1.1-beta.1 linux-x64 node-v12.18.4
+$ csdx --help [COMMAND]
+USAGE
+  $ csdx COMMAND
+...
+```
+<!-- usagestop -->
+# Commands
+<!-- commands -->
+* [`csdx cm:migration`](#csdx-cmmigration)
+
+## `csdx cm:migration`
+
+Contentstack migration script.
+
+```
+Contentstack migration script.
+
+USAGE
+  $ csdx cm:migration
+
+OPTIONS
+  -A, --authtoken                                      Use this flag to use the auth token of the current session. After
+                                                       logging in CLI, an auth token is generated for each new session.
+
+  -B, --branch=branch                                  Use this flag to add the branch name where you want to perform
+                                                       the migration.
+
+  -a, --management-token-alias=management-token-alias  Use this flag to add the management token alias.
+
+  -k, --api-key=api-key                                With this flag add the API key of your stack.
+
+  -n, --filePath=filePath                              Use this flag to provide the path of the file of the migration
+                                                       script provided by the user.
+
+  --multi                                              This flag helps you to migrate multiple content files in a single
+                                                       instance.
+```
+
+_See code: [src/commands/cm/migration.js](https://github.com/contentstack/cli-migration/blob/v0.1.1-beta.1/src/commands/cm/migration.js)_
+<!-- commandsstop -->
+
+### Points to remember
+
+* Currently, the Migration plugin does not support Group fields migration. You can pass a custom schema to the createField method to migrate Group fields. [here](packages/contentstack-migration/examples/)
+* Currently, the Migration plugin does not support Global fields migration. You can migrate Global fields by creating an SDK instance and adding it to content types using the createField method.[here](packages/contentstack-migration/examples/)
+* Currently, the Migration plugin does not support migration of Entries. You can migrate entries by creating an SDK instance to create/ update/ delete entries for your content type. [here](packages/contentstack-migration/examples/)

package/oclif.manifest.json

@@ -0,0 +1 @@
+{"version":"0.1.1-beta.1","commands":{"cm:migration":{"id":"cm:migration","description":"Contentstack migration script.","pluginName":"@contentstack/cli-migration","pluginType":"core","aliases":[],"flags":{"api-key":{"name":"api-key","type":"option","char":"k","description":"With this flag add the API key of your stack."},"authtoken":{"name":"authtoken","type":"boolean","char":"A","description":"Use this flag to use the auth token of the current session. After logging in CLI, an auth token is generated for each new session.","allowNo":false},"management-token-alias":{"name":"management-token-alias","type":"option","char":"a","description":"Use this flag to add the management token alias."},"filePath":{"name":"filePath","type":"option","char":"n","description":"Use this flag to provide the path of the file of the migration script provided by the user."},"branch":{"name":"branch","type":"option","char":"B","description":"Use this flag to add the branch name where you want to perform the migration."},"multi":{"name":"multi","type":"boolean","description":"This flag helps you to migrate multiple content files in a single instance.","allowNo":false}},"args":[]}}}

package/package.json

@@ -0,0 +1,68 @@
+{
+  "name": "@contentstack/cli-migration",
+  "version": "0.1.1-beta.1",
+  "author": "@contentstack",
+  "bugs": "https://github.com/contentstack/cli/issues",
+  "dependencies": {
+    "@contentstack/cli-command": "^0.1.1-beta.2",
+    "@contentstack/management": "github:contentstack/contentstack-management-javascript#65e8588",
+    "@oclif/command": "^1.6.1",
+    "@oclif/config": "^1.15.1",
+    "async": "^3.2.0",
+    "axios": "^0.21.1",
+    "callsites": "^3.1.0",
+    "cardinal": "^2.1.1",
+    "chalk": "^4.1.0",
+    "dot-object": "^2.1.4",
+    "dotenv": "^8.2.0",
+    "listr": "^0.14.3",
+    "winston": "^3.3.3"
+  },
+  "devDependencies": {
+    "@oclif/dev-cli": "^1.26.0",
+    "@oclif/plugin-help": "^2.2.3",
+    "@oclif/test": "^1.2.8",
+    "chai": "^4.3.4",
+    "eslint": "^5.16.0",
+    "eslint-config-oclif": "^3.1.0",
+    "eslint-config-oclif-typescript": "^0.2.0",
+    "globby": "^10.0.2",
+    "husky": "^4.2.5",
+    "jsdoc": "^3.6.7",
+    "jsdoc-to-markdown": "^7.0.1",
+    "mkdirp": "^1.0.4",
+    "mocha": "^5.2.0",
+    "nock": "^13.1.1",
+    "nyc": "^14.1.1"
+  },
+  "engines": {
+    "node": ">=8.3.0"
+  },
+  "files": [
+    "/npm-shrinkwrap.json",
+    "/oclif.manifest.json",
+    "/src",
+    "/yarn.lock"
+  ],
+  "homepage": "https://github.com/contentstack/cli",
+  "keywords": [
+    "oclif-plugin"
+  ],
+  "license": "MIT",
+  "oclif": {
+    "commands": "./src/commands",
+    "bin": "csdx",
+    "devPlugins": [
+      "@oclif/plugin-help"
+    ]
+  },
+  "repository": "contentstack/cli-migration",
+  "scripts": {
+    "postpack": "rm -f oclif.manifest.json",
+    "posttest": "eslint ./src",
+    "generate-api-ref-md-doc": "nyc jsdoc2md ./src/modules/* > ./docs/api-reference.md",
+    "prepack": "oclif-dev manifest && oclif-dev readme && npm run generate-api-ref-md-doc",
+    "test": "nyc mocha \"test/**/*.test.js\"",
+    "version": "oclif-dev readme && git add README.md"
+  }
+}

package/src/actions/action-list.js

@@ -0,0 +1,32 @@
+/* eslint-disable no-unused-expressions */
+'use strict'
+
+class ActionList {
+  constructor(actionList, typeErrors) {
+    typeErrors && (this.typeErrors = typeErrors)
+    actionList && (this.actionList = actionList)
+    this.validators = []
+  }
+
+  addValidators(validator) {
+    this.validators.push(validator)
+  }
+
+  validate() {
+    const {validators, actionList} = this
+
+    let errors = []
+    for (const action of actionList) {
+      for (const validator of validators) {
+        if (validator.isApplicable(action)) {
+          errors = validator.validate(action)
+          break
+        }
+      }
+    }
+
+    return errors
+  }
+}
+
+module.exports = ActionList

package/src/actions/index.js

@@ -0,0 +1,218 @@
+'use strict'
+
+// Utils
+const {constants} = require('../utils')
+// Properties
+const {actions, validationAction} = constants
+const {
+  create,
+  customTask,
+  edit,
+  transformEntries,
+  deriveLinkedEntries,
+  transformEntriesToType,
+  typeError,
+  apiError,
+  schema,
+  __migrationError,
+  field,
+} = validationAction
+
+const actionCreators = {
+  customTasks: (callsite, opts) => {
+    const {CUSTOM_TASK} = actions
+    return {
+      type: customTask,
+      meta: {
+        callsite: {
+          file: callsite.getFileName(),
+          line: callsite.getLineNumber(),
+        },
+      },
+      payload: {
+        options: opts,
+        action: CUSTOM_TASK,
+      },
+    }
+  },
+  contentType: {
+    create: (callsite, id, opts) => {
+      const {CREATE_CT} = actions
+      return {
+        type: create,
+        meta: {
+          callsite: {
+            file: callsite.getFileName(),
+            line: callsite.getLineNumber(),
+          },
+        },
+        payload: {
+          contentTypeId: id,
+          options: opts,
+          action: CREATE_CT,
+        },
+      }
+    },
+    edit: (callsite, id, opts) => {
+      const {EDIT_CT} = actions
+
+      return {
+        type: edit,
+        meta: {
+          callsite: {
+            file: callsite.getFileName(),
+            line: callsite.getLineNumber(),
+          },
+        },
+        payload: {
+          contentTypeId: id,
+          options: opts,
+          action: EDIT_CT,
+        },
+      }
+    },
+    // delete: () => { },
+    transformEntries: (callsite, id, opts) => {
+      return {
+        type: transformEntries,
+        meta: {
+          callsite: {
+            file: callsite.getFileName(),
+            line: callsite.getLineNumber(),
+          },
+        },
+        payload: {
+          options: opts,
+        },
+      }
+    },
+    deriveLinkedEntries: (callsite, id, opts) => {
+      return {
+        type: deriveLinkedEntries,
+        meta: {
+          callsite: {
+            file: callsite.getFileName(),
+            line: callsite.getLineNumber(),
+          },
+        },
+        payload: {
+          options: opts,
+        },
+      }
+    },
+    transformEntriesToType: (callsite, id, opts) => {
+      return {
+        type: transformEntriesToType,
+        meta: {
+          callsite: {
+            file: callsite.getFileName(),
+            line: callsite.getLineNumber(),
+          },
+        },
+        payload: {
+          options: opts,
+        },
+      }
+    },
+    typeError: (callsite, id, {typeErrors}) => {
+      return {
+        type: typeError,
+        meta: {
+          callsite: {
+            file: callsite.getFileName(),
+            line: callsite.getLineNumber(),
+          },
+        },
+        payload: {typeErrors},
+      }
+    },
+    apiError: (callsite, id, opts) => {
+      return {
+        type: apiError,
+        meta: {
+          callsite: {
+            file: callsite.getFileName(),
+            line: callsite.getLineNumber(),
+          },
+        },
+        payload: {apiError: opts},
+      }
+    },
+    fromFields: (callsite, id, opts) => {
+      return {
+        type: schema,
+        meta: {
+          callsite: {
+            file: callsite.getFileName(),
+            line: callsite.getLineNumber(),
+          },
+        },
+        payload: {fromField: opts.fromField},
+      }
+    },
+    toFields: (callsite, id, opts) => {
+      return {
+        type: schema,
+        meta: {
+          callsite: {
+            file: callsite.getFileName(),
+            line: callsite.getLineNumber(),
+          },
+        },
+        payload: {toField: opts.toField},
+      }
+    },
+    toReferenceFields: (callsite, id, opts) => {
+      return {
+        type: schema,
+        meta: {
+          callsite: {
+            file: callsite.getFileName(),
+            line: callsite.getLineNumber(),
+          },
+        },
+        payload: {toField: opts.toReferenceField},
+      }
+    },
+    deriveFields: (callsite, id, opts) => {
+      return {
+        type: schema,
+        meta: {
+          callsite: {
+            file: callsite.getFileName(),
+            line: callsite.getLineNumber(),
+          },
+        },
+        payload: {deriveField: opts.deriveField},
+      }
+    },
+    migrationError: (callsite, id, opts) => {
+      return {
+        type: __migrationError,
+        meta: {
+          callsite: {
+            file: callsite.getFileName(),
+            line: callsite.getLineNumber(),
+          },
+        },
+        payload: {migrationError: opts},
+      }
+    },
+    field: (callsite, id, opts) => {
+      return {
+        type: field,
+        meta: {
+          callsite: {
+            file: callsite.getFileName(),
+            line: callsite.getLineNumber(),
+          },
+        },
+        payload: {field: opts},
+      }
+    },
+
+  },
+}
+
+exports.actionCreators = actionCreators
+exports.ActionList = require('./action-list')

package/src/commands/cm/migration.js

@@ -0,0 +1,182 @@
+/* eslint-disable no-unused-expressions */
+/* eslint-disable no-warning-comments */
+/* eslint-disable camelcase */
+'use strict'
+
+// Dependencies
+const Listr = require('listr')
+const {resolve, extname} = require('path')
+const {Command, flags} = require('@contentstack/cli-command')
+const {waterfall} = require('async')
+const {Parser} = require('../../modules')
+const {ActionList} = require('../../actions')
+const fs = require('fs')
+const chalk = require('chalk')
+
+const {ApiError, SchemaValidator, MigrationError, FieldValidator} = require('../../validators')
+
+// Utils
+const {map: _map, constants, safePromise, errorHelper} = require('../../utils')
+const {success} = require('../../utils/logger')
+
+// Properties
+const {get, set, getMapInstance, resetMapInstance} = _map
+const {requests: _requests, actionMapper, MANAGEMENT_SDK, MANAGEMENT_TOKEN, AUTH_TOKEN, API_KEY, BRANCH} = constants
+
+class MigrationCommand extends Command {
+  async run() {
+    // TODO: filePath validation required.
+    const migrationCommandFlags = this.parse(MigrationCommand).flags
+    const {filePath, multi, branch} = migrationCommandFlags
+    const authtoken = migrationCommandFlags.authtoken
+    const apiKey = migrationCommandFlags['api-key']
+    const alias = migrationCommandFlags['management-token-alias']
+
+    let stackSDKInstance
+
+    // Reset map instance
+    const mapInstance = getMapInstance()
+    resetMapInstance(mapInstance)
+    if (branch) {
+      set(BRANCH, mapInstance, branch)
+    }
+
+    if (alias) {
+      let managementToken = this.getToken(alias)
+      if (managementToken) {
+        set(MANAGEMENT_TOKEN, mapInstance, managementToken)
+        set(API_KEY, mapInstance, managementToken.apiKey)
+        if (branch) {
+          stackSDKInstance = this.managementAPIClient.stack({management_token: managementToken.token, api_key: managementToken.apiKey, branch_uid: branch})
+        } else {
+          stackSDKInstance = this.managementAPIClient.stack({management_token: managementToken.token, api_key: managementToken.apiKey})
+        }
+      }
+    }
+
+    if (authtoken) {
+      set(AUTH_TOKEN, mapInstance, authtoken)
+      set(API_KEY, mapInstance, apiKey)
+      this.managementAPIClient = {authtoken: this.authToken}
+      if (branch) {
+        stackSDKInstance = this.managementAPIClient.stack({api_key: apiKey, branch_uid: branch})
+      } else {
+        stackSDKInstance = this.managementAPIClient.stack({api_key: apiKey})
+      }
+    }
+
+    set(MANAGEMENT_SDK, mapInstance, stackSDKInstance)
+
+    if (multi) {
+      await this.execMultiFiles(filePath, mapInstance)
+    } else {
+      await this.execSingleFile(filePath, mapInstance)
+    }
+  }
+
+  async execSingleFile(filePath, mapInstance) {
+    // Resolved absolute path
+    const resolvedMigrationPath = resolve(filePath)
+    // User provided migration function
+    const migrationFunc = require(resolvedMigrationPath)
+
+    const parser = new Parser()
+
+    try {
+      const migrationParser = await parser.getMigrationParser(migrationFunc)
+      if (migrationParser.hasErrors) {
+        errorHelper(migrationParser.hasErrors)
+        // When the process is child, send error message to parent
+        if (process.send) process.send({errorOccurred: true})
+        this.exit(1)
+      }
+
+      // Make calls from here
+      const requests = get(_requests, mapInstance)
+      // Fetches tasks array
+      const tasks = this.getTasks(requests)
+
+      const listr = new Listr(tasks)
+
+      await listr.run().catch(error => {
+        this.handleErrors(error)
+        // When the process is child, send error message to parent
+        if (process.send) process.send({errorOccurred: true})
+      })
+      requests.splice(0, requests.length)
+    } catch (error) {
+      // errorHandler(null, null, null, error)
+    }
+  }
+
+  async execMultiFiles(filePath, mapInstance) {
+    // Resolved absolute path
+    const resolvedMigrationPath = resolve(filePath)
+    try {
+      const files = fs.readdirSync(resolvedMigrationPath)
+      for (let index = 0; index < files.length; index++) {
+        const file = files[index]
+        if (extname(file) === '.js') {
+          success(chalk`{white Executing file:} {grey {bold ${file}}}`)
+          // eslint-disable-next-line no-await-in-loop
+          await this.execSingleFile(resolve(filePath, file), mapInstance)
+        }
+      }
+    } catch (error) {
+      error(error)
+    }
+  }
+
+  getTasks(requests) {
+    const _tasks = []
+    const results = []
+
+    for (let i = 0; i < requests.length; i++) {
+      let reqObj = requests[i]
+      const {title, failedTitle, successTitle, tasks} = reqObj
+      const task = {
+        title: title,
+        task: async (ctx, task) => {
+          const [err, result] = await safePromise(waterfall(tasks))
+          if (err) {
+            ctx.error = true
+            task.title = failedTitle
+            throw err
+          }
+          result && results.push(result)
+          task.title = successTitle
+          return result
+        },
+      }
+      _tasks.push(task)
+    }
+    return _tasks
+  }
+
+  handleErrors() {
+    const mapInstance = getMapInstance()
+    const actions = get(actionMapper, mapInstance)
+    const actionList = new ActionList(actions)
+
+    actionList.addValidators(new ApiError())
+    actionList.addValidators(new SchemaValidator())
+    actionList.addValidators(new MigrationError())
+    actionList.addValidators(new FieldValidator())
+
+    const errors = actionList.validate()
+    errorHelper(errors)
+  }
+}
+
+MigrationCommand.description = 'Contentstack migration script.'
+
+MigrationCommand.flags = {
+  'api-key': flags.string({char: 'k', description: 'With this flag add the API key of your stack.', dependsOn: ['authtoken'], exclusive: ['management-token-alias']}),
+  authtoken: flags.boolean({char: 'A', description: 'Use this flag to use the auth token of the current session. After logging in CLI, an auth token is generated for each new session.', dependsOn: ['api-key'], exclusive: ['management-token-alias']}),
+  'management-token-alias': flags.string({char: 'a', description: 'Use this flag to add the management token alias.', exclusive: ['authtoken']}), // Add a better description
+  filePath: flags.string({char: 'n', description: 'Use this flag to provide the path of the file of the migration script provided by the user.'}),
+  branch: flags.string({char: 'B', description: 'Use this flag to add the branch name where you want to perform the migration.'}),
+  multi: flags.boolean({description: 'This flag helps you to migrate multiple content files in a single instance.'}), // Add a better description
+}
+
+module.exports = MigrationCommand

package/src/config/api-config.js

@@ -0,0 +1,19 @@
+/* eslint-disable camelcase */
+'use strict'
+
+const {CONTENTSTACK_API_KEY, CONTENTSTACK_AUTHTOKEN} = process.env
+const {version} = require('../../package.json')
+module.exports = {
+  hostname: 'dev9-api.contentstack.com',
+  // hostname: 'stag-app.contentstack.com',
+  version: '/v3',
+  method: 'GET', // Default Http method
+  headers: {
+    'Content-Type': 'application/json',
+    'Content-Length': null,
+    'X-User-Agent': `@contentstack-migration/v${version}`,
+    authtoken: CONTENTSTACK_AUTHTOKEN,
+    api_key: CONTENTSTACK_API_KEY,
+    // management_token: CONTENTSTACK_MANAGEMENT_TOKEN
+  },
+}

package/src/config/default-options.js

@@ -0,0 +1,7 @@
+/* eslint-disable camelcase */
+'use strict'
+
+module.exports = {
+  is_page: true,
+  singleton: false,
+}

package/src/config/index.js

@@ -0,0 +1,7 @@
+'use strict'
+
+module.exports = {
+  apiConfig: require('./api-config'),
+  defaultOptions: require('./default-options'),
+  masterLocale: require('./master-locale'),
+}

package/src/config/master-locale.js

@@ -0,0 +1,10 @@
+/* eslint-disable camelcase */
+'use strict'
+
+module.exports = {
+  master_locale: {
+    // master locale of the stack
+    name: 'English - United States',
+    code: 'en-us',
+  },
+}