adapt-migrations 1.0.0

package/.eslintignore

@@ -0,0 +1 @@
+node_modules

package/.eslintrc.json

@@ -0,0 +1,14 @@
+{
+  "env": {
+      "browser": false,
+      "node": true,
+      "commonjs": false,
+      "es2020": true
+  },
+  "extends": [
+      "standard"
+  ],
+  "parserOptions": {
+      "ecmaVersion": 2022
+  }
+}

package/README.md

@@ -0,0 +1,43 @@
+# adapt-migrations
+
+### Todos
+https://github.com/cgkineo/adapt-migrations/issues/1
+
+### Commands API
+https://github.com/cgkineo/adapt-migrations/blob/master/api/commands.js
+* `load({ cwd, cachePath, scripts })` - loads all migration tasks
+* `capture({ cwd, content, fromPlugins })` - captures current plugins and content
+* `migrate({ cwd, toPlugins })` - migrates content from capture to new plugins
+* `test({ cwd })` - tests the migrations with dummy content
+
+### Migration script API
+Functions:
+* `describe(description, describeFunction)` Describe a migration
+* `whereContent(description, contentFilterFunction)` Limit when the migration runs, return true/false/throw Error
+* `whereFromPlugin(description, fromPluginFilterFunction)` Limit when the migration runs, return true/false/throw Error
+* `whereToPlugin(description, toPluginFilterFunction)` Limit when the migration runs, return true/false/throw Error
+* `mutateContent(contentFunction)` Change content, return true/false/throw Error
+* `checkContent(contentFunction)` Check content, return true/false/throw Error
+* `throwError(description)` Throw an error
+* `testSuccessWhere({ fromPlugins, toPlugins, content })` Supply some tests content which should end in success
+* `testStopWhere({ fromPlugins, toPlugins, content })` Supply some tests content which should end prematurely
+* `testErrorWhere({ fromPlugins, toPlugins, content })` Supply some tests content which will trigger an error
+
+Arguments:
+* `describeFunction = () => {}` Function body has a collection of migration script functions
+* `contentFilterFunction = content => {}` Function body should return true/false/throw Error
+* `fromPluginFilterFunction = fromPlugins => {}` Function body should return true/false/throw Error
+* `toPluginFilterFunction = toPlugins => {}` Function body should return true/false/throw Error
+* `contentFunction = content => { }` Function body should mutate or check the content, returning true/false/throw Error
+* `fromPlugins = [{ name: 'quickNav , version: '1.0.0' }]` Test data describing the original plugins
+* `toPlugins = [{ name: 'pageNav , version: '1.0.0' }]` Test data describing the destination plugins
+* `content = [{ _id: 'c-05, ... }]` Test content for the course content
+
+### Grunt Commands
+```sh
+grunt migration:capture # captures current plugins and content
+# do plugin/fw updates
+grunt migration:migrate # migrates content from capture to new plugins
+grunt migration:test # tests the migrations with dummy content
+grunt migration:test --file=adapt-contrib-text/migrations/text.js # tests the migrations with dummy content
+```

package/api/commands.js

@@ -0,0 +1,33 @@
+import Task from '../lib/Task.js'
+
+export async function load ({ cwd = process.cwd(), scripts = [], cachePath, logger } = {}) {
+  return Task.load({
+    cwd,
+    scripts,
+    cachePath,
+    logger
+  })
+}
+
+export async function capture ({ content, fromPlugins, logger }) {
+  return {
+    content,
+    fromPlugins,
+    logger
+  }
+};
+
+export async function migrate ({ cwd = process.cwd(), journal, logger }) {
+  return Task.runApplicable({
+    cwd,
+    journal,
+    logger
+  })
+}
+
+export async function test ({ cwd = process.cwd(), logger } = {}) {
+  return Task.runTests({
+    cwd,
+    logger
+  })
+}

package/api/data.js

@@ -0,0 +1,26 @@
+import { deferOrRunWrap, successStopOrErrorWrap } from '../lib/lifecycle.js'
+
+export function mutateContent (description, callback) {
+  return deferOrRunWrap(function (context) {
+    return successStopOrErrorWrap('mutateContent', description, async () => {
+      return callback(context.content)
+    })
+  }, { description, type: 'action' })
+};
+
+export function checkContent (description, callback) {
+  return deferOrRunWrap(function (context) {
+    return successStopOrErrorWrap('checkContent', description, async () => {
+      context.journal.freeze()
+      let result
+      try {
+        result = await callback(context.content)
+      } catch (err) {
+        context.journal.unfreeze()
+        throw err
+      }
+      context.journal.unfreeze()
+      return result
+    })
+  }, { description, type: 'action' })
+};

package/api/describe.js

@@ -0,0 +1,12 @@
+import Task from '../lib/Task.js'
+import Logger from './../lib/Logger.js'
+
+export function describe (description, load) {
+  const logger = Logger.getInstance();
+  logger.info(`Describe -- ${description} -- Registered`)
+  if (Task.current) {
+    logger.error(`Describe -- Cannot nest describe statements -- ${description}`)
+  }
+  // eslint-disable-next-line no-new
+  new Task({ description, load })
+};

package/api/errors.js

@@ -0,0 +1,28 @@
+import { deferOrRunWrap, successStopOrErrorWrap } from '../lib/lifecycle.js'
+import Logger from './../lib/Logger.js'
+
+const logger = Logger.getInstance();
+
+export function throwError (description) {
+  let error = description
+  if (!(description instanceof Error)) {
+    logger.error(`Errors -- ${description}`)
+    error = new Error(description)
+  } else {
+    description = description.message
+  }
+  return deferOrRunWrap(function (context) {
+    return successStopOrErrorWrap('throwError', description, async () => {
+      throw error
+    })
+  }, { type: 'action' })
+}
+
+export function ifErroredAsk (config) {
+  return deferOrRunWrap(function (context) {
+    return successStopOrErrorWrap('isErroredAsk', config.question, async () => {
+      if (!context.hasErrored) return true
+      // Ask a question
+    })
+  }, { type: 'error' })
+};

package/api/plugins.js

@@ -0,0 +1,44 @@
+import { deferOrRunWrap, successStopOrErrorWrap } from '../lib/lifecycle.js'
+
+export function removePlugin (description, config) {
+  return deferOrRunWrap(function (context) {
+    return successStopOrErrorWrap('removePlugin', description, async () => {
+      if (!description || !config) throw new Error('removePlugin - incorrectly configured')
+
+      context.fromPlugins = context.fromPlugins.filter(plugin => plugin.name !== config.name)
+
+      return true
+    })
+  }, { description, type: 'action' })
+};
+
+export function addPlugin (description, config) {
+  return deferOrRunWrap(function (context) {
+    return successStopOrErrorWrap('addPlugin', description, async () => {
+      if (!description || !config) throw new Error('addPlugin - incorrectly configured')
+
+      const newPlugin = context.toPlugins.find(plugin => (plugin.name === config.name))
+      if (!newPlugin) throw new Error(`addPlugin - ${config.name} not found`)
+      context.fromPlugins.push(newPlugin)
+
+      return true
+    })
+  }, { description, type: 'action' })
+};
+
+export function updatePlugin (description, config) {
+  return deferOrRunWrap(function (context) {
+    return successStopOrErrorWrap('updatePlugin', description, async () => {
+      if (!description || !config) throw new Error('updatePlugin - incorrectly configured')
+
+      context.fromPlugins.forEach(plugin => {
+        if (plugin.name !== config.name) return
+        plugin.version = config.version
+        if (!config.framework) return
+        plugin.framework = config.framework
+      })
+
+      return true
+    })
+  }, { description, type: 'action' })
+};

package/api/tests.js

@@ -0,0 +1,57 @@
+import TaskTest from '../lib/TaskTest.js'
+import { deferOrRunWrap } from '../lib/lifecycle.js'
+import Logger from '../lib/Logger.js'
+
+const logger = Logger.getInstance();
+
+export function testSuccessWhere (description, {
+  fromPlugins,
+  toPlugins,
+  content
+}) {
+  return deferOrRunWrap(() => {
+    logger.debug(`Tests -- testSuccessWhere ${description}`)
+    return new TaskTest({
+      description,
+      shouldRun: true,
+      fromPlugins,
+      toPlugins,
+      content
+    })
+  }, { type: 'test' })
+};
+
+export function testStopWhere (description, {
+  fromPlugins,
+  toPlugins,
+  content
+}) {
+  return deferOrRunWrap(() => {
+    logger.debug(`Tests -- testStopWhere ${description}`)
+    return new TaskTest({
+      description,
+      shouldStop: true,
+      shouldRun: false,
+      fromPlugins,
+      toPlugins,
+      content
+    })
+  }, { type: 'test' })
+};
+
+export function testErrorWhere (description, {
+  fromPlugins,
+  toPlugins,
+  content
+}) {
+  return deferOrRunWrap(() => {
+    logger.debug(`Tests -- testErrorWhere ${description}`)
+    return new TaskTest({
+      description,
+      shouldError: true,
+      fromPlugins,
+      toPlugins,
+      content
+    })
+  }, { type: 'test' })
+};

package/api/where.js

@@ -0,0 +1,46 @@
+import semver from 'semver'
+import { deferOrRunWrap, successStopOrErrorWrap } from '../lib/lifecycle.js'
+
+export function whereContent (description, callback) {
+  return deferOrRunWrap(({ content }) => {
+    return successStopOrErrorWrap('whereContent', description, async () => {
+      return callback(content)
+    })
+  }, { description, type: 'where' })
+};
+
+export function whereFromPlugin (description, callbackOrConfig) {
+  return deferOrRunWrap(({ fromPlugins }) => {
+    return successStopOrErrorWrap('whereFromPlugin', description, async () => {
+      const isCallback = (typeof callbackOrConfig === 'function')
+      if (isCallback) {
+        const callback = callbackOrConfig
+        return callback(fromPlugins)
+      }
+      const config = callbackOrConfig
+      return fromPlugins.some(plugin => {
+        if (config.name && plugin.name !== config.name) return false
+        if (config.version && !semver.satisfies(plugin.version, config.version)) return false
+        return true
+      })
+    })
+  }, { description, type: 'where' })
+};
+
+export function whereToPlugin (description, callbackOrConfig) {
+  return deferOrRunWrap(({ toPlugins }) => {
+    return successStopOrErrorWrap('whereToPlugin', description, async () => {
+      const isCallback = (typeof callbackOrConfig === 'function')
+      if (isCallback) {
+        const callback = callbackOrConfig
+        return callback(toPlugins)
+      }
+      const config = callbackOrConfig
+      return toPlugins.some(plugin => {
+        if (config.name && plugin.name !== config.name) return false
+        if (config.version && !semver.satisfies(plugin.version, config.version)) return false
+        return true
+      })
+    })
+  }, { description, type: 'where' })
+};

package/examples/migrations.js

@@ -0,0 +1,159 @@
+module.exports = function(grunt) {
+
+  const Helpers = require('../helpers')(grunt);
+  const globs = require('globs');
+  const path = require('path');
+  const fs = require('fs-extra');
+  const _ = require('underscore');
+
+  function unix(path) {
+    return path.replace(/\\/g, '/');
+  }
+
+  function dressPathIndex(fileItem) {
+    return {
+      ...fileItem.item,
+      __index__: fileItem.index,
+      __path__: unix(fileItem.file.path)
+    };
+  }
+
+  function undressPathIndex(object) {
+    const clone = { ...object };
+    delete clone.__index__;
+    delete clone.__path__;
+    return clone;
+  }
+
+  grunt.registerTask('migration', 'Migrate from on verion to another', function(mode) {
+    const next = this.async();
+    const buildConfig = Helpers.generateConfigData();
+    const fileNameIncludes = grunt.option('file');
+
+    (async function() {
+      const migrations = await import('adapt-migrations');
+      const logger = migrations.Logger.getInstance();
+      const cwd = process.cwd();
+      const outputPath = path.join(cwd, './migrations/');
+      const cache = new migrations.CacheManager();
+      const cachePath = await cache.getCachePath({
+        outputPath: buildConfig.outputdir,
+        tempPath: outputPath
+      });
+
+      const framework = Helpers.getFramework();
+      logger.debug(`Using ${framework.useOutputData ? framework.outputPath : framework.sourcePath} folder for course data...`);
+
+      const plugins = framework.getPlugins().getAllPackageJSONFileItems().map(fileItem => fileItem.item);
+      const migrationScripts = Array.from(await new Promise(resolve => {
+        globs([
+          '*/*/migrations/**/*.js',
+          'core/migrations/**/*.js'
+        ], { cwd: path.join(cwd, './src/'), absolute: true }, (err, files) => resolve(err ? null : files));
+      })).filter(filePath => {
+        if (!fileNameIncludes) return true;
+        return filePath.includes(fileNameIncludes);
+      });
+
+      await migrations.load({
+        cachePath,
+        scripts: migrationScripts,
+        logger
+      });
+
+      if (mode === 'capture') {
+
+        if (!fs.existsSync(outputPath)) fs.mkdirSync(outputPath);
+        const languages = framework.getData().languages.map((language) => language.name);
+        const languageFile = path.join(outputPath, 'captureLanguages.json');
+        fs.writeJSONSync(languageFile, languages);
+        languages.forEach(async (language, index) => {
+          logger.debug(`Migration -- Capture ${language}`)
+          const data = framework.getData();
+          // get all items from config.json file and all language files, append __index__ and __path__ to each item
+          const content = [
+            ...data.configFile.fileItems,
+            ...data.languages[index].getAllFileItems()
+          ].map(dressPathIndex);
+          const captured = await migrations.capture({ content, fromPlugins: plugins, logger });
+          const outputFile = path.join(outputPath, `capture_${language}.json`);
+          fs.writeJSONSync(outputFile, captured);
+        });
+
+        logger.output(outputPath, 'capture');
+        return next();
+      }
+
+      if (mode === 'migrate') {
+        try {
+          const languagesFile = path.join(outputPath, 'captureLanguages.json');
+          const languages = fs.readJSONSync(languagesFile);
+
+          for (const language of languages) {
+            logger.debug(`Migration -- Migrate ${language}`)
+            const Journal = migrations.Journal;
+            if (!fs.existsSync(outputPath)) fs.mkdirSync(outputPath);
+            const outputFile = path.join(outputPath, `capture_${language}.json`);
+            const { content, fromPlugins } = fs.readJSONSync(outputFile);
+            const originalFromPlugins = JSON.parse(JSON.stringify(fromPlugins));
+            const journal = new Journal({
+              logger,
+              data: {
+                content,
+                fromPlugins,
+                originalFromPlugins,
+                toPlugins: plugins,
+              },
+              supplementEntry: (entry, data) => {
+                entry._id = data[entry.keys[0]][entry.keys[1]]?._id ?? '';
+                entry._type = data[entry.keys[0]][entry.keys[1]]?._type ?? '';
+                if (entry._type && data[entry.keys[0]][entry.keys[1]]?.[`_${entry._type}`]) {
+                  entry[`_${entry._type}`] = data[entry.keys[0]][entry.keys[1]]?.[`_${entry._type}`] ?? '';
+                }
+                return entry;
+              }
+            });
+            await migrations.migrate({ journal, logger });
+
+            // Todo - {
+              // display changes success/failure and request user confirmation before completing
+              // move saving of content outside of the language loop
+              // only 1 confirmation per course rather than 1 per language
+              // output journal entries for revert
+            // }
+
+            // group all content items by path
+            const outputFilePathItems = _.groupBy(content, '__path__');
+            // sort items inside each path
+            Object.values(outputFilePathItems).forEach(outputFile => outputFile.sort((a, b) => a.__index__ - b.__index__));
+            // get paths
+            const outputFilePaths = Object.keys(outputFilePathItems);
+
+            outputFilePaths.forEach(outputPath => {
+              const outputItems = outputFilePathItems[outputPath];
+              if (!outputItems?.length) return;
+              const isSingleObject = (outputItems.length === 1 && outputItems[0].__index__ === null);
+              const stripped = isSingleObject
+                ? undressPathIndex(outputItems[0]) // config.json, course.json
+                : outputItems.map(undressPathIndex); // contentObjects.json, articles.json, blocks.json, components.json
+              // console.log(journal.entries)
+              fs.writeJSONSync(outputPath, stripped, { replacer: null, spaces: 2 });
+            });
+          }
+        } catch (error) {
+          logger.error(error.stack);
+        }
+        logger.output(outputPath, 'migrate');
+        return next();
+      }
+
+      if (mode === 'test') {
+        await migrations.test();
+        return next();
+      }
+
+      return next();
+    })();
+  });
+
+};

package/examples/script.js

@@ -0,0 +1,78 @@
+// Example migration script
+
+import { describe, whereContent, whereFromPlugin, whereToPlugin, mutateContent, checkContent, throwError, ifErroredAsk, testSuccessWhere, testErrorWhere, testStopWhere } from 'adapt-migrations';
+
+describe('update plugin from v6.1.4 to v6.1.5 and add "Ollie" to display title', async () => {
+  whereFromPlugin('text v6.1.4', { name: 'adapt-contrib-text', version: '<=6.1.4' });
+  whereContent('has configured displayTitles', async content =>
+    content.some(({ displayTitle }) => displayTitle)
+  );
+  mutateContent('change displayTitle', async content => {
+    const quicknavs = content.filter(({ displayTitle }) => displayTitle);
+    quicknavs.forEach(item => (item.displayTitle += ' ollie'));
+    return true;
+  });
+  checkContent('check everything is ok', async content => {
+    const isInvalid = content.some(({ displayTitle }) => displayTitle && !String(displayTitle).endsWith(' ollie'));
+    if (isInvalid) throw new Error('found displayTitle without ollie at the end');
+    return true;
+  });
+  updatePlugin('update text plugin', {name: 'adapt-contrib-text', version: '6.1.5', framework: '>=5.19.4'})
+});
+
+describe('quicknav to pagenav', async () => {
+  addPlugin('add pagenav plugin', { name: 'pagenav', version: '1.0.0'});
+  whereFromPlugin('quicknav v1.0.0', { name: 'quicknav', version: '1.0.0' });
+  whereToPlugin('pagenav v1.0.0', { name: 'pagenav', version: '1.0.0' });
+  removePlugin('remove quicknav plugin', {name: 'quicknav'})
+  whereContent('has configured quicknavs', async content =>
+    content.some(({ _component }) => _component === 'quicknav')
+  );
+  mutateContent('change _component name', async content => {
+    const quicknavs = content.filter(({ _component }) => _component === 'quicknav');
+    quicknavs.forEach(item => (item._component = 'pagenav'));
+    return true;
+  });
+  checkContent('check everything is ok', async content => {
+    const isInvalid = content.some(({ isInvalid }) => isInvalid);
+    if (isInvalid) throw new Error('found invalid content attribute');
+    return true;
+  });
+  // TODO: handle errors with question, allow to run without ui
+  // ifErroredAsk({ question: 'Skip error', yes: 'Yes', no: 'No', defaultSkipError: true });
+  // TODO: modify stack traces one errors to refer to the original migration script rather than the cached one,keep map of cached files to original files
+  testSuccessWhere('Valid plugins and content', {
+    fromPlugins: [{ name: 'quicknav', version: '1.0.0' }],
+    toPlugins: [{ name: 'pagenav', version: '1.0.0' }],
+    content: [{ _component: 'quicknav' }]
+  });
+  testStopWhere('Invalid content', {
+    fromPlugins: [{ name: 'quicknav', version: '1.0.0' }],
+    toPlugins: [{ name: 'pagenav', version: '1.0.0' }],
+    content: [{ _component: 'quicknav1' }]
+  });
+  testStopWhere('Invalid origin plugins', {
+    fromPlugins: [{ name: 'quicknav', version: '0.1.0' }],
+    toPlugins: [{ name: 'pagenav', version: '0.1.0' }],
+    content: [{ _component: 'quicknav' }]
+  });
+  testStopWhere('Invalid destination plugins', {
+    content: [{ _component: 'quicknav' }],
+    fromPlugins: [{ name: 'quicknav', version: '1.0.0' }],
+    toPlugins: [{ name: 'pagenav', version: '0.1.0' }]
+  });
+  testErrorWhere('Has invalid configuration', {
+    fromPlugins: [{ name: 'quicknav', version: '1.0.0' }],
+    toPlugins: [{ name: 'pagenav', version: '1.0.0' }],
+    content: [{ _component: 'quicknav', isInvalid: true }]
+  });
+});
+
+describe('where quicknav is weirdly configured', async () => {
+  checkContent('check everything is ok', async content => {
+    const isInvalid = content.some(({ isInvalid }) => isInvalid);
+    if (isInvalid) throw new Error('Something went wrong');
+    return true;
+  });
+  throwError('this is an error');
+});

package/index.js

@@ -0,0 +1,62 @@
+import {
+  load,
+  capture,
+  migrate,
+  test
+} from './api/commands.js'
+import {
+  describe
+} from './api/describe.js'
+import {
+  whereContent,
+  whereFromPlugin,
+  whereToPlugin
+} from './api/where.js'
+import {
+  checkContent,
+  mutateContent
+} from './api/data.js'
+import {
+  ifErroredAsk,
+  throwError
+} from './api/errors.js'
+import {
+  testErrorWhere,
+  testStopWhere,
+  testSuccessWhere
+} from './api/tests.js'
+import {
+  updatePlugin,
+  removePlugin,
+  addPlugin
+} from './api/plugins.js'
+import Journal from './lib/Journal.js'
+import CacheManager from './lib/CacheManager.js'
+import Logger from './lib/Logger.js'
+
+export {
+  // commands
+  load,
+  capture,
+  migrate,
+  test,
+  // migration script api
+  describe,
+  whereContent,
+  whereFromPlugin,
+  whereToPlugin,
+  checkContent,
+  mutateContent,
+  ifErroredAsk,
+  throwError,
+  testErrorWhere,
+  testStopWhere,
+  testSuccessWhere,
+  updatePlugin,
+  removePlugin,
+  addPlugin,
+  // environment objects
+  Journal,
+  CacheManager,
+  Logger
+}