adapt-authoring-adaptframework 0.0.1

package/lib/AdaptFrameworkBuild.js

@@ -0,0 +1,517 @@
+import _ from 'lodash'
+import { App, Hook } from 'adapt-authoring-core'
+import { createWriteStream } from 'fs'
+import AdaptCli from 'adapt-cli'
+import AdaptFrameworkUtils from './AdaptFrameworkUtils.js'
+import fs from 'fs-extra'
+import path from 'upath'
+import semver from 'semver'
+import zipper from 'zipper'
+
+/**
+ * Encapsulates all behaviour needed to build a single Adapt course instance
+ * @memberof adaptframework
+ */
+class AdaptFrameworkBuild {
+  /**
+   * Imports a course zip to the database
+   * @param {AdaptFrameworkBuildOptions} options
+   * @return {Promise} Resolves to this AdaptFrameworkBuild instance
+   */
+  static async run (options) {
+    const instance = new AdaptFrameworkBuild(options)
+    await instance.build()
+    return instance
+  }
+
+  /**
+   * Returns a timestring to be used for an adaptbuild expiry
+   * @return {String}
+   */
+  static async getBuildExpiry () {
+    const framework = await App.instance.waitForModule('adaptframework')
+    return new Date(Date.now() + framework.getConfig('buildLifespan')).toISOString()
+  }
+
+  /**
+   * Options to be passed to AdaptFrameworkBuild
+   * @typedef {Object} AdaptFrameworkBuildOptions
+   * @property {String} action The type of build to execute
+   * @property {String} courseId The course  to build
+   * @property {String} userId The user executing the build
+   * @property {String} expiresAt When the build expires
+   *
+   * @constructor
+   * @param {AdaptFrameworkBuildOptions} options
+   */
+  constructor ({ action, courseId, userId, expiresAt }) {
+    /**
+     * The MongoDB collection name
+     * @type {String}
+     */
+    this.collectionName = 'adaptbuilds'
+    /**
+     * The build action being performed
+     * @type {String}
+     */
+    this.action = action
+    /**
+     * Shorthand for checking if this build is a preview
+     * @type {Boolean}
+     */
+    this.isPreview = action === 'preview'
+    /**
+     * Shorthand for checking if this build is a publish
+     * @type {Boolean}
+     */
+    this.isPublish = action === 'publish'
+    /**
+     * Shorthand for checking if this build is an export
+     * @type {Boolean}
+     */
+    this.isExport = action === 'export'
+    /**
+     * The _id of the course being build
+     * @type {String}
+     */
+    this.courseId = courseId
+    /**
+     * All JSON data describing this course
+     * @type {Object}
+     */
+    this.expiresAt = expiresAt
+    /**
+     * All JSON data describing this course
+     * @type {Object}
+     */
+    this.courseData = {}
+    /**
+     * All metadata related to assets used in this course
+     * @type {Object}
+     */
+    this.assetData = {}
+    /**
+     * Metadata describing this build attempt
+     * @type {Object}
+     */
+    this.buildData = {}
+    /**
+     * A map of _ids for use with 'friendly' IDs
+     * @type {Object}
+     */
+    this.idMap = {}
+    /**
+     * _id of the user initiating the course build
+     * @type {String}
+     */
+    this.userId = userId
+    /**
+     * The build output directory
+     * @type {String}
+     */
+    this.dir = ''
+    /**
+     * The course build directory
+     * @type {String}
+     */
+    this.buildDir = ''
+    /**
+     * The course content directory
+     * @type {String}
+     */
+    this.courseDir = ''
+    /**
+     * The final location of the build
+     * @type {String}
+     */
+    this.location = ''
+    /**
+     * List of plugins used in this course
+     * @type {Array<Object>}
+     */
+    this.enabledPlugins = []
+    /**
+     * List of plugins NOT used in this course
+     * @type {Array<Object>}
+     */
+    this.disabledPlugins = []
+    /**
+     * Invoked prior to a course being built.
+     * @type {Hook}
+     */
+    this.preBuildHook = new Hook({ mutable: true })
+    /**
+      * Invoked after a course has been built.
+      * @type {Hook}
+      */
+    this.postBuildHook = new Hook({ mutable: true })
+  }
+
+  /**
+   * Makes sure the directory exists
+   * @param {string} dir
+   */
+  async ensureDir (dir) {
+    try {
+      await fs.mkdir(dir, { recursive: true })
+    } catch (e) {
+      if (e.code !== 'EEXIST') throw e
+    }
+  }
+
+  /**
+   * Runs the Adapt framework build tools to generate a course build
+   * @return {Promise} Resolves with the output directory
+   */
+  async build () {
+    await this.removeOldBuilds()
+
+    const framework = await App.instance.waitForModule('adaptframework')
+    if (!this.expiresAt) {
+      this.expiresAt = await AdaptFrameworkBuild.getBuildExpiry()
+    }
+    this.dir = path.resolve(framework.getConfig('buildDir'), new Date().getTime().toString())
+    this.buildDir = path.join(this.dir, 'build')
+    this.courseDir = path.join(this.buildDir, 'course')
+
+    const cacheDir = path.join(framework.getConfig('buildDir'), 'cache')
+
+    await this.ensureDir(this.dir)
+    await this.ensureDir(this.buildDir)
+    await this.ensureDir(cacheDir)
+
+    AdaptFrameworkUtils.logDir('dir', this.dir)
+    AdaptFrameworkUtils.logDir('buildDir', this.buildDir)
+    AdaptFrameworkUtils.logDir('cacheDir', this.cacheDir)
+
+    await this.loadCourseData()
+
+    await Promise.all([
+      this.copyAssets(),
+      this.copySource()
+    ])
+    await this.preBuildHook.invoke(this)
+    await framework.preBuildHook.invoke(this)
+
+    await this.writeContentJson()
+    
+    AdaptFrameworkUtils.logDir('courseDir', this.courseDir)
+
+    if (!this.isExport) {
+      try {
+        AdaptFrameworkUtils.logMemory()
+        await AdaptCli.buildCourse({
+          cwd: this.dir,
+          sourceMaps: !this.isPublish,
+          outputDir: this.buildDir,
+          cachePath: path.resolve(cacheDir, this.courseId),
+          logger: { log: (...args) => App.instance.logger.log('debug', 'adapt-cli', ...args) }
+        })
+        AdaptFrameworkUtils.logMemory()
+      } catch (e) {
+        AdaptFrameworkUtils.logMemory()
+        throw App.instance.errors.FW_CLI_BUILD_FAILED
+          .setData(e)
+      }
+    }
+    this.isPreview ? await this.createPreview() : await this.createZip()
+
+    await this.postBuildHook.invoke(this)
+    await framework.postBuildHook.invoke(this)
+
+    this.buildData = await this.recordBuildAttempt()
+  }
+
+  /**
+   * Collects and caches all the DB data for the course being built
+   * @return {Promise}
+   */
+  async loadCourseData () {
+    const content = await App.instance.waitForModule('content')
+    const [course] = await content.find({ _id: this.courseId, _type: 'course' })
+    if (!course) {
+      throw App.instance.errors.NOT_FOUND.setData({ type: 'course', id: this.courseId })
+    }
+    const langDir = path.join(this.courseDir, 'en')
+    this.courseData = {
+      course: { dir: langDir, fileName: 'course.json', data: undefined },
+      config: { dir: this.courseDir, fileName: 'config.json', data: undefined },
+      contentObject: { dir: langDir, fileName: 'contentObjects.json', data: [] },
+      article: { dir: langDir, fileName: 'articles.json', data: [] },
+      block: { dir: langDir, fileName: 'blocks.json', data: [] },
+      component: { dir: langDir, fileName: 'components.json', data: [] }
+    }
+    await this.loadAssetData()
+    const contentItems = [course, ...await content.find({ _courseId: course._id })]
+    this.createIdMap(contentItems)
+    this.sortContentItems(contentItems)
+    await this.cachePluginData()
+    await this.transformContentItems(contentItems)
+  }
+
+  /**
+   * Processes and caches the course's assets
+   * @return {Promise}
+   */
+  async loadAssetData () {
+    const [assets, courseassets, mongodb, tags] = await App.instance.waitForModule('assets', 'courseassets', 'mongodb', 'tags')
+
+    const caRecs = await courseassets.find({ courseId: this.courseId })
+    const uniqueAssetIds = new Set(caRecs.map(c => mongodb.ObjectId.parse(c.assetId)))
+    const usedAssets = await assets.find({ _id: { $in: [...uniqueAssetIds] } })
+
+    const usedTagIds = new Set(usedAssets.reduce((m, a) => [...m, ...(a.tags ?? [])], []))
+    const usedTags = await tags.find({ _id: { $in: [...usedTagIds] } })
+    const tagTitleLookup = t => usedTags.find(u => u._id.toString() === t.toString()).title
+
+    const idMap = {}
+    const assetDocs = []
+    const courseDir = this.courseData.course.dir
+
+    await Promise.all(usedAssets.map(async a => {
+      assetDocs.push({ ...a, tags: a?.tags?.map(tagTitleLookup) })
+      if (!idMap[a._id]) idMap[a._id] = a.url ? a.url : path.join(courseDir, 'assets', a.path)
+    }))
+    this.assetData = { dir: courseDir, fileName: 'assets.json', idMap, data: assetDocs }
+  }
+
+  /**
+   * Caches lists of which plugins are/aren't being used in this course
+   * @return {Promise}
+   */
+  async cachePluginData () {
+    const all = (await (await App.instance.waitForModule('contentplugin')).find({}))
+      .reduce((m, p) => Object.assign(m, { [p.name]: p }), {})
+
+    const _cachePluginDeps = (p, memo = {}) => {
+      Object.entries(p?.pluginDependencies ?? {}).forEach(([name, version]) => {
+        const p = memo[name] ?? all[name]
+        const e = !p
+          ? App.instance.errors.FW_MISSING_PLUGIN_DEP.setData({ name })
+          : !semver.satisfies(p.version, version) ? App.instance.errors.FW_INCOMPAT_PLUGIN_DEP.setData({ name, version }) : undefined
+        if (e) {
+          throw e.setData({ name, version })
+        }
+        if (!memo[name]) {
+          _cachePluginDeps(p, memo)
+          memo[name] = p
+        }
+      })
+      return memo
+    }
+    const enabled = (this.courseData.config.data._enabledPlugins || [])
+      .reduce((plugins, name) => {
+        const p = all[name]
+        return Object.assign(plugins, { [name]: p, ..._cachePluginDeps(p) })
+      }, {})
+
+    Object.entries(all).forEach(([name, p]) => (enabled[name] ? this.enabledPlugins : this.disabledPlugins).push(p))
+  }
+
+  /**
+   * Stores a map of friendlyId values to ObjectId _ids
+   */
+  createIdMap (items) {
+    items.forEach(i => {
+      this.idMap[i._id] = i._friendlyId
+    })
+  }
+
+  /**
+   * Sorts the course data into the types needed for each Adapt JSON file. Works by memoising items into an object using the relative sort order as a key used for sorting.
+   * @param {Array<Object>} items The list of content objects
+   */
+  sortContentItems (items) {
+    const getSortOrderStr = co => (co._type === 'course' ? '1' : co._sortOrder.toString()).padStart(4, '0') // note we pad to allow 9999 children
+    const coMap = items.reduce((m, item) => Object.assign(m, { [item._id]: item }), {}) // object mapping items to their _id for easy lookup
+    const sorted = {}
+    items.forEach(i => {
+      const type = i._type === 'page' || i._type === 'menu' ? 'contentObject' : i._type
+      if (type === 'course' || type === 'config') {
+        this.courseData[type].data = i
+        return // don't sort the course or config items
+      }
+      if (!sorted[type]) sorted[type] = {}
+      // recursively calculate a sort order which is relative to the entire course for comparison
+      let sortOrder = ''
+      for (let item = i; item; sortOrder = getSortOrderStr(item) + sortOrder, item = coMap[item._parentId]);
+      sorted[type][sortOrder.padEnd(64, '0')] = i // pad the final string for comparison purposes
+    }) // finally populate this.courseData with the sorted items
+    Object.entries(sorted).forEach(([type, data]) => {
+      this.courseData[type].data = Object.keys(data).sort().map(key => data[key])
+    })
+  }
+
+  /**
+   * Transforms content items into a format recognised by the Adapt framework
+   */
+  async transformContentItems (items) {
+    items.forEach(i => {
+      // slot any _friendlyIds into the _id field
+      ['_courseId', '_parentId'].forEach(k => {
+        i[k] = this.idMap[i[k]] || i[k]
+      })
+      if (i._friendlyId) {
+        i._id = i._friendlyId
+      }
+      // replace asset _ids with correct paths
+      const idMapEntries = Object.entries(this.assetData.idMap)
+      const itemString = idMapEntries.reduce((s, [_id, assetPath]) => {
+        const relPath = assetPath.replace(this.courseDir, 'course')
+        return s.replace(new RegExp(_id, 'g'), relPath)
+      }, JSON.stringify(i))
+      Object.assign(i, JSON.parse(itemString))
+      // insert expected _component values
+      if (i._component) {
+        i._component = this.enabledPlugins.find(p => p.name === i._component).targetAttribute.slice(1)
+      }
+    })
+    // move globals to a nested _extensions object as expected by the framework
+    this.enabledPlugins.forEach(({ targetAttribute, type }) => {
+      let key = `_${type}`
+      if (type === 'component' || type === 'extension') key += 's'
+      try {
+        _.merge(this.courseData.course.data._globals, {
+          [key]: { [targetAttribute]: this.courseData.course.data._globals[targetAttribute] }
+        })
+        delete this.courseData.course.data._globals[targetAttribute]
+      } catch (e) {}
+    })
+    // map course tag values (_id -> title)
+    const tags = await App.instance.waitForModule('tags')
+    const course = this.courseData.course.data
+    if (course?.tags?.length) {
+      course.tags = (await tags.find({ $or: course.tags.map(_id => Object.create({ _id })) }))
+        .map(t => t.title)
+    }
+  }
+
+  /**
+   * Copies the source code needed for this course. Plugin copy works as a whitelist any unknown plugin files aren't included erroneously.
+   * @return {Promise}
+   */
+  async copySource () {
+    const { path: fwPath } = await App.instance.waitForModule('adaptframework')
+    const BLACKLIST = ['.git', '.DS_Store', 'thumbs.db', 'node_modules', 'course']
+    const ENABLED_PLUGINS = this.enabledPlugins.map(p => p.name)
+    const srcDir = path.join(fwPath, 'src')
+    await fs.copy(fwPath, this.dir, { 
+      filter: f => {
+        f = path.normalize(f)
+        const [type, name] = path.relative(srcDir, f).split('/');
+        const isPlugin = f.startsWith(srcDir) && type && type !== 'core' && !!name
+        
+        if(isPlugin && !ENABLED_PLUGINS.includes(name)) {
+          return false
+        }
+        return !BLACKLIST.includes(path.basename(f))
+      }
+    })
+    if (!this.isExport) await fs.symlink(`${fwPath}/node_modules`, `${this.dir}/node_modules`, 'junction')
+  }
+
+  /**
+   * Deals with copying all assets used in this course
+   * @return {Promise}
+   */
+  async copyAssets () {
+    const assets = await App.instance.waitForModule('assets')
+    return Promise.all(this.assetData.data.map(async a => {
+      if (a.url) {
+        return
+      }
+      await this.ensureDir(path.dirname(this.assetData.idMap[a._id]))
+      const inputStream = await assets.createFsWrapper(a).read(a)
+      const outputStream = createWriteStream(this.assetData.idMap[a._id])
+      inputStream.pipe(outputStream)
+      return new Promise((resolve, reject) => {
+        inputStream.on('end', () => resolve())
+        outputStream.on('error', e => reject(e))
+      })
+    }))
+  }
+
+  /**
+   * Outputs all course data to the required JSON files
+   * @return {Promise}
+   */
+  async writeContentJson () {
+    const data = Object.values(this.courseData)
+    if (this.isExport && this.assetData.data.length) {
+      this.assetData.data = this.assetData.data.map(d => {
+        return {
+          title: d.title,
+          description: d.description,
+          filename: d.path,
+          tags: d.tags
+        }
+      })
+      data.push(this.assetData)
+    }
+    return Promise.all(data.map(async ({ dir, fileName, data }) => {
+      await this.ensureDir(dir)
+      const filepath = path.join(dir, fileName)
+      const returnData = await fs.writeJson(filepath, data, { spaces: 2 })
+      AdaptFrameworkUtils.log('debug', 'WRITE', filepath)
+      return returnData
+    }))
+  }
+
+  /**
+   * Makes sure the output folder is structured to allow the files to be served statically for previewing
+   * @return {Promise}
+   */
+  async createPreview () {
+    const tempName = `${this.dir}_temp`
+    await fs.move(path.join(this.dir, 'build'), tempName)
+    await fs.remove(this.dir)
+    await fs.move(tempName, this.dir)
+    this.location = this.dir
+  }
+
+  /**
+   * Creates a zip file containing all files relevant to the type of build being performed
+   * @return {Promise}
+   */
+  async createZip () {
+    const zipPath = path.join(this.dir, this.isPublish ? 'build' : '')
+    const outputPath = `${this.dir}.zip`
+    await zipper.zip(zipPath, outputPath, { removeSource: true })
+    await fs.remove(this.dir)
+    this.location = outputPath
+  }
+
+  /**
+   * Stored metadata about a build attempt in the DB
+   * @return {Promise} Resolves with the DB document
+   */
+  async recordBuildAttempt () {
+    const [framework, jsonschema, mongodb] = await App.instance.waitForModule('adaptframework', 'jsonschema', 'mongodb')
+    const schema = await jsonschema.getSchema('adaptbuild')
+    const validatedData = await schema.validate({
+      action: this.action,
+      courseId: this.courseId,
+      location: this.location,
+      expiresAt: this.expiresAt,
+      createdBy: this.userId,
+      versions: this.enabledPlugins.reduce((m, p) => {
+        return { ...m, [p.name]: p.version }
+      }, { adapt_framework: framework.version })
+    })
+    return mongodb.insert(this.collectionName, validatedData)
+  }
+
+  /**
+   * Removes all previous builds of this.action type
+   * @return {Promise}
+   */
+  async removeOldBuilds () {
+    const mongodb = await App.instance.waitForModule('mongodb')
+    const query = { action: this.action, createdBy: this.userId }
+    const oldBuilds = await mongodb.find(this.collectionName, query)
+    await Promise.all(oldBuilds.map(b => fs.remove(b.location)))
+    return mongodb.deleteMany(this.collectionName, query)
+  }
+}
+
+export default AdaptFrameworkBuild