adapt-authoring-adaptframework 2.6.0 → 3.0.0

package/.github/workflows/releases.yml

@@ -10,7 +10,8 @@ permissions:
   issues: write
   pull-requests: write
   id-token: write
+  packages: write
 
 jobs:
   release:
-    uses: adaptlearning/semantic-release-config/.github/workflows/release.yml@master
+    uses: adaptlearning/semantic-release-config/.github/workflows/release.yml@master

package/conf/config.schema.json

@@ -32,6 +32,11 @@
       "description": "URL of the Adapt framework git repository to install",
       "type": "string"
     },
+    "prebuildCache": {
+      "description": "When enabled, eagerly rebuilds the prebuilt cache for every (theme, menu) combination in the background after invalidation (e.g. plugin install/update/delete)",
+      "type": "boolean",
+      "default": false
+    },
     "importMaxFileSize": {
       "description": "Maximum file upload size for course imports",
       "type": "string",

package/index.js

@@ -3,6 +3,6 @@
  * @namespace adaptframework
  */
 export { default } from './lib/AdaptFrameworkModule.js'
-export { copyFrameworkSource } from './lib/utils.js'
+export { copyFrameworkSource, readFrameworkPluginVersions } from './lib/utils.js'
 export { default as AdaptFrameworkBuild } from './lib/AdaptFrameworkBuild.js'
 export { default as AdaptFrameworkImport } from './lib/AdaptFrameworkImport.js'

package/lib/AdaptFrameworkBuild.js

@@ -3,7 +3,8 @@ import { App, Hook, ensureDir, writeJson } from 'adapt-authoring-core'
 import { parseObjectId } from 'adapt-authoring-mongodb'
 import { createWriteStream } from 'node:fs'
 import AdaptCli from 'adapt-cli'
-import { log, logDir, logMemory, copyFrameworkSource } from './utils.js'
+import { log, logDir, logMemory, copyFrameworkSource, generateLanguageManifest, applyBuildReplacements } from './utils.js'
+import BuildCache from './BuildCache.js'
 import fs from 'node:fs/promises'
 import path from 'upath'
 import semver from 'semver'
@@ -199,11 +200,48 @@ class AdaptFrameworkBuild {
 
     await this.loadCourseData()
 
+    // Check for cached preview build
+    if (this.isPreview && !contentOnly) {
+      const cache = new BuildCache(path.join(framework.getConfig('buildDir'), 'prebuilt-cache'))
+      const pluginHash = await framework.getPluginHash()
+      const theme = this.courseData.config.data._theme
+      const menu = this.courseData.config.data._menu
+
+      if (await cache.has(pluginHash, theme, menu)) {
+        await cache.restore(pluginHash, theme, menu, this.buildDir)
+        await this.applySchemaDefaults()
+        await this.copyAssets()
+        await this.preBuildHook.invoke(this)
+        await this.writeContentJson()
+        await this.writeLanguageManifest()
+        await applyBuildReplacements(this.buildDir, {
+          defaultLanguage: this.courseData.config.data._defaultLanguage ?? 'en',
+          defaultDirection: this.courseData.config.data._defaultDirection ?? 'ltr',
+          buildType: 'development',
+          timestamp: Date.now()
+        })
+        this.location = path.join(this.dir, 'build')
+        await this.postBuildHook.invoke(this)
+        this.buildData = await this.recordBuildAttempt()
+        return this
+      }
+    }
+
     const tasks = [this.copyAssets()]
     if (!contentOnly) {
+      // preview cache is shared across courses, so include all installed plugins
+      // — except disabled themes/menus, since only one of each can be active per
+      // build and the framework's less:dev task globs every theme/menu in src/,
+      // which OOMs when more than one is present (see adapt_framework#3802).
+      const pluginsToInclude = this.isPreview
+        ? [
+            ...this.enabledPlugins,
+            ...this.disabledPlugins.filter(p => p.type !== 'theme' && p.type !== 'menu')
+          ]
+        : this.enabledPlugins
       tasks.push(copyFrameworkSource({
         destDir: this.dir,
-        enabledPlugins: this.enabledPlugins.map(p => p.name),
+        enabledPlugins: pluginsToInclude.map(p => p.name),
         linkNodeModules: !this.isExport
       }))
     }
@@ -233,6 +271,18 @@ class AdaptFrameworkBuild {
           .setData(e)
       }
     }
+    // Populate prebuilt cache after successful grunt build for preview
+    if (this.isPreview && !contentOnly) {
+      const cache = new BuildCache(path.join(framework.getConfig('buildDir'), 'prebuilt-cache'))
+      const pluginHash = await framework.getPluginHash()
+      const theme = this.courseData.config.data._theme
+      const menu = this.courseData.config.data._menu
+      try {
+        await cache.populate(this.buildDir, pluginHash, theme, menu)
+      } catch (e) {
+        log('warn', 'CACHE', `failed to populate prebuilt cache: ${e.message}`)
+      }
+    }
     if (this.compress) {
       this.location = await this.prepareZip()
     } else {
@@ -275,10 +325,10 @@ class AdaptFrameworkBuild {
    * @return {Promise}
    */
   async loadAssetData () {
-    const [assets, courseassets, tags] = await App.instance.waitForModule('assets', 'courseassets', 'tags')
+    const [assets, content, tags] = await App.instance.waitForModule('assets', 'content', 'tags')
 
-    const caRecs = await courseassets.find({ courseId: this.courseId })
-    const uniqueAssetIds = new Set(caRecs.map(c => parseObjectId(c.assetId)))
+    const courseContent = await content.find({ _courseId: this.courseId }, { validate: false }, { projection: { _assetIds: 1 } })
+    const uniqueAssetIds = new Set(courseContent.flatMap(c => (c._assetIds ?? []).map(id => parseObjectId(id))))
     const usedAssets = await assets.find({ _id: { $in: [...uniqueAssetIds] } })
 
     const usedTagIds = new Set(usedAssets.reduce((m, a) => [...m, ...(a.tags ?? [])], []))
@@ -426,12 +476,35 @@ class AdaptFrameworkBuild {
     }))
   }
 
+  /**
+   * Outputs all course data to the required JSON files
+   * @return {Promise}
+   */
+  async writeContentJson () {
+    const data = Object.values(this.courseData)
+    if (this.isExport && this.assetData.data.length) {
+      this.assetData.data = this.assetData.data.map(d => {
+        return {
+          title: d.title,
+          description: d.description,
+          filename: d.path,
+          tags: d.tags
+        }
+      })
+      data.push(this.assetData)
+    }
+    return Promise.all(data.map(async ({ dir, fileName, data }) => {
+      await ensureDir(dir)
+      const filepath = path.join(dir, fileName)
+      const returnData = await writeJson(filepath, data)
+      log('verbose', 'WRITE', filepath)
+      return returnData
+    }))
+  }
+
   /**
    * Applies schema defaults to the in-memory course and config data using
    * the jsonschema module. Replicates what grunt's schema-defaults task does.
-   *
-   * TODO: replace validateWithDefaults workaround with schema.validate(data, { ignoreErrors: true })
-   * once migrated to adapt-schemas v3.x (see #184)
    * @return {Promise}
    */
   async applySchemaDefaults () {
@@ -442,31 +515,19 @@ class AdaptFrameworkBuild {
     const extensionFilter = s => contentplugin.isPluginSchema(s) ? enabledPluginSchemas.includes(s) : true
     const getSchema = name => jsonschema.getSchema(name, { useCache: false, extensionFilter })
 
-    /**
-     * Applies defaults via validate(), catching and ignoring validation errors.
-     * The validated+defaulted data is returned from validate() on success, or
-     * extracted from the error on failure (validate clones internally).
-     */
-    const validateWithDefaults = (schema, data) => {
-      try {
-        return schema.validate(data, { useDefaults: true, ignoreRequired: true })
-      } catch (e) {
-        return e.data.data
-      }
-    }
-
+    // Apply defaults without running full validation (which rejects ObjectIds etc.)
     const [courseSchema, configSchema] = await Promise.all([
       getSchema('course'),
       getSchema('config')
     ])
-    Object.assign(this.courseData.course.data, validateWithDefaults(courseSchema, this.courseData.course.data))
-    Object.assign(this.courseData.config.data, validateWithDefaults(configSchema, this.courseData.config.data))
+    courseSchema.compiledWithDefaults(this.courseData.course.data)
+    configSchema.compiledWithDefaults(this.courseData.config.data)
 
     for (const type of ['contentObject', 'article', 'block']) {
       const schemaName = type === 'contentObject' ? 'contentobject' : type
       const schema = await getSchema(schemaName)
       for (const item of this.courseData[type].data) {
-        Object.assign(item, validateWithDefaults(schema, item))
+        schema.compiledWithDefaults(item)
       }
     }
 
@@ -476,34 +537,23 @@ class AdaptFrameworkBuild {
       if (!componentSchemas[schemaName]) {
         componentSchemas[schemaName] = await getSchema(schemaName)
       }
-      Object.assign(item, validateWithDefaults(componentSchemas[schemaName], item))
+      componentSchemas[schemaName].compiledWithDefaults(item)
     }
   }
 
   /**
-   * Outputs all course data to the required JSON files
+   * Writes the language_data_manifest.js for each language dir.
+   * Only needed on cache-hit builds where grunt is skipped.
    * @return {Promise}
    */
-  async writeContentJson () {
-    const data = Object.values(this.courseData)
-    if (this.isExport && this.assetData.data.length) {
-      this.assetData.data = this.assetData.data.map(d => {
-        return {
-          title: d.title,
-          description: d.description,
-          filename: d.path,
-          tags: d.tags
-        }
-      })
-      data.push(this.assetData)
-    }
-    return Promise.all(data.map(async ({ dir, fileName, data }) => {
-      await ensureDir(dir)
-      const filepath = path.join(dir, fileName)
-      const returnData = await writeJson(filepath, data)
-      log('verbose', 'WRITE', filepath)
-      return returnData
-    }))
+  async writeLanguageManifest () {
+    const langDir = this.courseData.course.dir
+    const fileNames = Object.values(this.courseData)
+      .filter(d => d.dir === langDir)
+      .map(d => d.fileName)
+    const manifest = generateLanguageManifest(fileNames)
+    await ensureDir(langDir)
+    await writeJson(path.join(langDir, 'language_data_manifest.js'), manifest)
   }
 
   /**

package/lib/AdaptFrameworkImport.js

@@ -1,13 +1,11 @@
-import { App, Hook, spawn, readJson, writeJson } from 'adapt-authoring-core'
-import { parseObjectId } from 'adapt-authoring-mongodb'
+import { App, Hook, readJson, writeJson } from 'adapt-authoring-core'
+import { isValidObjectId, parseObjectId } from 'adapt-authoring-mongodb'
 import fs from 'node:fs/promises'
 import { glob } from 'glob'
-import octopus from 'adapt-octopus'
 import path from 'upath'
-import { randomBytes } from 'node:crypto'
 import semver from 'semver'
 import { unzip } from 'zipper'
-import { log, logDir, getImportSummary, getImportContentCounts } from './utils.js'
+import { log, logDir, getImportSummary, getImportContentCounts, readFrameworkPluginVersions, collectMigrationScripts, runContentMigration } from './utils.js'
 
 import ComponentTransform from './migrations/component.js'
 import ConfigTransform from './migrations/config.js'
@@ -15,6 +13,7 @@ import GraphicSrcTransform from './migrations/graphic-src.js'
 import NavOrderTransform from './migrations/nav-order.js'
 import ParentIdTransform from './migrations/parent-id.js'
 import RemoveUndefTransform from './migrations/remove-undef.js'
+import VanillaBackgroundStylesTransform from './migrations/vanilla-background-styles.js'
 import StartPageTransform from './migrations/start-page.js'
 import ThemeUndefTransform from './migrations/theme-undef.js'
 
@@ -26,7 +25,8 @@ const ContentMigrations = [
   ParentIdTransform,
   RemoveUndefTransform,
   StartPageTransform,
-  ThemeUndefTransform
+  ThemeUndefTransform,
+  VanillaBackgroundStylesTransform
 ]
 
 /**
@@ -175,7 +175,8 @@ class AdaptFrameworkImport {
      */
     this.statusReport = {
       info: [],
-      warn: []
+      warn: [],
+      error: []
     }
     /**
      * Summary information for the import run
@@ -221,10 +222,9 @@ class AdaptFrameworkImport {
         assets,
         content,
         contentplugin,
-        courseassets,
         framework,
         jsonschema
-      ] = await App.instance.waitForModule('assets', 'content', 'contentplugin', 'courseassets', 'adaptframework', 'jsonschema')
+      ] = await App.instance.waitForModule('assets', 'content', 'contentplugin', 'adaptframework', 'jsonschema')
       /**
        * Cached module instance for easy access
        * @type {AssetsModule}
@@ -240,11 +240,6 @@ class AdaptFrameworkImport {
        * @type {ContentPluginModule}
        */
       this.contentplugin = contentplugin
-      /**
-       * Cached module instance for easy access
-       * @type {CourseAssetsModule}
-       */
-      this.courseassets = courseassets
       /**
        * Cached module instance for easy access
        * @type {AdaptFrameworkModule}
@@ -269,9 +264,8 @@ class AdaptFrameworkImport {
         [this.importCourseAssets, importContent],
         [this.importCoursePlugins, isDryRun && importPlugins],
         [this.importCoursePlugins, !isDryRun && importContent],
-        [this.loadCourseData, isDryRun && importContent],
+        [this.loadCourseData, importContent],
         [this.migrateCourseData, !isDryRun && migrateContent],
-        [this.loadCourseData, !isDryRun && importContent],
         [this.importCourseData, !isDryRun && importContent],
         [this.generateSummary]
       ]
@@ -344,34 +338,18 @@ class AdaptFrameworkImport {
       }
       this.statusReport.info.push({ code: 'MIGRATE_CONTENT', data })
     }
-    await this.convertSchemas()
     log('debug', 'preparation tasks completed successfully')
   }
 
   /**
-   * Converts all properties.schema files to a valid JSON schema format
-   * @return {Promise}
-   */
-  async convertSchemas () {
-    return octopus.runRecursive({
-      cwd: this.path,
-      logger: { log: (...args) => log('debug', ...args) }
-    })
-  }
-
-  /**
-   * Writes the contents of 2-customStyles.less to course.json file. Unfortunately it's necessary to do it this way to ensure it's included in migrations.
+   * Reads 2-customStyles.less (if present) and injects its contents as customStyle on the in-memory course, so migrations and the DB write see it. Existing customStyle on the course takes precedence.
    */
   async patchCustomStyle () {
     const [customStylePath] = await glob('**/2-customStyles.less', { cwd: this.path, absolute: true })
-    const courseJsonPath = `${this.langPath}/course.json`
-    if (!customStylePath) {
-      return
-    }
+    if (!customStylePath) return
     try {
       const customStyle = await fs.readFile(customStylePath, 'utf8')
-      const courseJson = await readJson(courseJsonPath)
-      await writeJson(courseJsonPath, { customStyle, ...courseJson })
+      this.contentJson.course = { customStyle, ...this.contentJson.course }
       log('info', 'patched course customStyle')
     } catch (e) {
       log('warn', 'failed to patch course customStyle', e)
@@ -379,15 +357,14 @@ class AdaptFrameworkImport {
   }
 
   /**
-   * Ensures _theme exists on the config
+   * Ensures _theme exists on the in-memory config
    */
   async patchThemeName () {
     try {
-      const configJsonPath = `${this.coursePath}/config.json`
-      const configJson = await readJson(configJsonPath)
-      if (configJson._theme) return
-      configJson._theme = Object.values(this.usedContentPlugins).find(p => p.type === 'theme').name
-      await writeJson(configJsonPath, configJson)
+      if (this.contentJson.config?._theme) return
+      const _theme = Object.values(this.usedContentPlugins).find(p => p.type === 'theme')?.name
+      if (!_theme || !this.contentJson.config) return
+      this.contentJson.config._theme = _theme
       log('info', 'patched config _theme')
     } catch (e) {
       log('warn', 'failed to patch config _theme', e)
@@ -492,47 +469,61 @@ class AdaptFrameworkImport {
   }
 
   /**
-   * Run grunt task
-   * @return {Promise}
-   */
-  async runGruntMigration (subTask, { outputDir, captureDir, outputFilePath }) {
-    const output = await spawn({
-      cmd: 'npx',
-      args: ['grunt', `migration:${subTask}`, `--outputdir=${outputDir}`, `--capturedir=${captureDir}`],
-      cwd: this.frameworkPath ?? this.framework.path
-    })
-    if (outputFilePath) await fs.writeFile(outputFilePath, output)
-  }
-
-  /**
-   * Handle migrate course data, installs adapt-migrations/capture data/adds updated scripts/migrates data
+   * Migrates course data in-memory using adapt-migrations
    */
   async migrateCourseData () {
     try {
       await this.patchThemeName()
       await this.patchCustomStyle()
 
-      const migrationId = `${this.userId}-${randomBytes(4).toString('hex')}`
-
-      const opts = {
-        outputDir: path.relative(this.framework.path, path.resolve(this.coursePath, '..')),
-        captureDir: path.join(`./${migrationId}-migrations`),
-        outputFilePath: path.join(this.framework.path, 'migrations', `${migrationId}.txt`)
-      }
-      log('debug', 'MIGRATION_ID', migrationId)
-      logDir('captureDir', opts.captureDir)
-      logDir('outputDir', opts.outputDir)
+      const content = this.flattenContentJson()
+      const fromPlugins = Object.values(this.usedContentPlugins).map(p => ({
+        name: p.name,
+        version: p.version
+      }))
+      const toPlugins = await readFrameworkPluginVersions(this.framework.path)
+      const scripts = await collectMigrationScripts(this.framework.path)
 
-      await this.runGruntMigration('capture', opts)
-      await this.runGruntMigration('migrate', opts)
+      const migrated = await runContentMigration({ content, fromPlugins, toPlugins, scripts })
 
-      await fs.rm(path.join(this.framework.path, opts.captureDir), { recursive: true })
+      this.unflattenContentJson(migrated)
+      log('info', 'in-memory content migration completed')
     } catch (error) {
       log('error', 'Migration process failed', error)
       throw App.instance.errors.FW_IMPORT_MIGRATION_FAILED.setData({ reason: error.message })
     }
   }
 
+  /**
+   * Flattens this.contentJson into a flat array for adapt-migrations
+   * @returns {Array}
+   */
+  flattenContentJson () {
+    const content = []
+    if (this.contentJson.course?._id) content.push(this.contentJson.course)
+    if (this.contentJson.config?._id) content.push(this.contentJson.config)
+    for (const item of Object.values(this.contentJson.contentObjects)) {
+      if (item?._id) content.push(item)
+    }
+    return content
+  }
+
+  /**
+   * Writes migrated content back into the contentJson structure
+   * @param {Array} migrated The migrated content array
+   */
+  unflattenContentJson (migrated) {
+    for (const item of migrated) {
+      if (item._type === 'course') {
+        this.contentJson.course = item
+      } else if (item._type === 'config') {
+        this.contentJson.config = item
+      } else {
+        this.contentJson.contentObjects[item._id] = item
+      }
+    }
+  }
+
   /**
    * Imports any specified tags
    * @return {Promise}
@@ -586,13 +577,15 @@ class AdaptFrameworkImport {
       if (this.settings.isDryRun) {
         return
       }
+      const stats = await fs.stat(filepath)
       try {
         const asset = await this.assets.insert({
           ...data,
           createdBy: this.userId,
           file: {
             filepath,
-            originalFilename: filepath
+            originalFilename: filepath,
+            size: stats.size
           },
           tags: data.tags
         })
@@ -600,7 +593,13 @@ class AdaptFrameworkImport {
         const resolved = path.relative(`${this.coursePath}/..`, filepath)
         this.assetMap[resolved] = asset._id.toString()
       } catch (e) {
-        this.statusReport.warn.push({ code: 'ASSET_IMPORT_FAILED', data: { filepath } })
+        if (e.code === 'DUPLICATE_ASSET') {
+          const resolved = path.relative(`${this.coursePath}/..`, filepath)
+          this.assetMap[resolved] = e.data.assetId
+        } else {
+          log('error', `asset import failed for '${filepath}'`, e)
+          this.statusReport.warn.push({ code: 'ASSET_IMPORT_FAILED', data: { filepath, reason: e?.message ?? String(e) } })
+        }
       }
       imagesImported++
     }))
@@ -738,8 +737,9 @@ class AdaptFrameworkImport {
     try {
       const course = await this.importContentObject({ ...this.contentJson.course, tags: this.tags })
       /*  config  */ await this.importContentObject(this.contentJson.config)
-      // we need to run an update with the same data to make sure all extension schema settings are applied
-      await this.importContentObject({ ...this.contentJson.course, _id: course._id }, { isUpdate: true })
+      // we need to run an update with the same data to make sure all extension schema settings are applied;
+      // ignoreRequired because some plugins declare top-level required properties with no default that Ajv can't materialise (e.g. adapt-contrib-glossary)
+      await this.importContentObject({ ...this.contentJson.course, _id: course._id }, { isUpdate: true, ignoreRequired: true })
     } catch (e) {
       throw App.instance.errors.FW_IMPORT_CONTENT_FAILED.setData({ errors: [formatError(e)] })
     }
@@ -751,8 +751,8 @@ class AdaptFrameworkImport {
         try {
           const itemJson = this.contentJson.contentObjects[_id]
           await this.importContentObject({
-            _sortOrder: hierarchy[itemJson._parentId].indexOf(_id) + 1,
-            ...itemJson // note that JSON sort order will override the deduced one
+            ...itemJson,
+            _sortOrder: hierarchy[itemJson._parentId].indexOf(_id) + 1 // trust the hierarchy: per-insert updateSortOrder is disabled, so we can't rely on bad export values being normalised later
           })
         } catch (e) {
           errors.push(formatError(e))
@@ -760,6 +760,8 @@ class AdaptFrameworkImport {
       }
     }
     if (errors.length) throw App.instance.errors.FW_IMPORT_CONTENT_FAILED.setData({ errors })
+    // single-pass sweep now all content is in place; per-insert sweep was disabled to avoid O(n²) work
+    await this.content.updateEnabledPlugins({ _courseId: this.idMap.course }, { forceUpdate: true })
     log('debug', 'imported course data successfully')
   }
 
@@ -798,6 +800,7 @@ class AdaptFrameworkImport {
     let insertData = await this.transformData({
       ...data,
       _id: undefined,
+      _assetIds: undefined, // recompute from resolved asset references; export ships paths, not ObjectIds
       _courseId: this.idMap.course,
       createdBy: this.userId
     })
@@ -810,7 +813,7 @@ class AdaptFrameworkImport {
     }
     insertData = schema.sanitise(insertData)
     let doc
-    const opts = { schemaName, validate: true, useCache: false }
+    const opts = { schemaName, validate: true, useCache: false, updateEnabledPlugins: false, updateSortOrder: false, ignoreRequired: options.ignoreRequired }
     if (options.isUpdate) {
       doc = await this.content.update({ _id: data._id }, insertData, opts)
     } else {
@@ -842,7 +845,13 @@ class AdaptFrameworkImport {
     schema.walk(data, field =>
       field?._backboneForms?.type === 'Asset' || field?._backboneForms === 'Asset'
     ).forEach(({ data: parent, key, value }) => {
-      value ? parent[key] = this.assetMap[value] ?? value : delete parent[key]
+      if (!value) return delete parent[key]
+      const mapped = this.assetMap[value]
+      if (mapped) return (parent[key] = mapped)
+      if (isValidObjectId(value)) return (parent[key] = value)
+      log('warn', `unable to resolve asset reference '${value}' — dropping field`)
+      this.statusReport.warn.push({ code: 'UNRESOLVED_ASSET_REF', data: { path: value } })
+      delete parent[key]
     })
   }
 
@@ -914,9 +923,7 @@ class AdaptFrameworkImport {
         const _courseId = parseObjectId(this.idMap[this.contentJson.course._id])
         tasks.push(
           this.content.deleteMany({ _courseId })
-            .catch(e => log('warn', 'failed to delete course content', e)),
-          this.courseassets.deleteMany({ courseId: _courseId })
-            .catch(e => log('warn', 'failed to delete course assets', e))
+            .catch(e => log('warn', 'failed to delete course content', e))
         )
       } catch (e) {} // courseId not available, no content to roll back
     }