adapt-authoring-adaptframework 2.5.6 → 3.0.0

package/lib/AdaptFrameworkModule.js

@@ -4,7 +4,8 @@ import AdaptFrameworkImport from './AdaptFrameworkImport.js'
 import fs from 'node:fs/promises'
 import { getHandler, postHandler, importHandler, postUpdateHandler, getUpdateHandler } from './handlers.js'
 import { loadRouteConfig, registerRoutes } from 'adapt-authoring-server'
-import { runCliCommand } from './utils.js'
+import { runCliCommand, readFrameworkPluginVersions, migrateExistingCourses, computePluginHash, prebuildCache } from './utils.js'
+import BuildCache from './BuildCache.js'
 import path from 'node:path'
 import semver from 'semver'
 
@@ -83,6 +84,17 @@ class AdaptFrameworkModule extends AbstractModule {
 
     await this.installFramework()
 
+    this.app.waitForModule('contentplugin').then(contentplugin => {
+      contentplugin.postInsertHook.tap(() => this.invalidatePrebuiltCache())
+      contentplugin.postUpdateHook.tap(() => this.invalidatePrebuiltCache())
+      contentplugin.postDeleteHook.tap(() => this.invalidatePrebuiltCache())
+    })
+    this.postUpdateHook.tap(() => this.invalidatePrebuiltCache())
+
+    if (this.getConfig('prebuildCache')) {
+      this.prebuildCache()
+    }
+
     process.env.BROWSERSLIST_IGNORE_OLD_DATA = '1'
 
     if (this.app.args['update-framework'] === true) {
@@ -209,6 +221,7 @@ class AdaptFrameworkModule extends AbstractModule {
    * @return {Promise}
    */
   async updateFramework (version) {
+    let migrationResult
     try {
       if (version) {
         this.checkVersionCompatibility(version)
@@ -216,13 +229,71 @@ class AdaptFrameworkModule extends AbstractModule {
       if (!version && this.targetVersionRange) {
         version = await this.getLatestVersion()
       }
+      const fromPlugins = await readFrameworkPluginVersions(this.path)
       await this.runCliCommand('updateFramework', { version })
       this._version = await this.runCliCommand('getCurrentFrameworkVersion')
+      const toPlugins = await readFrameworkPluginVersions(this.path)
+      migrationResult = await migrateExistingCourses({ fromPlugins, toPlugins, frameworkDir: this.path })
     } catch (e) {
       this.log('error', `failed to update framework, ${e.message}`)
       throw e.statusCode ? e : this.app.errors.FW_UPDATE_FAILED.setData({ reason: e.message })
     }
-    this.postUpdateHook.invoke()
+    await this.postUpdateHook.invoke()
+    return migrationResult
+  }
+
+  /**
+   * Returns a cached plugin hash, computing it on first call
+   * @return {Promise<String>}
+   */
+  async getPluginHash () {
+    if (!this._pluginHash) {
+      this._pluginHash = await computePluginHash(this.path)
+    }
+    return this._pluginHash
+  }
+
+  /**
+   * Invalidates the prebuilt compilation cache and optionally
+   * triggers an eager rebuild of the shared cache in the background
+   */
+  async invalidatePrebuiltCache () {
+    this._pluginHash = null
+
+    try {
+      await new BuildCache(path.join(this.getConfig('buildDir'), 'prebuilt-cache')).invalidate()
+    } catch (e) {
+      this.log('warn', `failed to invalidate prebuilt cache: ${e.message}`)
+    }
+
+    if (this.getConfig('prebuildCache')) {
+      this.prebuildCache()
+    }
+  }
+
+  /**
+   * Eagerly rebuilds the prebuilt cache in the background, iterating every
+   * (theme, menu) combination of installed plugins. Safe to call multiple
+   * times — if a build is already in progress, it will be reused.
+   * Idempotent — already-cached combos are skipped.
+   * @return {Promise<void>}
+   */
+  prebuildCache () {
+    if (this._eagerBuildPromise) {
+      return this._eagerBuildPromise
+    }
+    this._eagerBuildPromise = prebuildCache({
+      buildDir: this.getConfig('buildDir'),
+      frameworkDir: this.path
+    }).catch(e => {
+      this.log('warn', `eager prebuild failed: ${e.message}`)
+      if (e.cmd) this.log('warn', `cmd: ${e.cmd}`)
+      if (e.raw) this.log('warn', `output: ${e.raw}`)
+      if (e.stderr) this.log('warn', `stderr: ${e.stderr}`)
+    }).finally(() => {
+      this._eagerBuildPromise = null
+    })
+    return this._eagerBuildPromise
   }
 
   /**
@@ -245,7 +316,7 @@ class AdaptFrameworkModule extends AbstractModule {
   async loadSchemas () {
     const jsonschema = await this.app.waitForModule('jsonschema')
     const schemas = (await this.runCliCommand('getSchemaPaths')).filter(s => s.includes('/core/'))
-    await Promise.all(schemas.map(s => jsonschema.registerSchema(s)))
+    schemas.forEach(s => jsonschema.registerSchema(s))
   }
 
   /**
@@ -265,7 +336,7 @@ class AdaptFrameworkModule extends AbstractModule {
     if (!course) {
       return
     }
-    const shareWithUsers = course?._shareWithUsers.map(id => id.toString()) ?? []
+    const shareWithUsers = course._shareWithUsers?.map(id => id.toString()) ?? []
     const userId = req.auth.user._id.toString()
     return course.createdBy.toString() === userId || course._isShared || shareWithUsers.includes(userId)
   }
@@ -307,6 +378,18 @@ class AdaptFrameworkModule extends AbstractModule {
     this.contentMigrations.push(migration)
   }
 
+  /**
+   * Migrates content for specific courses. Called by contentplugin on plugin update.
+   * @param {Object} options
+   * @param {Array<{name: String, version: String}>} options.fromPlugins Plugin versions before the update
+   * @param {Array<{name: String, version: String}>} options.toPlugins Plugin versions after the update
+   * @param {String[]} options.courseIds Course IDs to migrate
+   * @returns {Promise<{migrated: Number, failed: Number, errors: Array}>}
+   */
+  async migrateCourses ({ fromPlugins, toPlugins, courseIds }) {
+    return migrateExistingCourses({ fromPlugins, toPlugins, frameworkDir: this.path, courseIds })
+  }
+
   /**
    * Builds a single Adapt framework course
    * @param {AdaptFrameworkBuildOptions} options

package/lib/BuildCache.js

@@ -0,0 +1,105 @@
+import fs from 'node:fs/promises'
+import path from 'upath'
+import { log } from './utils/log.js'
+
+/** Build output entries that aren't cached (rebuilt per-build from course data) */
+const SKIP_ENTRIES = new Set(['course'])
+
+/**
+ * Filesystem-level cache of grunt build output, keyed by (pluginHash, theme, menu).
+ * One instance per cache root; methods are stateless beyond the root path.
+ */
+class BuildCache {
+  /**
+   * @param {String} cacheRoot Root cache directory
+   */
+  constructor (cacheRoot) {
+    this.cacheRoot = cacheRoot
+  }
+
+  /**
+   * @returns {String} The cache directory path for the given combo
+   */
+  getPath (pluginHash, theme, menu) {
+    return path.join(this.cacheRoot, `${pluginHash}_${theme}_${menu}`)
+  }
+
+  /**
+   * @returns {Promise<Boolean>} Whether a cached build exists for the given combo
+   */
+  async has (pluginHash, theme, menu) {
+    try {
+      await fs.access(this.getPath(pluginHash, theme, menu))
+      return true
+    } catch {
+      return false
+    }
+  }
+
+  /**
+   * Copies the build output (minus per-course content) into the cache for the given combo.
+   * Uses a temp dir + atomic rename for parallel safety.
+   * @param {String} buildOutputDir The build output directory
+   */
+  async populate (buildOutputDir, pluginHash, theme, menu) {
+    const cacheDir = this.getPath(pluginHash, theme, menu)
+    await fs.mkdir(this.cacheRoot, { recursive: true })
+
+    const tmpDir = `${cacheDir}_tmp_${Date.now()}`
+    try {
+      await fs.mkdir(tmpDir, { recursive: true })
+      const entries = await fs.readdir(buildOutputDir)
+      for (const entry of entries) {
+        if (SKIP_ENTRIES.has(entry)) continue
+        await copyEntry(path.join(buildOutputDir, entry), path.join(tmpDir, entry))
+      }
+      await safeRename(tmpDir, cacheDir)
+      log('info', 'CACHE', `populated cache for ${pluginHash} (theme=${theme}, menu=${menu})`)
+    } catch (e) {
+      await fs.rm(tmpDir, { recursive: true, force: true })
+      throw e
+    }
+  }
+
+  /**
+   * Copies cached artifacts to a build directory.
+   * @param {String} destDir Destination build directory
+   */
+  async restore (pluginHash, theme, menu, destDir) {
+    await fs.mkdir(destDir, { recursive: true })
+    await fs.cp(this.getPath(pluginHash, theme, menu), destDir, { recursive: true })
+    log('info', 'CACHE', `restored from cache for ${pluginHash} (theme=${theme}, menu=${menu})`)
+  }
+
+  /**
+   * Removes the entire cache root.
+   */
+  async invalidate () {
+    await fs.rm(this.cacheRoot, { recursive: true, force: true })
+    log('info', 'CACHE', 'invalidated prebuilt cache')
+  }
+}
+
+async function copyEntry (src, dest) {
+  const stat = await fs.stat(src)
+  if (stat.isDirectory()) {
+    await fs.cp(src, dest, { recursive: true })
+  } else {
+    await fs.mkdir(path.dirname(dest), { recursive: true })
+    await fs.copyFile(src, dest)
+  }
+}
+
+async function safeRename (src, dest) {
+  try {
+    await fs.rename(src, dest)
+  } catch (e) {
+    if (e.code === 'ENOTEMPTY' || e.code === 'EEXIST') {
+      await fs.rm(src, { recursive: true, force: true })
+    } else {
+      throw e
+    }
+  }
+}
+
+export default BuildCache

package/lib/handlers.js

@@ -133,11 +133,12 @@ export async function postUpdateHandler (req, res, next) {
     }
     log('info', 'running framework update')
     const previousVersion = framework.version
-    await framework.updateFramework(req.body.version)
+    const migrationResult = await framework.updateFramework(req.body.version)
     const currentVersion = framework.version !== previousVersion ? framework.version : undefined
     res.json({
       from: previousVersion,
-      to: currentVersion
+      to: currentVersion,
+      migration: migrationResult
     })
   } catch (e) {
     return next(e)

package/lib/migrations/vanilla-background-styles.js

@@ -0,0 +1,17 @@
+const FIELDS = ['_backgroundRepeat', '_backgroundSize', '_backgroundPosition']
+
+function clean (styles) {
+  if (!styles || typeof styles !== 'object') return
+  for (const f of FIELDS) {
+    if (styles[f] === '' || styles[f] === null) delete styles[f]
+  }
+}
+
+async function VanillaBackgroundStyles (data) {
+  const v = data._vanilla
+  if (!v || typeof v !== 'object') return
+  clean(v._backgroundStyles)
+  clean(v._pageHeader?._backgroundStyles)
+}
+
+export default VanillaBackgroundStyles

package/lib/utils/applyBuildReplacements.js

@@ -0,0 +1,23 @@
+import fs from 'node:fs/promises'
+import path from 'upath'
+
+/**
+ * Applies @@placeholder substitutions in index.html
+ * @param {String} buildDir The build output directory
+ * @param {Object} data Replacement values
+ * @param {String} data.defaultLanguage The default language code
+ * @param {String} data.defaultDirection The default text direction
+ * @param {String} data.buildType The build type (e.g. 'development')
+ * @param {Number} data.timestamp The build timestamp
+ * @return {Promise}
+ */
+export async function applyBuildReplacements (buildDir, { defaultLanguage, defaultDirection, buildType, timestamp }) {
+  const indexPath = path.join(buildDir, 'index.html')
+  let html = await fs.readFile(indexPath, 'utf8')
+  html = html
+    .replace(/@@config\._defaultLanguage/g, defaultLanguage)
+    .replace(/@@config\._defaultDirection/g, defaultDirection)
+    .replace(/@@build\.type/g, buildType)
+    .replace(/@@build\.timestamp/g, String(timestamp))
+  await fs.writeFile(indexPath, html)
+}

package/lib/utils/collectMigrationScripts.js

@@ -0,0 +1,15 @@
+import { glob } from 'glob'
+import path from 'node:path'
+
+/**
+ * Collects all migration script paths from the framework's src directory
+ * @param {String} frameworkDir Absolute path to the framework directory
+ * @returns {Promise<String[]>} Absolute paths to migration scripts
+ */
+export async function collectMigrationScripts (frameworkDir) {
+  const srcDir = path.join(frameworkDir, 'src')
+  return glob([
+    'core/migrations/**/*.js',
+    '*/*/migrations/**/*.js'
+  ], { cwd: srcDir, absolute: true })
+}

package/lib/utils/computePluginHash.js

@@ -0,0 +1,14 @@
+import { createHash } from 'node:crypto'
+import Project from 'adapt-cli/lib/integration/Project.js'
+
+/**
+ * Computes a deterministic hash from the installed plugin set
+ * @param {String} frameworkDir Path to the local framework installation
+ * @return {Promise<String>} 16-char hex hash
+ */
+export async function computePluginHash (frameworkDir) {
+  const project = new Project({ cwd: frameworkDir })
+  const deps = await project.getInstalledDependencies()
+  const sorted = Object.entries(deps).sort(([a], [b]) => a.localeCompare(b))
+  return createHash('sha256').update(JSON.stringify(sorted)).digest('hex').slice(0, 16)
+}

package/lib/utils/copyFrameworkSource.js

@@ -17,7 +17,7 @@ export async function copyFrameworkSource (options) {
   if (options.copyNodeModules !== true) BLACKLIST.push('node_modules')
 
   const srcDir = path.join(fwPath, 'src')
-  const enabledPlugins = options.enabledPlugins ?? []
+  const enabledPlugins = options.enabledPlugins
   await fs.cp(fwPath, options.destDir, {
     recursive: true,
     filter: f => {
@@ -25,7 +25,7 @@ export async function copyFrameworkSource (options) {
       const [type, name] = path.relative(srcDir, f).split('/')
       const isPlugin = f.startsWith(srcDir) && type && type !== 'core' && !!name
 
-      if (isPlugin && !enabledPlugins.includes(name)) {
+      if (isPlugin && enabledPlugins && !enabledPlugins.includes(name)) {
         return false
       }
       return !BLACKLIST.includes(path.basename(f))

package/lib/utils/generateLanguageManifest.js

@@ -0,0 +1,9 @@
+/**
+ * Returns the list of JSON filenames that belong in a language manifest.
+ * The framework runtime reads this to know which data files to fetch.
+ * @param {Array<String>} jsonFileNames All JSON filenames written to the language dir
+ * @return {Array<String>} Filtered list excluding the manifest itself and assets.json
+ */
+export function generateLanguageManifest (jsonFileNames) {
+  return jsonFileNames.filter(f => f !== 'language_data_manifest.js' && f !== 'assets.json')
+}

package/lib/utils/log.js

@@ -3,15 +3,13 @@ import bytes from 'bytes'
 import fsSync from 'node:fs'
 import path from 'upath'
 
-let fw
-
 /**
- * Logs a message using the framework module
- * @param {...*} args Arguments to be logged
+ * Logs a message using the framework module's namespace
+ * @param {String} level Log level
+ * @param {...*} rest Arguments to be logged
  */
-export async function log (...args) {
-  if (!fw) fw = await App.instance.waitForModule('adaptframework')
-  return fw.log(...args)
+export function log (level, ...rest) {
+  App.instance?.logger?.log(level, 'adaptframework', ...rest)
 }
 
 /**

package/lib/utils/migrateExistingCourses.js

@@ -0,0 +1,87 @@
+import { App } from 'adapt-authoring-core'
+import { isDeepStrictEqual } from 'node:util'
+import { collectMigrationScripts } from './collectMigrationScripts.js'
+import { runContentMigration } from './runContentMigration.js'
+import { log } from './log.js'
+
+/**
+ * Migrates content for a set of courses by courseId
+ * @param {Object} options
+ * @param {Array<{name: String, version: String}>} options.fromPlugins Plugin versions before update
+ * @param {Array<{name: String, version: String}>} options.toPlugins Plugin versions after update
+ * @param {String} options.frameworkDir Absolute path to the framework directory
+ * @param {String[]} [options.courseIds] Specific course IDs to migrate (if omitted, migrates all)
+ * @returns {Promise<{migrated: Number, failed: Number, errors: Array}>}
+ */
+export async function migrateExistingCourses ({ fromPlugins, toPlugins, frameworkDir, courseIds }) {
+  const content = await App.instance.waitForModule('content')
+  const scripts = await collectMigrationScripts(frameworkDir)
+
+  if (!scripts.length) {
+    log('debug', 'no migration scripts found, skipping')
+    return { migrated: 0, failed: 0, errors: [] }
+  }
+
+  const foundCourses = courseIds
+    ? await Promise.all(courseIds.map(async _id => content.findOne({ _id, _type: 'course' }, { strict: false })))
+    : await content.find({ _type: 'course' })
+
+  let migrated = 0
+  let failed = 0
+  const errors = []
+
+  for (let ci = 0; ci < foundCourses.length; ci++) {
+    const course = foundCourses[ci]
+    if (!course) {
+      const courseId = courseIds?.[ci] ?? 'unknown'
+      log('warn', `course ${courseId} not found, skipping`)
+      errors.push({ courseId, error: 'course not found' })
+      failed++
+      continue
+    }
+    try {
+      const courseId = course._id.toString()
+      log('debug', `migrating course ${courseId}`)
+
+      const courseContent = await fetchCourseContent(content, course)
+      const originals = courseContent.map(item => JSON.parse(JSON.stringify(item)))
+
+      const migratedContent = await runContentMigration({
+        content: courseContent,
+        fromPlugins: JSON.parse(JSON.stringify(fromPlugins)),
+        toPlugins,
+        scripts
+      })
+
+      let updatedCount = 0
+      for (let i = 0; i < migratedContent.length; i++) {
+        const normalized = JSON.parse(JSON.stringify(migratedContent[i]))
+        if (!isDeepStrictEqual(originals[i], normalized)) {
+          await content.update({ _id: migratedContent[i]._id }, normalized)
+          updatedCount++
+        }
+      }
+      if (updatedCount > 0) {
+        log('info', `migrated ${updatedCount} items in course ${courseId}`)
+      }
+      migrated++
+    } catch (e) {
+      const courseId = course?._id?.toString() ?? 'unknown'
+      log('error', `migration failed for course ${courseId}`, e.message)
+      errors.push({ courseId, error: e.message })
+      failed++
+    }
+  }
+
+  log('info', `migration complete: ${migrated} succeeded, ${failed} failed`)
+  return { migrated, failed, errors }
+}
+
+async function fetchCourseContent (content, course) {
+  const config = await content.findOne({ _courseId: course._id, _type: 'config' }, { strict: false })
+  const items = await content.find({ _courseId: course._id, _type: { $nin: ['course', 'config'] } })
+  const result = [course]
+  if (config) result.push(config)
+  result.push(...items)
+  return result
+}

package/lib/utils/prebuildCache.js

@@ -0,0 +1,110 @@
+import { App, ensureDir, writeJson } from 'adapt-authoring-core'
+import AdaptCli from 'adapt-cli'
+import fs from 'node:fs/promises'
+import path from 'upath'
+import { copyFrameworkSource } from './copyFrameworkSource.js'
+import BuildCache from '../BuildCache.js'
+import { computePluginHash } from './computePluginHash.js'
+import { log } from './log.js'
+
+/**
+ * Eagerly populates the prebuilt cache for every (theme, menu) combination
+ * of installed plugins. Iterates serially: each iteration runs a full grunt
+ * build with the chosen theme/menu and caches the output.
+ *
+ * Idempotent — combos that already have a cache entry are skipped, so
+ * re-runs only build what's missing. Per-iteration failures are logged
+ * but don't abort the whole prebuild.
+ * @param {Object} options
+ * @param {String} options.buildDir Root build directory
+ * @param {String} options.frameworkDir Path to the adapt_framework source
+ * @return {Promise}
+ */
+export async function prebuildCache ({ buildDir, frameworkDir }) {
+  const app = App.instance
+  const cache = new BuildCache(path.join(buildDir, 'prebuilt-cache'))
+  const pluginHash = await computePluginHash(frameworkDir)
+
+  const contentplugin = await app.waitForModule('contentplugin')
+  const allPlugins = await contentplugin.find({})
+  const themes = allPlugins.filter(p => p.type === 'theme')
+  const menus = allPlugins.filter(p => p.type === 'menu')
+
+  if (!themes.length || !menus.length) {
+    throw new Error('Cannot prebuild cache: no theme or menu plugin installed')
+  }
+
+  log('info', 'CACHE', `starting eager prebuild for ${themes.length * menus.length} (theme,menu) combinations`)
+
+  for (const theme of themes) {
+    for (const menu of menus) {
+      try {
+        await prebuildOne({ buildDir, cache, pluginHash, theme, menu, allPlugins })
+      } catch (e) {
+        log('warn', 'CACHE', `eager prebuild failed for theme=${theme.name} menu=${menu.name}: ${e.message}`)
+        if (e.cmd) log('warn', 'CACHE', `cmd: ${e.cmd}`)
+        if (e.stderr) log('warn', 'CACHE', `stderr: ${e.stderr}`)
+      }
+    }
+  }
+
+  log('info', 'CACHE', 'eager prebuild complete')
+}
+
+async function prebuildOne ({ buildDir, cache, pluginHash, theme, menu, allPlugins }) {
+  const app = App.instance
+
+  if (await cache.has(pluginHash, theme.name, menu.name)) {
+    log('info', 'CACHE', `skipping cached combo theme=${theme.name} menu=${menu.name}`)
+    return
+  }
+
+  // Only one theme/menu can be active per build — drop the others so
+  // the framework's less:dev task doesn't glob multiple themes' LESS
+  // into a single adapt.css (see adapt_framework#3802).
+  const includedPlugins = allPlugins.filter(p =>
+    (p.type !== 'theme' && p.type !== 'menu') || p.name === theme.name || p.name === menu.name
+  )
+  const pluginNames = includedPlugins.map(p => p.name)
+
+  const tmpDir = path.join(buildDir, `_eager_cache_${Date.now()}_${theme.name}_${menu.name}`)
+  try {
+    log('info', 'CACHE', `building combo theme=${theme.name} menu=${menu.name}`)
+
+    await copyFrameworkSource({
+      destDir: tmpDir,
+      enabledPlugins: pluginNames,
+      linkNodeModules: true
+    })
+
+    const outputDir = path.join(tmpDir, 'build')
+    const buildCourseDir = path.join(outputDir, 'course', 'en')
+    await ensureDir(buildCourseDir)
+    await writeJson(path.join(outputDir, 'course', 'config.json'), {
+      _defaultLanguage: 'en',
+      _theme: theme.name,
+      _menu: menu.name,
+      _enabledPlugins: pluginNames
+    })
+    await writeJson(path.join(buildCourseDir, 'course.json'), {
+      title: '_eager_cache_build',
+      _latestTrackingId: 0
+    })
+    const cacheDir = path.join(buildDir, 'cache')
+    await ensureDir(cacheDir)
+
+    await AdaptCli.buildCourse({
+      cwd: tmpDir,
+      sourceMaps: true,
+      outputDir,
+      cachePath: path.join(cacheDir, '_eager_cache'),
+      logger: { log: (...args) => app.logger.log('debug', 'adapt-cli', ...args) }
+    })
+
+    if (!await cache.has(pluginHash, theme.name, menu.name)) {
+      await cache.populate(outputDir, pluginHash, theme.name, menu.name)
+    }
+  } finally {
+    await fs.rm(tmpDir, { recursive: true, force: true })
+  }
+}

package/lib/utils/readFrameworkPluginVersions.js

@@ -0,0 +1,21 @@
+import { readJson } from 'adapt-authoring-core'
+import { glob } from 'glob'
+import path from 'node:path'
+
+/**
+ * Reads bower.json files from the framework's src directory to build a list of plugin names and versions
+ * @param {String} frameworkDir Absolute path to the framework directory
+ * @returns {Promise<Array<{name: String, version: String}>>}
+ */
+export async function readFrameworkPluginVersions (frameworkDir) {
+  const srcDir = path.join(frameworkDir, 'src')
+  const bowerPaths = await glob([
+    'core/bower.json',
+    '{components,extensions,menu,theme}/*/bower.json'
+  ], { cwd: srcDir, absolute: true })
+  const plugins = await Promise.all(bowerPaths.map(async p => {
+    const { name, version } = await readJson(p)
+    return { name, version }
+  }))
+  return plugins
+}