@antora/assembler 1.0.0-alpha.9 → 1.0.0-beta.2

package/README.md

@@ -1,10 +1,13 @@
 # Antora Assembler
 
-A library for writing Antora extensions that combine multiple pages into one.
+Antora Assembler provides a JavaScript library and a set of Antora extensions for merging content in an Antora site into assembly files and converting those files to another format, such as PDF.
 
-Assembler works by constructing aggregate AsciiDoc documents from the pages based on the navigation tree per component version.
-It then invokes the specified callback to convert those documents to an output format.
-If a catalog is specified, Assembler pushes the converted files back into the catalog to be published as attachments alongside the other files in the site.
+The core of Assembler (this package) handles merging AsciiDoc content from multiple pages into an assembly.
+It repeats this process for each selected component version.
+It then delegates to a function or command (e.g., asciidoctor-pdf) designated by an exporter extension (a separate package) to convert the assembly files to an export format, such as PDF, resulting in an export.
+Assembler then publishes the export files along with the other files in the site.
+
+The exports can then be viewed offline, either by the browser or a reader application.
 
 [Antora](https://antora.org) is a modular static site generator designed for creating documentation sites from AsciiDoc documents.
 The Assembler extends the feature set of Antora by providing the foundation for page aggregation.

package/lib/assemble-content.js

@@ -1,46 +1,234 @@
 'use strict'
 
+const computeOut = require('./util/compute-out')
+const fs = require('node:fs')
+const { promises: fsp } = fs
+const LazyReadable = require('./util/lazy-readable')
 const loadConfig = require('./load-config')
-const produceAggregateDocuments = require('./produce-aggregate-documents')
+const ospath = require('node:path')
+const { posix: path } = ospath
+const produceAssemblyFiles = require('./produce-assembly-files')
 const PromiseQueue = require('./util/promise-queue')
-const runCommand = require('./util/run-command')
+const { stringify: toJSON } = JSON
 
-async function assembleContent (playbook, contentCatalog, convertDocument, { siteCatalog, configSource }) {
-  // Q: could we get ContentCatalog#getComponentVersionStartPage() in Antora core?
-  if (typeof contentCatalog.getComponentVersionStartPage !== 'function') {
-    contentCatalog.getComponentVersionStartPage = function (component, version) {
-      return this.resolvePage('index.adoc', { component, version })
-    }
-  }
+const invariably = { new: () => ({}), void: () => undefined }
+const PACKAGE_NAME = require('../package.json').name
+
+async function assembleContent (playbook, contentCatalog, converter, { configSource, navigationCatalog }) {
   const assemblerConfig = await loadConfig(playbook, configSource)
   if (!assemblerConfig) return [] // TODO consider removing and doing this another way
-  const generatorFunctions = this ? this.getFunctions() : {}
+  const context = isBound(this)
+    ? this
+    : {
+        getFunctions: invariably.new,
+        getLogger: invariably.void,
+        getVariables: invariably.new,
+      }
+  const generatorFunctions = context.getFunctions()
   const { loadAsciiDoc = require('@antora/asciidoc-loader') } = generatorFunctions
-  const aggregateDocuments = produceAggregateDocuments(loadAsciiDoc, contentCatalog, assemblerConfig)
-  if (!convertDocument) return aggregateDocuments
+  const targetBackend = converter == null ? undefined : (converter.backend ?? converter.extname?.slice(1))
+  const { assembly: assemblyConfig, build: buildConfig } = assemblerConfig
+  const { profile = targetBackend } = assemblyConfig
+  const intrinsicAttributes = { 'loader-assembler': '' }
+  if (profile) {
+    buildConfig.dir ??= ospath.join(playbook.dir ?? process.cwd(), `build/assembler-${profile}`)
+    intrinsicAttributes[`assembler-profile-${profile}`] = ''
+    intrinsicAttributes['assembler-profile'] = profile
+    if (targetBackend) {
+      intrinsicAttributes[`assembler-backend-${targetBackend}`] = ''
+      intrinsicAttributes['assembler-backend'] = targetBackend
+    }
+  } else {
+    buildConfig.dir ??= ospath.join(playbook.dir ?? process.cwd(), 'build/assembler')
+  }
+  Object.assign(assemblerConfig.asciidoc.attributes, intrinsicAttributes)
+  const assemblyFiles = produceAssemblyFiles(
+    loadAsciiDoc,
+    contentCatalog,
+    assemblerConfig,
+    createResolveAssemblyModel(context, contentCatalog, assemblyConfig, intrinsicAttributes, navigationCatalog)
+  )
+  const { convert = converter, extname: targetExtname, mediaType: targetMediaType } = converter ?? {}
+  if (!(convert && assemblyFiles.length)) return assemblyFiles
   const { publishSite: publishFiles = require('@antora/site-publisher') } = generatorFunctions
-  const buildConfig = assemblerConfig.build
-  await prepareWorkspace(publishFiles, aggregateDocuments, contentCatalog, buildConfig)
-  // TODO: pass more information to convertDocument so it doesn't have to compute internal stuff
-  // Q: don't we need to pass in the combined/resolved AsciiDoc attributes per file or component version?
+  await prepareWorkspace(publishFiles, assemblyFiles, contentCatalog, buildConfig)
+  const boundConvert = convert.bind(context)
   return new PromiseQueue({ concurrency: buildConfig.processLimit })
-    .add(aggregateDocuments.map((doc) => () => convertDocument.call(this, doc, buildConfig, runCommand)))
+    .add(
+      assemblyFiles.map((doc) => async () => {
+        const convertAttributes = prepareConvertAttributes(doc, targetExtname, buildConfig)
+        if (buildConfig.mkdirs) await fsp.mkdir(convertAttributes.outdir, { recursive: true, force: true })
+        return boundConvert(doc, convertAttributes, buildConfig).then(
+          (fileOrContents = new LazyReadable(() => fs.createReadStream(convertAttributes.outfile))) => {
+            return coerceToExportFormat(doc, targetBackend, targetExtname, targetMediaType, fileOrContents)
+          }
+        )
+      })
+    )
     .toPromise()
     .then((files) => {
-      if (buildConfig.publish && siteCatalog) siteCatalog.addFiles(files)
-      return files
+      if (!buildConfig.publish) {
+        for (const file of files) delete file.assembler.assembled
+        return files
+      }
+      const qualifyExports = buildConfig.qualifyExports
+      return files.map((file) => {
+        const pages = file.assembler.assembled.pages
+        delete file.assembler.assembled
+        file = contentCatalog.addFile(Object.assign(file, { out: computeOut.call(contentCatalog, file.src) }))
+        const extname = file.extname
+        const download = file.assembler.downloadStem + extname
+        if (qualifyExports) {
+          file.pub.url = '/' + (file.out.path = path.join(file.out.dirname, (file.out.basename = download)))
+        } else {
+          file.pub.download = download
+        }
+        pages.forEach((fragment, page) => {
+          const assemblerMeta = (page.assembler ??= {})
+          ;(assemblerMeta.exports ??= []).push({ fragment, file })
+          const extnameProp = extname.slice(1)
+          if (extnameProp in assemblerMeta) return
+          Object.defineProperty(assemblerMeta, extnameProp, {
+            configurable: true,
+            enumerable: true,
+            get: findFirstExportWithExtname.bind(assemblerMeta, extname),
+          })
+        })
+        return file
+      })
     })
 }
 
+function createResolveAssemblyModel (context, contentCatalog, shared, intrinsicAttributes, navigationCatalog) {
+  const { assemblerProfiles } = context.getVariables()
+  if (!assemblerProfiles) {
+    return (componentVersion) => {
+      const navigation =
+        navigationCatalog?.getNavigation(componentVersion.name, componentVersion.version) ?? componentVersion.navigation
+      return Object.assign({}, shared, { navigation })
+    }
+  }
+  const boundSendToLog = sendToLog.bind(context.getLogger ? context.getLogger(PACKAGE_NAME) : undefined)
+  const { buildNavigation = require('@antora/navigation-builder') } = context.getFunctions()
+  return (componentVersion) => {
+    const componentVersionProfiles = assemblerProfiles.get(componentVersion.version + '@' + componentVersion.name)
+    const overrides =
+      componentVersionProfiles?.get(intrinsicAttributes['assembler-profile']) ?? componentVersionProfiles?.get()
+    const { navFiles, messages } = overrides
+    const model = Object.assign({}, shared, overrides)
+    delete model.navFiles
+    delete model.messages
+    const navigationOverride = navigationCatalog?.getNavigation(componentVersion.name, componentVersion.version)
+    if (navigationOverride) return Object.assign(model, { navigation: navigationOverride })
+    messages?.forEach(boundSendToLog)
+    model.navigation = componentVersion.navigation
+    if (!navFiles) return model
+    if (!navFiles.length) return Object.assign(model, { navigation: [] })
+    const navigation_ = model.navigation
+    const asciidoc_ = componentVersion.asciidoc
+    componentVersion.asciidoc = Object.assign({}, asciidoc_, {
+      attributes: Object.assign({}, asciidoc_.attributes, intrinsicAttributes),
+    })
+    buildNavigation(
+      new Proxy(contentCatalog, {
+        get (target, property) {
+          const method = target[property]
+          if (property !== 'findBy') return method
+          return (criteria) => (toJSON(criteria) === '{"family":"nav"}' ? navFiles : method.call(target, criteria))
+        },
+      })
+    )
+    model.navigation = componentVersion.navigation
+    Object.assign(componentVersion, { asciidoc: asciidoc_, navigation: navigation_ })
+    return model
+  }
+}
+
+function prepareConvertAttributes (doc, targetExtname, buildConfig) {
+  const {
+    asciidoc: { attributes: docAttributes } = { attributes: {} },
+    extname: docfilesuffix,
+    path: reldocfile,
+    src: { family, relative },
+  } = doc
+  const { cwd = process.cwd(), dir = cwd } = buildConfig
+  const docname = family + '$' + relative.slice(0, relative.length - docfilesuffix.length)
+  const docfile = ospath.join(dir, reldocfile)
+  const docdir = dir
+  const imagesdir = ''
+  const outfile = docfile.slice(0, docfile.length - docfilesuffix.length) + targetExtname
+  const attributes = Object.assign({}, docAttributes, {
+    docdir,
+    docfile,
+    docfilesuffix,
+    'docname@': docname,
+    imagesdir,
+    outdir: ospath.dirname(docfile),
+    outfile,
+    outfilesuffix: targetExtname,
+    toArgs (optionFlag, command) {
+      const padCharRef = process.platform === 'win32' && command.startsWith('bundle exec ')
+      const args = []
+      for (let [name, val] of Object.entries(this)) {
+        if (val) {
+          val = `${name}=${padCharRef && typeof val.charAt === 'function' && val.charAt() === '&' ? ' ' : ''}${val}`
+        } else if (val === '') {
+          val = name
+        } else {
+          val = `!${name}${val === false ? '@' : ''}`
+        }
+        args.push('-a', val)
+      }
+      return args
+    },
+  })
+  return Object.defineProperty(attributes, 'toArgs', { enumerable: false })
+}
+
+function coerceToExportFormat (originalFile, targetBackend, targetExtname, targetMediaType, fileOrContents) {
+  const file =
+    fileOrContents == null || Buffer.isBuffer(fileOrContents) || typeof fileOrContents.pipe === 'function'
+      ? Object.assign(originalFile, { contents: fileOrContents })
+      : fileOrContents
+  ;(file.assembler ??= {}).backend = targetBackend
+  if (file.extname === targetExtname) return file
+  const sourcePath = file.path
+  const sourceExtname = file.extname
+  const relativeWithoutExtname = file.src.relative.slice(0, file.src.relative.length - sourceExtname.length)
+  const newPath = sourcePath.slice(0, sourcePath.length - sourceExtname.length) + targetExtname
+  Object.assign(file, { mediaType: (file.src.mediaType = targetMediaType), path: newPath })
+  file.src.basename = path.basename((file.src.relative = relativeWithoutExtname + (file.src.extname = targetExtname)))
+  return file
+}
+
+function findFirstExportWithExtname (extname) {
+  return this.exports.find((it) => it.file.extname === extname)
+}
+
 // TODO: if no workspace dir is defined, we shouldn't continue
-function prepareWorkspace (publishFiles, aggregateDocuments, contentCatalog, buildConfig) {
-  const { dir, clean, keepAggregateSource } = buildConfig
-  const files = contentCatalog.findBy({ family: 'image' }).filter(({ out }) => out?.assembled)
-  if (keepAggregateSource) {
-    files.push(...aggregateDocuments.map((file) => Object.assign(file, { out: { path: file.path } })))
+function prepareWorkspace (publishFiles, assemblyFiles, contentCatalog, buildConfig) {
+  const { dir, clean, keepSource } = buildConfig
+  const files = []
+  const outPaths = new Set()
+  for (const file of assemblyFiles) {
+    for (const asset of file.assembler.assembled.assets) {
+      if (outPaths.has(asset.out.path)) continue
+      files.push(asset)
+      outPaths.add(asset.out.path)
+    }
   }
-  // TODO: site publisher should accept a single catalog
-  return publishFiles({ output: { clean, dir } }, [{ getFiles: () => files }])
+  if (keepSource) files.push(...assemblyFiles.map((file) => Object.assign(file, { out: { path: file.path } })))
+  return publishFiles({ output: { clean, dir } }, { getFiles: () => files })
+}
+
+function isBound (obj) {
+  if (obj == null) return false
+  for (const p in obj) return true
+  return false
+}
+
+function sendToLog (levelAndArgs) {
+  if (this) this[levelAndArgs[0]].apply(this, levelAndArgs.slice(1))
 }
 
 module.exports = assembleContent

package/lib/configure.js

@@ -0,0 +1,96 @@
+'use strict'
+
+function configure (context, ...args) {
+  internalConfigure.apply(context, args)
+}
+
+function internalConfigure (converter, config = {}, providers = {}) {
+  this.once('componentsRegistered', ({ contentCatalog, assemblerProfiles }) => {
+    this.updateVariables(assemblerProfiles ? undefined : { assemblerProfiles: getAssemblerProfiles(contentCatalog) })
+  })
+
+  this.once('beforeProcess', ({ siteAsciiDocConfig }) => {
+    siteAsciiDocConfig.keepSource = true
+  })
+
+  this.once('navigationBuilt', async ({ playbook, contentCatalog }) => {
+    const { assembleContent = require('./assemble-content'), ...assembleContentConfig } = providers
+    assembleContentConfig.configSource ??= config.configFile
+    await assembleContent.call(this, playbook, contentCatalog, converter, assembleContentConfig)
+  })
+}
+
+function getAssemblerProfiles (contentCatalog, assemblerProfiles = new Map()) {
+  contentCatalog.getComponents().forEach((component) => {
+    component.versions.forEach((componentVersion) => {
+      const source = componentVersion.nav?.origin
+      const assemblerConfig = getAssemblerConfigFromDescriptor(source?.descriptor)
+      if (!assemblerConfig) return
+      const componentVersionRef = `${componentVersion.version}@${componentVersion.name}`
+      const filesByPath = componentVersion.files.reduce((accum, it) => accum.set(it.path, it), new Map())
+      const componentVersionProfiles = assemblerConfig.reduce((accum, entry) => {
+        const data = {}
+        const profile = entry.profile ?? undefined
+        if (accum.has(profile)) return accum
+        const nav = entry.nav
+        Object.entries(entry).forEach(([key, val]) => {
+          if (key === 'profile' || key === 'nav') return
+          const camelKey = key.toLowerCase().replace(/[_-]([a-z0-9])/g, (_, l, idx) => (idx ? l.toUpperCase() : l))
+          data[camelKey] = val
+        })
+        if (nav) {
+          data.navFiles = (nav.length ? [...new Set(nav)] : nav).reduce((navFiles, path_) => {
+            const navFile = filesByPath.get(path_)
+            if (navFile) {
+              navFiles.push(initNavFile(navFile, component.name, componentVersion.version, navFiles.length))
+            } else {
+              ;(data.messages ??= []).push([
+                'warn',
+                { source },
+                `Could not resolve nav file for ${profile || '<default>'} profile in ${componentVersionRef}: ${path_}`,
+              ])
+            }
+            return navFiles
+          }, [])
+        } else if (!Object.keys(data).length) {
+          return accum
+        }
+        return accum.set(profile, data)
+      }, new Map())
+      assemblerProfiles.set(componentVersionRef, componentVersionProfiles)
+    })
+  })
+  return assemblerProfiles
+}
+
+function getAssemblerConfigFromDescriptor (descriptor = {}) {
+  let assemblerConfig = descriptor.ext?.assembler
+  if (!assemblerConfig) return
+  if (Array.isArray(assemblerConfig)) {
+    if (!assemblerConfig.length) return
+  } else {
+    assemblerConfig = [assemblerConfig]
+  }
+  return assemblerConfig
+}
+
+function initNavFile (file, component, version, index) {
+  const src = Object.assign({}, file.src, { component, version, module: 'ROOT', family: 'nav' })
+  const filePathSegments = file.path.split('/')
+  let relativeStartIdx = 0
+  if (filePathSegments[0] === 'modules') {
+    relativeStartIdx += 1
+    if (filePathSegments.length > 2) {
+      relativeStartIdx += 1
+      src.module = filePathSegments[1]
+      if (filePathSegments.length > 3 && filePathSegments[2] === 'partials') {
+        relativeStartIdx += 1
+        src.family = 'partial'
+      }
+    }
+  }
+  src.relative = filePathSegments.slice(relativeStartIdx).join('/')
+  return Object.assign(new file.constructor(file), { nav: { index }, src })
+}
+
+module.exports = configure

package/lib/index.js

@@ -1,6 +1,6 @@
 'use strict'
 
 const assembleContent = require('./assemble-content')
-const runCommand = require('./util/run-command')
+const configure = require('./configure')
 
-module.exports = { assembleContent, runCommand }
+module.exports = { assembleContent, configure, configureAssembler: configure }

package/lib/load-config.js

@@ -10,14 +10,14 @@ function loadConfig (playbook, configSource = './antora-assembler.yml') {
     configSource.constructor === Object
       ? Promise.resolve(configSource)
       : fsp
-        .access((configSource = expandPath(configSource, { dot: playbook.dir })))
-        .then(
-          () => true,
-          () => false
-        )
-        .then((exists) =>
-          exists ? fsp.readFile(configSource).then((data) => camelCaseKeys(yaml.load(data), ['asciidoc'])) : {}
-        )
+          .access((configSource = expandPath(configSource, { dot: playbook.dir })))
+          .then(
+            () => true,
+            () => false
+          )
+          .then((exists) =>
+            exists ? fsp.readFile(configSource).then((data) => camelCaseKeys(yaml.load(data), ['asciidoc'])) : {}
+          )
   ).then((config) => {
     if (config.enabled === false) return undefined
     let asciidocAttrs
@@ -26,30 +26,47 @@ function loadConfig (playbook, configSource = './antora-assembler.yml') {
     } else if (!(asciidocAttrs = config.asciidoc.attributes)) {
       config.asciidoc.attributes = asciidocAttrs = {}
     }
-    if (!('doctype' in asciidocAttrs)) asciidocAttrs.doctype = 'book'
     if (!('revdate' in asciidocAttrs)) asciidocAttrs.revdate = getLocalDate()
     asciidocAttrs['page-partial'] = null
-    asciidocAttrs['loader-assembler'] = ''
-    if (config.componentVersions == null) {
-      config.componentVersions = ['*']
-    } else if (typeof config.componentVersions === 'string') {
-      config.componentVersions = config.componentVersions.split(', ')
+    const remapComponentVersionsKey = !('componentVersionFilter' in config)
+    const componentVersionFilter = (config.componentVersionFilter ??= {})
+    if (remapComponentVersionsKey && 'componentVersions' in config) {
+      componentVersionFilter.names = config.componentVersions
+      delete config.componentVersions
     }
-    if (!('rootLevel' in config)) config.rootLevel = 0
-    if (!('insertStartPage' in config)) config.insertStartPage = true
-    if (['discrete', 'fuse', 'enclose'].indexOf(config.sectionMergeStrategy) < 0) {
-      config.sectionMergeStrategy = 'discrete'
+    if (componentVersionFilter.names == null) {
+      componentVersionFilter.names = ['*']
+    } else if (typeof componentVersionFilter.names === 'string') {
+      componentVersionFilter.names = componentVersionFilter.names.split(', ')
     }
-    const build = config.build || (config.build = {})
+    const remapAssemblyKeys = !('assembly' in config)
+    const assembly = (config.assembly ??= {})
+    if (remapAssemblyKeys) {
+      for (const key of ['rootLevel', 'insertStartPage', 'sectionMergeStrategy']) {
+        if (!(key in config)) continue
+        assembly[key] = config[key]
+        delete config[key]
+      }
+    }
+    if (!('doctype' in assembly)) assembly.doctype = 'doctype' in asciidocAttrs ? asciidocAttrs.doctype : 'book'
+    delete asciidocAttrs.doctype
+    if (!('rootLevel' in assembly)) assembly.rootLevel = 0
+    if (!('insertStartPage' in assembly)) assembly.insertStartPage = true
+    if (['discrete', 'fuse', 'enclose'].indexOf(assembly.sectionMergeStrategy) < 0) {
+      assembly.sectionMergeStrategy = 'discrete'
+    }
+    const build = (config.build ??= {})
     if (build.dir === '$' + '{playbook.output.dir}') {
-      //build.dir = playbook.output.dir
       throw new Error('Not implemented')
-    } else {
-      build.dir = expandPath(build.dir || './build/assembler', { dot: playbook.dir })
     }
-    build.cwd = playbook.dir // use playbook.dir for the purpose of finding and loading require scripts
+    build.dir &&= expandPath(build.dir, { dot: playbook.dir })
+    build.cwd = playbook.dir // use playbook.dir for finding and loading require scripts
     if (!('clean' in build) && 'output' in playbook) build.clean = playbook.output.clean
     if (!('publish' in build)) build.publish = true
+    if ('keepAggregateSource' in build) {
+      build.keepSource = build.keepAggregateSource
+      delete build.keepAggregateSource
+    }
     if (!build.processLimit) {
       build.processLimit = 'processLimit' in build ? Infinity : Math.round(os.cpus().length * 0.5)
     }
@@ -57,7 +74,7 @@ function loadConfig (playbook, configSource = './antora-assembler.yml') {
   })
 }
 
-function camelCaseKeys (o, stopPaths = [], p) {
+function camelCaseKeys (o, stopPaths = [], p = undefined) {
   if (Array.isArray(o)) return o.map((it) => camelCaseKeys(it, stopPaths, p))
   if (o == null || o.constructor !== Object) return o
   const pathPrefix = p ? p + '.' : ''