@antora/assembler 1.0.0-alpha.10 → 1.0.0-alpha.12

package/README.md

@@ -1,10 +1,11 @@
 # Antora Assembler
 
-A library for writing Antora extensions that combine multiple pages into one.
+A library for merging the AsciiDoc from multiple pages into a single document (i.e., an assembly) and converting that document to another format such as PDF.
 
-Assembler works by constructing aggregate AsciiDoc documents from the pages based on the navigation tree per component version.
-It then invokes the specified callback to convert those documents to an output format.
-If a catalog is specified, Assembler pushes the converted files back into the catalog to be published as attachments alongside the other files in the site.
+Assembler works by constructing AsciiDoc documents from multiple pages using the navigation as a model.
+It then delegates to a converter (typically an external command) provided by an exporter extension to convert those assembly documents to an export format such as PDF.
+Finally, it publishes those exports along with the other files in the site.
+The exports can then be viewed offline, either by the browser or a reader application.
 
 [Antora](https://antora.org) is a modular static site generator designed for creating documentation sites from AsciiDoc documents.
 The Assembler extends the feature set of Antora by providing the foundation for page aggregation.

package/lib/assemble-content.js

@@ -1,46 +1,233 @@
 'use strict'
 
+const computeOut = require('./util/compute-out')
+const fs = require('node:fs')
+const { promises: fsp } = fs
+const LazyReadable = require('./util/lazy-readable')
 const loadConfig = require('./load-config')
-const produceAggregateDocuments = require('./produce-aggregate-documents')
+const ospath = require('node:path')
+const { posix: path } = ospath
+const produceAssemblyFiles = require('./produce-assembly-files')
 const PromiseQueue = require('./util/promise-queue')
-const runCommand = require('@antora/run-command-helper')
+const { stringify: toJSON } = JSON
 
-async function assembleContent (playbook, contentCatalog, convertDocument, { siteCatalog, configSource }) {
-  // Q: could we get ContentCatalog#getComponentVersionStartPage() in Antora core?
-  if (typeof contentCatalog.getComponentVersionStartPage !== 'function') {
-    contentCatalog.getComponentVersionStartPage = function (component, version) {
-      return this.resolvePage('index.adoc', { component, version })
-    }
-  }
+const invariably = { new: () => ({}), void: () => undefined }
+const PACKAGE_NAME = require('../package.json').name
+
+async function assembleContent (playbook, contentCatalog, converter, { configSource, navigationCatalog }) {
   const assemblerConfig = await loadConfig(playbook, configSource)
   if (!assemblerConfig) return [] // TODO consider removing and doing this another way
-  const generatorFunctions = this ? this.getFunctions() : {}
+  const context = isBound(this)
+    ? this
+    : {
+        getFunctions: invariably.new,
+        getLogger: invariably.void,
+        getVariables: invariably.new,
+      }
+  const generatorFunctions = context.getFunctions()
   const { loadAsciiDoc = require('@antora/asciidoc-loader') } = generatorFunctions
-  const aggregateDocuments = produceAggregateDocuments(loadAsciiDoc, contentCatalog, assemblerConfig)
-  if (!convertDocument) return aggregateDocuments
+  const targetBackend = converter == null ? undefined : (converter.backend ?? converter.extname?.slice(1))
+  const { assembly: assemblyConfig, build: buildConfig } = assemblerConfig
+  const { profile = targetBackend } = assemblyConfig
+  const intrinsicAttributes = { 'loader-assembler': '' }
+  if (profile) {
+    buildConfig.dir ??= ospath.join(playbook.dir ?? process.cwd(), `build/assembler-${profile}`)
+    intrinsicAttributes[`assembler-profile-${profile}`] = ''
+    intrinsicAttributes['assembler-profile'] = profile
+    if (targetBackend) {
+      intrinsicAttributes[`assembler-backend-${targetBackend}`] = ''
+      intrinsicAttributes['assembler-backend'] = targetBackend
+    }
+  } else {
+    buildConfig.dir ??= ospath.join(playbook.dir ?? process.cwd(), 'build/assembler')
+  }
+  Object.assign(assemblerConfig.asciidoc.attributes, intrinsicAttributes)
+  const assemblyFiles = produceAssemblyFiles(
+    loadAsciiDoc,
+    contentCatalog,
+    assemblerConfig,
+    createResolveAssemblyModel(context, contentCatalog, assemblyConfig, intrinsicAttributes, navigationCatalog)
+  )
+  const { convert = converter, extname: targetExtname, mediaType: targetMediaType } = converter ?? {}
+  if (!(convert && assemblyFiles.length)) return assemblyFiles
   const { publishSite: publishFiles = require('@antora/site-publisher') } = generatorFunctions
-  const buildConfig = assemblerConfig.build
-  await prepareWorkspace(publishFiles, aggregateDocuments, contentCatalog, buildConfig)
-  // TODO: pass more information to convertDocument so it doesn't have to compute internal stuff
-  // Q: don't we need to pass in the combined/resolved AsciiDoc attributes per file or component version?
+  await prepareWorkspace(publishFiles, assemblyFiles, contentCatalog, buildConfig)
+  const boundConvert = convert.bind(context)
   return new PromiseQueue({ concurrency: buildConfig.processLimit })
-    .add(aggregateDocuments.map((doc) => () => convertDocument.call(this, doc, buildConfig, runCommand)))
+    .add(
+      assemblyFiles.map((doc) => async () => {
+        const convertAttributes = prepareConvertAttributes(doc, targetExtname, buildConfig)
+        if (buildConfig.mkdirs) await fsp.mkdir(convertAttributes.outdir, { recursive: true, force: true })
+        return boundConvert(doc, convertAttributes, buildConfig).then(
+          (fileOrContents = new LazyReadable(() => fs.createReadStream(convertAttributes.outfile))) => {
+            return coerceToExportFormat(doc, targetBackend, targetExtname, targetMediaType, fileOrContents)
+          }
+        )
+      })
+    )
     .toPromise()
     .then((files) => {
-      if (buildConfig.publish && siteCatalog) siteCatalog.addFiles(files)
-      return files
+      if (!buildConfig.publish) {
+        for (const file of files) delete file.assembler.assembled
+        return files
+      }
+      const qualifyExports = buildConfig.qualifyExports
+      return files.map((file) => {
+        const pages = file.assembler.assembled.pages
+        delete file.assembler.assembled
+        file = contentCatalog.addFile(Object.assign(file, { out: computeOut.call(contentCatalog, file.src) }))
+        const extname = file.extname
+        const download = file.assembler.downloadStem + extname
+        if (qualifyExports) {
+          file.pub.url = '/' + (file.out.path = path.join(file.out.dirname, (file.out.basename = download)))
+        } else {
+          file.pub.download = download
+        }
+        pages.forEach((fragment, page) => {
+          const assemblerMeta = (page.assembler ??= {})
+          ;(assemblerMeta.exports ??= []).push({ fragment, file })
+          const extnameProp = extname.slice(1)
+          if (extnameProp in assemblerMeta) return
+          Object.defineProperty(assemblerMeta, extnameProp, {
+            configurable: true,
+            enumerable: true,
+            get: findFirstExportWithExtname.bind(assemblerMeta, extname),
+          })
+        })
+        return file
+      })
     })
 }
 
+function createResolveAssemblyModel (context, contentCatalog, defaults, intrinsicAttributes, navigationCatalog) {
+  const { assemblerProfiles } = context.getVariables()
+  const { insertStartPage, rootLevel, sectionMergeStrategy } = defaults
+  if (!assemblerProfiles) {
+    return (componentVersion) => {
+      const navigation =
+        navigationCatalog?.getNavigation(componentVersion.name, componentVersion.version) ?? componentVersion.navigation
+      return { insertStartPage, rootLevel, sectionMergeStrategy, navigation }
+    }
+  }
+  const boundSendToLog = sendToLog.bind(context.getLogger ? context.getLogger(PACKAGE_NAME) : undefined)
+  const { buildNavigation = require('@antora/navigation-builder') } = context.getFunctions()
+  return (componentVersion) => {
+    const componentVersionProfiles = assemblerProfiles.get(componentVersion.version + '@' + componentVersion.name)
+    const { navFiles, messages, ...model } = Object.assign(
+      { insertStartPage, rootLevel, sectionMergeStrategy },
+      componentVersionProfiles?.get(intrinsicAttributes['assembler-profile']) ?? componentVersionProfiles?.get()
+    )
+    const navigationOverride = navigationCatalog?.getNavigation(componentVersion.name, componentVersion.version)
+    if (navigationOverride) return Object.assign(model, { navigation: navigationOverride })
+    messages?.forEach(boundSendToLog)
+    model.navigation = componentVersion.navigation
+    if (!navFiles) return model
+    if (!navFiles.length) return Object.assign(model, { navigation: [] })
+    const navigation_ = model.navigation
+    const asciidoc_ = componentVersion.asciidoc
+    componentVersion.asciidoc = Object.assign({}, asciidoc_, {
+      attributes: Object.assign({}, asciidoc_.attributes, intrinsicAttributes),
+    })
+    buildNavigation(
+      new Proxy(contentCatalog, {
+        get (target, property) {
+          const method = target[property]
+          if (property !== 'findBy') return method
+          return (criteria) => (toJSON(criteria) === '{"family":"nav"}' ? navFiles : method.call(target, criteria))
+        },
+      })
+    )
+    model.navigation = componentVersion.navigation
+    Object.assign(componentVersion, { asciidoc: asciidoc_, navigation: navigation_ })
+    return model
+  }
+}
+
+function prepareConvertAttributes (doc, targetExtname, buildConfig) {
+  const {
+    asciidoc: { attributes: docAttributes } = { attributes: {} },
+    extname: docfilesuffix,
+    path: reldocfile,
+    src: { family, relative },
+  } = doc
+  const { cwd = process.cwd(), dir = cwd } = buildConfig
+  const docname = family + '$' + relative.slice(0, relative.length - docfilesuffix.length)
+  const docfile = ospath.join(dir, reldocfile)
+  const docdir = ospath.dirname(docfile)
+  const imagesdir = '/..'.repeat(reldocfile.split('/').length - 1).slice(1)
+  const outfile = docfile.slice(0, docfile.length - docfilesuffix.length) + targetExtname
+  const attributes = Object.assign({}, docAttributes, {
+    docdir,
+    docfile,
+    docfilesuffix,
+    'docname@': docname,
+    imagesdir,
+    outdir: docdir,
+    outfile,
+    outfilesuffix: targetExtname,
+    toArgs (optionFlag, command) {
+      const padCharRef = process.plaform === 'win32' && command.startsWith('bundle exec ')
+      const args = []
+      for (let [name, val] of Object.entries(this)) {
+        if (val) {
+          val = `${name}=${padCharRef && typeof val.charAt === 'function' && val.charAt() === '&' ? ' ' : ''}${val}`
+        } else if (val === '') {
+          val = name
+        } else {
+          val = `!${name}${val === false ? '@' : ''}`
+        }
+        args.push('-a', val)
+      }
+      return args
+    },
+  })
+  return Object.defineProperty(attributes, 'toArgs', { enumerable: false })
+}
+
+function coerceToExportFormat (originalFile, targetBackend, targetExtname, targetMediaType, fileOrContents) {
+  const file =
+    fileOrContents == null || Buffer.isBuffer(fileOrContents) || typeof fileOrContents.pipe === 'function'
+      ? Object.assign(originalFile, { contents: fileOrContents })
+      : fileOrContents
+  ;(file.assembler ??= {}).backend = targetBackend
+  if (file.extname === targetExtname) return file
+  const sourcePath = file.path
+  const sourceExtname = file.extname
+  const relativeWithoutExtname = file.src.relative.slice(0, file.src.relative.length - sourceExtname.length)
+  const newPath = sourcePath.slice(0, sourcePath.length - sourceExtname.length) + targetExtname
+  Object.assign(file, { mediaType: (file.src.mediaType = targetMediaType), path: newPath })
+  file.src.basename = path.basename((file.src.relative = relativeWithoutExtname + (file.src.extname = targetExtname)))
+  return file
+}
+
+function findFirstExportWithExtname (extname) {
+  return this.exports.find((it) => it.file.extname === extname)
+}
+
 // TODO: if no workspace dir is defined, we shouldn't continue
-function prepareWorkspace (publishFiles, aggregateDocuments, contentCatalog, buildConfig) {
-  const { dir, clean, keepAggregateSource } = buildConfig
-  const files = contentCatalog.findBy({ family: 'image' }).filter(({ out }) => out?.assembled)
-  if (keepAggregateSource) {
-    files.push(...aggregateDocuments.map((file) => Object.assign(file, { out: { path: file.path } })))
+function prepareWorkspace (publishFiles, assemblyFiles, contentCatalog, buildConfig) {
+  const { dir, clean, keepSource } = buildConfig
+  const files = []
+  const outPaths = new Set()
+  for (const file of assemblyFiles) {
+    for (const asset of file.assembler.assembled.assets) {
+      if (outPaths.has(asset.out.path)) continue
+      files.push(asset)
+      outPaths.add(asset.out.path)
+    }
   }
-  // TODO: site publisher should accept a single catalog
-  return publishFiles({ output: { clean, dir } }, [{ getFiles: () => files }])
+  if (keepSource) files.push(...assemblyFiles.map((file) => Object.assign(file, { out: { path: file.path } })))
+  return publishFiles({ output: { clean, dir } }, { getFiles: () => files })
+}
+
+function isBound (obj) {
+  if (obj == null) return false
+  for (const p in obj) return true
+  return false
+}
+
+function sendToLog (levelAndArgs) {
+  if (this) this[levelAndArgs[0]].apply(this, levelAndArgs.slice(1))
 }
 
 module.exports = assembleContent

package/lib/configure.js

@@ -0,0 +1,96 @@
+'use strict'
+
+function configure (context, ...args) {
+  internalConfigure.apply(context, args)
+}
+
+function internalConfigure (converter, config = {}, providers = {}) {
+  this.once('componentsRegistered', ({ contentCatalog, assemblerProfiles }) => {
+    this.updateVariables(assemblerProfiles ? undefined : { assemblerProfiles: getAssemblerProfiles(contentCatalog) })
+  })
+
+  this.once('beforeProcess', ({ siteAsciiDocConfig }) => {
+    siteAsciiDocConfig.keepSource = true
+  })
+
+  this.once('navigationBuilt', async ({ playbook, contentCatalog }) => {
+    const { assembleContent = require('./assemble-content'), ...assembleContentConfig } = providers
+    assembleContentConfig.configSource ??= config.configFile
+    await assembleContent.call(this, playbook, contentCatalog, converter, assembleContentConfig)
+  })
+}
+
+function getAssemblerProfiles (contentCatalog, assemblerProfiles = new Map()) {
+  contentCatalog.getComponents().forEach((component) => {
+    component.versions.forEach((componentVersion) => {
+      const source = componentVersion.nav?.origin
+      const assemblerConfig = getAssemblerConfigFromDescriptor(source?.descriptor)
+      if (!assemblerConfig) return
+      const componentVersionRef = `${componentVersion.version}@${componentVersion.name}`
+      const filesByPath = componentVersion.files.reduce((accum, it) => accum.set(it.path, it), new Map())
+      const componentVersionProfiles = assemblerConfig.reduce((accum, entry) => {
+        const data = {}
+        const profile = entry.profile ?? undefined
+        if (accum.has(profile)) return accum
+        const nav = entry.nav
+        Object.entries(entry).forEach(([key, val]) => {
+          if (key === 'profile' || key === 'nav') return
+          const camelKey = key.toLowerCase().replace(/[_-]([a-z0-9])/g, (_, l, idx) => (idx ? l.toUpperCase() : l))
+          data[camelKey] = val
+        })
+        if (nav) {
+          data.navFiles = (nav.length ? [...new Set(nav)] : nav).reduce((navFiles, path_) => {
+            const navFile = filesByPath.get(path_)
+            if (navFile) {
+              navFiles.push(initNavFile(navFile, component.name, componentVersion.version, navFiles.length))
+            } else {
+              ;(data.messages ??= []).push([
+                'warn',
+                { source },
+                `Could not resolve nav file for ${profile || '<default>'} profile in ${componentVersionRef}: ${path_}`,
+              ])
+            }
+            return navFiles
+          }, [])
+        } else if (!Object.keys(data).length) {
+          return accum
+        }
+        return accum.set(profile, data)
+      }, new Map())
+      assemblerProfiles.set(componentVersionRef, componentVersionProfiles)
+    })
+  })
+  return assemblerProfiles
+}
+
+function getAssemblerConfigFromDescriptor (descriptor = {}) {
+  let assemblerConfig = descriptor.ext?.assembler
+  if (!assemblerConfig) return
+  if (Array.isArray(assemblerConfig)) {
+    if (!assemblerConfig.length) return
+  } else {
+    assemblerConfig = [assemblerConfig]
+  }
+  return assemblerConfig
+}
+
+function initNavFile (file, component, version, index) {
+  const src = Object.assign({}, file.src, { component, version, module: 'ROOT', family: 'nav' })
+  const filePathSegments = file.path.split('/')
+  let relativeStartIdx = 0
+  if (filePathSegments[0] === 'modules') {
+    relativeStartIdx += 1
+    if (filePathSegments.length > 2) {
+      relativeStartIdx += 1
+      src.module = filePathSegments[1]
+      if (filePathSegments.length > 3 && filePathSegments[2] === 'partials') {
+        relativeStartIdx += 1
+        src.family = 'partial'
+      }
+    }
+  }
+  src.relative = filePathSegments.slice(relativeStartIdx).join('/')
+  return Object.assign(new file.constructor(file), { nav: { index }, src })
+}
+
+module.exports = configure

package/lib/index.js

@@ -1,5 +1,6 @@
 'use strict'
 
 const assembleContent = require('./assemble-content')
+const configure = require('./configure')
 
-module.exports = { assembleContent }
+module.exports = { assembleContent, configure, configureAssembler: configure }

package/lib/load-config.js

@@ -29,25 +29,43 @@ function loadConfig (playbook, configSource = './antora-assembler.yml') {
     if (!('doctype' in asciidocAttrs)) asciidocAttrs.doctype = 'book'
     if (!('revdate' in asciidocAttrs)) asciidocAttrs.revdate = getLocalDate()
     asciidocAttrs['page-partial'] = null
-    asciidocAttrs['loader-assembler'] = ''
-    if (config.componentVersions == null) {
-      config.componentVersions = ['*']
-    } else if (typeof config.componentVersions === 'string') {
-      config.componentVersions = config.componentVersions.split(', ')
+    const remapComponentVersionsKey = !('componentVersionFilter' in config)
+    const componentVersionFilter = (config.componentVersionFilter ??= {})
+    if (remapComponentVersionsKey && 'componentVersions' in config) {
+      componentVersionFilter.names = config.componentVersions
+      delete config.componentVersions
     }
-    if (!('rootLevel' in config)) config.rootLevel = 0
-    if (!('insertStartPage' in config)) config.insertStartPage = true
-    if (['discrete', 'fuse', 'enclose'].indexOf(config.sectionMergeStrategy) < 0) {
-      config.sectionMergeStrategy = 'discrete'
+    if (componentVersionFilter.names == null) {
+      componentVersionFilter.names = ['*']
+    } else if (typeof componentVersionFilter.names === 'string') {
+      componentVersionFilter.names = componentVersionFilter.names.split(', ')
     }
-    const build = config.build || (config.build = {})
+    const remapAssemblyKeys = !('assembly' in config)
+    const assembly = (config.assembly ??= {})
+    if (remapAssemblyKeys) {
+      for (const key of ['rootLevel', 'insertStartPage', 'sectionMergeStrategy']) {
+        if (!(key in config)) continue
+        assembly[key] = config[key]
+        delete config[key]
+      }
+    }
+    if (!('rootLevel' in assembly)) assembly.rootLevel = 0
+    if (!('insertStartPage' in assembly)) assembly.insertStartPage = true
+    if (['discrete', 'fuse', 'enclose'].indexOf(assembly.sectionMergeStrategy) < 0) {
+      assembly.sectionMergeStrategy = 'discrete'
+    }
+    const build = (config.build ??= {})
     if (build.dir === '$' + '{playbook.output.dir}') {
       throw new Error('Not implemented')
     }
-    build.dir = expandPath(build.dir || './build/assembler', { dot: playbook.dir })
-    build.cwd = playbook.dir // use playbook.dir for the purpose of finding and loading require scripts
+    build.dir &&= expandPath(build.dir, { dot: playbook.dir })
+    build.cwd = playbook.dir // use playbook.dir for finding and loading require scripts
     if (!('clean' in build) && 'output' in playbook) build.clean = playbook.output.clean
     if (!('publish' in build)) build.publish = true
+    if ('keepAggregateSource' in build) {
+      build.keepSource = build.keepAggregateSource
+      delete build.keepAggregateSource
+    }
     if (!build.processLimit) {
       build.processLimit = 'processLimit' in build ? Infinity : Math.round(os.cpus().length * 0.5)
     }

package/lib/{produce-aggregate-document.js → produce-assembly-file.js}

@@ -1,58 +1,55 @@
 'use strict'
 
+const createAsciiDocFile = require('./util/create-asciidoc-file')
 const path = require('node:path/posix')
 const sanitize = require('./util/sanitize')
 const unconvertInlineAsciiDoc = require('./util/unconvert-inline-asciidoc')
 
-function produceAggregateDocument (
+const BuiltInNamedEntities = { amp: '&', apos: "'", gt: '>', lt: '<', nbsp: ' ', quot: '"' }
+const CharRefRx = /&(?:([a-z][a-z]+\d{0,2})|#(?:(\d{2,6})|x([a-z\d]{2,5})));/g
+
+function produceAssemblyFile (
   loadAsciiDoc,
   contentCatalog,
   componentVersion,
   outline,
+  files,
   doctype,
-  pages,
   asciidocConfig,
   mutableAttributes,
   sectionMergeStrategy = 'discrete'
 ) {
-  const pagesInOutline = selectPagesInOutline(outline, pages)
+  const pagesByUrl = files.reduce((map, it) => (it.src.family === 'page' ? map.set(it.pub.url, it) : map), new Map())
+  if (outline.urlType === 'internal' && !pagesByUrl.get(outline.url) && !(outline.items || []).length) return
+  const pagesInOutline = selectPagesInOutline(outline, pagesByUrl)
   const navtitle = outline.content
-  const templateFile = contentCatalog.addFile({
-    src: {
-      component: componentVersion.name,
-      version: componentVersion.version,
-      module: 'ROOT',
-      family: 'page',
-      relative: generateSlug(navtitle),
-    },
-  })
-  const { dir: outDir, name: outName } = path.parse(templateFile.out.path)
-  const path_ = outName === templateFile.src.relative ? path.join(outDir, outName + '.adoc') : outDir + '.adoc'
-  contentCatalog.removeFile(templateFile)
-  const header = buildAsciiDocHeader(componentVersion, navtitle, doctype)
-  const body = aggregateAsciiDoc(
+  const buffer = mergeAsciiDoc(
     loadAsciiDoc,
     contentCatalog,
-    header,
+    buildAsciiDocHeader(componentVersion, navtitle, doctype),
     componentVersion,
     outline,
+    files,
     pagesInOutline,
     asciidocConfig,
     mutableAttributes,
     sectionMergeStrategy
   )
-  return new templateFile.constructor({
-    aggregate: true,
+  const level = outline.index ? 0 : 1
+  const stem = level === 0 ? 'index' : generateSlug(navtitle)
+  const downloadStem = [componentVersion.name, componentVersion.version, level === 0 ? '' : stem]
+    .filter((it) => it)
+    .join('-')
+  return createAsciiDocFile(contentCatalog, {
     asciidoc: asciidocConfig,
-    contents: Buffer.from([...header, ...body].join('\n') + '\n'),
-    mediaType: 'text/asciidoc',
-    path: path_,
+    assembler: { assembled: pagesInOutline.assembled, downloadStem, level },
+    contents: Buffer.from(buffer.join('\n') + '\n'),
     src: {
       component: componentVersion.name,
       version: componentVersion.version,
-      basename: path.basename(path_),
-      stem: path.basename(path_, '.adoc'),
-      extname: '.adoc',
+      module: 'ROOT',
+      family: 'export',
+      relative: stem + '.adoc',
     },
   })
 }
@@ -77,25 +74,25 @@ function buildAsciiDocHeader (componentVersion, navtitle, doctype = 'book') {
   ]
 }
 
-function selectPagesInOutline (outlineEntry, pages) {
-  const page = outlineEntry.urlType === 'internal' ? pages.find((it) => it.pub.url === outlineEntry.url) : undefined
-  return (outlineEntry.items || []).reduce(
-    (accum, item) => selectPagesInOutline(item, pages).forEach((v, k) => accum.set(k, v)) ?? accum,
-    new Map(
-      page && [
-        [`${page.src.module === 'ROOT' ? '' : page.src.module + ':'}${page.src.relative}`, page],
-        [page.pub.url, page],
-      ]
-    )
-  )
+function selectPagesInOutline (outlineEntry, pagesByUrl, accum) {
+  accum ??= Object.assign(new Map(), { assembled: { pages: new Map(), assets: new Set() } })
+  const page = outlineEntry.urlType === 'internal' ? pagesByUrl.get(outlineEntry.url) : undefined
+  if (page) {
+    accum
+      .set(`${page.src.module === 'ROOT' ? '' : page.src.module + ':'}${page.src.relative}`, page)
+      .set(page.pub.url, page)
+  }
+  for (const item of outlineEntry.items || []) selectPagesInOutline(item, pagesByUrl, accum)
+  return accum
 }
 
-function aggregateAsciiDoc (
+function mergeAsciiDoc (
   loadAsciiDoc,
   contentCatalog,
-  header,
+  buffer,
   componentVersion,
   outlineEntry,
+  files,
   pagesInOutline,
   asciidocConfig,
   mutableAttributes,
@@ -103,7 +100,6 @@ function aggregateAsciiDoc (
   lastComponentVersion = componentVersion,
   level = 0
 ) {
-  const buffer = []
   // TODO: we could try to be smart about it and make sure the page with fragment is included at least once
   if (outlineEntry.hash) return buffer
   const { content: navtitle, items = [], unresolved, urlType, url } = outlineEntry
@@ -116,7 +112,7 @@ function aggregateAsciiDoc (
   })(asciidocConfig.attributes['site-url'])
   // FIXME: ideally, resource ID would be stored in navigation so we can look up the page more efficiently
   let page = urlType === 'internal' && !unresolved ? pagesInOutline.get(url) : undefined
-  if (page && pagesInOutline.aggregated?.includes(page)) page = undefined
+  if (page && pagesInOutline.assembled.pages.has(page)) page = undefined
   if (page) {
     let contents = page.src.contents
     if (contents == null) return buffer
@@ -129,21 +125,19 @@ function aggregateAsciiDoc (
         .replace(/^(?:[ \t]*\r\n?|[ \t]*\n)+/, '')
         .trimRight()
     )
-    ;(pagesInOutline.aggregated ??= []).push(page)
-    page = new page.constructor(Object.assign({}, page, { contents, mediaType: 'text/asciidoc' }))
     const { component, version, module: module_, relative, origin } = page.src
     const topicPrefix = ~relative.indexOf('/') ? path.dirname(relative) + '/' : ''
-    const doc = loadAsciiDoc(page, contentCatalog, asciidocConfig)
+    const pageAsAsciiDoc = new page.constructor(Object.assign({}, page, { contents, mediaType: page.src.mediaType }))
+    const doc = loadAsciiDoc(pageAsAsciiDoc, contentCatalog, asciidocConfig)
     const refs = doc.getCatalog().refs
     // NOTE: in Antora, docname is relative src path from module without file extension
     const docname = doc.getAttribute('docname')
     const docnameForId = docname.replace(/[/]/g, '::').replace(/[.]/g, '-')
-    const scopeId = component !== componentVersion.name
-    const idprefix =
-      (scopeId ? component + ':' : '') +
-      (module_ === 'ROOT' ? (scopeId ? ':' : '') : module_ + ':') +
-      docnameForId +
-      ':::'
+    const qualifyId = component !== componentVersion.name
+    const idscope =
+      (qualifyId ? component + ':' : '') + (module_ === 'ROOT' ? (qualifyId ? ':' : '') : module_ + ':') + docnameForId
+    const idprefix = idscope + ':::'
+    let pageFragment = ''
     buffer.push('')
     buffer.push(`:docname: ${docname}`)
     if (component !== lastComponentVersion.name) {
@@ -174,17 +168,18 @@ function aggregateAsciiDoc (
       if (level === 1 && navtitlePlain === componentVersion.title) {
         level--
       } else {
+        pageFragment = `#${idscope}`
         let hlevel = level + 1
         if (hlevel > 6) {
           hlevel = 6
-          buffer.push(`[discrete#${idprefix}]`)
+          buffer.push(`[discrete${pageFragment}]`)
         } else {
-          buffer.push(`[#${idprefix}]`)
+          buffer.push(`[${pageFragment}]`)
         }
         buffer.push(`${'='.repeat(hlevel)} ${navtitleAsciiDoc}`)
       }
     } else {
-      header.unshift(`[#${idprefix}]`)
+      buffer.unshift(`[#${idscope}]`)
     }
     if (sectionMergeStrategy === 'enclose' && hasItems && doc.hasSections()) {
       enclosed = true
@@ -213,6 +208,7 @@ function aggregateAsciiDoc (
       buffer.push(`${'='.repeat(hlevel)} ${overviewTitle}`)
       if (toggleSectids) buffer.push(':sectids:')
     }
+    pagesInOutline.assembled.pages.set(page, pageFragment)
     const lines = doc.getSourceLines()
     const ignoreLines = []
     // TODO: think more about when multipart is allowed; perhaps configurable
@@ -335,7 +331,7 @@ function aggregateAsciiDoc (
             .replace(/[/]/g, '::')
             .replace(/\.adoc$/, '')
             .replace(/[.]/g, '-')
-          const refid = `${pagePart}:::${fragment}`
+          const refid = fragment ? `${pagePart}:::${fragment}` : pagePart
           return `<<${refid}${text && text !== targetPage.title ? ',' + text.replace(/\\]/g, ']') : ''}>>`
         })
       }
@@ -360,7 +356,7 @@ function aggregateAsciiDoc (
             const image = contentCatalog.resolveResource(target, page.src, 'image', ['image'])
             // TODO: handle (or report) unresolved image better
             if (image?.out) {
-              image.out.assembled = true
+              pagesInOutline.assembled.assets.add(image)
               return `image:${image.out.path.replace(/_/g, '{underscore}')}[${attrlist}]`
             }
           }
@@ -420,7 +416,7 @@ function aggregateAsciiDoc (
               // FIXME: handle (or report) case when image is not resolved
               if (image?.out) {
                 const boxedAttrlist = line.slice(line.indexOf('['))
-                image.out.assembled = true
+                pagesInOutline.assembled.assets.add(image)
                 lines[idx] = `${prefix}image::${image.out.path}${boxedAttrlist}`
               }
             }
@@ -433,7 +429,10 @@ function aggregateAsciiDoc (
         if (block.getId()) rewriteStyleAttribute(block, lines, idx, idprefix)
       }
     })
-    buffer.push(...lines.filter((it) => it !== undefined))
+    safePush(
+      buffer,
+      lines.filter((it) => it !== undefined)
+    )
     const attributeEntries = Object.entries(doc.source_header_attributes?.$$smap || {})
     if (attributeEntries.length) {
       const resolvedAttributeEntries = attributeEntries.reduce(
@@ -461,7 +460,7 @@ function aggregateAsciiDoc (
         },
         ['']
       )
-      if (resolvedAttributeEntries.length > 1) buffer.push(...resolvedAttributeEntries)
+      if (resolvedAttributeEntries.length > 1) safePush(buffer, resolvedAttributeEntries)
     }
   } else if (level) {
     if (level === 1 && navtitlePlain === componentVersion.title) {
@@ -486,7 +485,7 @@ function aggregateAsciiDoc (
       if (urlType === 'external') {
         sectionTitle = `${url}[${navtitleAsciiDoc.replace(/\]/g, '\\]')}]`
       } else if (urlType === 'internal' && !unresolved && siteUrl) {
-        const resource = contentCatalog.getFiles().find((it) => it.out && it.pub.url === url)
+        const resource = files.find((it) => it.pub.url === url)
         if (resource) sectionTitle = `${siteUrl}${resource.pub.url}[${navtitleAsciiDoc.replace(/\]/g, '\\]')}]`
       }
       let hlevel = level + 1
@@ -508,20 +507,19 @@ function aggregateAsciiDoc (
       ? items.slice(1)
       : items
     ).forEach((item) => {
-      buffer.push(
-        ...aggregateAsciiDoc(
-          loadAsciiDoc,
-          contentCatalog,
-          header,
-          componentVersion,
-          item,
-          pagesInOutline,
-          asciidocConfig,
-          mutableAttributes,
-          sectionMergeStrategy,
-          lastComponentVersion,
-          nextLevel
-        )
+      mergeAsciiDoc(
+        loadAsciiDoc,
+        contentCatalog,
+        buffer,
+        componentVersion,
+        item,
+        files,
+        pagesInOutline,
+        asciidocConfig,
+        mutableAttributes,
+        sectionMergeStrategy,
+        lastComponentVersion,
+        nextLevel
       )
     })
   }
@@ -531,9 +529,14 @@ function aggregateAsciiDoc (
 function generateSlug (title) {
   return title
     .toLowerCase()
-    .replace(/&.+?;|[^ \p{Alpha}0-9_\-.]/gu, '')
-    .replace(/[ _.]/g, '-')
-    .replace(/--+/g, '-')
+    .replace(/<[^>]+>/g, '')
+    .replace(CharRefRx, (_, name, dec, hex) => {
+      if (name) return BuiltInNamedEntities[name] ?? '?'
+      return String.fromCharCode(dec ? parseInt(dec, 10) : parseInt(hex, 16))
+    })
+    .replace(/[\x27\u2019]/g, '')
+    .replace(/[^\p{Alpha}0-9\-]/gu, '-')
+    .replace(/^-+|-+$|(-)-+/g, '$1')
 }
 
 function fixSectionLevels (sections, multipart) {
@@ -602,4 +605,14 @@ function getObjectId (obj) {
   return global.Opal.uid()
 }
 
-module.exports = produceAggregateDocument
+function safePush (onto, entries) {
+  try {
+    onto.push(...entries)
+  } catch (err) {
+    /* istanbul ignore if */
+    if (!(err instanceof RangeError)) throw err
+    for (const e of entries) onto.push(e)
+  }
+}
+
+module.exports = produceAssemblyFile

package/lib/{produce-aggregate-documents.js → produce-assembly-files.js}

@@ -1,22 +1,31 @@
 'use strict'
 
+const createAsciiDocFile = require('./util/create-asciidoc-file')
 const filterComponentVersions = require('./filter-component-versions')
-const produceAggregateDocument = require('./produce-aggregate-document')
+const produceAssemblyFile = require('./produce-assembly-file')
 const selectMutableAttributes = require('./select-mutable-attributes')
 
 const IMAGE_MACRO_RX = /^image::?(.+?)\[(.*?)\]$/
 
-function produceAggregateDocuments (loadAsciiDoc, contentCatalog, assemblerConfig) {
-  const { insertStartPage, rootLevel, sectionMergeStrategy, asciidoc: assemblerAsciiDocConfig } = assemblerConfig
+function produceAssemblyFiles (loadAsciiDoc, contentCatalog, assemblerConfig, resolveAssemblyModel) {
+  const { asciidoc: assemblerAsciiDocConfig, assembly: assemblyConfig } = assemblerConfig
+  resolveAssemblyModel ??= (componentVersion) => ({
+    insertStartPage: assemblyConfig.insertStartPage,
+    rootLevel: assemblyConfig.rootLevel,
+    sectionMergeStrategy: assemblyConfig.sectionMergeStrategy,
+    navigation: componentVersion.navigation,
+  })
   const assemblerAsciiDocAttributes = Object.assign({}, assemblerAsciiDocConfig.attributes)
   const { doctype, revdate, 'source-highlighter': sourceHighlighter } = assemblerAsciiDocAttributes
   delete assemblerAsciiDocAttributes.doctype
   delete assemblerAsciiDocAttributes.revdate
   delete assemblerAsciiDocAttributes['source-highlighter']
-  return filterComponentVersions(contentCatalog.getComponents(), assemblerConfig.componentVersions).reduce(
+  const publishableFiles = contentCatalog.getFiles().filter((file) => file.out)
+  return filterComponentVersions(contentCatalog.getComponents(), assemblerConfig.componentVersionFilter.names).reduce(
     (accum, componentVersion) => {
-      const { name: componentName, version, title, navigation } = componentVersion
+      const { insertStartPage, rootLevel, sectionMergeStrategy, navigation } = resolveAssemblyModel(componentVersion)
       if (!navigation) return accum
+      const { name: componentName, version, title } = componentVersion
       const componentVersionAsciiDocConfig = getAsciiDocConfigWithAsciidoctorReducerExtension(componentVersion)
       const mergedAsciiDocConfig = Object.assign({}, componentVersionAsciiDocConfig, {
         attributes: Object.assign({ revdate }, componentVersionAsciiDocConfig.attributes, assemblerAsciiDocAttributes),
@@ -32,37 +41,44 @@ function produceAggregateDocuments (loadAsciiDoc, contentCatalog, assemblerConfi
         image.out.assembled = true
       })
       const rootEntry = { content: title }
-      let startPage = contentCatalog.getComponentVersionStartPage(componentName, version)
+      let startPage =
+        'startPage' in componentVersion
+          ? componentVersion.startPage
+          : contentCatalog.resolvePage('index.adoc', { component: componentName, version })
       if (startPage && startPage.src.component === componentName && startPage.src.version === version) {
-        if (insertStartPage && !includedInNav(navigation, startPage.pub.url)) {
-          Object.assign(rootEntry, { url: startPage.pub.url, urlType: 'internal' })
+        if (insertStartPage) {
+          const navtitle = startPage.asciidoc?.navtitle || rootEntry.content
+          Object.assign(rootEntry, { navtitle, url: startPage.pub.url, urlType: 'internal' })
         }
       } else {
         // Q: should we always use a reference page as startPage for computing mutableAttributes?
-        startPage = createFile({
-          component: componentVersion.name,
-          version: componentVersion.version,
-          relative: '.reference-page.adoc',
-          origin: (componentVersion.origins || [])[0],
+        startPage = createAsciiDocFile(contentCatalog, {
+          src: {
+            component: componentVersion.name,
+            version: componentVersion.version,
+            module: 'ROOT',
+            family: 'page',
+            relative: '.start-page.adoc',
+            origin: (componentVersion.origins || [])[0],
+          },
         })
       }
       const mutableAttributes = selectMutableAttributes(loadAsciiDoc, contentCatalog, startPage, mergedAsciiDocConfig)
       delete mutableAttributes.doctype
-      accum = accum.concat(
-        prepareOutlines(navigation, rootEntry, rootLevel).map((outline) =>
-          produceAggregateDocument(
-            loadAsciiDoc,
-            contentCatalog,
-            componentVersion,
-            outline,
-            doctype,
-            contentCatalog.getPages((page) => page.out),
-            mergedAsciiDocConfig,
-            mutableAttributes,
-            sectionMergeStrategy
-          )
+      prepareOutlines(navigation, rootEntry, rootLevel).forEach((outline) => {
+        const assemblyFile = produceAssemblyFile(
+          loadAsciiDoc,
+          contentCatalog,
+          componentVersion,
+          outline,
+          publishableFiles,
+          doctype,
+          mergedAsciiDocConfig,
+          mutableAttributes,
+          sectionMergeStrategy
         )
-      )
+        if (assemblyFile) accum.push(assemblyFile)
+      })
       mergedAsciiDocAttributes.doctype = doctype
       sourceHighlighter
         ? (mergedAsciiDocAttributes['source-highlighter'] = sourceHighlighter)
@@ -73,33 +89,6 @@ function produceAggregateDocuments (loadAsciiDoc, contentCatalog, assemblerConfi
   )
 }
 
-function createFile (src) {
-  const familySegment = (src.family ??= 'page') + 's'
-  const relativeSegments = src.relative.split('/')
-  const segments = ['modules', (src.module ??= 'ROOT'), familySegment, ...relativeSegments]
-  const path = segments.join('/')
-  const moduleRootPath = Array(relativeSegments.length - 1)
-    .fill('..')
-    .join('/')
-  const outPath = [
-    src.component === 'ROOT' ? '' : src.component,
-    src.version,
-    src.module === 'ROOT' ? '' : src.module,
-    src.family === 'page' ? '' : '_' + familySegment,
-    src.family === 'page' ? src.relative.replace(/\.adoc$/, '.html') : src.relative,
-  ]
-    .filter((it) => it)
-    .join('/')
-  return {
-    path,
-    dirname: path.slice(0, path.lastIndexOf('/')),
-    contents: src.contents ?? Buffer.alloc(0),
-    src,
-    out: { path: outPath },
-    pub: { url: '/' + outPath, moduleRootPath },
-  }
-}
-
 function getAsciiDocConfigWithAsciidoctorReducerExtension (componentVersion) {
   const asciidoctorReducerExtension = require('@asciidoctor/reducer') // NOTE: must be required lazily
   const asciidocConfig = componentVersion.asciidoc
@@ -123,21 +112,27 @@ function isResourceRef (target) {
   return ~target.indexOf(':') && !(~target.indexOf('://') || (target.startsWith('data:') && ~target.indexOf(',')))
 }
 
-// when root level is 0, merge the navigation into the rootEntry
-// when root level is 1, create navigation per navigation menu
-// in this case, if there's only a single navigation menu with no title, promote each top-level item to a menu
 function prepareOutlines (navigation, rootEntry, rootLevel) {
-  if (rootLevel === 0 || navigation.length === 1) {
-    let navBranch
-    if (navigation.length === 1) {
-      navBranch = navigation[0]
+  const singleEntry = navigation.length === 1
+  const items = navigation.reduce((accum, it) => {
+    if (!it.content || (singleEntry && it.url ? it.url === rootEntry.url : it.content === rootEntry.content)) {
+      accum.push(...it.items)
     } else {
-      const items = navigation.reduce((accum, it) => accum.concat(it.content ? it : it.items), [])
-      navBranch = items.length ? { items } : {}
+      accum.push(it)
+    }
+    return accum
+  }, [])
+  if (rootLevel === 0) {
+    if (rootEntry.url && includedInNav(items, rootEntry.url)) {
+      for (const p of ['url', 'urlType']) delete rootEntry[p]
     }
-    return rootLevel === 0 || navBranch.content ? [Object.assign(rootEntry, navBranch)] : navBranch.items
+    return [Object.assign(rootEntry, { index: true, items })]
+  }
+  if (rootEntry.url && !includedInNav(items, rootEntry.url)) {
+    rootEntry.content = singleEntry && rootEntry.url === navigation[0].url ? navigation[0].content : rootEntry.navtitle
+    items.unshift(rootEntry)
   }
-  return navigation.reduce((navTree, it) => navTree.concat(it.content ? it : it.items), [rootEntry])
+  return items
 }
 
-module.exports = produceAggregateDocuments
+module.exports = produceAssemblyFiles

package/lib/util/compute-out.js

@@ -0,0 +1,57 @@
+'use strict'
+
+const { posix: path } = require('node:path')
+
+function computeOut (src) {
+  const { component, version, module: module_, family, relative } = src
+  const outRelative = family === 'page' ? relative.replace(/\.adoc$/, '.html') : relative
+  const { dir: dirname, base: basename, ext: extname, name: stem } = path.parse(outRelative)
+  const componentVersion = this.getComponentVersion(component, version)
+  const versionSegment =
+    'activeVersionSegment' in componentVersion
+      ? componentVersion.activeVersionSegment
+      : resolveActiveVersionSegment.call(this, component, version)
+  const outDirSegments = []
+  const moduleRootPathSegments = []
+  if (component !== 'ROOT') outDirSegments.push(component)
+  if (versionSegment) outDirSegments.push(versionSegment)
+  if (module_ !== 'ROOT') outDirSegments.push(module_)
+  const outModuleDirSegments = outDirSegments.slice()
+  if (family !== 'page') {
+    outDirSegments.push(`_${family}s`)
+    moduleRootPathSegments.push('..')
+  }
+  if (dirname) {
+    outDirSegments.push(dirname)
+    for (const _ of dirname.split('/')) moduleRootPathSegments.push('..')
+  }
+  const rootPathSegments = moduleRootPathSegments.slice()
+  for (const _ of outModuleDirSegments) rootPathSegments.push('..')
+  const outDirname = outDirSegments.join('/')
+  const result = {
+    dirname: outDirname,
+    basename,
+    path: outDirname + '/' + basename,
+    moduleRootPath: moduleRootPathSegments.length ? moduleRootPathSegments.join('/') : '.',
+    rootPath: rootPathSegments.length ? rootPathSegments.join('/') : '.',
+  }
+  return result
+}
+
+function resolveActiveVersionSegment (component, version) {
+  let startPage = this.resolvePage('index.adoc', { component, version })
+  let startPageSrc
+  if (startPage) {
+    startPageSrc = startPage.src
+  } else {
+    startPageSrc = { component, version, module: 'ROOT', family: 'page', relative: 'index.adoc' }
+    this.removeFile((startPage = this.addFile({ src: startPageSrc })))
+  }
+  const outPathSegments = startPage.out.path.split('/')
+  for (const depth of startPage.out.moduleRootPath.split('/')) outPathSegments.pop()
+  if (startPageSrc.module !== 'ROOT') outPathSegments.pop()
+  if (startPageSrc.component !== 'ROOT') outPathSegments.shift()
+  return outPathSegments.length ? outPathSegments[0] : ''
+}
+
+module.exports = computeOut

package/lib/util/create-asciidoc-file.js

@@ -0,0 +1,18 @@
+'use strict'
+
+const computeOut = require('./compute-out')
+const { posix: path } = require('node:path')
+
+function createAsciiDocFile (contentCatalog, file) {
+  file.mediaType = 'text/asciidoc'
+  const src = file.src
+  const out = computeOut.call(contentCatalog, src)
+  if (src.family === 'export') {
+    contentCatalog.removeFile((file = contentCatalog.addFile(Object.assign(file, { path: out.path, out: null }))))
+    return file
+  }
+  const pub = { url: '/' + out.path, moduleRootPath: out.moduleRootPath, rootPath: out.rootPath }
+  return { contents: src.contents ?? Buffer.alloc(0), src, out, pub }
+}
+
+module.exports = createAsciiDocFile

package/lib/util/lazy-readable.js

@@ -0,0 +1,18 @@
+'use strict'
+
+const { PassThrough } = require('node:stream')
+
+// adapted from https://github.com/jpommerening/node-lazystream/blob/master/lib/lazystream.js | license: MIT
+class LazyReadable extends PassThrough {
+  constructor (fn, options) {
+    super(options)
+    this._read = function () {
+      delete this._read // restores original method
+      fn.call(this, options).on('error', this.emit.bind(this, 'error')).pipe(this)
+      return this._read.apply(this, arguments)
+    }
+    this.emit('readable')
+  }
+}
+
+module.exports = LazyReadable

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@antora/assembler",
-  "version": "1.0.0-alpha.10",
+  "version": "1.0.0-alpha.12",
   "description": "An extension library for Antora that assembles content from multiple pages into a single AsciiDoc file to converted and publish.",
   "license": "MPL-2.0",
   "author": "OpenDevise Inc. (https://opendevise.com)",
@@ -9,7 +9,11 @@
     "Sarah White <sarah@opendevise.com>"
   ],
   "homepage": "https://antora.org",
-  "repository": "gitlab:antora/antora-assembler",
+  "repository": {
+    "type": "git",
+    "url": "git+https://gitlab.com/antora/antora-assembler.git",
+    "directory": "packages/assembler"
+  },
   "bugs": {
     "url": "https://gitlab.com/antora/antora-assembler/issues"
   },
@@ -21,26 +25,26 @@
   "main": "lib/index.js",
   "exports": {
     ".": "./lib/index.js",
-    "./asciidoctor/reducer-extension": "./lib/asciidoctor/reducer-extension.js",
     "./filter-component-versions": "./lib/filter-component-versions.js",
     "./load-config": "./lib/load-config.js",
-    "./produce-aggregate-document": "./lib/produce-aggregate-document.js",
-    "./produce-aggregate-documents": "./lib/produce-aggregate-documents.js",
+    "./produce-assembly-file": "./lib/produce-assembly-file.js",
+    "./produce-assembly-files": "./lib/produce-assembly-files.js",
     "./select-mutable-attributes": "./lib/select-mutable-attributes.js"
   },
   "imports": {
+    "#run-command": "@antora/run-command-helper",
     "#unconvert-inline-asciidoc": "./lib/util/unconvert-inline-asciidoc.js"
   },
   "dependencies": {
     "@asciidoctor/reducer": "~1.1",
-    "@antora/expand-path-helper": "~2.0",
-    "@antora/run-command-helper": "~1.0",
+    "@antora/expand-path-helper": "~3.0",
     "braces": "~3.0",
     "picomatch": "~3.0",
     "js-yaml": "~4.1"
   },
   "devDependencies": {
     "@antora/asciidoc-loader": "~3.1",
+    "@antora/navigation-builder": "~3.1",
     "@antora/site-publisher": "~3.1"
   },
   "engines": {
@@ -54,8 +58,5 @@
     "antora-extension",
     "asciidoc",
     "documentation"
-  ],
-  "publishConfig": {
-    "access": "public"
-  }
+  ]
 }