@inditextech/docouture-antora-extensions 0.1.0-SNAPSHOT.40.1

package/lib/llms-txt.js

@@ -0,0 +1,185 @@
+'use strict'
+
+const htmlToMarkdown = require('./html-to-markdown')
+const resolveUrl = require('./resolve-url')
+
+// Mirrors search-index.js's own constants and reasoning for both: a page
+// opts out of `:page-layout: home` because the landing is marketing surface
+// assembled from block extensions, not documentation prose, and everything
+// else defaults to Antora/page-composer's own 'default' layout.
+const DEFAULT_LAYOUT_NAME = 'default'
+const HOME_LAYOUT_NAME = 'home'
+
+/**
+ * Generates `llms.txt` and `llms-full.txt` at Antora build time (GH-95) and
+ * publishes both at the site root, alongside `sitemap.xml` — a Markdown
+ * index and a full aggregated Markdown dump of the site, meant for LLM
+ * ingestion (https://llmstxt.org) without scraping rendered HTML chrome.
+ *
+ * Site-wide, not per-component: unlike search-index.js, which publishes one
+ * JSON file per component version, both files here are a SINGLE aggregate
+ * across every component/version the aggregate has, because the issue asks
+ * for files "alongside index.html and sitemap.xml" at the output root, and
+ * llmstxt.org's own convention is one pair of files per site. A component's
+ * pages are grouped under a heading named after their nav module (the title
+ * nav-modules.js stamps onto `componentVersion.navigation`), falling back to
+ * the component's own title for a site with no `nav_modules`; the component
+ * title is prefixed too, but ONLY when the aggregate has more than one
+ * component, so the common single-component site doesn't get a redundant
+ * "Weave.js — Framework" when "Framework" alone is unambiguous.
+ *
+ * ORDERING DEPENDENCY, same one search-index.js documents at length: this
+ * reads `tree.module` / `tree.title` off `componentVersion.navigation`,
+ * stamped by nav-modules.js on the SAME `navigationBuilt` event, so index.js
+ * must register nav-modules before this module or every section silently
+ * falls back to the component title.
+ *
+ * Configuration is authored per component, under a new `llms` key in
+ * `docs/antora.yml` — the same place `nav_modules` and `footer` live, and for
+ * the same reason: `site.keys` in the playbook is a flat primitive map and
+ * cannot hold a list. See nav-modules.js's own header for the full argument.
+ *
+ *     llms:
+ *       summary: The visual collaborative apps framework, headless and store-agnostic.
+ *       exclude:
+ *         - main:some-internal-page.adoc
+ *
+ * `summary` becomes the blockquote under the site title in `llms.txt`; the
+ * first component to declare one wins when several do (there is only one
+ * site-wide summary line, not one per component). `exclude` is a list of
+ * page IDs — the same strings `xref:...[]` accepts — resolved against the
+ * component the way footer.js resolves its own link targets; a page ID that
+ * resolves to nothing is dropped with a warning rather than silently
+ * matching everything or nothing.
+ */
+module.exports = function registerLlmsTxt(context) {
+  const logger = context.getLogger('docouture-llms-txt')
+  // Keyed the same way nav-modules.js and footer.js key their own copies of
+  // the descriptor — see nav-modules.js's header for why this has to be a
+  // two-phase (contentAggregated, then navigationBuilt) extension at all.
+  const descriptors = new Map()
+
+  context.on('contentAggregated', ({ contentAggregate }) => {
+    for (const bucket of contentAggregate) {
+      if (!bucket.llms) continue
+      descriptors.set(bucket.version + '@' + bucket.name, bucket.llms)
+    }
+  })
+
+  context.on('navigationBuilt', ({ contentCatalog, siteCatalog, playbook }) => {
+    const defaultLayout = playbook.ui?.defaultLayout || DEFAULT_LAYOUT_NAME
+    const components = contentCatalog.getComponents()
+    const multiComponent = components.length > 1
+
+    const sectionsByHeading = new Map()
+    const fullParts = []
+    let summary
+
+    for (const component of components) {
+      for (const componentVersion of component.versions) {
+        const where = `${componentVersion.name}@${componentVersion.version || 'default'}`
+        const descriptor = descriptors.get(componentVersion.version + '@' + componentVersion.name) || {}
+        if (descriptor.summary && !summary) summary = descriptor.summary
+
+        const excluded = buildExcludedSet(descriptor.exclude, contentCatalog, componentVersion, logger, where)
+
+        const moduleTitleByModule = new Map()
+        for (const tree of componentVersion.navigation || []) {
+          if (tree.module) moduleTitleByModule.set(tree.module, tree.title || tree.module)
+        }
+
+        const pages = contentCatalog.getPages(
+          (page) =>
+            page.out &&
+            page.src.component === componentVersion.name &&
+            page.src.version === componentVersion.version &&
+            resolveLayout(page, defaultLayout) !== HOME_LAYOUT_NAME &&
+            !excluded.has(page.pub.url)
+        )
+
+        for (const page of pages) {
+          const title = page.asciidoc?.doctitle
+          if (!title) continue // no AsciiDoc header at all — nothing to list or dump, same rule search-index.js applies
+
+          const moduleTitle = moduleTitleByModule.get(page.src.module) || componentVersion.title
+          const heading = multiComponent ? `${componentVersion.title} — ${moduleTitle}` : moduleTitle
+
+          if (!sectionsByHeading.has(heading)) sectionsByHeading.set(heading, [])
+          sectionsByHeading.get(heading).push({
+            title,
+            url: page.pub.url,
+            description: page.asciidoc?.attributes?.description || '',
+          })
+
+          fullParts.push(buildFullEntry(title, page))
+        }
+      }
+    }
+
+    const llmsTxt = buildLlmsTxt(playbook.site?.title, summary, sectionsByHeading)
+    const llmsFullTxt = fullParts.join('\n\n---\n\n') + '\n'
+
+    siteCatalog.addFile({
+      contents: Buffer.from(llmsTxt),
+      out: { path: 'llms.txt' },
+      pub: { url: '/llms.txt' },
+    })
+    siteCatalog.addFile({
+      contents: Buffer.from(llmsFullTxt),
+      out: { path: 'llms-full.txt' },
+      pub: { url: '/llms-full.txt' },
+    })
+  })
+}
+
+// Mirrors @antora/page-composer's own resolution (see search-index.js's own
+// note on why this reads `file.asciidoc.attributes` rather than the UI
+// model's `page.layout`, which does not exist yet at `navigationBuilt`).
+function resolveLayout(page, defaultLayout) {
+  return page.asciidoc?.attributes?.['page-layout'] || defaultLayout
+}
+
+function buildExcludedSet(exclude, contentCatalog, componentVersion, logger, where) {
+  const excluded = new Set()
+  if (exclude === undefined) return excluded
+  if (!Array.isArray(exclude)) {
+    logger.warn('Ignoring llms.exclude in %s: expected a list of page IDs, got %s', where, typeof exclude)
+    return excluded
+  }
+
+  // Page IDs are resolved as if from the component's ROOT module, the same
+  // default context footer.js uses for its own link targets, so a bare
+  // `index.adoc` means what an author would expect and a module-qualified
+  // `sdk:index.adoc` works without naming the component.
+  const resolveContext = { component: componentVersion.name, version: componentVersion.version, module: 'ROOT' }
+  for (const spec of exclude) {
+    const url = resolveUrl(spec, contentCatalog, resolveContext)
+    if (url) {
+      excluded.add(url)
+    } else {
+      logger.warn('llms.exclude entry %s in %s resolves to no page', spec, where)
+    }
+  }
+  return excluded
+}
+
+function buildFullEntry(title, page) {
+  const markdown = htmlToMarkdown(page.contents.toString())
+  return `# ${title}\n\nSource: ${page.pub.url}\n\n${markdown}`
+}
+
+function buildLlmsTxt(siteTitle, summary, sectionsByHeading) {
+  let out = `# ${siteTitle || 'Documentation'}\n`
+  if (summary) out += `\n> ${summary}\n`
+
+  for (const [heading, entries] of sectionsByHeading) {
+    if (!entries.length) continue
+    out += `\n## ${heading}\n\n`
+    for (const entry of entries) {
+      const description = entry.description ? `: ${entry.description}` : ''
+      out += `- [${entry.title}](${entry.url})${description}\n`
+    }
+  }
+
+  return out.trim() + '\n'
+}

package/lib/llms-txt.spec.js

@@ -0,0 +1,172 @@
+'use strict'
+
+import { describe, expect, it } from 'vitest'
+
+const registerLlmsTxt = require('./llms-txt')
+
+function createContext() {
+  const listeners = {}
+  return {
+    getLogger: () => ({ warn: () => {}, info: () => {} }),
+    on(event, fn) {
+      ;(listeners[event] ||= []).push(fn)
+    },
+    async emit(event, payload) {
+      for (const fn of listeners[event] || []) await fn(payload)
+    },
+  }
+}
+
+function createPage({ module: mod, relative, url, title, description, html, layout }) {
+  return {
+    out: true,
+    src: { component: 'weavejs', version: '', module: mod, relative },
+    pub: { url },
+    asciidoc: {
+      doctitle: title,
+      attributes: { description, ...(layout ? { 'page-layout': layout } : {}) },
+    },
+    contents: Buffer.from(html || '<p>Body</p>'),
+  }
+}
+
+function createContentCatalog(componentVersion, pages) {
+  return {
+    getComponents: () => [{ versions: [componentVersion] }],
+    getPages: (filterFn) => pages.filter(filterFn),
+    resolvePage(spec, context) {
+      const parts = spec.split(':')
+      const relative = parts.pop()
+      const mod = parts.pop() || context.module
+      const page = pages.find(
+        (p) =>
+          p.src.component === context.component &&
+          p.src.version === context.version &&
+          p.src.module === mod &&
+          p.src.relative === relative
+      )
+      return page ? { pub: page.pub } : undefined
+    },
+  }
+}
+
+async function run({ llms, navigation, pages }) {
+  const context = createContext()
+  registerLlmsTxt(context)
+
+  const componentVersion = {
+    name: 'weavejs',
+    version: '',
+    title: 'Weave.js',
+    navigation: navigation || [],
+  }
+  const contentCatalog = createContentCatalog(componentVersion, pages)
+  const files = []
+  const siteCatalog = { addFile: (f) => files.push(f) }
+  const playbook = { site: { title: 'Weave.js' }, ui: {} }
+
+  await context.emit('contentAggregated', {
+    contentAggregate: llms ? [{ name: 'weavejs', version: '', llms }] : [],
+  })
+  await context.emit('navigationBuilt', { contentCatalog, siteCatalog, playbook })
+
+  const llmsTxt = files.find((f) => f.out.path === 'llms.txt').contents.toString()
+  const llmsFullTxt = files.find((f) => f.out.path === 'llms-full.txt').contents.toString()
+  return { llmsTxt, llmsFullTxt, files }
+}
+
+describe('registerLlmsTxt', () => {
+  it('builds an index grouped by nav module, with the summary as a blockquote', async () => {
+    const pages = [
+      createPage({
+        module: 'main',
+        relative: 'index.adoc',
+        url: '/weavejs/main/index.html',
+        title: 'Getting Started',
+        description: 'How to start',
+        html: '<h2 id="x">Sub</h2><p>Hello world</p>',
+      }),
+    ]
+    const { llmsTxt, llmsFullTxt } = await run({
+      llms: { summary: 'The visual collaborative apps framework.' },
+      navigation: [{ module: 'main', title: 'Framework' }],
+      pages,
+    })
+
+    expect(llmsTxt).toContain('# Weave.js')
+    expect(llmsTxt).toContain('> The visual collaborative apps framework.')
+    expect(llmsTxt).toContain('## Framework')
+    expect(llmsTxt).toContain('- [Getting Started](/weavejs/main/index.html): How to start')
+
+    expect(llmsFullTxt).toContain('# Getting Started')
+    expect(llmsFullTxt).toContain('Source: /weavejs/main/index.html')
+    expect(llmsFullTxt).toContain('## Sub')
+    expect(llmsFullTxt).toContain('Hello world')
+  })
+
+  it('falls back to the component title when no nav_modules metadata is present', async () => {
+    const pages = [createPage({ module: 'ROOT', relative: 'index.adoc', url: '/weavejs/index.html', title: 'Home' })]
+    const { llmsTxt } = await run({ pages })
+
+    expect(llmsTxt).toContain('## Weave.js')
+    expect(llmsTxt).not.toContain('>') // no summary declared, no blockquote line
+  })
+
+  it('drops excluded pages from both files', async () => {
+    const pages = [
+      createPage({
+        module: 'main',
+        relative: 'index.adoc',
+        url: '/weavejs/main/index.html',
+        title: 'Getting Started',
+      }),
+      createPage({
+        module: 'main',
+        relative: 'secret.adoc',
+        url: '/weavejs/main/secret.html',
+        title: 'Secret',
+      }),
+    ]
+    const { llmsTxt, llmsFullTxt } = await run({
+      llms: { exclude: ['main:secret.adoc'] },
+      navigation: [{ module: 'main', title: 'Framework' }],
+      pages,
+    })
+
+    expect(llmsTxt).toContain('Getting Started')
+    expect(llmsTxt).not.toContain('Secret')
+    expect(llmsFullTxt).not.toContain('# Secret')
+  })
+
+  it('excludes pages using the home layout', async () => {
+    const pages = [
+      createPage({
+        module: 'ROOT',
+        relative: 'index.adoc',
+        url: '/weavejs/index.html',
+        title: 'Landing',
+        layout: 'home',
+      }),
+      createPage({
+        module: 'main',
+        relative: 'index.adoc',
+        url: '/weavejs/main/index.html',
+        title: 'Getting Started',
+      }),
+    ]
+    const { llmsTxt } = await run({ navigation: [{ module: 'main', title: 'Framework' }], pages })
+
+    expect(llmsTxt).not.toContain('Landing')
+    expect(llmsTxt).toContain('Getting Started')
+  })
+
+  it('skips pages with no AsciiDoc title', async () => {
+    const pages = [
+      createPage({ module: 'main', relative: 'index.adoc', url: '/weavejs/main/index.html', title: undefined }),
+    ]
+    const { llmsTxt, llmsFullTxt } = await run({ pages })
+
+    expect(llmsTxt.trim()).toBe('# Weave.js')
+    expect(llmsFullTxt.trim()).toBe('')
+  })
+})

package/lib/nav-modules.js

@@ -0,0 +1,233 @@
+'use strict'
+
+const resolveUrl = require('./resolve-url')
+
+// `modules/<module>/nav.adoc` — the shape of every entry in a component
+// descriptor's own `nav` list. Anything else (a nav file outside modules/,
+// which Antora does allow) simply has no module to attach metadata to.
+const MODULE_NAV_PATH_RX = /^modules\/([^/]+)\//
+
+/**
+ * Attaches per-module metadata to Antora's navigation trees, so the UI can
+ * show ONE module's navigation at a time with a switcher between them.
+ *
+ * Background: @antora/navigation-builder already produces one root tree per
+ * unordered list per nav file, in the order the descriptor's `nav` key lists
+ * them (build-navigation.js: `tree.order = navFile.nav.index`). A component
+ * with seven nav files therefore already has seven root trees in
+ * `page.navigation` — Antora's own equivalent of a Fumadocs `root: true`
+ * folder. What it has no notion of is which MODULE a tree belongs to, or what
+ * that module should be called in a switcher.
+ *
+ * There is no built-in place to put that, which is why this extension exists.
+ * Every alternative was checked and is a dead end:
+ *
+ *   - custom keys in antora.yml are dropped: ContentCatalog#registerComponentVersion
+ *     destructures only asciidoc/displayVersion/prerelease/startPage/title
+ *   - playbook `site.keys` is declared `format: 'primitive-map'`, so flat
+ *     primitives only — no nested map, no list
+ *   - a nav file's own AsciiDoc attributes are unreachable: buildNavigation
+ *     hardcodes `extensions: []` and keeps only the ulists
+ *   - a list TITLE on the nav file would give a title and nothing else, and
+ *     it would also push a module entry into every page's `page.breadcrumbs`
+ *     (page-composer's attachNavProperties collects ancestors that have
+ *     `content`), which is a visible change to every page in the site
+ *
+ * So the metadata is authored in antora.yml under `nav_modules` and read HERE,
+ * from the raw aggregate, before classification throws the unknown key away.
+ *
+ * Authored shape — a LIST, not a map keyed by module:
+ *
+ *     nav_modules:
+ *       - module: main
+ *         title: Framework
+ *         description: The visual collaborative apps framework
+ *         icon: design/menu-tab-outlined
+ *
+ * Two optional keys beyond the ones the switcher draws:
+ *
+ *   - `switcher: false` annotates a tree without offering it as a module to
+ *     switch to. That is what a LANDING page's own navigation is: a menu that
+ *     belongs to no module (`modules/ROOT/nav.adoc`, rendered only on the
+ *     landing — see the UI's `page-nav-module` attribute), which should not
+ *     appear beside the real modules in the switcher. Declaring it is also
+ *     what stops the "no nav_modules entry" warning below from failing a build
+ *     whose playbook sets `failure_level: warn`.
+ *   - `start_page` overrides where the switcher and the site footer send
+ *     someone who picks this module. It is a page ID — the same string you
+ *     would write inside `xref:...[]` — resolved against this component.
+ *     Without it, the target is the first internal page in the module's own
+ *     navigation, which for a conventionally laid out module is its index.
+ *
+ * The list is load-bearing. @antora/content-aggregator runs the whole
+ * descriptor through `camelCaseKeys`, which recurses into nested objects and
+ * rewrites their KEYS (only `asciidoc` is exempted). A map keyed by module
+ * would silently rename every kebab-case module — `store-azure-web-pubsub`
+ * would arrive here as `storeAzureWebPubsub` and match nothing. Values are
+ * left alone, so carrying the slug as a value rather than a key is what makes
+ * it survive. `nav_modules` itself still camelCases to `navModules`, which is
+ * why that is what this file reads.
+ *
+ * Result: each tree in `page.navigation` gains `module`, `title`, and
+ * optionally `description` and `icon`. Sites that don't declare `nav_modules`
+ * are untouched, and the UI falls back to rendering every tree — which is the
+ * behaviour every single-nav-file site already had.
+ */
+module.exports = function registerNavModules(context) {
+  const logger = context.getLogger('docouture-nav-modules')
+  // Keyed the same way @antora/navigation-builder keys its own accumulator.
+  const descriptors = new Map()
+
+  // The aggregate is the LAST point at which the descriptor's unknown keys
+  // still exist: generate-site.js calls `vars.remove('contentAggregate')` in
+  // the very next step, and classifyContent would have dropped `navModules`
+  // anyway. Copy out what's needed now; annotate later.
+  context.on('contentAggregated', ({ contentAggregate }) => {
+    for (const bucket of contentAggregate) {
+      if (!bucket.navModules) continue
+      descriptors.set(bucket.version + '@' + bucket.name, {
+        nav: bucket.nav || [],
+        navModules: bucket.navModules,
+      })
+    }
+  })
+
+  // navigationBuilt fires after buildNavigation has assigned the trees to
+  // `componentVersion.navigation` and before pagesComposed, which is when
+  // page-composer hands that same array to the UI model as `page.navigation`.
+  // These are the same objects throughout, so annotating in place is enough —
+  // nothing needs to be threaded through the page composer.
+  context.on('navigationBuilt', ({ contentCatalog }) => {
+    for (const component of contentCatalog.getComponents()) {
+      for (const componentVersion of component.versions) {
+        const descriptor = descriptors.get(componentVersion.version + '@' + componentVersion.name)
+        if (descriptor) annotate(componentVersion, descriptor, contentCatalog, logger)
+      }
+    }
+  })
+}
+
+function stampItems(items, module_) {
+  for (const item of items || []) {
+    item.module = module_
+    stampItems(item.items, module_)
+  }
+}
+
+// Depth first, because a module's first entry is often an unlinked category
+// heading (Fumadocs' `"---Overview---"` separators survived the migration as
+// nav entries with `content` and neither `url` nor children) and the first
+// real page hangs off a later sibling.
+function findFirstInternalUrl(items) {
+  for (const item of items || []) {
+    if (item.urlType === 'internal' && item.url) return item.url
+    const nested = findFirstInternalUrl(item.items)
+    if (nested) return nested
+  }
+}
+
+function annotate(componentVersion, { nav, navModules }, contentCatalog, logger) {
+  const where = `${componentVersion.name}@${componentVersion.version || 'default'}`
+
+  if (!Array.isArray(navModules)) {
+    logger.warn('Ignoring nav_modules in %s: expected a list of entries, got %s', where, typeof navModules)
+    return
+  }
+
+  const declared = new Map()
+  for (const entry of navModules) {
+    if (!entry || entry.constructor !== Object || !entry.module) {
+      logger.warn('Ignoring nav_modules entry in %s: every entry needs a module key', where)
+      continue
+    }
+    if (declared.has(entry.module)) {
+      logger.warn('Ignoring duplicate nav_modules entry for module %s in %s', entry.module, where)
+      continue
+    }
+    declared.set(entry.module, entry)
+  }
+
+  const matched = new Set()
+  for (const tree of componentVersion.navigation || []) {
+    // `tree.order` is the index of the nav file in the descriptor's own `nav`
+    // list — an integer, except when one nav file holds more than one list, in
+    // which case the extra lists get a fraction on top of that same index
+    // (build-navigation.js). Flooring therefore gets back to the file either
+    // way, and every list in a file belongs to that file's module.
+    const navPath = nav[Math.floor(tree.order)]
+    const module_ = navPath && MODULE_NAV_PATH_RX.exec(navPath)?.[1]
+    if (!module_) continue
+
+    tree.module = module_
+    // Stamped all the way down, not just on the root. `page.previous` and
+    // `page.next` are these very item objects (page-composer's findNavItem
+    // returns matches out of this same tree), so carrying the module on each
+    // one is what lets the footer pagination tell "the next page" from "the
+    // next page IN THIS MODULE" without re-deriving anything from URLs.
+    stampItems(tree.items, module_)
+
+    const meta = declared.get(module_)
+    if (!meta) {
+      // Not fatal: the tree still renders, and the UI keys segmentation off
+      // `tree.module`, which is set. Only the switcher entry is missing.
+      logger.warn('No nav_modules entry for module %s in %s', module_, where)
+      continue
+    }
+    matched.add(module_)
+
+    // Fall back to the slug rather than leaving the switcher entry blank.
+    tree.title = meta.title || module_
+    if (meta.description) tree.description = meta.description
+    if (meta.icon) tree.icon = meta.icon
+
+    // `switcher: false` — annotated, but not a module anyone can switch TO.
+    // A landing page's own navigation is the case this exists for; see this
+    // file's header. The UI keys "offer this in the switcher" off `startUrl`,
+    // so withholding it is the whole implementation, and it is also why the
+    // flag has to skip the "no page to link to" warning below rather than
+    // fall through to it.
+    if (meta.switcher === false) {
+      tree.switcher = false
+      continue
+    }
+
+    // Where the switcher and the site footer should send someone who picks
+    // this module. `start_page` is the authored override, a page ID resolved
+    // against this component; absent one, it is the first internal page in
+    // the module's own navigation, found here rather than in the template
+    // because finding it means a depth-first walk, which Handlebars has no
+    // way to express. Named `startUrl`, not `url`, on purpose: a nav node
+    // WITH a `url` is a link as far as nav-tree.hbs is concerned, and a root
+    // menu is not one.
+    let startUrl
+    if (meta.startPage) {
+      startUrl = resolveUrl(meta.startPage, contentCatalog, {
+        component: componentVersion.name,
+        version: componentVersion.version,
+        module: module_,
+      })
+      if (!startUrl) {
+        logger.warn(
+          'nav_modules start_page %s for module %s in %s resolves to no page; falling back to the navigation',
+          meta.startPage,
+          module_,
+          where
+        )
+      }
+    }
+    if (!startUrl) startUrl = findFirstInternalUrl(tree.items)
+    if (startUrl) {
+      tree.startUrl = startUrl
+    } else {
+      logger.warn('Module %s in %s has no internal page to link to from the navigation switcher', module_, where)
+    }
+  }
+
+  for (const module_ of declared.keys()) {
+    if (matched.has(module_)) continue
+    // Almost always a typo in the slug, or a module whose nav file is missing
+    // from the descriptor's `nav` list — both of which silently produce a
+    // module the switcher can never reach.
+    logger.warn('nav_modules entry for module %s in %s matches no navigation file', module_, where)
+  }
+}