@inditextech/docouture-antora-extensions 0.1.0-SNAPSHOT.40.1

package/lib/search-index.js

@@ -0,0 +1,279 @@
+'use strict'
+
+const { parse, NodeType } = require('node-html-parser')
+
+// h1-h4 are destinations worth their own record. h5/h6 are a label inside a
+// section rather than a destination — see this file's header — so they are
+// deliberately absent here and fall through to the generic recursion branch,
+// which folds their text into whatever chunk is currently open.
+const SECTION_HEADING_TAGS = new Set(['h1', 'h2', 'h3', 'h4'])
+
+// Elements whose entire subtree is dropped before any text is collected —
+// see the module doc comment for why each one is here.
+const STRIPPED_TAGS = new Set(['pre', 'svg'])
+const STRIPPED_CLASSES = new Set(['toc', 'icon'])
+
+const ENVELOPE_VERSION = 1
+// Multilingual search is out of scope for this epic (see #64); every record
+// is authored English prose today, so this is a constant, not a lookup.
+const LANGUAGE = 'english'
+
+// @antora/page-composer's own default when neither a page nor the playbook
+// names one — see build-ui-model.js's buildPageUiModel and constants.js.
+const DEFAULT_LAYOUT_NAME = 'default'
+// The layout a landing page opts into with `:page-layout: home`
+// (ui-bundle/src/layouts/home.hbs). It is a shell of card grids, feature
+// tables and a CTA — marketing surface assembled from block extensions, not
+// documentation prose — so it is excluded rather than indexed as if it were
+// a regular doc page.
+const HOME_LAYOUT_NAME = 'home'
+
+/**
+ * Builds the site's full-text search index at Antora build time and
+ * publishes it as one static JSON asset per component version — no UI reads
+ * it yet (that starts at #66/#67, S2/S3 of the search epic, #64).
+ *
+ * Why the HTML and not the AsciiDoc source, why `navigationBuilt`
+ * specifically, the record shape, the section-splitting rule and the
+ * empty-page fallback are all argued at length in #65 — repeating that
+ * reasoning inline would drift out of sync with the issue, so only the
+ * mechanics are documented here.
+ *
+ * ORDERING DEPENDENCY: this module reads `tree.module` / `tree.title` off
+ * `componentVersion.navigation`, which `nav-modules.js` stamps on the SAME
+ * `navigationBuilt` event. `GeneratorContext#notify` awaits listeners in
+ * registration order (see nav-modules.js's own header for the citation), so
+ * this only produces correct categories because `index.js` registers
+ * nav-modules BEFORE this module. Reorder those calls and every category
+ * silently falls back to the component title — no error, just wrong
+ * grouping.
+ */
+module.exports = function registerSearchIndex(context) {
+  const logger = context.getLogger('docouture-search-index')
+
+  context.on('navigationBuilt', ({ contentCatalog, siteCatalog, playbook }) => {
+    const outputDir = playbook.ui?.outputDir ?? '_'
+    const defaultLayout = playbook.ui?.defaultLayout || DEFAULT_LAYOUT_NAME
+
+    // Site-wide feedback (GH-103-adjacent): a component version producing
+    // no search records used to fail silently — the index file for it
+    // simply never got written, with nothing in the log to explain why.
+    // These totals, plus the per-component-version summary below, make what
+    // actually got indexed (or didn't) visible on every build without
+    // reading the produced JSON files.
+    let totalPages = 0
+    let totalRecords = 0
+    let totalFiles = 0
+
+    for (const component of contentCatalog.getComponents()) {
+      for (const componentVersion of component.versions) {
+        const where = `${componentVersion.name}@${componentVersion.version || 'default'}`
+        const pages = contentCatalog.getPages(
+          (page) =>
+            page.out &&
+            page.src.component === componentVersion.name &&
+            page.src.version === componentVersion.version &&
+            resolveLayout(page, defaultLayout) !== HOME_LAYOUT_NAME
+        )
+        const records = buildComponentVersionRecords(componentVersion, contentCatalog, defaultLayout, logger)
+
+        if (!records.length) {
+          logger.warn('%s produced no search records; no search index written', where)
+          continue
+        }
+
+        const basename = componentVersion.version
+          ? `${componentVersion.name}-${componentVersion.version}.json`
+          : `${componentVersion.name}.json`
+        const outPath = `${outputDir}/search/${basename}`
+
+        const envelope = {
+          version: ENVELOPE_VERSION,
+          language: LANGUAGE,
+          component: componentVersion.name,
+          componentVersion: componentVersion.version || '',
+          records,
+        }
+
+        siteCatalog.addFile({
+          contents: Buffer.from(JSON.stringify(envelope)),
+          out: { path: outPath },
+          pub: { url: '/' + outPath },
+        })
+
+        // Basename only — see footer.js's own note on componentVersion.footer
+        // for why the UI resolves this against uiRootPath rather than being
+        // handed an absolute URL.
+        componentVersion.searchIndex = basename
+
+        logger.info('%s: %s pages, %s search records -> %s', where, pages.length, records.length, outPath)
+
+        totalPages += pages.length
+        totalRecords += records.length
+        totalFiles += 1
+      }
+    }
+
+    logger.info(
+      'search index totals: %s pages, %s records, %s index file(s) written',
+      totalPages,
+      totalRecords,
+      totalFiles
+    )
+  })
+}
+
+function buildComponentVersionRecords(componentVersion, contentCatalog, defaultLayout, logger) {
+  // Module -> category title. Empty when the component declares no
+  // nav_modules at all, which is what makes a single-module site fall back
+  // to one category (its own title) for every page.
+  const categoryByModule = new Map()
+  for (const tree of componentVersion.navigation || []) {
+    if (tree.module) categoryByModule.set(tree.module, tree.title || tree.module)
+  }
+
+  const pages = contentCatalog.getPages(
+    (page) =>
+      page.out &&
+      page.src.component === componentVersion.name &&
+      page.src.version === componentVersion.version &&
+      resolveLayout(page, defaultLayout) !== HOME_LAYOUT_NAME
+  )
+
+  const records = []
+  for (const page of pages) {
+    const category = categoryByModule.get(page.src.module) || componentVersion.title
+    records.push(...buildPageRecords(page, componentVersion.title, category, logger))
+  }
+  return records
+}
+
+function buildPageRecords(page, componentTitle, category, logger) {
+  const title = page.asciidoc?.doctitle
+  if (!title) {
+    // No AsciiDoc header at all (rare, but a page-partial fragment or the
+    // synthetic 404 page are both real cases) — nothing to title a record
+    // with, so there is nothing useful to index.
+    return []
+  }
+
+  const url = page.pub.url
+  const baseHierarchy = dedupeAdjacent([componentTitle, category])
+
+  // Pass 1: flatten the article into chunks in document order — one for the
+  // prose before the first heading (level 0, no heading text), then one per
+  // h1-h4 that carries an id. Deliberately not a walk of Asciidoctor's own
+  // `.sect1 > h2 + .sectionbody` nesting; see #65 for why that shape is the
+  // wrong one to reproduce.
+  const root = parse(page.contents.toString())
+  const chunks = [{ level: 0, headingText: undefined, id: undefined, textParts: [] }]
+  walk(root, chunks)
+
+  // Pass 2: turn chunks into records, tracking an explicit stack of open
+  // ancestor headings by level (headings are siblings in the flattened HTML,
+  // not DOM ancestors of the sections they open, so this can't be read off
+  // the tree itself).
+  const stack = []
+  const records = []
+  for (const chunk of chunks) {
+    const content = collapseWhitespace(chunk.textParts.join(''))
+    if (chunk.level === 0) {
+      if (content) {
+        records.push({ title, hierarchy: baseHierarchy, content, url, category })
+      }
+      continue
+    }
+
+    // Ancestors are whatever is still open at a shallower level than this
+    // heading; this heading's own text is not one of its own ancestors.
+    const ancestors = stack.slice(0, chunk.level - 1).filter(Boolean)
+    records.push({
+      title,
+      section: chunk.headingText,
+      hierarchy: [...baseHierarchy, ...ancestors],
+      content,
+      url: `${url}#${chunk.id}`,
+      category,
+    })
+
+    stack[chunk.level - 1] = chunk.headingText
+    stack.length = chunk.level
+  }
+
+  if (records.some((record) => record.content)) return records
+
+  // The empty-page case (#65): a page whose entire body was `pre` blocks
+  // produces only content-less chunks above (no prose at all, whether or
+  // not a heading happened to wrap the code) and would otherwise be
+  // unfindable by its own title. One fallback record carries what
+  // survives: title, hierarchy, and the page's own :description: attribute
+  // where it has one.
+  logger.info('Page %s produced no search content; emitting a fallback record', url)
+  return [
+    {
+      title,
+      hierarchy: baseHierarchy,
+      content: page.asciidoc?.attributes?.description || '',
+      url,
+      category,
+    },
+  ]
+}
+
+// Depth-first, document-order walk. Pushes a new chunk onto `chunks`
+// whenever a heading with an id is found; every other text node is appended
+// to whichever chunk is currently last.
+function walk(node, chunks) {
+  for (const child of node.childNodes) {
+    if (child.nodeType === NodeType.TEXT_NODE) {
+      chunks[chunks.length - 1].textParts.push(child.text)
+      continue
+    }
+    if (child.nodeType !== NodeType.ELEMENT_NODE) continue
+
+    const tag = child.rawTagName?.toLowerCase()
+    if (STRIPPED_TAGS.has(tag)) continue
+    if ((child.classList?.value || []).some((cls) => STRIPPED_CLASSES.has(cls))) continue
+
+    const id = child.getAttribute('id')
+    if (SECTION_HEADING_TAGS.has(tag) && id) {
+      chunks.push({
+        level: Number(tag[1]),
+        headingText: collapseWhitespace(child.text),
+        id,
+        textParts: [],
+      })
+      continue
+    }
+
+    // A wrapper containing a heading — this project's own block extensions
+    // (cards, steps, feature tabs, accordion) among them — has to be opened
+    // up, or its prose gets filed under whichever chunk was current when the
+    // wrapper opened. Recursing here is what "opens it up". Inline `<code>`
+    // falls through this same branch and its text lands in the current
+    // chunk, which is why it needs no special case to be kept.
+    walk(child, chunks)
+  }
+}
+
+function collapseWhitespace(text) {
+  return text.replace(/\s+/g, ' ').trim()
+}
+
+// Mirrors @antora/page-composer's own resolution (build-ui-model.js:86):
+// a page's `:page-layout:` attribute, else the playbook's ui.default_layout,
+// else Antora's own 'default'. Read directly off `file.asciidoc.attributes`
+// rather than `page.attributes`/`page.layout` because neither of those UI
+// model fields exists yet at `navigationBuilt` — page-composer builds them
+// at the next step, `pagesComposed`.
+function resolveLayout(page, defaultLayout) {
+  return page.asciidoc?.attributes?.['page-layout'] || defaultLayout
+}
+
+function dedupeAdjacent(parts) {
+  const out = []
+  for (const part of parts) {
+    if (part && part !== out[out.length - 1]) out.push(part)
+  }
+  return out
+}

package/lib/search-index.spec.js

@@ -0,0 +1,104 @@
+'use strict'
+
+import { describe, expect, it } from 'vitest'
+
+const registerSearchIndex = require('./search-index')
+
+function createContext() {
+  const listeners = {}
+  const logs = { info: [], warn: [] }
+  return {
+    logs,
+    getLogger: () => ({
+      info: (...args) => logs.info.push(args),
+      warn: (...args) => logs.warn.push(args),
+    }),
+    on(event, fn) {
+      ;(listeners[event] ||= []).push(fn)
+    },
+    async emit(event, payload) {
+      for (const fn of listeners[event] || []) await fn(payload)
+    },
+  }
+}
+
+function createPage({ component = 'test-docs', version = '', module: mod = 'ROOT', url, title, html }) {
+  return {
+    out: true,
+    src: { component, version, module: mod },
+    pub: { url },
+    asciidoc: { doctitle: title, attributes: {} },
+    contents: Buffer.from(html || `<h2 id="x">Section</h2><p>Body text.</p>`),
+  }
+}
+
+function createContentCatalog(components, pages) {
+  return {
+    getComponents: () => components,
+    getPages: (filterFn) => pages.filter(filterFn),
+  }
+}
+
+async function run({ components, pages }) {
+  const context = createContext()
+  registerSearchIndex(context)
+
+  const contentCatalog = createContentCatalog(components, pages)
+  const files = []
+  const siteCatalog = { addFile: (f) => files.push(f) }
+  const playbook = { ui: {} }
+
+  await context.emit('navigationBuilt', { contentCatalog, siteCatalog, playbook })
+
+  return { files, logs: context.logs }
+}
+
+describe('registerSearchIndex feedback', () => {
+  it('logs a per-component-version summary and a site-wide total', async () => {
+    const componentVersion = { name: 'test-docs', version: '', title: 'Test Docs', navigation: [] }
+    const component = { versions: [componentVersion] }
+    const pages = [createPage({ url: '/test-docs/index.html', title: 'Home' })]
+
+    const { files, logs } = await run({ components: [component], pages })
+
+    expect(files).toHaveLength(1)
+    expect(logs.info).toContainEqual([
+      '%s: %s pages, %s search records -> %s',
+      'test-docs@default',
+      1,
+      1,
+      '_/search/test-docs.json',
+    ])
+    expect(logs.info).toContainEqual(['search index totals: %s pages, %s records, %s index file(s) written', 1, 1, 1])
+    expect(logs.warn).toHaveLength(0)
+  })
+
+  it('warns, instead of silently skipping, when a component version produces no search records', async () => {
+    const componentVersion = { name: 'empty-docs', version: '', title: 'Empty Docs', navigation: [] }
+    const component = { versions: [componentVersion] }
+
+    const { files, logs } = await run({ components: [component], pages: [] })
+
+    expect(files).toHaveLength(0)
+    expect(logs.warn).toHaveLength(1)
+    expect(logs.warn[0].join(' ')).toContain('empty-docs@default')
+    expect(logs.warn[0].join(' ')).toContain('no search index written')
+  })
+
+  it('accumulates totals across multiple component versions', async () => {
+    const versionA = { name: 'a', version: '', title: 'A', navigation: [] }
+    const versionB = { name: 'b', version: '', title: 'B', navigation: [] }
+    const components = [{ versions: [versionA] }, { versions: [versionB] }]
+    const pages = [
+      createPage({ component: 'a', url: '/a/index.html', title: 'A Home' }),
+      createPage({ component: 'b', url: '/b/index.html', title: 'B Home' }),
+    ]
+
+    const { files, logs } = await run({ components, pages })
+
+    expect(files).toHaveLength(2)
+    const totalsLine = logs.info.find((args) => args[0].startsWith('search index totals'))
+    expect(totalsLine).toEqual(['search index totals: %s pages, %s records, %s index file(s) written', 2, 2, 2])
+    expect(logs.warn).toHaveLength(0)
+  })
+})

package/lib/shiki-prewarm.js

@@ -0,0 +1,76 @@
+'use strict'
+
+// GH-89: pre-warms the one Shiki highlighter instance the whole build's
+// source blocks share, asynchronously, before any page is converted.
+//
+// This is an ANTORA pipeline extension (`antora.extensions` in the
+// playbook) — see index.js's own header for why this package's exports are
+// that kind and not the `asciidoc.extensions` kind. It exists only to solve
+// GH-89's own "Key Risk / Spike Required" section: `@asciidoctor/core ~2.2`
+// converts every page through a fully-synchronous (Opal) loop, but building
+// a Shiki highlighter — instantiating its WASM oniguruma engine, loading
+// every bundled language/theme grammar — is asynchronous. So that async
+// work has to happen ONCE, up front, outside the conversion loop entirely,
+// and its result handed to the synchronous side through a plain shared
+// module (`shiki-instance.js`, in the SIBLING
+// @inditextech/docouture-asciidoc-extensions package — that's where the
+// synchronous consumer, `shiki-syntax-highlighter.js`'s `highlight()`,
+// lives too).
+//
+// `contentAggregated` is the event to hook: @antora/site-generator awaits it
+// (`generate-site.js`, `await context.notify('contentAggregated', …)`)
+// before `contentClassified`, itself before `documentsConverted` — the
+// phase that actually calls into Asciidoctor. Same ordering guarantee
+// nav-modules.js already relies on for its own `contentAggregated` listener
+// (see that file's header) — this one just doesn't touch `contentAggregate`
+// itself, so it declares zero parameters.
+const { createOnigurumaEngine } = require('shiki/engine/oniguruma')
+const { createHighlighterCoreSync } = require('shiki/core')
+const shikiInstance = require('@inditextech/docouture-asciidoc-extensions/lib/shiki-instance')
+const { LANGS, LIGHT_THEME, DARK_THEME } = require('@inditextech/docouture-asciidoc-extensions/lib/shiki-config')
+
+async function loadLangs() {
+  return Promise.all(LANGS.map((id) => import('@shikijs/langs/' + id).then((mod) => mod.default)))
+}
+
+async function loadThemes() {
+  return Promise.all([LIGHT_THEME, DARK_THEME].map((id) => import('@shikijs/themes/' + id).then((mod) => mod.default)))
+}
+
+module.exports = function registerShikiPrewarm(context) {
+  context.on('contentAggregated', async function docoutureShikiPrewarm() {
+    const [engine, langs, themes] = await Promise.all([
+      // `shiki/wasm` ships the pre-built oniguruma binary — no network fetch,
+      // no separate build step. Instantiating it is the one truly
+      // asynchronous piece; everything downstream of this `await` in
+      // shiki-syntax-highlighter.js's `highlight()` is synchronous.
+      createOnigurumaEngine(import('shiki/wasm')),
+      loadLangs(),
+      loadThemes(),
+    ])
+
+    // Sync from here: `createHighlighterCoreSync` (note: NOT
+    // `createHighlighterCore`) requires its engine and grammars already
+    // resolved, in exchange for a highlighter whose own `codeToHtml` is a
+    // plain synchronous function — the whole point of doing all of the
+    // above up front instead of on first use.
+    const highlighter = createHighlighterCoreSync({ engine, langs, themes })
+
+    // Shiki's own default-colour custom properties normally live on the
+    // `<pre>` it generates itself; shiki-syntax-highlighter.js discards that
+    // wrapper (see its own header) and needs them separately. Probing a
+    // trivial block through the real highlighter — rather than reading
+    // theme JSON fields directly — guarantees this matches whatever Shiki
+    // itself would have put there, without this file needing to know how
+    // Shiki derives a theme's default foreground/background.
+    const probe = highlighter.codeToHtml('', {
+      lang: 'text',
+      themes: { light: LIGHT_THEME, dark: DARK_THEME },
+      defaultColor: false,
+    })
+    const match = /<pre[^>]*\sstyle="([^"]*)"/.exec(probe)
+    const rootStyle = match ? match[1] : ''
+
+    shikiInstance.set(highlighter, rootStyle)
+  })
+}

package/lib/version-report.js

@@ -0,0 +1,40 @@
+'use strict'
+
+/**
+ * Reports, on every build, exactly what Antora computed for each
+ * component's versions — which ones exist, and which one it picked as
+ * `latest`. Purely diagnostic: nothing here changes any build output.
+ *
+ * Exists because that computation (@antora/content-classifier's
+ * `registerComponentVersion`, `component.versions.find(v => !v.prerelease)`)
+ * is otherwise invisible until someone notices a version tag/toggle
+ * rendering the wrong colour on the built site — by which point it could
+ * just as easily be a stale/never-actually-published deploy as a real
+ * version-data problem (see the docouture-publish.yml / gh-pages fixes this
+ * shipped alongside). Logging it plainly at build time answers "did Antora
+ * pick the version I expected as latest" without needing to inspect a live
+ * site at all.
+ *
+ * Hooked on `contentClassified` — the earliest point after
+ * `classifyContent` has populated the content catalog (every component's
+ * `versions[]` and `.latest` fully resolved) and before navigation/pages are
+ * built. Always `logger.info`, never `logger.warn`: this is a plain report
+ * of what Antora decided, not a claim that anything is wrong — a component
+ * legitimately has only one (prerelease) version before its first release,
+ * and that is not an error.
+ */
+module.exports = function registerVersionReport(context) {
+  const logger = context.getLogger('docouture-version-report')
+
+  context.on('contentClassified', ({ contentCatalog }) => {
+    for (const component of contentCatalog.getComponents()) {
+      const rows = component.versions.map((componentVersion) => {
+        const flags = []
+        if (componentVersion === component.latest) flags.push('latest')
+        if (componentVersion.prerelease) flags.push('prerelease')
+        return flags.length ? `${componentVersion.version} (${flags.join(', ')})` : componentVersion.version
+      })
+      logger.info('%s versions: %s', component.name, rows.join(', '))
+    }
+  })
+}

package/lib/version-report.spec.js

@@ -0,0 +1,83 @@
+'use strict'
+
+import { describe, expect, it } from 'vitest'
+
+const registerVersionReport = require('./version-report')
+
+function createContext(logs) {
+  const listeners = {}
+  return {
+    getLogger: () => ({
+      warn: () => {},
+      info: (...args) => logs.push(args),
+    }),
+    on(event, fn) {
+      ;(listeners[event] ||= []).push(fn)
+    },
+    async emit(event, payload) {
+      for (const fn of listeners[event] || []) await fn(payload)
+    },
+  }
+}
+
+function componentVersion({ version, prerelease }) {
+  return { version, prerelease }
+}
+
+async function run(components) {
+  const logs = []
+  const context = createContext(logs)
+  registerVersionReport(context)
+
+  const contentCatalog = {
+    getComponents: () => components,
+  }
+
+  await context.emit('contentClassified', { contentCatalog })
+  return logs.map(([, ...args]) => args.join(': '))
+}
+
+describe('registerVersionReport', () => {
+  it('flags the version Antora picked as latest', async () => {
+    const stable = componentVersion({ version: 'stable' })
+    const prerelease = componentVersion({ version: 'prerelease', prerelease: true })
+    const component = { name: 'test-docs', versions: [stable, prerelease], latest: stable }
+
+    const rows = await run([component])
+
+    expect(rows).toEqual(['test-docs: stable (latest), prerelease (prerelease)'])
+  })
+
+  it('reports a single-version component with no latest/prerelease flags implied incorrectly', async () => {
+    const onlyVersion = componentVersion({ version: 'prerelease', prerelease: true })
+    const component = { name: 'test-docs', versions: [onlyVersion], latest: onlyVersion }
+
+    const rows = await run([component])
+
+    // A component with only a prerelease version still has SOME latest
+    // (Antora's own fallback — there's no "no latest" state) — both flags
+    // legitimately apply to the same single entry.
+    expect(rows).toEqual(['test-docs: prerelease (latest, prerelease)'])
+  })
+
+  it('reports every component in the catalog', async () => {
+    const v1 = componentVersion({ version: '1.0' })
+    const v2 = componentVersion({ version: '2.0' })
+    const componentA = { name: 'a', versions: [v1], latest: v1 }
+    const componentB = { name: 'b', versions: [v2], latest: v2 }
+
+    const rows = await run([componentA, componentB])
+
+    expect(rows).toEqual(['a: 1.0 (latest)', 'b: 2.0 (latest)'])
+  })
+
+  it('reports a non-latest, non-prerelease version plainly', async () => {
+    const older = componentVersion({ version: '1.0' })
+    const newer = componentVersion({ version: '2.0' })
+    const component = { name: 'test-docs', versions: [older, newer], latest: newer }
+
+    const rows = await run([component])
+
+    expect(rows).toEqual(['test-docs: 1.0, 2.0 (latest)'])
+  })
+})

package/package.json

@@ -0,0 +1,33 @@
+{
+  "name": "@inditextech/docouture-antora-extensions",
+  "version": "0.1.0-SNAPSHOT.40.1",
+  "description": "Antora pipeline extensions shared by docouture documentation sites — per-module navigation metadata (nav_modules)",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/InditexTech/test-antoradocs.git",
+    "directory": "code/packages/antora-extensions"
+  },
+  "license": "MPL-2.0",
+  "main": "index.js",
+  "files": [
+    "index.js",
+    "lib",
+    "resources"
+  ],
+  "engines": {
+    "node": ">=24.0.0"
+  },
+  "dependencies": {
+    "@antora/page-composer": "~3.1.15",
+    "@antora/redirect-producer": "~3.1.15",
+    "@shikijs/langs": "~4.4.0",
+    "@shikijs/themes": "~4.4.0",
+    "node-html-parser": "^9.0.1",
+    "shiki": "~4.4.0",
+    "@inditextech/docouture-asciidoc-extensions": "0.1.0-SNAPSHOT.40.1"
+  },
+  "scripts": {
+    "lint": "eslint .",
+    "test": "vitest run"
+  }
+}