@redpanda-data/docs-extensions-and-macros 5.35.1 → 5.37.1

package/extensions/kapa-source-groups.js

@@ -0,0 +1,180 @@
+/* Publishes the Kapa source-group mapping to the UI as an AsciiDoc attribute.
+ *
+ * Example use in the playbook:
+ *   antora:
+ *     extensions:
+ *       - require: '@redpanda-data/docs-extensions-and-macros/extensions/kapa-source-groups'
+ *
+ * WHY THIS EXISTS
+ * ---------------
+ * docs-ui's get-kapa-source-groups helper needs to turn the page's version
+ * segment into a Kapa source group id, so Ask AI retrieval is scoped to the docs
+ * version the reader is actually on (DOC-1807, DOC-2450). docs-ui does not depend
+ * on this package and must not carry a second copy of the mapping, which would
+ * drift silently. So the mapping travels as an attribute: generated here by
+ * `doc-tools generate kapa-source-groups`, committed to docs-data, and merged
+ * onto every component version at build time.
+ *
+ * Set on every component version rather than only the versioned ones, because the
+ * helper needs the mapping on unversioned pages too. Those resolve to the
+ * mapping's default_segment, and sending no filter there is exactly what caused
+ * DOC-2450: the reporter was on a page with no version of its own.
+ *
+ * DEGRADATION
+ * -----------
+ * Every failure sets nothing and warns. The helper treats a missing attribute as
+ * "send no source group", which is the pre-DOC-2450 behaviour of searching every
+ * version. That is the right way to fail: a wrong group is worse than no group,
+ * because scoping to a group that does not hold the reader's version returns only
+ * Kapa's global sources, silently and with no error.
+ */
+
+const fs = require('fs')
+const path = require('path')
+const { raiseListenerLimit } = require('./util/raise-listener-limit')
+
+const ATTRIBUTE_NAME = 'kapa-source-groups'
+// Sits beside assets/widgets/, which proxy-api-docs.js already fetches from the
+// site origin, so the /api/ path needs no new hostname or auth.
+const ASSET_DIR = 'assets/data'
+const ASSET_FILENAME = 'kapa-source-groups.json'
+const DEFAULT_MAPPING_PATH = path.join(__dirname, '..', 'docs-data', 'kapa-source-groups.json')
+
+/**
+ * Validate the shape the helper depends on, so a malformed mapping is caught at
+ * build time with a named reason rather than producing pages that quietly lose
+ * version scoping.
+ *
+ * @param {*} mapping
+ * @returns {string|null} Reason it is unusable, or null when it is fine
+ */
+function validateMapping (mapping) {
+  if (!mapping || typeof mapping !== 'object') return 'not an object'
+  if (!mapping.segments || typeof mapping.segments !== 'object') return 'no segments object'
+  const segments = Object.keys(mapping.segments)
+  if (segments.length === 0) return 'segments is empty'
+  if (!mapping.default_segment) return 'no default_segment'
+  if (!mapping.segments[mapping.default_segment]) {
+    // The helper falls back to default_segment for every unversioned page, so a
+    // dangling default silently disables scoping across most of the site.
+    return `default_segment "${mapping.default_segment}" is not one of the segments (${segments.join(', ')})`
+  }
+  const missing = segments.filter((s) => !mapping.segments[s] || !mapping.segments[s].group_id)
+  if (missing.length) return `segments missing group_id: ${missing.join(', ')}`
+  return null
+}
+
+module.exports.register = function ({ config = {} } = {}) {
+  const logger = this.getLogger('kapa-source-groups-extension')
+
+  // Three listeners on a generator context that already carries 30+ extensions,
+  // which trips Node's default max-listeners warning. Raised before subscribing,
+  // per __tests__/extensions/raise-listener-limit.test.js.
+  raiseListenerLimit(this)
+  const mappingPath = config.mapping_file ? path.resolve(config.mapping_file) : DEFAULT_MAPPING_PATH
+
+  // Loaded on first use and reused by all three listeners, so a missing or
+  // broken mapping warns once per build rather than once per event.
+  let loaded = false
+  let cached = null
+  function load () {
+    if (loaded) return cached
+    loaded = true
+    cached = readMapping()
+    return cached
+  }
+
+  /** Read + validate, or return null having warned with the consequence. */
+  function readMapping () {
+    let mapping
+    try {
+      if (!fs.existsSync(mappingPath)) {
+        logger.warn(
+          `Kapa source-group mapping not found at ${mappingPath}. Ask AI will search every docs version. ` +
+          'Generate it with: doc-tools generate kapa-source-groups'
+        )
+        return null
+      }
+      mapping = JSON.parse(fs.readFileSync(mappingPath, 'utf8'))
+    } catch (err) {
+      logger.warn(`Could not read the Kapa source-group mapping at ${mappingPath}: ${err.message}. Ask AI will search every docs version.`)
+      return null
+    }
+    const problem = validateMapping(mapping)
+    if (problem) {
+      logger.warn(`Kapa source-group mapping at ${mappingPath} is unusable (${problem}). Ask AI will search every docs version.`)
+      return null
+    }
+    return mapping
+  }
+
+  // site.keys reaches EVERY page, including ones with no component: the 404 page
+  // renders the Ask AI panel but has no page.component or page.componentVersion,
+  // so a component-version attribute alone leaves it unscoped. Its layout pulls
+  // in head and header, both of which already read site.keys, so this is the one
+  // channel that covers it. Set in playbookBuilt because the site UI model is
+  // derived from the playbook.
+  this.on('playbookBuilt', ({ playbook }) => {
+    const mapping = load()
+    if (!mapping) return
+    playbook.site = playbook.site || {}
+    playbook.site.keys = playbook.site.keys || {}
+    if (playbook.site.keys[ATTRIBUTE_NAME] === undefined) {
+      playbook.site.keys[ATTRIBUTE_NAME] = JSON.stringify(mapping)
+    }
+  })
+
+  // The /api/ reference is Bump.sh HTML assembled by docs-site's proxy-api-docs
+  // edge function, which has no Antora page context and cannot import from
+  // node_modules. Publishing the mapping as a site asset lets that function read
+  // the default group at request time, in parallel with the widget fetches it
+  // already does. The alternative -- a UUID pasted into docs-ui's static widget
+  // context or a Netlify env var -- is a second copy of the mapping that drifts
+  // silently the first time a group is recreated.
+  this.on('beforePublish', ({ siteCatalog }) => {
+    const mapping = load()
+    if (!mapping) return
+    siteCatalog.addFile({
+      contents: Buffer.from(JSON.stringify(mapping, null, 2) + '\n', 'utf8'),
+      out: { path: path.join(ASSET_DIR, ASSET_FILENAME) },
+    })
+    logger.info(`Kapa source groups: published ${ASSET_DIR}/${ASSET_FILENAME} for the /api/ proxy.`)
+  })
+
+  this.on('contentClassified', async ({ contentCatalog }) => {
+    const mapping = load()
+    if (!mapping) return
+
+    // Serialised once. The helper accepts a string or an object, but a string is
+    // what survives Antora's attribute handling unchanged, and it keeps the
+    // attribute value comparable between builds.
+    const serialised = JSON.stringify(mapping)
+
+    let versions = 0
+    const components = await contentCatalog.getComponents()
+    for (const component of components) {
+      component.versions.forEach(({ asciidoc }) => {
+        if (!asciidoc) return
+        asciidoc.attributes = asciidoc.attributes || {}
+        // Do not clobber an explicit override: a playbook or antora.yml that
+        // already sets this wins, matching how add-global-attributes.js lets
+        // component attributes beat the shared file.
+        if (asciidoc.attributes[ATTRIBUTE_NAME] === undefined) {
+          asciidoc.attributes[ATTRIBUTE_NAME] = serialised
+        }
+        versions += 1
+      })
+    }
+
+    logger.info(
+      `Kapa source groups: set ${ATTRIBUTE_NAME} on ${versions} component version(s); ` +
+      `${Object.keys(mapping.segments).length} version segments, default "${mapping.default_segment}".`
+    )
+  })
+}
+
+module.exports.validateMapping = validateMapping
+module.exports.ATTRIBUTE_NAME = ATTRIBUTE_NAME
+module.exports.DEFAULT_MAPPING_PATH = DEFAULT_MAPPING_PATH
+module.exports.ASSET_DIR = ASSET_DIR
+module.exports.ASSET_FILENAME = ASSET_FILENAME

package/extensions/util/scan-content-urls.js

@@ -13,6 +13,8 @@
  *
  * A block closes only on a delimiter of the same length as the one that opened
  * it; a longer run, such as the row of dashes in a rendered table, is content.
+ * - indented literal paragraphs: an indented line that opens a block is
+ *   rendered verbatim, so a URL on it is text and not a link
  * - line comments (//), which Asciidoctor drops entirely
  * - inline code spans (`...`)
  *
@@ -60,6 +62,23 @@ const ATTRIBUTE_REFERENCE_RX = /\{[a-zA-Z0-9_][a-zA-Z0-9_-]*\}/
 // prose cannot swallow the rest of the document.
 const MAX_WRAPPED_LABEL_LENGTH = 500
 
+// An indented line that opens a block is a literal paragraph: Asciidoctor
+// renders it verbatim, so any URL on it is text rather than a link. This is
+// where connector descriptions put their shell examples, which is how the
+// three comma-joined gcloud scope strings in google_drive_list_labels were
+// read as one URL and reported broken.
+const INDENTED_RX = /^[ \t]+\S/
+// Indentation does not make a literal paragraph out of a list item, a table
+// cell or a description list: Asciidoctor still parses those, so their links
+// are real. Verified against @asciidoctor/core, marker by marker (-, *, **,
+// ., .., 1., a., |, term::).
+const INDENTED_PARSED_RX = /^[ \t]+(?:[*.-]+[ \t]|\d+\.[ \t]|[a-zA-Z]\.[ \t]|\||\S.*::(?:[ \t]|$))/
+// Lines a block can start after, so an indented line that follows one opens a
+// literal paragraph: a list continuation (+), a block attribute list
+// ([source,yaml]) or a block title (.Title). A blank line and the close of a
+// delimited block do the same and are handled in the scan loop.
+const BLOCK_START_RX = /^(?:\+|\[.*\]|\.\S.*)$/
+
 // A delimiter run, plus the info string a fence may carry. Comment blocks
 // (////) come first in the alternation so a //// line is not mistaken for the
 // start of a line comment.
@@ -108,22 +127,48 @@ function closesBlock (delimiter, open) {
   return delimiter.char === open.char && delimiter.length === open.length && !delimiter.info
 }
 
+/**
+ * True when an indented line at the start of a block opens a literal
+ * paragraph, rather than continuing a construct Asciidoctor still parses.
+ */
+function opensLiteralParagraph (line) {
+  return INDENTED_RX.test(line) && !INDENTED_PARSED_RX.test(line)
+}
+
 function scanContentUrls (content) {
   const matches = []
   const lines = content.split('\n')
   let offset = 0
   let openDelimiter = null
+  // A literal paragraph runs from the indented line that opened it to the next
+  // blank line, and takes any unindented lines in between with it.
+  let inLiteralParagraph = false
+  // A block can start at the top of the file, and again after every line that
+  // ended the last one.
+  let atBlockStart = true
   for (const rawLine of lines) {
     const line = rawLine.endsWith('\r') ? rawLine.slice(0, -1) : rawLine
     const delimiter = blockDelimiter(line)
     if (openDelimiter) {
       // Inside a block, only its own matching delimiter gets out. Anything
       // else on this line is content and is never scanned.
-      if (delimiter && closesBlock(delimiter, openDelimiter)) openDelimiter = null
+      if (delimiter && closesBlock(delimiter, openDelimiter)) {
+        openDelimiter = null
+        atBlockStart = true
+      }
     } else if (delimiter) {
       openDelimiter = delimiter
-    } else if (!LINE_COMMENT_RX.test(line)) {
-      scanLine(line, offset, matches, content)
+      inLiteralParagraph = false
+    } else {
+      const blank = !line.trim()
+      if (inLiteralParagraph) {
+        if (blank) inLiteralParagraph = false
+      } else if (atBlockStart && opensLiteralParagraph(line)) {
+        inLiteralParagraph = true
+      } else if (!LINE_COMMENT_RX.test(line)) {
+        scanLine(line, offset, matches, content)
+      }
+      atBlockStart = blank || BLOCK_START_RX.test(line)
     }
     offset += rawLine.length + 1
   }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@redpanda-data/docs-extensions-and-macros",
-  "version": "5.35.1",
+  "version": "5.37.1",
   "description": "Antora extensions and macros developed for Redpanda documentation.",
   "keywords": [
     "antora",
@@ -92,7 +92,9 @@
     "./macros/enterprise": "./macros/enterprise.js",
     "./macros/prop": "./macros/prop.js",
     "./extensions/property-page-index": "./extensions/property-page-index.js",
-    "./extensions/render-property-descriptions": "./extensions/render-property-descriptions.js"
+    "./extensions/render-property-descriptions": "./extensions/render-property-descriptions.js",
+    "./extensions/kapa-source-groups": "./extensions/kapa-source-groups.js",
+    "./docs-data/kapa-source-groups.json": "./docs-data/kapa-source-groups.json"
   },
   "files": [
     "extensions",
@@ -104,7 +106,9 @@
     "tools",
     "docker-compose",
     "mcp",
-    "docs-data/rpk-overrides.schema.json"
+    "docs-data/rpk-overrides.schema.json",
+    "docs-data/kapa-source-groups.json",
+    "docs-data/kapa-source-groups.schema.json"
   ],
   "license": "ISC",
   "engines": {

package/tools/generate-cli-docs.js

@@ -138,11 +138,19 @@ function parseJSDocComments(sourceFile) {
   // was documented with the first command's prose. That is exactly what
   // happened to `generate migrate-rpcn-descriptions`, which shipped a
   // regenerated section describing migrate-rpcn-metadata.
-  const pattern = /\/\*\*\s*((?:(?!\*\/)[\s\S])*?)\s*\*\/\s*(?:programCli|automation|validation)\s*\.command\(['"]([^'"]+)['"]\)/g;
+  //
+  // The group is captured too, because a command NAME is not unique: the same
+  // name can be registered under both `automation` (generate) and `validation`
+  // (validate). `kapa-source-groups` is, and keying on the bare name alone made
+  // the later registration silently overwrite the earlier, so the
+  // `generate kapa-source-groups` section shipped describing the validate
+  // command, complete with validate examples under a generate usage line.
+  // Same failure as the migrate-rpcn one above, one level up.
+  const pattern = /\/\*\*\s*((?:(?!\*\/)[\s\S])*?)\s*\*\/\s*(programCli|automation|validation)\s*\.command\(['"]([^'"]+)['"]\)/g;
 
   let match;
   while ((match = pattern.exec(content)) !== null) {
-    const [, commentText, commandName] = match;
+    const [, commentText, group, commandName] = match;
 
     // Parse the comment into sections
     const parsed = {
@@ -189,7 +197,12 @@ function parseJSDocComments(sourceFile) {
         .trim();
     }
 
-    comments[commandName] = parsed;
+    // Keyed by the CLI path a reader would type, so `generate x` and
+    // `validate x` cannot collide. The bare name is kept as well, for the
+    // top-level commands registered straight onto programCli.
+    const groupPrefix = { automation: 'generate ', validation: 'validate ', programCli: '' }[group]
+    comments[`${groupPrefix}${commandName}`] = parsed;
+    if (comments[commandName] === undefined) comments[commandName] = parsed;
   }
 
   return comments;
@@ -290,7 +303,9 @@ IMPORTANT: This documentation is auto-generated. Do not edit manually. Run \`npm
     const label = commandPath.join(' ');
     console.log(`${'  '.repeat(level - 1)}Generating docs for: ${label}`);
     const data = parseHelp(getHelpOutput(label));
-    const jsdoc = jsdocs[commandPath[commandPath.length - 1]];
+    // Prefer the full command path, falling back to the bare name so that
+    // commands registered directly on programCli still resolve.
+    const jsdoc = jsdocs[label] || jsdocs[commandPath[commandPath.length - 1]];
     doc += generateCommandDoc(label, data, jsdoc, level);
     for (const child of data.commands) {
       // Commander lists `help [command]` in every group; it is not a command

package/tools/kapa-source-groups/fetch-with-deadline.js

@@ -0,0 +1,33 @@
+/**
+ * fetch() with a deadline that covers the WHOLE exchange, body included.
+ *
+ * The obvious shape, `try { res = await fetch(url, { signal }) } finally
+ * { clearTimeout(timer) }` and then `await res.text()`, only bounds the time to
+ * headers. fetch resolves as soon as headers arrive, the timer is cleared, and a
+ * server that then stalls mid-body holds the caller forever. For a scheduled
+ * check that is a job that never finishes rather than one that exits 2, so
+ * nobody is told and the next run queues behind it.
+ *
+ * @param {typeof fetch} fetchImpl
+ * @param {string} url
+ * @param {RequestInit} init - Merged with the abort signal
+ * @param {number} timeoutMs
+ * @param {(res: Response, signal: AbortSignal) => Promise<*>} read
+ *   Consumes the body while the timer is still armed. Return null for a
+ *   response you do not want to read (a non-OK status, say) so the caller can
+ *   inspect res.status first.
+ * @returns {Promise<{res: Response, body: *}>}
+ */
+async function fetchWithDeadline (fetchImpl, url, init, timeoutMs, read) {
+  const controller = new AbortController()
+  const timer = setTimeout(() => controller.abort(), timeoutMs)
+  try {
+    const res = await fetchImpl(url, { ...init, signal: controller.signal })
+    const body = await read(res, controller.signal)
+    return { res, body }
+  } finally {
+    clearTimeout(timer)
+  }
+}
+
+module.exports = { fetchWithDeadline }