@redpanda-data/docs-extensions-and-macros 5.35.0 → 5.37.0

package/docs-data/kapa-source-groups.json

@@ -0,0 +1,119 @@
+{
+  "project_id": "97f44223-f930-4fb9-ae1e-ecd436a4d85c",
+  "parent_group": {
+    "id": "238b3c08-5cf6-4615-937c-fa2f5a2e83a4",
+    "name": "Streaming",
+    "type": "product"
+  },
+  "default_segment": "current",
+  "segments": {
+    "23.3": {
+      "group_id": "625e5538-aa73-4969-a5c1-4c35a0108a94",
+      "group_name": "23.3",
+      "source_ids": [
+        "21672ac9-cc94-4529-8835-55334ede0a18"
+      ],
+      "source_names": [
+        "Documentation (23.3)"
+      ]
+    },
+    "24.1": {
+      "group_id": "29357fc1-30b1-46d4-a9de-e30f9b696b2f",
+      "group_name": "24.1",
+      "source_ids": [
+        "0d68df8c-1f9b-45f6-baa3-25d8bb46b164"
+      ],
+      "source_names": [
+        "Documentation (24.1)"
+      ]
+    },
+    "24.2": {
+      "group_id": "3442b9cb-67db-4800-a320-9def40551b93",
+      "group_name": "24.2",
+      "source_ids": [
+        "11659ad5-b7df-4a06-9e8b-4e9913795889"
+      ],
+      "source_names": [
+        "Documentation (24.2)"
+      ]
+    },
+    "24.3": {
+      "group_id": "11288265-8a9e-4a49-8c63-2f9fbe459b76",
+      "group_name": "24.3",
+      "source_ids": [
+        "f1377953-8bcc-48c9-a0e7-acb8efae701e"
+      ],
+      "source_names": [
+        "Documentation (24.3)"
+      ]
+    },
+    "25.1": {
+      "group_id": "317edba8-6c78-4cca-bad5-1dfe47c9d3dc",
+      "group_name": "25.1",
+      "source_ids": [
+        "6cfe84a2-4d5b-4123-b302-f372a5500d02"
+      ],
+      "source_names": [
+        "Documentation (25.1)"
+      ]
+    },
+    "25.2": {
+      "group_id": "639427c4-06fb-4b8f-8df3-d6a444b85101",
+      "group_name": "25.2",
+      "source_ids": [
+        "4d4d1a66-08e7-46aa-ac72-fea301f96b5f"
+      ],
+      "source_names": [
+        "Documentation (25.2)"
+      ]
+    },
+    "25.3": {
+      "group_id": "65723bc1-941c-4b3b-b5ba-2c601f7b3ba9",
+      "group_name": "25.3",
+      "source_ids": [
+        "d1213d06-6132-47e2-abca-43fe0a85dde6"
+      ],
+      "source_names": [
+        "Documentation (25.3)"
+      ]
+    },
+    "26.1": {
+      "group_id": "f0751d2d-de07-4a9c-8d75-7960121178b7",
+      "group_name": "26.1",
+      "source_ids": [
+        "ec6810ab-6208-4e6b-acef-83da55d45762"
+      ],
+      "source_names": [
+        "Documentation (26.1)"
+      ]
+    },
+    "current": {
+      "group_id": "35567e94-30c7-4608-ab5d-1ff982339d05",
+      "group_name": "current",
+      "source_ids": [
+        "f5c1e522-be3a-45f6-8463-11abc1b35ea9"
+      ],
+      "source_names": [
+        "Documentation (current)"
+      ]
+    }
+  },
+  "global_sources": [
+    "Admin API",
+    "Agentic Data Plane",
+    "Cloud Controlplane API",
+    "Cloud Dataplane API",
+    "Corrected Responses",
+    "Documentation (Cloud)",
+    "Documentation (RPCN)",
+    "GitHub Discussions",
+    "HTTP Proxy API",
+    "pylon",
+    "Recruitment Scam Page",
+    "Redpanda Connect Release Notes",
+    "Redpanda Core Release Notes",
+    "Schema Registry API",
+    "Terraform provider",
+    "YouTube Tutorials"
+  ]
+}

package/docs-data/kapa-source-groups.schema.json

@@ -0,0 +1,73 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "$id": "https://redpanda.com/schemas/kapa-source-groups.schema.json",
+  "title": "Kapa source groups",
+  "description": "Generated mapping from a docs URL version segment to the Kapa source group that scopes retrieval to that version. Produced by `doc-tools generate kapa-source-groups` from Kapa's ingestion API; do not hand-edit. Kapa publishes no write API, so the groups and their source assignments are created in the Kapa dashboard and this file only records what is there. See tools/kapa-source-groups/generate-kapa-source-groups.js.",
+  "type": "object",
+  "additionalProperties": false,
+  "required": ["project_id", "parent_group", "default_segment", "segments", "global_sources"],
+  "properties": {
+    "project_id": {
+      "type": "string",
+      "format": "uuid",
+      "description": "Kapa project the groups belong to. Matches KAPA_PROJECT_ID. Recorded so a mapping generated against the wrong project is obvious rather than silently wrong."
+    },
+    "parent_group": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": ["id", "name", "type"],
+      "description": "The product-level group the version groups hang off. Kapa's hierarchy is two levels and is expressed top-down via sub_groups; there is no parent_id field anywhere in its schema, so the generator finds this by 'the group that has children' rather than by name. Renaming it in the dashboard is therefore safe.",
+      "properties": {
+        "id": { "type": "string", "format": "uuid" },
+        "name": { "type": "string" },
+        "type": {
+          "type": ["string", "null"],
+          "enum": ["product", "version", null],
+          "description": "Kapa requires every group to be typed. The enum appears only in Kapa's API schema, not in any prose doc. Nothing in Kapa's documented retrieval semantics reads it, so it is recorded for humans rather than relied on."
+        }
+      }
+    },
+    "default_segment": {
+      "type": "string",
+      "description": "Segment used for any page with no version of its own: Cloud, Connect, Agentic Data Plane, labs, home, search, the 404 page, and the Bump-rendered /api/ pages. Must be a key of `segments`. Deliberately set rather than left unset: sending no group filter makes Kapa search every version at once, which is the DOC-2450 failure (a reader on an unversioned page was answered from 25.2 instead of current)."
+    },
+    "segments": {
+      "type": "object",
+      "minProperties": 1,
+      "description": "Keyed by the docs URL version segment, not by the Antora `page.version` attribute. The two disagree for the latest release: `latest_version_segment: 'current'` publishes 26.2 at /streaming/current/ while page.version reads 26.2. Kapa source_url values and the URL prefixes in docs-site's lib/platform-filter.mjs both use the segment, so that is what this keys on.",
+      "additionalProperties": { "$ref": "#/$defs/segment" }
+    },
+    "global_sources": {
+      "type": "array",
+      "items": { "type": "string" },
+      "description": "Names of the Kapa sources deliberately assigned to no group. These are the 'global' set: always returned alongside whichever group a query scopes to. Recorded because the design depends on it: Agentic Data Plane, Cloud and Connect content must stay reachable from every page, whatever version the reader is on. Note 'Global' is NOT a real Kapa group despite the dashboard showing a Global row: the API returns no such object and an unassigned source reports source_groups: []."
+    }
+  },
+  "$defs": {
+    "segment": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": ["group_id", "group_name", "source_ids", "source_names"],
+      "properties": {
+        "group_id": {
+          "type": "string",
+          "format": "uuid",
+          "description": "The value to pass as sourceGroupIdsInclude (Agent SDK), sourceGroupIDsInclude (Chat SDK, note the capital ID), or _meta.source_group_ids_include (hosted MCP server). Kapa group IDs are stable across renames."
+        },
+        "group_name": { "type": "string" },
+        "source_ids": {
+          "type": "array",
+          "minItems": 1,
+          "items": { "type": "string", "format": "uuid" },
+          "description": "Sources assigned to this group. minItems is 1 on purpose: a version group with no sources is worse than no group at all, because scoping to it returns only the global set and the reader silently gets nothing version-specific. The generator refuses to emit such a mapping."
+        },
+        "source_names": {
+          "type": "array",
+          "minItems": 1,
+          "items": { "type": "string" },
+          "description": "Human-readable counterpart to source_ids, so a reviewer can sanity-check the mapping without opening the Kapa dashboard."
+        }
+      }
+    }
+  }
+}

package/extensions/kapa-source-groups.js

@@ -0,0 +1,180 @@
+/* Publishes the Kapa source-group mapping to the UI as an AsciiDoc attribute.
+ *
+ * Example use in the playbook:
+ *   antora:
+ *     extensions:
+ *       - require: '@redpanda-data/docs-extensions-and-macros/extensions/kapa-source-groups'
+ *
+ * WHY THIS EXISTS
+ * ---------------
+ * docs-ui's get-kapa-source-groups helper needs to turn the page's version
+ * segment into a Kapa source group id, so Ask AI retrieval is scoped to the docs
+ * version the reader is actually on (DOC-1807, DOC-2450). docs-ui does not depend
+ * on this package and must not carry a second copy of the mapping, which would
+ * drift silently. So the mapping travels as an attribute: generated here by
+ * `doc-tools generate kapa-source-groups`, committed to docs-data, and merged
+ * onto every component version at build time.
+ *
+ * Set on every component version rather than only the versioned ones, because the
+ * helper needs the mapping on unversioned pages too. Those resolve to the
+ * mapping's default_segment, and sending no filter there is exactly what caused
+ * DOC-2450: the reporter was on a page with no version of its own.
+ *
+ * DEGRADATION
+ * -----------
+ * Every failure sets nothing and warns. The helper treats a missing attribute as
+ * "send no source group", which is the pre-DOC-2450 behaviour of searching every
+ * version. That is the right way to fail: a wrong group is worse than no group,
+ * because scoping to a group that does not hold the reader's version returns only
+ * Kapa's global sources, silently and with no error.
+ */
+
+const fs = require('fs')
+const path = require('path')
+const { raiseListenerLimit } = require('./util/raise-listener-limit')
+
+const ATTRIBUTE_NAME = 'kapa-source-groups'
+// Sits beside assets/widgets/, which proxy-api-docs.js already fetches from the
+// site origin, so the /api/ path needs no new hostname or auth.
+const ASSET_DIR = 'assets/data'
+const ASSET_FILENAME = 'kapa-source-groups.json'
+const DEFAULT_MAPPING_PATH = path.join(__dirname, '..', 'docs-data', 'kapa-source-groups.json')
+
+/**
+ * Validate the shape the helper depends on, so a malformed mapping is caught at
+ * build time with a named reason rather than producing pages that quietly lose
+ * version scoping.
+ *
+ * @param {*} mapping
+ * @returns {string|null} Reason it is unusable, or null when it is fine
+ */
+function validateMapping (mapping) {
+  if (!mapping || typeof mapping !== 'object') return 'not an object'
+  if (!mapping.segments || typeof mapping.segments !== 'object') return 'no segments object'
+  const segments = Object.keys(mapping.segments)
+  if (segments.length === 0) return 'segments is empty'
+  if (!mapping.default_segment) return 'no default_segment'
+  if (!mapping.segments[mapping.default_segment]) {
+    // The helper falls back to default_segment for every unversioned page, so a
+    // dangling default silently disables scoping across most of the site.
+    return `default_segment "${mapping.default_segment}" is not one of the segments (${segments.join(', ')})`
+  }
+  const missing = segments.filter((s) => !mapping.segments[s] || !mapping.segments[s].group_id)
+  if (missing.length) return `segments missing group_id: ${missing.join(', ')}`
+  return null
+}
+
+module.exports.register = function ({ config = {} } = {}) {
+  const logger = this.getLogger('kapa-source-groups-extension')
+
+  // Three listeners on a generator context that already carries 30+ extensions,
+  // which trips Node's default max-listeners warning. Raised before subscribing,
+  // per __tests__/extensions/raise-listener-limit.test.js.
+  raiseListenerLimit(this)
+  const mappingPath = config.mapping_file ? path.resolve(config.mapping_file) : DEFAULT_MAPPING_PATH
+
+  // Loaded on first use and reused by all three listeners, so a missing or
+  // broken mapping warns once per build rather than once per event.
+  let loaded = false
+  let cached = null
+  function load () {
+    if (loaded) return cached
+    loaded = true
+    cached = readMapping()
+    return cached
+  }
+
+  /** Read + validate, or return null having warned with the consequence. */
+  function readMapping () {
+    let mapping
+    try {
+      if (!fs.existsSync(mappingPath)) {
+        logger.warn(
+          `Kapa source-group mapping not found at ${mappingPath}. Ask AI will search every docs version. ` +
+          'Generate it with: doc-tools generate kapa-source-groups'
+        )
+        return null
+      }
+      mapping = JSON.parse(fs.readFileSync(mappingPath, 'utf8'))
+    } catch (err) {
+      logger.warn(`Could not read the Kapa source-group mapping at ${mappingPath}: ${err.message}. Ask AI will search every docs version.`)
+      return null
+    }
+    const problem = validateMapping(mapping)
+    if (problem) {
+      logger.warn(`Kapa source-group mapping at ${mappingPath} is unusable (${problem}). Ask AI will search every docs version.`)
+      return null
+    }
+    return mapping
+  }
+
+  // site.keys reaches EVERY page, including ones with no component: the 404 page
+  // renders the Ask AI panel but has no page.component or page.componentVersion,
+  // so a component-version attribute alone leaves it unscoped. Its layout pulls
+  // in head and header, both of which already read site.keys, so this is the one
+  // channel that covers it. Set in playbookBuilt because the site UI model is
+  // derived from the playbook.
+  this.on('playbookBuilt', ({ playbook }) => {
+    const mapping = load()
+    if (!mapping) return
+    playbook.site = playbook.site || {}
+    playbook.site.keys = playbook.site.keys || {}
+    if (playbook.site.keys[ATTRIBUTE_NAME] === undefined) {
+      playbook.site.keys[ATTRIBUTE_NAME] = JSON.stringify(mapping)
+    }
+  })
+
+  // The /api/ reference is Bump.sh HTML assembled by docs-site's proxy-api-docs
+  // edge function, which has no Antora page context and cannot import from
+  // node_modules. Publishing the mapping as a site asset lets that function read
+  // the default group at request time, in parallel with the widget fetches it
+  // already does. The alternative -- a UUID pasted into docs-ui's static widget
+  // context or a Netlify env var -- is a second copy of the mapping that drifts
+  // silently the first time a group is recreated.
+  this.on('beforePublish', ({ siteCatalog }) => {
+    const mapping = load()
+    if (!mapping) return
+    siteCatalog.addFile({
+      contents: Buffer.from(JSON.stringify(mapping, null, 2) + '\n', 'utf8'),
+      out: { path: path.join(ASSET_DIR, ASSET_FILENAME) },
+    })
+    logger.info(`Kapa source groups: published ${ASSET_DIR}/${ASSET_FILENAME} for the /api/ proxy.`)
+  })
+
+  this.on('contentClassified', async ({ contentCatalog }) => {
+    const mapping = load()
+    if (!mapping) return
+
+    // Serialised once. The helper accepts a string or an object, but a string is
+    // what survives Antora's attribute handling unchanged, and it keeps the
+    // attribute value comparable between builds.
+    const serialised = JSON.stringify(mapping)
+
+    let versions = 0
+    const components = await contentCatalog.getComponents()
+    for (const component of components) {
+      component.versions.forEach(({ asciidoc }) => {
+        if (!asciidoc) return
+        asciidoc.attributes = asciidoc.attributes || {}
+        // Do not clobber an explicit override: a playbook or antora.yml that
+        // already sets this wins, matching how add-global-attributes.js lets
+        // component attributes beat the shared file.
+        if (asciidoc.attributes[ATTRIBUTE_NAME] === undefined) {
+          asciidoc.attributes[ATTRIBUTE_NAME] = serialised
+        }
+        versions += 1
+      })
+    }
+
+    logger.info(
+      `Kapa source groups: set ${ATTRIBUTE_NAME} on ${versions} component version(s); ` +
+      `${Object.keys(mapping.segments).length} version segments, default "${mapping.default_segment}".`
+    )
+  })
+}
+
+module.exports.validateMapping = validateMapping
+module.exports.ATTRIBUTE_NAME = ATTRIBUTE_NAME
+module.exports.DEFAULT_MAPPING_PATH = DEFAULT_MAPPING_PATH
+module.exports.ASSET_DIR = ASSET_DIR
+module.exports.ASSET_FILENAME = ASSET_FILENAME

package/extensions/version-fetcher/set-latest-version.js

@@ -10,10 +10,13 @@ module.exports.register = function ({ config }) {
   const GetLatestConnectVersion = require('./get-latest-connect');
   const logger = this.getLogger('set-latest-version-extension');
 
-  const { getGitHubToken } = require('../../cli-utils/github-token');
+  const { getGitHubApiToken } = require('../../cli-utils/github-token');
   // Shared with tools/bundle-openapi.js so there is one major.minor derivation.
   const { toShortVersion } = require('../../cli-utils/version');
-  const token = getGitHubToken();
+  // API token, not the git one: GIT_CREDENTIALS (Antora's clone credential)
+  // is only a last resort here, because on Netlify it is scoped to the docs
+  // content repos and cannot see streaming-enterprise.
+  const token = getGitHubApiToken();
 
   if (!token) {
     // Unauthenticated requests still work for the public repos this
@@ -21,7 +24,7 @@ module.exports.register = function ({ config }) {
     // version now comes from the private streaming-enterprise repo, whose
     // API 404s without a token -- the latest-redpanda-* attributes are then
     // left unset and pages fall back to their antora.yml values.
-    logger.warn('GitHub token not set (GIT_CREDENTIALS, REDPANDA_GITHUB_TOKEN, GITHUB_TOKEN, or GH_TOKEN). The Redpanda version lookup against the private streaming-enterprise repo will fail and leave latest-redpanda-* attributes unset; other lookups proceed unauthenticated.');
+    logger.warn('GitHub token not set (REDPANDA_GITHUB_TOKEN, GITHUB_TOKEN, GH_TOKEN, or GIT_CREDENTIALS). The Redpanda version lookup against the private streaming-enterprise repo will fail and leave latest-redpanda-* attributes unset; other lookups proceed unauthenticated.');
   }
 
   this.on('contentClassified', async ({ contentCatalog }) => {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@redpanda-data/docs-extensions-and-macros",
-  "version": "5.35.0",
+  "version": "5.37.0",
   "description": "Antora extensions and macros developed for Redpanda documentation.",
   "keywords": [
     "antora",
@@ -92,7 +92,9 @@
     "./macros/enterprise": "./macros/enterprise.js",
     "./macros/prop": "./macros/prop.js",
     "./extensions/property-page-index": "./extensions/property-page-index.js",
-    "./extensions/render-property-descriptions": "./extensions/render-property-descriptions.js"
+    "./extensions/render-property-descriptions": "./extensions/render-property-descriptions.js",
+    "./extensions/kapa-source-groups": "./extensions/kapa-source-groups.js",
+    "./docs-data/kapa-source-groups.json": "./docs-data/kapa-source-groups.json"
   },
   "files": [
     "extensions",
@@ -104,7 +106,9 @@
     "tools",
     "docker-compose",
     "mcp",
-    "docs-data/rpk-overrides.schema.json"
+    "docs-data/rpk-overrides.schema.json",
+    "docs-data/kapa-source-groups.json",
+    "docs-data/kapa-source-groups.schema.json"
   ],
   "license": "ISC",
   "engines": {

package/tools/generate-cli-docs.js

@@ -138,11 +138,19 @@ function parseJSDocComments(sourceFile) {
   // was documented with the first command's prose. That is exactly what
   // happened to `generate migrate-rpcn-descriptions`, which shipped a
   // regenerated section describing migrate-rpcn-metadata.
-  const pattern = /\/\*\*\s*((?:(?!\*\/)[\s\S])*?)\s*\*\/\s*(?:programCli|automation|validation)\s*\.command\(['"]([^'"]+)['"]\)/g;
+  //
+  // The group is captured too, because a command NAME is not unique: the same
+  // name can be registered under both `automation` (generate) and `validation`
+  // (validate). `kapa-source-groups` is, and keying on the bare name alone made
+  // the later registration silently overwrite the earlier, so the
+  // `generate kapa-source-groups` section shipped describing the validate
+  // command, complete with validate examples under a generate usage line.
+  // Same failure as the migrate-rpcn one above, one level up.
+  const pattern = /\/\*\*\s*((?:(?!\*\/)[\s\S])*?)\s*\*\/\s*(programCli|automation|validation)\s*\.command\(['"]([^'"]+)['"]\)/g;
 
   let match;
   while ((match = pattern.exec(content)) !== null) {
-    const [, commentText, commandName] = match;
+    const [, commentText, group, commandName] = match;
 
     // Parse the comment into sections
     const parsed = {
@@ -189,7 +197,12 @@ function parseJSDocComments(sourceFile) {
         .trim();
     }
 
-    comments[commandName] = parsed;
+    // Keyed by the CLI path a reader would type, so `generate x` and
+    // `validate x` cannot collide. The bare name is kept as well, for the
+    // top-level commands registered straight onto programCli.
+    const groupPrefix = { automation: 'generate ', validation: 'validate ', programCli: '' }[group]
+    comments[`${groupPrefix}${commandName}`] = parsed;
+    if (comments[commandName] === undefined) comments[commandName] = parsed;
   }
 
   return comments;
@@ -290,7 +303,9 @@ IMPORTANT: This documentation is auto-generated. Do not edit manually. Run \`npm
     const label = commandPath.join(' ');
     console.log(`${'  '.repeat(level - 1)}Generating docs for: ${label}`);
     const data = parseHelp(getHelpOutput(label));
-    const jsdoc = jsdocs[commandPath[commandPath.length - 1]];
+    // Prefer the full command path, falling back to the bare name so that
+    // commands registered directly on programCli still resolve.
+    const jsdoc = jsdocs[label] || jsdocs[commandPath[commandPath.length - 1]];
     doc += generateCommandDoc(label, data, jsdoc, level);
     for (const child of data.commands) {
       // Commander lists `help [command]` in every group; it is not a command

package/tools/get-redpanda-version.js

@@ -2,7 +2,7 @@
 
 const GetLatestRedpandaVersion = require('../extensions/version-fetcher/get-latest-redpanda-version.js');
 const { getPrereleaseFromAntora } = require('../cli-utils/antora-utils.js');
-const { getGitHubToken } = require('../cli-utils/github-token');
+const { getGitHubApiToken } = require('../cli-utils/github-token');
 
 /**
  * Fetches and prints the latest Redpanda version and Docker repository.
@@ -20,7 +20,7 @@ module.exports = async function getRedpandaVersion({ beta = false, fromAntora =
   const owner = 'redpanda-data';
   const repo = 'streaming-enterprise';
 
-  if (!getGitHubToken()) {
+  if (!getGitHubApiToken()) {
     console.error(`❌ redpanda-data/${repo} is a private repository.`);
     console.error('   Set GH_TOKEN, REDPANDA_GITHUB_TOKEN, or GITHUB_TOKEN to a token with access.');
     process.exit(1);

package/tools/kapa-source-groups/fetch-with-deadline.js

@@ -0,0 +1,33 @@
+/**
+ * fetch() with a deadline that covers the WHOLE exchange, body included.
+ *
+ * The obvious shape, `try { res = await fetch(url, { signal }) } finally
+ * { clearTimeout(timer) }` and then `await res.text()`, only bounds the time to
+ * headers. fetch resolves as soon as headers arrive, the timer is cleared, and a
+ * server that then stalls mid-body holds the caller forever. For a scheduled
+ * check that is a job that never finishes rather than one that exits 2, so
+ * nobody is told and the next run queues behind it.
+ *
+ * @param {typeof fetch} fetchImpl
+ * @param {string} url
+ * @param {RequestInit} init - Merged with the abort signal
+ * @param {number} timeoutMs
+ * @param {(res: Response, signal: AbortSignal) => Promise<*>} read
+ *   Consumes the body while the timer is still armed. Return null for a
+ *   response you do not want to read (a non-OK status, say) so the caller can
+ *   inspect res.status first.
+ * @returns {Promise<{res: Response, body: *}>}
+ */
+async function fetchWithDeadline (fetchImpl, url, init, timeoutMs, read) {
+  const controller = new AbortController()
+  const timer = setTimeout(() => controller.abort(), timeoutMs)
+  try {
+    const res = await fetchImpl(url, { ...init, signal: controller.signal })
+    const body = await read(res, controller.signal)
+    return { res, body }
+  } finally {
+    clearTimeout(timer)
+  }
+}
+
+module.exports = { fetchWithDeadline }