@redpanda-data/docs-extensions-and-macros 5.35.0 → 5.37.0

package/tools/kapa-source-groups/generate-kapa-source-groups.js

@@ -0,0 +1,319 @@
+// Standalone module to read Kapa source groups and emit the mapping that tells
+// each docs surface which group to scope a query to.
+// Usage: generateKapaSourceGroups({ apiKey, projectId, defaultSegment })
+//
+// WHY THIS EXISTS
+// ---------------
+// Kapa indexes several separately crawled copies of the Self-Managed docs (one
+// per version). Without scoping, a reader on a 25.1 page gets answers drawn from
+// any of them: one measured call returned the same architecture page six times at
+// six different versions. Source groups fix that at retrieval time, but the group
+// UUIDs live in the Kapa dashboard while the version list lives in Antora, so
+// something has to join the two. This generator is that join.
+//
+// The Kapa side is read-only by design: Kapa publishes no endpoint to create a
+// group or assign a source to one ("we do not provide an API endpoint for
+// uploading or managing sources"). So this reads the state a human created in the
+// dashboard and turns it into a committed artifact the build and the browser can
+// both consume. It never mutates Kapa.
+//
+// RESPONSE SHAPES (observed live 2026-09-04, project 97f44223)
+// -----------------------------------------------------------
+// GET /ingestion/v1/projects/:id/source-groups/
+//   { count, next, previous, results: [
+//       { id, name, type: 'product'|'version', description, sub_groups: [...], created_at, updated_at }
+//   ]}
+//
+// GET /ingestion/v1/projects/:id/sources/
+//   { count, next, previous, results: [
+//       { id, name, type: 'scrape'|'openapi'|'custom_qa'|'youtube'|..., source_groups: [...], ... }
+//   ]}
+//
+// Kapa's published OpenAPI schema declares BOTH `sub_groups` and `source_groups`
+// as `"type":"string"`. That is a spec bug: live responses return arrays. The
+// parsers below accept an array, a single object, a comma-separated string, or
+// null, so a schema fix on Kapa's side cannot break us either way.
+//
+// "GLOBAL" IS NOT A GROUP
+// ----------------------
+// The dashboard shows a "Global sources" row, and Kapa's own docs call it "a
+// special Global group that exists by default" with an ID. Both are misleading.
+// The API returns no such group, and an unassigned source reports
+// `source_groups: []`. Global is simply "assigned to no group". Assigning a
+// source to a version group is what takes it out of the global set; unassigned
+// sources stay global and are returned alongside whichever group a query scopes
+// to. That is what keeps Agentic Data Plane content reachable from every page.
+
+const { KAPA_API_BASE, kapaAuthHeaders } = require('../../cli-utils/kapa-credentials')
+const { fetchWithDeadline } = require('./fetch-with-deadline')
+
+/**
+ * Raised when Kapa's group tree itself is unusable: no groups, no version
+ * sub-groups, a duplicated group name, a group with no sources, or the default
+ * segment's group missing.
+ *
+ * `generate` treats all of these as fatal and names the reason, because every
+ * one of them describes a change someone made in the dashboard that the mapping
+ * cannot represent. The scheduled `validate` command does not call `generate`;
+ * it reads the sources endpoint directly and reports its own four gaps.
+ */
+class KapaGroupTreeError extends Error {
+  constructor (message) {
+    super(message)
+    this.name = 'KapaGroupTreeError'
+    this.code = 'KAPA_GROUP_TREE'
+  }
+}
+
+const REQUEST_TIMEOUT_MS = 15000
+
+/**
+ * Normalise the fields Kapa mis-declares as strings. Accepts an array, a single
+ * object, a comma-separated string of ids, or null/undefined.
+ *
+ * @param {*} value
+ * @returns {Array<object|string>}
+ */
+function toList (value) {
+  if (Array.isArray(value)) return value
+  if (value === null || value === undefined || value === '') return []
+  if (typeof value === 'string') {
+    return value.split(',').map((s) => s.trim()).filter(Boolean)
+  }
+  if (typeof value === 'object') return [value]
+  return []
+}
+
+/** Pull an id out of either an object or a bare id string. */
+function idOf (entry) {
+  if (typeof entry === 'string') return entry
+  return entry && typeof entry === 'object' ? entry.id || null : null
+}
+
+/**
+ * Fetch every page of a Kapa list endpoint.
+ *
+ * Both ingestion endpoints return count/next/previous but declare NO pagination
+ * query parameters, so the only supported way through is to follow `next`. At
+ * 22 sources this project fits in one page today; follow the cursor anyway so it
+ * does not quietly truncate later.
+ *
+ * @param {string} url - Absolute first-page URL
+ * @param {string} apiKey
+ * @param {Function} [fetchImpl] - Injectable for tests
+ * @returns {Promise<object[]>} Concatenated results
+ */
+async function fetchAllPages (url, apiKey, fetchImpl = globalThis.fetch) {
+  const out = []
+  let next = url
+  let guard = 0
+
+  while (next) {
+    if (++guard > 50) throw new Error(`Kapa pagination did not terminate after ${guard - 1} pages: ${url}`)
+
+    // The deadline covers the body read too; see fetch-with-deadline.js for
+    // why clearing the timer after headers alone is not a timeout.
+    const { res, body } = await fetchWithDeadline(
+      fetchImpl, next, { headers: kapaAuthHeaders(apiKey) }, REQUEST_TIMEOUT_MS,
+      (r) => (r.ok ? r.json() : null)
+    )
+
+    if (!res.ok) {
+      // 401/403 is the overwhelmingly likely failure and deserves its own hint:
+      // a wrong-project key returns 403, not an empty list.
+      const hint = res.status === 401 || res.status === 403
+        ? ' Check that KAPA_API_KEY is valid and belongs to the project in KAPA_PROJECT_ID.'
+        : ''
+      throw new Error(`Kapa API request failed: ${res.status} ${res.statusText} for ${next}.${hint}`)
+    }
+
+    if (!Array.isArray(body?.results)) {
+      throw new Error(`Kapa API returned an unexpected shape for ${next}: no results array.`)
+    }
+    out.push(...body.results)
+    next = body.next || null
+  }
+
+  return out
+}
+
+/**
+ * Flatten the group tree into parent + children.
+ *
+ * Kapa's hierarchy is two levels and is expressed top-down: a group carries
+ * `sub_groups`, and there is no parent_id/parent/level field anywhere in the
+ * schema. So children are only discoverable by descending from parents.
+ *
+ * @param {object[]} groups - `results` from the source-groups endpoint
+ * @returns {{parents: object[], childrenByParentId: Map<string, object[]>}}
+ */
+function flattenGroups (groups) {
+  const parents = []
+  const childrenByParentId = new Map()
+
+  for (const group of groups) {
+    const subs = toList(group.sub_groups).filter((s) => typeof s === 'object')
+    parents.push(group)
+    childrenByParentId.set(group.id, subs)
+  }
+
+  return { parents, childrenByParentId }
+}
+
+/**
+ * Build the segment -> group mapping.
+ *
+ * @param {object} options
+ * @param {string} options.apiKey
+ * @param {string} options.projectId
+ * @param {string} [options.defaultSegment='current'] - Segment used for pages with no version
+ * @param {string} [options.parentGroupName] - Restrict to one parent by name (default: the only parent that has children)
+ * @param {Function} [options.fetchImpl]
+ * @returns {Promise<string>} Pretty-printed JSON, newline-terminated
+ */
+async function generateKapaSourceGroups ({
+  apiKey,
+  projectId,
+  defaultSegment = 'current',
+  parentGroupName,
+  fetchImpl,
+} = {}) {
+  if (!apiKey) throw new Error('generateKapaSourceGroups requires an apiKey')
+  if (!projectId) throw new Error('generateKapaSourceGroups requires a projectId')
+
+  const base = `${KAPA_API_BASE}/ingestion/v1/projects/${encodeURIComponent(projectId)}`
+  const [groups, sources] = await Promise.all([
+    fetchAllPages(`${base}/source-groups/`, apiKey, fetchImpl),
+    fetchAllPages(`${base}/sources/`, apiKey, fetchImpl),
+  ])
+
+  if (groups.length === 0) {
+    // An empty list looks identical to "the dashboard work has not been done
+    // yet". Refuse rather than writing an empty mapping that would make every
+    // consumer silently fall back to unscoped retrieval.
+    throw new KapaGroupTreeError(
+      'Kapa returned no source groups. Create the parent group and its version sub groups in the ' +
+      'Kapa dashboard (Sources > Manage groups) before running this generator.'
+    )
+  }
+
+  const { parents, childrenByParentId } = flattenGroups(groups)
+
+  const candidates = parentGroupName
+    ? parents.filter((p) => p.name === parentGroupName)
+    : parents.filter((p) => (childrenByParentId.get(p.id) || []).length > 0)
+
+  if (candidates.length === 0) {
+    const names = parents.map((p) => `"${p.name}"`).join(', ')
+    throw new KapaGroupTreeError(
+      parentGroupName
+        ? `No source group named "${parentGroupName}". Groups present: ${names}.`
+        : 'No source group has any sub groups, so there are no version groups to map. ' +
+          `Add version sub groups under a parent in the Kapa dashboard. Groups present: ${names}.`
+    )
+  }
+  if (candidates.length > 1) {
+    const names = candidates.map((p) => `"${p.name}"`).join(', ')
+    throw new KapaGroupTreeError(
+      `Ambiguous parent group: ${names} all have sub groups. Pass --parent-group to choose one.`
+    )
+  }
+
+  const parent = candidates[0]
+  const children = childrenByParentId.get(parent.id) || []
+
+  // Which sources are assigned to each group. A version group with no sources
+  // is worse than no group at all: scoping to it returns only the global set, so
+  // the reader silently gets no version-specific content. Surface it.
+  const sourcesByGroupId = new Map()
+  for (const source of sources) {
+    for (const entry of toList(source.source_groups)) {
+      const gid = idOf(entry)
+      if (!gid) continue
+      if (!sourcesByGroupId.has(gid)) sourcesByGroupId.set(gid, [])
+      sourcesByGroupId.get(gid).push({ id: source.id, name: source.name })
+    }
+  }
+
+  const segments = {}
+  const empty = []
+  for (const child of children) {
+    const assigned = (sourcesByGroupId.get(child.id) || [])
+      .slice()
+      .sort((a, b) => a.name.localeCompare(b.name))
+    if (assigned.length === 0) empty.push(child.name)
+    if (Object.prototype.hasOwnProperty.call(segments, child.name)) {
+      // Silently overwriting makes half that version's sources unreachable at
+      // retrieval, and which group_id survives depends on the order Kapa
+      // returns sub_groups in, which it does not guarantee. So the committed
+      // file flaps between runs and the drift issue reads only "group id
+      // changed", with no hint that two groups share a name. Duplicating a name
+      // is an easy dashboard slip: re-creating a group instead of editing it.
+      throw new KapaGroupTreeError(
+        `Two sub groups under "${parent.name}" are both named "${child.name}" ` +
+        `(${segments[child.name].group_id} and ${child.id}). Rename or delete one in the Kapa ` +
+        'dashboard: a segment can map to only one group, and which one won here would depend on ' +
+        'the order Kapa happens to return them in.'
+      )
+    }
+    segments[child.name] = {
+      group_id: child.id,
+      group_name: child.name,
+      // Sorted so a reordering upstream cannot produce a docs diff.
+      source_ids: assigned.map((s) => s.id).sort(),
+      source_names: assigned.map((s) => s.name),
+    }
+  }
+
+  if (!segments[defaultSegment]) {
+    throw new KapaGroupTreeError(
+      `Default segment "${defaultSegment}" is not one of the version groups ` +
+      `(${Object.keys(segments).sort().join(', ') || 'none'}). Pages with no version would have ` +
+      'nothing to scope to, so this must be fixed before the mapping is usable.'
+    )
+  }
+
+  if (empty.length) {
+    throw new KapaGroupTreeError(
+      `These version groups have no sources assigned: ${empty.sort().join(', ')}. ` +
+      'Scoping a query to an empty group returns only global sources, so the reader would get no ' +
+      'version-specific content at all. Assign each Documentation (X) source to its group in the ' +
+      'Kapa dashboard (Sources > Configure), then re-run.'
+    )
+  }
+
+  // Sources deliberately left unassigned. These are the global set: always
+  // returned alongside whichever group a query scopes to. Recorded because the
+  // whole design depends on it (Agentic Data Plane must stay reachable from
+  // every page), so a future reader can see it was a choice, not an oversight.
+  const globalSources = sources
+    .filter((s) => toList(s.source_groups).length === 0)
+    .map((s) => s.name)
+    .sort((a, b) => a.localeCompare(b))
+
+  // Keys sorted throughout: this file is diff-checked in CI, so a stable order
+  // is what makes "no changes" mean "nothing changed upstream". Note there is
+  // deliberately no generated-at timestamp -- that is exactly what stops
+  // cloud-regions from being diff-checkable.
+  const mapping = {
+    project_id: projectId,
+    parent_group: { id: parent.id, name: parent.name, type: parent.type || null },
+    default_segment: defaultSegment,
+    segments: Object.fromEntries(
+      Object.keys(segments).sort((a, b) => a.localeCompare(b, 'en', { numeric: true })).map((k) => [k, segments[k]])
+    ),
+    global_sources: globalSources,
+  }
+
+  return `${JSON.stringify(mapping, null, 2)}\n`
+}
+
+module.exports = {
+  KapaGroupTreeError,
+  generateKapaSourceGroups,
+  // Exported for unit tests.
+  toList,
+  idOf,
+  flattenGroups,
+  fetchAllPages,
+}

package/tools/kapa-source-groups/kapa-source-versions.js

@@ -0,0 +1,83 @@
+/**
+ * Which streaming docs versions Kapa has a source for.
+ *
+ * Kapa names one crawl per published version, `Documentation (24.2)`,
+ * `Documentation (current)`, and so on. The sources endpoint exposes no URL,
+ * only the name, so the version is read from the name. That convention is the
+ * contract this check depends on: a crawl named anything else is invisible to it
+ * and reported as missing, which is the right direction to be wrong in.
+ *
+ * A source that exists but sits in no group is reported separately. Unassigned
+ * sources are GLOBAL in Kapa, returned for every query on every version, so a
+ * `Documentation (26.2)` left unassigned would leak 26.2 content into 24.2
+ * readers' answers with no error anywhere.
+ *
+ * The group ids each version's sources sit in are returned too, so the caller
+ * can compare them with the committed mapping. Every deployed surface scopes a
+ * query by the mapping's group_id, not by Kapa's current state, so a group that
+ * was deleted and recreated, or a source moved into another version's group,
+ * leaves the mapping pointing at a group that no longer holds that version.
+ */
+
+const { KAPA_API_BASE } = require('../../cli-utils/kapa-credentials')
+const { fetchAllPages, toList, idOf } = require('./generate-kapa-source-groups.js')
+
+const SOURCE_NAME_RE = /^Documentation \(([^)]+)\)$/
+
+/**
+ * @param {string} name - A Kapa source name
+ * @returns {string|null} The version segment, or null when the name is not a docs crawl
+ */
+function versionFromSourceName (name) {
+  const m = typeof name === 'string' ? name.match(SOURCE_NAME_RE) : null
+  if (!m) return null
+  const v = m[1].trim()
+  // Cloud and RPCN crawls use the same naming but are not streaming versions.
+  return /^(\d+\.\d+|current)$/.test(v) ? v : null
+}
+
+/**
+ * Classify a list of Kapa source objects.
+ *
+ * @param {Array<{name: string, source_groups?: unknown}>} sources
+ * @returns {{covered: Set<string>, unassigned: Set<string>, groups: Map<string, Set<string>>}}
+ *   covered: versions with a source that is in at least one group
+ *   unassigned: versions with a source that is in no group (global)
+ *   groups: for each covered version, the ids of every group its sources sit in
+ */
+function classifySources (sources) {
+  const covered = new Set()
+  const unassigned = new Set()
+  const groups = new Map()
+  for (const s of sources || []) {
+    const v = versionFromSourceName(s && s.name)
+    if (!v) continue
+    const ids = toList(s.source_groups).map(idOf).filter(Boolean)
+    if (ids.length) {
+      covered.add(v)
+      if (!groups.has(v)) groups.set(v, new Set())
+      for (const id of ids) groups.get(v).add(String(id))
+    } else {
+      unassigned.add(v)
+    }
+  }
+  // A version with two sources, one grouped and one not, is covered: the
+  // grouped one scopes correctly. The stray global is still worth a report,
+  // but it is a generator concern (it lists global_sources), not a gap here.
+  for (const v of covered) unassigned.delete(v)
+  return { covered, unassigned, groups }
+}
+
+/**
+ * @param {{apiKey: string, projectId: string, fetchImpl?: typeof fetch}} opts
+ * @returns {Promise<{covered: Set<string>, unassigned: Set<string>, groups: Map<string, Set<string>>}>}
+ */
+async function fetchKapaSourceVersions ({ apiKey, projectId, fetchImpl }) {
+  if (!apiKey) throw new Error('fetchKapaSourceVersions requires an apiKey')
+  if (!projectId) throw new Error('fetchKapaSourceVersions requires a projectId')
+  const url = `${KAPA_API_BASE}/ingestion/v1/projects/${encodeURIComponent(projectId)}/sources/`
+  const sources = await fetchAllPages(url, apiKey, fetchImpl)
+  return classifySources(sources)
+}
+
+module.exports = { versionFromSourceName, classifySources, fetchKapaSourceVersions, SOURCE_NAME_RE }

package/tools/kapa-source-groups/published-segments.js

@@ -0,0 +1,98 @@
+// Reads the docs site's own sitemap to find out which version segments are
+// actually published.
+//
+// WHY THIS EXISTS
+// ---------------
+// `validate kapa-source-groups` compares Kapa's sources and the committed
+// mapping, but both of those are blind to the failure that actually matters: a
+// new docs version being published with no Kapa source group behind it.
+//
+// Cutting a v/X.Y branch in redpanda-data/docs publishes /streaming/X.Y/ because
+// the playbook globs `branches: v/*`, and no file changes in this repo or
+// docs-site. Kapa has no write API, so no source or group appears either. The
+// mapping and Kapa still agree with each other, and every reader on the new
+// version silently falls back to the default segment.
+//
+// So the check needs a third input: what the site publishes. The sitemap is the
+// right source because it reflects what is actually live, rather than what a
+// playbook or a branch list intends.
+
+const { fetchWithDeadline } = require('./fetch-with-deadline')
+
+const SITEMAP_TIMEOUT_MS = 15000
+
+/**
+ * Extract the distinct version segments from a streaming sitemap.
+ *
+ * Parsed with a regex rather than an XML parser on purpose: the only thing wanted
+ * is the path segment after /streaming/, this runs in CI where a dependency is
+ * cost, and a malformed sitemap should yield "no segments found" (which the
+ * caller treats as an error) rather than a parser exception.
+ *
+ * @param {string} xml - Sitemap XML
+ * @returns {string[]} Sorted, de-duplicated version segments
+ */
+function parsePublishedSegments (xml) {
+  const found = new Set()
+  for (const m of String(xml || '').matchAll(/\/streaming\/([^/"'<\s]+)\//g)) {
+    found.add(m[1])
+  }
+  return [...found].sort((a, b) => a.localeCompare(b, 'en', { numeric: true }))
+}
+
+/**
+ * Fetch the published version segments from a docs site.
+ *
+ * @param {object} options
+ * @param {string} options.siteUrl - Site origin, e.g. https://docs.redpanda.com
+ * @param {Function} [options.fetchImpl] - Injectable for tests
+ * @returns {Promise<string[]>}
+ * @throws {Error} On a non-OK response, a network failure, or an empty result
+ */
+async function fetchPublishedSegments ({ siteUrl, fetchImpl = globalThis.fetch } = {}) {
+  if (!siteUrl) throw new Error('fetchPublishedSegments requires a siteUrl')
+  const url = `${String(siteUrl).replace(/\/+$/, '')}/sitemap-streaming.xml`
+
+  let res, body
+  try {
+    // The deadline covers the body read too; see fetch-with-deadline.js for
+    // why clearing the timer after headers alone is not a timeout.
+    ;({ res, body } = await fetchWithDeadline(
+      fetchImpl, url, {}, SITEMAP_TIMEOUT_MS,
+      (r) => (r.ok ? r.text() : null)
+    ))
+  } catch (err) {
+    // A network failure means "could not find out", which the caller must keep
+    // distinct from "a version is missing" so a scheduled run does not file an
+    // issue every time the site is briefly unreachable.
+    throw new Error(`Could not fetch ${url}: ${err.message}`)
+  }
+
+  if (!res.ok) throw new Error(`Could not fetch ${url}: ${res.status} ${res.statusText}`)
+
+  const segments = parsePublishedSegments(body)
+  if (segments.length === 0) {
+    // An empty sitemap and a moved sitemap look identical, and treating either as
+    // "nothing is published" would report every mapped segment as stale.
+    throw new Error(`No /streaming/<version>/ URLs found in ${url}. The sitemap may have moved or changed shape.`)
+  }
+  return segments
+}
+
+/**
+ * URL segments that publish without a durable version behind them.
+ *
+ * docs-site sets `latest_prerelease_version_segment: 'beta'`, so a branch with
+ * `prerelease: true` publishes at /streaming/beta/ for the whole pre-GA cycle
+ * (26.2's ran 2026-06-11 to 2026-07-28). No Kapa group is expected for it, and
+ * a check that reported it as missing would file the same false issue every
+ * Monday of every beta.
+ */
+const PRERELEASE_SEGMENTS = new Set(['beta'])
+
+/** @param {string} segment */
+function isPrereleaseSegment (segment) {
+  return PRERELEASE_SEGMENTS.has(segment)
+}
+
+module.exports = { parsePublishedSegments, fetchPublishedSegments, isPrereleaseSegment, PRERELEASE_SEGMENTS, SITEMAP_TIMEOUT_MS }

package/tools/property-extractor/rp-util-fetch.js

@@ -4,7 +4,7 @@ const { spawnSync } = require('child_process')
 const fs = require('fs')
 const os = require('os')
 const path = require('path')
-const { getGitHubToken } = require('../../cli-utils/github-token')
+const { getGitHubToken, getGitHubApiToken } = require('../../cli-utils/github-token')
 
 /**
  * rp_util's config schema JSON, for any streaming-enterprise ref (tag,
@@ -287,7 +287,7 @@ const RELEASE_ASSET_TO_SCHEMA_KEY = {
  *   no release exists for this tag (caller should fall back to building)
  */
 async function fetchPublishedSchema(tag) {
-  const token = getGitHubToken()
+  const token = getGitHubApiToken()
   const headers = token ? { Authorization: `Bearer ${token}` } : {}
 
   const releaseResp = await fetch(

package/tools/rpk-docs/rpk-docs-handler.js

@@ -1324,8 +1324,8 @@ function downloadRpkRelease(tag, destDir) {
   // redpanda-data/redpanda. This function already has a fallback (the
   // caller builds from source instead), so a missing token warns and
   // returns null here rather than throwing the way the clone-only paths do.
-  const { getGitHubToken } = require('../../cli-utils/github-token')
-  const token = getGitHubToken()
+  const { getGitHubApiToken } = require('../../cli-utils/github-token')
+  const token = getGitHubApiToken()
   if (!token) {
     console.warn('No GitHub token available for the private streaming-enterprise release download; falling back to a source build.')
     return null