@redpanda-data/docs-extensions-and-macros 5.35.1 → 5.37.0

package/tools/kapa-source-groups/kapa-source-versions.js

@@ -0,0 +1,83 @@
+/**
+ * Which streaming docs versions Kapa has a source for.
+ *
+ * Kapa names one crawl per published version, `Documentation (24.2)`,
+ * `Documentation (current)`, and so on. The sources endpoint exposes no URL,
+ * only the name, so the version is read from the name. That convention is the
+ * contract this check depends on: a crawl named anything else is invisible to it
+ * and reported as missing, which is the right direction to be wrong in.
+ *
+ * A source that exists but sits in no group is reported separately. Unassigned
+ * sources are GLOBAL in Kapa, returned for every query on every version, so a
+ * `Documentation (26.2)` left unassigned would leak 26.2 content into 24.2
+ * readers' answers with no error anywhere.
+ *
+ * The group ids each version's sources sit in are returned too, so the caller
+ * can compare them with the committed mapping. Every deployed surface scopes a
+ * query by the mapping's group_id, not by Kapa's current state, so a group that
+ * was deleted and recreated, or a source moved into another version's group,
+ * leaves the mapping pointing at a group that no longer holds that version.
+ */
+
+const { KAPA_API_BASE } = require('../../cli-utils/kapa-credentials')
+const { fetchAllPages, toList, idOf } = require('./generate-kapa-source-groups.js')
+
+const SOURCE_NAME_RE = /^Documentation \(([^)]+)\)$/
+
+/**
+ * @param {string} name - A Kapa source name
+ * @returns {string|null} The version segment, or null when the name is not a docs crawl
+ */
+function versionFromSourceName (name) {
+  const m = typeof name === 'string' ? name.match(SOURCE_NAME_RE) : null
+  if (!m) return null
+  const v = m[1].trim()
+  // Cloud and RPCN crawls use the same naming but are not streaming versions.
+  return /^(\d+\.\d+|current)$/.test(v) ? v : null
+}
+
+/**
+ * Classify a list of Kapa source objects.
+ *
+ * @param {Array<{name: string, source_groups?: unknown}>} sources
+ * @returns {{covered: Set<string>, unassigned: Set<string>, groups: Map<string, Set<string>>}}
+ *   covered: versions with a source that is in at least one group
+ *   unassigned: versions with a source that is in no group (global)
+ *   groups: for each covered version, the ids of every group its sources sit in
+ */
+function classifySources (sources) {
+  const covered = new Set()
+  const unassigned = new Set()
+  const groups = new Map()
+  for (const s of sources || []) {
+    const v = versionFromSourceName(s && s.name)
+    if (!v) continue
+    const ids = toList(s.source_groups).map(idOf).filter(Boolean)
+    if (ids.length) {
+      covered.add(v)
+      if (!groups.has(v)) groups.set(v, new Set())
+      for (const id of ids) groups.get(v).add(String(id))
+    } else {
+      unassigned.add(v)
+    }
+  }
+  // A version with two sources, one grouped and one not, is covered: the
+  // grouped one scopes correctly. The stray global is still worth a report,
+  // but it is a generator concern (it lists global_sources), not a gap here.
+  for (const v of covered) unassigned.delete(v)
+  return { covered, unassigned, groups }
+}
+
+/**
+ * @param {{apiKey: string, projectId: string, fetchImpl?: typeof fetch}} opts
+ * @returns {Promise<{covered: Set<string>, unassigned: Set<string>, groups: Map<string, Set<string>>}>}
+ */
+async function fetchKapaSourceVersions ({ apiKey, projectId, fetchImpl }) {
+  if (!apiKey) throw new Error('fetchKapaSourceVersions requires an apiKey')
+  if (!projectId) throw new Error('fetchKapaSourceVersions requires a projectId')
+  const url = `${KAPA_API_BASE}/ingestion/v1/projects/${encodeURIComponent(projectId)}/sources/`
+  const sources = await fetchAllPages(url, apiKey, fetchImpl)
+  return classifySources(sources)
+}
+
+module.exports = { versionFromSourceName, classifySources, fetchKapaSourceVersions, SOURCE_NAME_RE }

package/tools/kapa-source-groups/published-segments.js

@@ -0,0 +1,98 @@
+// Reads the docs site's own sitemap to find out which version segments are
+// actually published.
+//
+// WHY THIS EXISTS
+// ---------------
+// `validate kapa-source-groups` compares Kapa's sources and the committed
+// mapping, but both of those are blind to the failure that actually matters: a
+// new docs version being published with no Kapa source group behind it.
+//
+// Cutting a v/X.Y branch in redpanda-data/docs publishes /streaming/X.Y/ because
+// the playbook globs `branches: v/*`, and no file changes in this repo or
+// docs-site. Kapa has no write API, so no source or group appears either. The
+// mapping and Kapa still agree with each other, and every reader on the new
+// version silently falls back to the default segment.
+//
+// So the check needs a third input: what the site publishes. The sitemap is the
+// right source because it reflects what is actually live, rather than what a
+// playbook or a branch list intends.
+
+const { fetchWithDeadline } = require('./fetch-with-deadline')
+
+const SITEMAP_TIMEOUT_MS = 15000
+
+/**
+ * Extract the distinct version segments from a streaming sitemap.
+ *
+ * Parsed with a regex rather than an XML parser on purpose: the only thing wanted
+ * is the path segment after /streaming/, this runs in CI where a dependency is
+ * cost, and a malformed sitemap should yield "no segments found" (which the
+ * caller treats as an error) rather than a parser exception.
+ *
+ * @param {string} xml - Sitemap XML
+ * @returns {string[]} Sorted, de-duplicated version segments
+ */
+function parsePublishedSegments (xml) {
+  const found = new Set()
+  for (const m of String(xml || '').matchAll(/\/streaming\/([^/"'<\s]+)\//g)) {
+    found.add(m[1])
+  }
+  return [...found].sort((a, b) => a.localeCompare(b, 'en', { numeric: true }))
+}
+
+/**
+ * Fetch the published version segments from a docs site.
+ *
+ * @param {object} options
+ * @param {string} options.siteUrl - Site origin, e.g. https://docs.redpanda.com
+ * @param {Function} [options.fetchImpl] - Injectable for tests
+ * @returns {Promise<string[]>}
+ * @throws {Error} On a non-OK response, a network failure, or an empty result
+ */
+async function fetchPublishedSegments ({ siteUrl, fetchImpl = globalThis.fetch } = {}) {
+  if (!siteUrl) throw new Error('fetchPublishedSegments requires a siteUrl')
+  const url = `${String(siteUrl).replace(/\/+$/, '')}/sitemap-streaming.xml`
+
+  let res, body
+  try {
+    // The deadline covers the body read too; see fetch-with-deadline.js for
+    // why clearing the timer after headers alone is not a timeout.
+    ;({ res, body } = await fetchWithDeadline(
+      fetchImpl, url, {}, SITEMAP_TIMEOUT_MS,
+      (r) => (r.ok ? r.text() : null)
+    ))
+  } catch (err) {
+    // A network failure means "could not find out", which the caller must keep
+    // distinct from "a version is missing" so a scheduled run does not file an
+    // issue every time the site is briefly unreachable.
+    throw new Error(`Could not fetch ${url}: ${err.message}`)
+  }
+
+  if (!res.ok) throw new Error(`Could not fetch ${url}: ${res.status} ${res.statusText}`)
+
+  const segments = parsePublishedSegments(body)
+  if (segments.length === 0) {
+    // An empty sitemap and a moved sitemap look identical, and treating either as
+    // "nothing is published" would report every mapped segment as stale.
+    throw new Error(`No /streaming/<version>/ URLs found in ${url}. The sitemap may have moved or changed shape.`)
+  }
+  return segments
+}
+
+/**
+ * URL segments that publish without a durable version behind them.
+ *
+ * docs-site sets `latest_prerelease_version_segment: 'beta'`, so a branch with
+ * `prerelease: true` publishes at /streaming/beta/ for the whole pre-GA cycle
+ * (26.2's ran 2026-06-11 to 2026-07-28). No Kapa group is expected for it, and
+ * a check that reported it as missing would file the same false issue every
+ * Monday of every beta.
+ */
+const PRERELEASE_SEGMENTS = new Set(['beta'])
+
+/** @param {string} segment */
+function isPrereleaseSegment (segment) {
+  return PRERELEASE_SEGMENTS.has(segment)
+}
+
+module.exports = { parsePublishedSegments, fetchPublishedSegments, isPrereleaseSegment, PRERELEASE_SEGMENTS, SITEMAP_TIMEOUT_MS }