@redpanda-data/docs-extensions-and-macros 5.11.0 → 5.12.0

package/bin/doc-tools.js

@@ -2223,7 +2223,7 @@ const validation = new Command('validate').description('Validate docs data again
  * @why A feature must only be documented as enterprise under its approved
  * external name. This check catches new license-gated features in core that
  * have no registry entry yet, registry pointers that no longer match core,
- * and stale connect enterprise lists.
+ * and registry connect-plugin entries that no longer match info.csv.
  *
  * @example
  * # Check everything against the default remote sources
@@ -2244,58 +2244,27 @@ validation
   .description('Check the enterprise features registry against core, connect, and docs sources of truth')
   .option('--registry <path>', 'Local path to enterprise-features.yml (default: fetch from docs repo main)')
   .option('--tag <ref>', 'Redpanda git ref for the core headers', 'dev')
-  .option('--connect-ref <ref>', 'Connect git ref for info.csv', 'main')
+  .option('--connect-ref <ref>', "Connect git ref for info.csv ('latest' resolves the newest release tag; the registry documents released state)", 'latest')
   .option('--docs-ref <ref>', 'Docs repo git ref for remote fetches', 'main')
   .option('--disable-page <path>', 'Local path to disable-enterprise-features.adoc (default: fetch from docs repo)')
-  .option('--antora <path>', 'Local path to rp-connect-docs antora.yml (default: fetch from rp-connect-docs main)')
   .option('--properties <path>', 'Local path to a generated cluster-properties JSON; enables existence checks for gating properties')
-  .option('--skip-connect', 'Skip the connect info.csv comparison')
+  .option('--skip-connect', 'Skip the connect check (info.csv is not fetched, so registry connect-plugin entries go unverified)')
   .option('--format <format>', 'Output format: text or json', 'text')
   .option('--write-mapping <path>', 'Write the internal-to-external name mapping partial to this path')
   .action(async (options) => {
     const { runChecks, buildMappingPartial } = require('../tools/enterprise-features/verify')
-    const { extractAntoraEnterpriseComponents } = require('../tools/enterprise-features/parsers')
-    const yamlLib = require('js-yaml')
-
-    const RAW = 'https://raw.githubusercontent.com'
-    // Sources that fail to load are reported as error-level findings rather
-    // than silently skipped, so CI cannot pass on a check that never ran.
-    const failedSources = []
-    async function fetchText (url, sourceName) {
-      const resp = await fetch(url)
-      if (!resp.ok) {
-        if (sourceName) {
-          failedSources.push({ level: 'error', check: 'fetch', message: `Could not load ${sourceName} (${url}): ${resp.status} ${resp.statusText}. The related checks did not run.` })
-          return undefined
-        }
-        throw new Error(`Failed to fetch ${url}: ${resp.status} ${resp.statusText}`)
-      }
-      return resp.text()
-    }
-    const readLocal = (p) => fs.readFileSync(path.resolve(p), 'utf8')
+    // Fetch wiring (GitHub auth, rejected-token retry, transient-failure
+    // retries, --skip-connect short-circuits) lives in
+    // cli-utils/enterprise-sources so it is testable without a network.
+    const { loadEnterpriseSources } = require('../cli-utils/enterprise-sources')
 
     try {
-      const registryYaml = options.registry
-        ? readLocal(options.registry)
-        : await fetchText(`${RAW}/redpanda-data/docs/${options.docsRef}/shared/modules/ROOT/partials/enterprise-features.yml`)
-      const coreHeader = await fetchText(`${RAW}/redpanda-data/redpanda/${options.tag}/src/v/features/enterprise_features.h`, 'core enterprise_features.h')
-      const configurationHeader = await fetchText(`${RAW}/redpanda-data/redpanda/${options.tag}/src/v/config/configuration.h`, 'core configuration.h')
-      const infoCsv = options.skipConnect
-        ? undefined
-        : await fetchText(`${RAW}/redpanda-data/connect/${options.connectRef}/internal/plugins/info.csv`, 'connect info.csv')
-      const disablePage = options.disablePage
-        ? readLocal(options.disablePage)
-        : await fetchText(`${RAW}/redpanda-data/docs/${options.docsRef}/modules/get-started/pages/licensing/disable-enterprise-features.adoc`, 'disable-enterprise-features.adoc')
-      const antoraYaml = options.antora
-        ? readLocal(options.antora)
-        : await fetchText(`${RAW}/redpanda-data/rp-connect-docs/main/antora.yml`, 'rp-connect-docs antora.yml')
-      const antoraEnterpriseComponents = antoraYaml
-        ? extractAntoraEnterpriseComponents(yamlLib.load(antoraYaml))
-        : undefined
+      const { registryYaml, coreHeader, configurationHeader, infoCsv, connectRef, disablePage, failedSources } =
+        await loadEnterpriseSources(options)
 
       let allPropertyNames
       if (options.properties) {
-        const propertyData = JSON.parse(readLocal(options.properties))
+        const propertyData = JSON.parse(fs.readFileSync(path.resolve(options.properties), 'utf8'))
         allPropertyNames = Object.keys(propertyData.properties || propertyData)
       }
 
@@ -2304,7 +2273,7 @@ validation
         coreHeader,
         configurationHeader,
         infoCsv,
-        antoraEnterpriseComponents,
+        connectRef,
         disablePage,
         allPropertyNames,
       })

package/cli-utils/enterprise-sources.js

@@ -0,0 +1,244 @@
+'use strict'
+
+/**
+ * Source loading for `doc-tools validate enterprise-features`.
+ *
+ * Assembles the raw text of every source of truth the validator compares
+ * against: the enterprise features registry, the core headers, the connect
+ * plugin list, and the disable-enterprise-features page. Extracted from
+ * bin/doc-tools.js so the fetch wiring (auth, retries, --skip-connect
+ * short-circuits) is testable without a network.
+ */
+
+const fs = require('fs')
+const path = require('path')
+const { getGitHubToken } = require('./github-token')
+
+const RAW = 'https://raw.githubusercontent.com'
+
+// Repos known to be private, so a 404 can legitimately mean "needs auth".
+// Public sources (redpanda core headers, connect info.csv) 404 only when the
+// file is genuinely missing, and suggesting credentials there sends the
+// reader after a problem that does not exist (e.g. during a transient
+// GitHub blip).
+const PRIVATE_REPO_PREFIXES = [
+  `${RAW}/redpanda-data/docs/`,
+  `${RAW}/redpanda-data/rp-connect-docs/`,
+]
+
+const TOKEN_HINT = ' This source is in a private repository: set GIT_CREDENTIALS (or GITHUB_TOKEN / REDPANDA_GITHUB_TOKEN / ACTIONS_BOT_TOKEN) so the fetch can authenticate.'
+const REJECTED_HINT = ' A GitHub token was sent but rejected, so it may be expired or not grant access to this repository (the default Actions GITHUB_TOKEN is scoped to its own repository).'
+
+const TRANSIENT_RETRIES = 3
+const TRANSIENT_RETRY_DELAY_MS = 2000
+
+function isPrivateSource (url) {
+  return PRIVATE_REPO_PREFIXES.some((prefix) => url.startsWith(prefix))
+}
+
+// The response body of a failed fetch is never read; cancel it so the
+// underlying connection is released instead of lingering until GC.
+async function discardBody (resp) {
+  await resp.body?.cancel().catch(() => {})
+}
+
+/**
+ * Build a fetchText function that authenticates raw.githubusercontent.com
+ * requests and records failures for named sources.
+ *
+ * Some sources live in private repos (the docs repo), which
+ * raw.githubusercontent serves as 404 rather than 401 when unauthenticated —
+ * indistinguishable from a genuinely missing file. A token is sent when one
+ * is available so those fetches resolve; public sources are unaffected.
+ *
+ * raw.githubusercontent also returns 404 (not 401) for a REJECTED token,
+ * even on public files, so a stale token in any of the token env vars would
+ * break fetches that work tokenless. On a 404 with a token, the fetch is
+ * retried once without authentication before the 404 is treated as real.
+ *
+ * Transient failures (network errors, 429, 5xx) are retried a few times
+ * with a delay — the same treatment the CGO test gives curl — because a
+ * single blip on any source would otherwise fail the whole run, and the
+ * check runs on a weekly cron where nobody is watching live.
+ *
+ * @param {object} [deps]
+ * @param {string|null} [deps.token] - GitHub token (defaults to getGitHubToken())
+ * @param {Function} [deps.fetchImpl] - fetch implementation (defaults to global fetch)
+ * @param {Function} [deps.warn] - warning sink (defaults to console.warn)
+ * @param {Function} [deps.sleep] - delay implementation, for tests
+ * @returns {{fetchText: Function, failedSources: Array}} fetchText and the
+ *   findings array it appends to. Sources that fail to load are reported as
+ *   error-level findings rather than silently skipped, so CI cannot pass on a
+ *   check that never ran. Failures without a sourceName throw instead.
+ */
+function createSourceFetcher (deps = {}) {
+  const ghToken = deps.token !== undefined ? deps.token : getGitHubToken()
+  const fetchImpl = deps.fetchImpl || ((...args) => fetch(...args))
+  const warn = deps.warn || ((msg) => console.warn(msg))
+  const sleep = deps.sleep || ((ms) => new Promise((resolve) => setTimeout(resolve, ms)))
+  const failedSources = []
+  let warnedRejectedToken = false
+
+  // One HTTP attempt with transient-failure retries. Returns the final
+  // response; network errors that persist through every retry are rethrown.
+  async function attemptFetch (url, opts) {
+    let lastError
+    for (let attempt = 1; attempt <= TRANSIENT_RETRIES; attempt++) {
+      let resp
+      try {
+        resp = await fetchImpl(url, opts)
+      } catch (err) {
+        lastError = err
+        if (attempt < TRANSIENT_RETRIES) await sleep(TRANSIENT_RETRY_DELAY_MS)
+        continue
+      }
+      if (resp.status === 429 || resp.status >= 500) {
+        await discardBody(resp)
+        lastError = null
+        if (attempt < TRANSIENT_RETRIES) {
+          await sleep(TRANSIENT_RETRY_DELAY_MS)
+          continue
+        }
+        return resp
+      }
+      return resp
+    }
+    throw lastError
+  }
+
+  async function fetchText (url, sourceName) {
+    let resp
+    try {
+      resp = await attemptFetch(url, ghToken ? { headers: { Authorization: `Bearer ${ghToken}` } } : undefined)
+    } catch (err) {
+      if (sourceName) {
+        failedSources.push({ level: 'error', check: 'fetch', message: `Could not load ${sourceName} (${url}): ${err.message}. The related checks did not run.` })
+        return undefined
+      }
+      throw new Error(`Failed to fetch ${url}: ${err.message}`)
+    }
+    if (resp.ok) return resp.text()
+    await discardBody(resp)
+
+    let hint = ''
+    if (resp.status === 404 && ghToken) {
+      const retry = await attemptFetch(url)
+      if (retry.ok) {
+        if (!warnedRejectedToken) {
+          warnedRejectedToken = true
+          warn('Warning: the configured GitHub token was rejected by raw.githubusercontent.com (it may be expired); fetching without authentication instead.')
+        }
+        return retry.text()
+      }
+      await discardBody(retry)
+      if (isPrivateSource(url)) hint = REJECTED_HINT
+    } else if (resp.status === 404 && isPrivateSource(url)) {
+      // A tokenless 404 on a known-private repo is the signature of an
+      // unauthenticated fetch, so say so instead of leaving it to look
+      // like a missing file. Public sources get no credential hint: their
+      // 404s mean the file is missing (or GitHub is having a bad moment).
+      hint = TOKEN_HINT
+    }
+
+    if (sourceName) {
+      failedSources.push({ level: 'error', check: 'fetch', message: `Could not load ${sourceName} (${url}): ${resp.status} ${resp.statusText}. The related checks did not run.${hint}` })
+      return undefined
+    }
+    throw new Error(`Failed to fetch ${url}: ${resp.status} ${resp.statusText}.${hint}`)
+  }
+
+  return { fetchText, failedSources }
+}
+
+/**
+ * Resolve the ref to compare connect plugins at.
+ *
+ * Primary: the newest published release tag from the GitHub API. Fallback
+ * when the API is unavailable: the `latest-connect-version` asciidoc
+ * attribute in rp-connect-docs antora.yml — the latest version the docs
+ * actually document, which is the state the registry describes. (This is a
+ * different use of that file than the retired enterprise-components list:
+ * the version attribute is actively maintained by the connect version-update
+ * automation.) If both routes fail, the connect check is skipped with an
+ * error finding rather than silently comparing against main.
+ */
+async function resolveLatestConnectRef ({ fetchText, failedSources, warn }) {
+  try {
+    const releaseJson = await fetchText('https://api.github.com/repos/redpanda-data/connect/releases/latest')
+    const tag = JSON.parse(releaseJson).tag_name
+    if (tag) return String(tag)
+    throw new Error('the GitHub API response has no tag_name')
+  } catch (err) {
+    warn(`Warning: could not resolve the latest connect release from the GitHub API (${err.message}); falling back to the latest documented connect version in rp-connect-docs antora.yml.`)
+  }
+  const antoraYaml = await fetchText(`${RAW}/redpanda-data/rp-connect-docs/main/antora.yml`, 'latest documented connect version (rp-connect-docs antora.yml fallback)')
+  if (antoraYaml === undefined) return undefined // fetch failure already reported
+  const yamlLib = require('js-yaml')
+  let documented
+  try {
+    const attributes = yamlLib.load(antoraYaml)?.asciidoc?.attributes
+    documented = attributes && attributes['latest-connect-version']
+  } catch (err) {
+    failedSources.push({ level: 'error', check: 'fetch', message: `Could not resolve the connect ref: rp-connect-docs antora.yml did not parse (${err.message}). The connect check did not run.` })
+    return undefined
+  }
+  if (!documented) {
+    failedSources.push({ level: 'error', check: 'fetch', message: 'Could not resolve the connect ref: rp-connect-docs antora.yml has no latest-connect-version attribute. The connect check did not run.' })
+    return undefined
+  }
+  const version = String(documented).trim()
+  return version.startsWith('v') ? version : `v${version}`
+}
+
+/**
+ * Load every source the enterprise-features validation consumes.
+ *
+ * The rp-connect-docs antora.yml is deliberately NOT a source anymore:
+ * rp-connect-docs#485 removed the enterprise-components list it carried, so
+ * fetching it bought a cross-repo credential requirement in exchange for a
+ * comparison against nothing. The connect check now validates the registry's
+ * connect-plugin entries directly against info.csv.
+ *
+ * The connect ref defaults to 'latest', which resolves to the latest
+ * published connect release tag. The registry documents released state, so
+ * comparing it against main would flag plugins renamed or newly marked
+ * enterprise ahead of any release. The core headers deliberately stay on
+ * 'dev' for the opposite reason: their check exists to give early warning
+ * of NEW license-gated features before they ship.
+ *
+ * @param {object} options - Parsed CLI options: registry, tag, connectRef,
+ *   docsRef, disablePage, skipConnect.
+ * @param {object} [deps] - Injectable dependencies for tests: token,
+ *   fetchImpl, warn, sleep (see createSourceFetcher), and readLocal.
+ * @returns {Promise<object>} { registryYaml, coreHeader, configurationHeader,
+ *   infoCsv, connectRef, disablePage, failedSources }
+ */
+async function loadEnterpriseSources (options, deps = {}) {
+  const { fetchText, failedSources } = createSourceFetcher(deps)
+  const readLocal = deps.readLocal || ((p) => fs.readFileSync(path.resolve(p), 'utf8'))
+
+  const registryYaml = options.registry
+    ? readLocal(options.registry)
+    : await fetchText(`${RAW}/redpanda-data/docs/${options.docsRef}/shared/modules/ROOT/partials/enterprise-features.yml`)
+  const coreHeader = await fetchText(`${RAW}/redpanda-data/redpanda/${options.tag}/src/v/features/enterprise_features.h`, 'core enterprise_features.h')
+  const configurationHeader = await fetchText(`${RAW}/redpanda-data/redpanda/${options.tag}/src/v/config/configuration.h`, 'core configuration.h')
+
+  let connectRef = options.connectRef
+  let infoCsv
+  if (!options.skipConnect) {
+    if (!connectRef || connectRef === 'latest') {
+      connectRef = await resolveLatestConnectRef({ fetchText, failedSources, warn: deps.warn || ((msg) => console.warn(msg)) })
+    }
+    if (connectRef) {
+      infoCsv = await fetchText(`${RAW}/redpanda-data/connect/${connectRef}/internal/plugins/info.csv`, `connect info.csv (${connectRef})`)
+    }
+  }
+
+  const disablePage = options.disablePage
+    ? readLocal(options.disablePage)
+    : await fetchText(`${RAW}/redpanda-data/docs/${options.docsRef}/modules/get-started/pages/licensing/disable-enterprise-features.adoc`, 'disable-enterprise-features.adoc')
+
+  return { registryYaml, coreHeader, configurationHeader, infoCsv, connectRef, disablePage, failedSources }
+}
+
+module.exports = { createSourceFetcher, loadEnterpriseSources, RAW }

package/cli-utils/self-managed-docs-branch.js

@@ -1,37 +1,45 @@
-const https = require('https');
 const yaml = require('js-yaml');
 const { spawnSync } = require('child_process');
+const { getGitHubToken } = require('./github-token');
+
+const ANTORA_URL = 'https://raw.githubusercontent.com/redpanda-data/docs/main/antora.yml'
+
+const TOKEN_HINT = ' The docs repository is private, so set GIT_CREDENTIALS (or GITHUB_TOKEN / REDPANDA_GITHUB_TOKEN / ACTIONS_BOT_TOKEN) so the fetch can authenticate.'
+const REJECTED_HINT = ' A GitHub token was sent but rejected, so it may be expired or not grant access to the docs repository (the default Actions GITHUB_TOKEN is scoped to its own repository).'
 
 /**
  * Retrieves the current Self-Managed documentation version from the remote antora.yml file.
  *
+ * The docs repository is private, and raw.githubusercontent serves private
+ * content as 404 rather than 401 when unauthenticated, so the fetch sends a
+ * Bearer token when one is available. It also returns 404 (not 401) for a
+ * REJECTED token, so a 404 with a token is retried once without
+ * authentication before being treated as real.
+ *
+ * @param {object} [deps] - Injectable for tests: { fetchImpl, token }.
  * @returns {Promise<string>} Resolves with the version string (for example, "25.1").
  *
  * @throws {Error} If the antora.yml file cannot be fetched, parsed, or if the version field is missing.
  */
-function fetchRemoteAntoraVersion() {
-  const url = 'https://raw.githubusercontent.com/redpanda-data/docs/main/antora.yml'
-  return new Promise((resolve, reject) => {
-    https.get(url, res => {
-      if (res.statusCode !== 200) {
-        return reject(new Error(`Failed to fetch antora.yml: ${res.statusCode}`))
-      }
-      let body = ''
-      res.on('data', chunk => (body += chunk))
-      res.on('end', () => {
-        try {
-          const cfg = yaml.load(body)
-          if (cfg.version == null) {
-            throw new Error('version field missing')
-          }
-          const version = String(cfg.version).trim()
-          resolve(version)
-        } catch (err) {
-          reject(err)
-        }
-      })
-    }).on('error', reject)
-  })
+async function fetchRemoteAntoraVersion(deps = {}) {
+  const fetchImpl = deps.fetchImpl || fetch
+  const ghToken = 'token' in deps ? deps.token : getGitHubToken()
+
+  let resp = await fetchImpl(ANTORA_URL, ghToken ? { headers: { Authorization: `Bearer ${ghToken}` } } : undefined)
+  const hint = ghToken ? REJECTED_HINT : TOKEN_HINT
+  if (resp.status === 404 && ghToken) {
+    await resp.body?.cancel().catch(() => {})
+    resp = await fetchImpl(ANTORA_URL)
+  }
+  if (!resp.ok) {
+    await resp.body?.cancel().catch(() => {})
+    throw new Error(`Failed to fetch antora.yml: ${resp.status}${resp.status === 404 ? hint : ''}`)
+  }
+  const cfg = yaml.load(await resp.text())
+  if (cfg.version == null) {
+    throw new Error('version field missing')
+  }
+  return String(cfg.version).trim()
 }
 
 /**

package/cli-utils/setup-mcp.js

@@ -141,7 +141,8 @@ async function setupMCP(options = {}) {
 
         // Check if configuration matches desired mode
         const isCorrectNpxConfig = mode === 'npx' && existingCommand === 'npx' &&
-                                    existingArgs.includes('doc-tools-mcp');
+                                    existingArgs.includes('doc-tools-mcp') &&
+                                    existingArgs.some(a => a === `--package=${packageName}`);
         const isCorrectLocalConfig = mode === 'local' && existingCommand === 'node' &&
                                       existingArgs[0] === localPath;
 
@@ -149,7 +150,7 @@ async function setupMCP(options = {}) {
           needsUpdate = true;
           console.log(chalk.yellow('⚠ ') + ' MCP server configured but with different setup:');
           console.log(chalk.gray(`  Current: ${existingCommand} ${existingArgs.join(' ')}`));
-          const newSetup = mode === 'local' ? `node ${localPath}` : 'npx -y doc-tools-mcp';
+          const newSetup = mode === 'local' ? `node ${localPath}` : `npx -y --package=${packageName} doc-tools-mcp`;
           console.log(chalk.gray(`  New:     ${newSetup}\n`));
         } else {
           console.log(chalk.green('✓') + ` MCP server already configured correctly\n`);
@@ -197,7 +198,7 @@ async function setupMCP(options = {}) {
   if (mode === 'local') {
     command = `claude mcp add --transport stdio --scope user ${serverName} -- node ${localPath}`;
   } else {
-    command = `claude mcp add --transport stdio --scope user ${serverName} -- npx -y doc-tools-mcp`;
+    command = `claude mcp add --transport stdio --scope user ${serverName} -- npx -y --package=${packageName} doc-tools-mcp`;
   }
 
   // Execute the command