@redpanda-data/docs-extensions-and-macros 5.10.0 → 5.11.1

package/bin/doc-tools.js

@@ -1279,6 +1279,56 @@ automation
     }
   })
 
+/**
+ * generate rpk-env-partial
+ *
+ * @description
+ * Generates the -X option -> RPK_* environment variable mapping table as an
+ * AsciiDoc partial from rpk's own -X option data, so the table cannot drift
+ * from the CLI. Prefers the structured x_options array in `rpk --print-tree`
+ * (which carries the env var names rpk itself derives) and falls back to
+ * parsing `-X list` text for rpk versions that predate it. Hidden -X options
+ * appear in neither source, so they are excluded automatically.
+ *
+ * The main rpk-docs pipeline also writes this partial from the tree it
+ * already holds; this standalone command is for targeted refreshes without
+ * a full generation run.
+ *
+ * @example
+ * # Generate for a release tag (sparse-clones redpanda, builds rpk with Go)
+ * npx doc-tools generate rpk-env-partial --ref v26.2.1 --output modules/reference/partials/rpk-env-vars.adoc
+ *
+ * # Use a local source checkout (as-is, no checkout changes), an existing
+ * # rpk binary, or a versioned tree snapshot (no clone or build at all)
+ * npx doc-tools generate rpk-env-partial --from-source ~/redpanda --output modules/reference/partials/rpk-env-vars.adoc
+ * npx doc-tools generate rpk-env-partial --rpk-bin "$(command -v rpk)" --output modules/reference/partials/rpk-env-vars.adoc
+ * npx doc-tools generate rpk-env-partial --from-json docs-data/rpk-v26.2.1.json --output modules/reference/partials/rpk-env-vars.adoc
+ */
+automation
+  .command('rpk-env-partial')
+  .description('Generate the -X -> RPK_* env var mapping partial from rpk -X list output.')
+  .option('-r, --ref <ref>', 'Git branch or tag to build rpk from (e.g., dev, v26.2.1). Clones from GitHub.')
+  .option('--from-source <path>', 'Path to local rpk source (src/go/rpk directory)')
+  .option('--rpk-bin <path>', 'Path to an existing rpk binary (skips clone and build)')
+  .option('--from-json <path>', 'Versioned tree snapshot from docs-data (skips clone and build; requires a snapshot with x_options)')
+  .option('--output <path>', 'Path to write the partial to', 'modules/reference/partials/rpk-env-vars.adoc')
+  .action((options) => {
+    try {
+      const { handleXEnvPartialGeneration } = require('../tools/rpk-docs/generate-x-env-partial.js')
+      const result = handleXEnvPartialGeneration({
+        ref: options.ref,
+        fromSource: options.fromSource,
+        rpkBin: options.rpkBin,
+        fromJson: options.fromJson,
+        output: options.output
+      })
+      console.log(`Wrote ${result.keyCount} -X option mappings to ${result.output} (source: ${result.source})`)
+    } catch (err) {
+      console.error(`Error: ${err.message}`)
+      process.exit(1)
+    }
+  })
+
 /**
  * generate rpk-plugin-stubs
  *
@@ -1677,7 +1727,50 @@ automation
       if (r.status !== 0) process.exit(r.status)
 
       const { formatHelmSpec } = require('../cli-utils/format-helm-spec')
-      fs.writeFileSync(outFile, formatHelmSpec(fs.readFileSync(outFile, 'utf8')), 'utf8')
+      let doc = formatHelmSpec(fs.readFileSync(outFile, 'utf8'))
+
+      // helm-docs only renders keys that exist in the YAML tree, so optional
+      // values that ship commented out never appear in the reference. Pull
+      // documented commented-out keys from values.yaml and inject them. This
+      // runs after formatHelmSpec so the injected body sections are not affected
+      // by the blank-line normalization that its header fix performs.
+      const valuesFile = path.join(chartPath, 'values.yaml')
+      if (fs.existsSync(valuesFile)) {
+        const { extractCommentedValueDocs, injectIntoAsciiDoc, filterEntriesBySchema } = require('../cli-utils/helm-commented-values')
+        try {
+          let entries = extractCommentedValueDocs(fs.readFileSync(valuesFile, 'utf8'))
+
+          // Deprecation notices and other prose can produce paths the chart
+          // no longer accepts (for example the removed
+          // storage.tiered.credentialsSecretRef.configurationKey). When the
+          // chart ships a values schema, drop any path it provably rejects.
+          const schemaFile = path.join(chartPath, 'values.schema.json')
+          if (entries.length > 0 && fs.existsSync(schemaFile)) {
+            const schema = JSON.parse(fs.readFileSync(schemaFile, 'utf8'))
+            const { accepted, rejected } = filterEntriesBySchema(entries, schema)
+            if (rejected.length > 0) {
+              console.warn(`Warning: Skipping ${rejected.length} commented-out value(s) rejected by values.schema.json: ${rejected.map((e) => e.path).join(', ')}`)
+            }
+            entries = accepted
+          }
+
+          if (entries.length > 0) {
+            const { doc: withInjected, injected, sectionsFound } = injectIntoAsciiDoc(doc, entries)
+            doc = withInjected
+            if (injected.length > 0) {
+              console.log(`Documented ${injected.length} commented-out value(s): ${injected.join(', ')}`)
+            } else if (sectionsFound === 0) {
+              // Without this, a heading-level change upstream would silently
+              // drop every documented commented-out value from the reference.
+              console.warn(`Warning: No value section headings found in ${name} to anchor commented-out value docs; dropped: ${entries.map((e) => e.path).join(', ')}`)
+            }
+          }
+        } catch (err) {
+          console.warn(`Warning: Skipping commented-out value docs for ${name}: ${err.message}`)
+        }
+      }
+
+      fs.writeFileSync(outFile, doc, 'utf8')
 
       console.log(`Done: Wrote ${outFile}`)
     }
@@ -2130,7 +2223,7 @@ const validation = new Command('validate').description('Validate docs data again
  * @why A feature must only be documented as enterprise under its approved
  * external name. This check catches new license-gated features in core that
  * have no registry entry yet, registry pointers that no longer match core,
- * and stale connect enterprise lists.
+ * and registry connect-plugin entries that no longer match info.csv.
  *
  * @example
  * # Check everything against the default remote sources
@@ -2151,58 +2244,27 @@ validation
   .description('Check the enterprise features registry against core, connect, and docs sources of truth')
   .option('--registry <path>', 'Local path to enterprise-features.yml (default: fetch from docs repo main)')
   .option('--tag <ref>', 'Redpanda git ref for the core headers', 'dev')
-  .option('--connect-ref <ref>', 'Connect git ref for info.csv', 'main')
+  .option('--connect-ref <ref>', "Connect git ref for info.csv ('latest' resolves the newest release tag; the registry documents released state)", 'latest')
   .option('--docs-ref <ref>', 'Docs repo git ref for remote fetches', 'main')
   .option('--disable-page <path>', 'Local path to disable-enterprise-features.adoc (default: fetch from docs repo)')
-  .option('--antora <path>', 'Local path to rp-connect-docs antora.yml (default: fetch from rp-connect-docs main)')
   .option('--properties <path>', 'Local path to a generated cluster-properties JSON; enables existence checks for gating properties')
-  .option('--skip-connect', 'Skip the connect info.csv comparison')
+  .option('--skip-connect', 'Skip the connect check (info.csv is not fetched, so registry connect-plugin entries go unverified)')
   .option('--format <format>', 'Output format: text or json', 'text')
   .option('--write-mapping <path>', 'Write the internal-to-external name mapping partial to this path')
   .action(async (options) => {
     const { runChecks, buildMappingPartial } = require('../tools/enterprise-features/verify')
-    const { extractAntoraEnterpriseComponents } = require('../tools/enterprise-features/parsers')
-    const yamlLib = require('js-yaml')
-
-    const RAW = 'https://raw.githubusercontent.com'
-    // Sources that fail to load are reported as error-level findings rather
-    // than silently skipped, so CI cannot pass on a check that never ran.
-    const failedSources = []
-    async function fetchText (url, sourceName) {
-      const resp = await fetch(url)
-      if (!resp.ok) {
-        if (sourceName) {
-          failedSources.push({ level: 'error', check: 'fetch', message: `Could not load ${sourceName} (${url}): ${resp.status} ${resp.statusText}. The related checks did not run.` })
-          return undefined
-        }
-        throw new Error(`Failed to fetch ${url}: ${resp.status} ${resp.statusText}`)
-      }
-      return resp.text()
-    }
-    const readLocal = (p) => fs.readFileSync(path.resolve(p), 'utf8')
+    // Fetch wiring (GitHub auth, rejected-token retry, transient-failure
+    // retries, --skip-connect short-circuits) lives in
+    // cli-utils/enterprise-sources so it is testable without a network.
+    const { loadEnterpriseSources } = require('../cli-utils/enterprise-sources')
 
     try {
-      const registryYaml = options.registry
-        ? readLocal(options.registry)
-        : await fetchText(`${RAW}/redpanda-data/docs/${options.docsRef}/shared/modules/ROOT/partials/enterprise-features.yml`)
-      const coreHeader = await fetchText(`${RAW}/redpanda-data/redpanda/${options.tag}/src/v/features/enterprise_features.h`, 'core enterprise_features.h')
-      const configurationHeader = await fetchText(`${RAW}/redpanda-data/redpanda/${options.tag}/src/v/config/configuration.h`, 'core configuration.h')
-      const infoCsv = options.skipConnect
-        ? undefined
-        : await fetchText(`${RAW}/redpanda-data/connect/${options.connectRef}/internal/plugins/info.csv`, 'connect info.csv')
-      const disablePage = options.disablePage
-        ? readLocal(options.disablePage)
-        : await fetchText(`${RAW}/redpanda-data/docs/${options.docsRef}/modules/get-started/pages/licensing/disable-enterprise-features.adoc`, 'disable-enterprise-features.adoc')
-      const antoraYaml = options.antora
-        ? readLocal(options.antora)
-        : await fetchText(`${RAW}/redpanda-data/rp-connect-docs/main/antora.yml`, 'rp-connect-docs antora.yml')
-      const antoraEnterpriseComponents = antoraYaml
-        ? extractAntoraEnterpriseComponents(yamlLib.load(antoraYaml))
-        : undefined
+      const { registryYaml, coreHeader, configurationHeader, infoCsv, connectRef, disablePage, failedSources } =
+        await loadEnterpriseSources(options)
 
       let allPropertyNames
       if (options.properties) {
-        const propertyData = JSON.parse(readLocal(options.properties))
+        const propertyData = JSON.parse(fs.readFileSync(path.resolve(options.properties), 'utf8'))
         allPropertyNames = Object.keys(propertyData.properties || propertyData)
       }
 
@@ -2211,7 +2273,7 @@ validation
         coreHeader,
         configurationHeader,
         infoCsv,
-        antoraEnterpriseComponents,
+        connectRef,
         disablePage,
         allPropertyNames,
       })

package/cli-utils/enterprise-sources.js

@@ -0,0 +1,244 @@
+'use strict'
+
+/**
+ * Source loading for `doc-tools validate enterprise-features`.
+ *
+ * Assembles the raw text of every source of truth the validator compares
+ * against: the enterprise features registry, the core headers, the connect
+ * plugin list, and the disable-enterprise-features page. Extracted from
+ * bin/doc-tools.js so the fetch wiring (auth, retries, --skip-connect
+ * short-circuits) is testable without a network.
+ */
+
+const fs = require('fs')
+const path = require('path')
+const { getGitHubToken } = require('./github-token')
+
+const RAW = 'https://raw.githubusercontent.com'
+
+// Repos known to be private, so a 404 can legitimately mean "needs auth".
+// Public sources (redpanda core headers, connect info.csv) 404 only when the
+// file is genuinely missing, and suggesting credentials there sends the
+// reader after a problem that does not exist (e.g. during a transient
+// GitHub blip).
+const PRIVATE_REPO_PREFIXES = [
+  `${RAW}/redpanda-data/docs/`,
+  `${RAW}/redpanda-data/rp-connect-docs/`,
+]
+
+const TOKEN_HINT = ' This source is in a private repository: set GIT_CREDENTIALS (or GITHUB_TOKEN / REDPANDA_GITHUB_TOKEN / ACTIONS_BOT_TOKEN) so the fetch can authenticate.'
+const REJECTED_HINT = ' A GitHub token was sent but rejected, so it may be expired or not grant access to this repository (the default Actions GITHUB_TOKEN is scoped to its own repository).'
+
+const TRANSIENT_RETRIES = 3
+const TRANSIENT_RETRY_DELAY_MS = 2000
+
+function isPrivateSource (url) {
+  return PRIVATE_REPO_PREFIXES.some((prefix) => url.startsWith(prefix))
+}
+
+// The response body of a failed fetch is never read; cancel it so the
+// underlying connection is released instead of lingering until GC.
+async function discardBody (resp) {
+  await resp.body?.cancel().catch(() => {})
+}
+
+/**
+ * Build a fetchText function that authenticates raw.githubusercontent.com
+ * requests and records failures for named sources.
+ *
+ * Some sources live in private repos (the docs repo), which
+ * raw.githubusercontent serves as 404 rather than 401 when unauthenticated —
+ * indistinguishable from a genuinely missing file. A token is sent when one
+ * is available so those fetches resolve; public sources are unaffected.
+ *
+ * raw.githubusercontent also returns 404 (not 401) for a REJECTED token,
+ * even on public files, so a stale token in any of the token env vars would
+ * break fetches that work tokenless. On a 404 with a token, the fetch is
+ * retried once without authentication before the 404 is treated as real.
+ *
+ * Transient failures (network errors, 429, 5xx) are retried a few times
+ * with a delay — the same treatment the CGO test gives curl — because a
+ * single blip on any source would otherwise fail the whole run, and the
+ * check runs on a weekly cron where nobody is watching live.
+ *
+ * @param {object} [deps]
+ * @param {string|null} [deps.token] - GitHub token (defaults to getGitHubToken())
+ * @param {Function} [deps.fetchImpl] - fetch implementation (defaults to global fetch)
+ * @param {Function} [deps.warn] - warning sink (defaults to console.warn)
+ * @param {Function} [deps.sleep] - delay implementation, for tests
+ * @returns {{fetchText: Function, failedSources: Array}} fetchText and the
+ *   findings array it appends to. Sources that fail to load are reported as
+ *   error-level findings rather than silently skipped, so CI cannot pass on a
+ *   check that never ran. Failures without a sourceName throw instead.
+ */
+function createSourceFetcher (deps = {}) {
+  const ghToken = deps.token !== undefined ? deps.token : getGitHubToken()
+  const fetchImpl = deps.fetchImpl || ((...args) => fetch(...args))
+  const warn = deps.warn || ((msg) => console.warn(msg))
+  const sleep = deps.sleep || ((ms) => new Promise((resolve) => setTimeout(resolve, ms)))
+  const failedSources = []
+  let warnedRejectedToken = false
+
+  // One HTTP attempt with transient-failure retries. Returns the final
+  // response; network errors that persist through every retry are rethrown.
+  async function attemptFetch (url, opts) {
+    let lastError
+    for (let attempt = 1; attempt <= TRANSIENT_RETRIES; attempt++) {
+      let resp
+      try {
+        resp = await fetchImpl(url, opts)
+      } catch (err) {
+        lastError = err
+        if (attempt < TRANSIENT_RETRIES) await sleep(TRANSIENT_RETRY_DELAY_MS)
+        continue
+      }
+      if (resp.status === 429 || resp.status >= 500) {
+        await discardBody(resp)
+        lastError = null
+        if (attempt < TRANSIENT_RETRIES) {
+          await sleep(TRANSIENT_RETRY_DELAY_MS)
+          continue
+        }
+        return resp
+      }
+      return resp
+    }
+    throw lastError
+  }
+
+  async function fetchText (url, sourceName) {
+    let resp
+    try {
+      resp = await attemptFetch(url, ghToken ? { headers: { Authorization: `Bearer ${ghToken}` } } : undefined)
+    } catch (err) {
+      if (sourceName) {
+        failedSources.push({ level: 'error', check: 'fetch', message: `Could not load ${sourceName} (${url}): ${err.message}. The related checks did not run.` })
+        return undefined
+      }
+      throw new Error(`Failed to fetch ${url}: ${err.message}`)
+    }
+    if (resp.ok) return resp.text()
+    await discardBody(resp)
+
+    let hint = ''
+    if (resp.status === 404 && ghToken) {
+      const retry = await attemptFetch(url)
+      if (retry.ok) {
+        if (!warnedRejectedToken) {
+          warnedRejectedToken = true
+          warn('Warning: the configured GitHub token was rejected by raw.githubusercontent.com (it may be expired); fetching without authentication instead.')
+        }
+        return retry.text()
+      }
+      await discardBody(retry)
+      if (isPrivateSource(url)) hint = REJECTED_HINT
+    } else if (resp.status === 404 && isPrivateSource(url)) {
+      // A tokenless 404 on a known-private repo is the signature of an
+      // unauthenticated fetch, so say so instead of leaving it to look
+      // like a missing file. Public sources get no credential hint: their
+      // 404s mean the file is missing (or GitHub is having a bad moment).
+      hint = TOKEN_HINT
+    }
+
+    if (sourceName) {
+      failedSources.push({ level: 'error', check: 'fetch', message: `Could not load ${sourceName} (${url}): ${resp.status} ${resp.statusText}. The related checks did not run.${hint}` })
+      return undefined
+    }
+    throw new Error(`Failed to fetch ${url}: ${resp.status} ${resp.statusText}.${hint}`)
+  }
+
+  return { fetchText, failedSources }
+}
+
+/**
+ * Resolve the ref to compare connect plugins at.
+ *
+ * Primary: the newest published release tag from the GitHub API. Fallback
+ * when the API is unavailable: the `latest-connect-version` asciidoc
+ * attribute in rp-connect-docs antora.yml — the latest version the docs
+ * actually document, which is the state the registry describes. (This is a
+ * different use of that file than the retired enterprise-components list:
+ * the version attribute is actively maintained by the connect version-update
+ * automation.) If both routes fail, the connect check is skipped with an
+ * error finding rather than silently comparing against main.
+ */
+async function resolveLatestConnectRef ({ fetchText, failedSources, warn }) {
+  try {
+    const releaseJson = await fetchText('https://api.github.com/repos/redpanda-data/connect/releases/latest')
+    const tag = JSON.parse(releaseJson).tag_name
+    if (tag) return String(tag)
+    throw new Error('the GitHub API response has no tag_name')
+  } catch (err) {
+    warn(`Warning: could not resolve the latest connect release from the GitHub API (${err.message}); falling back to the latest documented connect version in rp-connect-docs antora.yml.`)
+  }
+  const antoraYaml = await fetchText(`${RAW}/redpanda-data/rp-connect-docs/main/antora.yml`, 'latest documented connect version (rp-connect-docs antora.yml fallback)')
+  if (antoraYaml === undefined) return undefined // fetch failure already reported
+  const yamlLib = require('js-yaml')
+  let documented
+  try {
+    const attributes = yamlLib.load(antoraYaml)?.asciidoc?.attributes
+    documented = attributes && attributes['latest-connect-version']
+  } catch (err) {
+    failedSources.push({ level: 'error', check: 'fetch', message: `Could not resolve the connect ref: rp-connect-docs antora.yml did not parse (${err.message}). The connect check did not run.` })
+    return undefined
+  }
+  if (!documented) {
+    failedSources.push({ level: 'error', check: 'fetch', message: 'Could not resolve the connect ref: rp-connect-docs antora.yml has no latest-connect-version attribute. The connect check did not run.' })
+    return undefined
+  }
+  const version = String(documented).trim()
+  return version.startsWith('v') ? version : `v${version}`
+}
+
+/**
+ * Load every source the enterprise-features validation consumes.
+ *
+ * The rp-connect-docs antora.yml is deliberately NOT a source anymore:
+ * rp-connect-docs#485 removed the enterprise-components list it carried, so
+ * fetching it bought a cross-repo credential requirement in exchange for a
+ * comparison against nothing. The connect check now validates the registry's
+ * connect-plugin entries directly against info.csv.
+ *
+ * The connect ref defaults to 'latest', which resolves to the latest
+ * published connect release tag. The registry documents released state, so
+ * comparing it against main would flag plugins renamed or newly marked
+ * enterprise ahead of any release. The core headers deliberately stay on
+ * 'dev' for the opposite reason: their check exists to give early warning
+ * of NEW license-gated features before they ship.
+ *
+ * @param {object} options - Parsed CLI options: registry, tag, connectRef,
+ *   docsRef, disablePage, skipConnect.
+ * @param {object} [deps] - Injectable dependencies for tests: token,
+ *   fetchImpl, warn, sleep (see createSourceFetcher), and readLocal.
+ * @returns {Promise<object>} { registryYaml, coreHeader, configurationHeader,
+ *   infoCsv, connectRef, disablePage, failedSources }
+ */
+async function loadEnterpriseSources (options, deps = {}) {
+  const { fetchText, failedSources } = createSourceFetcher(deps)
+  const readLocal = deps.readLocal || ((p) => fs.readFileSync(path.resolve(p), 'utf8'))
+
+  const registryYaml = options.registry
+    ? readLocal(options.registry)
+    : await fetchText(`${RAW}/redpanda-data/docs/${options.docsRef}/shared/modules/ROOT/partials/enterprise-features.yml`)
+  const coreHeader = await fetchText(`${RAW}/redpanda-data/redpanda/${options.tag}/src/v/features/enterprise_features.h`, 'core enterprise_features.h')
+  const configurationHeader = await fetchText(`${RAW}/redpanda-data/redpanda/${options.tag}/src/v/config/configuration.h`, 'core configuration.h')
+
+  let connectRef = options.connectRef
+  let infoCsv
+  if (!options.skipConnect) {
+    if (!connectRef || connectRef === 'latest') {
+      connectRef = await resolveLatestConnectRef({ fetchText, failedSources, warn: deps.warn || ((msg) => console.warn(msg)) })
+    }
+    if (connectRef) {
+      infoCsv = await fetchText(`${RAW}/redpanda-data/connect/${connectRef}/internal/plugins/info.csv`, `connect info.csv (${connectRef})`)
+    }
+  }
+
+  const disablePage = options.disablePage
+    ? readLocal(options.disablePage)
+    : await fetchText(`${RAW}/redpanda-data/docs/${options.docsRef}/modules/get-started/pages/licensing/disable-enterprise-features.adoc`, 'disable-enterprise-features.adoc')
+
+  return { registryYaml, coreHeader, configurationHeader, infoCsv, connectRef, disablePage, failedSources }
+}
+
+module.exports = { createSourceFetcher, loadEnterpriseSources, RAW }