@redpanda-data/docs-extensions-and-macros 5.9.0 → 5.11.0

package/tools/rpk-docs/generate-rpk-docs.js

@@ -2588,6 +2588,10 @@ async function generateRpkDocs(options = {}) {
     overrides,
     outputDir,
     cloudSecretDir, // Directory for rpk cloud and rpk security secret commands
+    // Directory for the shared -X -> RPK_* env vars partial. The handler passes
+    // the partials dir it already derived; when absent (direct callers, tests)
+    // it is derived from outputDir the same way.
+    envPartialDir,
     rpkVersion,
     pluginVersions = {},
     draftMissing = false,
@@ -3155,6 +3159,21 @@ async function generateRpkDocs(options = {}) {
     navResult = updateNavFile(navFile, commands, resolvedOverrides, topLevelWithSubcommands, effectiveProtectedPlugins)
   }
 
+  // Regenerate the shared -X -> RPK_* env vars partial from the same tree,
+  // so it always matches the rpk build the command pages came from. Trees
+  // from rpk versions that predate x_options (redpanda#31520) skip with a
+  // note; `doc-tools generate rpk-env-partial` covers those via its
+  // -X list fallback.
+  const envPartialResult = writeEnvVarsPartial(
+    tree,
+    envPartialDir || derivePartialsDir(outputDir)
+  )
+  if (envPartialResult.written) {
+    console.log(`  Regenerated env vars partial (${envPartialResult.keyCount} -X options): ${envPartialResult.output}`)
+  } else {
+    console.log('  Env vars partial not regenerated: tree has no x_options (rpk predates redpanda#31520)')
+  }
+
   return {
     commandCount: commands.length,
     filesGenerated,
@@ -3162,12 +3181,55 @@ async function generateRpkDocs(options = {}) {
     filesFailed,
     filesDeleted,
     subdirectoriesCreated: createdSubdirs.size,
+    envPartial: envPartialResult,
     ...navResult
   }
 }
 
+/**
+ * Resolve the partials directory of the Antora module that owns the rpk pages.
+ *
+ * For the docs layout (`.../modules/reference/pages/rpk`) this is the sibling
+ * of `pages`, so `.../modules/reference/partials`. For an arbitrary
+ * `--output-dir` there is no `pages` segment to anchor on, so fall back to a
+ * `partials` directory beside the output. Walking a fixed number of levels up
+ * from outputDir is wrong for arbitrary paths: it escapes the output tree and
+ * can land on an unrelated directory.
+ *
+ * @param {string} outputDir - The rpk pages output directory
+ * @returns {string} Absolute or relative path to the partials directory
+ */
+function derivePartialsDir (outputDir) {
+  const normalized = String(outputDir).replace(/\\/g, '/')
+  const pagesIndex = normalized.lastIndexOf('/pages/')
+  if (pagesIndex !== -1) {
+    return path.join(normalized.substring(0, pagesIndex), 'partials')
+  }
+  return path.join(path.dirname(normalized), 'partials')
+}
+
+/**
+ * Write the -X -> RPK_* env vars partial from a command tree's x_options.
+ * @param {Object} tree - Parsed rpk command tree
+ * @param {string} partialsDir - Directory the partial is written into
+ * @returns {{written: boolean, keyCount?: number, output?: string}}
+ */
+function writeEnvVarsPartial (tree, partialsDir) {
+  if (!Array.isArray(tree.x_options) || tree.x_options.length === 0) {
+    return { written: false }
+  }
+  const { renderPartial, keyToEnvVar } = require('./generate-x-env-partial.js')
+  const options = tree.x_options.map(o => ({ name: o.name, env: o.env || keyToEnvVar(o.name) }))
+  const output = path.join(partialsDir, 'rpk-env-vars.adoc')
+  fs.mkdirSync(path.dirname(output), { recursive: true })
+  fs.writeFileSync(output, renderPartial(options))
+  return { written: true, keyCount: options.length, output }
+}
+
 module.exports = {
   generateRpkDocs,
+  writeEnvVarsPartial,
+  derivePartialsDir,
   mergeCommandOverrides,
   applyOverridesToTree,
   resolveReferences,

package/tools/rpk-docs/generate-x-env-partial.js

@@ -0,0 +1,284 @@
+'use strict'
+
+/**
+ * Generates the rpk env vars partial (the `-X` option -> RPK_* environment
+ * variable mapping table) from rpk itself, so the table cannot drift from
+ * the CLI.
+ *
+ * Data source, in order of preference:
+ * 1. The `x_options` array in `rpk --print-tree` JSON (added in
+ *    redpanda#31520): structured, versioned with the release, and carries
+ *    descriptions.
+ * 2. `rpk -X list` text output, for rpk versions that predate x_options.
+ *
+ * The partial is included by both reference:rpk/rpk-x-options.adoc and
+ * reference:environment-variables.adoc in the docs repo. Hidden -X options
+ * (for example cloud_environment, whose values are deliberately
+ * undocumented) appear in neither source, so they are excluded
+ * automatically.
+ */
+
+const fs = require('fs')
+const os = require('os')
+const path = require('path')
+const { spawnSync } = require('child_process')
+
+// A healthy rpk exposes ~30 -X options. Refuse to overwrite the partial
+// with a suspiciously small table (for example, if -X list output ever
+// changes format and only partially parses).
+const MIN_OPTIONS = 20
+
+const GENERATED_BANNER = `// tag::generated[]
+// Generated from rpk's own -X option data. Do not edit manually:
+// regenerate with doc-tools generate rpk-docs (which rewrites this
+// file alongside the rpk command pages) or, for a targeted refresh,
+// doc-tools generate rpk-env-partial.
+`
+
+/**
+ * Extract -X options from `rpk --print-tree` JSON.
+ * @param {string} output - Raw stdout from `rpk --print-tree`
+ * @returns {Array<{name: string, env: string}>|null} options in display
+ *   order, or null when the tree has no x_options (rpk predates
+ *   redpanda#31520). The env name comes from rpk itself when present
+ *   (the tree's env field is produced by the same derivation rpk reads).
+ */
+function xOptionsFromTree (output) {
+  let tree
+  try {
+    tree = JSON.parse(output)
+  } catch {
+    return null
+  }
+  if (!Array.isArray(tree.x_options) || tree.x_options.length === 0) return null
+  return tree.x_options.map(o => ({ name: o.name, env: o.env || keyToEnvVar(o.name) }))
+}
+
+/**
+ * Parse `rpk -X list` output into option keys (fallback for rpk versions
+ * whose --print-tree has no x_options).
+ * Each line has the form `key=value-hint`; keys contain lowercase
+ * letters, digits, dots, and underscores.
+ * @param {string} output - Raw stdout from `rpk -X list`
+ * @returns {Array<string>} option keys in CLI order
+ */
+function parseXList (output) {
+  const keys = []
+  for (const line of output.split('\n')) {
+    const eq = line.indexOf('=')
+    if (eq < 1) continue
+    const key = line.slice(0, eq).trim()
+    if (/^[a-z0-9._]+$/.test(key)) keys.push(key)
+  }
+  return keys
+}
+
+/**
+ * Convert an -X option key to its RPK_* environment variable name:
+ * prefix RPK_, uppercase, dots -> underscores.
+ * @param {string} key
+ * @returns {string}
+ */
+function keyToEnvVar (key) {
+  return 'RPK_' + key.toUpperCase().replace(/\./g, '_')
+}
+
+/**
+ * Convert an -X option key to its section anchor on the rpk-x-options page.
+ * The docs repo's AsciiDoc ID convention turns dots into hyphens but keeps
+ * underscores (tls.insecure_skip_verify -> tls-insecure_skip_verify),
+ * verified against the rendered page IDs. The previous hand-written table
+ * hyphenated underscores too, which left 12 of its 31 in-page links dead.
+ * @param {string} key
+ * @returns {string}
+ */
+function keyToAnchor (key) {
+  return key.replace(/\./g, '-')
+}
+
+/**
+ * Render the partial's AsciiDoc content.
+ * @param {Array<{name: string, env: string}>} options - -X options
+ * @returns {string}
+ */
+function renderPartial (options) {
+  const rows = options.map(({ name, env }) => {
+    const xrefTarget = `xref:reference:rpk/rpk-x-options.adoc#${keyToAnchor(name)}[${name}]`
+    return `|${xrefTarget} |${env}`
+  })
+  return `${GENERATED_BANNER}
+Every \`-X\` option has a corresponding \`RPK_*\` environment variable. Convert by prefixing with \`RPK_\` and replacing dots with underscores:
+
+[cols="1m,1m"]
+|===
+|\`-X\` Option |Environment Variable
+
+${rows.join('\n')}
+|===
+// end::generated[]
+`
+}
+
+/**
+ * Run an rpk invocation from a Go source checkout.
+ * @param {string} sourcePath - Path to src/go/rpk in a redpanda checkout
+ * @param {Array<string>} args - rpk arguments
+ * @returns {string} Raw stdout
+ */
+function runRpkFromSource (sourcePath, args) {
+  const result = spawnSync('go', ['run', 'cmd/rpk/main.go', ...args], {
+    cwd: sourcePath,
+    encoding: 'utf8',
+    timeout: 300000, // includes build time
+    maxBuffer: 50 * 1024 * 1024
+  })
+  if (result.status !== 0) {
+    throw new Error(`Failed to run rpk ${args.join(' ')} from source: ${result.stderr}`)
+  }
+  return result.stdout
+}
+
+/**
+ * Run an rpk invocation with an existing binary.
+ * @param {string} rpkBin - Path to the rpk binary
+ * @param {Array<string>} args - rpk arguments
+ * @returns {string} Raw stdout
+ */
+function runRpkBinary (rpkBin, args) {
+  const result = spawnSync(rpkBin, args, {
+    encoding: 'utf8',
+    timeout: 60000,
+    maxBuffer: 50 * 1024 * 1024
+  })
+  if (result.status !== 0) {
+    throw new Error(`Failed to run ${rpkBin} ${args.join(' ')}: ${result.stderr}`)
+  }
+  return result.stdout
+}
+
+/**
+ * Resolve a local rpk source path (repo root or src/go/rpk) without
+ * touching the checkout.
+ * @param {string} p - User-supplied path
+ * @returns {string} Path to src/go/rpk
+ */
+function resolveRpkSourcePath (p) {
+  const abs = path.resolve(p)
+  if (abs.endsWith(path.join('src', 'go', 'rpk'))) return abs
+  const nested = path.join(abs, 'src', 'go', 'rpk')
+  if (fs.existsSync(nested)) return nested
+  throw new Error(`Cannot find rpk source under ${abs}: pass the repo root or the src/go/rpk directory`)
+}
+
+/**
+ * Find the temp sparse-clone root that prepareSourceFromRef created
+ * (os.tmpdir()/rpk-source-<random>/redpanda/src/go/rpk), or null when the path is
+ * not one of ours — never delete anything outside the temp dir.
+ * @param {string} sourcePath - The src/go/rpk path returned by prepareSourceFromRef
+ * @returns {string|null}
+ */
+function tempCloneRoot (sourcePath) {
+  let dir = sourcePath
+  const tmp = fs.realpathSync(os.tmpdir())
+  const real = fs.realpathSync(dir)
+  dir = real
+  while (dir.startsWith(tmp) && dir !== tmp) {
+    if (path.basename(dir).startsWith('rpk-source-') && path.dirname(dir) === tmp) return dir
+    dir = path.dirname(dir)
+  }
+  return null
+}
+
+/**
+ * Generate the partial and write it to disk.
+ * @param {Object} options
+ * @param {string} [options.ref] - Git ref of redpanda to build rpk from
+ * @param {string} [options.fromSource] - Local src/go/rpk path (skips clone)
+ * @param {string} [options.rpkBin] - Existing rpk binary (skips build)
+ * @param {string} [options.fromJson] - Versioned tree snapshot from docs-data
+ *   (skips clone and build; the snapshot must carry x_options)
+ * @param {string} options.output - Path to write the partial to
+ * @returns {{keyCount: number, output: string, source: string}}
+ */
+function handleXEnvPartialGeneration (options) {
+  const { ref, fromSource, rpkBin, fromJson, output } = options
+  if (!output) throw new Error('Missing required --output path')
+
+  let xopts
+  let source
+  if (fromJson) {
+    // Snapshots saved by the rpk-docs handler wrap the tree in raw_tree/tree;
+    // accept a bare tree too. There is no rpk to run in this mode, so a
+    // snapshot from before x_options existed (redpanda#31520) cannot be used.
+    const j = JSON.parse(fs.readFileSync(fromJson, 'utf8'))
+    for (const candidate of [j.raw_tree, j.tree, j]) {
+      if (!candidate) continue
+      xopts = xOptionsFromTree(JSON.stringify(candidate))
+      if (xopts) break
+    }
+    if (!xopts) {
+      throw new Error(
+        `${fromJson} has no x_options; the snapshot predates redpanda#31520. ` +
+        'Regenerate with --ref/--from-source/--rpk-bin instead.'
+      )
+    }
+    source = `snapshot ${path.basename(fromJson)}`
+  } else {
+    let run
+    let cleanupDir = null
+    if (rpkBin) {
+      run = args => runRpkBinary(rpkBin, args)
+    } else if (fromSource && !ref) {
+      // Local-source mode with no explicit ref: use the checkout exactly as
+      // it stands. prepareSourceFromRef would fetch and check out a ref,
+      // silently moving the user's working tree.
+      const sourcePath = resolveRpkSourcePath(fromSource)
+      run = args => runRpkFromSource(sourcePath, args)
+    } else {
+      const { prepareSourceFromRef } = require('./rpk-docs-handler.js')
+      const sourcePath = prepareSourceFromRef(ref || 'dev', fromSource || null)
+      if (!fromSource) {
+        // prepareSourceFromRef sparse-cloned into a temp dir this run owns;
+        // remove it when done so repeated runs don't accumulate clones.
+        cleanupDir = tempCloneRoot(sourcePath)
+      }
+      run = args => runRpkFromSource(sourcePath, args)
+    }
+
+    try {
+      // Preferred: structured x_options from the command tree JSON.
+      xopts = xOptionsFromTree(run(['--print-tree']))
+      source = 'print-tree x_options'
+      if (!xopts) {
+        // Fallback for rpk versions that predate x_options in --print-tree.
+        // The env name is derived locally with the documented mapping rule.
+        xopts = parseXList(run(['-X', 'list'])).map(name => ({ name, env: keyToEnvVar(name) }))
+        source = '-X list (fallback)'
+      }
+    } finally {
+      if (cleanupDir) {
+        fs.rmSync(cleanupDir, { recursive: true, force: true })
+      }
+    }
+  }
+
+  if (xopts.length < MIN_OPTIONS) {
+    throw new Error(
+      `Parsed only ${xopts.length} -X options from ${source} (expected at least ${MIN_OPTIONS}); ` +
+      'refusing to write a suspiciously small partial. The output format may have changed.'
+    )
+  }
+
+  fs.mkdirSync(path.dirname(output), { recursive: true })
+  fs.writeFileSync(output, renderPartial(xopts))
+  return { keyCount: xopts.length, output, source }
+}
+
+module.exports = {
+  xOptionsFromTree,
+  parseXList,
+  keyToEnvVar,
+  keyToAnchor,
+  renderPartial,
+  handleXEnvPartialGeneration
+}

package/tools/rpk-docs/rpk-docs-handler.js

@@ -7,7 +7,7 @@ const fs = require('fs')
 const os = require('os')
 const semver = require('semver')
 const { findRepoRoot } = require('../../cli-utils/doc-tools-utils')
-const { generateRpkDocs, applyOverridesToTree, resolveReferences, shouldExcludeCommand, shouldUsePartialDir } = require('./generate-rpk-docs')
+const { generateRpkDocs, applyOverridesToTree, resolveReferences, shouldExcludeCommand, shouldUsePartialDir, derivePartialsDir } = require('./generate-rpk-docs')
 const { detectLinuxOnlyFromSource, warnIfDetectionLooksBroken } = require('./detect-platform-commands')
 const { generateRpkDiff, printDiffReport, generateWhatsNewSection, flattenToMap } = require('./report-delta')
 const { loadAndValidateOverrides, ValidationResult } = require('./validate-overrides')
@@ -1914,26 +1914,14 @@ async function handleRpkDocsGeneration(options = {}) {
   const defaultOutputDir = path.join(repoRoot, 'modules', 'reference', 'pages', 'rpk')
   const finalOutputDir = outputDir || defaultOutputDir
 
-  // cloudSecretDir should be relative to outputDir, not repoRoot
-  // If outputDir is .../modules/reference/pages/rpk, cloudSecretDir should be .../modules/reference/partials
-  let defaultCloudSecretDir
-  if (outputDir) {
-    // Check if outputDir follows docs repo structure (ends with pages/rpk or similar)
-    const normalizedOutput = finalOutputDir.replace(/\\/g, '/')
-    if (normalizedOutput.includes('/pages/')) {
-      // Derive cloudSecretDir from outputDir by going up from pages/rpk to partials
-      const pagesIndex = normalizedOutput.lastIndexOf('/pages/')
-      const referenceDir = normalizedOutput.substring(0, pagesIndex)
-      defaultCloudSecretDir = path.join(referenceDir, 'partials')
-    } else {
-      // Arbitrary output directory - use sibling partials directory
-      const outputParent = path.dirname(finalOutputDir)
-      defaultCloudSecretDir = path.join(outputParent, 'partials')
-    }
-  } else {
-    defaultCloudSecretDir = path.join(repoRoot, 'modules', 'reference', 'partials')
-  }
-  const finalCloudSecretDir = cloudSecretDir || defaultCloudSecretDir
+  // Partials live relative to outputDir, not repoRoot: for the docs layout
+  // (.../modules/reference/pages/rpk) that is .../modules/reference/partials,
+  // and for an arbitrary --output-dir it is a partials dir beside the output.
+  // derivePartialsDir owns that rule so each consumer does not re-derive it.
+  const derivedPartialsDir = outputDir
+    ? derivePartialsDir(finalOutputDir)
+    : path.join(repoRoot, 'modules', 'reference', 'partials')
+  const finalCloudSecretDir = cloudSecretDir || derivedPartialsDir
 
   // nav.adoc lives at modules/ROOT/nav.adoc relative to the repo root
   const navFile = path.join(repoRoot, 'modules', 'ROOT', 'nav.adoc')
@@ -2139,6 +2127,7 @@ async function handleRpkDocsGeneration(options = {}) {
         overrides: overridesData,
         outputDir: finalOutputDir,
         cloudSecretDir: finalCloudSecretDir,
+        envPartialDir: derivedPartialsDir,
         rpkVersion,
         pluginVersions,
         draftMissing,
@@ -2526,6 +2515,7 @@ async function handleRpkDocsGeneration(options = {}) {
       overrides: overridesData,
       outputDir: finalOutputDir,
       cloudSecretDir: finalCloudSecretDir,
+      envPartialDir: derivedPartialsDir,
       rpkVersion,
       pluginVersions,
       draftMissing,

package/tools/rpk-docs/validate-output.js

@@ -23,12 +23,13 @@ const VALIDATION_RULES = [
       let match
       while ((match = urlPattern.exec(content)) !== null) {
         const url = match[1]
-        // Check if it's a docs.redpanda.com URL (should always be xref)
+        // docs.redpanda.com URLs are converted to xrefs at site build time
+        // by the url-to-xref Antora extension, so a raw one is informational
         if (url.includes('docs.redpanda.com')) {
           issues.push({
             line: getLineNumber(content, match.index),
-            message: `Raw docs.redpanda.com URL should be an xref: ${url}`,
-            severity: 'error'
+            message: `Raw docs.redpanda.com URL (converted to an xref by the url-to-xref extension at build time): ${url}`,
+            severity: 'info'
           })
         }
       }

package/cli-utils/convert-doc-links.js

@@ -1,58 +0,0 @@
-const { URL } = require('url');
-
-/**
- * Converts a docs.redpanda.com URL, optionally suffixed with a label in brackets, into an Antora xref resource ID string.
- *
- * If the input includes a label in square brackets (for example, `[Label]`), the label is preserved and appended to the resulting xref.
- *
- * @param {string} input - A docs.redpanda.com URL, optionally followed by a label in square brackets.
- * @returns {string} The corresponding Antora xref resource ID, with the label preserved if present.
- *
- * @throws {Error} If the input is not a valid URL or does not belong to docs.redpanda.com.
- */
-function urlToXref(input) {
-  // Peel off an optional “[label]”
-  let urlPart  = input;
-  let label    = '';
-  const mLabel = input.match(/^(.*)\[([^\]]+)\]$/);
-  if (mLabel) {
-    urlPart = mLabel[1];
-    label   = mLabel[2];
-  }
-
-  //Parse & validate
-  let url;
-  try {
-    url = new URL(urlPart);
-  } catch {
-    throw new Error(`Invalid URL: ${input}`);
-  }
-  if (!/docs\.redpanda\.com$/.test(url.hostname)) {
-    throw new Error(`Not a docs.redpanda.com URL: ${input}`);
-  }
-
-  // Strip any leading “/docs”, “/docs/vX.Y”, “/docs/current”, “/vX.Y” or “/current”
-  let p = url.pathname.replace(
-    /^\/(?:docs(?:\/(?:v?\d+\.\d+|current))?|v?\d+\.\d+|current)\/?/,
-    ''
-  );
-  // Drop trailing slash
-  p = p.replace(/\/$/, '');
-  const segments = p.split('/').filter(Boolean);
-
-  // Build module + path + .adoc
-  let xref;
-  if (segments.length === 0) {
-    xref = 'xref:index.adoc';
-  } else {
-    const moduleName = segments.shift();
-    const pagePath   = segments.join('/');
-    const fileName   = (pagePath || moduleName) + '.adoc';
-    xref = `xref:${moduleName}:${fileName}`;
-  }
-
-  // Re-attach label if there was one
-  return label ? `${xref}[${label}]` : xref;
-}
-
-module.exports = { urlToXref };