@redpanda-data/docs-extensions-and-macros 5.10.0 → 5.11.0

package/bin/doc-tools.js

@@ -1279,6 +1279,56 @@ automation
     }
   })
 
+/**
+ * generate rpk-env-partial
+ *
+ * @description
+ * Generates the -X option -> RPK_* environment variable mapping table as an
+ * AsciiDoc partial from rpk's own -X option data, so the table cannot drift
+ * from the CLI. Prefers the structured x_options array in `rpk --print-tree`
+ * (which carries the env var names rpk itself derives) and falls back to
+ * parsing `-X list` text for rpk versions that predate it. Hidden -X options
+ * appear in neither source, so they are excluded automatically.
+ *
+ * The main rpk-docs pipeline also writes this partial from the tree it
+ * already holds; this standalone command is for targeted refreshes without
+ * a full generation run.
+ *
+ * @example
+ * # Generate for a release tag (sparse-clones redpanda, builds rpk with Go)
+ * npx doc-tools generate rpk-env-partial --ref v26.2.1 --output modules/reference/partials/rpk-env-vars.adoc
+ *
+ * # Use a local source checkout (as-is, no checkout changes), an existing
+ * # rpk binary, or a versioned tree snapshot (no clone or build at all)
+ * npx doc-tools generate rpk-env-partial --from-source ~/redpanda --output modules/reference/partials/rpk-env-vars.adoc
+ * npx doc-tools generate rpk-env-partial --rpk-bin "$(command -v rpk)" --output modules/reference/partials/rpk-env-vars.adoc
+ * npx doc-tools generate rpk-env-partial --from-json docs-data/rpk-v26.2.1.json --output modules/reference/partials/rpk-env-vars.adoc
+ */
+automation
+  .command('rpk-env-partial')
+  .description('Generate the -X -> RPK_* env var mapping partial from rpk -X list output.')
+  .option('-r, --ref <ref>', 'Git branch or tag to build rpk from (e.g., dev, v26.2.1). Clones from GitHub.')
+  .option('--from-source <path>', 'Path to local rpk source (src/go/rpk directory)')
+  .option('--rpk-bin <path>', 'Path to an existing rpk binary (skips clone and build)')
+  .option('--from-json <path>', 'Versioned tree snapshot from docs-data (skips clone and build; requires a snapshot with x_options)')
+  .option('--output <path>', 'Path to write the partial to', 'modules/reference/partials/rpk-env-vars.adoc')
+  .action((options) => {
+    try {
+      const { handleXEnvPartialGeneration } = require('../tools/rpk-docs/generate-x-env-partial.js')
+      const result = handleXEnvPartialGeneration({
+        ref: options.ref,
+        fromSource: options.fromSource,
+        rpkBin: options.rpkBin,
+        fromJson: options.fromJson,
+        output: options.output
+      })
+      console.log(`Wrote ${result.keyCount} -X option mappings to ${result.output} (source: ${result.source})`)
+    } catch (err) {
+      console.error(`Error: ${err.message}`)
+      process.exit(1)
+    }
+  })
+
 /**
  * generate rpk-plugin-stubs
  *
@@ -1677,7 +1727,50 @@ automation
       if (r.status !== 0) process.exit(r.status)
 
       const { formatHelmSpec } = require('../cli-utils/format-helm-spec')
-      fs.writeFileSync(outFile, formatHelmSpec(fs.readFileSync(outFile, 'utf8')), 'utf8')
+      let doc = formatHelmSpec(fs.readFileSync(outFile, 'utf8'))
+
+      // helm-docs only renders keys that exist in the YAML tree, so optional
+      // values that ship commented out never appear in the reference. Pull
+      // documented commented-out keys from values.yaml and inject them. This
+      // runs after formatHelmSpec so the injected body sections are not affected
+      // by the blank-line normalization that its header fix performs.
+      const valuesFile = path.join(chartPath, 'values.yaml')
+      if (fs.existsSync(valuesFile)) {
+        const { extractCommentedValueDocs, injectIntoAsciiDoc, filterEntriesBySchema } = require('../cli-utils/helm-commented-values')
+        try {
+          let entries = extractCommentedValueDocs(fs.readFileSync(valuesFile, 'utf8'))
+
+          // Deprecation notices and other prose can produce paths the chart
+          // no longer accepts (for example the removed
+          // storage.tiered.credentialsSecretRef.configurationKey). When the
+          // chart ships a values schema, drop any path it provably rejects.
+          const schemaFile = path.join(chartPath, 'values.schema.json')
+          if (entries.length > 0 && fs.existsSync(schemaFile)) {
+            const schema = JSON.parse(fs.readFileSync(schemaFile, 'utf8'))
+            const { accepted, rejected } = filterEntriesBySchema(entries, schema)
+            if (rejected.length > 0) {
+              console.warn(`Warning: Skipping ${rejected.length} commented-out value(s) rejected by values.schema.json: ${rejected.map((e) => e.path).join(', ')}`)
+            }
+            entries = accepted
+          }
+
+          if (entries.length > 0) {
+            const { doc: withInjected, injected, sectionsFound } = injectIntoAsciiDoc(doc, entries)
+            doc = withInjected
+            if (injected.length > 0) {
+              console.log(`Documented ${injected.length} commented-out value(s): ${injected.join(', ')}`)
+            } else if (sectionsFound === 0) {
+              // Without this, a heading-level change upstream would silently
+              // drop every documented commented-out value from the reference.
+              console.warn(`Warning: No value section headings found in ${name} to anchor commented-out value docs; dropped: ${entries.map((e) => e.path).join(', ')}`)
+            }
+          }
+        } catch (err) {
+          console.warn(`Warning: Skipping commented-out value docs for ${name}: ${err.message}`)
+        }
+      }
+
+      fs.writeFileSync(outFile, doc, 'utf8')
 
       console.log(`Done: Wrote ${outFile}`)
     }

package/cli-utils/helm-commented-values.js

@@ -0,0 +1,361 @@
+'use strict'
+
+/**
+ * Extracts documentation for commented-out values in a Helm chart's
+ * values.yaml, and injects the resulting entries into the AsciiDoc reference
+ * that `doc-tools generate helm-spec` produces.
+ *
+ * helm-docs only renders rows for keys that exist in the YAML tree, so
+ * optional values that ship commented out (for example `external.domain` in
+ * the Redpanda chart) never appear in the generated reference, even when the
+ * chart author wrote a `# --` description for them. This module closes that
+ * gap without requiring any change to chart behavior.
+ *
+ * Two comment conventions are recognized:
+ *
+ * 1. helm-docs style, attached to a commented-out key. A comment block that
+ *    contains a `# -- description` marker and is immediately followed by a
+ *    commented-out key documents that key. The key's path is derived from
+ *    the enclosing uncommented keys by indentation:
+ *
+ *      external:
+ *        # -- Optional domain advertised to external clients.
+ *        # domain: local
+ *
+ *    yields `external.domain`.
+ *
+ * 2. Explicit, fully-qualified syntax that works anywhere in the file:
+ *
+ *      # @doc external.addresses -- Optional list of advertised addresses.
+ *      # @default -- `nil`
+ *
+ * In both forms, an optional `# @default -- text` line overrides the
+ * displayed default, which is otherwise `nil`.
+ */
+
+// Key names never contain dots or slashes: a dotted or slashed "key" in a
+// comment is prose or a URL (`# docker.redpanda.com/...:4.101.0`), and
+// treating it as a key both fabricates a path and truncates the description.
+// Fully-qualified dotted paths are supported through the @doc syntax instead.
+const COMMENTED_KEY_RE = /^(\s*)#(\s{0,4})([a-z][A-Za-z0-9_-]*):\s*(.*)$/
+const REAL_KEY_RE = /^(\s*)([A-Za-z0-9_."/-]+):(.*)$/
+// `(?!-)` keeps comment dividers such as `# ----------------` from being
+// mistaken for an empty description marker.
+const DESC_MARKER_RE = /^\s*#\s*--\s*(?!-)(.*)$/
+const AT_DOC_RE = /^\s*#\s*@doc\s+([A-Za-z0-9_./-]+)\s+--\s*(.*)$/
+const AT_DEFAULT_RE = /^\s*#\s*@default\s+--\s*(.*)$/
+const COMMENT_RE = /^\s*#\s?(.*)$/
+// A block scalar header may carry an explicit indentation indicator digit on
+// either side of the chomping indicator: `|2`, `|-2`, `|2-`, `>2`, ...
+const BLOCK_SCALAR_RE = /[|>][0-9]?[+-]?[0-9]?\s*(#.*)?$/
+const LIST_ITEM_RE = /^\s*-\s/
+
+/**
+ * Parse values.yaml text and return entries for documented commented-out keys.
+ *
+ * @param {string} yamlText - Raw contents of values.yaml.
+ * @returns {Array<{path: string, description: string, default: string}>}
+ */
+function extractCommentedValueDocs (yamlText) {
+  const lines = yamlText.split(/\r?\n/)
+  const entries = []
+  const stack = [] // enclosing real keys: { name, indent }
+
+  // Accumulating comment block: { descLines, default, candidate }. The
+  // candidate is the key the block documents; emission is deferred to the
+  // end of the block so that a later key line at the same depth supersedes a
+  // prose line that merely looks like a key (`# example:`, `# default: 30s`).
+  let block = null
+  let atDoc = null // accumulating @doc entry: { path, descLines, default }
+  let skipScalarIndent = -1 // inside a block scalar when >= 0
+  let suppressInnerIndent = -1 // inside an emitted commented key's subtree when >= 0
+
+  const flushAtDoc = () => {
+    if (atDoc) {
+      entries.push({
+        path: atDoc.path,
+        description: atDoc.descLines.join('\n').trim(),
+        default: atDoc.default || '`nil`',
+      })
+      atDoc = null
+    }
+  }
+
+  // Emit the active block's candidate, if any, and close the block. The
+  // emitted key suppresses its own commented subtree until real content
+  // appears, so nested example structures do not emit bogus paths.
+  const flushBlock = () => {
+    if (block && block.candidate) {
+      const { name, effIndent } = block.candidate
+      const parents = stack.filter((k) => k.indent < effIndent).map((k) => k.name)
+      entries.push({
+        path: [...parents, name].join('.'),
+        description: block.descLines.join('\n').trim(),
+        default: block.default || '`nil`',
+      })
+      suppressInnerIndent = effIndent
+    }
+    block = null
+  }
+
+  for (const line of lines) {
+    // Skip the body of block scalars so their content is never mistaken for
+    // keys or comments.
+    if (skipScalarIndent >= 0) {
+      const indent = line.match(/^\s*/)[0].length
+      if (line.trim() === '' || indent > skipScalarIndent) continue
+      skipScalarIndent = -1
+    }
+
+    const atDocMatch = line.match(AT_DOC_RE)
+    if (atDocMatch) {
+      flushAtDoc()
+      flushBlock()
+      atDoc = { path: atDocMatch[1], descLines: [atDocMatch[2]], default: '' }
+      continue
+    }
+
+    const atDefault = line.match(AT_DEFAULT_RE)
+    if (atDefault) {
+      if (atDoc) atDoc.default = atDefault[1].trim()
+      else if (block) block.default = atDefault[1].trim()
+      continue
+    }
+
+    const commentedKey = line.match(COMMENTED_KEY_RE)
+    if (commentedKey) {
+      // A commented-out key line terminates an @doc entry rather than being
+      // swallowed into its description, so @doc lines can sit directly above
+      // the commented key they document.
+      flushAtDoc()
+      // Authors indent a commented key either before the marker ("  # child:")
+      // or after it ("#   child:"), and the two forms mean the same nesting.
+      // Counting only one side made them disagree: a child indented before the
+      // marker escaped its parent's subtree and was emitted as a top-level
+      // path. Fold both sides into one depth, treating the single space that
+      // separates "#" from its text as punctuation rather than indentation, so
+      // the result lines up with the real-key indents held on the stack.
+      const effIndent = commentedKey[1].length + Math.max(0, commentedKey[2].length - 1)
+      // A previously emitted commented key suppresses its own commented
+      // subtree, so nested example structures do not emit bogus paths.
+      if (suppressInnerIndent >= 0 && effIndent > suppressInnerIndent) {
+        block = null
+        continue
+      }
+      suppressInnerIndent = -1
+      // Only bare keys or simple scalar values look like real chart values.
+      // Prose such as `# Warning: If you use LoadBalancers...` has spaces in
+      // the trailing text and is rejected.
+      const value = commentedKey[4].trim()
+      // Reject URL-like lines such as `# https://github.com/...`, where the
+      // "key" is a URL scheme and the "value" is the scheme-relative part.
+      const simpleValue = (value === '' || !/\s/.test(value)) && !value.startsWith('/')
+      if (block && simpleValue) {
+        if (!block.candidate || effIndent <= block.candidate.effIndent) {
+          // The last key-shaped line at the block's shallowest depth wins:
+          // an earlier line at the same depth was prose shaped like a key,
+          // and deeper lines are nested example structures.
+          block.candidate = { name: commentedKey[3], effIndent }
+        }
+        continue
+      }
+    }
+
+    const descMarker = line.match(DESC_MARKER_RE)
+    if (descMarker) {
+      flushAtDoc()
+      flushBlock()
+      block = { descLines: [descMarker[1]], default: '', candidate: null }
+      continue
+    }
+
+    const comment = line.match(COMMENT_RE)
+    if (comment && line.trim().startsWith('#')) {
+      if (atDoc) atDoc.descLines.push(comment[1])
+      // Description lines precede the documented key; once a candidate
+      // exists, trailing comment lines belong to its example structure.
+      else if (block && !block.candidate) block.descLines.push(comment[1])
+      continue
+    }
+
+    // Any non-comment line ends comment accumulation. Only real content also
+    // ends subtree suppression: a blank line inside a commented-out example
+    // structure must not re-enable emission against a stale parent key, since
+    // the real-key stack is not rebuilt by comment lines.
+    flushAtDoc()
+    flushBlock()
+    if (line.trim() === '') continue
+    suppressInnerIndent = -1
+
+    if (LIST_ITEM_RE.test(line)) continue
+
+    const realKey = line.match(REAL_KEY_RE)
+    if (realKey) {
+      const indent = realKey[1].length
+      while (stack.length > 0 && stack[stack.length - 1].indent >= indent) {
+        stack.pop()
+      }
+      stack.push({ name: realKey[2].replace(/^"|"$/g, ''), indent })
+      if (BLOCK_SCALAR_RE.test(realKey[3])) skipScalarIndent = indent
+    }
+  }
+
+  flushAtDoc()
+  flushBlock()
+  return entries
+}
+
+/**
+ * Check a dotted value path against a chart's values.schema.json. A path is
+ * rejected only when the walk can prove the schema forbids it: an object
+ * level whose `properties` lack the segment, whose `patternProperties` (if
+ * any) do not match it, and whose `additionalProperties` is `false`.
+ * Anything the walk cannot interpret ($ref, allOf, open objects) is allowed,
+ * so an incomplete schema never hides documentation.
+ *
+ * @param {object} schema - Parsed values.schema.json.
+ * @param {string} path - Dotted value path, for example `external.domain`.
+ * @returns {boolean}
+ */
+function isPathAllowedBySchema (schema, path) {
+  let node = schema
+  for (const segment of path.split('.')) {
+    if (!node || typeof node !== 'object' || node.$ref || node.allOf || node.anyOf || node.oneOf) {
+      return true
+    }
+    const props = node.properties
+    if (props && Object.prototype.hasOwnProperty.call(props, segment)) {
+      node = props[segment]
+      continue
+    }
+    if (node.patternProperties) {
+      const pattern = Object.keys(node.patternProperties)
+        .find((p) => { try { return new RegExp(p).test(segment) } catch { return true } })
+      if (pattern !== undefined) {
+        node = node.patternProperties[pattern]
+        continue
+      }
+    }
+    return node.additionalProperties !== false
+  }
+  return true
+}
+
+/**
+ * Split extracted entries into those permitted by the chart's
+ * values.schema.json and those provably rejected by it, such as keys whose
+ * `# --` comment is a deprecation notice for a removed value.
+ *
+ * @param {Array<{path: string}>} entries
+ * @param {object} schema - Parsed values.schema.json.
+ * @returns {{accepted: Array, rejected: Array}}
+ */
+function filterEntriesBySchema (entries, schema) {
+  const accepted = []
+  const rejected = []
+  for (const entry of entries) {
+    (isPathAllowedBySchema(schema, entry.path) ? accepted : rejected).push(entry)
+  }
+  return { accepted, rejected }
+}
+
+// Neutralize description lines that would otherwise parse as AsciiDoc
+// structure when injected at block level: section titles re-parent the
+// following sections, and an unclosed block delimiter swallows the rest of
+// the document. `{empty}` renders as nothing but demotes the line to a
+// plain paragraph.
+function sanitizeDescription (text) {
+  return text
+    .split('\n')
+    .map((line) => (/^(=+\s|-{2,}\s*$|[=*_.+]{4,}\s*$|\|===)/.test(line) ? `{empty}${line}` : line))
+    .join('\n')
+}
+
+/**
+ * Inject extracted entries into a generated AsciiDoc Helm reference.
+ *
+ * Sections are matched by their `=== link:++<url>?...path=<key>++[<key>]`
+ * headings. New sections reuse the Artifact Hub URL prefix of the existing
+ * sections and are inserted in alphabetical key order. Keys that are already
+ * documented are skipped.
+ *
+ * @param {string} adoc - The converted AsciiDoc document.
+ * @param {Array<{path: string, description: string, default: string}>} entries
+ * @returns {{doc: string, injected: string[], sectionsFound: number}}
+ */
+function injectIntoAsciiDoc (adoc, entries) {
+  // The label may itself contain brackets (`[storage.volume[0].name]` in the
+  // connectors chart), so it is matched to end of line rather than to the
+  // first closing bracket.
+  const headingRe = /^=== link:\+\+(https?:\/\/[^+]*[?&]path=)([^+]+)\+\+\[.*\]\s*$/gm
+  const sections = []
+  let match
+  while ((match = headingRe.exec(adoc)) !== null) {
+    sections.push({ index: match.index, urlPrefix: match[1], key: match[2] })
+  }
+  if (sections.length === 0) {
+    return { doc: adoc, injected: [], sectionsFound: 0 }
+  }
+
+  const urlPrefix = sections[0].urlPrefix
+  // Tracks both keys present in the document and keys queued for injection,
+  // so a path extracted twice (for example, from both @doc and # -- comments
+  // for the same key) is only injected once.
+  const existingKeys = new Set(sections.map((s) => s.key))
+  const additions = []
+  for (const entry of entries) {
+    if (existingKeys.has(entry.path)) continue
+    existingKeys.add(entry.path)
+    additions.push(entry)
+  }
+  additions.sort((a, b) => a.path.localeCompare(b.path))
+
+  let doc = adoc
+  const injected = []
+
+  for (const entry of additions) {
+    // Recompute section offsets after each insertion.
+    const current = []
+    headingRe.lastIndex = 0
+    while ((match = headingRe.exec(doc)) !== null) {
+      current.push({ index: match.index, key: match[2] })
+    }
+    const after = current.find((s) => s.key.localeCompare(entry.path) > 0)
+
+    const section = [
+      `=== link:++${urlPrefix}${entry.path}++[${entry.path}]`,
+      '',
+      sanitizeDescription(entry.description),
+      '',
+      `*Default:* ${entry.default}`,
+      '',
+      '',
+    ].join('\n')
+
+    if (after) {
+      doc = doc.slice(0, after.index) + section + doc.slice(after.index)
+    } else {
+      // Append after the final section, before the next heading (levels 1-3)
+      // if one exists, so a trailing non-values subsection stays last.
+      const last = current[current.length - 1]
+      const rest = doc.slice(last.index)
+      const nextHeading = rest.search(/\n={1,3} /)
+      if (nextHeading > 0) {
+        const cut = last.index + nextHeading + 1
+        doc = doc.slice(0, cut) + section + doc.slice(cut)
+      } else {
+        doc = doc.trimEnd() + '\n\n' + section
+      }
+    }
+    injected.push(entry.path)
+  }
+
+  return { doc, injected, sectionsFound: sections.length }
+}
+
+module.exports = {
+  extractCommentedValueDocs,
+  injectIntoAsciiDoc,
+  filterEntriesBySchema,
+  isPathAllowedBySchema,
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@redpanda-data/docs-extensions-and-macros",
-  "version": "5.10.0",
+  "version": "5.11.0",
   "description": "Antora extensions and macros developed for Redpanda documentation.",
   "keywords": [
     "antora",

package/tools/generate-cli-docs.js

@@ -299,6 +299,7 @@ IMPORTANT: This documentation is auto-generated. Do not edit manually. Run \`npm
     'property-docs',
     'metrics-docs',
     'rpk-docs',
+    'rpk-env-partial',
     'rpcn-connector-docs',
     'helm-spec',
     'cloud-regions',

package/tools/rpk-docs/generate-rpk-docs.js

@@ -2588,6 +2588,10 @@ async function generateRpkDocs(options = {}) {
     overrides,
     outputDir,
     cloudSecretDir, // Directory for rpk cloud and rpk security secret commands
+    // Directory for the shared -X -> RPK_* env vars partial. The handler passes
+    // the partials dir it already derived; when absent (direct callers, tests)
+    // it is derived from outputDir the same way.
+    envPartialDir,
     rpkVersion,
     pluginVersions = {},
     draftMissing = false,
@@ -3155,6 +3159,21 @@ async function generateRpkDocs(options = {}) {
     navResult = updateNavFile(navFile, commands, resolvedOverrides, topLevelWithSubcommands, effectiveProtectedPlugins)
   }
 
+  // Regenerate the shared -X -> RPK_* env vars partial from the same tree,
+  // so it always matches the rpk build the command pages came from. Trees
+  // from rpk versions that predate x_options (redpanda#31520) skip with a
+  // note; `doc-tools generate rpk-env-partial` covers those via its
+  // -X list fallback.
+  const envPartialResult = writeEnvVarsPartial(
+    tree,
+    envPartialDir || derivePartialsDir(outputDir)
+  )
+  if (envPartialResult.written) {
+    console.log(`  Regenerated env vars partial (${envPartialResult.keyCount} -X options): ${envPartialResult.output}`)
+  } else {
+    console.log('  Env vars partial not regenerated: tree has no x_options (rpk predates redpanda#31520)')
+  }
+
   return {
     commandCount: commands.length,
     filesGenerated,
@@ -3162,12 +3181,55 @@ async function generateRpkDocs(options = {}) {
     filesFailed,
     filesDeleted,
     subdirectoriesCreated: createdSubdirs.size,
+    envPartial: envPartialResult,
     ...navResult
   }
 }
 
+/**
+ * Resolve the partials directory of the Antora module that owns the rpk pages.
+ *
+ * For the docs layout (`.../modules/reference/pages/rpk`) this is the sibling
+ * of `pages`, so `.../modules/reference/partials`. For an arbitrary
+ * `--output-dir` there is no `pages` segment to anchor on, so fall back to a
+ * `partials` directory beside the output. Walking a fixed number of levels up
+ * from outputDir is wrong for arbitrary paths: it escapes the output tree and
+ * can land on an unrelated directory.
+ *
+ * @param {string} outputDir - The rpk pages output directory
+ * @returns {string} Absolute or relative path to the partials directory
+ */
+function derivePartialsDir (outputDir) {
+  const normalized = String(outputDir).replace(/\\/g, '/')
+  const pagesIndex = normalized.lastIndexOf('/pages/')
+  if (pagesIndex !== -1) {
+    return path.join(normalized.substring(0, pagesIndex), 'partials')
+  }
+  return path.join(path.dirname(normalized), 'partials')
+}
+
+/**
+ * Write the -X -> RPK_* env vars partial from a command tree's x_options.
+ * @param {Object} tree - Parsed rpk command tree
+ * @param {string} partialsDir - Directory the partial is written into
+ * @returns {{written: boolean, keyCount?: number, output?: string}}
+ */
+function writeEnvVarsPartial (tree, partialsDir) {
+  if (!Array.isArray(tree.x_options) || tree.x_options.length === 0) {
+    return { written: false }
+  }
+  const { renderPartial, keyToEnvVar } = require('./generate-x-env-partial.js')
+  const options = tree.x_options.map(o => ({ name: o.name, env: o.env || keyToEnvVar(o.name) }))
+  const output = path.join(partialsDir, 'rpk-env-vars.adoc')
+  fs.mkdirSync(path.dirname(output), { recursive: true })
+  fs.writeFileSync(output, renderPartial(options))
+  return { written: true, keyCount: options.length, output }
+}
+
 module.exports = {
   generateRpkDocs,
+  writeEnvVarsPartial,
+  derivePartialsDir,
   mergeCommandOverrides,
   applyOverridesToTree,
   resolveReferences,

package/tools/rpk-docs/generate-x-env-partial.js

@@ -0,0 +1,284 @@
+'use strict'
+
+/**
+ * Generates the rpk env vars partial (the `-X` option -> RPK_* environment
+ * variable mapping table) from rpk itself, so the table cannot drift from
+ * the CLI.
+ *
+ * Data source, in order of preference:
+ * 1. The `x_options` array in `rpk --print-tree` JSON (added in
+ *    redpanda#31520): structured, versioned with the release, and carries
+ *    descriptions.
+ * 2. `rpk -X list` text output, for rpk versions that predate x_options.
+ *
+ * The partial is included by both reference:rpk/rpk-x-options.adoc and
+ * reference:environment-variables.adoc in the docs repo. Hidden -X options
+ * (for example cloud_environment, whose values are deliberately
+ * undocumented) appear in neither source, so they are excluded
+ * automatically.
+ */
+
+const fs = require('fs')
+const os = require('os')
+const path = require('path')
+const { spawnSync } = require('child_process')
+
+// A healthy rpk exposes ~30 -X options. Refuse to overwrite the partial
+// with a suspiciously small table (for example, if -X list output ever
+// changes format and only partially parses).
+const MIN_OPTIONS = 20
+
+const GENERATED_BANNER = `// tag::generated[]
+// Generated from rpk's own -X option data. Do not edit manually:
+// regenerate with doc-tools generate rpk-docs (which rewrites this
+// file alongside the rpk command pages) or, for a targeted refresh,
+// doc-tools generate rpk-env-partial.
+`
+
+/**
+ * Extract -X options from `rpk --print-tree` JSON.
+ * @param {string} output - Raw stdout from `rpk --print-tree`
+ * @returns {Array<{name: string, env: string}>|null} options in display
+ *   order, or null when the tree has no x_options (rpk predates
+ *   redpanda#31520). The env name comes from rpk itself when present
+ *   (the tree's env field is produced by the same derivation rpk reads).
+ */
+function xOptionsFromTree (output) {
+  let tree
+  try {
+    tree = JSON.parse(output)
+  } catch {
+    return null
+  }
+  if (!Array.isArray(tree.x_options) || tree.x_options.length === 0) return null
+  return tree.x_options.map(o => ({ name: o.name, env: o.env || keyToEnvVar(o.name) }))
+}
+
+/**
+ * Parse `rpk -X list` output into option keys (fallback for rpk versions
+ * whose --print-tree has no x_options).
+ * Each line has the form `key=value-hint`; keys contain lowercase
+ * letters, digits, dots, and underscores.
+ * @param {string} output - Raw stdout from `rpk -X list`
+ * @returns {Array<string>} option keys in CLI order
+ */
+function parseXList (output) {
+  const keys = []
+  for (const line of output.split('\n')) {
+    const eq = line.indexOf('=')
+    if (eq < 1) continue
+    const key = line.slice(0, eq).trim()
+    if (/^[a-z0-9._]+$/.test(key)) keys.push(key)
+  }
+  return keys
+}
+
+/**
+ * Convert an -X option key to its RPK_* environment variable name:
+ * prefix RPK_, uppercase, dots -> underscores.
+ * @param {string} key
+ * @returns {string}
+ */
+function keyToEnvVar (key) {
+  return 'RPK_' + key.toUpperCase().replace(/\./g, '_')
+}
+
+/**
+ * Convert an -X option key to its section anchor on the rpk-x-options page.
+ * The docs repo's AsciiDoc ID convention turns dots into hyphens but keeps
+ * underscores (tls.insecure_skip_verify -> tls-insecure_skip_verify),
+ * verified against the rendered page IDs. The previous hand-written table
+ * hyphenated underscores too, which left 12 of its 31 in-page links dead.
+ * @param {string} key
+ * @returns {string}
+ */
+function keyToAnchor (key) {
+  return key.replace(/\./g, '-')
+}
+
+/**
+ * Render the partial's AsciiDoc content.
+ * @param {Array<{name: string, env: string}>} options - -X options
+ * @returns {string}
+ */
+function renderPartial (options) {
+  const rows = options.map(({ name, env }) => {
+    const xrefTarget = `xref:reference:rpk/rpk-x-options.adoc#${keyToAnchor(name)}[${name}]`
+    return `|${xrefTarget} |${env}`
+  })
+  return `${GENERATED_BANNER}
+Every \`-X\` option has a corresponding \`RPK_*\` environment variable. Convert by prefixing with \`RPK_\` and replacing dots with underscores:
+
+[cols="1m,1m"]
+|===
+|\`-X\` Option |Environment Variable
+
+${rows.join('\n')}
+|===
+// end::generated[]
+`
+}
+
+/**
+ * Run an rpk invocation from a Go source checkout.
+ * @param {string} sourcePath - Path to src/go/rpk in a redpanda checkout
+ * @param {Array<string>} args - rpk arguments
+ * @returns {string} Raw stdout
+ */
+function runRpkFromSource (sourcePath, args) {
+  const result = spawnSync('go', ['run', 'cmd/rpk/main.go', ...args], {
+    cwd: sourcePath,
+    encoding: 'utf8',
+    timeout: 300000, // includes build time
+    maxBuffer: 50 * 1024 * 1024
+  })
+  if (result.status !== 0) {
+    throw new Error(`Failed to run rpk ${args.join(' ')} from source: ${result.stderr}`)
+  }
+  return result.stdout
+}
+
+/**
+ * Run an rpk invocation with an existing binary.
+ * @param {string} rpkBin - Path to the rpk binary
+ * @param {Array<string>} args - rpk arguments
+ * @returns {string} Raw stdout
+ */
+function runRpkBinary (rpkBin, args) {
+  const result = spawnSync(rpkBin, args, {
+    encoding: 'utf8',
+    timeout: 60000,
+    maxBuffer: 50 * 1024 * 1024
+  })
+  if (result.status !== 0) {
+    throw new Error(`Failed to run ${rpkBin} ${args.join(' ')}: ${result.stderr}`)
+  }
+  return result.stdout
+}
+
+/**
+ * Resolve a local rpk source path (repo root or src/go/rpk) without
+ * touching the checkout.
+ * @param {string} p - User-supplied path
+ * @returns {string} Path to src/go/rpk
+ */
+function resolveRpkSourcePath (p) {
+  const abs = path.resolve(p)
+  if (abs.endsWith(path.join('src', 'go', 'rpk'))) return abs
+  const nested = path.join(abs, 'src', 'go', 'rpk')
+  if (fs.existsSync(nested)) return nested
+  throw new Error(`Cannot find rpk source under ${abs}: pass the repo root or the src/go/rpk directory`)
+}
+
+/**
+ * Find the temp sparse-clone root that prepareSourceFromRef created
+ * (os.tmpdir()/rpk-source-<random>/redpanda/src/go/rpk), or null when the path is
+ * not one of ours — never delete anything outside the temp dir.
+ * @param {string} sourcePath - The src/go/rpk path returned by prepareSourceFromRef
+ * @returns {string|null}
+ */
+function tempCloneRoot (sourcePath) {
+  let dir = sourcePath
+  const tmp = fs.realpathSync(os.tmpdir())
+  const real = fs.realpathSync(dir)
+  dir = real
+  while (dir.startsWith(tmp) && dir !== tmp) {
+    if (path.basename(dir).startsWith('rpk-source-') && path.dirname(dir) === tmp) return dir
+    dir = path.dirname(dir)
+  }
+  return null
+}
+
+/**
+ * Generate the partial and write it to disk.
+ * @param {Object} options
+ * @param {string} [options.ref] - Git ref of redpanda to build rpk from
+ * @param {string} [options.fromSource] - Local src/go/rpk path (skips clone)
+ * @param {string} [options.rpkBin] - Existing rpk binary (skips build)
+ * @param {string} [options.fromJson] - Versioned tree snapshot from docs-data
+ *   (skips clone and build; the snapshot must carry x_options)
+ * @param {string} options.output - Path to write the partial to
+ * @returns {{keyCount: number, output: string, source: string}}
+ */
+function handleXEnvPartialGeneration (options) {
+  const { ref, fromSource, rpkBin, fromJson, output } = options
+  if (!output) throw new Error('Missing required --output path')
+
+  let xopts
+  let source
+  if (fromJson) {
+    // Snapshots saved by the rpk-docs handler wrap the tree in raw_tree/tree;
+    // accept a bare tree too. There is no rpk to run in this mode, so a
+    // snapshot from before x_options existed (redpanda#31520) cannot be used.
+    const j = JSON.parse(fs.readFileSync(fromJson, 'utf8'))
+    for (const candidate of [j.raw_tree, j.tree, j]) {
+      if (!candidate) continue
+      xopts = xOptionsFromTree(JSON.stringify(candidate))
+      if (xopts) break
+    }
+    if (!xopts) {
+      throw new Error(
+        `${fromJson} has no x_options; the snapshot predates redpanda#31520. ` +
+        'Regenerate with --ref/--from-source/--rpk-bin instead.'
+      )
+    }
+    source = `snapshot ${path.basename(fromJson)}`
+  } else {
+    let run
+    let cleanupDir = null
+    if (rpkBin) {
+      run = args => runRpkBinary(rpkBin, args)
+    } else if (fromSource && !ref) {
+      // Local-source mode with no explicit ref: use the checkout exactly as
+      // it stands. prepareSourceFromRef would fetch and check out a ref,
+      // silently moving the user's working tree.
+      const sourcePath = resolveRpkSourcePath(fromSource)
+      run = args => runRpkFromSource(sourcePath, args)
+    } else {
+      const { prepareSourceFromRef } = require('./rpk-docs-handler.js')
+      const sourcePath = prepareSourceFromRef(ref || 'dev', fromSource || null)
+      if (!fromSource) {
+        // prepareSourceFromRef sparse-cloned into a temp dir this run owns;
+        // remove it when done so repeated runs don't accumulate clones.
+        cleanupDir = tempCloneRoot(sourcePath)
+      }
+      run = args => runRpkFromSource(sourcePath, args)
+    }
+
+    try {
+      // Preferred: structured x_options from the command tree JSON.
+      xopts = xOptionsFromTree(run(['--print-tree']))
+      source = 'print-tree x_options'
+      if (!xopts) {
+        // Fallback for rpk versions that predate x_options in --print-tree.
+        // The env name is derived locally with the documented mapping rule.
+        xopts = parseXList(run(['-X', 'list'])).map(name => ({ name, env: keyToEnvVar(name) }))
+        source = '-X list (fallback)'
+      }
+    } finally {
+      if (cleanupDir) {
+        fs.rmSync(cleanupDir, { recursive: true, force: true })
+      }
+    }
+  }
+
+  if (xopts.length < MIN_OPTIONS) {
+    throw new Error(
+      `Parsed only ${xopts.length} -X options from ${source} (expected at least ${MIN_OPTIONS}); ` +
+      'refusing to write a suspiciously small partial. The output format may have changed.'
+    )
+  }
+
+  fs.mkdirSync(path.dirname(output), { recursive: true })
+  fs.writeFileSync(output, renderPartial(xopts))
+  return { keyCount: xopts.length, output, source }
+}
+
+module.exports = {
+  xOptionsFromTree,
+  parseXList,
+  keyToEnvVar,
+  keyToAnchor,
+  renderPartial,
+  handleXEnvPartialGeneration
+}

package/tools/rpk-docs/rpk-docs-handler.js

@@ -7,7 +7,7 @@ const fs = require('fs')
 const os = require('os')
 const semver = require('semver')
 const { findRepoRoot } = require('../../cli-utils/doc-tools-utils')
-const { generateRpkDocs, applyOverridesToTree, resolveReferences, shouldExcludeCommand, shouldUsePartialDir } = require('./generate-rpk-docs')
+const { generateRpkDocs, applyOverridesToTree, resolveReferences, shouldExcludeCommand, shouldUsePartialDir, derivePartialsDir } = require('./generate-rpk-docs')
 const { detectLinuxOnlyFromSource, warnIfDetectionLooksBroken } = require('./detect-platform-commands')
 const { generateRpkDiff, printDiffReport, generateWhatsNewSection, flattenToMap } = require('./report-delta')
 const { loadAndValidateOverrides, ValidationResult } = require('./validate-overrides')
@@ -1914,26 +1914,14 @@ async function handleRpkDocsGeneration(options = {}) {
   const defaultOutputDir = path.join(repoRoot, 'modules', 'reference', 'pages', 'rpk')
   const finalOutputDir = outputDir || defaultOutputDir
 
-  // cloudSecretDir should be relative to outputDir, not repoRoot
-  // If outputDir is .../modules/reference/pages/rpk, cloudSecretDir should be .../modules/reference/partials
-  let defaultCloudSecretDir
-  if (outputDir) {
-    // Check if outputDir follows docs repo structure (ends with pages/rpk or similar)
-    const normalizedOutput = finalOutputDir.replace(/\\/g, '/')
-    if (normalizedOutput.includes('/pages/')) {
-      // Derive cloudSecretDir from outputDir by going up from pages/rpk to partials
-      const pagesIndex = normalizedOutput.lastIndexOf('/pages/')
-      const referenceDir = normalizedOutput.substring(0, pagesIndex)
-      defaultCloudSecretDir = path.join(referenceDir, 'partials')
-    } else {
-      // Arbitrary output directory - use sibling partials directory
-      const outputParent = path.dirname(finalOutputDir)
-      defaultCloudSecretDir = path.join(outputParent, 'partials')
-    }
-  } else {
-    defaultCloudSecretDir = path.join(repoRoot, 'modules', 'reference', 'partials')
-  }
-  const finalCloudSecretDir = cloudSecretDir || defaultCloudSecretDir
+  // Partials live relative to outputDir, not repoRoot: for the docs layout
+  // (.../modules/reference/pages/rpk) that is .../modules/reference/partials,
+  // and for an arbitrary --output-dir it is a partials dir beside the output.
+  // derivePartialsDir owns that rule so each consumer does not re-derive it.
+  const derivedPartialsDir = outputDir
+    ? derivePartialsDir(finalOutputDir)
+    : path.join(repoRoot, 'modules', 'reference', 'partials')
+  const finalCloudSecretDir = cloudSecretDir || derivedPartialsDir
 
   // nav.adoc lives at modules/ROOT/nav.adoc relative to the repo root
   const navFile = path.join(repoRoot, 'modules', 'ROOT', 'nav.adoc')
@@ -2139,6 +2127,7 @@ async function handleRpkDocsGeneration(options = {}) {
         overrides: overridesData,
         outputDir: finalOutputDir,
         cloudSecretDir: finalCloudSecretDir,
+        envPartialDir: derivedPartialsDir,
         rpkVersion,
         pluginVersions,
         draftMissing,
@@ -2526,6 +2515,7 @@ async function handleRpkDocsGeneration(options = {}) {
       overrides: overridesData,
       outputDir: finalOutputDir,
       cloudSecretDir: finalCloudSecretDir,
+      envPartialDir: derivedPartialsDir,
       rpkVersion,
       pluginVersions,
       draftMissing,