@redpanda-data/docs-extensions-and-macros 5.1.0 → 5.1.2

package/extensions/algolia-indexer/generate-index.js

@@ -292,40 +292,95 @@ function generateIndex (playbook, contentCatalog, { indexLatestOnly = false, exc
       ? page.asciidoc.attributes['page-commercial-names'].split(',').map(name => name.trim())
       : []
 
-    // FIXED: keywords now included in index item
-    const indexItem = {
-      title: documentTitle,
-      version: version,
-      text: text,
-      intro: intro,
-      objectID: urlPath + page.pub.url,
-      titles: titles,
-      keywords: keywords,
-      categories: categories,
-      commercialNames: commercialNames,
-      unixTimestamp: unixTimestamp
-    }
+    // Algolia only indexes roughly the first ~290 words of any single record. On
+    // long reference pages (cluster/topic properties, metrics) every property or
+    // metric name lives in the `titles` array, and there can be hundreds of them, so
+    // every name past that window is stored but NOT searchable (DOC-1878). Split the
+    // titles of such pages across multiple records, each small enough to be fully
+    // indexed. Normal pages keep a single record. Chunks after the first get a unique
+    // objectID anchored to their first heading so search results deep-link into the
+    // page; the first chunk keeps the page's base objectID.
+    const baseObjectID = urlPath + page.pub.url
+    const titleChunks = isPropertiesPage ? chunkTitles(titles) : [titles]
+
+    titleChunks.forEach((chunk, chunkIndex) => {
+      const objectID = chunkIndex === 0
+        ? baseObjectID
+        : `${baseObjectID}#${chunk[0].h}`
+
+      // keywords included in index item
+      const indexItem = {
+        title: documentTitle,
+        version: version,
+        text: text,
+        intro: intro,
+        objectID: objectID,
+        // Clean page path (no #fragment) shared by every chunk of a page. The search
+        // UI builds result links from `url` and appends its own matched-heading
+        // anchor, so chunk objectIDs (which carry a #anchor for uniqueness) must not
+        // be used for the href. `url` is also the attribute to dedupe chunks on.
+        url: baseObjectID,
+        titles: chunk,
+        keywords: keywords,
+        categories: categories,
+        commercialNames: commercialNames,
+        unixTimestamp: unixTimestamp
+      }
 
-    if (component.name !== 'labs') {
-      indexItem.product = component.title
-      indexItem.breadcrumbs = breadcrumbs
-      indexItem.type = 'Doc'
-      indexItem._tags = Array.isArray(tag) ? tag : [tag]
-    } else {
-      indexItem.deployment = deployment
-      indexItem.type = 'Lab'
-      indexItem.interactive = false
-      indexItem._tags = Array.isArray(tag) ? tag : [tag]
-    }
+      if (component.name !== 'labs') {
+        indexItem.product = component.title
+        indexItem.breadcrumbs = breadcrumbs
+        indexItem.type = 'Doc'
+        indexItem._tags = Array.isArray(tag) ? tag : [tag]
+      } else {
+        indexItem.deployment = deployment
+        indexItem.type = 'Lab'
+        indexItem.interactive = false
+        indexItem._tags = Array.isArray(tag) ? tag : [tag]
+      }
 
-    algolia[cname][version].push(indexItem)
-    algoliaCount++
+      algolia[cname][version].push(indexItem)
+      algoliaCount++
+    })
   }
 
   logger.info(`Indexed ${algoliaCount} pages`)
   return algolia
 }
 
+/**
+ * Split a titles array into chunks whose combined token count stays within
+ * Algolia's per-record indexing window (~290 words; we leave a safety margin).
+ * Each returned chunk is small enough that every heading it contains is actually
+ * indexed for search, which keeps property/metric names on long reference pages
+ * searchable (DOC-1878).
+ *
+ * @param {Array<{t: string, h: string}>} titles - Section headings for a page.
+ * @param {number} [maxTokens=180] - Approximate token budget per chunk.
+ * @returns {Array<Array<{t: string, h: string}>>} One or more title chunks. An
+ *   empty input yields a single empty chunk so the page still gets a record.
+ */
+function chunkTitles (titles, maxTokens = 180) {
+  if (!titles.length) return [[]]
+  const chunks = []
+  let current = []
+  let tokens = 0
+  for (const entry of titles) {
+    // Algolia tokenizes on separators (whitespace, underscores, dots, etc.), so a
+    // name like `partition_autobalancing_mode` counts as several tokens.
+    const entryTokens = String(entry.t || '').split(/[\s_./:-]+/).filter(Boolean).length || 1
+    if (current.length && tokens + entryTokens > maxTokens) {
+      chunks.push(current)
+      current = []
+      tokens = 0
+    }
+    current.push(entry)
+    tokens += entryTokens
+  }
+  if (current.length) chunks.push(current)
+  return chunks
+}
+
 /**
  * Extract the path from a URL
  * @param {string} url - The URL to extract path from

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@redpanda-data/docs-extensions-and-macros",
-  "version": "5.1.0",
+  "version": "5.1.2",
   "description": "Antora extensions and macros developed for Redpanda documentation.",
   "keywords": [
     "antora",

package/tools/redpanda-connect/pr-summary-formatter.js

@@ -235,7 +235,7 @@ function generateMultiVersionPRSummary(masterDiff, binaryAnalysis = null, drafte
         else if (isSelfHostedOnly) platformIndicator = ' 🖥️';
         if (isCgoOnly) platformIndicator += ' 🔧';
 
-        lines.push(`##### \`${comp.name}\` (${comp.type}, ${comp.status})${platformIndicator}`);
+        lines.push(`##### \`${comp.name}\` (${comp.status ? `${comp.type}, ${comp.status}` : comp.type})${platformIndicator}`);
 
         // Add description (truncated to 2 sentences)
         if (comp.description) {
@@ -982,7 +982,7 @@ function generatePRSummary(diffData, binaryAnalysis = null, draftedConnectors =
         lines.push('**☁️ Cloud Supported:**');
         lines.push('');
         cloudSupportedNew.forEach(c => {
-          lines.push(`- **${c.name}** (${c.type}, ${c.status})`);
+          lines.push(`- **${c.name}** (${c.status ? `${c.type}, ${c.status}` : c.type})`);
           const desc = c.summary || c.description;
           if (desc) {
             const shortDesc = truncateToSentence(desc, 2);
@@ -996,7 +996,7 @@ function generatePRSummary(diffData, binaryAnalysis = null, draftedConnectors =
         lines.push('**Self-Hosted Only:**');
         lines.push('');
         selfHostedOnlyNew.forEach(c => {
-          lines.push(`- **${c.name}** (${c.type}, ${c.status})`);
+          lines.push(`- **${c.name}** (${c.status ? `${c.type}, ${c.status}` : c.type})`);
           const desc = c.summary || c.description;
           if (desc) {
             const shortDesc = truncateToSentence(desc, 2);
@@ -1008,7 +1008,7 @@ function generatePRSummary(diffData, binaryAnalysis = null, draftedConnectors =
     } else {
       // No cloud support info, just list all
       diffData.details.newComponents.forEach(c => {
-        lines.push(`- **${c.name}** (${c.type}, ${c.status})`);
+        lines.push(`- **${c.name}** (${c.status ? `${c.type}, ${c.status}` : c.type})`);
         const desc = c.summary || c.description;
         if (desc) {
           const shortDesc = truncateToSentence(desc, 2);
@@ -1247,7 +1247,7 @@ function generatePRSummary(diffData, binaryAnalysis = null, draftedConnectors =
     Object.entries(gapsByType).forEach(([type, connectors]) => {
       lines.push(`**${type}:**`);
       connectors.forEach(c => {
-        lines.push(`- ${c.name} (${c.status})`);
+        lines.push(`- ${c.name}${c.status ? ` (${c.status})` : ''}`);
       });
       lines.push('');
     });

package/tools/redpanda-connect/report-delta.js

@@ -21,7 +21,7 @@ function generateConnectorDiffJson(oldIndex, newIndex, opts = {}) {
     return {
       name,
       type,
-      status: raw.status || raw.type || '',
+      status: raw.status || '',
       version: raw.version || raw.introducedInVersion || '',
       description: raw.description || '',
       requiresCgo: metadata.requiresCgo || false,
@@ -39,7 +39,7 @@ function generateConnectorDiffJson(oldIndex, newIndex, opts = {}) {
     return {
       name,
       type,
-      status: raw.status || raw.type || '',
+      status: raw.status || '',
       version: raw.version || raw.introducedInVersion || '',
       description: raw.description || '',
       requiresCgo: metadata.requiresCgo || false,
@@ -101,7 +101,7 @@ function generateConnectorDiffJson(oldIndex, newIndex, opts = {}) {
       deprecatedComponents.push({
         name,
         type,
-        status: raw.status || raw.type || '',
+        status: raw.status || '',
         version: raw.version || raw.introducedInVersion || '',
         description: raw.description || ''
       });
@@ -561,7 +561,7 @@ function printDeltaReport(oldIndex, newIndex) {
     newComponentKeys.forEach(key => {
       const [type, name] = key.split(':');
       const raw = newMap[key].raw;
-      const status = raw.status || raw.type || '';
+      const status = raw.status || '';
       const version = raw.version || raw.introducedInVersion || '';
       console.log(
         `   • ${type}/${name}${

package/tools/redpanda-connect/rpcn-connector-docs-handler.js

@@ -30,6 +30,8 @@ function capToTwoSentences (description) {
 
   const abbreviations = [
     /https?:\/\/[^\s]+?(?=[.!?](?:\s|$)|\s|$)/gi,  // Protect URLs from being split by sentence detection (non-greedy, preserve trailing punctuation)
+    /\bxref:[^\s[\]]+\[[^\]]*\]/g,  // Protect Antora xref macros (e.g. xref:components:processors/try.adoc[`try`]) whose targets contain periods
+    /\b[\w/-]+\.(?:adoc|md|ya?ml|json|html)\b/gi,  // Protect bare doc/config filenames (try.adoc, config.yaml) from splitting
     /\bv\d+\.\d+(?:\.\d+)?/gi,
     /\d+\.\d+/g,
     /\be\.g\./gi,
@@ -64,6 +66,16 @@ function capToTwoSentences (description) {
   const sentenceRegex = /[^.!?]+[.!?]+(?:\s|$)/g
   const sentences = normalized.match(sentenceRegex)
 
+  // If an unprotected mid-token period prevented the leading text from
+  // forming a complete match, merge that dropped prefix into the first
+  // sentence rather than silently discarding it.
+  if (sentences && sentences.length > 0) {
+    const firstIndex = normalized.indexOf(sentences[0])
+    if (firstIndex > 0) {
+      sentences[0] = normalized.slice(0, firstIndex) + sentences[0]
+    }
+  }
+
   if (!sentences || sentences.length === 0) {
     let result = normalized
     placeholders.forEach(({ placeholder, original }) => {
@@ -135,7 +147,7 @@ function augmentConnectorData (connectorData, binaryAnalysis) {
   let addedCloudOnlyCount = 0
 
   const connectorTypes = ['inputs', 'outputs', 'processors', 'caches', 'rate_limits',
-    'buffers', 'metrics', 'scanners', 'tracers']
+    'buffers', 'metrics', 'scanners', 'tracers', 'config']
 
   for (const type of connectorTypes) {
     if (!Array.isArray(augmentedData[type])) {
@@ -195,6 +207,47 @@ function augmentConnectorData (connectorData, binaryAnalysis) {
   return { augmentedData, augmentedCount, addedCgoCount, addedCloudOnlyCount }
 }
 
+/**
+ * Build a pure-OSS snapshot from a (possibly augmented) connector index, for use
+ * as the baseline in binary analysis. Removes augmentation-only entries (the
+ * cloud-only and cgo-only connectors that augmentConnectorData() adds and persists
+ * back into the data file) and strips platform metadata so the snapshot reflects
+ * only what the standard OSS rpk build reports.
+ *
+ * @param {Object} connectorIndex - Connector index object with arrays per type
+ * @returns {Object} Deep-cloned clean copy safe to diff as pure OSS data
+ */
+function buildCleanOssData (connectorIndex) {
+  const cleanData = JSON.parse(JSON.stringify(connectorIndex))
+  const connectorTypes = ['inputs', 'outputs', 'processors', 'caches', 'rate_limits',
+    'buffers', 'metrics', 'scanners', 'tracers']
+
+  for (const type of connectorTypes) {
+    if (Array.isArray(cleanData[type])) {
+      // Keep only connectors from OSS rpk (have config/fields)
+      // Remove augmentation-only connectors (added by previous binary analysis)
+      cleanData[type] = cleanData[type].filter(c => c.config || c.fields)
+
+      // Remove platform metadata from remaining connectors
+      stripPlatformMetadata(cleanData[type])
+    }
+  }
+
+  // Config components have no config/fields wrappers, so the filter above
+  // can't distinguish OSS entries from augmentation-only ones. augmentConnectorData()
+  // adds cloud-only and cgo-only config entries and persists them back to the data
+  // file, so on subsequent runs they would otherwise be copied into the clean OSS
+  // snapshot and pollute binary analysis. Drop those augmentation-only entries by
+  // their platform markers (genuine OSS config entries never carry cloudOnly, and
+  // a standard-build component is never cgo-only) before stripping metadata.
+  if (Array.isArray(cleanData.config)) {
+    cleanData.config = cleanData.config.filter(c => c.cloudOnly !== true && c.requiresCgo !== true)
+    stripPlatformMetadata(cleanData.config)
+  }
+
+  return cleanData
+}
+
 /**
  * Update whats-new.adoc with new release information
  * @param {Object} params - Parameters
@@ -1097,20 +1150,7 @@ async function handleRpcnConnectorDocs (options) {
   const cleanOssDataPath = path.join(dataDir, `._connect-${newVersion}-clean.json`)
 
   // Create clean version by removing augmented connectors
-  const cleanData = JSON.parse(JSON.stringify(newIndex))
-  const connectorTypes = ['inputs', 'outputs', 'processors', 'caches', 'rate_limits',
-    'buffers', 'metrics', 'scanners', 'tracers']
-
-  for (const type of connectorTypes) {
-    if (Array.isArray(cleanData[type])) {
-      // Keep only connectors from OSS rpk (have config/fields)
-      // Remove augmentation-only connectors (added by previous binary analysis)
-      cleanData[type] = cleanData[type].filter(c => c.config || c.fields)
-
-      // Remove platform metadata from remaining connectors
-      stripPlatformMetadata(cleanData[type])
-    }
-  }
+  const cleanData = buildCleanOssData(newIndex)
 
   fs.writeFileSync(cleanOssDataPath, JSON.stringify(cleanData, null, 2), 'utf8')
 
@@ -1357,7 +1397,7 @@ async function handleRpcnConnectorDocs (options) {
       // Strip CGO/cloud-only connectors and metadata from old data
       oldIndexForDiff = JSON.parse(JSON.stringify(oldIndex))
       const connectorTypes = ['inputs', 'outputs', 'processors', 'caches', 'rate_limits',
-        'buffers', 'metrics', 'scanners', 'tracers']
+        'buffers', 'metrics', 'scanners', 'tracers', 'config']
 
       let totalStripped = 0
       for (const type of connectorTypes) {
@@ -1970,5 +2010,7 @@ async function handleRpcnConnectorDocs (options) {
 module.exports = {
   handleRpcnConnectorDocs,
   updateWhatsNew,
-  capToTwoSentences
+  capToTwoSentences,
+  augmentConnectorData,
+  buildCleanOssData
 }

package/tools/rpk-docs/generate-rpk-docs.js

@@ -274,6 +274,23 @@ function shouldExcludeCommand(overrides, commandPath) {
   return commandOverride?.exclude === true
 }
 
+/**
+ * Check if a command should be written to the partials directory (cloudSecretDir).
+ * Walks up the command path so setting asPartial: true on "rpk ai" applies to all children.
+ * @param {Object} overrides - Resolved overrides object
+ * @param {string} commandPath - Full command path (e.g., "rpk ai agent list")
+ * @returns {boolean}
+ */
+function shouldUsePartialDir(overrides, commandPath) {
+  if (!overrides?.commands) return false
+  const parts = commandPath.split(' ')
+  for (let i = parts.length; i >= 1; i--) {
+    const ancestor = parts.slice(0, i).join(' ')
+    if (overrides.commands[ancestor]?.asPartial === true) return true
+  }
+  return false
+}
+
 /**
  * Get command metadata from overrides
  * @param {Object} overrides - Overrides object (resolved)
@@ -390,13 +407,21 @@ function processContentArray(contentArray, context = '') {
     }
 
     switch (type) {
-      case 'section':
+      case 'section': {
+        // Escape {word} (no hyphens) in table cell lines — CLI format specifiers like {hex}
+        // are not AsciiDoc attribute refs and must be escaped to prevent substitution.
+        const escapedContent = typeof content === 'string'
+          ? content.replace(/^\|(.*)$/gm, (m, cell) =>
+              /\{[a-z_][a-z0-9_]*\}/.test(cell)
+                ? '|' + cell.replace(/\{([a-z_][a-z0-9_]*)\}/g, '\\{$1}')
+                : m)
+          : content
         // Pass through all section properties (subsections, parent, headingLevel, exclude)
         result.sections[position].push({
           type: 'section',
           id,
           title,
-          content,
+          content: escapedContent,
           subsections: item.subsections,
           parent: item.parent,
           headingLevel: item.headingLevel,
@@ -408,6 +433,7 @@ function processContentArray(contentArray, context = '') {
           console.warn(`   Use seeAlso key instead`)
         }
         break
+      }
 
       case 'example':
         // Structured single example
@@ -1184,7 +1210,10 @@ function convertIndentedTablesToAsciiDoc(text, options = {}) {
             // Remove trailing colon from value (e.g., "Organization:" -> "Organization")
             const cleanValue = value.replace(/:$/, '')
             const desc = descParts.join(' ')
-            result.push(`|${cleanValue} |${desc}`)
+            // Escape {word} (no hyphens) in table cells — CLI format specifiers like {hex}, {json}
+            // are not AsciiDoc attribute refs and must be escaped to prevent substitution.
+            const escapeAttrs = s => s.replace(/\{([a-z_][a-z0-9_]*)\}/g, '\\{$1}')
+            result.push(`|${escapeAttrs(cleanValue)} |${escapeAttrs(desc)}`)
           }
           result.push('|===')
           result.push('')
@@ -1203,6 +1232,28 @@ function convertIndentedTablesToAsciiDoc(text, options = {}) {
   return result.join('\n')
 }
 
+/**
+ * Apply only the text replacements from textTransformations to a raw string.
+ * Used for examples content where we want string substitutions (e.g. rpai → rpk ai)
+ * but NOT AsciiDoc structural formatting or inlineCode wrapping.
+ * @param {string} text
+ * @param {Object|null} customTransformations
+ * @returns {string}
+ */
+function applyTextTransformations(text, customTransformations) {
+  if (!text || !customTransformations?.replacements) return text
+  let result = text
+  for (const rule of customTransformations.replacements) {
+    try {
+      const flags = rule.flags || 'g'
+      result = result.replace(new RegExp(rule.pattern, flags), rule.replacement)
+    } catch (err) {
+      console.warn(`⚠ Invalid replacement pattern: ${rule.pattern}`)
+    }
+  }
+  return result
+}
+
 /**
  * Format description by adding backticks around flags and code
  * @param {string} desc - Description text
@@ -1289,10 +1340,18 @@ function formatDescription(desc, customTransformations = null, options = {}) {
   })
 
   // Protect inline code (backticked content)
+  // Escape {word} (no hyphens) inside backtick spans — these are CLI format specifiers or
+  // path placeholders, not AsciiDoc attribute refs (which use hyphens like {full-version}).
+  // AsciiDoc single-backtick spans still apply attribute substitution, so {hex}/{namespace}/etc.
+  // would expand to empty string without this escaping.
   const inlineCode = []
   preProcessed = preProcessed.replace(/`[^`]+`/g, (match) => {
     const placeholder = `__INLINE_CODE_${inlineCode.length}__`
-    inlineCode.push(match)
+    // Skip `+...+` passthrough spans — they already prevent attribute substitution
+    const escaped = (match.startsWith('`+') && match.endsWith('+`'))
+      ? match
+      : match.replace(/\{([a-z_][a-z0-9_]*)\}/g, '\\{$1}')
+    inlineCode.push(escaped)
     return placeholder
   })
 
@@ -1536,28 +1595,46 @@ function formatDescription(desc, customTransformations = null, options = {}) {
   // Restore early code blocks FIRST (from indented command/YAML detection)
   // This must happen before inline code restoration so that placeholders
   // inside the early code blocks (like __INLINE_CODE_X__) get resolved
+  // Use function replacements to prevent $ special-pattern interpretation (e.g. `$` → before-match)
   earlyCodeBlocks.forEach((block, i) => {
-    result = result.replace(`__EARLY_CODE_BLOCK_${i}__`, block)
+    result = result.replace(`__EARLY_CODE_BLOCK_${i}__`, () => block)
   })
 
   // Restore inline code
   inlineCode.forEach((code, i) => {
-    result = result.replace(`__INLINE_CODE_${i}__`, code)
+    result = result.replace(`__INLINE_CODE_${i}__`, () => code)
   })
 
   // Restore xrefs
   xrefs.forEach((xref, i) => {
-    result = result.replace(`__XREF_${i}__`, xref)
+    result = result.replace(`__XREF_${i}__`, () => xref)
   })
 
   // Restore code blocks
   codeBlocks.forEach((block, i) => {
-    result = result.replace(`__CODE_BLOCK_${i}__`, block)
+    result = result.replace(`__CODE_BLOCK_${i}__`, () => block)
   })
 
   // Clean up double backticks (moved AFTER restoration to catch nested backticks)
   result = result.replace(/``+/g, '`')
 
+  // Final pass: escape {word} (no hyphens) inside any backtick spans that were created
+  // dynamically during processing (after the initial inline-code protection step).
+  // Skip `+...+` passthrough spans — they already prevent attribute substitution.
+  // Use negative lookbehind (?<!\\) to avoid double-escaping already-escaped \{word}.
+  result = result.replace(/`[^`]+`/g, match => {
+    if (match.startsWith('`+') && match.endsWith('+`')) return match
+    return match.replace(/(?<!\\)\{([a-z_][a-z0-9_]*)\}/g, '\\{$1}')
+  })
+
+  // Escape {word} (no hyphens) in AsciiDoc table cell lines that come from pre-formatted
+  // AsciiDoc in CLI descriptions (e.g. format specifier tables with {hex}, {json}).
+  // These never pass through backtick-span protection, so they need a separate pass.
+  result = result.replace(/^\|(.*)$/gm, (match, cellContent) => {
+    if (!/(?<!\\)\{[a-z_][a-z0-9_]*\}/.test(cellContent)) return match
+    return '|' + cellContent.replace(/(?<!\\)\{([a-z_][a-z0-9_]*)\}/g, '\\{$1}')
+  })
+
   // Remove trailing period after backticked commands on their own line
   // e.g., "`rpk cluster status`." → "`rpk cluster status`"
   // Only consume trailing spaces, not newlines (to preserve paragraph breaks)
@@ -1581,40 +1658,48 @@ function formatUsage(usage) {
 }
 
 /**
- * Format EXAMPLES section - convert indented commands and inline code to AsciiDoc code blocks
+ * Format EXAMPLES section - convert indented commands and inline code to AsciiDoc code blocks.
+ *
+ * Handles the standard Go CLI example format where each group is separated by a blank line,
+ * comment lines (# ...) describe the example, and command lines follow:
+ *
+ *   # Description of the example
+ *   rpk some command --flag
+ *
+ *   # Another example
+ *   rpk other command
+ *   rpk other command --variant
+ *
+ * Comments become plain text descriptions; consecutive command lines share one code block.
+ *
  * @param {string} text - Examples text with indented commands or inline code
  * @returns {string} Formatted examples with code blocks
  */
 function formatExamples(text) {
   if (!text) return text
 
-  // Convert both indented commands and inline backticked commands to AsciiDoc code blocks
-  // Pattern 1: description line, then indented command (starts with 2+ spaces)
-  // Pattern 2: description line ending with colon/period, followed by `rpk command`
   const lines = text.split('\n')
   const result = []
   let i = 0
 
   while (i < lines.length) {
     const line = lines[i]
+    const isIndented = /^\s{2,}\S/.test(line)
 
-    // Check if next line is an indented command (2+ spaces followed by non-space)
-    if (i + 1 < lines.length && /^\s{2,}\S/.test(lines[i + 1])) {
-      // Current line is description, next line is command
-      result.push(line)
-      result.push('')
-      result.push('[,bash]')
-      result.push('----')
-      result.push(lines[i + 1].trim())
-      result.push('----')
-      i += 2
-    } else if (/^\s{2,}\S/.test(line)) {
-      // Standalone indented command without preceding description
+    if (isIndented && line.trim().startsWith('#')) {
+      // Comment line — render as a description paragraph outside the code block
+      result.push(line.trim().replace(/^#\s*/, ''))
+      i++
+    } else if (isIndented) {
+      // Command line — collect all consecutive command lines into one code block
+      // (stops at a blank line or a comment line)
       result.push('[,bash]')
       result.push('----')
-      result.push(line.trim())
+      while (i < lines.length && /^\s{2,}\S/.test(lines[i]) && !lines[i].trim().startsWith('#')) {
+        result.push(lines[i].trim())
+        i++
+      }
       result.push('----')
-      i++
     } else if (/^\s*`rpk .+`\s*$/.test(line)) {
       // Inline backticked command on its own line - convert to code block
       const command = line.trim().replace(/^`|`$/g, '')
@@ -1637,7 +1722,7 @@ function formatExamples(text) {
       result.push('----')
       i += 2
     } else {
-      // Regular line (description, blank line, etc.)
+      // Regular line (blank line, non-indented description text, etc.)
       result.push(line)
       i++
     }
@@ -1997,6 +2082,91 @@ function findTopLevelWithSubcommands(tree) {
   return result
 }
 
+/**
+ * Static entries that always appear at the top of the rpk nav section,
+ * before the auto-generated command entries. These are hand-written pages
+ * that are not part of the CLI command tree. The root `rpk` command is not
+ * listed here: it is represented by the section's hand-written index.adoc
+ * landing page (the section header), so it is skipped during generation.
+ */
+const STATIC_RPK_NAV_ENTRIES = [
+  '*** xref:reference:rpk/rpk-commands.adoc[]',
+  '*** xref:reference:rpk/rpk-x-options.adoc[rpk -X]',
+]
+
+/**
+ * Update the rpk section of nav.adoc with the current command tree.
+ * Replaces auto-generated entries while preserving the section header and
+ * the static hand-written entries defined in STATIC_RPK_NAV_ENTRIES.
+ * @param {string} navFile - Absolute path to nav.adoc
+ * @param {Array} commands - Flat command list from flattenCommands()
+ * @param {Object} resolvedOverrides - Resolved overrides
+ * @param {Set} topLevelWithSubcommands - Set of top-level command names with subcommands
+ * @returns {Object} { navUpdated, navEntriesGenerated }
+ */
+function updateNavFile(navFile, commands, resolvedOverrides, topLevelWithSubcommands) {
+  if (!fs.existsSync(navFile)) {
+    console.warn(`Warning: nav file not found, skipping nav update: ${navFile}`)
+    return { navUpdated: false }
+  }
+
+  const content = fs.readFileSync(navFile, 'utf8')
+  const lines = content.split('\n')
+
+  // Locate the rpk section header
+  const sectionStart = lines.findIndex(l => /xref:reference:rpk\/index\.adoc/.test(l))
+  if (sectionStart === -1) {
+    console.warn('Warning: rpk section not found in nav file, skipping nav update')
+    return { navUpdated: false }
+  }
+
+  // Find where the section ends — next line with the same or fewer asterisks as the header
+  const headerStars = (lines[sectionStart].match(/^\*+/) || [''])[0].length
+  let sectionEnd = lines.length
+  for (let i = sectionStart + 1; i < lines.length; i++) {
+    const m = lines[i].match(/^\*+/)
+    if (m && m[0].length <= headerStars && lines[i].trim() !== '') {
+      sectionEnd = i
+      break
+    }
+  }
+
+  // Generate nav entries from the command tree (DFS order from flattenCommands)
+  const newEntries = []
+  for (const { path: commandPath } of commands) {
+    if (commandPath === 'rpk') continue
+    if (shouldExcludeCommand(resolvedOverrides, commandPath)) continue
+    if (shouldUsePartialDir(resolvedOverrides, commandPath)) continue
+    if (commandPath.startsWith('rpk cloud')) continue
+    if (commandPath.startsWith('rpk security secret')) continue
+
+    const parts = commandPath.split(' ')
+    const stars = '*'.repeat(parts.length + 1)
+    const { subdir, fileName } = getOutputPath(commandPath, topLevelWithSubcommands)
+    const xrefPath = subdir
+      ? `reference:rpk/${subdir}/${fileName}`
+      : `reference:rpk/${fileName}`
+
+    newEntries.push(`${stars} xref:${xrefPath}[]`)
+  }
+
+  // Rebuild: header + static entries + generated entries
+  const newSection = [
+    lines[sectionStart],
+    ...STATIC_RPK_NAV_ENTRIES,
+    ...newEntries,
+  ]
+
+  const newLines = [
+    ...lines.slice(0, sectionStart),
+    ...newSection,
+    ...lines.slice(sectionEnd),
+  ]
+
+  fs.writeFileSync(navFile, newLines.join('\n'), 'utf8')
+  return { navUpdated: true, navEntriesGenerated: newEntries.length }
+}
+
 /**
  * Generate AsciiDoc documentation for all rpk commands
  * @param {Object} options - Generation options
@@ -2011,7 +2181,8 @@ async function generateRpkDocs(options = {}) {
     rpkVersion,
     pluginVersions = {},
     draftMissing = false,
-    flatOutput = false // If true, output all files flat (legacy behavior)
+    flatOutput = false, // If true, output all files flat (legacy behavior)
+    navFile // Optional: path to nav.adoc for automatic nav updates
   } = options
 
   // Register partials
@@ -2058,8 +2229,18 @@ async function generateRpkDocs(options = {}) {
   let filesGenerated = 0
   let filesSkipped = 0
   let filesFailed = 0
+  let filesDeleted = 0
+  const writtenFiles = new Set()
 
   for (const { path: commandPath, command } of commands) {
+    // Skip the root `rpk` command: its reference is the section's hand-written
+    // index.adoc landing page, which generation must not overwrite. The nav
+    // builder skips it for the same reason (see updateNavFile).
+    if (commandPath === 'rpk') {
+      filesSkipped++
+      continue
+    }
+
     // Check if command should be excluded
     if (shouldExcludeCommand(resolvedOverrides, commandPath)) {
       filesSkipped++
@@ -2087,7 +2268,8 @@ async function generateRpkDocs(options = {}) {
         if (mergedCommand.excludeExamples && mergedCommand.excludeExamples.length > 0) {
           examplesContent = filterExamples(sectionContent, mergedCommand.excludeExamples)
         }
-        // Format examples: convert indented commands to code blocks
+        // Apply text transformations (e.g. rpai → rpk ai) then format
+        examplesContent = applyTextTransformations(examplesContent, textTransformations)
         sections[sectionName] = formatExamples(examplesContent)
       } else {
         sections[sectionName] = formatDescription(sectionContent, textTransformations)
@@ -2097,11 +2279,15 @@ async function generateRpkDocs(options = {}) {
     // Add examples from rpk source if not already in sections
     // rpk --print-tree provides examples in a separate field
     if (command.examples && !sections.EXAMPLES) {
-      let examplesContent = command.examples.trim()
+      // Strip surrounding blank lines only — preserve per-line indentation so formatExamples
+      // can detect indented command lines vs plain description text
+      let examplesContent = command.examples.replace(/^\n+/, '').replace(/\n+$/, '')
       // Apply excludeExamples filter BEFORE formatting
       if (mergedCommand.excludeExamples && mergedCommand.excludeExamples.length > 0) {
         examplesContent = filterExamples(examplesContent, mergedCommand.excludeExamples)
       }
+      // Apply text transformations (e.g. rpai → rpk ai) then format
+      examplesContent = applyTextTransformations(examplesContent, textTransformations)
       sections.EXAMPLES = formatExamples(examplesContent)
     }
 
@@ -2172,11 +2358,12 @@ async function generateRpkDocs(options = {}) {
     }
 
     // Build subcommands with correct xref paths
-    // Filter out excluded subcommands so they don't appear in the parent's list
+    // Filter out excluded and asPartial subcommands — excluded have no file,
+    // asPartial ones live in the partials directory with no linkable xref.
     const subcommands = (command.commands || [])
       .filter(sub => {
         const subPath = `${commandPath} ${sub.name}`
-        return !shouldExcludeCommand(resolvedOverrides, subPath)
+        return !shouldExcludeCommand(resolvedOverrides, subPath) && !shouldUsePartialDir(resolvedOverrides, subPath)
       })
       .map(sub => {
         const subPath = `${commandPath} ${sub.name}`
@@ -2357,7 +2544,9 @@ async function generateRpkDocs(options = {}) {
     // Check if this command should go to cloudSecretDir
     const isCloudCommand = commandPath.startsWith('rpk cloud')
     const isSecuritySecretCommand = commandPath.startsWith('rpk security secret')
-    const useCloudSecretDir = cloudSecretDir && (isCloudCommand || isSecuritySecretCommand)
+    const useCloudSecretDir = cloudSecretDir && (
+      isCloudCommand || isSecuritySecretCommand || shouldUsePartialDir(resolvedOverrides, commandPath)
+    )
     const effectiveOutputDir = useCloudSecretDir ? cloudSecretDir : outputDir
 
     let filePath
@@ -2384,6 +2573,7 @@ async function generateRpkDocs(options = {}) {
 
     try {
       fs.writeFileSync(filePath, content, 'utf8')
+      writtenFiles.add(filePath)
       filesGenerated++
 
       if (filesGenerated % 50 === 0) {
@@ -2395,12 +2585,71 @@ async function generateRpkDocs(options = {}) {
     }
   }
 
+  // Delete stale .adoc files from removed/excluded commands, including whole
+  // subtrees that no longer receive any writes this run.
+  //
+  // We only clean the "rpk-*" subdirectories (e.g. rpk-topic/, rpk-cluster/),
+  // which are created and owned exclusively by generation. We deliberately do
+  // NOT touch the root output directory: it holds hand-written pages alongside
+  // generated ones (e.g. index.adoc, rpk-commands.adoc, rpk-x-options.adoc), and
+  // those hand-written pages share the "rpk-*.adoc" naming, so there is no safe
+  // way to distinguish a stale generated root page from a hand-written one. A
+  // rare stale root page is preferable to deleting hand-written content.
+  //
+  // Deletions are further restricted to the generated "rpk.adoc"/"rpk-*.adoc"
+  // pattern so a stray non-rpk file inside a subdir is never removed. This also
+  // lets us clean the rpk-* subdirs of the shared partials directory
+  // (cloudSecretDir) without touching hand-written partials at its root.
+  const generatedFilePattern = /^rpk(-[^/\\]*)?\.adoc$/
+
+  const rpkSubdirs = (baseDir) => {
+    if (!baseDir || !fs.existsSync(baseDir)) return []
+    return fs.readdirSync(baseDir, { withFileTypes: true })
+      .filter(entry => entry.isDirectory() && entry.name.startsWith('rpk-'))
+      .map(entry => path.join(baseDir, entry.name))
+  }
+
+  const scanDirs = new Set()
+  for (const dir of rpkSubdirs(outputDir)) scanDirs.add(dir)
+  for (const dir of rpkSubdirs(cloudSecretDir)) scanDirs.add(dir)
+
+  for (const dir of scanDirs) {
+    for (const entry of fs.readdirSync(dir, { withFileTypes: true })) {
+      if (!entry.isFile() || !generatedFilePattern.test(entry.name)) continue
+      const full = path.join(dir, entry.name)
+      if (!writtenFiles.has(full)) {
+        try {
+          fs.unlinkSync(full)
+          filesDeleted++
+        } catch (err) {
+          console.warn(`Warning: Failed to delete stale file "${full}": ${err.message}`)
+        }
+      }
+    }
+    // Remove the rpk-* subdirectory itself if it is now empty (removed subtree).
+    if (fs.readdirSync(dir).length === 0) {
+      try {
+        fs.rmdirSync(dir)
+      } catch (err) {
+        console.warn(`Warning: Failed to remove empty directory "${dir}": ${err.message}`)
+      }
+    }
+  }
+
+  // Update nav.adoc if a path was provided
+  let navResult = { navUpdated: false }
+  if (navFile) {
+    navResult = updateNavFile(navFile, commands, resolvedOverrides, topLevelWithSubcommands)
+  }
+
   return {
     commandCount: commands.length,
     filesGenerated,
     filesSkipped,
     filesFailed,
-    subdirectoriesCreated: createdSubdirs.size
+    filesDeleted,
+    subdirectoriesCreated: createdSubdirs.size,
+    ...navResult
   }
 }
 
@@ -2425,6 +2674,8 @@ module.exports = {
   loadTemplate,
   registerPartial,
   shouldExcludeCommand,
+  shouldUsePartialDir,
+  updateNavFile,
   getCommandMetadata,
   processContentArray,
   // Exported for testing

package/tools/rpk-docs/rpk-docs-handler.js

@@ -387,18 +387,27 @@ function fetchRpkTreeFromLinuxSource(sourcePath) {
 
   // Start a container with the Go image, mount source, build rpk, install plugins, then print-tree
   // Use a long-running container so we can run multiple commands
+  // Pin the Go image to the exact version required by go.mod to prevent
+  // "go.mod requires go >= X.Y.Z (running X.Y.Z-1)" build failures when
+  // golang:1 resolves to a patch release behind the requirement.
+  const requiredGoVersion = getRequiredGoVersion(absoluteSourcePath)
+  const goImage = requiredGoVersion ? `golang:${requiredGoVersion}` : 'golang:1'
+
   console.log('Starting build container...')
   const createResult = spawnSync('docker', [
     'run', '-d', '--rm',
     '-v', `${absoluteSourcePath}:/rpk-source:ro`,
     '-w', '/rpk-source',
-    'golang:1',
+    goImage,
     'sh', '-c', 'sleep 600' // Keep container alive for 10 minutes
   ], {
     encoding: 'utf8',
     timeout: 60000
   })
 
+  let activeResult = createResult
+  let activeImage = goImage
+
   if (createResult.status !== 0) {
     const stderr = createResult.stderr || ''
     if (stderr.includes('Cannot connect to the Docker daemon')) {
@@ -407,13 +416,40 @@ function fetchRpkTreeFromLinuxSource(sourcePath) {
         'Start Docker Desktop or the Docker service and try again.'
       )
     }
-    throw new Error(
-      `Failed to create build container: ${stderr}\n` +
-      'Make sure Docker is running and has sufficient resources.'
+
+    // If the pinned tag couldn't be pulled (not yet on Docker Hub for a new
+    // patch release), retry with golang:1 before giving up.
+    const isPullFailure = requiredGoVersion && (
+      stderr.includes('manifest unknown') ||
+      stderr.includes('pull access denied') ||
+      stderr.includes('not found') ||
+      stderr.includes('repository does not exist')
     )
+    if (isPullFailure) {
+      console.warn(`⚠ Could not pull ${goImage}: ${stderr.trim()}`)
+      console.log('Retrying with golang:1...')
+      activeImage = 'golang:1'
+      activeResult = spawnSync('docker', [
+        'run', '-d', '--rm',
+        '-v', `${absoluteSourcePath}:/rpk-source:ro`,
+        '-w', '/rpk-source',
+        activeImage,
+        'sh', '-c', 'sleep 600'
+      ], {
+        encoding: 'utf8',
+        timeout: 60000
+      })
+    }
+
+    if (activeResult.status !== 0) {
+      throw new Error(
+        `Failed to create build container: ${activeResult.stderr || stderr}\n` +
+        'Make sure Docker is running and has sufficient resources.'
+      )
+    }
   }
 
-  const containerId = createResult.stdout.trim()
+  const containerId = activeResult.stdout.trim()
   console.log(`Build container started: ${containerId.substring(0, 12)}`)
 
   try {
@@ -1041,6 +1077,9 @@ async function handleRpkDocsGeneration(options = {}) {
   }
   const finalCloudSecretDir = cloudSecretDir || defaultCloudSecretDir
 
+  // nav.adoc lives at modules/ROOT/nav.adoc relative to the repo root
+  const navFile = path.join(repoRoot, 'modules', 'ROOT', 'nav.adoc')
+
   // Determine which version to document
   const version = effectiveRef || 'dev'
   console.log(`\nGenerating rpk documentation for version: ${version}`)
@@ -1088,12 +1127,19 @@ async function handleRpkDocsGeneration(options = {}) {
         rpkVersion,
         pluginVersions: {},
         draftMissing,
-        preservationsDir: preserveFrom
+        preservationsDir: preserveFrom,
+        navFile
       })
 
       console.log(`\nGeneration complete!`)
       console.log(`  - Commands documented: ${result.commandCount}`)
       console.log(`  - Files generated: ${result.filesGenerated}`)
+      if (result.filesDeleted > 0) {
+        console.log(`  - Stale files deleted: ${result.filesDeleted}`)
+      }
+      if (result.navUpdated) {
+        console.log(`  - Nav entries updated: ${result.navEntriesGenerated}`)
+      }
       console.log(`  - Output directory: ${finalOutputDir}`)
 
       // Run validation on generated output
@@ -1387,12 +1433,19 @@ async function handleRpkDocsGeneration(options = {}) {
       rpkVersion,
       pluginVersions,
       draftMissing,
-      preservationsDir: preserveFrom
+      preservationsDir: preserveFrom,
+      navFile
     })
 
     console.log(`\nGeneration complete!`)
     console.log(`  - Commands documented: ${result.commandCount}`)
     console.log(`  - Files generated: ${result.filesGenerated}`)
+    if (result.filesDeleted > 0) {
+      console.log(`  - Stale files deleted: ${result.filesDeleted}`)
+    }
+    if (result.navUpdated) {
+      console.log(`  - Nav entries updated: ${result.navEntriesGenerated}`)
+    }
     console.log(`  - Output directory: ${finalOutputDir}`)
 
     // Run validation on generated output
@@ -1626,6 +1679,7 @@ module.exports = {
   handleRpkDocsGeneration,
   fetchRpkTreeFromSource,
   fetchRpkTreeFromLinuxSource,
+  getRequiredGoVersion,
   prepareSourceFromRef,
   detectLinuxOnlyFromSource,
   addPlatformMarkersFromSource,

package/tools/rpk-docs/templates/command.hbs

@@ -18,7 +18,7 @@ ifndef::env-cloud[]
 :page-deprecated: true
 {{/if}}
 
-// This content is autogenerated. Do not edit manually. To customize content, see the writer's guide: https://github.com/redpanda-data/docs-extensions-and-macros/blob/main/docs-data/RPK_OVERRIDES_GUIDE.adoc
+// This content is autogenerated. Do not edit manually. To customize content, see the writer's guide: https://github.com/redpanda-data/docs/blob/main/docs-data/RPK_OVERRIDES_GUIDE.adoc
 {{#each pageAttributes}}
 :{{@key}}: {{this}}
 {{/each}}

package/tools/rpk-docs/validate-overrides.js

@@ -107,9 +107,23 @@ function normalizeCommandPath(cmdPath) {
 
 /**
  * Load and compile the JSON Schema for validation
+ * @param {string} [overridesPath] - Path to the overrides file; the schema is
+ *   looked up as a sibling (same directory) before falling back to the
+ *   package-relative path.
  * @returns {Function} Compiled schema validator
  */
-function loadSchemaValidator() {
+function loadSchemaValidator(overridesPath) {
+  // Primary: look for the schema next to the overrides file (docs repo's docs-data/).
+  if (overridesPath) {
+    const siblingPath = path.join(path.dirname(path.resolve(overridesPath)), 'rpk-overrides.schema.json')
+    if (fs.existsSync(siblingPath)) {
+      const schema = JSON.parse(fs.readFileSync(siblingPath, 'utf8'))
+      const ajv = new Ajv2020({ allErrors: true, verbose: true })
+      return ajv.compile(schema)
+    }
+  }
+
+  // Fallback: package-relative path (legacy / monorepo usage).
   const schemaPath = path.resolve(__dirname, '../../docs-data/rpk-overrides.schema.json')
 
   if (!fs.existsSync(schemaPath)) {
@@ -124,13 +138,14 @@ function loadSchemaValidator() {
 /**
  * Validate overrides against JSON Schema
  * @param {Object} overrides - Overrides object to validate
+ * @param {string} [overridesPath] - Path to the overrides file (used to locate schema)
  * @returns {ValidationResult}
  */
-function validateSchema(overrides) {
+function validateSchema(overrides, overridesPath) {
   const result = new ValidationResult()
 
   try {
-    const validate = loadSchemaValidator()
+    const validate = loadSchemaValidator(overridesPath)
     const valid = validate(overrides)
 
     if (!valid) {
@@ -531,7 +546,7 @@ function validateDescriptions(commandOverride, context) {
  * @param {Object} [commandTree] - Optional command tree for path validation
  * @returns {ValidationResult}
  */
-function validateOverrides(overrides, commandTree = null) {
+function validateOverrides(overrides, commandTree = null, overridesPath = null) {
   const result = new ValidationResult()
 
   if (!overrides || typeof overrides !== 'object') {
@@ -540,7 +555,7 @@ function validateOverrides(overrides, commandTree = null) {
   }
 
   // 1. Schema validation
-  result.merge(validateSchema(overrides))
+  result.merge(validateSchema(overrides, overridesPath))
 
   // 2. Reference validation
   result.merge(validateReferences(overrides))
@@ -596,7 +611,7 @@ function loadAndValidateOverrides(overridesPath, commandTree = null) {
   }
 
   // Run validations
-  validation.merge(validateOverrides(overrides, commandTree))
+  validation.merge(validateOverrides(overrides, commandTree, overridesPath))
 
   return { overrides, validation }
 }