@redpanda-data/docs-extensions-and-macros 5.0.4 → 5.1.0

package/bin/doc-tools-mcp.js

@@ -151,18 +151,46 @@ const tools = [
   },
   {
     name: 'generate_rpk_docs',
-    description: 'Generate RPK command-line documentation for a specific version. Creates AsciiDoc files for all rpk commands. Writers use this when updating CLI docs for a new release. Can run in background with progress streaming.',
+    description: 'Generate rpk command-line documentation using the JSON-based `rpk --print-tree` command (requires rpk v26.2.0+). Creates AsciiDoc files for all rpk commands including plugins (connect, ai, check, etc.). Supports overrides for description improvements and generates diffs between versions.',
     inputSchema: {
       type: 'object',
       properties: {
         tag: {
           type: 'string',
-          description: 'Git tag for released content (for example "25.3.1" or "v25.3.1"). Auto-prepends "v" if not present. Use tags for GA or beta releases.'
+          description: 'Git tag for released content (for example "26.2.0" or "v26.2.0"). Auto-prepends "v" if not present. Use tags for GA or beta releases.'
         },
         branch: {
           type: 'string',
           description: 'Branch name for in-progress content (for example "dev", "main"). Use branches for documentation under development.'
         },
+        overrides: {
+          type: 'string',
+          description: 'Path to overrides JSON file for improving auto-generated descriptions (defaults to docs-data/rpk-overrides.json if present)'
+        },
+        fetch_binary: {
+          type: 'boolean',
+          description: 'Download rpk binary for the specified version instead of using local rpk. Required for versions with --print-tree support.'
+        },
+        diff: {
+          type: 'string',
+          description: 'Generate diff against previous version (for example "v26.1.9"). Shows new commands, removed commands, and flag changes.'
+        },
+        update_whats_new: {
+          type: 'boolean',
+          description: 'Update whats-new.adoc with changes from the diff (optional, defaults to false)'
+        },
+        draft_missing: {
+          type: 'boolean',
+          description: 'Generate draft pages for new commands that appear in the diff (optional, defaults to false)'
+        },
+        output_dir: {
+          type: 'string',
+          description: 'Output directory for generated AsciiDoc files (defaults to modules/reference/pages/rpk)'
+        },
+        data_dir: {
+          type: 'string',
+          description: 'Directory for versioned JSON and diff files (defaults to docs-data)'
+        },
         background: {
           type: 'boolean',
           description: 'Run as background job with progress updates. Returns job ID immediately instead of waiting. Default: false (run synchronously)'

package/bin/doc-tools.js

@@ -181,6 +181,76 @@ programCli
     }
   })
 
+/**
+ * get-antora-value
+ *
+ * @description
+ * Reads a value from antora.yml using a dot-separated key path.
+ * Useful for retrieving version attributes and other configuration in CI/CD pipelines.
+ *
+ * @why
+ * Automation workflows need to read the currently documented versions from antora.yml
+ * to determine what version to diff against when generating updated documentation.
+ *
+ * @example
+ * # Get the latest documented Redpanda version
+ * npx doc-tools get-antora-value asciidoc.attributes.latest-redpanda-tag
+ * # Output: v26.1.9
+ *
+ * # Get the component name
+ * npx doc-tools get-antora-value name
+ * # Output: ROOT
+ *
+ * @requirements
+ * - Must run from directory containing antora.yml or antora.yaml
+ */
+programCli
+  .command('get-antora-value')
+  .description('Read a value from antora.yml by dot-path (e.g., asciidoc.attributes.latest-redpanda-tag)')
+  .argument('<keyPath>', 'Dot-separated path to the value (e.g., asciidoc.attributes.latest-redpanda-tag)')
+  .action((keyPath) => {
+    const value = getAntoraValue(keyPath)
+    if (value === undefined) {
+      process.exit(1)
+    }
+    console.log(value)
+  })
+
+/**
+ * set-antora-value
+ *
+ * @description
+ * Sets a value in antora.yml using a dot-separated key path.
+ * Uses surgical text replacement to preserve formatting and comments.
+ *
+ * @why
+ * Automation workflows need to update version attributes in antora.yml
+ * after generating documentation to track the currently documented version.
+ *
+ * @example
+ * # Update the latest rpk version
+ * npx doc-tools set-antora-value asciidoc.attributes.latest-rpk-version v26.2.0
+ *
+ * # Update the latest Redpanda version
+ * npx doc-tools set-antora-value asciidoc.attributes.latest-redpanda-tag v26.2.0
+ *
+ * @requirements
+ * - Must run from directory containing antora.yml or antora.yaml
+ */
+programCli
+  .command('set-antora-value')
+  .description('Set a value in antora.yml by dot-path (e.g., asciidoc.attributes.latest-rpk-version)')
+  .argument('<keyPath>', 'Dot-separated path to the value (e.g., asciidoc.attributes.latest-rpk-version)')
+  .argument('<value>', 'The value to set')
+  .action((keyPath, value) => {
+    const success = setAntoraValue(keyPath, value)
+    if (!success) {
+      console.error(`Failed to set ${keyPath}`)
+      process.exit(1)
+    }
+    console.log(`Set ${keyPath} = ${value}`)
+  })
+
 /**
  * link-readme
  *
@@ -956,80 +1026,219 @@ automation
  * generate rpk-docs
  *
  * @description
- * Generates comprehensive CLI reference documentation for RPK (Redpanda Keeper), the official
- * Redpanda command-line tool. Starts Redpanda in Docker (RPK is bundled with Redpanda), executes
- * `rpk --help` for all commands and subcommands recursively, parses the help output, and generates
- * structured AsciiDoc documentation for each command with usage, flags, and descriptions.
- * Optionally compares RPK commands between versions to identify new or changed commands.
+ * Generates comprehensive CLI reference documentation for rpk (Redpanda Keeper).
+ * Clones the Redpanda source, builds rpk with Go, and parses `rpk --print-tree` JSON output.
+ * Detects Linux-only commands by analyzing Go build tags in the source code.
+ *
+ * Key features:
+ * - Clones source from GitHub (sparse checkout for speed)
+ * - Builds rpk from source using Go
+ * - Parses Go build tags to detect Linux-only commands
+ * - Automatically includes rpk plugins (connect, ai, check, etc.)
+ * - Supports overrides.json for description improvements
+ * - Generates versioned JSON files for downstream consumers (tooltips, etc.)
+ * - Generates diffs between versions for release notes
  *
  * @why
- * RPK has dozens of commands and subcommands with complex flags and options. The built-in help
- * text is the source of truth for RPK's CLI interface. Manual documentation becomes outdated as
- * RPK evolves. This automation extracts documentation directly from RPK's help output, ensuring
- * accuracy. Running RPK from Docker guarantees the exact version being documented, and diffing
- * between versions automatically highlights CLI changes for release notes.
+ * Building from source provides accurate platform detection by analyzing Go build tags
+ * (//go:build linux) rather than comparing binaries. This is faster and more reliable.
  *
  * @example
- * # Basic: Generate RPK docs for a specific version
- * npx doc-tools generate rpk-docs --tag v25.3.1
+ * # Generate docs for a specific version
+ * npx doc-tools generate rpk-docs --ref v26.2.0
  *
- * # Compare RPK commands between versions
- * npx doc-tools generate rpk-docs \
- *   --tag v25.3.1 \
- *   --diff v25.2.1
+ * # Generate docs for latest development branch
+ * npx doc-tools generate rpk-docs --ref dev
  *
- * # Use custom Docker repository
- * npx doc-tools generate rpk-docs \
- *   --tag v25.3.1 \
- *   --docker-repo docker.redpanda.com/redpandadata/redpanda
+ * # Auto-detect local redpanda checkout (if available)
+ * npx doc-tools generate rpk-docs
  *
- * # Full workflow: document new release
- * VERSION=$(npx doc-tools get-redpanda-version)
- * npx doc-tools generate rpk-docs --tag $VERSION
+ * # Generate with diff against previous version
+ * npx doc-tools generate rpk-docs --ref v26.2.0 --diff v26.1.9
+ *
+ * # Use custom overrides file
+ * npx doc-tools generate rpk-docs --ref dev --overrides custom-overrides.json
  *
  * @requirements
- * - Docker must be installed and running
- * - Sufficient disk space for Docker image
- * - Internet connection to pull Docker images
+ * - Go must be installed (https://go.dev/)
+ * - Git must be installed (for cloning source)
  */
 automation
   .command('rpk-docs')
-  .description('Generate AsciiDoc documentation for rpk CLI commands. Defaults to branch "dev" if neither --tag nor --branch is specified.')
-  .option('-t, --tag <tag>', 'Git tag for released content (GA/beta)')
-  .option('-b, --branch <branch>', 'Branch name for in-progress content')
-  .option('--docker-repo <repo>', 'Docker repository to use', commonOptions.dockerRepo)
-  .option('--console-tag <tag>', 'Redpanda Console version to use', commonOptions.consoleTag)
-  .option('--console-docker-repo <repo>', 'Docker repository for Console', commonOptions.consoleDockerRepo)
-  .option('--diff <oldTag>', 'Also diff autogenerated rpk docs from <oldTag> → <tag>')
-  .action((options) => {
-    verifyMetricsDependencies()
+  .description('Generate rpk CLI documentation from source. Builds rpk and parses source for platform detection.')
+  .option('-r, --ref <ref>', 'Git branch or tag to document (e.g., dev, v26.2.0). Clones from GitHub.')
+  .option('--from-source <path>', 'Path to local rpk source (src/go/rpk directory)')
+  .option('--from-json <path>', 'Regenerate docs from an existing versioned JSON file (skips building)')
+  .option('--overrides <path>', 'Path to overrides JSON file', 'docs-data/rpk-overrides.json')
+  .option('--diff <oldVersion>', 'Generate diff against previous version')
+  .option('--update-whats-new [path]', 'Update what\'s-new file with rpk changes from diff (default: modules/get-started/pages/release-notes/redpanda.adoc)')
+  .option('--draft-missing', 'Generate draft pages for new commands')
+  .option('--output-dir <dir>', 'Output directory for generated AsciiDoc', 'modules/reference/pages/rpk')
+  .option('--cloud-secret-dir <dir>', 'Output directory for rpk cloud and rpk security secret commands (defaults to partials relative to output-dir)')
+  .option('--data-dir <dir>', 'Directory for versioned JSON and diff files', 'docs-data')
+  .option('--preserve-from <path>', 'Path to existing docs to preserve cloud conditionals from')
+  .option('--print-summary', 'Print PR summary (for GitHub Actions)')
+  .option('--summary-file <path>', 'Write PR summary to file (for GitHub Actions)')
+  .option('--show-info', 'Include info-level validation messages')
+  .action(async (options) => {
+    try {
+      const { handleRpkDocsGeneration } = require('../tools/rpk-docs/rpk-docs-handler.js')
+
+      // Handle --update-whats-new with optional path
+      let whatsNewPath = null
+      if (options.updateWhatsNew !== undefined) {
+        // If true (flag without path), use default; if string, use that path
+        whatsNewPath = typeof options.updateWhatsNew === 'string'
+          ? options.updateWhatsNew
+          : 'modules/get-started/pages/release-notes/redpanda.adoc'
+      }
 
-    if (options.tag && options.branch) {
-      console.error('Error: Cannot specify both --tag and --branch')
+      const result = await handleRpkDocsGeneration({
+        ref: options.ref,
+        fromSource: options.fromSource,
+        fromJson: options.fromJson,
+        overrides: options.overrides,
+        diff: options.diff,
+        updateWhatsNew: whatsNewPath,
+        draftMissing: options.draftMissing,
+        outputDir: options.outputDir,
+        cloudSecretDir: options.cloudSecretDir,
+        dataDir: options.dataDir,
+        preserveFrom: options.preserveFrom,
+        printSummary: options.printSummary,
+        showInfo: options.showInfo
+      })
+
+      if (result.success) {
+        console.log('\n✓ rpk documentation generated successfully')
+
+        // Write PR summary to file if requested (useful for GitHub Actions)
+        if (options.summaryFile && result.prSummary) {
+          fs.writeFileSync(options.summaryFile, result.prSummary, 'utf8')
+          console.log(`PR summary written to: ${options.summaryFile}`)
+        }
+
+        // Exit with warning if validation errors found
+        if (result.validationResult?.summary?.totalErrors > 0) {
+          console.error(`\n⚠ Validation found ${result.validationResult.summary.totalErrors} error(s)`)
+          process.exit(1)
+        }
+
+        process.exit(0)
+      } else {
+        console.error('Error: Generation failed')
+        process.exit(1)
+      }
+    } catch (err) {
+      console.error(`Error: ${err.message}`)
       process.exit(1)
     }
+  })
 
-    const newTag = options.tag || options.branch || 'dev'
-    const oldTag = options.diff
+/**
+ * validate rpk-overrides
+ *
+ * @description
+ * Validates the rpk-overrides.json file against the JSON schema and checks for common issues:
+ * - Schema compliance (required fields, valid types)
+ * - Valid $ref references (no broken or circular refs)
+ * - Valid command paths (compared against actual rpk command tree)
+ * - Valid admonition locations (after_flags, after_usage, etc.)
+ * - Valid platform values (linux, darwin, windows)
+ *
+ * @why
+ * Catching override errors early prevents broken documentation. This command lets writers
+ * validate their changes before generation, avoiding cryptic errors during the build process.
+ *
+ * @example
+ * # Validate overrides with default paths
+ * npx doc-tools validate rpk-overrides
+ *
+ * # Validate against a specific rpk tree (for complete command path validation)
+ * npx doc-tools validate rpk-overrides --tree docs-data/rpk-v26.2.0.json
+ *
+ * # Validate a custom overrides file
+ * npx doc-tools validate rpk-overrides --overrides my-overrides.json
+ *
+ * # Strict mode - exit with error code on validation failures
+ * npx doc-tools validate rpk-overrides --strict
+ *
+ * @requirements
+ * - rpk-overrides.schema.json in docs-data/
+ */
+automation
+  .command('rpk-overrides')
+  .description('Validate rpk-overrides.json against schema and check for common issues')
+  .option('--overrides <path>', 'Path to overrides JSON file', 'docs-data/rpk-overrides.json')
+  .option('--tree <path>', 'Path to rpk tree JSON file for command path validation (e.g., docs-data/rpk-v26.2.0.json)')
+  .option('--strict', 'Exit with error code on validation failures')
+  .action((options) => {
+    try {
+      const { loadAndValidateOverrides } = require('../tools/rpk-docs/validate-overrides.js')
+      const repoRoot = findRepoRoot()
+      const overridesPath = path.resolve(repoRoot, options.overrides)
+
+      // Load tree if provided for command path validation
+      let commandTree = null
+      if (options.tree) {
+        const treePath = path.resolve(repoRoot, options.tree)
+        if (!fs.existsSync(treePath)) {
+          console.error(`Error: Tree file not found: ${treePath}`)
+          process.exit(1)
+        }
+        const treeData = JSON.parse(fs.readFileSync(treePath, 'utf8'))
+        commandTree = treeData.tree || treeData
+      }
 
-    if (oldTag) {
-      const oldDir = path.join('autogenerated', oldTag, 'rpk')
-      if (!fs.existsSync(oldDir)) {
-        console.log(`Generating rpk docs for old tag ${oldTag}…`)
-        runClusterDocs('rpk', oldTag, options)
+      console.log(`Validating: ${overridesPath}`)
+      if (commandTree) {
+        console.log(`Comparing against tree from: ${options.tree}`)
+      } else {
+        console.log('Note: Skipping command path validation (no --tree provided)')
       }
-    }
+      console.log('')
 
-    console.log(`Generating rpk docs for new tag ${newTag}…`)
-    runClusterDocs('rpk', newTag, options)
+      const { overrides, validation } = loadAndValidateOverrides(overridesPath, commandTree)
 
-    if (oldTag) {
-      diffDirs('rpk', oldTag, newTag)
-    }
+      if (!overrides) {
+        console.error('Error: Could not load overrides file')
+        process.exit(1)
+      }
 
-    process.exit(0)
+      // Print results
+      console.log('=' .repeat(60))
+      console.log('VALIDATION RESULTS')
+      console.log('='.repeat(60))
+
+      if (validation.errors.length === 0 && validation.warnings.length === 0) {
+        console.log('✓ No issues found')
+      } else {
+        console.log(validation.format())
+      }
+
+      console.log('='.repeat(60))
+      console.log(`Errors: ${validation.errors.length}`)
+      console.log(`Warnings: ${validation.warnings.length}`)
+      console.log('='.repeat(60))
+
+      // Exit with appropriate code
+      if (options.strict && !validation.valid) {
+        console.log('\n✗ Validation failed (strict mode)')
+        process.exit(1)
+      } else if (validation.valid) {
+        console.log('\n✓ Validation passed')
+        process.exit(0)
+      } else {
+        console.log('\n⚠ Validation completed with errors (use --strict to fail)')
+        process.exit(0)
+      }
+    } catch (err) {
+      console.error(`Error: ${err.message}`)
+      process.exit(1)
+    }
   })
 
+
 /**
  * generate helm-spec
  *

package/bin/mcp-tools/rpk-docs.js

@@ -1,5 +1,15 @@
 /**
- * MCP Tools - RPK Documentation Generation
+ * MCP Tools - rpk Documentation Generation
+ *
+ * Generates rpk CLI reference documentation by building from Go source
+ * and parsing build tags to detect Linux-only commands.
+ *
+ * Features:
+ * - Builds rpk from source (requires Go)
+ * - Detects Linux-only commands from Go build tags
+ * - Supports overrides.json for description improvements
+ * - Generates diffs between versions for release notes
+ * - Publishes versioned JSON for downstream consumers
  *
  * OPTIMIZATION: This tool calls CLI, doesn't use LLM directly.
  * - No model recommendation (CLI tool)
@@ -12,14 +22,22 @@ const { getAntoraStructure } = require('./antora');
 const { createJob } = require('./job-queue');
 
 /**
- * Generate RPK command documentation
+ * Generate rpk command documentation from source
  *
- * Use tags for released content (GA or beta), branches for in-progress content.
- * Defaults to branch "dev" if neither tag nor branch is provided.
+ * Clones redpanda source (or uses local checkout) and builds rpk to generate
+ * documentation. Parses Go build tags to detect Linux-only commands.
  *
  * @param {Object} args - Arguments
- * @param {string} [args.tag] - Git tag for released content (for example, "v25.3.1")
- * @param {string} [args.branch] - Branch name for in-progress content (for example, "dev", "main")
+ * @param {string} [args.ref] - Git ref to document (e.g., "dev", "v26.2.0", "main")
+ * @param {string} [args.tag] - Git tag (alias for ref, backward compatibility)
+ * @param {string} [args.branch] - Git branch (alias for ref, backward compatibility)
+ * @param {string} [args.from_source] - Path to local rpk source directory
+ * @param {string} [args.overrides] - Path to overrides JSON file (defaults to docs-data/rpk-overrides.json)
+ * @param {string} [args.diff] - Generate diff against this previous version
+ * @param {boolean} [args.update_whats_new] - Update whats-new.adoc with changes
+ * @param {boolean} [args.draft_missing] - Generate draft pages for new commands
+ * @param {string} [args.output_dir] - Output directory for generated AsciiDoc
+ * @param {string} [args.data_dir] - Directory for versioned JSON and diff files
  * @param {boolean} [args.background] - Run as background job
  * @returns {Object} Generation results
  */
@@ -35,23 +53,23 @@ function generateRpkDocs(args) {
     };
   }
 
-  // Validate that tag and branch are mutually exclusive
-  if (args.tag && args.branch) {
-    return {
-      success: false,
-      error: 'Cannot specify both tag and branch',
-      suggestion: 'Use either --tag or --branch, not both'
-    };
-  }
-
-  // Default to 'dev' branch if neither provided
-  const gitRef = args.tag || args.branch || 'dev';
-  const refType = args.tag ? 'tag' : 'branch';
-
-  // Normalize version (add 'v' prefix if not present and not a branch name like 'dev' or 'main')
-  let version = gitRef;
-  if (args.tag && !version.startsWith('v')) {
-    version = `v${version}`;
+  // Determine version from ref, tag, or branch (backward compatibility)
+  // Priority: ref > tag > branch > 'dev' (default)
+  let version;
+  let refType;
+  if (args.ref) {
+    version = args.ref;
+    refType = args.ref.match(/^v?\d/) ? 'tag' : 'branch';
+  } else if (args.tag) {
+    version = args.tag.startsWith('v') ? args.tag : `v${args.tag}`;
+    refType = 'tag';
+  } else if (args.branch) {
+    version = args.branch;
+    refType = 'branch';
+  } else {
+    // No ref specified - default to 'dev'
+    version = 'dev';
+    refType = 'branch';
   }
 
   // Get doc-tools command (handles both local and installed)
@@ -60,12 +78,40 @@ function generateRpkDocs(args) {
   // Build command arguments array
   const baseArgs = ['generate', 'rpk-docs'];
 
-  if (args.tag) {
-    baseArgs.push('--tag');
-    baseArgs.push(version);
-  } else {
-    baseArgs.push('--branch');
-    baseArgs.push(version);
+  // Always use --ref (defaults to 'dev' if not specified)
+  baseArgs.push('--ref', version);
+
+  // Add optional arguments
+  if (args.from_source) {
+    baseArgs.push('--from-source', args.from_source);
+  }
+
+  if (args.overrides) {
+    baseArgs.push('--overrides', args.overrides);
+  }
+
+  if (args.diff) {
+    let diffVersion = args.diff;
+    if (!diffVersion.startsWith('v')) {
+      diffVersion = `v${diffVersion}`;
+    }
+    baseArgs.push('--diff', diffVersion);
+  }
+
+  if (args.update_whats_new) {
+    baseArgs.push('--update-whats-new');
+  }
+
+  if (args.draft_missing) {
+    baseArgs.push('--draft-missing');
+  }
+
+  if (args.output_dir) {
+    baseArgs.push('--output-dir', args.output_dir);
+  }
+
+  if (args.data_dir) {
+    baseArgs.push('--data-dir', args.data_dir);
   }
 
   // If background mode, create job and return immediately
@@ -79,8 +125,10 @@ function generateRpkDocs(args) {
       success: true,
       background: true,
       job_id: jobId,
-      message: `RPK docs generation started in background. Use get_job_status with job_id: ${jobId} to check progress.`,
-      [refType]: version
+      message: `rpk docs generation started in background. Use get_job_status with job_id: ${jobId} to check progress.`,
+      ref: version,
+      version: version,
+      ref_type: refType
     };
   }
 
@@ -115,24 +163,46 @@ function generateRpkDocs(args) {
 
     const output = result.stdout;
     const commandCountMatch = output.match(/(\d+) commands/i);
+    const filesGeneratedMatch = output.match(/Generated (\d+) files/i);
 
-    return {
+    const response = {
       success: true,
-      [refType]: version,
-      files_generated: [`autogenerated/${version}/rpk/*.adoc`],
+      ref: version,
+      version: version,
+      ref_type: refType,
       commands_documented: commandCountMatch ? parseInt(commandCountMatch[1]) : null,
-      output: output.trim(),
-      summary: `Generated RPK documentation for Redpanda ${refType} ${version}`
+      files_generated: filesGeneratedMatch ? parseInt(filesGeneratedMatch[1]) : null,
+      output: output.trim()
     };
+
+    // Add diff summary if available
+    if (args.diff) {
+      response.diff_from = args.diff.startsWith('v') ? args.diff : `v${args.diff}`;
+      response.diff_to = version;
+    }
+
+    response.summary = `Generated rpk documentation for ${version}`;
+
+    return response;
   } catch (err) {
-    return {
+    const response = {
       success: false,
       error: err.message,
       stdout: err.stdout || '',
       stderr: err.stderr || '',
-      exitCode: err.status,
-      suggestion: 'Check that the version exists in the Redpanda repository'
+      exitCode: err.status
     };
+
+    // Provide helpful suggestions based on error
+    if (err.message.includes('Go is required')) {
+      response.suggestion = 'Install Go from https://go.dev/ and ensure it\'s in your PATH.';
+    } else if (err.message.includes('not found') || err.message.includes('ENOENT')) {
+      response.suggestion = 'Ensure Go and Git are installed and in your PATH.';
+    } else {
+      response.suggestion = 'Check that the version/ref exists in the Redpanda repository.';
+    }
+
+    return response;
   }
 }