@redpanda-data/docs-extensions-and-macros 4.12.6 → 4.13.1

package/bin/mcp-tools/prompt-discovery.js

@@ -0,0 +1,283 @@
+/**
+ * Prompt Discovery and Caching
+ *
+ * Automatically discovers prompts from the mcp/prompts directory.
+ * Caches prompt content and metadata for performance.
+ * Supports file watching in development mode.
+ */
+
+const fs = require('fs');
+const path = require('path');
+const { parsePromptFile, validateFrontmatter } = require('./frontmatter');
+const { validatePromptName } = require('./mcp-validation');
+
+/**
+ * In-memory prompt cache
+ */
+class PromptCache {
+  constructor() {
+    this.prompts = new Map(); // name -> { metadata, content, filePath }
+    this.watchers = [];
+  }
+
+  /**
+   * Set prompt in cache
+   * @param {string} name - Prompt name
+   * @param {Object} data - Prompt data
+   */
+  set(name, data) {
+    this.prompts.set(name, data);
+  }
+
+  /**
+   * Get prompt from cache
+   * @param {string} name - Prompt name
+   * @returns {Object|null} Prompt data or null
+   */
+  get(name) {
+    return this.prompts.get(name) || null;
+  }
+
+  /**
+   * Get all prompts
+   * @returns {Array} Array of prompt objects
+   */
+  getAll() {
+    return Array.from(this.prompts.values());
+  }
+
+  /**
+   * Get all prompt names
+   * @returns {Array} Array of prompt names
+   */
+  getNames() {
+    return Array.from(this.prompts.keys());
+  }
+
+  /**
+   * Check if prompt exists
+   * @param {string} name - Prompt name
+   * @returns {boolean}
+   */
+  has(name) {
+    return this.prompts.has(name);
+  }
+
+  /**
+   * Clear all prompts
+   */
+  clear() {
+    this.prompts.clear();
+  }
+
+  /**
+   * Stop all file watchers
+   */
+  stopWatching() {
+    this.watchers.forEach(watcher => watcher.close());
+    this.watchers = [];
+  }
+
+  /**
+   * Add a file watcher
+   * @param {FSWatcher} watcher - File system watcher
+   */
+  addWatcher(watcher) {
+    this.watchers.push(watcher);
+  }
+}
+
+/**
+ * Default argument formatters for different types
+ */
+const argumentFormatters = {
+  'content-append': (args, schema) => {
+    if (!args) return '';
+
+    let result = '\n\n---\n\n';
+
+    // If there's a 'content' argument, add it specially
+    if (args.content) {
+      result += `**Content to review:**\n\n${args.content}`;
+      return result;
+    }
+
+    // Otherwise, format all arguments
+    Object.entries(args).forEach(([key, value]) => {
+      const argDef = schema?.find(a => a.name === key);
+      const label = argDef?.name || key;
+      result += `**${label.charAt(0).toUpperCase() + label.slice(1)}:** ${value}\n`;
+    });
+
+    return result;
+  },
+
+  'structured': (args, schema) => {
+    if (!args) return '';
+
+    let result = '\n\n---\n\n';
+    Object.entries(args).forEach(([key, value]) => {
+      result += `**${key}:** ${value}\n`;
+    });
+
+    return result;
+  }
+};
+
+/**
+ * Discover all prompts in the prompts directory
+ * @param {string} promptsDir - Path to prompts directory
+ * @returns {Array} Array of discovered prompts
+ */
+function discoverPrompts(promptsDir) {
+  if (!fs.existsSync(promptsDir)) {
+    throw new Error(`Prompts directory not found: ${promptsDir}`);
+  }
+
+  const files = fs.readdirSync(promptsDir).filter(f => f.endsWith('.md'));
+  const prompts = [];
+
+  for (const file of files) {
+    try {
+      const name = path.basename(file, '.md');
+      validatePromptName(name); // Ensure safe name
+
+      const filePath = path.join(promptsDir, file);
+      const fileContent = fs.readFileSync(filePath, 'utf8');
+
+      const { metadata, content } = parsePromptFile(fileContent, file);
+
+      // Build prompt object
+      const prompt = {
+        name,
+        description: metadata.description || `Prompt: ${name}`,
+        version: metadata.version || '1.0.0',
+        arguments: metadata.arguments || [],
+        argumentFormat: metadata.argumentFormat || 'content-append',
+        content,
+        filePath,
+        _rawMetadata: metadata
+      };
+
+      prompts.push(prompt);
+    } catch (err) {
+      console.error(`Error loading prompt ${file}: ${err.message}`);
+      // Continue loading other prompts
+    }
+  }
+
+  return prompts;
+}
+
+/**
+ * Load all prompts into cache
+ * @param {string} baseDir - Base directory (repo root)
+ * @param {PromptCache} cache - Prompt cache instance
+ * @returns {Array} Loaded prompts
+ */
+function loadAllPrompts(baseDir, cache) {
+  const promptsDir = path.join(baseDir, 'mcp', 'prompts');
+  const prompts = discoverPrompts(promptsDir);
+
+  // Clear and reload cache
+  cache.clear();
+
+  prompts.forEach(prompt => {
+    cache.set(prompt.name, prompt);
+  });
+
+  return prompts;
+}
+
+/**
+ * Watch prompts directory for changes (development mode)
+ * @param {string} baseDir - Base directory
+ * @param {PromptCache} cache - Prompt cache instance
+ * @param {Function} onChange - Callback when prompts change
+ */
+function watchPrompts(baseDir, cache, onChange) {
+  const promptsDir = path.join(baseDir, 'mcp', 'prompts');
+
+  if (!fs.existsSync(promptsDir)) {
+    console.error(`Cannot watch prompts directory (not found): ${promptsDir}`);
+    return;
+  }
+
+  const watcher = fs.watch(promptsDir, (eventType, filename) => {
+    if (!filename || !filename.endsWith('.md')) {
+      return;
+    }
+
+    console.error(`Prompt file changed: ${filename} (${eventType})`);
+    console.error('Reloading all prompts...');
+
+    try {
+      const prompts = loadAllPrompts(baseDir, cache);
+      console.error(`Reloaded ${prompts.length} prompts`);
+
+      if (onChange) {
+        onChange(prompts);
+      }
+    } catch (err) {
+      console.error(`Error reloading prompts: ${err.message}`);
+    }
+  });
+
+  cache.addWatcher(watcher);
+  console.error('File watching enabled for prompts (dev mode)');
+}
+
+/**
+ * Build prompt text with arguments
+ * @param {Object} prompt - Prompt object from cache
+ * @param {Object} args - Arguments provided by user
+ * @returns {string} Complete prompt text
+ */
+function buildPromptWithArguments(prompt, args) {
+  let promptText = prompt.content;
+
+  if (!args || Object.keys(args).length === 0) {
+    return promptText;
+  }
+
+  // Use the formatter specified by the prompt
+  const formatter = argumentFormatters[prompt.argumentFormat];
+  if (!formatter) {
+    console.error(
+      `Unknown argument format: ${prompt.argumentFormat} for prompt ${prompt.name}`
+    );
+    return promptText;
+  }
+
+  const formattedArgs = formatter(args, prompt.arguments);
+  promptText += formattedArgs;
+
+  return promptText;
+}
+
+/**
+ * Convert prompts to MCP protocol format
+ * @param {Array} prompts - Discovered prompts
+ * @returns {Array} Prompts in MCP format
+ */
+function promptsToMcpFormat(prompts) {
+  return prompts.map(prompt => ({
+    name: prompt.name,
+    description: prompt.description,
+    arguments: prompt.arguments.map(arg => ({
+      name: arg.name,
+      description: arg.description,
+      required: arg.required
+    }))
+  }));
+}
+
+module.exports = {
+  PromptCache,
+  discoverPrompts,
+  loadAllPrompts,
+  watchPrompts,
+  buildPromptWithArguments,
+  promptsToMcpFormat,
+  argumentFormatters
+};

package/bin/mcp-tools/property-docs.js

@@ -0,0 +1,157 @@
+/**
+ * MCP Tools - Property Documentation Generation
+ *
+ * OPTIMIZATION: This tool calls CLI, doesn't use LLM directly.
+ * - No model recommendation (CLI tool)
+ * - Cost comes from doc-tools CLI execution
+ */
+
+const { spawnSync } = require('child_process');
+const { findRepoRoot, getDocToolsCommand, MAX_EXEC_BUFFER_SIZE, DEFAULT_COMMAND_TIMEOUT } = require('./utils');
+const { getAntoraStructure } = require('./antora');
+const { createJob } = require('./job-queue');
+
+/**
+ * Generate Redpanda property documentation
+ *
+ * Use tags for released content (GA or beta), branches for in-progress content.
+ * Defaults to branch "dev" if neither tag nor branch is provided.
+ *
+ * @param {Object} args - Arguments
+ * @param {string} [args.tag] - Git tag for released content (for example, "v25.3.1")
+ * @param {string} [args.branch] - Branch name for in-progress content (for example, "dev", "main")
+ * @param {boolean} [args.generate_partials] - Whether to generate AsciiDoc partials
+ * @param {boolean} [args.background] - Run as background job
+ * @returns {Object} Generation results
+ */
+function generatePropertyDocs(args) {
+  const repoRoot = findRepoRoot();
+  const structure = getAntoraStructure(repoRoot);
+
+  if (!structure.hasDocTools) {
+    return {
+      success: false,
+      error: 'doc-tools not found in this repository',
+      suggestion: 'Navigate to the docs-extensions-and-macros repository'
+    };
+  }
+
+  // Validate that tag and branch are mutually exclusive
+  if (args.tag && args.branch) {
+    return {
+      success: false,
+      error: 'Cannot specify both tag and branch',
+      suggestion: 'Use either --tag or --branch, not both'
+    };
+  }
+
+  // Default to 'dev' branch if neither provided
+  const gitRef = args.tag || args.branch || 'dev';
+  const refType = args.tag ? 'tag' : 'branch';
+
+  // Normalize version (add 'v' prefix if tag and not present and not "latest")
+  let version = gitRef;
+  if (args.tag && version !== 'latest' && !version.startsWith('v')) {
+    version = `v${version}`;
+  }
+
+  // Get doc-tools command (handles both local and installed)
+  const docTools = getDocToolsCommand(repoRoot);
+
+  // Build command arguments
+  const baseArgs = ['generate', 'property-docs'];
+
+  if (args.tag) {
+    baseArgs.push('--tag');
+    baseArgs.push(version);
+  } else {
+    baseArgs.push('--branch');
+    baseArgs.push(version);
+  }
+
+  if (args.generate_partials) {
+    baseArgs.push('--generate-partials');
+  }
+
+  // If background mode, create job and return immediately
+  if (args.background) {
+    const cmdArgs = [docTools.program, ...docTools.getArgs(baseArgs)];
+    const jobId = createJob('generate_property_docs', cmdArgs, {
+      cwd: repoRoot.root
+    });
+
+    return {
+      success: true,
+      background: true,
+      job_id: jobId,
+      message: `Property docs generation started in background. Use get_job_status with job_id: ${jobId} to check progress.`,
+      [refType]: version
+    };
+  }
+
+  // Otherwise run synchronously
+  try {
+    const result = spawnSync(docTools.program, docTools.getArgs(baseArgs), {
+      cwd: repoRoot.root,
+      encoding: 'utf8',
+      stdio: 'pipe',
+      maxBuffer: MAX_EXEC_BUFFER_SIZE,
+      timeout: DEFAULT_COMMAND_TIMEOUT
+    });
+
+    // Check for spawn errors
+    if (result.error) {
+      const err = new Error(`Failed to execute command: ${result.error.message}`);
+      err.stdout = result.stdout || '';
+      err.stderr = result.stderr || '';
+      err.status = result.status;
+      throw err;
+    }
+
+    // Check for non-zero exit codes
+    if (result.status !== 0) {
+      const errorMsg = result.stderr || `Command failed with exit code ${result.status}`;
+      const err = new Error(errorMsg);
+      err.stdout = result.stdout || '';
+      err.stderr = result.stderr || '';
+      err.status = result.status;
+      throw err;
+    }
+
+    const output = result.stdout;
+
+    // Parse output to extract useful information
+    const propertyCountMatch = output.match(/(\d+) properties/i);
+    const filesGenerated = [];
+
+    if (args.generate_partials) {
+      filesGenerated.push('modules/reference/partials/cluster-properties.adoc');
+      filesGenerated.push('modules/reference/partials/broker-properties.adoc');
+      filesGenerated.push('modules/reference/partials/tunable-properties.adoc');
+      filesGenerated.push('modules/reference/partials/topic-properties.adoc');
+    }
+    filesGenerated.push('modules/reference/partials/properties.json');
+
+    return {
+      success: true,
+      [refType]: version,
+      files_generated: filesGenerated,
+      property_count: propertyCountMatch ? parseInt(propertyCountMatch[1]) : null,
+      output: output.trim(),
+      summary: `Generated property documentation for Redpanda ${refType} ${version}`
+    };
+  } catch (err) {
+    return {
+      success: false,
+      error: err.message,
+      stdout: err.stdout || '',
+      stderr: err.stderr || '',
+      exitCode: err.status,
+      suggestion: 'Check that the version exists in the Redpanda repository'
+    };
+  }
+}
+
+module.exports = {
+  generatePropertyDocs
+};

package/bin/mcp-tools/rpcn-docs.js

@@ -0,0 +1,113 @@
+/**
+ * MCP Tools - Redpanda Connect Documentation Generation
+ *
+ * OPTIMIZATION: This tool calls CLI, doesn't use LLM directly.
+ * - Caches connector metadata when possible
+ * - Use --data-dir to avoid re-fetching
+ * - Consider incremental generation for updates
+ * - No model recommendation (CLI tool)
+ */
+
+const { execSync, spawnSync } = require('child_process');
+const { findRepoRoot, MAX_EXEC_BUFFER_SIZE, DEFAULT_COMMAND_TIMEOUT } = require('./utils');
+const { getAntoraStructure } = require('./antora');
+const cache = require('./cache');
+
+/**
+ * Generate Redpanda Connect connector documentation
+ * @param {Object} args - Arguments
+ * @returns {Object} Generation results
+ */
+function generateRpConnectDocs(args = {}) {
+  const repoRoot = findRepoRoot();
+  const structure = getAntoraStructure(repoRoot);
+
+  if (!structure.hasDocTools) {
+    return {
+      success: false,
+      error: 'doc-tools not found in this repository',
+      suggestion: 'Navigate to the docs-extensions-and-macros repository'
+    };
+  }
+
+  try {
+    // Build command arguments array (no shell interpolation)
+    const cmdArgs = ['doc-tools', 'generate', 'rpcn-connector-docs'];
+    
+    // Add flags only when present, each as separate array entries
+    if (args.fetch_connectors) cmdArgs.push('--fetch-connectors');
+    if (args.draft_missing) cmdArgs.push('--draft-missing');
+    if (args.update_whats_new) cmdArgs.push('--update-whats-new');
+    if (args.include_bloblang) cmdArgs.push('--include-bloblang');
+    if (args.data_dir) {
+      cmdArgs.push('--data-dir');
+      cmdArgs.push(args.data_dir);
+    }
+    if (args.old_data) {
+      cmdArgs.push('--old-data');
+      cmdArgs.push(args.old_data);
+    }
+    if (args.csv) {
+      cmdArgs.push('--csv');
+      cmdArgs.push(args.csv);
+    }
+    if (args.overrides) {
+      cmdArgs.push('--overrides');
+      cmdArgs.push(args.overrides);
+    }
+
+    const result = spawnSync('npx', cmdArgs, {
+      cwd: repoRoot.root,
+      encoding: 'utf8',
+      stdio: 'pipe',
+      maxBuffer: MAX_EXEC_BUFFER_SIZE,
+      timeout: DEFAULT_COMMAND_TIMEOUT
+    });
+
+    // Check for spawn errors
+    if (result.error) {
+      const err = new Error(`Failed to execute command: ${result.error.message}`);
+      err.stdout = result.stdout || '';
+      err.stderr = result.stderr || '';
+      err.status = result.status;
+      throw err;
+    }
+
+    // Check for non-zero exit codes
+    if (result.status !== 0) {
+      const errorMsg = result.stderr || `Command failed with exit code ${result.status}`;
+      const err = new Error(errorMsg);
+      err.stdout = result.stdout || '';
+      err.stderr = result.stderr || '';
+      err.status = result.status;
+      throw err;
+    }
+
+    const output = result.stdout;
+
+    const connectorsMatch = output.match(/(\d+) connectors/i);
+
+    return {
+      success: true,
+      connectors_documented: connectorsMatch ? parseInt(connectorsMatch[1]) : null,
+      files_generated: ['modules/reference/pages/redpanda-connect/components/'],
+      output: output.trim(),
+      summary: 'Generated Redpanda Connect connector documentation',
+      // Cost optimization metadata
+      _optimizationNotes: 'This tool calls CLI. To reduce costs: 1) Cache connector data with --data-dir, 2) Use incremental generation when possible, 3) Avoid --fetch-connectors unless needed'
+    };
+  } catch (err) {
+    return {
+      success: false,
+      error: err.message,
+      stdout: err.stdout || '',
+      stderr: err.stderr || '',
+      exitCode: err.status,
+      suggestion: 'Check that you have network access to fetch connector data if using --fetch-connectors'
+    };
+  }
+}
+
+module.exports = {
+  generateRpConnectDocs
+};

package/bin/mcp-tools/rpk-docs.js

@@ -0,0 +1,141 @@
+/**
+ * MCP Tools - RPK Documentation Generation
+ *
+ * OPTIMIZATION: This tool calls CLI, doesn't use LLM directly.
+ * - No model recommendation (CLI tool)
+ * - Cost comes from doc-tools CLI execution
+ */
+
+const { spawnSync } = require('child_process');
+const { findRepoRoot, getDocToolsCommand, MAX_EXEC_BUFFER_SIZE, DEFAULT_COMMAND_TIMEOUT } = require('./utils');
+const { getAntoraStructure } = require('./antora');
+const { createJob } = require('./job-queue');
+
+/**
+ * Generate RPK command documentation
+ *
+ * Use tags for released content (GA or beta), branches for in-progress content.
+ * Defaults to branch "dev" if neither tag nor branch is provided.
+ *
+ * @param {Object} args - Arguments
+ * @param {string} [args.tag] - Git tag for released content (for example, "v25.3.1")
+ * @param {string} [args.branch] - Branch name for in-progress content (for example, "dev", "main")
+ * @param {boolean} [args.background] - Run as background job
+ * @returns {Object} Generation results
+ */
+function generateRpkDocs(args) {
+  const repoRoot = findRepoRoot();
+  const structure = getAntoraStructure(repoRoot);
+
+  if (!structure.hasDocTools) {
+    return {
+      success: false,
+      error: 'doc-tools not found in this repository',
+      suggestion: 'Navigate to the docs-extensions-and-macros repository'
+    };
+  }
+
+  // Validate that tag and branch are mutually exclusive
+  if (args.tag && args.branch) {
+    return {
+      success: false,
+      error: 'Cannot specify both tag and branch',
+      suggestion: 'Use either --tag or --branch, not both'
+    };
+  }
+
+  // Default to 'dev' branch if neither provided
+  const gitRef = args.tag || args.branch || 'dev';
+  const refType = args.tag ? 'tag' : 'branch';
+
+  // Normalize version (add 'v' prefix if not present and not a branch name like 'dev' or 'main')
+  let version = gitRef;
+  if (args.tag && !version.startsWith('v')) {
+    version = `v${version}`;
+  }
+
+  // Get doc-tools command (handles both local and installed)
+  const docTools = getDocToolsCommand(repoRoot);
+
+  // Build command arguments array
+  const baseArgs = ['generate', 'rpk-docs'];
+
+  if (args.tag) {
+    baseArgs.push('--tag');
+    baseArgs.push(version);
+  } else {
+    baseArgs.push('--branch');
+    baseArgs.push(version);
+  }
+
+  // If background mode, create job and return immediately
+  if (args.background) {
+    const cmdArgs = [docTools.program, ...docTools.getArgs(baseArgs)];
+    const jobId = createJob('generate_rpk_docs', cmdArgs, {
+      cwd: repoRoot.root
+    });
+
+    return {
+      success: true,
+      background: true,
+      job_id: jobId,
+      message: `RPK docs generation started in background. Use get_job_status with job_id: ${jobId} to check progress.`,
+      [refType]: version
+    };
+  }
+
+  // Otherwise run synchronously
+  try {
+    const result = spawnSync(docTools.program, docTools.getArgs(baseArgs), {
+      cwd: repoRoot.root,
+      encoding: 'utf8',
+      stdio: 'pipe',
+      maxBuffer: MAX_EXEC_BUFFER_SIZE,
+      timeout: DEFAULT_COMMAND_TIMEOUT
+    });
+
+    // Check for spawn errors
+    if (result.error) {
+      const err = new Error(`Failed to execute command: ${result.error.message}`);
+      err.stdout = result.stdout || '';
+      err.stderr = result.stderr || '';
+      err.status = result.status;
+      throw err;
+    }
+
+    // Check for non-zero exit codes
+    if (result.status !== 0) {
+      const errorMsg = result.stderr || `Command failed with exit code ${result.status}`;
+      const err = new Error(errorMsg);
+      err.stdout = result.stdout || '';
+      err.stderr = result.stderr || '';
+      err.status = result.status;
+      throw err;
+    }
+
+    const output = result.stdout;
+    const commandCountMatch = output.match(/(\d+) commands/i);
+
+    return {
+      success: true,
+      [refType]: version,
+      files_generated: [`autogenerated/${version}/rpk/*.adoc`],
+      commands_documented: commandCountMatch ? parseInt(commandCountMatch[1]) : null,
+      output: output.trim(),
+      summary: `Generated RPK documentation for Redpanda ${refType} ${version}`
+    };
+  } catch (err) {
+    return {
+      success: false,
+      error: err.message,
+      stdout: err.stdout || '',
+      stderr: err.stderr || '',
+      exitCode: err.status,
+      suggestion: 'Check that the version exists in the Redpanda repository'
+    };
+  }
+}
+
+module.exports = {
+  generateRpkDocs
+};