@redpanda-data/docs-extensions-and-macros 4.13.5 → 4.14.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@redpanda-data/docs-extensions-and-macros",
-  "version": "4.13.5",
+  "version": "4.14.0",
   "description": "Antora extensions and macros developed for Redpanda documentation.",
   "keywords": [
     "antora",

package/tools/redpanda-connect/rpcn-connector-docs-handler.js

@@ -1172,12 +1172,40 @@ async function handleRpcnConnectorDocs (options) {
       const missingFromCloudDocs = []
       const cloudDocsErrors = []
       if (cloudSupportedSet.size > 0 && options.checkCloudDocs !== false) {
-        console.log('\n   ℹ️  Checking cloud-docs repository for missing connector pages...')
+        console.log('\n   INFO: Checking cloud-docs repository for missing connector pages...')
 
         // Use shared Octokit instance
         const octokit = require('../../cli-utils/octokit-client')
 
         try {
+          // Optimization: Fetch entire directory tree in 1 API call instead of 471 individual calls
+          console.log('   Fetching cloud-docs directory tree (1 API call)...')
+
+          let existingFiles = new Set()
+
+          try {
+            // Get the tree for the components directory
+            const { data: tree } = await octokit.git.getTree({
+              owner: 'redpanda-data',
+              repo: 'cloud-docs',
+              tree_sha: 'main:modules/develop/pages/connect/components',
+              recursive: true
+            })
+
+            // Build a set of existing file paths for O(1) lookup
+            tree.tree.forEach(item => {
+              if (item.type === 'blob' && item.path.endsWith('.adoc')) {
+                existingFiles.add(item.path)
+              }
+            })
+
+            console.log(`   Loaded ${existingFiles.size} existing connector pages from cloud-docs`)
+          } catch (treeError) {
+            console.log(`   WARNING: Could not fetch tree (${treeError.status}), falling back to individual checks`)
+            // If tree API fails, fall back to individual checks (old behavior)
+            existingFiles = null
+          }
+
           // Check each cloud-supported connector
           // Filter to only check actual connector/component types that need individual pages
           const connectorTypes = ['inputs', 'outputs', 'processors', 'caches', 'buffers', 'scanners', 'metrics', 'tracers']
@@ -1198,95 +1226,67 @@ async function handleRpcnConnectorDocs (options) {
               }
             }
 
-            const cloudDocsPath = `modules/develop/pages/connect/components/${type}/${name}.adoc`
+            const relativePath = `${type}/${name}.adoc`
+            const fullPath = `modules/develop/pages/connect/components/${relativePath}`
+
+            // Fast path: Check against tree if we have it
+            if (existingFiles !== null) {
+              if (!existingFiles.has(relativePath)) {
+                missingFromCloudDocs.push({ type, name, path: fullPath })
+              }
+              continue
+            }
 
+            // Fallback path: Individual API calls (only if tree fetch failed)
             try {
               await octokit.repos.getContent({
                 owner: 'redpanda-data',
                 repo: 'cloud-docs',
-                path: cloudDocsPath,
+                path: fullPath,
                 ref: 'main'
               })
               // File exists, no action needed
             } catch (error) {
               if (error.status === 404) {
                 // File doesn't exist in cloud-docs
-                missingFromCloudDocs.push({ type, name, path: cloudDocsPath })
+                missingFromCloudDocs.push({ type, name, path: fullPath })
               } else {
-                // Non-404 error (auth, rate-limit, network, etc.)
-                // Try fallback: check raw URL without authentication
-                const rawUrl = `https://raw.githubusercontent.com/redpanda-data/cloud-docs/main/${cloudDocsPath}`
-                try {
-                  const https = require('https')
-                  const { URL } = require('url')
-                  const parsedUrl = new URL(rawUrl)
-
-                  await new Promise((resolve, reject) => {
-                    const req = https.request({
-                      hostname: parsedUrl.hostname,
-                      path: parsedUrl.pathname,
-                      method: 'HEAD',
-                      timeout: 5000
-                    }, (res) => {
-                      if (res.statusCode === 200) {
-                        resolve() // File exists
-                      } else if (res.statusCode === 404) {
-                        reject(new Error('404'))
-                      } else {
-                        reject(new Error(`Status ${res.statusCode}`))
-                      }
-                    })
-                    req.on('error', reject)
-                    req.on('timeout', () => {
-                      req.destroy()
-                      reject(new Error('Timeout'))
-                    })
-                    req.end()
-                  })
-                  // Fallback succeeded - file exists, no action needed
-                } catch (fallbackError) {
-                  if (fallbackError.message === '404') {
-                    // Confirmed missing via fallback
-                    missingFromCloudDocs.push({ type, name, path: cloudDocsPath })
-                  } else {
-                    // Both API and fallback failed
-                    cloudDocsErrors.push({
-                      type,
-                      name,
-                      path: cloudDocsPath,
-                      status: error.status || 'unknown',
-                      message: `API: ${error.message}; Fallback: ${fallbackError.message}`
-                    })
-                  }
-                }
+                // Non-404 error - record as error
+                cloudDocsErrors.push({
+                  type,
+                  name,
+                  path: fullPath,
+                  status: error.status || 'unknown',
+                  message: error.message
+                })
               }
             }
           }
 
           // Report results
           if (cloudDocsErrors.length > 0) {
-            console.log(`   ⚠️  Encountered ${cloudDocsErrors.length} error(s) while checking cloud-docs (check inconclusive):`)
+            console.log(`   WARNING: Encountered ${cloudDocsErrors.length} error(s) while checking cloud-docs (check inconclusive):`)
             cloudDocsErrors.forEach(({ type, name, status, message }) => {
-              console.log(`      • ${type}/${name} - Status ${status}: ${message}`)
+              console.log(`      - ${type}/${name} - Status ${status}: ${message}`)
             })
-            console.log(`   ℹ️  Please resolve these errors (e.g., check GITHUB_TOKEN or VBOT_GITHUB_API_TOKEN, API rate limits, network connectivity)`)
+            console.log(`   INFO: Please resolve these errors (e.g., check GITHUB_TOKEN or VBOT_GITHUB_API_TOKEN, API rate limits, network connectivity)`)
             if (missingFromCloudDocs.length > 0) {
-              console.log(`   ℹ️  Additionally, ${missingFromCloudDocs.length} connector(s) confirmed missing from cloud-docs:`)
+              console.log(`   INFO: Additionally, ${missingFromCloudDocs.length} connector(s) confirmed missing from cloud-docs:`)
               missingFromCloudDocs.forEach(({ type, name }) => {
-                console.log(`      • ${type}/${name}`)
+                console.log(`      - ${type}/${name}`)
               })
             }
           } else if (missingFromCloudDocs.length > 0) {
-            console.log(`   ⚠️  Found ${missingFromCloudDocs.length} cloud-supported connector(s) missing from cloud-docs:`)
+            console.log(`   WARNING: Found ${missingFromCloudDocs.length} cloud-supported connector(s) missing from cloud-docs:`)
             missingFromCloudDocs.forEach(({ type, name }) => {
-              console.log(`      • ${type}/${name}`)
+              console.log(`      - ${type}/${name}`)
             })
-            console.log(`   ℹ️  These connectors need pages added to https://github.com/redpanda-data/cloud-docs`)
+            console.log(`   INFO: These connectors need pages added to https://github.com/redpanda-data/cloud-docs`)
           } else {
-            console.log(`   ✓  All cloud-supported connectors have pages in cloud-docs`)
+            console.log(`   All cloud-supported connectors have pages in cloud-docs`)
           }
         } catch (error) {
-          console.log(`   ⚠️  Could not check cloud-docs: ${error.message}`)
+          console.log(`   WARNING: Could not check cloud-docs: ${error.message}`)
         }
       }
 

package/bin/mcp-tools/content-review.js

@@ -1,196 +0,0 @@
-/**
- * Content Review Tool
- *
- * Provides comprehensive content review with automatic style guide context.
- * This tool automatically fetches the style guide and provides review instructions,
- * so the LLM has everything needed in a single call.
- *
- * OPTIMIZATION: Caches style guide and review instructions.
- * - Style guide cached for 1 hour (rarely changes)
- * - Review instructions cached permanently (static)
- */
-
-const fs = require('fs');
-const path = require('path');
-const cache = require('./cache');
-
-/**
- * Review content with automatic style guide context
- * @param {Object} args - Tool arguments
- * @param {string} args.content - Content to review
- * @param {string} [args.focus] - What to focus on (comprehensive, style, terminology, clarity)
- * @param {string} [baseDir] - Base directory for finding resources
- * @returns {Object} Review context including style guide
- */
-function reviewContent(args, baseDir) {
-  if (!args || !args.content) {
-    return {
-      success: false,
-      error: 'Missing required parameter: content'
-    };
-  }
-
-  const focus = args.focus || 'comprehensive';
-
-  // Validate focus parameter
-  const validFocus = ['comprehensive', 'style', 'terminology', 'clarity'];
-  if (!validFocus.includes(focus)) {
-    return {
-      success: false,
-      error: `Invalid focus parameter. Must be one of: ${validFocus.join(', ')}`
-    };
-  }
-
-  // Find base directory (either provided or from repo root)
-  const actualBaseDir = baseDir || path.join(__dirname, '../..');
-
-  // Load style guide resource (with caching)
-  const styleGuidePath = path.join(actualBaseDir, 'mcp', 'team-standards', 'style-guide.md');
-  const cacheKey = 'style-guide-content';
-
-  let styleGuide = cache.get(cacheKey);
-
-  if (!styleGuide) {
-    if (!fs.existsSync(styleGuidePath)) {
-      return {
-        success: false,
-        error: `Style guide not found at: ${styleGuidePath}`,
-        suggestion: 'Ensure the MCP server is running from the correct directory'
-      };
-    }
-
-    try {
-      styleGuide = fs.readFileSync(styleGuidePath, 'utf8');
-      cache.set(cacheKey, styleGuide, 60 * 60 * 1000); // Cache for 1 hour
-    } catch (err) {
-      return {
-        success: false,
-        error: `Failed to read style guide: ${err.message}`
-      };
-    }
-  }
-
-  // Build review instructions based on focus (cached permanently as they're static)
-  const instructionsCacheKey = `review-instructions:${focus}`;
-  let instructions = cache.get(instructionsCacheKey);
-
-  if (!instructions) {
-    instructions = buildReviewInstructions(focus);
-    cache.set(instructionsCacheKey, instructions, 24 * 60 * 60 * 1000); // Cache for 24 hours
-  }
-
-  // Return the bundled context
-  return {
-    success: true,
-    message: 'Style guide loaded. Please review the content according to the instructions below.',
-    styleGuide,
-    instructions,
-    content: args.content,
-    focus,
-    // Provide formatted output that LLM can easily parse
-    reviewContext: formatReviewContext(styleGuide, instructions, args.content, focus)
-  };
-}
-
-/**
- * Build review instructions based on focus area
- * @param {string} focus - What to focus on
- * @returns {string} Review instructions
- */
-function buildReviewInstructions(focus) {
-  const baseInstructions = `
-# Content Review Instructions
-
-You are reviewing documentation for the Redpanda documentation team.
-
-The style guide has been automatically loaded for your reference.
-
-## Your task
-
-Review the provided content and provide detailed, actionable feedback.
-`;
-
-  const focusInstructions = {
-    comprehensive: `
-Review all aspects of the content:
-
-1. **Style violations** - Check against the style guide (capitalization, formatting, structure)
-2. **Terminology issues** - Verify correct usage of approved terms
-3. **Voice and tone** - Ensure consistent, appropriate tone
-4. **Clarity and readability** - Identify confusing or unclear sections
-5. **Technical accuracy** - Flag any technical issues (if detectable)
-6. **Formatting** - Check AsciiDoc formatting, code blocks, lists
-7. **Accessibility** - Verify heading hierarchy, alt text, link text
-
-Provide specific line numbers or sections for each issue found.
-`,
-    style: `
-Focus on style guide compliance:
-
-1. **Formatting violations** - Capitalization, punctuation, spacing
-2. **Structural issues** - Heading hierarchy, section organization
-3. **Voice and tone** - Too formal, too casual, or inconsistent
-4. **Style guide rules** - Any violations of documented standards
-
-Reference specific style guide sections when identifying issues.
-`,
-    terminology: `
-Focus on terminology:
-
-1. **Incorrect terms** - Wrong capitalization, spelling, or usage
-2. **Deprecated terms** - Outdated terms that should be replaced
-3. **Inconsistent usage** - Same concept referred to differently
-4. **Missing approved terms** - Concepts that should use glossary terms
-
-Check against the terminology section of the style guide.
-`,
-    clarity: `
-Focus on clarity and readability:
-
-1. **Complex sentences** - Sentences that are too long or convoluted
-2. **Unclear explanations** - Technical concepts that need more context
-3. **Poor structure** - Content organization issues
-4. **Missing context** - Assumptions about reader knowledge
-5. **Confusing language** - Jargon without explanation
-
-Suggest specific improvements for each issue.
-`
-  };
-
-  return baseInstructions + (focusInstructions[focus] || focusInstructions.comprehensive);
-}
-
-/**
- * Format the complete review context for the LLM
- * @param {string} styleGuide - Style guide content
- * @param {string} instructions - Review instructions
- * @param {string} content - Content to review
- * @param {string} focus - Focus area
- * @returns {string} Formatted review context
- */
-function formatReviewContext(styleGuide, instructions, content, focus) {
-  return `${instructions}
-
----
-
-# Style Guide Reference
-
-${styleGuide}
-
----
-
-# Content to Review
-
-${content}
-
----
-
-**Focus Area**: ${focus}
-
-Please provide your review above, organized by issue type with specific line/section references.
-`;
-}
-
-module.exports = {
-  reviewContent
-};

package/bin/mcp-tools/frontmatter.js

@@ -1,138 +0,0 @@
-/**
- * Frontmatter Parser for MCP Prompts
- *
- * Parses YAML frontmatter from markdown files to extract metadata.
- * Supports validation against JSON schemas.
- */
-
-const yaml = require('js-yaml');
-
-/**
- * Parse frontmatter from markdown content
- * @param {string} content - Markdown content with optional frontmatter
- * @returns {{ metadata: Object, content: string }} Parsed frontmatter and remaining content
- */
-function parseFrontmatter(content) {
-  const frontmatterRegex = /^---\s*\n([\s\S]*?)\n---\s*\n([\s\S]*)$/;
-  const match = content.match(frontmatterRegex);
-
-  if (!match) {
-    // No frontmatter found - return empty metadata and full content
-    return {
-      metadata: {},
-      content: content
-    };
-  }
-
-  try {
-    const metadata = yaml.load(match[1]) || {};
-    const remainingContent = match[2];
-
-    return {
-      metadata,
-      content: remainingContent
-    };
-  } catch (err) {
-    throw new Error(`Failed to parse YAML frontmatter: ${err.message}`);
-  }
-}
-
-/**
- * JSON Schema for prompt frontmatter
- */
-const promptFrontmatterSchema = {
-  type: 'object',
-  properties: {
-    description: {
-      type: 'string',
-      minLength: 10,
-      description: 'Description of what this prompt does'
-    },
-    version: {
-      type: 'string',
-      pattern: '^\\d+\\.\\d+\\.\\d+$',
-      description: 'Semantic version (for example, 1.0.0)'
-    },
-    arguments: {
-      type: 'array',
-      description: 'Arguments this prompt accepts',
-      items: {
-        type: 'object',
-        required: ['name', 'description', 'required'],
-        properties: {
-          name: {
-            type: 'string',
-            pattern: '^[a-z_][a-z0-9_]*$',
-            description: 'Argument name (lowercase, underscores allowed)'
-          },
-          description: {
-            type: 'string',
-            minLength: 5,
-            description: 'What this argument is for'
-          },
-          required: {
-            type: 'boolean',
-            description: 'Whether this argument is required'
-          }
-        },
-        additionalProperties: false
-      }
-    },
-    argumentFormat: {
-      type: 'string',
-      enum: ['content-append', 'structured'],
-      description: 'How to format arguments when building the prompt'
-    }
-  },
-  additionalProperties: false
-};
-
-/**
- * Validate frontmatter against schema
- * @param {Object} metadata - Parsed frontmatter metadata
- * @param {string} filename - File name (for error messages)
- * @param {Object} schema - JSON schema to validate against
- * @throws {Error} If validation fails
- */
-function validateFrontmatter(metadata, filename, schema = promptFrontmatterSchema) {
-  const Ajv = require('ajv');
-  const ajv = new Ajv({ allErrors: true });
-
-  const valid = ajv.validate(schema, metadata);
-
-  if (!valid) {
-    const errors = ajv.errors.map(err => {
-      const path = err.instancePath || 'root';
-      return `  - ${path}: ${err.message}`;
-    }).join('\n');
-
-    throw new Error(`Invalid frontmatter in ${filename}:\n${errors}`);
-  }
-}
-
-/**
- * Parse and validate prompt file
- * @param {string} content - File content
- * @param {string} filename - File name
- * @returns {{ metadata: Object, content: string }} Validated metadata and content
- */
-function parsePromptFile(content, filename) {
-  const { metadata, content: promptContent } = parseFrontmatter(content);
-
-  // If metadata exists, validate it
-  if (Object.keys(metadata).length > 0) {
-    validateFrontmatter(metadata, filename);
-  }
-
-  return {
-    metadata,
-    content: promptContent
-  };
-}
-
-module.exports = {
-  parseFrontmatter,
-  parsePromptFile,
-  validateFrontmatter,
-  promptFrontmatterSchema
-};