@redpanda-data/docs-extensions-and-macros 4.16.4 → 4.17.1

package/bin/mcp-tools/index.js

@@ -5,7 +5,7 @@
  */
 
 // Utilities
-const { findRepoRoot, executeCommand, normalizeVersion, formatDate } = require('./utils');
+const { findRepoRoot, executeCommand, normalizeVersion, formatDate, serializeResult } = require('./utils');
 
 // Antora
 const { getAntoraStructure } = require('./antora');
@@ -39,58 +39,73 @@ function executeTool(toolName, args = {}) {
   const repoRoot = findRepoRoot();
 
   try {
+    let result;
     switch (toolName) {
       case 'get_antora_structure':
-        return getAntoraStructure(repoRoot);
+        result = getAntoraStructure(repoRoot);
+        break;
 
       case 'get_redpanda_version':
-        return getRedpandaVersion(args);
+        result = getRedpandaVersion(args);
+        break;
 
       case 'get_console_version':
-        return getConsoleVersion();
+        result = getConsoleVersion();
+        break;
 
       case 'generate_property_docs':
-        return generatePropertyDocs(args);
+        result = generatePropertyDocs(args);
+        break;
 
       case 'generate_metrics_docs':
-        return generateMetricsDocs(args);
+        result = generateMetricsDocs(args);
+        break;
 
       case 'generate_rpk_docs':
-        return generateRpkDocs(args);
+        result = generateRpkDocs(args);
+        break;
 
       case 'generate_rpcn_connector_docs':
-        return generateRpConnectDocs(args);
+        result = generateRpConnectDocs(args);
+        break;
 
       case 'generate_helm_docs':
-        return generateHelmDocs(args);
+        result = generateHelmDocs(args);
+        break;
 
       case 'generate_cloud_regions':
-        return generateCloudRegions(args);
+        result = generateCloudRegions(args);
+        break;
 
       case 'generate_crd_docs':
-        return generateCrdDocs(args);
+        result = generateCrdDocs(args);
+        break;
 
       case 'generate_bundle_openapi':
-        return generateBundleOpenApi(args);
+        result = generateBundleOpenApi(args);
+        break;
 
       case 'review_generated_docs':
-        return reviewGeneratedDocs(args);
+        result = reviewGeneratedDocs(args);
+        break;
 
       case 'run_doc_tools_command': {
         // Validate and execute raw doc-tools command
         if (!args || typeof args !== 'object') {
-          return {
+          result = {
             success: false,
             error: 'Invalid arguments: expected an object'
           };
+          break;
         }
 
         const validation = validateDocToolsCommand(args.command);
         if (!validation.valid) {
-          return {
+          result = {
             success: false,
             error: validation.error
           };
+          break;
         }
 
         try {
@@ -110,34 +125,38 @@ function executeTool(toolName, args = {}) {
             cwd: repoRoot.root
           });
 
-          return {
+          result = {
             success: true,
             output: output.trim(),
             command: `${docTools.program} ${fullArgs.join(' ')}`
           };
         } catch (err) {
-          return {
+          result = {
             success: false,
             error: err.message,
             stdout: err.stdout || '',
             stderr: err.stderr || ''
           };
         }
+        break;
       }
 
       default:
-        return {
+        result = {
           success: false,
           error: `Unknown tool: ${toolName}`,
           suggestion: 'Check the tool name and try again'
         };
     }
+
+    // Serialize the result to handle any Error objects that might have slipped through
+    return serializeResult(result);
   } catch (err) {
-    return {
+    return serializeResult({
       success: false,
       error: err.message,
       suggestion: 'An unexpected error occurred while executing the tool'
-    };
+    });
   }
 }
 

package/bin/mcp-tools/utils.js

@@ -117,6 +117,100 @@ function formatDate(date = new Date()) {
   return date.toISOString().split('T')[0];
 }
 
+/**
+ * Serialize an Error object to a plain object
+ * Prevents circular reference errors when JSON.stringify'ing results
+ * @param {Error} error - The error to serialize
+ * @returns {Object|string|null} Plain object with error properties, string, or null
+ */
+function serializeError(error) {
+  if (!error) return null;
+
+  // If it's already a string, return as-is
+  if (typeof error === 'string') return error;
+
+  // If it's not an Error object, try to convert to string
+  if (!(error instanceof Error)) {
+    return String(error);
+  }
+
+  // Serialize Error object - use try-catch in case any property access fails
+  try {
+    const serialized = {
+      name: error.name || 'Error',
+      message: error.message || String(error)
+    };
+
+    // Safely add stack if available
+    if (error.stack) {
+      serialized.stack = String(error.stack);
+    }
+
+    // Include any additional properties (like stdout, stderr from exec errors)
+    // Use hasOwnProperty to avoid accessing inherited properties that might cause issues
+    for (const key of ['stdout', 'stderr', 'code', 'status']) {
+      if (Object.prototype.hasOwnProperty.call(error, key) && error[key] != null) {
+        // Convert to string to ensure serializability
+        serialized[key] = String(error[key]);
+      }
+    }
+
+    return serialized;
+  } catch (err) {
+    // If serialization fails for any reason, return a safe fallback
+    return {
+      name: 'Error',
+      message: 'Error object could not be fully serialized',
+      originalError: String(error)
+    };
+  }
+}
+
+/**
+ * Deep serialize a result object, converting any Error objects to plain objects
+ * Handles circular references by tracking visited objects
+ * @param {*} obj - The object to serialize
+ * @param {WeakSet} [visited] - Set of visited objects (for circular reference detection)
+ * @returns {*} Object with all errors serialized
+ */
+function serializeResult(obj, visited = new WeakSet()) {
+  if (!obj || typeof obj !== 'object') {
+    return obj;
+  }
+
+  // Check for circular references
+  if (visited.has(obj)) {
+    return '[Circular]';
+  }
+
+  // Handle Error objects first
+  if (obj instanceof Error) {
+    return serializeError(obj);
+  }
+
+  // Add to visited set
+  visited.add(obj);
+
+  // Handle arrays
+  if (Array.isArray(obj)) {
+    return obj.map(item => serializeResult(item, visited));
+  }
+
+  // Handle plain objects
+  const result = {};
+  for (const [key, value] of Object.entries(obj)) {
+    if (value instanceof Error) {
+      result[key] = serializeError(value);
+    } else if (value && typeof value === 'object') {
+      result[key] = serializeResult(value, visited);
+    } else {
+      result[key] = value;
+    }
+  }
+
+  return result;
+}
+
 module.exports = {
   MAX_RECURSION_DEPTH,
   MAX_EXEC_BUFFER_SIZE,
@@ -127,5 +221,7 @@ module.exports = {
   getDocToolsCommand,
   executeCommand,
   normalizeVersion,
-  formatDate
+  formatDate,
+  serializeError,
+  serializeResult
 };

package/extension-utils/llms-utils.js

@@ -17,10 +17,14 @@ const LLMS_DIRECTIVE_BASE = 'For the complete documentation index, see [llms.txt
  * @returns {string} Formatted markdown blockquote directive
  */
 function formatLlmsDirective(componentName) {
+  const feedbackInstructions = `
+>
+> **Agent Feedback**: If you encounter incorrect, outdated, or confusing documentation, submit feedback via \`POST https://docs.redpanda.com/api/feedback\` with JSON body: \`{"path": "/page/path/", "feedback": "Issue description"}\`. Only submit when you have specific, actionable feedback.`;
+
   if (componentName) {
-    return `> ${LLMS_DIRECTIVE_BASE}. Component-specific: [${componentName}-full.txt](/${componentName}-full.txt)`;
+    return `> ${LLMS_DIRECTIVE_BASE}. Component-specific: [${componentName}-full.txt](/${componentName}-full.txt)${feedbackInstructions}`;
   }
-  return `> ${LLMS_DIRECTIVE_BASE}`;
+  return `> ${LLMS_DIRECTIVE_BASE}${feedbackInstructions}`;
 }
 
 /**

package/extensions/README.adoc

@@ -60,6 +60,7 @@ IMPORTANT: Extensions must be registered under the `antora.extensions` key in yo
 * **generate-index-data** - Generate searchable indexes from content
 * **generate-rp-connect-categories** - Generate Redpanda Connect component categories
 * **generate-rp-connect-info** - Generate Redpanda Connect component information
+* **generate-fields-only-pages** - Generate field-only reference pages for Redpanda Connect components
 
 === Navigation
 * **unlisted-pages** - Manage pages that appear in navigation but aren't listed

package/extensions/add-llms-directive.js

@@ -25,6 +25,9 @@ module.exports.register = function () {
     pages.forEach(page => {
       if (!page.contents) return
 
+      // Skip field-only pages (marked by generate-fields-only-pages extension)
+      if (page.isFieldOnlyPage === true) return
+
       try {
         const html = page.contents.toString('utf8')
 
@@ -46,14 +49,23 @@ module.exports.register = function () {
         const directiveMarkdown = formatLlmsDirective(componentName)
 
         // Convert markdown blockquote to HTML blockquote
-        // Remove leading '> ' and convert markdown links to HTML
-        let directiveText = directiveMarkdown.replace(/^>\s*/, '')
+        // Remove leading '> ' from each line and convert markdown links to HTML
+        let directiveText = directiveMarkdown
+          .split('\n')
+          .map(line => line.replace(/^>\s*/, ''))
+          .join('\n')
 
         // Convert markdown links [text](url) to HTML <a> tags
         // Add space after <a to match afdocs test pattern expectations
         // Add tabindex="-1" and aria-hidden="true" to remove from tab order and hide from assistive tech
         directiveText = directiveText.replace(/\[([^\]]+)\]\(([^)]+)\)/g, '<a href="$2" tabindex="-1" aria-hidden="true">$1</a>')
 
+        // Convert markdown bold **text** to HTML <strong>
+        directiveText = directiveText.replace(/\*\*([^*]+)\*\*/g, '<strong>$1</strong>')
+
+        // Convert markdown code `text` to HTML <code>
+        directiveText = directiveText.replace(/`([^`]+)`/g, '<code>$1</code>')
+
         // Add tabindex="-1" and aria-hidden="true" to blockquote to fully hide from assistive tech
         const directiveHtml = `\n<blockquote class="llms-directive" tabindex="-1" aria-hidden="true">\n<p>${directiveText}</p>\n</blockquote>\n`
 

package/extensions/convert-to-markdown.js

@@ -485,34 +485,83 @@ module.exports.register = function () {
             )
           }
 
-          // Generate YAML frontmatter from AsciiDoc attributes
-          const frontmatter = generateFrontmatter(page)
-          if (frontmatter) {
-            logger.debug(`Generated frontmatter for ${page.src?.path}`)
-          }
+          // Remove escaped underscores from headings (TurndownService escapes them unnecessarily)
+          markdown = markdown.replace(/^(#{1,6}\s+.+)$/gm, (heading) => {
+            return heading.replace(/\\_/g, '_')
+          })
+
+          // Skip directive for field-only pages (marked by generate-fields-only-pages extension)
+          const isFieldOnlyPage = page.isFieldOnlyPage === true
+
+          // Field-only pages: basic markdown only, no frontmatter, no directive, no source comments
+          if (isFieldOnlyPage) {
+            // Strip anchor links from headings: [](#anchor-id)heading → heading
+            markdown = markdown.replace(/\[]\(#[^)]+\)/g, '')
+
+            // Just use the markdown as-is with basic cleanup
+            markdown = markdown.trim()
+          } else {
+            // Regular pages: full treatment with frontmatter and directive
+            // Generate YAML frontmatter from AsciiDoc attributes
+            const frontmatter = generateFrontmatter(page)
+            if (frontmatter) {
+              logger.debug(`Generated frontmatter for ${page.src?.path}`)
+            }
 
-          // Extract H1 heading if present (only at document start)
-          const h1Match = markdown.match(/^(#\s+.+?)(\n|$)/)
-          let h1Heading = ''
-          let restOfMarkdown = markdown
+            // Extract H1 heading if present (only at document start)
+            const h1Match = markdown.match(/^(#\s+.+?)(\n|$)/)
+            let h1Heading = ''
+            let restOfMarkdown = markdown
 
-          if (h1Match) {
-            h1Heading = h1Match[0]
-            restOfMarkdown = markdown.substring(h1Match[0].length).trimStart()
+            if (h1Match) {
+              h1Heading = h1Match[0]
+              restOfMarkdown = markdown.substring(h1Match[0].length).trimStart()
+            }
+
+            // Structure: H1 → llms.txt directive (blockquote) → frontmatter → source → content
+            // The directive must appear near the top for agent-friendly docs spec compliance
+            if (canonicalUrl) {
+              const componentName = page.src?.component || '';
+              // Use markdown blockquote format for the directive (visible, can be hidden with CSS)
+              const llmsDirective = formatLlmsDirective(componentName);
+
+              markdown = `${h1Heading}\n${llmsDirective}\n\n${frontmatter}<!-- Source: ${canonicalUrl} -->\n\n${restOfMarkdown}`
+            } else if (frontmatter) {
+              // If no canonical URL but we have frontmatter, still add directive after H1
+              const llmsDirective = formatLlmsDirective();
+              markdown = `${h1Heading}\n${llmsDirective}\n\n${frontmatter}${restOfMarkdown}`
+            }
           }
 
-          // Structure: H1 → llms.txt directive (blockquote) → frontmatter → source → content
-          // The directive must appear near the top for agent-friendly docs spec compliance
-          if (canonicalUrl) {
-            const componentName = page.src?.component || '';
-            // Use markdown blockquote format for the directive (visible, can be hidden with CSS)
-            const llmsDirective = formatLlmsDirective(componentName);
-
-            markdown = `${h1Heading}\n${llmsDirective}\n\n${frontmatter}<!-- Source: ${canonicalUrl} -->\n\n${restOfMarkdown}`
-          } else if (frontmatter) {
-            // If no canonical URL but we have frontmatter, still add directive after H1
-            const llmsDirective = formatLlmsDirective();
-            markdown = `${h1Heading}\n${llmsDirective}\n\n${frontmatter}${restOfMarkdown}`
+          // Convert relative URLs to absolute URLs (after directive is added)
+          if (siteUrl && page.pub?.url) {
+            try {
+              const baseUrl = new URL(siteUrl)
+              const pageUrl = new URL(page.pub.url, baseUrl)
+
+              // Convert absolute paths: [text](/path) → [text](https://domain/path)
+              markdown = markdown.replace(/\[([^\]]+)\]\(\/([^)]+)\)/g, (match, text, path) => {
+                try {
+                  const fullUrl = new URL('/' + path, baseUrl).toString()
+                  return `[${text}](${fullUrl})`
+                } catch (e) {
+                  return match // Keep original if URL construction fails
+                }
+              })
+
+              // Convert relative paths: [text](../../path) → [text](https://domain/resolved/path)
+              markdown = markdown.replace(/\[([^\]]+)\]\((\.\.\/[^)]+)\)/g, (match, text, relativePath) => {
+                try {
+                  // Resolve relative path against the current page URL
+                  const fullUrl = new URL(relativePath, pageUrl).toString()
+                  return `[${text}](${fullUrl})`
+                } catch (e) {
+                  return match // Keep original if URL construction fails
+                }
+              })
+            } catch (e) {
+              logger.debug(`Failed to resolve relative URLs for ${page.src?.path}: ${e.message}`)
+            }
           }
 
           // Clean up unnecessary whitespace

package/extensions/generate-fields-only-pages.js

@@ -0,0 +1,305 @@
+'use strict'
+
+const fs = require('fs')
+const path = require('path')
+const os = require('os')
+
+
+const handlebars = require('handlebars')
+
+// Import and register Redpanda Connect helpers
+const helpers = require('../tools/redpanda-connect/helpers')
+Object.entries(helpers).forEach(([name, fn]) => {
+  if (typeof fn === 'function') {
+    handlebars.registerHelper(name, fn)
+  }
+})
+
+// Register table format helper
+handlebars.registerHelper('renderConnectFieldsTable', function (children) {
+  if (!children || !Array.isArray(children) || children.length === 0) {
+    return 'No configuration fields available.\n\n'
+  }
+
+  const rows = []
+
+  function collectFields (fieldsList, pathPrefix = '') {
+    if (!Array.isArray(fieldsList)) return
+
+    fieldsList.forEach(field => {
+      if (field.is_deprecated || !field.name) return
+
+      const isArray = field.kind === 'array'
+      const nameWithArray = (typeof field.name === 'string' && isArray && !field.name.endsWith('[]'))
+        ? `${field.name}[]`
+        : field.name
+      const currentPath = pathPrefix ? `${pathPrefix}.${nameWithArray}` : nameWithArray
+
+      // Normalize type
+      let displayType
+      const isArrayTitle = typeof field.name === 'string' && field.name.endsWith('[]')
+      if (isArrayTitle) {
+        displayType = 'array<object>'
+      } else if (field.type === 'string' && field.kind === 'array') {
+        displayType = 'array'
+      } else if (field.type === 'unknown' && field.kind === 'map') {
+        displayType = 'object'
+      } else if (field.type === 'unknown' && (field.kind === 'array' || field.kind === 'list')) {
+        displayType = 'array'
+      } else {
+        displayType = field.type
+      }
+
+      // Format default value
+      let defaultValue = ''
+      if (field.default !== undefined) {
+        if (Array.isArray(field.default) && field.default.length === 0) {
+          defaultValue = '`[]`'
+        } else if (
+          field.default !== null &&
+          typeof field.default === 'object' &&
+          !Array.isArray(field.default) &&
+          Object.keys(field.default).length === 0
+        ) {
+          defaultValue = '`{}`'
+        } else if (typeof field.default === 'string') {
+          const escaped = field.default.replace(/`/g, '\\`')
+          defaultValue = `\`${escaped}\``
+        } else if (typeof field.default === 'number' || typeof field.default === 'boolean') {
+          defaultValue = `\`${field.default}\``
+        } else if (field.default === null) {
+          defaultValue = '`null`'
+        } else {
+          defaultValue = '_(complex)_'
+        }
+      }
+
+      // Clean description for table (single line)
+      let desc = (field.description || '').replace(/\n+/g, ' ').trim()
+      if (desc.length > 150) {
+        desc = desc.substring(0, 147) + '...'
+      }
+
+      rows.push({
+        field: `\`${currentPath}\``,
+        type: `\`${displayType}\``,
+        default: defaultValue || '-',
+        description: desc || '-'
+      })
+
+      // Recurse for children
+      if (field.children && Array.isArray(field.children) && field.children.length > 0) {
+        collectFields(field.children, currentPath)
+      }
+    })
+  }
+
+  collectFields(children, '')
+
+  if (rows.length === 0) return 'No configuration fields available.\n\n'
+
+  let table = '[cols="2,1,1,4"]\n'
+  table += '|===\n'
+  table += '|Field |Type |Default |Description\n\n'
+
+  rows.forEach(row => {
+    table += `|${row.field}\n`
+    table += `|${row.type}\n`
+    table += `|${row.default}\n`
+    table += `|${row.description}\n\n`
+  })
+
+  table += '|===\n'
+
+  return new handlebars.SafeString(table)
+})
+
+// Default configuration
+const DEFAULTS = {
+  format: 'nested',   // 'nested' or 'table'
+  enabled: true       // Allow disabling the extension
+}
+
+module.exports.register = function ({ config }) {
+  const logger = this.getLogger('generate-fields-only-pages-extension')
+
+  // Merge config with defaults
+  const {
+    format = DEFAULTS.format,
+    enabled = DEFAULTS.enabled
+  } = config || {}
+
+  if (!enabled) {
+    logger.info('Extension disabled via config')
+    return
+  }
+
+  // Validate format
+  if (format !== 'nested' && format !== 'table') {
+    logger.error(`Invalid format '${format}'. Must be 'nested' or 'table'. Disabling extension.`)
+    return
+  }
+
+  // Compile template based on format (without title - these pages are meant to be included)
+  const helperName = format === 'table' ? 'renderConnectFieldsTable' : 'renderConnectFields'
+  const fieldOnlyTemplate = handlebars.compile(`{{{${helperName} children}}}`)
+
+  this.on('contentClassified', ({ contentCatalog, siteCatalog }) => {
+    const component = contentCatalog.getComponent('redpanda-connect')
+    if (!component) {
+      logger.warn('redpanda-connect component not found. Skipping field-only page generation.')
+      return
+    }
+
+    const componentVersion = component.latest
+    if (!componentVersion) {
+      logger.warn('No latest version found for redpanda-connect component.')
+      return
+    }
+
+    let connectorData
+    // Look for any versioned JSON attachment in the components module
+    // (i.e. modules/components/attachments/connect-X.Y.Z.json)
+    const attachments = contentCatalog.findBy({
+      component: 'redpanda-connect',
+      version: componentVersion.version,
+      module: 'components',
+      family: 'attachment'
+    })
+
+    // Find all versioned connector JSON attachments and sort by semver
+    const versionedAttachmentPattern = /^connect-(\d+)\.(\d+)\.(\d+)\.json$/
+    const matchingAttachments = attachments
+      .map((file) => {
+        const relative = file.src?.relative || ''
+        const basename = relative.split('/').pop()
+        const match = versionedAttachmentPattern.exec(basename)
+        if (match) {
+          return {
+            file,
+            version: {
+              major: parseInt(match[1], 10),
+              minor: parseInt(match[2], 10),
+              patch: parseInt(match[3], 10),
+              string: `${match[1]}.${match[2]}.${match[3]}`
+            }
+          }
+        }
+        return null
+      })
+      .filter(Boolean)
+      .sort((a, b) => {
+        // Sort by major, then minor, then patch (descending)
+        if (a.version.major !== b.version.major) return b.version.major - a.version.major
+        if (a.version.minor !== b.version.minor) return b.version.minor - a.version.minor
+        return b.version.patch - a.version.patch
+      })
+
+    if (matchingAttachments.length > 1) {
+      const versions = matchingAttachments.map(m => m.version.string).join(', ')
+      logger.warn(`Multiple versioned connector JSON attachments found (${versions}). Using highest version: ${matchingAttachments[0].version.string}`)
+    }
+
+    const attachment = matchingAttachments[0]?.file
+
+    if (!attachment) {
+      logger.warn('No JSON attachment found in the components module of the redpanda-connect content catalog. Skipping field-only page generation.')
+      return
+    }
+
+    try {
+      connectorData = JSON.parse(attachment.contents.toString('utf8'))
+      logger.info(`Loaded connector data from content catalog attachment: ${attachment.src?.relative || 'unknown'}`)
+    } catch (err) {
+      logger.error(`Failed to parse connector data from content catalog attachment: ${err.message}`)
+      return
+    }
+
+    let pagesGenerated = 0
+
+    // Get origin from first existing page in component (for git metadata)
+    const existingPages = contentCatalog.getPages((page) => page.src.component === 'redpanda-connect')
+    const origin = existingPages.length > 0 ? existingPages[0].src.origin : { type: 'generated' }
+
+    // Iterate over each type (inputs, outputs, processors, etc.)
+    for (const [type, items] of Object.entries(connectorData)) {
+      if (!Array.isArray(items)) continue
+
+      // Skip bloblang functions/methods (they don't have config fields)
+      if (type === 'bloblang-functions' || type === 'bloblang-methods') continue
+
+      for (const item of items) {
+        if (!item.name) continue
+
+        // Only generate if there are fields
+        const hasFields = (item.config && item.config.children && item.config.children.length > 0) ||
+                         (item.children && item.children.length > 0)
+
+        if (!hasFields) continue
+
+        const fields = item.config?.children || item.children || []
+
+        try {
+          // Use Handlebars template to render content
+          const content = fieldOnlyTemplate({
+            name: item.name,
+            children: fields
+          })
+
+          const typeDir = type.endsWith('s') ? type : `${type}s`
+          const relative = `fields/${typeDir}/${item.name}.adoc`
+
+          // Create a fake absolute path for generated files (used by logger)
+          const fakeAbspath = path.join(os.tmpdir(), 'generated-fields-only', relative)
+
+          // Create a stat object like real files have
+          const contentBuffer = Buffer.from(content)
+          const stat = Object.assign(new fs.Stats(), {
+            mode: 0o100644,
+            mtime: new Date(),
+            size: contentBuffer.byteLength
+          })
+
+          // Create file spec with all required properties
+          const file = contentCatalog.addFile({
+            path: `modules/components/pages/${relative}`,
+            contents: contentBuffer,
+            stat: stat,
+            src: {
+              component: 'redpanda-connect',
+              version: componentVersion.version,
+              module: 'components',
+              family: 'page',
+              relative: relative,
+              mediaType: 'text/asciidoc',
+              origin: origin,
+              abspath: fakeAbspath  // Needed by logger for error messages
+            }
+          })
+
+          // Mark as field-only page (used by convert-to-markdown and add-llms-directive to skip directive)
+          file.isFieldOnlyPage = true
+
+          pagesGenerated++
+        } catch (err) {
+          logger.error(`Failed to generate field-only page for ${type}/${item.name}: ${err.message}`)
+        }
+      }
+    }
+
+    logger.info(`Generated ${pagesGenerated} field-only pages`)
+  })
+
+  // Unpublish field-only pages as HTML (they should only exist as markdown)
+  // Do this in beforePublish so pages go through full processing (composition, markdown conversion)
+  // but don't get written as HTML files
+  this.on('beforePublish', ({ contentCatalog }) => {
+    const fieldOnlyPages = contentCatalog.getPages((page) => page.isFieldOnlyPage === true && page.out)
+    fieldOnlyPages.forEach((page) => {
+      delete page.out
+    })
+    if (fieldOnlyPages.length > 0) {
+      logger.debug(`Unpublished ${fieldOnlyPages.length} field-only pages from HTML output`)
+    }
+  })
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@redpanda-data/docs-extensions-and-macros",
-  "version": "4.16.4",
+  "version": "4.17.1",
   "description": "Antora extensions and macros developed for Redpanda documentation.",
   "keywords": [
     "antora",
@@ -58,6 +58,7 @@
     "./extensions/generate-rp-connect-categories": "./extensions/generate-rp-connect-categories.js",
     "./extensions/generate-index-data": "./extensions/generate-index-data.js",
     "./extensions/generate-rp-connect-info": "./extensions/generate-rp-connect-info.js",
+    "./extensions/generate-fields-only-pages": "./extensions/generate-fields-only-pages.js",
     "./extensions/add-global-attributes": "./extensions/add-global-attributes.js",
     "./extensions/git-full-clone": "./extensions/git-full-clone.js",
     "./extensions/add-git-dates": "./extensions/add-git-dates.js",

package/tools/redpanda-connect/generate-rpcn-connector-docs.js

@@ -321,7 +321,7 @@ async function generateRpcnConnectorDocs(options) {
   // Compile the "main" template (used when writeFullDrafts = true)
   const compiledTemplate = handlebars.compile(fs.readFileSync(template, 'utf8'));
 
-  // Determine which templates to use for “fields” and “examples”
+  // Determine which templates to use for "fields" and "examples"
   // If templateFields is not provided, fall back to the single `template`.
   // If templateExamples is not provided, skip examples entirely.
   const fieldsTemplatePath   = templateFields   || template;
@@ -367,6 +367,10 @@ async function generateRpcnConnectorDocs(options) {
       if (!item.name) continue;
       const name = item.name;
 
+      // Compute typeDir once for this item (used in templates and file paths)
+      const typeDir = type.endsWith('s') ? type : `${type}s`;
+      item.typeDir = typeDir;
+
       // Always generate field and example partials (needed for both standalone and draft modes)
       // Render fields using the registered "fields" partial
       const fieldsOut = handlebars
@@ -430,6 +434,8 @@ async function generateRpcnConnectorDocs(options) {
           item.support = csvData.support;
         }
 
+        // typeDir is already computed at the beginning of the loop
+
         let content;
         try {
           content = compiledTemplate(item);
@@ -439,7 +445,6 @@ async function generateRpcnConnectorDocs(options) {
 
         // Determine output location based on availability
         let destFile;
-        const typeDir = type.endsWith('s') ? type : `${type}s`;
 
         if (isCloudOnly) {
           // Cloud-only connectors go to partials/components/cloud-only/{type}s/{name}.adoc

package/tools/redpanda-connect/rpcn-connector-docs-handler.js

@@ -28,6 +28,7 @@ function capToTwoSentences (description) {
   }
 
   const abbreviations = [
+    /https?:\/\/[^\s]+?(?=[.!?](?:\s|$)|\s|$)/gi,  // Protect URLs from being split by sentence detection (non-greedy, preserve trailing punctuation)
     /\bv\d+\.\d+(?:\.\d+)?/gi,
     /\d+\.\d+/g,
     /\be\.g\./gi,
@@ -1127,9 +1128,11 @@ async function handleRpcnConnectorDocs (options) {
             if (cgoConn.type === type) {
               const exists = connectorData[type].some(c => c.name === cgoConn.name)
               if (!exists) {
+                // Singularize type for consistency with stored data (except "metrics" which stays plural)
+                const componentType = cgoConn.type === 'metrics' ? 'metrics' : cgoConn.type.replace(/s$/, '')
                 connectorData[type].push({
                   ...cgoConn,
-                  type: cgoConn.type.replace(/s$/, ''),
+                  type: componentType,
                   cloudSupported: false,
                   requiresCgo: true
                 })
@@ -1144,9 +1147,11 @@ async function handleRpcnConnectorDocs (options) {
             if (cloudConn.type === type) {
               const exists = connectorData[type].some(c => c.name === cloudConn.name)
               if (!exists) {
+                // Singularize type for consistency with stored data (except "metrics" which stays plural)
+                const componentType = cloudConn.type === 'metrics' ? 'metrics' : cloudConn.type.replace(/s$/, '')
                 connectorData[type].push({
                   ...cloudConn,
-                  type: cloudConn.type.replace(/s$/, ''),
+                  type: componentType,
                   cloudSupported: true,
                   requiresCgo: false,
                   cloudOnly: true
@@ -1368,8 +1373,14 @@ async function handleRpcnConnectorDocs (options) {
           return !wasInOldOss && !wasInOldCgo && !docsExist
         })
       } else {
+        // Fallback when oldBinaryAnalysis is unavailable
+        // NOTE: oldIndex is loaded from saved JSON files that have platform metadata stripped
+        // (see stripPlatformMetadata() call before saving). This means checking requiresCgo === true
+        // will always fail. We rely on the name match and docs existence check as a heuristic.
+        // If a component with the same name exists in oldIndex OR docs exist, treat it as existing.
         newCgoComponents = binaryAnalysis.cgoOnly.filter(cgoComp => {
-          const wasInOldOss = oldIndex[cgoComp.type]?.some(c => c.name === cgoComp.name)
+          // Check if component with same name existed in old index (metadata unavailable, just name match)
+          const wasInOldIndex = oldIndex[cgoComp.type]?.some(c => c.name === cgoComp.name)
 
           // Check if docs already exist
           const typePlural = cgoComp.type.endsWith('s') ? cgoComp.type : `${cgoComp.type}s`
@@ -1378,7 +1389,8 @@ async function handleRpcnConnectorDocs (options) {
             fs.existsSync(path.join(root, relPath))
           )
 
-          return !wasInOldOss && !docsExist
+          // Only treat as new if it wasn't in the old index AND docs don't exist
+          return !wasInOldIndex && !docsExist
         })
         if (newCgoComponents.length > 0) {
           console.log(`   ℹ️  No old binary analysis found - treating ${newCgoComponents.length} cgo components not in old OSS data as new`)

package/tools/redpanda-connect/templates/connector.hbs

@@ -17,7 +17,7 @@ Common::
 --
 
 ```yml
-include::components:example$common/{{type}}s/{{name}}.yaml[]
+include::components:example$common/{{typeDir}}/{{name}}.yaml[]
 ```
 
 --
@@ -26,7 +26,7 @@ Advanced::
 --
 
 ```yml
-include::components:example$advanced/{{type}}s/{{name}}.yaml[]
+include::components:example$advanced/{{typeDir}}/{{name}}.yaml[]
 ```
 
 --
@@ -47,11 +47,11 @@ You can either xref:install:prebuilt-binary.adoc[download a prebuilt cgo-enabled
 
 {{/if}}
 {{#if (or this.config.children children)}}
-include::redpanda-connect:components:partial$fields/{{type}}s/{{name}}.adoc[]
+include::redpanda-connect:components:partial$fields/{{typeDir}}/{{name}}.adoc[]
 {{/if}}
 
 {{#if this.examples}}
-include::redpanda-connect:components:partial$examples/{{type}}s/{{name}}.adoc[]
+include::redpanda-connect:components:partial$examples/{{typeDir}}/{{name}}.adoc[]
 {{/if}}