@redpanda-data/docs-extensions-and-macros 4.12.0 → 4.12.2

package/bin/doc-tools.js

@@ -719,6 +719,18 @@ automation
         fs.unlinkSync(tmpFile);
         dataFile = finalFile;
         console.log(`✅ Fetched and saved: ${finalFile}`);
+
+        // Keep only 2 most recent versions in docs-data
+        const dataFiles = fs.readdirSync(dataDir)
+          .filter(f => /^connect-\d+\.\d+\.\d+\.json$/.test(f))
+          .sort();
+
+        while (dataFiles.length > 2) {
+          const oldestFile = dataFiles.shift();
+          const oldestPath = path.join(dataDir, oldestFile);
+          fs.unlinkSync(oldestPath);
+          console.log(`🧹 Deleted old version from docs-data: ${oldestFile}`);
+        }
       } catch (err) {
         console.error(`❌ Failed to fetch connectors: ${err.message}`);
         process.exit(1);
@@ -845,6 +857,45 @@ automation
     }
 
     const newIndex = JSON.parse(fs.readFileSync(dataFile, 'utf8'));
+
+    // Publish merged version with overrides to modules/components/attachments
+    if (options.overrides && fs.existsSync(options.overrides)) {
+      try {
+        const { mergeOverrides, resolveReferences } = require('../tools/redpanda-connect/generate-rpcn-connector-docs.js');
+
+        // Create a copy of newIndex to merge overrides into
+        const mergedData = JSON.parse(JSON.stringify(newIndex));
+
+        // Read and apply overrides
+        const ovRaw = fs.readFileSync(options.overrides, 'utf8');
+        const ovObj = JSON.parse(ovRaw);
+        const resolvedOverrides = resolveReferences(ovObj, ovObj);
+        mergeOverrides(mergedData, resolvedOverrides);
+
+        // Publish to modules/components/attachments
+        const attachmentsRoot = path.resolve(process.cwd(), 'modules/components/attachments');
+        fs.mkdirSync(attachmentsRoot, { recursive: true });
+
+        // Delete older versions from modules/components/attachments
+        const existingFiles = fs.readdirSync(attachmentsRoot)
+          .filter(f => /^connect-\d+\.\d+\.\d+\.json$/.test(f))
+          .sort();
+
+        for (const oldFile of existingFiles) {
+          const oldFilePath = path.join(attachmentsRoot, oldFile);
+          fs.unlinkSync(oldFilePath);
+          console.log(`🧹 Deleted old version: ${oldFile}`);
+        }
+
+        // Save merged version to modules/components/attachments
+        const destFile = path.join(attachmentsRoot, `connect-${newVersion}.json`);
+        fs.writeFileSync(destFile, JSON.stringify(mergedData, null, 2), 'utf8');
+        console.log(`✅ Published merged version to: ${path.relative(process.cwd(), destFile)}`);
+      } catch (err) {
+        console.error(`❌ Failed to publish merged version: ${err.message}`);
+      }
+    }
+
     printDeltaReport(oldIndex, newIndex);
 
     // Generate JSON diff file for whats-new.adoc
@@ -895,6 +946,93 @@ automation
 
     // Optionally update whats-new.adoc
     if (options.updateWhatsNew) {
+      // Helper function to cap description to two sentences
+      const capToTwoSentences = (description) => {
+        if (!description) return '';
+
+        // Helper to check if text contains problematic content
+        const hasProblematicContent = (text) => {
+          return /```[\s\S]*?```/.test(text) ||  // code blocks
+                 /`[^`]+`/.test(text) ||          // inline code
+                 /^[=#]+\s+.+$/m.test(text) ||    // headings
+                 /\n/.test(text);                 // newlines
+        };
+
+        // Step 1: Replace common abbreviations and ellipses with placeholders
+        const abbreviations = [
+          /\bv\d+\.\d+(?:\.\d+)?/gi, // version numbers like v4.12 or v4.12.0 (must come before decimal)
+          /\d+\.\d+/g,               // decimal numbers
+          /\be\.g\./gi,              // e.g.
+          /\bi\.e\./gi,              // i.e.
+          /\betc\./gi,               // etc.
+          /\bvs\./gi,                // vs.
+          /\bDr\./gi,                // Dr.
+          /\bMr\./gi,                // Mr.
+          /\bMs\./gi,                // Ms.
+          /\bMrs\./gi,               // Mrs.
+          /\bSt\./gi,                // St.
+          /\bNo\./gi                 // No.
+        ];
+
+        let normalized = description;
+        const placeholders = [];
+
+        // Replace abbreviations with placeholders
+        abbreviations.forEach((abbrevRegex, idx) => {
+          normalized = normalized.replace(abbrevRegex, (match) => {
+            const placeholder = `__ABBREV${idx}_${placeholders.length}__`;
+            placeholders.push({ placeholder, original: match });
+            return placeholder;
+          });
+        });
+
+        // Replace ellipses (three or more dots) with placeholder
+        normalized = normalized.replace(/\.{3,}/g, (match) => {
+          const placeholder = `__ELLIPSIS_${placeholders.length}__`;
+          placeholders.push({ placeholder, original: match });
+          return placeholder;
+        });
+
+        // Step 2: Split sentences using the regex
+        const sentenceRegex = /[^.!?]+[.!?]+(?:\s|$)/g;
+        const sentences = normalized.match(sentenceRegex);
+
+        if (!sentences || sentences.length === 0) {
+          // Restore placeholders and return original
+          let result = normalized;
+          placeholders.forEach(({ placeholder, original }) => {
+            result = result.replace(placeholder, original);
+          });
+          return result;
+        }
+
+        // Step 3: Determine how many sentences to include
+        let maxSentences = 2;
+
+        // If we have at least 2 sentences, check if the second one has problematic content
+        if (sentences.length >= 2) {
+          // Restore placeholders in second sentence to check original content
+          let secondSentence = sentences[1];
+          placeholders.forEach(({ placeholder, original }) => {
+            secondSentence = secondSentence.replace(new RegExp(placeholder, 'g'), original);
+          });
+
+          // If second sentence has problematic content, only take first sentence
+          if (hasProblematicContent(secondSentence)) {
+            maxSentences = 1;
+          }
+        }
+
+        let result = sentences.slice(0, maxSentences).join('');
+
+        // Step 4: Restore placeholders back to original text
+        placeholders.forEach(({ placeholder, original }) => {
+          result = result.replace(new RegExp(placeholder, 'g'), original);
+        });
+
+        return result.trim();
+      };
+
       try {
         const whatsNewPath = path.join(findRepoRoot(), 'modules/get-started/pages/whats-new.adoc');
         if (!fs.existsSync(whatsNewPath)) {
@@ -956,7 +1094,7 @@ automation
             for (const comp of comps) {
               section += `** xref:components:${type}/${comp.name}.adoc[\`${comp.name}\`]`;
               if (comp.status) section += ` (${comp.status})`;
-              if (comp.description) section += `: ${comp.description}`;
+              if (comp.description) section += `: ${capToTwoSentences(comp.description)}`;
               section += '\n';
             }
           }
@@ -977,6 +1115,60 @@ automation
               description: field.description || '',
             });
           }
+          for (const [type, fields] of Object.entries(fieldsByType)) {
+            section += `* ${type.charAt(0).toUpperCase() + type.slice(1)}:\n`;
+            // Group by component name
+            const byComp = {};
+            for (const f of fields) {
+              if (!byComp[f.compName]) byComp[f.compName] = [];
+              byComp[f.compName].push(f);
+            }
+            for (const [comp, compFields] of Object.entries(byComp)) {
+              section += `** xref:components:${type}/${comp}.adoc[\`${comp}\`]:`;
+              section += '\n';
+              for (const f of compFields) {
+                section += `*** xref:components:${type}/${comp}.adoc#${f.field}[\`${f.field}\`]`;
+                if (f.description) section += `: ${capToTwoSentences(f.description)}`;
+                section += '\n';
+              }
+            }
+          }
+        }
+
+        // Deprecated components
+        if (diff.details.deprecatedComponents && diff.details.deprecatedComponents.length) {
+          section += '\n=== Deprecations\n\n';
+          section += 'The following components are now deprecated:\n\n';
+          // Group by type
+          const byType = {};
+          for (const comp of diff.details.deprecatedComponents) {
+            if (!byType[comp.type]) byType[comp.type] = [];
+            byType[comp.type].push(comp);
+          }
+          for (const [type, comps] of Object.entries(byType)) {
+            section += `* ${type.charAt(0).toUpperCase() + type.slice(1)}:\n`;
+            for (const comp of comps) {
+              section += `** xref:components:${type}/${comp.name}.adoc[\`${comp.name}\`]\n`;
+            }
+          }
+        }
+
+        // Deprecated fields
+        if (diff.details.deprecatedFields && diff.details.deprecatedFields.length) {
+          if (!diff.details.deprecatedComponents || diff.details.deprecatedComponents.length === 0) {
+            section += '\n=== Deprecations\n\n';
+          }
+          section += '\nThe following fields are now deprecated:\n\n';
+          // Group deprecated fields by component type
+          const fieldsByType = {};
+          for (const field of diff.details.deprecatedFields) {
+            const [type, compName] = field.component.split(':');
+            if (!fieldsByType[type]) fieldsByType[type] = [];
+            fieldsByType[type].push({
+              compName,
+              field: field.field
+            });
+          }
           for (const [type, fields] of Object.entries(fieldsByType)) {
             section += `* ${type.charAt(0).toUpperCase() + type.slice(1)} components\n`;
             // Group by component name
@@ -986,23 +1178,20 @@ automation
               byComp[f.compName].push(f);
             }
             for (const [comp, compFields] of Object.entries(byComp)) {
-              section += `** xref:components:${type}/${comp}.adoc['${comp}']`;
+              section += `** xref:components:${type}/${comp}.adoc[\`${comp}\`]`;
               if (compFields.length === 1) {
                 const f = compFields[0];
-                section += `: xref:components:${type}/${comp}.adoc#${f.field}['${f.field}']`;
-                if (f.description) section += ` - ${f.description}`;
-                section += '\n';
+                section += `: xref:components:${type}/${comp}.adoc#${f.field}[\`${f.field}\`]\n`;
               } else {
                 section += '\n';
                 for (const f of compFields) {
-                  section += `*** xref:components:${type}/${comp}.adoc#${f.field}['${f.field}']`;
-                  if (f.description) section += ` - ${f.description}`;
-                  section += '\n';
+                  section += `*** xref:components:${type}/${comp}.adoc#${f.field}[\`${f.field}\`]\n`;
                 }
               }
             }
           }
         }
+
         let updated;
         if (startIdx !== -1) {
           // Replace the existing section

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@redpanda-data/docs-extensions-and-macros",
-  "version": "4.12.0",
+  "version": "4.12.2",
   "description": "Antora extensions and macros developed for Redpanda documentation.",
   "keywords": [
     "antora",

package/tools/redpanda-connect/helpers/renderConnectFields.js

@@ -138,7 +138,57 @@ module.exports = function renderConnectFields(children, prefix = '') {
     if (child.examples && child.examples.length) {
       block += `[source,yaml]\n----\n# Examples:\n`;
       if (child.kind === 'array') {
-        block += renderYamlList(child.name, child.examples);
+        child.examples.forEach(example => {
+          if (Array.isArray(example)) {
+            // Check if array contains any objects
+            const hasObjects = example.some(item => typeof item === 'object' && item !== null);
+
+            if (hasObjects) {
+              // Use block style for arrays of objects
+              block += renderYamlList(child.name, [example]);
+            } else {
+              // Use flow style for arrays of primitives: fieldName: [item1, item2, ...]
+              const items = example.map(item => {
+                if (typeof item === 'string') {
+                  // Check if already quoted (starts and ends with quotes)
+                  const alreadyQuoted = item.startsWith('"') && item.endsWith('"');
+
+                  if (alreadyQuoted) {
+                    // Already quoted, return as-is
+                    return item;
+                  }
+
+                  // Check if quoting is needed
+                  const needsQuoting = item === '' ||
+                                       item === '*' ||
+                                       /^(true|false|null|yes|no|on|off)$/i.test(item) ||
+                                       /^[-+]?[0-9]*\.?[0-9]+([eE][-+]?[0-9]+)?$/.test(item) ||
+                                       /[:\[\]\{\},&>|%@`"]/.test(item) ||
+                                       /\s/.test(item); // any whitespace
+
+                  if (needsQuoting) {
+                    // Escape backslashes first, then double quotes
+                    const escaped = item.replace(/\\/g, '\\\\').replace(/"/g, '\\"');
+                    return `"${escaped}"`;
+                  }
+                  return item;
+                }
+                // For primitives (numbers, booleans, etc.), convert to string
+                const strValue = String(item);
+                // Check if the stringified value needs quoting
+                if (/[:\[\]\{\},&>|%@`"]/.test(strValue) || /\s/.test(strValue)) {
+                  const escaped = strValue.replace(/\\/g, '\\\\').replace(/"/g, '\\"');
+                  return `"${escaped}"`;
+                }
+                return strValue;
+              });
+              block += `${child.name}: [${items.join(', ')}]\n`;
+            }
+          } else {
+            // Fallback for non-array examples (shouldn't happen for array fields)
+            block += `${child.name}: ${example}\n`;
+          }
+        });
       } else {
         child.examples.forEach(example => {
           if (typeof example === 'object') {

package/tools/redpanda-connect/report-delta.js

@@ -1,3 +1,5 @@
+const { execSync } = require('child_process');
+
 /**
  * Generate a JSON diff report between two connector index objects.
  * @param {object} oldIndex - Previous version connector index
@@ -78,6 +80,66 @@ function generateConnectorDiffJson(oldIndex, newIndex, opts = {}) {
     });
   });
 
+  // Newly deprecated components (exist in both versions but became deprecated)
+  const deprecatedComponents = [];
+  Object.keys(newMap).forEach(cKey => {
+    if (!(cKey in oldMap)) return;
+    const oldStatus = (oldMap[cKey].raw.status || '').toLowerCase();
+    const newStatus = (newMap[cKey].raw.status || '').toLowerCase();
+    if (oldStatus !== 'deprecated' && newStatus === 'deprecated') {
+      const [type, name] = cKey.split(':');
+      const raw = newMap[cKey].raw;
+      deprecatedComponents.push({
+        name,
+        type,
+        status: raw.status || raw.type || '',
+        version: raw.version || raw.introducedInVersion || '',
+        description: raw.description || ''
+      });
+    }
+  });
+
+  // Newly deprecated fields (exist in both versions but became deprecated)
+  const deprecatedFields = [];
+  Object.keys(newMap).forEach(cKey => {
+    if (!(cKey in oldMap)) return;
+    const oldFieldsArr = oldMap[cKey].fields || [];
+    const newFieldsArr = newMap[cKey].fields || [];
+
+    // Check fields that exist in both versions
+    const commonFields = oldFieldsArr.filter(f => newFieldsArr.includes(f));
+    commonFields.forEach(fName => {
+      const [type, compName] = cKey.split(':');
+
+      // Get old field object
+      let oldFieldObj = null;
+      if (type === 'config') {
+        oldFieldObj = (oldMap[cKey].raw.children || []).find(f => f.name === fName);
+      } else {
+        oldFieldObj = (oldMap[cKey].raw.config?.children || []).find(f => f.name === fName);
+      }
+
+      // Get new field object
+      let newFieldObj = null;
+      if (type === 'config') {
+        newFieldObj = (newMap[cKey].raw.children || []).find(f => f.name === fName);
+      } else {
+        newFieldObj = (newMap[cKey].raw.config?.children || []).find(f => f.name === fName);
+      }
+
+      const oldDeprecated = oldFieldObj && (oldFieldObj.is_deprecated === true || oldFieldObj.deprecated === true || (oldFieldObj.status || '').toLowerCase() === 'deprecated');
+      const newDeprecated = newFieldObj && (newFieldObj.is_deprecated === true || newFieldObj.deprecated === true || (newFieldObj.status || '').toLowerCase() === 'deprecated');
+
+      if (!oldDeprecated && newDeprecated) {
+        deprecatedFields.push({
+          component: cKey,
+          field: fName,
+          description: newFieldObj && newFieldObj.description
+        });
+      }
+    });
+  });
+
   return {
     comparison: {
       oldVersion: opts.oldVersion || '',
@@ -88,23 +150,20 @@ function generateConnectorDiffJson(oldIndex, newIndex, opts = {}) {
       newComponents: newComponents.length,
       removedComponents: removedComponents.length,
       newFields: newFields.length,
-      removedFields: removedFields.length
+      removedFields: removedFields.length,
+      deprecatedComponents: deprecatedComponents.length,
+      deprecatedFields: deprecatedFields.length
     },
     details: {
       newComponents,
       removedComponents,
       newFields,
-      removedFields
+      removedFields,
+      deprecatedComponents,
+      deprecatedFields
     }
   };
 }
-// tools/redpanda-connect/report-delta.js
-'use strict';
-
-const fs = require('fs');
-const path = require('path');
-const yaml = require('yaml');
-const { execSync } = require('child_process');
 
 function discoverComponentKeys(obj) {
   return Object.keys(obj).filter(key => Array.isArray(obj[key]));
@@ -209,6 +268,47 @@ function printDeltaReport(oldIndex, newIndex) {
     });
   });
 
+  // Newly deprecated components
+  const deprecatedComponentKeys = [];
+  Object.keys(newMap).forEach(cKey => {
+    if (!(cKey in oldMap)) return;
+    const oldStatus = (oldMap[cKey].raw.status || '').toLowerCase();
+    const newStatus = (newMap[cKey].raw.status || '').toLowerCase();
+    if (oldStatus !== 'deprecated' && newStatus === 'deprecated') {
+      deprecatedComponentKeys.push(cKey);
+    }
+  });
+
+  // Newly deprecated fields
+  const deprecatedFieldsList = [];
+  Object.keys(newMap).forEach(cKey => {
+    if (!(cKey in oldMap)) return;
+    const oldFieldsArr = oldMap[cKey].fields || [];
+    const newFieldsArr = newMap[cKey].fields || [];
+    const commonFields = oldFieldsArr.filter(f => newFieldsArr.includes(f));
+
+    commonFields.forEach(fName => {
+      const [type, compName] = cKey.split(':');
+      let oldFieldObj = null;
+      if (type === 'config') {
+        oldFieldObj = (oldMap[cKey].raw.children || []).find(f => f.name === fName);
+      } else {
+        oldFieldObj = (oldMap[cKey].raw.config?.children || []).find(f => f.name === fName);
+      }
+      let newFieldObj = null;
+      if (type === 'config') {
+        newFieldObj = (newMap[cKey].raw.children || []).find(f => f.name === fName);
+      } else {
+        newFieldObj = (newMap[cKey].raw.config?.children || []).find(f => f.name === fName);
+      }
+      const oldDeprecated = oldFieldObj && (oldFieldObj.is_deprecated === true || oldFieldObj.deprecated === true || (oldFieldObj.status || '').toLowerCase() === 'deprecated');
+      const newDeprecated = newFieldObj && (newFieldObj.is_deprecated === true || newFieldObj.deprecated === true || (newFieldObj.status || '').toLowerCase() === 'deprecated');
+      if (!oldDeprecated && newDeprecated) {
+        deprecatedFieldsList.push({ component: cKey, field: fName });
+      }
+    });
+  });
+
   console.log('\n📋 RPCN Connector Delta Report\n');
 
   if (newComponentKeys.length) {
@@ -242,6 +342,29 @@ function printDeltaReport(oldIndex, newIndex) {
   } else {
     console.log('➤ No newly added fields.\n');
   }
+
+  if (deprecatedComponentKeys.length) {
+    console.log('➤ Newly deprecated components:');
+    deprecatedComponentKeys.forEach(key => {
+      const [type, name] = key.split(':');
+      const raw = newMap[key].raw;
+      console.log(`   • ${type}/${name}`);
+    });
+    console.log('');
+  } else {
+    console.log('➤ No newly deprecated components.\n');
+  }
+
+  if (deprecatedFieldsList.length) {
+    console.log('➤ Newly deprecated fields:');
+    deprecatedFieldsList.forEach(entry => {
+      const { component, field } = entry;
+      console.log(`   • ${component} → ${field}`);
+    });
+    console.log('');
+  } else {
+    console.log('➤ No newly deprecated fields.\n');
+  }
 }
 
 module.exports = {