@redpanda-data/docs-extensions-and-macros 4.9.0 → 4.10.0

package/tools/redpanda-connect/generate-rpcn-connector-docs.js

@@ -209,6 +209,31 @@ function resolveReferences(obj, root) {
  * When generating full drafts, components with a `status` of `'deprecated'` are skipped.
  */
 async function generateRpcnConnectorDocs(options) {
+  // Types and output folders for bloblang function/method partials
+  const bloblangTypes = [
+    { key: 'bloblang-functions', folder: 'bloblang-functions' },
+    { key: 'bloblang-methods', folder: 'bloblang-methods' }
+  ];
+  // Recursively mark is_beta on any field/component with description starting with BETA:
+  function markBeta(obj) {
+    if (!obj || typeof obj !== 'object') return;
+    if (Array.isArray(obj)) {
+      obj.forEach(markBeta);
+      return;
+    }
+    // Mark as beta if description starts with 'BETA:' (case-insensitive, trims leading whitespace)
+    if (typeof obj.description === 'string' && /^\s*BETA:\s*/i.test(obj.description)) {
+      obj.is_beta = true;
+    }
+    // Recurse into children/config/fields
+    if (Array.isArray(obj.children)) obj.children.forEach(markBeta);
+    if (obj.config && Array.isArray(obj.config.children)) obj.config.children.forEach(markBeta);
+    // For connector/component arrays
+    for (const key of Object.keys(obj)) {
+      if (Array.isArray(obj[key])) obj[key].forEach(markBeta);
+    }
+  }
+
   const {
     data,
     overrides,
@@ -216,6 +241,7 @@ async function generateRpcnConnectorDocs(options) {
     templateIntro,
     templateFields,
     templateExamples,
+    templateBloblang,
     writeFullDrafts
   } = options;
 
@@ -223,16 +249,38 @@ async function generateRpcnConnectorDocs(options) {
   const raw = fs.readFileSync(data, 'utf8');
   const ext = path.extname(data).toLowerCase();
   const dataObj = ext === '.json' ? JSON.parse(raw) : yaml.parse(raw);
+  // Mark beta fields/components before overrides
+  markBeta(dataObj);
 
   // Apply overrides if provided
   if (overrides) {
     const ovRaw = fs.readFileSync(overrides, 'utf8');
     const ovObj = JSON.parse(ovRaw);
-    
     // Resolve any $ref references in the overrides
     const resolvedOverrides = resolveReferences(ovObj, ovObj);
-    
     mergeOverrides(dataObj, resolvedOverrides);
+
+    // Special: merge bloblang_methods and bloblang_functions from overrides into main data
+    for (const [overrideKey, mainKey] of [
+      ['bloblang_methods', 'bloblang-methods'],
+      ['bloblang_functions', 'bloblang-functions']
+    ]) {
+      if (Array.isArray(resolvedOverrides[overrideKey])) {
+        if (!Array.isArray(dataObj[mainKey])) dataObj[mainKey] = [];
+        // Merge by name
+        const mainArr = dataObj[mainKey];
+        const overrideArr = resolvedOverrides[overrideKey];
+        for (const overrideItem of overrideArr) {
+          if (!overrideItem.name) continue;
+          const idx = mainArr.findIndex(i => i.name === overrideItem.name);
+          if (idx !== -1) {
+            mainArr[idx] = { ...mainArr[idx], ...overrideItem };
+          } else {
+            mainArr.push(overrideItem);
+          }
+        }
+      }
+    }
   }
 
   // Compile the “main” template (used when writeFullDrafts = true)
@@ -260,6 +308,7 @@ async function generateRpcnConnectorDocs(options) {
   const fieldsOutRoot   = path.join(outputRoot, 'fields');
   const examplesOutRoot = path.join(outputRoot, 'examples');
   const draftsRoot      = path.join(outputRoot, 'drafts');
+  const configExamplesRoot = path.resolve(process.cwd(), 'modules/components/examples');
 
   if (!writeFullDrafts) {
     fs.mkdirSync(fieldsOutRoot,   { recursive: true });
@@ -332,6 +381,51 @@ async function generateRpcnConnectorDocs(options) {
     }
   }
 
+  // Bloblang function/method partials (only if includeBloblang is true)
+  if (options.includeBloblang) {
+    for (const { key, folder } of bloblangTypes) {
+      const items = dataObj[key];
+      if (!Array.isArray(items)) continue;
+      const outRoot = path.join(outputRoot, folder);
+      fs.mkdirSync(outRoot, { recursive: true });
+      // Use custom or default template
+      const bloblangTemplatePath = templateBloblang || path.resolve(__dirname, './templates/bloblang-function.hbs');
+      const bloblangTemplate = handlebars.compile(fs.readFileSync(bloblangTemplatePath, 'utf8'));
+      for (const fn of items) {
+        if (!fn.name) continue;
+        const adoc = bloblangTemplate(fn);
+        const outPath = path.join(outRoot, `${fn.name}.adoc`);
+        fs.writeFileSync(outPath, adoc, 'utf8');
+        partialsWritten++;
+        partialFiles.push(path.relative(process.cwd(), outPath));
+      }
+    }
+  }
+
+  // Common/Advanced config snippet YAMLs in modules/components/examples
+  const commonConfig = helpers.commonConfig;
+  const advancedConfig = helpers.advancedConfig;
+  for (const [type, items] of Object.entries(dataObj)) {
+    if (!Array.isArray(items)) continue;
+    for (const item of items) {
+      if (!item.name || !item.config || !Array.isArray(item.config.children)) continue;
+  // Common config
+  const commonYaml = commonConfig(type, item.name, item.config.children);
+  const commonPath = path.join(configExamplesRoot, 'common', type, `${item.name}.yaml`);
+  fs.mkdirSync(path.dirname(commonPath), { recursive: true });
+  fs.writeFileSync(commonPath, commonYaml.toString(), 'utf8');
+  partialsWritten++;
+  partialFiles.push(path.relative(process.cwd(), commonPath));
+  // Advanced config
+  const advYaml = advancedConfig(type, item.name, item.config.children);
+  const advPath = path.join(configExamplesRoot, 'advanced', type, `${item.name}.yaml`);
+  fs.mkdirSync(path.dirname(advPath), { recursive: true });
+  fs.writeFileSync(advPath, advYaml.toString(), 'utf8');
+  partialsWritten++;
+  partialFiles.push(path.relative(process.cwd(), advPath));
+    }
+  }
+
   return {
     partialsWritten,
     draftsWritten,

package/tools/redpanda-connect/helpers/bloblangExample.js

@@ -0,0 +1,42 @@
+// Bloblang example formatting helper for Handlebars
+function bloblangExample(example) {
+  if (typeof example === 'object' && example !== null && example.mapping) {
+    let codeBlock = '';
+    if (example.summary && example.summary.trim()) {
+      codeBlock += `# ${example.summary.trim().replace(/\n/g, '\n# ')}\n\n`;
+    }
+    if (typeof example.mapping === 'string') {
+      codeBlock += example.mapping.trim() + '\n';
+    }
+    if (Array.isArray(example.results)) {
+      for (const pair of example.results) {
+        if (Array.isArray(pair) && pair.length === 2) {
+          codeBlock += `\n# In:  ${pair[0]}\n# Out: ${pair[1]}\n`;
+        }
+      }
+    }
+    return `[,coffeescript]\n----\n${codeBlock.trim()}\n----\n`;
+  } else {
+    let exStr = '';
+    if (typeof example === 'string') {
+      exStr = example;
+    } else if (typeof example === 'object' && example !== null) {
+      if (example.code) {
+        exStr = example.code;
+      } else if (example.example) {
+        exStr = example.example;
+      } else {
+        try {
+          exStr = require('yaml').stringify(example).trim();
+        } catch {
+          exStr = JSON.stringify(example, null, 2);
+        }
+      }
+    } else {
+      exStr = String(example);
+    }
+    return `[source,coffeescript]\n----\n${exStr}\n----\n`;
+  }
+}
+
+module.exports = bloblangExample;

package/tools/redpanda-connect/helpers/index.js

@@ -4,10 +4,10 @@ module.exports = {
   uppercase:       require('./uppercase.js'),
   eq:              require('./eq.js'),
   ne:              require('./ne.js'),
-  join:              require('./join.js'),
+  join:            require('./join.js'),
   or:              require('./or.js'),
-  toYaml:              require('./toYaml.js'),
-  isObject:              require('./isObject.js'),
+  toYaml:          require('./toYaml.js'),
+  isObject:        require('./isObject.js'),
   renderYamlList:  require('./renderYamlList.js'),
   renderConnectFields:    require('./renderConnectFields.js'),
   renderConnectExamples:  require('./renderConnectExamples.js'),
@@ -16,4 +16,5 @@ module.exports = {
   buildConfigYaml:        require('./buildConfigYaml.js'),
   commonConfig:           require('./commonConfig.js'),
   advancedConfig:         require('./advancedConfig.js'),
+  bloblangExample:        require('./bloblangExample.js'),
 };

package/tools/redpanda-connect/helpers/renderConnectFields.js

@@ -28,7 +28,10 @@ module.exports = function renderConnectFields(children, prefix = '') {
 
     // Normalize types
     let displayType;
-    if (child.type === 'string' && child.kind === 'array') {
+    const isArrayTitle = typeof child.name === 'string' && child.name.endsWith('[]');
+    if (isArrayTitle) {
+      displayType = 'array<object>';
+    } else if (child.type === 'string' && child.kind === 'array') {
       displayType = 'array';
     } else if (child.type === 'unknown' && child.kind === 'map') {
       displayType = 'object';
@@ -42,14 +45,38 @@ module.exports = function renderConnectFields(children, prefix = '') {
 
     let block = '';
     const isArray = child.kind === 'array';
+    // Only append [] if kind is array and name does not already end with []
+    const nameWithArray = (typeof child.name === 'string' && isArray && !child.name.endsWith('[]'))
+      ? `${child.name}[]`
+      : child.name;
     const currentPath = prefix
-      ? `${prefix}.${child.name}${isArray ? '[]' : ''}`
-      : `${child.name}${isArray ? '[]' : ''}`;
+      ? `${prefix}.${nameWithArray}`
+      : `${nameWithArray}`;
 
     block += `=== \`${currentPath}\`\n\n`;
 
-    if (child.description) {
-      block += `${child.description}\n\n`;
+    // --- Beta badge logic (now uses is_beta) ---
+    let desc = child.description || '';
+    if (child.is_beta) {
+      // Remove any leading "BETA:" label (case-insensitive, trims leading whitespace)
+      desc = 'badge::[label=Beta, size=large, tooltip={page-beta-text}]\n\n' + desc.replace(/^\s*BETA:\s*/i, '');
+    }
+
+    // --- Interpolation support notice ---
+    const interpolationNotice = 'This field supports xref:configuration:interpolation.adoc#bloblang-queries[interpolation functions].';
+    if (child.interpolated === true) {
+      // Only add if not already present (case-insensitive)
+      const descLower = desc.toLowerCase();
+      if (!descLower.includes('interpolation functions')) {
+        desc = desc.trim() + (desc.trim() ? '\n\n' : '') + interpolationNotice;
+      }
+    } else {
+      // If interpolation is not true, remove the notice if present
+      desc = desc.replace(new RegExp(interpolationNotice.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'), 'g'), '').replace(/\n{2,}/g, '\n\n');
+    }
+
+    if (desc) {
+      block += `${desc}\n\n`;
     }
     if (child.is_secret) {
       block += `include::redpanda-connect:components:partial$secret_warning.adoc[]\n\n`;

package/tools/redpanda-connect/report-delta.js

@@ -1,3 +1,103 @@
+/**
+ * Generate a JSON diff report between two connector index objects.
+ * @param {object} oldIndex - Previous version connector index
+ * @param {object} newIndex - Current version connector index
+ * @param {object} opts - { oldVersion, newVersion, timestamp }
+ * @returns {object} JSON diff report
+ */
+function generateConnectorDiffJson(oldIndex, newIndex, opts = {}) {
+  const oldMap = buildComponentMap(oldIndex);
+  const newMap = buildComponentMap(newIndex);
+
+  // New components
+  const newComponentKeys = Object.keys(newMap).filter(k => !(k in oldMap));
+  const newComponents = newComponentKeys.map(key => {
+    const [type, name] = key.split(':');
+    const raw = newMap[key].raw;
+    return {
+      name,
+      type,
+      status: raw.status || raw.type || '',
+      version: raw.version || raw.introducedInVersion || '',
+      description: raw.description || ''
+    };
+  });
+
+  // Removed components
+  const removedComponentKeys = Object.keys(oldMap).filter(k => !(k in newMap));
+  const removedComponents = removedComponentKeys.map(key => {
+    const [type, name] = key.split(':');
+    const raw = oldMap[key].raw;
+    return {
+      name,
+      type,
+      status: raw.status || raw.type || '',
+      version: raw.version || raw.introducedInVersion || '',
+      description: raw.description || ''
+    };
+  });
+
+  // New fields under existing components
+  const newFields = [];
+  Object.keys(newMap).forEach(cKey => {
+    if (!(cKey in oldMap)) return;
+    const oldFields = new Set(oldMap[cKey].fields || []);
+    const newFieldsArr = newMap[cKey].fields || [];
+    newFieldsArr.forEach(fName => {
+      if (!oldFields.has(fName)) {
+        const [type, compName] = cKey.split(':');
+        let rawFieldObj = null;
+        if (type === 'config') {
+          rawFieldObj = (newMap[cKey].raw.children || []).find(f => f.name === fName);
+        } else {
+          rawFieldObj = (newMap[cKey].raw.config?.children || []).find(f => f.name === fName);
+        }
+        newFields.push({
+          component: cKey,
+          field: fName,
+          introducedIn: rawFieldObj && (rawFieldObj.introducedInVersion || rawFieldObj.version),
+          description: rawFieldObj && rawFieldObj.description
+        });
+      }
+    });
+  });
+
+  // Removed fields under existing components
+  const removedFields = [];
+  Object.keys(oldMap).forEach(cKey => {
+    if (!(cKey in newMap)) return;
+    const newFieldsSet = new Set(newMap[cKey].fields || []);
+    const oldFieldsArr = oldMap[cKey].fields || [];
+    oldFieldsArr.forEach(fName => {
+      if (!newFieldsSet.has(fName)) {
+        removedFields.push({
+          component: cKey,
+          field: fName
+        });
+      }
+    });
+  });
+
+  return {
+    comparison: {
+      oldVersion: opts.oldVersion || '',
+      newVersion: opts.newVersion || '',
+      timestamp: opts.timestamp || new Date().toISOString()
+    },
+    summary: {
+      newComponents: newComponents.length,
+      removedComponents: removedComponents.length,
+      newFields: newFields.length,
+      removedFields: removedFields.length
+    },
+    details: {
+      newComponents,
+      removedComponents,
+      newFields,
+      removedFields
+    }
+  };
+}
 // tools/redpanda-connect/report-delta.js
 'use strict';
 
@@ -149,4 +249,5 @@ module.exports = {
   buildComponentMap,
   getRpkConnectVersion,
   printDeltaReport,
+  generateConnectorDiffJson,
 };

package/tools/redpanda-connect/templates/bloblang-function.hbs

@@ -0,0 +1,28 @@
+// This content is autogenerated. Do not edit manually. To override descriptions, use the doc-tools CLI with the --overrides option: https://redpandadata.atlassian.net/wiki/spaces/DOC/pages/1247543314/Generate+reference+docs+for+Redpanda+Connect
+
+= {{name}}
+
+{{#if signature}}
+*Signature*: `{{signature}}`
+{{/if}}
+
+{{#if description}}
+{{description}}
+{{/if}}
+
+
+{{#if examples}}
+== Examples
+
+{{#each examples}}
+{{{bloblangExample this}}}
+{{/each}}
+{{/if}}
+
+{{#if related}}
+== Related
+
+{{#each related}}
+* {{this}}
+{{/each}}
+{{/if}}