@redpanda-data/docs-extensions-and-macros 4.5.0 → 4.6.0

package/tools/redpanda-connect/helpers/renderConnectFields.js

@@ -0,0 +1,146 @@
+'use strict';
+
+const yaml = require('yaml');
+const renderYamlList = require('./renderYamlList');
+const handlebars = require('handlebars');
+
+
+/**
+ * Renders the children of a configuration object into AsciiDoc.
+ *
+ * @param {Array<Object>} children - An array of child objects.
+ * @param {string} [prefix=''] - The prefix path for nested fields.
+ * @returns {handlebars.SafeString} The rendered SafeString containing the configuration details.
+ */
+module.exports = function renderConnectFields(children, prefix = '') {
+  if (!children || !Array.isArray(children) || children.length === 0) {
+    return '';
+  }
+
+  const sorted = [...children].sort((a, b) => {
+    const an = a.name || '';
+    const bn = b.name || '';
+    return an.localeCompare(bn, undefined, { sensitivity: 'base' });
+  });
+
+  let output = '';
+  prefix = typeof prefix === 'string' ? prefix : '';
+
+  sorted.forEach(child => {
+    if (child.is_deprecated) {
+      return;
+    }
+    const isArray = child.kind === 'array';
+    if (!child.name) return;
+    const currentPath = prefix
+      ? `${prefix}.${child.name}${isArray ? '[]' : ''}`
+      : `${child.name}${isArray ? '[]' : ''}`;
+
+    output += `=== \`${currentPath}\`\n\n`;
+
+    if (child.description) {
+      output += `${child.description}\n\n`;
+    }
+
+    if (child.is_secret === true) {
+      output += `include::redpanda-connect:components:partial$secret_warning.adoc[]\n\n`;
+    }
+
+    if (child.version) {
+      output += `ifndef::env-cloud[]\nRequires version ${child.version} or later.\nendif::[]\n\n`;
+    }
+
+    output += `*Type*: \`${child.type}\`\n\n`;
+
+    if (child.type !== 'object' && child.default !== undefined) {
+      if (typeof child.default !== 'object') {
+        const display = child.default === '' ? '""' : String(child.default);
+        output += `*Default*: \`${display}\`\n\n`;
+      } else {
+        const defYaml = yaml.stringify(child.default).trim();
+        output += `*Default*:\n[source,yaml]\n----\n${defYaml}\n----\n\n`;
+      }
+    }
+
+    if (
+      child.annotated_options &&
+      Array.isArray(child.annotated_options) &&
+      child.annotated_options.length > 0
+    ) {
+      output += '[cols="1m,2a"]\n';
+      output += '|===\n';
+      output += '|Option |Summary\n\n';
+      child.annotated_options.forEach(optionPair => {
+        if (Array.isArray(optionPair) && optionPair.length >= 2) {
+          output += `|${optionPair[0]}\n|${optionPair[1]}\n\n`;
+        }
+      });
+      output += '|===\n\n';
+    }
+
+    if (child.options && Array.isArray(child.options) && child.options.length > 0) {
+      output += `*Options*: ${child.options.map(opt => `\`${opt}\``).join(', ')}\n\n`;
+    }
+
+    if (child.examples && child.examples.length) {
+      output += '[source,yaml]\n----\n';
+      output += '# Examples:\n';
+
+      if (child.type === 'string') {
+        if (child.kind === 'array') {
+          output += renderYamlList(child.name, child.examples);
+        } else {
+          child.examples.forEach(example => {
+            if (typeof example === 'string' && example.includes('\n')) {
+              output += `${child.name}: |-\n`;
+              const indentedLines = example.split('\n').map(line => '  ' + line).join('\n');
+              output += `${indentedLines}\n`;
+            } else {
+              output += `${child.name}: ${example}\n`;
+            }
+          });
+          output += '\n';
+        }
+      } else if (child.type === 'processor') {
+        if (child.kind === 'array') {
+          output += renderYamlList(child.name, child.examples);
+        } else {
+          child.examples.forEach(example => {
+            output += `${child.name}: ${example}\n`;
+          });
+          output += '\n';
+        }
+      } else if (child.type === 'object') {
+        if (child.kind === 'array') {
+          output += renderYamlList(child.name, child.examples);
+        } else {
+          child.examples.forEach(example => {
+            if (typeof example === 'object') {
+              const snippet = yaml.stringify(example).trim();
+              const lines = snippet.split('\n');
+              // Prefix two spaces to every line
+              const formattedLines = lines.map(line => '  ' + line).join('\n');
+              output += `${child.name}:\n${formattedLines}\n`;
+            } else {
+              output += `${child.name}: ${example}\n`;
+            }
+          });
+          output += '\n';
+        }
+      } else {
+        child.examples.forEach(example => {
+          output += `${child.name}: ${example}\n`;
+        });
+        output += '\n';
+      }
+
+      output += '----\n\n';
+    }
+
+    if (child.children && Array.isArray(child.children) && child.children.length > 0) {
+      output += renderConnectFields(child.children, currentPath);
+    }
+  });
+
+  return new handlebars.SafeString(output);
+}

package/tools/redpanda-connect/helpers/renderLeafField.js

@@ -0,0 +1,64 @@
+const yaml = require('yaml');
+
+/**
+ * Renders a single “leaf” field (scalar or array) at the given indentation.
+ * If `field.default` is present, prints that. Otherwise prints an empty-string
+ * or empty-array plus an inline comment (# No default (optional/required)).
+ *
+ * @param {Object}   field       – one field object from “children”
+ * @param {number}   indentLevel – number of spaces to indent
+ * @returns {string}             – one line, including comment if needed
+ */
+module.exports = function renderLeafField(field, indentLevel) {
+  if (!field || typeof field !== 'object') {
+    throw new Error('renderLeafField: field must be an object');
+  }
+  if (typeof indentLevel !== 'number' || indentLevel < 0) {
+    throw new Error('renderLeafField: indentLevel must be a non-negative number');
+  }
+  if (!field.name || typeof field.name !== 'string') {
+    throw new Error('renderLeafField: field.name must be a non-empty string');
+  }
+
+  const indent = ' '.repeat(indentLevel);
+  const name = field.name;
+
+  // Decide whether optional or required
+  const optional = Boolean(field.is_optional);
+  const comment = optional
+    ? '# No default (optional)'
+    : '# No default (required)';
+
+  // If a default is provided, use it:
+  if (field.default !== undefined) {
+    // If default is itself an object or array → dump as YAML block
+    if (typeof field.default === 'object') {
+        try {
+          // Turn the object/array into a YAML string. We also need to indent that block
+          const rawYaml = yaml.stringify(field.default).trim();
+          // Indent each line of rawYaml by (indentLevel + 2) spaces:
+          const indentedYaml = rawYaml
+          .split('\n')
+          .map(line => ' '.repeat(indentLevel + 2) + line)
+          .join('\n');
+          return `${indent}${name}:\n${indentedYaml}`;
+        } catch (error) {
+          console.warn(`Failed to serialize default value for field ${field.name}:`, error);
+          return `${indent}${name}: {} # Error serializing default value`;
+        }
+    }
+
+    // Otherwise, default is a primitive (string/number/bool)
+    if (field.type === 'string') {
+      return `${indent}${name}: ${yaml.stringify(field.default)}`;
+    }
+    return `${indent}${name}: ${field.default}`;
+  }
+
+  // No default → choose representation based on kind
+  if (field.kind === 'array') {
+    return `${indent}${name}: [] ${comment}`;
+  } else {
+    return `${indent}${name}: "" ${comment}`;
+  }
+}

package/tools/redpanda-connect/helpers/renderObjectField.js

@@ -0,0 +1,41 @@
+const renderLeafField = require('./renderLeafField');
+
+/**
+ * Recursively renders an object‐typed field ( has .children[]) at the given indentation.
+ * Skips any sub-field with is_deprecated=true. Even if the parent has no default,
+ * the parent node itself is printed without a value (just a “key:”), then children.
+ *
+ * @param {Object}   field       – one field object of type==="object"
+ * @param {number}   indentLevel – number of spaces to indent the “key:”
+ * @returns {string[]}           – an array of lines (parent + children)
+ */
+module.exports = function renderObjectField(field, indentLevel) {
+  if (!field || !field.name || !Array.isArray(field.children)) {
+    throw new Error('renderObjectField requires a field object with name and children array');
+  }
+  if (typeof indentLevel !== 'number' || indentLevel < 0) {
+    throw new Error('indentLevel must be a non-negative number');
+  }
+  const lines = [];
+  const indent = ' '.repeat(indentLevel);
+
+  // Print the parent heading (no default comment here, because parent is just a namespace)
+  lines.push(`${indent}${field.name}:`);
+
+  // Recurse into children at indentLevel + 2
+  const childIndent = indentLevel + 2;
+  field.children.forEach(child => {
+    if (child.is_deprecated) {
+      return; // skip entirely
+    }
+    if (Array.isArray(child.children) && child.children.length > 0) {
+      // Nested object → recurse
+      lines.push(...renderObjectField(child, childIndent));
+    } else {
+      // Leaf → render via renderLeafField
+      lines.push(renderLeafField(child, childIndent));
+    }
+  });
+
+  return lines;
+}

package/tools/redpanda-connect/helpers/renderYamlList.js

@@ -0,0 +1,24 @@
+const yaml = require('yaml');
+
+/**
+ * Renders a list of objects or scalar items as a YAML list
+ *
+ * @param {string} name - The field name to use as the YAML key.
+ * @param {Array<Object|string|Array>} exampleGroups - An array of example groups.
+ * @returns {string} The rendered YAML list string.
+ */
+module.exports = function renderYamlList(name, exampleGroups) {
+  let out = `${name}:\n`;
+  exampleGroups.forEach(group => {
+    const items = Array.isArray(group) ? group : [group];
+    items.forEach(item => {
+      const snippet = yaml.stringify(item).trim();
+      const lines = snippet.split('\n');
+      out += lines
+        .map((line, idx) => (idx === 0 ? `  - ${line}` : `    ${line}`))
+        .join('\n') + '\n';
+    });
+    out += '\n';
+  });
+  return out;
+}

package/tools/redpanda-connect/helpers/toYaml.js

@@ -0,0 +1,11 @@
+const yaml = require('yaml');
+	
+/**
+ * Converts an object to a YAML string.
+ *
+ * @param {Object} obj - The object to convert.
+ * @returns {string} The YAML representation of the object, trimmed.
+ */
+module.exports = function toYaml(obj) {
+  return yaml.stringify(obj).trim();
+}

package/tools/redpanda-connect/helpers/uppercase.js

@@ -0,0 +1,9 @@
+/**
+ * Converts a string to uppercase.
+ *
+ * @param {string} str - The string to convert.
+ * @returns {string} The uppercase version of the input string.
+ */
+module.exports = function uppercase(str) {
+  return String(str).toUpperCase();
+}

package/tools/redpanda-connect/parse-csv-connectors.js

@@ -0,0 +1,63 @@
+const fs = require('fs');
+const path = require('path');
+const os = require('os');
+const Papa = require('papaparse');
+const fetchFromGithub = require('../fetch-from-github.js');
+
+const CSV_PATH = 'internal/plugins/info.csv';
+const GITHUB = { owner: 'redpanda-data', repo: 'connect', remotePath: CSV_PATH };
+
+async function parseCSVConnectors(localCsvPath, logger) {
+  logger = logger || console;
+  let csvText;
+
+  try {
+    if (localCsvPath && fs.existsSync(localCsvPath) && path.extname(localCsvPath) === '.csv') {
+      logger.info(`📄 Loading CSV from local file: ${localCsvPath}`);
+      csvText = fs.readFileSync(localCsvPath, 'utf8');
+    } else {
+      const tmpDir = path.join(os.tmpdir(), 'redpanda-connect-csv');
+      await fetchFromGithub(GITHUB.owner, GITHUB.repo, GITHUB.remotePath, tmpDir);
+      const downloaded = path.join(tmpDir, path.basename(GITHUB.remotePath));
+      if (!fs.existsSync(downloaded)) {
+        throw new Error(`Expected CSV at ${downloaded} but did not find it`);
+      }
+      logger.info(`📥 Loaded CSV from GitHub into: ${downloaded}`);
+      csvText = fs.readFileSync(downloaded, 'utf8');
+      fs.rmSync(tmpDir, { recursive: true, force: true });
+    }
+
+    const parsed = Papa.parse(csvText, {
+      header: true,
+      skipEmptyLines: true,
+      transformHeader: h => h.trim()
+    });
+
+    if (!parsed.meta.fields.includes('name') || !parsed.meta.fields.includes('type')) {
+      throw new Error('CSV is missing required headers: name and type');
+    }
+
+    const cleaned = parsed.data
+      .map(row => {
+        const trimmed = Object.fromEntries(
+          Object.entries(row).map(([k, v]) => [k.trim(), (v || '').trim()])
+        );
+
+        if (!trimmed.name || !trimmed.type) return null;
+
+        return {
+          name: trimmed.name,
+          type: trimmed.type,
+          is_cloud_supported: (trimmed.cloud || '').toLowerCase() === 'y' ? 'y' : 'n'
+        };
+      })
+      .filter(Boolean);
+
+    logger.info(`✅ Parsed ${cleaned.length} connector records.`);
+    return cleaned;
+  } catch (err) {
+    throw new Error(`CSV parsing failed: ${err.message}`);
+  }
+}
+
+module.exports = parseCSVConnectors;

package/tools/redpanda-connect/report-delta.js

@@ -0,0 +1,152 @@
+// tools/redpanda-connect/report-delta.js
+'use strict';
+
+const fs = require('fs');
+const path = require('path');
+const yaml = require('yaml');
+const { execSync } = require('child_process');
+
+function discoverComponentKeys(obj) {
+  return Object.keys(obj).filter(key => Array.isArray(obj[key]));
+}
+
+function buildComponentMap(indexObj) {
+  const map = {};
+  const types = discoverComponentKeys(indexObj);
+
+  types.forEach(type => {
+    (indexObj[type] || []).forEach(component => {
+      const name = component.name;
+      if (!name) return;
+
+      const lookupKey = `${type}:${name}`;
+      let childArray = [];
+
+      if (type === 'config') {
+        if (Array.isArray(component.children)) {
+          childArray = component.children;
+        }
+      } else {
+        if (component.config && Array.isArray(component.config.children)) {
+          childArray = component.config.children;
+        }
+      }
+
+      const fieldNames = childArray.map(f => f.name);
+      map[lookupKey] = { raw: component, fields: fieldNames };
+    });
+  });
+
+  return map;
+}
+
+function getRpkConnectVersion() {
+  try {
+    // Make sure the connect plugin is upgraded first (silent)
+    execSync('rpk connect upgrade', { stdio: 'ignore' });
+
+    // Now capture the --version output
+    const raw = execSync('rpk connect --version', {
+      stdio: ['ignore', 'pipe', 'ignore'],
+    })
+      .toString()
+      .trim();
+
+    // raw looks like:
+    //   Version: 4.53.0
+    //   Date:    2025-04-18T17:49:53Z
+    // We want to extract “4.53.0”
+    const match = raw.match(/^Version:\s*(.+)$/m);
+    if (!match) {
+      throw new Error(`Unexpected format from "rpk connect --version":\n${raw}`);
+    }
+    return match[1];
+  } catch (err) {
+    throw new Error(`Unable to run "rpk connect --version": ${err.message}`);
+  }
+}
+
+/**
+ * Given two “index objects” (parsed from connect.json), produce a console summary of:
+ *  • which connectors/components are brand-new
+ *  • which new fields appeared under existing connectors (including “config” entries)
+ *  • for each new component/field, if the raw object contains “version” or “introducedInVersion” or “requiresVersion” metadata, print it
+ */
+function printDeltaReport(oldIndex, newIndex) {
+  const oldMap = buildComponentMap(oldIndex);
+  const newMap = buildComponentMap(newIndex);
+
+  // 1) brand-new components
+  const newComponentKeys = Object.keys(newMap).filter(k => !(k in oldMap));
+
+  // 2) brand-new fields under shared components
+  const newFields = [];
+  Object.keys(newMap).forEach(cKey => {
+    if (!(cKey in oldMap)) return; // skip brand-new components here
+    const oldFields = new Set(oldMap[cKey].fields || []);
+    const newFieldsArr = newMap[cKey].fields || [];
+    newFieldsArr.forEach(fName => {
+      if (!oldFields.has(fName)) {
+        // fetch raw field metadata if available
+        const [type, compName] = cKey.split(':');
+        let rawFieldObj = null;
+        if (type === 'config') {
+          rawFieldObj = (newMap[cKey].raw.children || []).find(f => f.name === fName);
+        } else {
+          rawFieldObj = (newMap[cKey].raw.config?.children || []).find(f => f.name === fName);
+        }
+
+        let introducedIn = rawFieldObj && (rawFieldObj.introducedInVersion || rawFieldObj.version);
+        let requiresVer = rawFieldObj && rawFieldObj.requiresVersion;
+
+        newFields.push({
+          component: cKey,
+          field: fName,
+          introducedIn,
+          requiresVersion: requiresVer,
+        });
+      }
+    });
+  });
+
+  console.log('\n📋 RPCN Connector Delta Report\n');
+
+  if (newComponentKeys.length) {
+    console.log('➤ Newly added components:');
+    newComponentKeys.forEach(key => {
+      const [type, name] = key.split(':');
+      const raw = newMap[key].raw;
+      const status = raw.status || raw.type || '';
+      const version = raw.version || raw.introducedInVersion || '';
+      console.log(
+        `   • ${type}/${name}${
+          status ? ` (${status})` : ''
+        }${version ? ` — introduced in ${version}` : ''}`
+      );
+    });
+    console.log('');
+  } else {
+    console.log('➤ No newly added components.\n');
+  }
+
+  if (newFields.length) {
+    console.log('➤ Newly added fields:');
+    newFields.forEach(entry => {
+      const { component, field, introducedIn, requiresVersion } = entry;
+      process.stdout.write(`   • ${component} → ${field}`);
+      if (introducedIn) process.stdout.write(` (introducedIn: ${introducedIn})`);
+      if (requiresVersion) process.stdout.write(` (requiresVersion: ${requiresVersion})`);
+      console.log('');
+    });
+    console.log('');
+  } else {
+    console.log('➤ No newly added fields.\n');
+  }
+}
+
+module.exports = {
+  discoverComponentKeys,
+  buildComponentMap,
+  getRpkConnectVersion,
+  printDeltaReport,
+};

package/tools/redpanda-connect/templates/connector.hbs

@@ -0,0 +1,20 @@
+= {{{name}}}
+// tag::single-source[]
+:type: {{type}}
+:status: {{status}}
+:categories: [{{join categories ", "}}]
+:description: {{#if summary}}{{{summary}}}{{else if description}}{{description}}{{else}}TODO: Missing description. Add a description in the overrides file or source data.{{/if}}
+
+component_type_dropdown::[]
+
+{{> intro}}
+
+{{#if (or this.config.children children)}}
+include::redpanda-connect:components:partial$fields/{{type}}s/{{name}}.adoc[]
+{{/if}}
+
+{{#if this.examples}}
+include::redpanda-connect:components:partial$examples/{{type}}s/{{name}}.adoc[]
+{{/if}}
+
+// end::single-source[]

package/tools/redpanda-connect/templates/examples-partials.hbs

@@ -0,0 +1,7 @@
+{{#with examples}}
+// This content is autogenerated. Do not edit manually.
+
+== Examples
+
+{{{renderConnectExamples this}}}
+{{/with}}

package/tools/redpanda-connect/templates/fields-partials.hbs

@@ -0,0 +1,13 @@
+{{#if this.config.children}}
+// This content is autogenerated. Do not edit manually. To override descriptions, use the doc-tools CLI with the --overrides option: https://redpandadata.atlassian.net/wiki/spaces/DOC/pages/1247543314/Generate+reference+docs+for+Redpanda+Connect
+
+== Fields
+
+{{{renderConnectFields this.config.children}}}
+{{else if this.children}}
+// This content is autogenerated. Do not edit manually. To override descriptions, use the doc-tools CLI with the --overrides option: https://redpandadata.atlassian.net/wiki/spaces/DOC/pages/1247543314/Generate+reference+docs+for+Redpanda+Connect
+
+== Fields
+
+{{{renderConnectFields this.children}}}
+{{/if}}

package/tools/redpanda-connect/templates/intro.hbs

@@ -0,0 +1,35 @@
+// This content is autogenerated. Do not edit manually. To override descriptions or summaries, use the doc-tools CLI with the --overrides option: https://redpandadata.atlassian.net/wiki/spaces/DOC/pages/1247543314/Generate+reference+docs+for+Redpanda+Connect
+
+{{{summary}}}
+
+{{#if version}}
+ifndef::env-cloud[]
+Introduced in version {{version}}.
+endif::[]
+{{/if}}
+
+{{#if this.config.children}}
+[tabs]
+======
+Common::
++
+--
+```yaml
+{{{commonConfig this.type this.name this.config.children}}}
+```
+--
+Advanced::
++
+--
+```yaml
+{{{advancedConfig this.type this.name this.config.children}}}
+```
+--
+======
+{{/if}}
+
+{{#if description}}
+{{{description}}}
+{{else}}
+TODO: Missing description. Add a description in the overrides file or source data.
+{{/if}}