@redpanda-data/docs-extensions-and-macros 4.5.0 → 4.6.1

package/tools/redpanda-connect/helpers/renderObjectField.js

@@ -0,0 +1,41 @@
+const renderLeafField = require('./renderLeafField');
+
+/**
+ * Recursively renders an object‐typed field ( has .children[]) at the given indentation.
+ * Skips any sub-field with is_deprecated=true. Even if the parent has no default,
+ * the parent node itself is printed without a value (just a “key:”), then children.
+ *
+ * @param {Object}   field       – one field object of type==="object"
+ * @param {number}   indentLevel – number of spaces to indent the “key:”
+ * @returns {string[]}           – an array of lines (parent + children)
+ */
+module.exports = function renderObjectField(field, indentLevel) {
+  if (!field || !field.name || !Array.isArray(field.children)) {
+    throw new Error('renderObjectField requires a field object with name and children array');
+  }
+  if (typeof indentLevel !== 'number' || indentLevel < 0) {
+    throw new Error('indentLevel must be a non-negative number');
+  }
+  const lines = [];
+  const indent = ' '.repeat(indentLevel);
+
+  // Print the parent heading (no default comment here, because parent is just a namespace)
+  lines.push(`${indent}${field.name}:`);
+
+  // Recurse into children at indentLevel + 2
+  const childIndent = indentLevel + 2;
+  field.children.forEach(child => {
+    if (child.is_deprecated) {
+      return; // skip entirely
+    }
+    if (Array.isArray(child.children) && child.children.length > 0) {
+      // Nested object → recurse
+      lines.push(...renderObjectField(child, childIndent));
+    } else {
+      // Leaf → render via renderLeafField
+      lines.push(renderLeafField(child, childIndent));
+    }
+  });
+
+  return lines;
+}

package/tools/redpanda-connect/helpers/renderYamlList.js

@@ -0,0 +1,24 @@
+const yaml = require('yaml');
+
+/**
+ * Renders a list of objects or scalar items as a YAML list
+ *
+ * @param {string} name - The field name to use as the YAML key.
+ * @param {Array<Object|string|Array>} exampleGroups - An array of example groups.
+ * @returns {string} The rendered YAML list string.
+ */
+module.exports = function renderYamlList(name, exampleGroups) {
+  let out = `${name}:\n`;
+  exampleGroups.forEach(group => {
+    const items = Array.isArray(group) ? group : [group];
+    items.forEach(item => {
+      const snippet = yaml.stringify(item).trim();
+      const lines = snippet.split('\n');
+      out += lines
+        .map((line, idx) => (idx === 0 ? `  - ${line}` : `    ${line}`))
+        .join('\n') + '\n';
+    });
+    out += '\n';
+  });
+  return out;
+}

package/tools/redpanda-connect/helpers/toYaml.js

@@ -0,0 +1,11 @@
+const yaml = require('yaml');
+	
+/**
+ * Converts an object to a YAML string.
+ *
+ * @param {Object} obj - The object to convert.
+ * @returns {string} The YAML representation of the object, trimmed.
+ */
+module.exports = function toYaml(obj) {
+  return yaml.stringify(obj).trim();
+}

package/tools/redpanda-connect/helpers/uppercase.js

@@ -0,0 +1,9 @@
+/**
+ * Converts a string to uppercase.
+ *
+ * @param {string} str - The string to convert.
+ * @returns {string} The uppercase version of the input string.
+ */
+module.exports = function uppercase(str) {
+  return String(str).toUpperCase();
+}

package/tools/redpanda-connect/parse-csv-connectors.js

@@ -0,0 +1,63 @@
+const fs = require('fs');
+const path = require('path');
+const os = require('os');
+const Papa = require('papaparse');
+const fetchFromGithub = require('../fetch-from-github.js');
+
+const CSV_PATH = 'internal/plugins/info.csv';
+const GITHUB = { owner: 'redpanda-data', repo: 'connect', remotePath: CSV_PATH };
+
+async function parseCSVConnectors(localCsvPath, logger) {
+  logger = logger || console;
+  let csvText;
+
+  try {
+    if (localCsvPath && fs.existsSync(localCsvPath) && path.extname(localCsvPath) === '.csv') {
+      logger.info(`📄 Loading CSV from local file: ${localCsvPath}`);
+      csvText = fs.readFileSync(localCsvPath, 'utf8');
+    } else {
+      const tmpDir = path.join(os.tmpdir(), 'redpanda-connect-csv');
+      await fetchFromGithub(GITHUB.owner, GITHUB.repo, GITHUB.remotePath, tmpDir);
+      const downloaded = path.join(tmpDir, path.basename(GITHUB.remotePath));
+      if (!fs.existsSync(downloaded)) {
+        throw new Error(`Expected CSV at ${downloaded} but did not find it`);
+      }
+      logger.info(`📥 Loaded CSV from GitHub into: ${downloaded}`);
+      csvText = fs.readFileSync(downloaded, 'utf8');
+      fs.rmSync(tmpDir, { recursive: true, force: true });
+    }
+
+    const parsed = Papa.parse(csvText, {
+      header: true,
+      skipEmptyLines: true,
+      transformHeader: h => h.trim()
+    });
+
+    if (!parsed.meta.fields.includes('name') || !parsed.meta.fields.includes('type')) {
+      throw new Error('CSV is missing required headers: name and type');
+    }
+
+    const cleaned = parsed.data
+      .map(row => {
+        const trimmed = Object.fromEntries(
+          Object.entries(row).map(([k, v]) => [k.trim(), (v || '').trim()])
+        );
+
+        if (!trimmed.name || !trimmed.type) return null;
+
+        return {
+          name: trimmed.name,
+          type: trimmed.type,
+          is_cloud_supported: (trimmed.cloud || '').toLowerCase() === 'y' ? 'y' : 'n'
+        };
+      })
+      .filter(Boolean);
+
+    logger.info(`✅ Parsed ${cleaned.length} connector records.`);
+    return cleaned;
+  } catch (err) {
+    throw new Error(`CSV parsing failed: ${err.message}`);
+  }
+}
+
+module.exports = parseCSVConnectors;

package/tools/redpanda-connect/report-delta.js

@@ -0,0 +1,152 @@
+// tools/redpanda-connect/report-delta.js
+'use strict';
+
+const fs = require('fs');
+const path = require('path');
+const yaml = require('yaml');
+const { execSync } = require('child_process');
+
+function discoverComponentKeys(obj) {
+  return Object.keys(obj).filter(key => Array.isArray(obj[key]));
+}
+
+function buildComponentMap(indexObj) {
+  const map = {};
+  const types = discoverComponentKeys(indexObj);
+
+  types.forEach(type => {
+    (indexObj[type] || []).forEach(component => {
+      const name = component.name;
+      if (!name) return;
+
+      const lookupKey = `${type}:${name}`;
+      let childArray = [];
+
+      if (type === 'config') {
+        if (Array.isArray(component.children)) {
+          childArray = component.children;
+        }
+      } else {
+        if (component.config && Array.isArray(component.config.children)) {
+          childArray = component.config.children;
+        }
+      }
+
+      const fieldNames = childArray.map(f => f.name);
+      map[lookupKey] = { raw: component, fields: fieldNames };
+    });
+  });
+
+  return map;
+}
+
+function getRpkConnectVersion() {
+  try {
+    // Make sure the connect plugin is upgraded first (silent)
+    execSync('rpk connect upgrade', { stdio: 'ignore' });
+
+    // Now capture the --version output
+    const raw = execSync('rpk connect --version', {
+      stdio: ['ignore', 'pipe', 'ignore'],
+    })
+      .toString()
+      .trim();
+
+    // raw looks like:
+    //   Version: 4.53.0
+    //   Date:    2025-04-18T17:49:53Z
+    // We want to extract “4.53.0”
+    const match = raw.match(/^Version:\s*(.+)$/m);
+    if (!match) {
+      throw new Error(`Unexpected format from "rpk connect --version":\n${raw}`);
+    }
+    return match[1];
+  } catch (err) {
+    throw new Error(`Unable to run "rpk connect --version": ${err.message}`);
+  }
+}
+
+/**
+ * Given two “index objects” (parsed from connect.json), produce a console summary of:
+ *  • which connectors/components are brand-new
+ *  • which new fields appeared under existing connectors (including “config” entries)
+ *  • for each new component/field, if the raw object contains “version” or “introducedInVersion” or “requiresVersion” metadata, print it
+ */
+function printDeltaReport(oldIndex, newIndex) {
+  const oldMap = buildComponentMap(oldIndex);
+  const newMap = buildComponentMap(newIndex);
+
+  // 1) brand-new components
+  const newComponentKeys = Object.keys(newMap).filter(k => !(k in oldMap));
+
+  // 2) brand-new fields under shared components
+  const newFields = [];
+  Object.keys(newMap).forEach(cKey => {
+    if (!(cKey in oldMap)) return; // skip brand-new components here
+    const oldFields = new Set(oldMap[cKey].fields || []);
+    const newFieldsArr = newMap[cKey].fields || [];
+    newFieldsArr.forEach(fName => {
+      if (!oldFields.has(fName)) {
+        // fetch raw field metadata if available
+        const [type, compName] = cKey.split(':');
+        let rawFieldObj = null;
+        if (type === 'config') {
+          rawFieldObj = (newMap[cKey].raw.children || []).find(f => f.name === fName);
+        } else {
+          rawFieldObj = (newMap[cKey].raw.config?.children || []).find(f => f.name === fName);
+        }
+
+        let introducedIn = rawFieldObj && (rawFieldObj.introducedInVersion || rawFieldObj.version);
+        let requiresVer = rawFieldObj && rawFieldObj.requiresVersion;
+
+        newFields.push({
+          component: cKey,
+          field: fName,
+          introducedIn,
+          requiresVersion: requiresVer,
+        });
+      }
+    });
+  });
+
+  console.log('\n📋 RPCN Connector Delta Report\n');
+
+  if (newComponentKeys.length) {
+    console.log('➤ Newly added components:');
+    newComponentKeys.forEach(key => {
+      const [type, name] = key.split(':');
+      const raw = newMap[key].raw;
+      const status = raw.status || raw.type || '';
+      const version = raw.version || raw.introducedInVersion || '';
+      console.log(
+        `   • ${type}/${name}${
+          status ? ` (${status})` : ''
+        }${version ? ` — introduced in ${version}` : ''}`
+      );
+    });
+    console.log('');
+  } else {
+    console.log('➤ No newly added components.\n');
+  }
+
+  if (newFields.length) {
+    console.log('➤ Newly added fields:');
+    newFields.forEach(entry => {
+      const { component, field, introducedIn, requiresVersion } = entry;
+      process.stdout.write(`   • ${component} → ${field}`);
+      if (introducedIn) process.stdout.write(` (introducedIn: ${introducedIn})`);
+      if (requiresVersion) process.stdout.write(` (requiresVersion: ${requiresVersion})`);
+      console.log('');
+    });
+    console.log('');
+  } else {
+    console.log('➤ No newly added fields.\n');
+  }
+}
+
+module.exports = {
+  discoverComponentKeys,
+  buildComponentMap,
+  getRpkConnectVersion,
+  printDeltaReport,
+};

package/tools/redpanda-connect/templates/connector.hbs

@@ -0,0 +1,20 @@
+= {{{name}}}
+// tag::single-source[]
+:type: {{type}}
+:status: {{status}}
+:categories: [{{join categories ", "}}]
+:description: {{#if summary}}{{{summary}}}{{else if description}}{{description}}{{else}}TODO: Missing description. Add a description in the overrides file or source data.{{/if}}
+
+component_type_dropdown::[]
+
+{{> intro}}
+
+{{#if (or this.config.children children)}}
+include::redpanda-connect:components:partial$fields/{{type}}s/{{name}}.adoc[]
+{{/if}}
+
+{{#if this.examples}}
+include::redpanda-connect:components:partial$examples/{{type}}s/{{name}}.adoc[]
+{{/if}}
+
+// end::single-source[]

package/tools/redpanda-connect/templates/examples-partials.hbs

@@ -0,0 +1,7 @@
+{{#with examples}}
+// This content is autogenerated. Do not edit manually.
+
+== Examples
+
+{{{renderConnectExamples this}}}
+{{/with}}

package/tools/redpanda-connect/templates/fields-partials.hbs

@@ -0,0 +1,13 @@
+{{#if this.config.children}}
+// This content is autogenerated. Do not edit manually. To override descriptions, use the doc-tools CLI with the --overrides option: https://redpandadata.atlassian.net/wiki/spaces/DOC/pages/1247543314/Generate+reference+docs+for+Redpanda+Connect
+
+== Fields
+
+{{{renderConnectFields this.config.children}}}
+{{else if this.children}}
+// This content is autogenerated. Do not edit manually. To override descriptions, use the doc-tools CLI with the --overrides option: https://redpandadata.atlassian.net/wiki/spaces/DOC/pages/1247543314/Generate+reference+docs+for+Redpanda+Connect
+
+== Fields
+
+{{{renderConnectFields this.children}}}
+{{/if}}

package/tools/redpanda-connect/templates/intro.hbs

@@ -0,0 +1,33 @@
+{{{summary}}}
+
+{{#if version}}
+ifndef::env-cloud[]
+Introduced in version {{version}}.
+endif::[]
+{{/if}}
+
+{{#if this.config.children}}
+[tabs]
+======
+Common::
++
+--
+```yaml
+{{{commonConfig this.type this.name this.config.children}}}
+```
+--
+Advanced::
++
+--
+```yaml
+{{{advancedConfig this.type this.name this.config.children}}}
+```
+--
+======
+{{/if}}
+
+{{#if description}}
+{{{description}}}
+{{else}}
+TODO: Missing description. Add a description in the overrides file or source data.
+{{/if}}