@redpanda-data/docs-extensions-and-macros 4.5.0 → 4.6.1

package/tools/redpanda-connect/generate-rpcn-connector-docs.js

@@ -0,0 +1,233 @@
+'use strict';
+
+const fs = require('fs');
+const path = require('path');
+const handlebars = require('handlebars');
+const yaml = require('yaml');
+const helpers = require('./helpers');
+
+// Register each helper under handlebars, verifying that it’s a function
+Object.entries(helpers).forEach(([name, fn]) => {
+  if (typeof fn !== 'function') {
+    console.error(`❌ Helper "${name}" is not a function`);
+    process.exit(1);
+  }
+  handlebars.registerHelper(name, fn);
+});
+
+// Default “main” template (connector.hbs) which invokes partials {{> intro}}, {{> fields}}, {{> examples}}
+const DEFAULT_TEMPLATE = path.resolve(__dirname, './templates/connector.hbs');
+
+/**
+ * Reads a file at `filePath` and registers it as a Handlebars partial called `name`.
+ * Throws if the file cannot be read.
+ */
+function registerPartial(name, filePath) {
+  const resolved = path.resolve(filePath);
+  let source;
+  try {
+    source = fs.readFileSync(resolved, 'utf8');
+  } catch (err) {
+    throw new Error(`Unable to read "${name}" template at ${resolved}: ${err.message}`);
+  }
+  handlebars.registerPartial(name, source);
+}
+
+/**
+ * Deep-merge `overrides` into `target`. Only 'description', 'type',
+ * plus nested array/object entries get overridden; other keys remain intact.
+ */
+function mergeOverrides(target, overrides) {
+  if (!overrides || typeof overrides !== 'object') return target;
+  if (!target || typeof target !== 'object') {
+    throw new Error('Target must be a valid object');
+  }
+  for (const key in overrides) {
+    if (Array.isArray(target[key]) && Array.isArray(overrides[key])) {
+      // Merge two parallel arrays by matching items on `.name`
+      target[key] = target[key].map(item => {
+        const overrideItem = overrides[key].find(o => o.name === item.name);
+        if (overrideItem) {
+          // Overwrite description/type if present
+          ['description', 'type'].forEach(field => {
+            if (Object.hasOwn(overrideItem, field)) {
+              item[field] = overrideItem[field];
+            }
+          });
+          // Copy through selfManagedOnly flag
+          if (Object.hasOwn(overrideItem, 'selfManagedOnly')) {
+            item.selfManagedOnly = overrideItem.selfManagedOnly;
+          }
+          // Recurse for nested children
+          item = mergeOverrides(item, overrideItem);
+        }
+        return item;
+      });
+    } else if (
+      typeof target[key] === 'object' &&
+      typeof overrides[key] === 'object' &&
+      !Array.isArray(target[key]) &&
+      !Array.isArray(overrides[key])
+    ) {
+      // Deep-merge plain objects
+      target[key] = mergeOverrides(target[key], overrides[key]);
+    } else if (['description', 'type'].includes(key) && Object.hasOwn(overrides, key)) {
+      // Overwrite the primitive
+      target[key] = overrides[key];
+    }
+  }
+  return target;
+}
+
+/**
+ * Generates documentation files for RPCN connectors using Handlebars templates.
+ *
+ * Depending on the {@link writeFullDrafts} flag, generates either partial documentation files for connector fields and examples, or full draft documentation for each connector component. Supports merging override data and skips draft generation for components marked as deprecated.
+ *
+ * @param {Object} options - Configuration options for documentation generation.
+ * @param {string} options.data - Path to the connector data file (JSON or YAML).
+ * @param {string} [options.overrides] - Optional path to a JSON file with override data.
+ * @param {string} options.template - Path to the main Handlebars template.
+ * @param {string} [options.templateIntro] - Path to the intro partial template (used in full draft mode).
+ * @param {string} [options.templateFields] - Path to the fields partial template.
+ * @param {string} [options.templateExamples] - Path to the examples partial template.
+ * @param {boolean} options.writeFullDrafts - If true, generates full draft documentation; otherwise, generates partials.
+ * @returns {Promise<Object>} An object summarizing the number and paths of generated partials and drafts.
+ *
+ * @throws {Error} If reading or parsing input files fails, or if template rendering fails for a component.
+ *
+ * @remark
+ * When generating full drafts, components with a `status` of `'deprecated'` are skipped.
+ */
+async function generateRpcnConnectorDocs(options) {
+  const {
+    data,
+    overrides,
+    template,            // main Handlebars template (for full-draft mode)
+    templateIntro,
+    templateFields,
+    templateExamples,
+    writeFullDrafts
+  } = options;
+
+  // Read connector index (JSON or YAML)
+  const raw = fs.readFileSync(data, 'utf8');
+  const ext = path.extname(data).toLowerCase();
+  const dataObj = ext === '.json' ? JSON.parse(raw) : yaml.parse(raw);
+
+  // Apply overrides if provided
+  if (overrides) {
+    const ovRaw = fs.readFileSync(overrides, 'utf8');
+    const ovObj = JSON.parse(ovRaw);
+    mergeOverrides(dataObj, ovObj);
+  }
+
+  // Compile the “main” template (used when writeFullDrafts = true)
+  const compiledTemplate = handlebars.compile(fs.readFileSync(template, 'utf8'));
+
+  // Determine which templates to use for “fields” and “examples”
+  // If templateFields is not provided, fall back to the single `template`.
+  // If templateExamples is not provided, skip examples entirely.
+  const fieldsTemplatePath   = templateFields   || template;
+  const examplesTemplatePath = templateExamples || null;
+
+  // Register partials
+  if (!writeFullDrafts) {
+    if (fieldsTemplatePath) {
+      registerPartial('fields', fieldsTemplatePath);
+    }
+    if (examplesTemplatePath) {
+      registerPartial('examples', examplesTemplatePath);
+    }
+  } else {
+    registerPartial('intro', templateIntro);
+  }
+
+  const outputRoot     = path.resolve(process.cwd(), 'modules/components/partials');
+  const fieldsOutRoot   = path.join(outputRoot, 'fields');
+  const examplesOutRoot = path.join(outputRoot, 'examples');
+  const draftsRoot      = path.join(outputRoot, 'drafts');
+
+  if (!writeFullDrafts) {
+    fs.mkdirSync(fieldsOutRoot,   { recursive: true });
+    fs.mkdirSync(examplesOutRoot, { recursive: true });
+  }
+
+  let partialsWritten = 0;
+  let draftsWritten   = 0;
+  const partialFiles  = [];
+  const draftFiles    = [];
+
+  for (const [type, items] of Object.entries(dataObj)) {
+    if (!Array.isArray(items)) continue;
+
+    for (const item of items) {
+      if (!item.name) continue;
+      const name = item.name;
+
+      if (!writeFullDrafts) {
+        // Render fields using the registered “fields” partial
+        const fieldsOut = handlebars
+          .compile('{{> fields children=config.children}}')(item);
+
+        // Render examples only if an examples template was provided
+        let examplesOut = '';
+        if (examplesTemplatePath) {
+          examplesOut = handlebars
+            .compile('{{> examples examples=examples}}')(item);
+        }
+
+        if (fieldsOut.trim()) {
+          const fPath = path.join(fieldsOutRoot, type, `${name}.adoc`);
+          fs.mkdirSync(path.dirname(fPath), { recursive: true });
+          fs.writeFileSync(fPath, fieldsOut);
+          partialsWritten++;
+          partialFiles.push(path.relative(process.cwd(), fPath));
+        }
+
+        if (examplesOut.trim()) {
+          const ePath = path.join(examplesOutRoot, type, `${name}.adoc`);
+          fs.mkdirSync(path.dirname(ePath), { recursive: true });
+          fs.writeFileSync(ePath, examplesOut);
+          partialsWritten++;
+          partialFiles.push(path.relative(process.cwd(), ePath));
+        }
+      }
+
+      if (writeFullDrafts) {
+        if (String(item.status || '').toLowerCase() === 'deprecated') {
+          console.log(`Skipping draft for deprecated component: ${type}/${name}`);
+          continue;
+        }
+        let content;
+        try {
+          content = compiledTemplate(item);
+        } catch (err) {
+          throw new Error(`Template render failed for component "${name}": ${err.message}`);
+        }
+
+        const draftSubdir = name === 'gateway'
+          ? path.join(draftsRoot, 'cloud-only')
+          : draftsRoot;
+
+        const destFile = path.join(draftSubdir, `${name}.adoc`);
+        fs.mkdirSync(path.dirname(destFile), { recursive: true });
+        fs.writeFileSync(destFile, content, 'utf8');
+        draftsWritten++;
+        draftFiles.push(path.relative(process.cwd(), destFile));
+      }
+    }
+  }
+
+  return {
+    partialsWritten,
+    draftsWritten,
+    partialFiles,
+    draftFiles
+  };
+}
+
+module.exports = {
+  generateRpcnConnectorDocs,
+  mergeOverrides
+};

package/tools/redpanda-connect/helpers/advancedConfig.js

@@ -0,0 +1,17 @@
+const buildConfigYaml = require('./buildConfigYaml.js');
+const handlebars = require('handlebars');
+
+/**
+ * Handlebars helper “advancedConfig”.  Omits only deprecated.
+ *
+ * Usage in template:
+ *   {{advancedConfig this.type this.name this.config.children}}
+ */
+module.exports = function advancedConfig(type, connectorName, children) {
+  if (typeof type !== 'string' || typeof connectorName !== 'string' || !Array.isArray(children)) {
+    return '';
+  }
+
+  const yamlText = buildConfigYaml(type, connectorName, children, /*includeAdvanced=*/ true);
+  return new handlebars.SafeString(yamlText);
+}

package/tools/redpanda-connect/helpers/buildConfigYaml.js

@@ -0,0 +1,53 @@
+const renderLeafField = require('./renderLeafField');
+const renderObjectField = require('./renderObjectField');
+
+/**
+ * Builds either “Common” or “Advanced” YAML for one connector.
+ *
+ * - type            = “input” or “output” (or whatever type)
+ * - connectorName   = such as “amqp_1”
+ * - children        = the array of field‐definitions (entry.config.children)
+ * - includeAdvanced = if false → only fields where is_advanced !== true
+ *                      if true  → all fields (except deprecated)
+ *
+ * Structure produced:
+ *
+ *   type:
+ *     label: ""
+ *     connectorName:
+ *       ...child fields (with comments for “no default”)
+ */
+module.exports = function buildConfigYaml(type, connectorName, children, includeAdvanced) {
+  const lines = [];
+
+  // “type:” top‐level
+  lines.push(`${type}:`);
+
+  // Two‐space indent for “label”
+  lines.push(`  label: ""`);
+
+  // Two‐space indent for connectorName heading
+  lines.push(`  ${connectorName}:`);
+
+  // Four‐space indent for children
+  const baseIndent = 4;
+  children.forEach(field => {
+    if (field.is_deprecated) {
+      return; // skip deprecated fields
+    }
+    if (!includeAdvanced && field.is_advanced) {
+      return; // skip advanced fields in “common” mode
+    }
+
+    if (field.type === 'object' && Array.isArray(field.children)) {
+      // Render nested object
+      const nestedLines = renderObjectField(field, baseIndent);
+      lines.push(...nestedLines);
+    } else {
+      // Render a scalar or array leaf
+      lines.push(renderLeafField(field, baseIndent));
+    }
+  });
+
+  return lines.join('\n');
+}

package/tools/redpanda-connect/helpers/commonConfig.js

@@ -0,0 +1,31 @@
+const buildConfigYaml = require('./buildConfigYaml.js');
+const handlebars = require('handlebars');
+
+/**
+ * Handlebars helper “commonConfig”.  Omits deprecated + advanced.
+ *
+ * Usage in template:
+ *   {{commonConfig this.type this.name this.config.children}}
+ */
+module.exports = function commonConfig(type, connectorName, children) {
+  if (typeof type !== 'string' || !type.trim()) {
+    console.warn('commonConfig: type must be a non-empty string');
+    return '';
+  }
+  if (typeof connectorName !== 'string' || !connectorName.trim()) {
+    console.warn('commonConfig: connectorName must be a non-empty string');
+    return '';
+  }
+  if (!Array.isArray(children)) {
+    console.warn('commonConfig: children must be an array');
+    return '';
+  }
+
+  try {
+    const yamlText = buildConfigYaml(type, connectorName, children, /*includeAdvanced=*/ false);
+    return new handlebars.SafeString(yamlText);
+  } catch (error) {
+    console.error('Error in commonConfig helper:', error);
+    return new handlebars.SafeString('<!-- Error generating configuration -->');
+  }
+}

package/tools/redpanda-connect/helpers/eq.js

@@ -0,0 +1,10 @@
+/**
+ * Checks if two values are equal.
+ *
+ * @param {*} a - The first value.
+ * @param {*} b - The second value.
+ * @returns {boolean} True if the values are equal.
+ */
+module.exports = function eq(a, b) {
+  return a === b;
+}

package/tools/redpanda-connect/helpers/index.js

@@ -0,0 +1,19 @@
+'use strict';
+
+module.exports = {
+  uppercase:       require('./uppercase.js'),
+  eq:              require('./eq.js'),
+  ne:              require('./ne.js'),
+  join:              require('./join.js'),
+  or:              require('./or.js'),
+  toYaml:              require('./toYaml.js'),
+  isObject:              require('./isObject.js'),
+  renderYamlList:  require('./renderYamlList.js'),
+  renderConnectFields:    require('./renderConnectFields.js'),
+  renderConnectExamples:  require('./renderConnectExamples.js'),
+  renderLeafField:        require('./renderLeafField.js'),
+  renderObjectField:      require('./renderObjectField.js'),
+  buildConfigYaml:        require('./buildConfigYaml.js'),
+  commonConfig:           require('./commonConfig.js'),
+  advancedConfig:         require('./advancedConfig.js'),
+};

package/tools/redpanda-connect/helpers/isObject.js

@@ -0,0 +1 @@
+module.exports = v => v !== null && typeof v === 'object' && !Array.isArray(v)

package/tools/redpanda-connect/helpers/join.js

@@ -0,0 +1,6 @@
+module.exports = function join(array, separator) {
+  if (!Array.isArray(array)) {
+    return '';
+  }
+  return array.join(separator);
+}

package/tools/redpanda-connect/helpers/ne.js

@@ -0,0 +1,10 @@
+/**
+ * Checks if two values are not equal.
+ *
+ * @param {*} a - The first value.
+ * @param {*} b - The second value.
+ * @returns {boolean} True if the values are not equal.
+ */
+module.exports = function ne(a, b) {
+  return a !== b;
+}

package/tools/redpanda-connect/helpers/or.js

@@ -0,0 +1,4 @@
+module.exports = function (...args) {
+    const options = args.pop(); // Last argument is always the options object
+    return args.some(Boolean);
+};

package/tools/redpanda-connect/helpers/renderConnectExamples.js

@@ -0,0 +1,37 @@
+const handlebars = require('handlebars');
+
+/**
+ * Renders a list of examples.
+ *
+ * @param {Array<Object>} examples - An array of example objects.
+ * @returns {handlebars.SafeString} The rendered SafeString containing the examples.
+ */
+module.exports = function renderConnectExamples(examples) {
+  if (!examples || !Array.isArray(examples) || examples.length === 0) {
+    return '';
+  }
+  let output = '';
+  examples.forEach(example => {
+    if (example.title) {
+      const sanitizedTitle = example.title.replace(/[=]/g, '\\=');
+      output += `=== ${sanitizedTitle}\n\n`;
+    }
+    if (example.summary) {
+      output += `${example.summary}\n\n`;
+    }
+    if (example.config) {
+      if (typeof example.config !== 'string') {
+        console.warn('Example config must be a string, skipping');
+        return;
+      }
+      const configContent = example.config.trim();
+      if (configContent.includes('----')) {
+        console.warn('Example config contains AsciiDoc delimiters, this may break rendering');
+      }
+      output += '[source,yaml]\n----\n';
+      output += configContent + '\n';
+      output += '----\n\n';
+    }
+  });
+  return new handlebars.SafeString(output);
+}

package/tools/redpanda-connect/helpers/renderConnectFields.js

@@ -0,0 +1,148 @@
+const yaml = require('yaml');
+const renderYamlList = require('./renderYamlList');
+const handlebars = require('handlebars');
+
+/**
+ * Renders the children of a configuration object into AsciiDoc.
+ *
+ * @param {Array<Object>} children - An array of child objects.
+ * @param {string} [prefix=''] - The prefix path for nested fields.
+ * @returns {handlebars.SafeString} The rendered SafeString containing the configuration details.
+ */
+module.exports = function renderConnectFields(children, prefix = '') {
+  if (!children || !Array.isArray(children) || children.length === 0) {
+    return '';
+  }
+
+  const sorted = [...children].sort((a, b) => {
+    const an = a.name || '';
+    const bn = b.name || '';
+    return an.localeCompare(bn, undefined, { sensitivity: 'base' });
+  });
+
+  let output = '';
+  prefix = typeof prefix === 'string' ? prefix : '';
+
+  sorted.forEach(child => {
+    if (child.is_deprecated || !child.name) return;
+
+    // Normalize type
+    let displayType;
+    if (child.type === 'string' && child.kind === 'array') {
+      displayType = 'array';
+    } else if (child.type === 'unknown' && child.kind === 'map') {
+      displayType = 'object';
+    } else {
+      displayType = child.type;
+    }
+
+    let block = '';
+    const isArray = child.kind === 'array';
+    const currentPath = prefix
+      ? `${prefix}.${child.name}${isArray ? '[]' : ''}`
+      : `${child.name}${isArray ? '[]' : ''}`;
+
+    block += `=== \`${currentPath}\`\n\n`;
+
+    if (child.description) {
+      block += `${child.description}\n\n`;
+    }
+    if (child.is_secret) {
+      block += `include::redpanda-connect:components:partial$secret_warning.adoc[]\n\n`;
+    }
+    if (child.version) {
+      block += `ifndef::env-cloud[]\nRequires version ${child.version} or later.\nendif::[]\n\n`;
+    }
+
+    block += `*Type*: \`${displayType}\`\n\n`;
+
+    // Default
+    if (child.type !== 'object' && child.default !== undefined) {
+      if (typeof child.default !== 'object') {
+        const display = child.default === '' ? '""' : String(child.default);
+        block += `*Default*: \`${display}\`\n\n`;
+      } else {
+        const defYaml = yaml.stringify(child.default).trim();
+        block += `*Default*:\n[source,yaml]\n----\n${defYaml}\n----\n\n`;
+      }
+    }
+
+    // Annotated options
+    if (child.annotated_options && child.annotated_options.length) {
+      block += `[cols=\"1m,2a\"]\n|===\n|Option |Summary\n\n`;
+      child.annotated_options.forEach(([opt, summary]) => {
+        block += `|${opt}\n|${summary}\n\n`;
+      });
+      block += `|===\n\n`;
+    }
+
+    // Options list
+    if (child.options && child.options.length) {
+      block += `*Options*: ${child.options.map(opt => `\`${opt}\``).join(', ')}\n\n`;
+    }
+
+    // Examples
+    if (child.examples && child.examples.length) {
+      block += `[source,yaml]\n----\n# Examples:\n`;
+      if (child.type === 'string') {
+        if (child.kind === 'array') {
+          block += renderYamlList(child.name, child.examples);
+        } else {
+          child.examples.forEach(example => {
+            if (typeof example === 'string' && example.includes('\n')) {
+              block += `${child.name}: |-\n`;
+              block += example.split('\n').map(line => '  ' + line).join('\n') + '\n';
+            } else {
+              block += `${child.name}: \`${example}\`\n`;
+            }
+          });
+          block += '\n';
+        }
+      } else if (child.type === 'processor') {
+        if (child.kind === 'array') {
+          block += renderYamlList(child.name, child.examples);
+        } else {
+          child.examples.forEach(example => {
+            block += `${child.name}: \`${String(example)}\`\n`;
+          });
+          block += '\n';
+        }
+      } else if (child.type === 'object') {
+        if (child.kind === 'array') {
+          block += renderYamlList(child.name, child.examples);
+        } else {
+          child.examples.forEach(example => {
+            if (typeof example === 'object') {
+              const snippet = yaml.stringify(example).trim();
+              block += `${child.name}:\n`;
+              block += snippet.split('\n').map(line => '  ' + line).join('\n') + '\n';
+            } else {
+              block += `${child.name}: \`${String(example)}\`\n`;
+            }
+          });
+          block += '\n';
+        }
+      } else {
+        child.examples.forEach(example => {
+          block += `${child.name}: \`${String(example)}\`\n`;
+        });
+        block += '\n';
+      }
+      block += `----\n\n`;
+    }
+
+    // Nested
+    if (child.children && child.children.length) {
+      block += renderConnectFields(child.children, currentPath);
+    }
+
+    // Cloud guard
+    if (child.selfManagedOnly) {
+      output += `ifndef::env-cloud[]\n${block}endif::[]\n\n`;
+    } else {
+      output += block;
+    }
+  });
+
+  return new handlebars.SafeString(output);
+};

package/tools/redpanda-connect/helpers/renderLeafField.js

@@ -0,0 +1,64 @@
+const yaml = require('yaml');
+
+/**
+ * Renders a single “leaf” field (scalar or array) at the given indentation.
+ * If `field.default` is present, prints that. Otherwise prints an empty-string
+ * or empty-array plus an inline comment (# No default (optional/required)).
+ *
+ * @param {Object}   field       – one field object from “children”
+ * @param {number}   indentLevel – number of spaces to indent
+ * @returns {string}             – one line, including comment if needed
+ */
+module.exports = function renderLeafField(field, indentLevel) {
+  if (!field || typeof field !== 'object') {
+    throw new Error('renderLeafField: field must be an object');
+  }
+  if (typeof indentLevel !== 'number' || indentLevel < 0) {
+    throw new Error('renderLeafField: indentLevel must be a non-negative number');
+  }
+  if (!field.name || typeof field.name !== 'string') {
+    throw new Error('renderLeafField: field.name must be a non-empty string');
+  }
+
+  const indent = ' '.repeat(indentLevel);
+  const name = field.name;
+
+  // Decide whether optional or required
+  const optional = Boolean(field.is_optional);
+  const comment = optional
+    ? '# No default (optional)'
+    : '# No default (required)';
+
+  // If a default is provided, use it:
+  if (field.default !== undefined) {
+    // If default is itself an object or array → dump as YAML block
+    if (typeof field.default === 'object') {
+        try {
+          // Turn the object/array into a YAML string. We also need to indent that block
+          const rawYaml = yaml.stringify(field.default).trim();
+          // Indent each line of rawYaml by (indentLevel + 2) spaces:
+          const indentedYaml = rawYaml
+          .split('\n')
+          .map(line => ' '.repeat(indentLevel + 2) + line)
+          .join('\n');
+          return `${indent}${name}:\n${indentedYaml}`;
+        } catch (error) {
+          console.warn(`Failed to serialize default value for field ${field.name}:`, error);
+          return `${indent}${name}: {} # Error serializing default value`;
+        }
+    }
+
+    // Otherwise, default is a primitive (string/number/bool)
+    if (field.type === 'string') {
+      return `${indent}${name}: ${yaml.stringify(field.default)}`;
+    }
+    return `${indent}${name}: ${field.default}`;
+  }
+
+  // No default → choose representation based on kind
+  if (field.kind === 'array') {
+    return `${indent}${name}: [] ${comment}`;
+  } else {
+    return `${indent}${name}: "" ${comment}`;
+  }
+}