@redpanda-data/docs-extensions-and-macros 4.15.6 → 4.15.8

package/tools/redpanda-connect/generate-rpcn-connector-docs.js

@@ -244,7 +244,8 @@ async function generateRpcnConnectorDocs(options) {
     templateBloblang,
     writeFullDrafts,
     cgoOnly = [],        // Array of cgo-only connectors from cgo binary inspection
-    cloudOnly = []       // Array of cloud-only connectors from cloud binary inspection
+    cloudOnly = [],      // Array of cloud-only connectors from cloud binary inspection
+    csvMetadata = []     // Array of CSV metadata with support levels
   } = options;
 
   // Read connector index (JSON or YAML)
@@ -274,6 +275,16 @@ async function generateRpcnConnectorDocs(options) {
     });
   }
 
+  // Build a Map of CSV metadata for fast support level lookup
+  const csvMetadataMap = new Map();
+  if (Array.isArray(csvMetadata)) {
+    csvMetadata.forEach(item => {
+      if (item.type && item.name) {
+        csvMetadataMap.set(`${item.type}:${item.name}`, item);
+      }
+    });
+  }
+
   // Apply overrides if provided
   if (overrides && fs.existsSync(overrides)) {
     const ovRaw = fs.readFileSync(overrides, 'utf8');
@@ -395,6 +406,7 @@ async function generateRpcnConnectorDocs(options) {
         }
 
         // Check if this connector is cgo-only or cloud-only and mark it
+        // Use plural type for CGO/cloud detection (matches test expectations)
         const connectorKey = `${type}:${name}`;
         const isCloudOnly = cloudOnlySet.has(connectorKey);
         const isCgoOnly = cgoOnlySet.has(connectorKey);
@@ -411,6 +423,13 @@ async function generateRpcnConnectorDocs(options) {
           item.cloudOnly = true;
         }
 
+        // Lookup support level from CSV metadata using singular type
+        const csvKey = `${item.type}:${name}`;
+        const csvData = csvMetadataMap.get(csvKey);
+        if (csvData && csvData.support) {
+          item.support = csvData.support;
+        }
+
         let content;
         try {
           content = compiledTemplate(item);
@@ -453,6 +472,28 @@ async function generateRpcnConnectorDocs(options) {
       if (!Array.isArray(items)) continue;
       const outRoot = path.join(outputRoot, folder);
       fs.mkdirSync(outRoot, { recursive: true });
+
+      // Get current item names
+      const currentNames = new Set(items.filter(fn => fn.name).map(fn => fn.name));
+
+      // Delete partials that no longer exist in the data
+      if (fs.existsSync(outRoot)) {
+        const existingFiles = fs.readdirSync(outRoot).filter(f => f.endsWith('.adoc'));
+        for (const file of existingFiles) {
+          const name = file.replace('.adoc', '');
+          // Skip hand-authored files (by convention, start with underscore)
+          if (name.startsWith('_')) {
+            console.log(`Skipping hand-authored file: ${file}`);
+            continue;
+          }
+          if (!currentNames.has(name)) {
+            const filePath = path.join(outRoot, file);
+            fs.unlinkSync(filePath);
+            console.log(`Deleted removed ${folder} partial: ${file}`);
+          }
+        }
+      }
+
       // Use custom or default template
       const bloblangTemplatePath = templateBloblang || path.resolve(__dirname, './templates/bloblang-function.hbs');
       const bloblangTemplate = handlebars.compile(fs.readFileSync(bloblangTemplatePath, 'utf8'));
@@ -465,6 +506,71 @@ async function generateRpcnConnectorDocs(options) {
         partialFiles.push(path.relative(process.cwd(), outPath));
       }
     }
+
+    // Generate overview pages for Bloblang methods and functions
+    const guidesRoot = path.join(process.cwd(), 'modules/guides/pages/bloblang');
+    fs.mkdirSync(guidesRoot, { recursive: true });
+
+    // Generate methods.adoc with categorized methods
+    if (dataObj['bloblang-methods']) {
+      const methodsData = dataObj['bloblang-methods'];
+      const categoriesMap = new Map();
+
+      // Extract categories and their methods
+      for (const method of methodsData) {
+        if (!method.name || !method.categories || !Array.isArray(method.categories)) continue;
+
+        for (const cat of method.categories) {
+          const categoryName = cat.Category;
+          if (!categoryName) continue;
+
+          if (!categoriesMap.has(categoryName)) {
+            categoriesMap.set(categoryName, []);
+          }
+          categoriesMap.get(categoryName).push(method.name);
+        }
+      }
+
+      // Sort categories and prepare data for template
+      const categories = Array.from(categoriesMap.entries())
+        .map(([name, methods]) => ({
+          name,
+          methods: methods.sort()
+        }))
+        .sort((a, b) => {
+          // Special ordering: General first, Deprecated last
+          if (a.name === 'General') return -1;
+          if (b.name === 'General') return 1;
+          if (a.name === 'Deprecated') return 1;
+          if (b.name === 'Deprecated') return -1;
+          return a.name.localeCompare(b.name);
+        });
+
+      const methodsTemplatePath = path.resolve(__dirname, './templates/bloblang-methods-overview.hbs');
+      const methodsTemplate = handlebars.compile(fs.readFileSync(methodsTemplatePath, 'utf8'));
+      const methodsAdoc = methodsTemplate({ categories });
+      const methodsOutPath = path.join(guidesRoot, 'methods.adoc');
+      fs.writeFileSync(methodsOutPath, methodsAdoc, 'utf8');
+      console.log('Generated methods.adoc overview page');
+      partialFiles.push(path.relative(process.cwd(), methodsOutPath));
+    }
+
+    // Generate functions.adoc with all functions
+    if (dataObj['bloblang-functions']) {
+      const functionsData = dataObj['bloblang-functions'];
+      const functionNames = functionsData
+        .filter(fn => fn.name)
+        .map(fn => fn.name)
+        .sort();
+
+      const functionsTemplatePath = path.resolve(__dirname, './templates/bloblang-functions-overview.hbs');
+      const functionsTemplate = handlebars.compile(fs.readFileSync(functionsTemplatePath, 'utf8'));
+      const functionsAdoc = functionsTemplate({ functions: functionNames });
+      const functionsOutPath = path.join(guidesRoot, 'functions.adoc');
+      fs.writeFileSync(functionsOutPath, functionsAdoc, 'utf8');
+      console.log('Generated functions.adoc overview page');
+      partialFiles.push(path.relative(process.cwd(), functionsOutPath));
+    }
   }
 
   // Common/Advanced config snippet YAMLs in modules/components/examples

package/tools/redpanda-connect/github-release-utils.js

@@ -0,0 +1,275 @@
+'use strict';
+
+const octokit = require('../../cli-utils/octokit-client');
+const { hasGitHubToken } = require('../../cli-utils/github-token');
+const semver = require('semver');
+
+/**
+ * GitHub release discovery utilities for Redpanda Connect
+ *
+ * Provides functions to discover and filter GitHub releases between versions.
+ */
+
+// Cache for GitHub releases to avoid repeated API calls
+let releaseCache = null;
+let cacheTimestamp = null;
+const CACHE_TTL_MS = 5 * 60 * 1000; // 5 minutes
+
+/**
+ * Fetch all releases from redpanda-data/connect repository
+ * @param {boolean} useCache - Whether to use cached results
+ * @returns {Promise<Array>} Array of release objects
+ */
+async function fetchAllReleases(useCache = true) {
+  // Check cache
+  if (useCache && releaseCache && cacheTimestamp) {
+    const age = Date.now() - cacheTimestamp;
+    if (age < CACHE_TTL_MS) {
+      console.log(`✓ Using cached releases (${Math.round(age / 1000)}s old)`);
+      return releaseCache;
+    }
+  }
+
+  console.log('Fetching releases from GitHub...');
+
+  // Warn if no token available (only once per execution)
+  if (!hasGitHubToken() && !fetchAllReleases._warnedAboutToken) {
+    console.warn('⚠️  No GitHub token found. API rate limits will be more restrictive.');
+    console.warn('   Set GITHUB_TOKEN or GH_TOKEN environment variable for higher limits.');
+    fetchAllReleases._warnedAboutToken = true;
+  }
+
+  try {
+    // Fetch all releases (paginated)
+    const releases = await octokit.paginate(
+      octokit.rest.repos.listReleases,
+      {
+        owner: 'redpanda-data',
+        repo: 'connect',
+        per_page: 100
+      }
+    );
+
+    console.log(`✓ Fetched ${releases.length} releases from GitHub`);
+
+    // Update cache
+    releaseCache = releases;
+    cacheTimestamp = Date.now();
+
+    return releases;
+  } catch (error) {
+    if (error.status === 403 && error.response?.headers?.['x-ratelimit-remaining'] === '0') {
+      const resetTime = new Date(parseInt(error.response.headers['x-ratelimit-reset']) * 1000);
+      throw new Error(
+        `GitHub API rate limit exceeded. Resets at ${resetTime.toLocaleTimeString()}. ` +
+        `Consider setting GITHUB_TOKEN environment variable for higher limits.`
+      );
+    }
+
+    throw new Error(`Failed to fetch GitHub releases: ${error.message}`);
+  }
+}
+
+/**
+ * Parse version from GitHub release tag
+ * Handles formats: "v4.50.0", "4.50.0", "v4.50.0-beta.1"
+ * @param {string} tag - Release tag name
+ * @returns {string|null} Normalized version string or null if invalid
+ */
+function parseVersionFromTag(tag) {
+  if (!tag) return null;
+
+  // Remove 'v' prefix if present
+  const normalized = tag.startsWith('v') ? tag.slice(1) : tag;
+
+  // Validate semver format
+  const version = semver.valid(normalized);
+  return version;
+}
+
+/**
+ * Check if a version is a pre-release (beta, RC, alpha, etc.)
+ * @param {string} version - Semver version string
+ * @returns {boolean} True if pre-release
+ */
+function isPrerelease(version) {
+  const parsed = semver.parse(version);
+  if (!parsed) return false;
+
+  return parsed.prerelease.length > 0;
+}
+
+/**
+ * Filter releases to stable GA releases only
+ * @param {Array} releases - Array of GitHub release objects
+ * @returns {Array} Filtered releases with only stable versions
+ */
+function filterToStableReleases(releases) {
+  return releases.filter(release => {
+    // Skip drafts
+    if (release.draft) {
+      return false;
+    }
+
+    // Parse version
+    const version = parseVersionFromTag(release.tag_name);
+    if (!version) {
+      return false;
+    }
+
+    // Skip pre-releases (beta, RC, etc.)
+    if (isPrerelease(version)) {
+      return false;
+    }
+
+    return true;
+  });
+}
+
+/**
+ * Discover all intermediate releases between two versions
+ * @param {string} fromVersion - Starting version (e.g., "4.50.0")
+ * @param {string} toVersion - Ending version (e.g., "4.54.0")
+ * @param {Object} options - Optional configuration
+ * @param {boolean} options.includePrerelease - Include beta/RC versions (default: false)
+ * @param {boolean} options.useCache - Use cached GitHub data (default: true)
+ * @returns {Promise<Array>} Array of version objects: [{version, tag, date, url}]
+ */
+async function discoverIntermediateReleases(fromVersion, toVersion, options = {}) {
+  const {
+    includePrerelease = false,
+    useCache = true
+  } = options;
+
+  // Validate versions are strings
+  if (typeof fromVersion !== 'string') {
+    throw new Error(`Invalid starting version: ${fromVersion}`);
+  }
+  if (typeof toVersion !== 'string') {
+    throw new Error(`Invalid ending version: ${toVersion}`);
+  }
+
+  // Normalize versions (remove 'v' prefix if present)
+  const normalizedFrom = fromVersion.startsWith('v') ? fromVersion.slice(1) : fromVersion;
+  const normalizedTo = toVersion.startsWith('v') ? toVersion.slice(1) : toVersion;
+
+  // Validate versions
+  if (!semver.valid(normalizedFrom)) {
+    throw new Error(`Invalid starting version: ${fromVersion}`);
+  }
+  if (!semver.valid(normalizedTo)) {
+    throw new Error(`Invalid ending version: ${toVersion}`);
+  }
+
+  console.log('');
+  console.log(`Discovering releases between ${normalizedFrom} and ${normalizedTo}...`);
+
+  // Fetch all releases
+  const allReleases = await fetchAllReleases(useCache);
+
+  // Filter to stable releases unless includePrerelease is true
+  const filteredReleases = includePrerelease
+    ? allReleases.filter(r => !r.draft && parseVersionFromTag(r.tag_name))
+    : filterToStableReleases(allReleases);
+
+  // Parse and filter versions in range
+  const versionsInRange = [];
+
+  for (const release of filteredReleases) {
+    const version = parseVersionFromTag(release.tag_name);
+    if (!version) continue;
+
+    // Check if version is in range (inclusive)
+    if (
+      semver.gte(version, normalizedFrom) &&
+      semver.lte(version, normalizedTo)
+    ) {
+      versionsInRange.push({
+        version,
+        tag: release.tag_name,
+        date: release.published_at,
+        url: release.html_url,
+        isPrerelease: isPrerelease(version)
+      });
+    }
+  }
+
+  // Sort by semver (oldest to newest)
+  versionsInRange.sort((a, b) => semver.compare(a.version, b.version));
+
+  console.log(`✓ Found ${versionsInRange.length} release(s) in range`);
+
+  if (versionsInRange.length === 0) {
+    console.warn('⚠️  No releases found in the specified range');
+    return [];
+  }
+
+  // Log the discovered versions
+  console.log('');
+  console.log('Releases to process:');
+  versionsInRange.forEach((v, i) => {
+    const prereleaseTag = v.isPrerelease ? ' (pre-release)' : '';
+    const date = new Date(v.date).toLocaleDateString();
+    console.log(`  ${i + 1}. ${v.version}${prereleaseTag} - ${date}`);
+  });
+  console.log('');
+
+  return versionsInRange;
+}
+
+/**
+ * Find the appropriate cloud version for a given OSS release date
+ * Returns the latest stable cloud version published on or before the OSS release date
+ * @param {string} ossReleaseDate - ISO date string of the OSS release
+ * @param {Object} options - Optional configuration
+ * @param {boolean} options.useCache - Use cached GitHub data (default: true)
+ * @returns {Promise<string|null>} Cloud version string or null if not found
+ */
+async function findCloudVersionForDate(ossReleaseDate, options = {}) {
+  const { useCache = true } = options;
+
+  // Fetch all releases from the main connect repo (includes cloud builds)
+  const allReleases = await fetchAllReleases(useCache);
+
+  // Filter to stable releases
+  const stableReleases = filterToStableReleases(allReleases);
+
+  // Filter to releases on or before the OSS release date
+  const ossDate = new Date(ossReleaseDate);
+  const eligibleReleases = stableReleases.filter(release => {
+    const releaseDate = new Date(release.published_at);
+    return releaseDate <= ossDate;
+  });
+
+  if (eligibleReleases.length === 0) {
+    return null;
+  }
+
+  // Sort by date (most recent first)
+  eligibleReleases.sort((a, b) => {
+    return new Date(b.published_at) - new Date(a.published_at);
+  });
+
+  // Return the most recent version
+  const cloudVersion = parseVersionFromTag(eligibleReleases[0].tag_name);
+  return cloudVersion;
+}
+
+/**
+ * Clear the release cache
+ * Useful for testing or when you need fresh data
+ */
+function clearCache() {
+  releaseCache = null;
+  cacheTimestamp = null;
+}
+
+module.exports = {
+  discoverIntermediateReleases,
+  fetchAllReleases,
+  parseVersionFromTag,
+  isPrerelease,
+  filterToStableReleases,
+  findCloudVersionForDate,
+  clearCache
+};

package/tools/redpanda-connect/helpers/bloblangExample.js

@@ -1,10 +1,30 @@
 // Bloblang example formatting helper for Handlebars
 function bloblangExample(example) {
   if (typeof example === 'object' && example !== null && example.mapping) {
+    let leadIn = '';
     let codeBlock = '';
+
+    // Extract summary as lead-in prose (not a comment in code)
     if (example.summary && example.summary.trim()) {
-      codeBlock += `# ${example.summary.trim().replace(/\n/g, '\n# ')}\n\n`;
+      let summary = example.summary.trim();
+
+      // Convert Markdown headings to AsciiDoc
+      // ##### Heading -> ==== Heading (H5 -> H4 in AsciiDoc)
+      summary = summary.replace(/^#####\s+(.+)$/gm, '==== $1');
+      // #### Heading -> === Heading (H4 -> H3 in AsciiDoc)
+      summary = summary.replace(/^####\s+(.+)$/gm, '=== $1');
+      // ### Heading -> == Heading (H3 -> H2 in AsciiDoc)
+      summary = summary.replace(/^###\s+(.+)$/gm, '== $1');
+
+      // Ensure lead-in ends with a colon (replace period/exclamation/question mark if present)
+      if (summary.endsWith('.') || summary.endsWith('!') || summary.endsWith('?')) {
+        summary = summary.slice(0, -1) + ':';
+      } else if (!summary.endsWith(':')) {
+        summary += ':';
+      }
+      leadIn = summary + '\n\n';
     }
+
     if (typeof example.mapping === 'string') {
       codeBlock += example.mapping.trim() + '\n';
     }
@@ -15,7 +35,7 @@ function bloblangExample(example) {
         }
       }
     }
-    return `[,coffeescript]\n----\n${codeBlock.trim()}\n----\n`;
+    return `${leadIn}[,bloblang]\n----\n${codeBlock.trim()}\n----\n`;
   } else {
     let exStr = '';
     if (typeof example === 'string') {
@@ -35,7 +55,7 @@ function bloblangExample(example) {
     } else {
       exStr = String(example);
     }
-    return `[source,coffeescript]\n----\n${exStr}\n----\n`;
+    return `[source,bloblang]\n----\n${exStr}\n----\n`;
   }
 }
 

package/tools/redpanda-connect/helpers/buildConfigYaml.js

@@ -1,11 +1,14 @@
 const renderLeafField = require('./renderLeafField');
 const renderObjectField = require('./renderObjectField');
 
+// Component types that support the 'label' field
+const TYPES_WITH_LABEL = new Set(['inputs', 'outputs', 'processors']);
+
 /**
- * Builds either “Common” or “Advanced” YAML for one connector.
+ * Builds either "Common" or "Advanced" YAML for one connector.
  *
- * - type            = “input” or “output” (or whatever type)
- * - connectorName   = such as “amqp_1”
+ * - type            = "input" or "output" (or whatever type)
+ * - connectorName   = such as "amqp_1"
  * - children        = the array of field‐definitions (entry.config.children)
  * - includeAdvanced = if false → only fields where is_advanced !== true
  *                      if true  → all fields (except deprecated)
@@ -13,18 +16,20 @@ const renderObjectField = require('./renderObjectField');
  * Structure produced:
  *
  *   type:
- *     label: ""
+ *     label: ""          (only for inputs, outputs, processors)
  *     connectorName:
- *       ...child fields (with comments for “no default”)
+ *       ...child fields (with comments for "no default")
  */
 module.exports = function buildConfigYaml(type, connectorName, children, includeAdvanced) {
   const lines = [];
 
-  // “type:” top‐level
+  // "type:" top‐level
   lines.push(`${type}:`);
 
-  // Two‐space indent for “label”
-  lines.push(`  label: ""`);
+  // Two‐space indent for "label" (only for types that support it)
+  if (TYPES_WITH_LABEL.has(type)) {
+    lines.push(`  label: ""`);
+  }
 
   // Two‐space indent for connectorName heading
   lines.push(`  ${connectorName}:`);

package/tools/redpanda-connect/helpers/ensurePeriod.js

@@ -0,0 +1,27 @@
+/**
+ * Ensures a text string ends with a period (or other terminal punctuation).
+ * Used for normalizing Bloblang descriptions.
+ *
+ * @param {string} text - The text to process
+ * @returns {string} Text ending with appropriate punctuation
+ */
+function ensurePeriod(text) {
+  if (!text || typeof text !== 'string') {
+    return text;
+  }
+
+  const trimmed = text.trim();
+  if (!trimmed) {
+    return text;
+  }
+
+  // Check if already ends with terminal punctuation
+  if (trimmed.endsWith('.') || trimmed.endsWith('!') || trimmed.endsWith('?')) {
+    return text;
+  }
+
+  // Add period
+  return text.trim() + '.';
+}
+
+module.exports = ensurePeriod;

package/tools/redpanda-connect/helpers/hasOptionalParams.js

@@ -0,0 +1,7 @@
+// Check if any parameters are optional
+function hasOptionalParams(params) {
+  if (!Array.isArray(params)) return false;
+  return params.some(param => param.is_optional);
+}
+
+module.exports = hasOptionalParams;

package/tools/redpanda-connect/helpers/index.js

@@ -17,4 +17,7 @@ module.exports = {
   commonConfig:           require('./commonConfig.js'),
   advancedConfig:         require('./advancedConfig.js'),
   bloblangExample:        require('./bloblangExample.js'),
+  hasOptionalParams:      require('./hasOptionalParams.js'),
+  ensurePeriod:           require('./ensurePeriod.js'),
+  toSentenceCase:         require('./toSentenceCase.js'),
 };

package/tools/redpanda-connect/helpers/toSentenceCase.js

@@ -0,0 +1,69 @@
+'use strict'
+
+/**
+ * Convert a string to sentence case, preserving acronyms and proper nouns
+ * @param {string} text - The text to convert
+ * @returns {string} The text in sentence case
+ */
+function toSentenceCase(text) {
+  if (!text) return ''
+
+  // Map of exact word matches to preserve (case-sensitive)
+  const exactPreserve = new Map([
+    ['geoip', 'GeoIP'],
+    ['GEOIP', 'GeoIP'],
+    ['GeoIP', 'GeoIP']
+  ])
+
+  // List of acronyms to preserve (will be uppercased)
+  const preserveWords = new Set([
+    'SQL',
+    'JSON',
+    'JWT',
+    'XML',
+    'HTML',
+    'URL',
+    'URI',
+    'HTTP',
+    'HTTPS',
+    'TLS',
+    'SSL',
+    'AWS',
+    'GCP',
+    'API',
+    'ID',
+    'UUID',
+    'CSV'
+  ])
+
+  // Split into words
+  const words = text.split(/\s+/)
+
+  return words.map((word, index) => {
+    // Check if word is in exact preserve map first
+    if (exactPreserve.has(word) || exactPreserve.has(word.toLowerCase()) || exactPreserve.has(word.toUpperCase())) {
+      const key = exactPreserve.has(word) ? word : (exactPreserve.has(word.toLowerCase()) ? word.toLowerCase() : word.toUpperCase())
+      return exactPreserve.get(key)
+    }
+
+    // Check if word is in preserve list (case-insensitive check)
+    if (preserveWords.has(word.toUpperCase())) {
+      return word.toUpperCase()
+    }
+
+    // Check if word contains special characters like &
+    if (word === '&') {
+      return word
+    }
+
+    // First word: capitalize first letter, lowercase rest
+    if (index === 0) {
+      return word.charAt(0).toUpperCase() + word.slice(1).toLowerCase()
+    }
+
+    // Subsequent words: lowercase
+    return word.toLowerCase()
+  }).join(' ')
+}
+
+module.exports = toSentenceCase