@redpanda-data/docs-extensions-and-macros 4.6.12 → 4.7.0

package/bin/doc-tools.js

@@ -962,6 +962,63 @@ automation
     if (tmpClone) fs.rmSync(tmpClone, { recursive: true, force: true });
   });
 
+/**
+ * Generate Markdown table of cloud regions and tiers from master-data.yaml
+ */
+automation
+  .command('cloud-regions')
+  .description('Generate Markdown table of cloud regions and tiers from GitHub YAML file')
+  .option('--output <file>', 'Output file (relative to repo root)', 'cloud-controlplane/x-topics/cloud-regions.md')
+  .option('--format <fmt>', 'Output format: md (Markdown) or adoc (AsciiDoc)', 'md')
+  .option('--owner <owner>', 'GitHub repository owner', 'redpanda-data')
+  .option('--repo <repo>', 'GitHub repository name', 'cloudv2-infra')
+  .option('--path <path>', 'Path to YAML file in repository', 'apps/master-data-reconciler/manifests/overlays/production/master-data.yaml')
+  .option('--ref <ref>', 'Git reference (branch, tag, or commit SHA)', 'integration')
+  .option('--template <path>', 'Path to custom Handlebars template (relative to repo root)')
+  .option('--dry-run', 'Print output to stdout instead of writing file')
+  .action(async (options) => {
+    const { generateCloudRegions } = require('../tools/cloud-regions/generate-cloud-regions.js');
+
+    try {
+      const token = process.env.GITHUB_TOKEN || process.env.GH_TOKEN;
+      if (!token) {
+        throw new Error('GITHUB_TOKEN environment variable is required to fetch from private cloudv2-infra repo.');
+      }
+      const fmt = (options.format || 'md').toLowerCase();
+      let templatePath = undefined;
+      if (options.template) {
+        const repoRoot = findRepoRoot();
+        templatePath = path.resolve(repoRoot, options.template);
+        if (!fs.existsSync(templatePath)) {
+          throw new Error(`Custom template not found: ${templatePath}`);
+        }
+      }
+      const out = await generateCloudRegions({
+        owner: options.owner,
+        repo: options.repo,
+        path: options.path,
+        ref: options.ref,
+        format: fmt,
+        token,
+        template: templatePath,
+      });
+      if (options.dryRun) {
+        process.stdout.write(out);
+        console.log(`\n✅ (dry-run) ${fmt === 'adoc' ? 'AsciiDoc' : 'Markdown'} output printed to stdout.`);
+      } else {
+        // Always resolve output relative to repo root
+        const repoRoot = findRepoRoot();
+        const absOutput = path.resolve(repoRoot, options.output);
+        fs.mkdirSync(path.dirname(absOutput), { recursive: true });
+        fs.writeFileSync(absOutput, out, 'utf8');
+        console.log(`✅ Wrote ${absOutput}`);
+      }
+    } catch (err) {
+      console.error(`❌ Failed to generate cloud regions: ${err.message}`);
+      process.exit(1);
+    }
+  });
+
 automation
   .command('crd-spec')
   .description('Generate Asciidoc documentation for Kubernetes CRD references')

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@redpanda-data/docs-extensions-and-macros",
-  "version": "4.6.12",
+  "version": "4.7.0",
   "description": "Antora extensions and macros developed for Redpanda documentation.",
   "keywords": [
     "antora",

package/tools/cloud-regions/cloud-regions-table-adoc.hbs

@@ -0,0 +1,39 @@
+{{!-- AsciiDoc Cloud Regions Table Template --}}
+
+////
+This content is auto-generated. Do not edit manually.
+
+To regenerate this content, run:
+  npx doc-tools generate cloud-regions --help
+
+For source code and documentation:
+- Source: https://github.com/redpanda-data/docs-extensions-and-macros/tree/main/tools/cloud-regions
+- Docs: https://redpandadata.atlassian.net/wiki/spaces/DOC/pages/1185054748/Doc+Tools+CLI
+////
+
+Usage tiers define the sizing of a cluster and provide tested and guaranteed workload configurations for throughput, logical partitions, and connections. Availability depends on the region and the cluster type (BYOC, Dedicated). See link:https://docs.redpanda.com/redpanda-cloud/reference/tiers/byoc-tiers/[BYOC tiers] and link:https://docs.redpanda.com/redpanda-cloud/reference/tiers/dedicated-tiers/[Dedicated tiers] for further details.
+
+{{!--{{#if lastUpdated}}
+_Last Updated: {{lastUpdated}}_
+{{/if}}--}}
+
+{{#each providers}}
+
+=== {{name}}
+.Regions for {{name}}
+[%collapsible]
+====
+[cols="1,1,2",options="header"]
+|===
+|Region
+|Zones
+|Throughput Tiers
+{{#each regions}}
+|{{name}} 
+|{{zones}} 
+|{{#each tiers}}* {{this}}
+{{/each}}
+{{/each}}
+|===
+====
+{{/each}}

package/tools/cloud-regions/cloud-regions-table-md.hbs

@@ -0,0 +1,46 @@
+{{!-- Markdown Cloud Regions Table Template --}}
+
+<!-- 
+This content is auto-generated. Do not edit manually.
+
+To regenerate this content, run:
+  npx doc-tools generate cloud-regions -h
+
+For source code and documentation:
+- Source: https://github.com/redpanda-data/docs-extensions-and-macros/tree/main/tools/cloud-regions
+- Docs: https://redpandadata.atlassian.net/wiki/spaces/DOC/pages/1185054748/Doc+Tools+CLI
+-->
+
+Usage tiers define the sizing of a cluster and provide tested and guaranteed workload configurations for throughput, logical partitions, and connections. Availability depends on the region and the cluster type (BYOC, Dedicated). See [BYOC tiers](https://docs.redpanda.com/redpanda-cloud/reference/tiers/byoc-tiers/) and [Dedicated tiers](https://docs.redpanda.com/redpanda-cloud/reference/tiers/dedicated-tiers/) for further details.
+
+{{!--{{#if lastUpdated}}
+<p><em>Last Updated: {{lastUpdated}}</em></p>
+{{/if}}--}}
+
+{{#each providers}}
+<h3>{{name}}</h3>
+<details><summary>Regions for {{name}}</summary>
+<p class="sr-only">Table of cloud regions and throughput tiers for {{name}}. Use the table headers for more information.</p>
+<table aria-label="Cloud regions and throughput tiers for {{name}}">
+  <tr>
+    <th title="Cloud region name (such as us-central1)">Region</th>
+    <th title="Availability zones in the region (comma-separated)">Zones</th>
+    <th title="Available throughput tiers and cluster types in this region">Throughput Tiers</th>
+  </tr>
+  {{#each regions}}
+  <tr>
+    <td>{{name}}</td>
+    <td>{{zones}}</td>
+    <td>
+      <ul>
+        {{#each tiers}}
+        <li>{{this}}</li>
+        {{/each}}
+      </ul>
+    </td>
+  </tr>
+  {{/each}}
+</table>
+
+</details>
+{{/each}}

package/tools/cloud-regions/generate-cloud-regions.js

@@ -0,0 +1,243 @@
+// Standalone module to fetch, parse, filter, and render cloud regions/tier data
+// Usage: generateCloudRegions({ sourceUrl, format, token })
+
+/**
+ * Expected YAML source data shape:
+ *
+ * {
+ *   regions: [
+ *     {
+ *       name: string,                // Region name, such as "us-west1"
+ *       cloudProvider: string,       // One of CLOUD_PROVIDER_AWS, CLOUD_PROVIDER_GCP, CLOUD_PROVIDER_AZURE
+ *       zones: [string] | string,    // List of zones or comma-separated string
+ *       redpandaProductAvailability: {
+ *         [key: string]: {
+ *           redpandaProductName: string, // Product name (must match a public product in products[])
+ *           clusterTypes: [string],      // List of cluster type enums
+ *         }
+ *       }
+ *     }, ...
+ *   ],
+ *   products: [
+ *     {
+ *       name: string,      // Product name (used for filtering and output)
+ *       isPublic: boolean, // Only public products are documented
+ *     }, ...
+ *   ]
+ * }
+ *
+ * All keys are required unless otherwise noted. Only public products/tiers are included in output.
+ */
+
+const path = require('path');
+const fs = require('fs');
+const jsYaml = require('js-yaml');
+const renderCloudRegions = require('./render-cloud-regions');
+
+const providerMap = {
+  CLOUD_PROVIDER_AWS: 'AWS',
+  CLOUD_PROVIDER_GCP: 'GCP',
+  CLOUD_PROVIDER_AZURE: 'Azure',
+};
+const providerOrder = ['GCP', 'AWS', 'Azure'];
+const clusterTypeMap = {
+  CLUSTER_TYPE_BYOC: 'BYOC',
+  CLUSTER_TYPE_DEDICATED: 'Dedicated',
+  CLUSTER_TYPE_FMC: 'Dedicated',
+};
+/**
+ * Returns the display name for a given cluster type, or the original value if unmapped.
+ * @param {string} ct - The internal cluster type identifier.
+ * @return {string} The display name for the cluster type.
+ */
+function displayClusterType(ct) {
+  return clusterTypeMap[ct] || ct;
+}
+
+/**
+ * Fetches YAML content from GitHub using the GitHub API.
+ *
+ * Uses the GitHub API to fetch file content, which avoids caching issues that can occur with raw URLs.
+ *
+ * @param {Object} options - Options for fetching the YAML content.
+ * @param {string} options.owner - GitHub repository owner.
+ * @param {string} options.repo - GitHub repository name.
+ * @param {string} options.path - Path to the file within the repository.
+ * @param {string} [options.ref='main'] - Git reference (branch, tag, or commit SHA).
+ * @param {string} [options.token] - Optional GitHub token for authorization.
+ * @returns {Promise<string>} The fetched YAML content as a string.
+ * @throws {Error} If the GitHub API call fails or the file cannot be found.
+ */
+async function fetchYaml({ owner, repo, path, ref = 'main', token }) {
+  try {
+    const { Octokit } = await import('@octokit/rest');
+    const octokit = new Octokit(token ? { auth: token } : {});
+
+    console.log(`[cloud-regions] INFO: Fetching ${owner}/${repo}/${path}@${ref} via GitHub API`);
+
+    const response = await octokit.repos.getContent({
+      owner,
+      repo,
+      path,
+      ref,
+    });
+
+    if (Array.isArray(response.data)) {
+      throw new Error(`Path ${path} is a directory, not a file`);
+    }
+
+    if (response.data.type !== 'file') {
+      throw new Error(`Path ${path} is not a file`);
+    }
+
+    // Decode base64 content
+    const content = Buffer.from(response.data.content, 'base64').toString('utf8');
+
+    if (!content || content.trim() === '') {
+      throw new Error('Empty YAML content received from GitHub API');
+    }
+
+    return content;
+  } catch (err) {
+    console.error(`[cloud-regions] ERROR: Failed to fetch from GitHub API: ${err.message}`);
+    throw new Error(`GitHub API fetch failed: ${err.message}`);
+  }
+}
+
+/**
+ * Parses YAML text describing cloud regions and products, filters for public products, and organizes regions by provider.
+ *
+ * The function expects YAML content with a top-level `regions` array and an optional `products` array. It groups regions by cloud provider, includes only those with at least one public product tier, and formats tier and cluster type information for each region. Providers and regions without public tiers are excluded from the result.
+ *
+ * @param {string} yamlText - The YAML content to parse and process.
+ * @return {Array<Object>} An array of provider objects, each containing a name and a list of regions with their available public product tiers.
+ * @throws {Error} If the YAML is malformed or missing the required `regions` array.
+ */
+function processCloudRegions(yamlText) {
+  let data;
+  try {
+    data = jsYaml.load(yamlText);
+  } catch (e) {
+    console.error('[cloud-regions] ERROR: Malformed YAML.');
+    throw new Error('Malformed YAML: ' + e.message);
+  }
+  if (!data || !Array.isArray(data.regions)) {
+    console.error('[cloud-regions] ERROR: YAML missing top-level regions array.');
+    throw new Error('YAML does not contain a top-level regions array.');
+  }
+  // Ensure grouped keys match providerOrder and providerMap values
+  const grouped = { AWS: [], GCP: [], Azure: [] };
+  for (const region of data.regions) {
+    const key = providerMap[region.cloudProvider];
+    if (!key) {
+      console.warn(`[cloud-regions] WARN: Unknown cloudProvider '${region.cloudProvider}' in region '${region.name}'. Skipping.`);
+      continue;
+    }
+    grouped[key].push(region);
+  }
+  // Build a set of public product names
+  const publicProductNames = new Set();
+  if (Array.isArray(data.products)) {
+    for (const product of data.products) {
+      if (product.isPublic && product.name) {
+        publicProductNames.add(product.name);
+      }
+    }
+  } else {
+    console.warn('[cloud-regions] WARN: No products array found in YAML.');
+  }
+  // Prepare providers array for template, only including public products and using name
+  const providers = providerOrder
+    .filter((prov) => grouped[prov] && grouped[prov].length > 0)
+    .map((prov) => {
+      // Only include regions that have at least one public product/tier
+      const filteredRegions = grouped[prov].map((region) => {
+        const zones = Array.isArray(region.zones) ? region.zones.join(',') : (region.zones || '');
+        let tiers = [];
+        if (region.redpandaProductAvailability && typeof region.redpandaProductAvailability === 'object') {
+          // Group by tier name, collect all cluster types for that tier
+          const tierMap = {};
+          for (const t of Object.values(region.redpandaProductAvailability)) {
+            if (!t.redpandaProductName || !publicProductNames.has(t.redpandaProductName)) {
+              continue;
+            }
+            const productName = t.redpandaProductName;
+            if (!tierMap[productName]) tierMap[productName] = new Set();
+            if (Array.isArray(t.clusterTypes)) {
+              for (const ct of t.clusterTypes) tierMap[productName].add(displayClusterType(ct));
+            }
+          }
+          tiers = Object.entries(tierMap)
+            .map(([productName, cts]) => `${productName}: ${Array.from(cts).sort().join(', ')}`)
+            .sort((a, b) => a.localeCompare(b));
+        }
+        return {
+          name: region.name,
+          zones,
+          tiers,
+        };
+      }).filter(region => region.tiers && region.tiers.length > 0);
+      if (filteredRegions.length === 0) {
+        console.info(`[cloud-regions] INFO: No public tiers found for provider '${prov}'.`);
+      }
+      return {
+        name: prov,
+        regions: filteredRegions,
+      };
+    })
+    .filter(provider => provider.regions && provider.regions.length > 0);
+  if (providers.length === 0) {
+    console.warn('[cloud-regions] WARN: No providers/regions found after filtering.');
+  }
+  return providers;
+}
+
+/**
+ * Fetches, processes, and renders cloud region and tier data from a GitHub YAML file.
+ *
+ * Retrieves YAML data from GitHub using the GitHub API (to avoid caching issues),
+ * parses and filters it to include only public cloud regions and tiers, and renders the result in the requested format.
+ *
+ * @param {Object} options - Options for generating cloud regions.
+ * @param {string} options.owner - GitHub repository owner.
+ * @param {string} options.repo - GitHub repository name.
+ * @param {string} options.path - Path to the YAML file within the repository.
+ * @param {string} [options.ref='main'] - Git reference (branch, tag, or commit SHA).
+ * @param {string} [options.format='md'] - The output format (e.g., 'md' for Markdown).
+ * @param {string} [options.token] - Optional GitHub token for authentication.
+ * @param {string} [options.template] - Optional path to custom Handlebars template.
+ * @returns {string} The rendered cloud regions output.
+ * @throws {Error} If fetching, processing, or rendering fails, or if no valid providers or regions are found.
+ */
+async function generateCloudRegions({ owner, repo, path, ref = 'main', format = 'md', token, template }) {
+  let yamlText;
+  try {
+    yamlText = await fetchYaml({ owner, repo, path, ref, token });
+  } catch (err) {
+    console.error(`[cloud-regions] ERROR: Failed to fetch YAML: ${err.message}`);
+    throw err;
+  }
+  let providers;
+  try {
+    providers = processCloudRegions(yamlText);
+  } catch (err) {
+    console.error(`[cloud-regions] ERROR: Failed to process cloud regions: ${err.message}`);
+    throw err;
+  }
+  if (providers.length === 0) {
+    console.error('[cloud-regions] ERROR: No providers/regions found in YAML after filtering.');
+    throw new Error('No providers/regions found in YAML after filtering.');
+  }
+  const lastUpdated = new Date().toISOString();
+  try {
+    return renderCloudRegions({ providers, format, lastUpdated, template });
+  } catch (err) {
+    console.error(`[cloud-regions] ERROR: Failed to render cloud regions: ${err.message}`);
+    throw err;
+  }
+}
+
+module.exports = {
+  generateCloudRegions,
+  processCloudRegions,
+};

package/tools/cloud-regions/render-cloud-regions.js

@@ -0,0 +1,48 @@
+const fs = require('fs');
+const path = require('path');
+const handlebars = require('handlebars');
+
+
+/**
+ * Generates a formatted string representing cloud provider regions using a Handlebars template.
+ *
+ * Sorts regions alphabetically within each provider and renders the data using a template file corresponding to the specified format ('md' or 'adoc'). Optionally includes a last updated timestamp.
+ *
+ * @param {Object} opts - Options for rendering.
+ * @param {Array} opts.providers - List of cloud provider objects, each with a name and an array of regions.
+ * @param {string} opts.format - Output format, either 'md' (Markdown) or 'adoc' (AsciiDoc).
+ * @param {string} [opts.lastUpdated] - Optional ISO timestamp indicating when the data was last updated.
+ * @returns {string} The rendered output string.
+ * @throws {Error} If the providers array is missing or empty.
+ */
+function renderCloudRegions({ providers, format, lastUpdated }) {
+  if (!Array.isArray(providers) || providers.length === 0) {
+    throw new Error('No providers/regions found in YAML.');
+  }
+  if (!['md', 'adoc'].includes(format)) {
+    throw new Error(`Unsupported format: ${format}. Use 'md' or 'adoc'.`);
+  }
+  // Sort regions alphabetically within each provider
+  const sortedProviders = providers.map(provider => ({
+    ...provider,
+    regions: [...provider.regions].sort((a, b) => a.name.localeCompare(b.name))
+  }));
+  const templateFile = path.join(__dirname, `cloud-regions-table-${format}.hbs`);
+  if (!fs.existsSync(templateFile)) {
+    throw new Error(`Template file not found: ${templateFile}`);
+  }
+  let templateSrc, template;
+  try {
+    templateSrc = fs.readFileSync(templateFile, 'utf8');
+    template = handlebars.compile(templateSrc);
+  } catch (err) {
+    throw new Error(`Failed to compile Handlebars template at ${templateFile}: ${err.message}`);
+  }
+  try {
+    return template({ providers: sortedProviders, lastUpdated });
+  } catch (err) {
+    throw new Error(`Failed to render Handlebars template at ${templateFile}: ${err.message}`);
+  }
+}
+
+module.exports = renderCloudRegions;

package/tools/redpanda-connect/generate-rpcn-connector-docs.js

@@ -67,7 +67,11 @@ function mergeOverrides(target, overrides) {
     }
 
     // === Handle examples ===
-    if (key === 'examples' && Array.isArray(overrides[key]) && Array.isArray(target[key])) {
+    if (key === 'examples' && Array.isArray(overrides[key])) {
+      // If target[key] is not an array, initialize it
+      if (!Array.isArray(target[key])) {
+        target[key] = [];
+      }
       const overrideMap = new Map(overrides[key].map(o => [o.title, o]));
 
       target[key] = target[key].map(example => {