@redpanda-data/docs-extensions-and-macros 5.1.2 → 5.2.0

package/bin/doc-tools.js

@@ -770,6 +770,7 @@ automation
   .option('--template-intro <path>', 'Intro section partial template', path.resolve(__dirname, '../tools/redpanda-connect/templates/intro.hbs'))
   .option('--template-fields <path>', 'Fields section partial template', path.resolve(__dirname, '../tools/redpanda-connect/templates/fields-partials.hbs'))
   .option('--template-examples <path>', 'Examples section partial template', path.resolve(__dirname, '../tools/redpanda-connect/templates/examples-partials.hbs'))
+  .option('--template-metadata <path>', 'Metadata section partial template', path.resolve(__dirname, '../tools/redpanda-connect/templates/metadata-partials.hbs'))
   .option('--template-bloblang <path>', 'Custom Handlebars template for bloblang function/method partials')
   .option('--overrides <path>', 'Optional JSON file with overrides', 'docs-data/overrides.json')
   .option('--include-bloblang', 'Include Bloblang functions and methods in generation')
@@ -792,6 +793,38 @@ automation
     await handleRpcnConnectorDocs(options)
   })
 
+/**
+ * generate migrate-rpcn-metadata
+ *
+ * @description
+ * One-time migration that moves inline `== Metadata` blocks out of connector
+ * reference pages (modules/components/pages/<type>/<name>.adoc) and into
+ * regenerated partials (modules/components/partials/metadata/<type>/<name>.adoc),
+ * replacing the inline block in the page with an include directive. Afterwards
+ * `generate rpcn-connector-docs` keeps the partial in sync with the connector's
+ * upstream description on every run.
+ *
+ * @why
+ * Metadata documented inline in the main page is never refreshed by normal
+ * regeneration, so it drifts from the source. Extracting it into a partial makes
+ * it flow through automatically, matching the fields and examples partials.
+ *
+ * @example
+ * # Dry run (default): report the pages that would change
+ * npx doc-tools generate migrate-rpcn-metadata
+ *
+ * # Apply the migration
+ * npx doc-tools generate migrate-rpcn-metadata --write
+ */
+automation
+  .command('migrate-rpcn-metadata')
+  .description('One-time migration of inline connector == Metadata blocks into regenerated partials. Dry run unless --write is given.')
+  .option('--write', 'Apply changes (default is a dry run that only reports)')
+  .action((options) => {
+    const { migrateMetadataToPartials } = require('../tools/redpanda-connect/migrate-metadata-to-partials.js')
+    migrateMetadataToPartials({ write: Boolean(options.write) })
+  })
+
 /**
  * generate property-docs
  *

package/bin/mcp-tools/metrics-docs.js

@@ -133,6 +133,9 @@ function generateMetricsDocs(args) {
     return {
       success: false,
       error: err.message,
+      // Report the resolved ref even on failure so callers can confirm
+      // parameter defaulting worked independently of the build outcome.
+      [refType]: version,
       stdout: err.stdout || '',
       stderr: err.stderr || '',
       exitCode: err.status,

package/bin/mcp-tools/property-docs.js

@@ -162,6 +162,9 @@ function generatePropertyDocs(args) {
     return {
       success: false,
       error: err.message,
+      // Report the resolved ref even on failure so callers can confirm
+      // parameter defaulting worked independently of the build outcome.
+      [refType]: version,
       stdout: err.stdout || '',
       stderr: err.stderr || '',
       exitCode: err.status,

package/bin/mcp-tools/rpk-docs.js

@@ -188,6 +188,12 @@ function generateRpkDocs(args) {
     const response = {
       success: false,
       error: err.message,
+      // Report the ref that was resolved and attempted, even on failure, so
+      // callers can confirm parameter defaulting worked independently of the
+      // (environment-dependent) build outcome.
+      ref: version,
+      version: version,
+      ref_type: refType,
       stdout: err.stdout || '',
       stderr: err.stderr || '',
       exitCode: err.status

package/cli-utils/install-test-dependencies.sh

@@ -104,7 +104,11 @@ install_buf() {
         # Download and install buf for Linux
         BIN="/usr/local/bin/buf"
         VERSION="1.28.1"
-        curl -sSL "https://github.com/bufbuild/buf/releases/download/v${VERSION}/buf-$(uname -s)-$(uname -m)" -o "${BIN}" || { log_error "Failed to download buf"; exit 1; }
+        # Same download hardening as install_rpk: fail on HTTP errors and
+        # retry transient GitHub/CDN failures with exponential backoff.
+        curl -fsSL --retry 5 --retry-all-errors \
+            --connect-timeout 30 --max-time 300 --retry-max-time 600 \
+            "https://github.com/bufbuild/buf/releases/download/v${VERSION}/buf-$(uname -s)-$(uname -m)" -o "${BIN}" || { log_error "Failed to download buf"; rm -f "${BIN}" 2>/dev/null; exit 1; }
         chmod +x "${BIN}" || { log_error "Failed to make buf executable"; exit 1; }
         ;;
       Darwin)
@@ -464,8 +468,15 @@ install_rpk() {
     
     log_info "Detected ${os_name} ${arch_name}, downloading ${rpk_filename}..."
     
-    # Try to download and install rpk
-    if curl -LO "$rpk_url"; then
+    # Try to download and install rpk.
+    # Use -f so HTTP errors (e.g. 403/404/429/503) fail instead of writing an
+    # error page into the zip, and retry with curl's default exponential
+    # backoff (1s, 2s, 4s, ...) to ride out transient GitHub/CDN and network
+    # errors, including rate limits that need time to clear. --retry-max-time
+    # caps the whole retry window so CI fails fast on persistent errors.
+    if curl -fL --retry 5 --retry-all-errors \
+            --connect-timeout 30 --max-time 300 --retry-max-time 600 \
+            -o "$rpk_filename" "$rpk_url"; then
         if unzip "$rpk_filename" 2>/dev/null; then
             mkdir -p ~/.local/bin
             if mv rpk ~/.local/bin/ 2>/dev/null; then
@@ -505,6 +516,8 @@ install_rpk() {
         fi
     else
         log_warn "Failed to download $rpk_url"
+        # A mid-transfer failure can leave a partial file behind even with -f.
+        rm -f "$rpk_filename" 2>/dev/null
         log_warn "Please install rpk manually:"
         log_warn "https://docs.redpanda.com/current/get-started/rpk-install/"
         return 1

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@redpanda-data/docs-extensions-and-macros",
-  "version": "5.1.2",
+  "version": "5.2.0",
   "description": "Antora extensions and macros developed for Redpanda documentation.",
   "keywords": [
     "antora",

package/tools/generate-cli-docs.js

@@ -304,7 +304,8 @@ IMPORTANT: This documentation is auto-generated. Do not edit manually. Run \`npm
     'cloud-regions',
     'crd-spec',
     'bundle-openapi',
-    'update-connect-version'
+    'update-connect-version',
+    'migrate-rpcn-metadata'
   ];
 
   generateSubcommands.forEach(subcmd => {

package/tools/redpanda-connect/generate-rpcn-connector-docs.js

@@ -18,6 +18,17 @@ Object.entries(helpers).forEach(([name, fn]) => {
 // Default “main” template (connector.hbs) which invokes partials {{> intro}}, {{> fields}}, {{> examples}}
 const DEFAULT_TEMPLATE = path.resolve(__dirname, './templates/connector.hbs');
 
+// Default template for the regenerated per-connector metadata partial.
+const DEFAULT_METADATA_TEMPLATE = path.resolve(__dirname, './templates/metadata-partials.hbs');
+
+// Banner-only content written to a metadata partial when its `== Metadata`
+// section is removed upstream. Kept as an AsciiDoc comment so the file renders
+// nothing while any lingering include directive still resolves cleanly.
+const STALE_METADATA_PARTIAL =
+  '// This content is autogenerated. Do not edit manually. ' +
+  'The connector description no longer contains a `== Metadata` section, so this partial is intentionally empty. ' +
+  'You can remove the include directive that points at this file from the connector page.\n';
+
 /**
  * Reads a file at `filePath` and registers it as a Handlebars partial called `name`.
  * Throws if the file cannot be read.
@@ -241,6 +252,7 @@ async function generateRpcnConnectorDocs(options) {
     templateIntro,
     templateFields,
     templateExamples,
+    templateMetadata,
     templateBloblang,
     writeFullDrafts,
     cgoOnly = [],        // Array of cgo-only connectors from cgo binary inspection
@@ -336,6 +348,12 @@ async function generateRpcnConnectorDocs(options) {
     registerPartial('examples', examplesTemplatePath);
   }
 
+  // Always register the metadata partial. Like fields and examples, it is
+  // regenerated on every run so that metadata fields added to a connector's
+  // upstream description flow into the docs automatically.
+  const metadataTemplatePath = templateMetadata || DEFAULT_METADATA_TEMPLATE;
+  registerPartial('metadata', metadataTemplatePath);
+
   // In draft mode, also register the intro partial
   if (writeFullDrafts && templateIntro) {
     registerPartial('intro', templateIntro);
@@ -344,6 +362,7 @@ async function generateRpcnConnectorDocs(options) {
   const outputRoot     = path.resolve(process.cwd(), 'modules/components/partials');
   const fieldsOutRoot   = path.join(outputRoot, 'fields');
   const examplesOutRoot = path.join(outputRoot, 'examples');
+  const metadataOutRoot = path.join(outputRoot, 'metadata');
   // Drafts go to pages/, not partials/
   const componentsRoot  = writeFullDrafts
     ? path.resolve(process.cwd(), 'modules/components/pages')
@@ -353,6 +372,7 @@ async function generateRpcnConnectorDocs(options) {
   if (!writeFullDrafts) {
     fs.mkdirSync(fieldsOutRoot,   { recursive: true });
     fs.mkdirSync(examplesOutRoot, { recursive: true });
+    fs.mkdirSync(metadataOutRoot, { recursive: true });
   }
 
   let partialsWritten = 0;
@@ -403,6 +423,37 @@ async function generateRpcnConnectorDocs(options) {
         }
       }
 
+      // Render the metadata partial from the connector's description. Only
+      // written when the description contains a `== Metadata` section.
+      const metadataOut = handlebars
+        .compile('{{> metadata description=description type=type typeDir=typeDir name=name}}')(item);
+
+      if (type !== 'bloblang-functions' && type !== 'bloblang-methods') {
+        // Write under typeDir (not the raw data key) so the file path always
+        // matches the include directive built by metadataIncludeLine().
+        const mPath = path.join(metadataOutRoot, typeDir, `${name}.adoc`);
+        if (metadataOut.trim()) {
+          fs.mkdirSync(path.dirname(mPath), { recursive: true });
+          fs.writeFileSync(mPath, metadataOut);
+          if (!writeFullDrafts) {
+            partialsWritten++;
+            partialFiles.push(path.relative(process.cwd(), mPath));
+          }
+        } else if (fs.existsSync(mPath)) {
+          // The upstream description dropped its `== Metadata` section, but a
+          // previously generated partial is still on disk and an already
+          // published page may include it. Deleting the file would break that
+          // include (unresolved include directive + build warning), so blank it
+          // to a banner-only file: the include still resolves and renders
+          // nothing. Log it so a human knows to remove the include line.
+          fs.writeFileSync(mPath, STALE_METADATA_PARTIAL);
+          console.log(
+            `Metadata section removed upstream; emptied stale partial ${path.relative(process.cwd(), mPath)} ` +
+            `(remove its include directive from the ${type}/${name} page)`
+          );
+        }
+      }
+
       if (writeFullDrafts) {
         if (String(item.status || '').toLowerCase() === 'deprecated') {
           console.log(`Skipping draft for deprecated component: ${type}/${name}`);

package/tools/redpanda-connect/helpers/descriptionWithMetadataInclude.js

@@ -0,0 +1,15 @@
+'use strict';
+
+const { descriptionWithMetadataInclude } = require('../metadata-utils.js');
+
+/**
+ * Handlebars helper: render a connector description with its inline
+ * `== Metadata` block replaced by an include of the regenerated metadata
+ * partial, so newly drafted pages pick up metadata changes automatically.
+ * Falls back to the unchanged description when no metadata section exists.
+ * @param {object} item connector data (needs description, type/typeDir, name)
+ * @returns {string}
+ */
+module.exports = function descriptionWithMetadataIncludeHelper (item) {
+  return descriptionWithMetadataInclude(item);
+};

package/tools/redpanda-connect/helpers/index.js

@@ -11,6 +11,8 @@ module.exports = {
   renderYamlList:  require('./renderYamlList.js'),
   renderConnectFields:    require('./renderConnectFields.js'),
   renderConnectExamples:  require('./renderConnectExamples.js'),
+  renderConnectMetadata:  require('./renderConnectMetadata.js'),
+  descriptionWithMetadataInclude: require('./descriptionWithMetadataInclude.js'),
   renderLeafField:        require('./renderLeafField.js'),
   renderObjectField:      require('./renderObjectField.js'),
   buildConfigYaml:        require('./buildConfigYaml.js'),

package/tools/redpanda-connect/helpers/renderConnectMetadata.js

@@ -0,0 +1,13 @@
+'use strict';
+
+const { extractMetadata } = require('../metadata-utils.js');
+
+/**
+ * Handlebars helper: return the `== Metadata` block extracted from a connector
+ * description, or '' when there is none. Used by the metadata partial template.
+ * @param {string} description
+ * @returns {string}
+ */
+module.exports = function renderConnectMetadata (description) {
+  return extractMetadata(description);
+};

package/tools/redpanda-connect/metadata-utils.js

@@ -0,0 +1,112 @@
+'use strict';
+
+/**
+ * Utilities for extracting the "== Metadata" section out of a connector's
+ * `description` prose so it can be emitted as a regenerated partial (like the
+ * fields and examples partials) instead of being frozen into a hand-maintained
+ * main page.
+ *
+ * Convention: the connector `.Description()` in the Connect source contains a
+ * level-2 AsciiDoc heading `== Metadata` followed by a bullet list, terminated
+ * by the next level-2 heading (for example `== Permissions`) or end of string.
+ * As of Connect 4.99.0, 52 of the 65 metadata-documenting components already
+ * follow this exact heading convention.
+ */
+
+const METADATA_HEADING = /^==\s+Metadata\s*$/;
+const LEVEL2_HEADING = /^==\s+\S/;
+// AsciiDoc listing/literal block delimiter (`----`, possibly longer). Lines
+// inside such blocks must not be treated as headings.
+const BLOCK_DELIMITER = /^-{4,}$/;
+
+/**
+ * Locate the `== Metadata` section within a description.
+ * @param {string} description
+ * @returns {{start:number, end:number, block:string}|null} character offsets of
+ *   the section (heading through the last content line, trailing blank lines
+ *   excluded) and the extracted block text, or null when no section is present.
+ */
+function locateMetadata (description) {
+  if (!description || typeof description !== 'string') return null;
+
+  const lines = description.split('\n');
+  let headingLine = -1;
+  let inBlock = false;
+  for (let i = 0; i < lines.length; i++) {
+    if (BLOCK_DELIMITER.test(lines[i])) { inBlock = !inBlock; continue; }
+    if (!inBlock && METADATA_HEADING.test(lines[i])) { headingLine = i; break; }
+  }
+  if (headingLine === -1) return null;
+
+  // Find the terminating heading (next level-2 heading after the metadata one).
+  let endLine = lines.length;
+  inBlock = false;
+  for (let i = headingLine + 1; i < lines.length; i++) {
+    if (BLOCK_DELIMITER.test(lines[i])) { inBlock = !inBlock; continue; }
+    if (!inBlock && LEVEL2_HEADING.test(lines[i])) { endLine = i; break; }
+  }
+
+  // Trim trailing blank lines inside the section so the block ends cleanly.
+  let lastContent = endLine - 1;
+  while (lastContent > headingLine && lines[lastContent].trim() === '') lastContent--;
+
+  const startOffset = lines.slice(0, headingLine).join('\n').length + (headingLine > 0 ? 1 : 0);
+  const block = lines.slice(headingLine, lastContent + 1).join('\n');
+  const endOffset = startOffset + block.length;
+
+  return { start: startOffset, end: endOffset, block };
+}
+
+/**
+ * Return the extracted `== Metadata` block, or '' when there is none.
+ * @param {string} description
+ * @returns {string}
+ */
+function extractMetadata (description) {
+  const found = locateMetadata(description);
+  return found ? found.block : '';
+}
+
+/**
+ * Derive the plural type directory (for example `input` -> `inputs`).
+ * @param {object} item connector data with `type` and/or `typeDir`
+ * @returns {string}
+ */
+function typeDirFor (item) {
+  if (item && item.typeDir) return item.typeDir;
+  const type = item && item.type;
+  if (!type) return '';
+  return type.endsWith('s') ? type : `${type}s`;
+}
+
+/**
+ * Build the Antora include directive for a connector's metadata partial.
+ * @param {object} item connector data with `type`/`typeDir` and `name`
+ * @returns {string}
+ */
+function metadataIncludeLine (item) {
+  return `include::connect:components:partial$metadata/${typeDirFor(item)}/${item.name}.adoc[]`;
+}
+
+/**
+ * Replace the inline `== Metadata` block in a description with an include
+ * directive pointing at the regenerated metadata partial, preserving position
+ * relative to surrounding sections (such as `== Permissions`). Returns the
+ * description unchanged when no metadata section is present.
+ * @param {object} item connector data (needs `description`, `type`/`typeDir`, `name`)
+ * @returns {string}
+ */
+function descriptionWithMetadataInclude (item) {
+  const description = (item && item.description) || '';
+  const found = locateMetadata(description);
+  if (!found) return description;
+  return description.slice(0, found.start) + metadataIncludeLine(item) + description.slice(found.end);
+}
+
+module.exports = {
+  locateMetadata,
+  extractMetadata,
+  typeDirFor,
+  metadataIncludeLine,
+  descriptionWithMetadataInclude,
+};

package/tools/redpanda-connect/migrate-metadata-to-partials.js

@@ -0,0 +1,103 @@
+'use strict';
+
+/**
+ * One-time migration for the metadata-partial change.
+ *
+ * Existing connector reference pages carry their `== Metadata` block inline in
+ * the main page (`modules/components/pages/<type>/<name>.adoc`), where normal
+ * regeneration never refreshes it. This script moves that block into a
+ * regenerated partial (`modules/components/partials/metadata/<type>/<name>.adoc`)
+ * and replaces the inline block in the page with an include directive, matching
+ * the pattern already used for the fields and examples partials.
+ *
+ * After migration, `doc-tools generate rpcn-connector-docs` keeps the partial in
+ * sync with the connector's upstream description on every run.
+ *
+ * Usage (run from the docs repo root):
+ *   npx doc-tools generate migrate-rpcn-metadata            # dry run, reports changes
+ *   npx doc-tools generate migrate-rpcn-metadata --write    # apply changes
+ */
+
+const fs = require('fs');
+const path = require('path');
+const { globSync } = require('glob');
+const { locateMetadata, metadataIncludeLine } = require('./metadata-utils.js');
+
+const PARTIAL_BANNER =
+  '// This content is autogenerated. Do not edit manually. Metadata fields come from the connector\'s description in the Connect source. To change them, update the connector description upstream, or use the doc-tools CLI with the --overrides option: https://redpandadata.atlassian.net/wiki/spaces/DOC/pages/1247543314/Generate+reference+docs+for+Redpanda+Connect';
+
+const PAGES_ROOT = path.resolve(process.cwd(), 'modules/components/pages');
+const PARTIALS_ROOT = path.resolve(process.cwd(), 'modules/components/partials/metadata');
+
+/** Recursively collect .adoc files under a directory (minimatch glob). */
+function collectAdocFiles (dir) {
+  if (!fs.existsSync(dir)) return [];
+  return globSync('**/*.adoc', { cwd: dir, absolute: true, nodir: true });
+}
+
+/**
+ * Migrate inline `== Metadata` blocks in connector pages to regenerated
+ * partials plus include directives.
+ * @param {object} [options]
+ * @param {boolean} [options.write=false] Apply changes (otherwise dry run).
+ * @returns {{migrated:number, skipped:number}}
+ */
+function migrateMetadataToPartials ({ write = false } = {}) {
+  const pages = collectAdocFiles(PAGES_ROOT);
+
+  let migrated = 0;
+  let skipped = 0;
+
+  for (const page of pages) {
+    // Connector pages live in a <type> subdirectory (pages/<type>/<name>.adoc).
+    // Skip pages directly under pages/ — they are not connector pages, and the
+    // parent-directory name would be a bogus type.
+    if (path.dirname(page) === PAGES_ROOT) { skipped++; continue; }
+
+    const content = fs.readFileSync(page, 'utf8');
+    const found = locateMetadata(content);
+    if (!found) { skipped++; continue; }
+
+    // Derive type directory and connector name from the page path.
+    const typeDir = path.basename(path.dirname(page));
+    const name = path.basename(page, '.adoc');
+    const item = { typeDir, name };
+
+    // If the block is already an include, nothing to do.
+    if (/include::connect:components:partial\$metadata\//.test(found.block)) { skipped++; continue; }
+
+    const partialPath = path.join(PARTIALS_ROOT, typeDir, `${name}.adoc`);
+    const partialContent = `${PARTIAL_BANNER}\n\n${found.block}\n`;
+    const newPage = content.slice(0, found.start) + metadataIncludeLine(item) + content.slice(found.end);
+
+    // Prefer a partial already produced by the generator (source of truth).
+    // Only seed one from the page's inline block when none exists yet, so the
+    // migration works standalone but never clobbers generated content.
+    const partialExists = fs.existsSync(partialPath);
+
+    console.log(`${write ? 'MIGRATE' : 'WOULD MIGRATE'}: ${typeDir}/${name}`);
+    console.log(`  -> partial: ${path.relative(process.cwd(), partialPath)}${partialExists ? ' (exists, kept)' : ' (seeded from page)'}`);
+
+    if (write) {
+      if (!partialExists) {
+        fs.mkdirSync(path.dirname(partialPath), { recursive: true });
+        fs.writeFileSync(partialPath, partialContent);
+      }
+      fs.writeFileSync(page, newPage);
+    }
+    migrated++;
+  }
+
+  console.log(`\n${write ? 'Migrated' : 'Would migrate'} ${migrated} page(s); ${skipped} without an inline metadata section.`);
+  if (!write) console.log('Dry run only. Re-run with --write to apply.');
+
+  return { migrated, skipped };
+}
+
+module.exports = { migrateMetadataToPartials };
+
+// Allow direct execution for local development; the supported entry point is
+// `npx doc-tools generate migrate-rpcn-metadata`.
+if (require.main === module) {
+  migrateMetadataToPartials({ write: process.argv.includes('--write') });
+}

package/tools/redpanda-connect/rpcn-connector-docs-handler.js

@@ -1093,6 +1093,7 @@ async function handleRpcnConnectorDocs (options) {
       templateIntro: options.templateIntro,
       templateFields: options.templateFields,
       templateExamples: options.templateExamples,
+      templateMetadata: options.templateMetadata,
       templateBloblang: options.templateBloblang,
       writeFullDrafts: false,
       includeBloblang: !!options.includeBloblang,
@@ -1882,6 +1883,7 @@ async function handleRpcnConnectorDocs (options) {
           template: options.templateMain,
           templateFields: options.templateFields,
           templateExamples: options.templateExamples,
+          templateMetadata: options.templateMetadata,
           templateIntro: options.templateIntro,
           writeFullDrafts: true,
           cgoOnly: binaryAnalysis?.cgoOnly || [],

package/tools/redpanda-connect/templates/intro.hbs

@@ -7,7 +7,7 @@ endif::[]
 {{/if}}
 
 {{#if description}}
-{{{description}}}
+{{{descriptionWithMetadataInclude this}}}
 {{else}}
 // TODO: Missing description. Add a description in the overrides file or source data.
 {{/if}}

package/tools/redpanda-connect/templates/metadata-partials.hbs

@@ -0,0 +1,5 @@
+{{#if (renderConnectMetadata description)}}
+// This content is autogenerated. Do not edit manually. Metadata fields come from the connector's description in the Connect source. To change them, update the connector description upstream, or use the doc-tools CLI with the --overrides option: https://redpandadata.atlassian.net/wiki/spaces/DOC/pages/1247543314/Generate+reference+docs+for+Redpanda+Connect
+
+{{{renderConnectMetadata description}}}
+{{/if}}