@redpanda-data/docs-extensions-and-macros 5.12.4 → 5.13.1

package/tools/redpanda-connect/metadata-utils.js

@@ -18,8 +18,18 @@ const path = require('path');
 
 const METADATA_HEADING = /^==\s+Metadata\s*$/;
 // AsciiDoc listing/literal block delimiter (`----`, possibly longer). Lines
-// inside such blocks must not be treated as headings.
+// inside such blocks must not be treated as headings. Asciidoctor only reads a
+// delimiter at column 0 -- an indented `----` is a literal paragraph -- so the
+// test is against the raw line, never a trimmed one.
 const BLOCK_DELIMITER = /^-{4,}$/;
+// AsciiDoc literal block delimiter (`....`, possibly longer) -- as
+// substitution-free as a ---- listing block, and just as liable to contain a
+// brace or heading-shaped line that must not be touched.
+const LITERAL_DELIMITER = /^\.{4,}$/;
+// Markdown-style fence delimiter (``` or ~~~, possibly with a language tag).
+// Descriptions and metadata blocks carry fenced examples alongside AsciiDoc
+// ---- blocks.
+const FENCE_DELIMITER = /^(`{3,}|~{3,})/;
 // The metadata block ends at the next structural element. Besides the next
 // level-2 heading, this also covers page-level constructs that follow the
 // section when locateMetadata runs against a full reference page (not just a
@@ -29,6 +39,54 @@ const BLOCK_DELIMITER = /^-{4,}$/;
 // swallow the trailing `include::...partial$fields[]` and `// end::single-source[]`.
 const SECTION_END = /^(?:==\s+\S|include::|\/\/\s*(?:tag|end)::)/;
 
+/**
+ * Split `text` into lines annotated with whether each line sits inside (or
+ * delimits) a verbatim region: an AsciiDoc `----` listing block or a markdown
+ * ```/~~~ fence. Every scanner over connector prose walks the text through
+ * this, so fence interiors are treated exactly like listing-block interiors:
+ * content, never headings, never escapable prose, never section terminators.
+ *
+ * Layered state: while one delimiter kind is open, only its own closer
+ * matters. A `----` line inside a fence (or a fence line inside a `----`
+ * block) is content -- treating it as a delimiter leaks the state and
+ * swallows everything after it.
+ *
+ * This is the only such state machine in the tree on purpose. There used to
+ * be three, and they had already drifted: two tested the trimmed line and one
+ * the raw line, so an indented `----` opened a verbatim region for one
+ * scanner and was plain content for another, on the same description in the
+ * same pipeline. Asciidoctor itself only honours a delimiter at column 0, so
+ * the raw-line test is the correct one and the only one here.
+ *
+ * @param {string} text
+ * @returns {Array<{line: string, verbatim: boolean}>}
+ */
+function annotateVerbatimLines (text) {
+  let inBlock = false;
+  let inLiteral = false;
+  let fence = null;
+  return String(text == null ? '' : text).split('\n').map((line) => {
+    if (inBlock) {
+      if (BLOCK_DELIMITER.test(line)) inBlock = false;
+      return { line, verbatim: true };
+    }
+    if (inLiteral) {
+      if (LITERAL_DELIMITER.test(line)) inLiteral = false;
+      return { line, verbatim: true };
+    }
+    if (fence) {
+      const closer = line.match(FENCE_DELIMITER);
+      if (closer && closer[1][0] === fence) fence = null;
+      return { line, verbatim: true };
+    }
+    const fenceMatch = line.match(FENCE_DELIMITER);
+    if (fenceMatch) { fence = fenceMatch[1][0]; return { line, verbatim: true }; }
+    if (BLOCK_DELIMITER.test(line)) { inBlock = true; return { line, verbatim: true }; }
+    if (LITERAL_DELIMITER.test(line)) { inLiteral = true; return { line, verbatim: true }; }
+    return { line, verbatim: false };
+  });
+}
+
 /**
  * Locate the `== Metadata` section within a description.
  * @param {string} description
@@ -39,22 +97,19 @@ const SECTION_END = /^(?:==\s+\S|include::|\/\/\s*(?:tag|end)::)/;
 function locateMetadata (description) {
   if (!description || typeof description !== 'string') return null;
 
-  const lines = description.split('\n');
+  const annotated = annotateVerbatimLines(description);
+  const lines = annotated.map((a) => a.line);
   let headingLine = -1;
-  let inBlock = false;
-  for (let i = 0; i < lines.length; i++) {
-    if (BLOCK_DELIMITER.test(lines[i])) { inBlock = !inBlock; continue; }
-    if (!inBlock && METADATA_HEADING.test(lines[i])) { headingLine = i; break; }
+  for (let i = 0; i < annotated.length; i++) {
+    if (!annotated[i].verbatim && METADATA_HEADING.test(lines[i])) { headingLine = i; break; }
   }
   if (headingLine === -1) return null;
 
   // Find the terminating element after the metadata heading: the next level-2
   // heading, an Antora include directive, or a single-source tag comment.
   let endLine = lines.length;
-  inBlock = false;
-  for (let i = headingLine + 1; i < lines.length; i++) {
-    if (BLOCK_DELIMITER.test(lines[i])) { inBlock = !inBlock; continue; }
-    if (!inBlock && SECTION_END.test(lines[i])) { endLine = i; break; }
+  for (let i = headingLine + 1; i < annotated.length; i++) {
+    if (!annotated[i].verbatim && SECTION_END.test(lines[i])) { endLine = i; break; }
   }
 
   // Trim trailing blank lines inside the section so the block ends cleanly.
@@ -78,16 +133,64 @@ function extractMetadata (description) {
   return found ? found.block : '';
 }
 
+// Rate limits are the one component family the upstream data and the docs
+// repo spell differently: the dataset key is `rate-limits`, `item.type` is
+// `rate_limit`, and the pages directory is `rate_limits`. Every spelling has
+// to collapse to one partial directory or the write path and the include path
+// disagree and the include does not resolve.
+const TYPE_DIR_ALIASES = new Map([
+  ['rate-limit', 'rate_limits'],
+  ['rate-limits', 'rate_limits'],
+  ['rate_limit', 'rate_limits'],
+]);
+
+/**
+ * The component families that have their own reference pages under
+ * `modules/components/pages/<typeDir>/`, in canonical typeDir spelling.
+ *
+ * This list existed in eight places and the copies disagreed three ways:
+ * `config` was in two of them, `rate_limits` was missing from one, and two
+ * carried both rate-limit spellings. A component family added upstream that
+ * is missed by one copy silently gets no partial, no page backfill or no
+ * cloud-docs check, depending on which copy forgot it.
+ * @type {string[]}
+ */
+const CONNECTOR_TYPE_DIRS = [
+  'inputs', 'outputs', 'processors', 'caches', 'rate_limits',
+  'buffers', 'metrics', 'scanners', 'tracers',
+];
+
+/**
+ * Every top-level key of the connector dataset that holds component objects:
+ * the page-backed families plus `config`, which has entries but no per-entry
+ * page. Callers that walk the dataset (augmenting, diffing, stripping) want
+ * this; callers that write per-page artifacts want CONNECTOR_TYPE_DIRS.
+ * @type {string[]}
+ */
+const CONNECTOR_DATA_KEYS = [...CONNECTOR_TYPE_DIRS, 'config'];
+
+/**
+ * Canonical partial directory for a plural type directory. Applied to every
+ * derivation of a type directory so `typeDirFor` is the single source of truth
+ * for both the file the generator writes and the include line a page carries.
+ * @param {string} typeDir
+ * @returns {string}
+ */
+function normalizeTypeDir (typeDir) {
+  return TYPE_DIR_ALIASES.get(typeDir) || typeDir;
+}
+
 /**
- * Derive the plural type directory (for example `input` -> `inputs`).
+ * Derive the plural, canonical type directory (for example `input` ->
+ * `inputs`, `rate-limits` -> `rate_limits`).
  * @param {object} item connector data with `type` and/or `typeDir`
  * @returns {string}
  */
 function typeDirFor (item) {
-  if (item && item.typeDir) return item.typeDir;
+  if (item && item.typeDir) return normalizeTypeDir(item.typeDir);
   const type = item && item.type;
   if (!type) return '';
-  return type.endsWith('s') ? type : `${type}s`;
+  return normalizeTypeDir(type.endsWith('s') ? type : `${type}s`);
 }
 
 /**
@@ -139,9 +242,22 @@ function descriptionWithMetadataInclude (item) {
   return description.slice(0, found.start) + metadataIncludeLine(item) + description.slice(found.end);
 }
 
-// Markdown-style fence delimiter (``` or ~~~, possibly with a language tag).
-// Metadata blocks can carry fenced examples alongside AsciiDoc ---- blocks.
-const FENCE_DELIMITER = /^(`{3,}|~{3,})/;
+/**
+ * Build the Antora include directive for a connector's description partial.
+ * Mirrors {@link metadataIncludeLine} so a drafted page can pull its
+ * auto-generated summary and description from a regenerated partial instead of
+ * freezing them into the page body at first draft.
+ *
+ * The partial defines two tag regions, so pages address it twice: `attrs` in
+ * the page header (sets `:description:`) and `body` where the prose renders.
+ * @param {object} item connector data with `type`/`typeDir` and `name`
+ * @param {string} [tag] tag region to include (`attrs` or `body`); omit for the whole file
+ * @returns {string}
+ */
+function descriptionIncludeLine (item, tag) {
+  const attrs = tag ? `tag=${tag}` : '';
+  return `include::connect:components:partial$descriptions/${typeDirFor(item)}/${item.name}.adoc[${attrs}]`;
+}
 
 /**
  * Collect the section heading titles in an AsciiDoc block, ignoring lines
@@ -155,25 +271,8 @@ const FENCE_DELIMITER = /^(`{3,}|~{3,})/;
 function sectionHeadings (text) {
   if (!text || typeof text !== 'string') return [];
   const headings = [];
-  let inBlock = false;
-  let fence = null;
-  for (const line of text.split('\n')) {
-    // Layered state: while inside one delimiter kind, the only thing that
-    // matters is its own closer. A fence-like line inside a ---- literal
-    // block (or a ---- line inside a fence) is content, not a delimiter —
-    // treating it as one leaks the state and swallows every later heading.
-    if (inBlock) {
-      if (BLOCK_DELIMITER.test(line)) inBlock = false;
-      continue;
-    }
-    if (fence) {
-      const closer = line.match(FENCE_DELIMITER);
-      if (closer && closer[1][0] === fence) fence = null;
-      continue;
-    }
-    const fenceMatch = line.match(FENCE_DELIMITER);
-    if (fenceMatch) { fence = fenceMatch[1][0]; continue; }
-    if (BLOCK_DELIMITER.test(line)) { inBlock = true; continue; }
+  for (const { line, verbatim } of annotateVerbatimLines(text)) {
+    if (verbatim) continue;
     const m = line.match(/^=+\s+(\S.*)$/);
     if (m) headings.push(m[1].trim());
   }
@@ -196,13 +295,62 @@ function lostMetadataSections (oldContent, newContent) {
   return sectionHeadings(oldContent).filter((h) => !newHeadings.has(h));
 }
 
+/**
+ * Flatten AsciiDoc prose into a value usable as a `:description:` attribute
+ * and, downstream, as the page's `<meta name="description">`.
+ *
+ * Meta descriptions are read as plain text by search results, link previews
+ * and assistants, so markup has to come out mechanically: xref and link
+ * macros become their labels, bare URL macros become their labels, `<<>>`
+ * shorthand becomes its text, the "open in new tab" caret goes, inline code
+ * loses its backticks, and whitespace (including hard line breaks, which an
+ * attribute value cannot carry) collapses to single spaces. No prose is
+ * edited.
+ *
+ * This is the one implementation: `summaryAttribute` (the description
+ * partial's attrs region) and `backfillPageDescriptions` (the page-header
+ * self-heal) both call it, so the two paths cannot drift into publishing
+ * different meta descriptions for the same summary.
+ *
+ * @param {string} text
+ * @returns {string}
+ */
+function flattenToAttributeValue (text) {
+  if (!text) return '';
+  return String(text)
+    .replace(/(?:xref|link):[^\[\]]+\[([^\]]*)\]/g, '$1')
+    // Bare URL macros (no xref:/link: prefix) and internal xref shorthand
+    // also appear in source summaries and render literally if left in place.
+    .replace(/https?:\/\/[^\s\[\]]+\[([^\]]*)\]/g, '$1')
+    // image:: and inline image: macros have no textual value in a meta
+    // description; drop them rather than leaking the target path.
+    .replace(/image:{1,2}[^\[\]\s]*\[[^\]]*\]/g, '')
+    .replace(/<<[^,>]+,([^>]+)>>/g, '$1')
+    .replace(/<<([^>]+)>>/g, '$1')
+    // Strip the "open in new tab" caret that AsciiDoc URL macros carry
+    // inside the link text, e.g. [Open Telemetry collector^].
+    .replace(/\^/g, '')
+    .replace(/`([^`]*)`/g, '$1')
+    .replace(/\s+/g, ' ')
+    .trim();
+}
+
 module.exports = {
   locateMetadata,
+  flattenToAttributeValue,
+  CONNECTOR_TYPE_DIRS,
+  CONNECTOR_DATA_KEYS,
+  normalizeTypeDir,
   resolvePageTypeDir,
   extractMetadata,
   typeDirFor,
   metadataIncludeLine,
   descriptionWithMetadataInclude,
+  descriptionIncludeLine,
   sectionHeadings,
   lostMetadataSections,
+  annotateVerbatimLines,
+  BLOCK_DELIMITER,
+  LITERAL_DELIMITER,
+  FENCE_DELIMITER,
 };

package/tools/redpanda-connect/migrate-descriptions-to-partials.js

@@ -0,0 +1,275 @@
+'use strict';
+
+/**
+ * One-time migration that makes published connector pages consume the
+ * regenerated description partial.
+ *
+ * Connector reference pages are one-time first drafts: the summary, the
+ * "Introduced in version" line and the description prose were frozen into
+ * `modules/components/pages/<type>/<name>.adoc` when the page was drafted, and
+ * nothing has refreshed them since. The generator now writes the same content
+ * to `modules/components/partials/descriptions/<type>/<name>.adoc` on every
+ * run, with two tag regions. This script rewires existing pages to it:
+ *
+ *   header: include::...[tag=attrs]  inside the page's `// tag::meta[]` region
+ *   body:   include::...[tag=body]   where the frozen prose is today
+ *
+ * The header rewire is mechanical and safe: it swaps one `:description:` line
+ * for an include that sets the same attribute from the partial.
+ *
+ * The body rewire is a rewrite of published prose, so it is guarded. The page's
+ * intro region is only replaced when every non-blank line of it also appears in
+ * the partial's body region. Pages whose intro carries anything the partial does
+ * not reproduce (hand-added prose, a hand-written config listing, an extra
+ * admonition) are reported and left alone for a human, because migrating them
+ * would silently delete published content.
+ *
+ * Usage (run from the docs repo root, after a generator run so the partials
+ * exist):
+ *   npx doc-tools generate migrate-rpcn-descriptions             # dry run
+ *   npx doc-tools generate migrate-rpcn-descriptions --write     # apply
+ *   npx doc-tools generate migrate-rpcn-descriptions --skip-body # headers only
+ */
+
+const fs = require('fs');
+const path = require('path');
+const { descriptionIncludeLine, normalizeTypeDir, flattenToAttributeValue } = require('./metadata-utils.js');
+
+// The page's meta tag region: the block single-source stubs inherit
+// :description: from, so the attrs include belongs inside it.
+const META_TAG_OPEN = /^\/\/\s*tag::meta\[\]\s*$/;
+const META_TAG_CLOSE = /^\/\/\s*end::meta\[\]\s*$/;
+
+// The intro region starts after the component-type dropdown macro, which every
+// connector page carries directly below its header.
+const INTRO_START = /^component_type_dropdown::\[\]\s*$/;
+
+// ...and ends at the first structural element after it. `include::` covers the
+// metadata, fields and examples partials; `[tabs]` covers the generated config
+// listing; `== ` covers the first real section. Fenced code blocks are
+// deliberately NOT terminators: a hand-written config listing inside the intro
+// has to stay inside the region so the content check below sees it and refuses.
+const INTRO_END = /^(?:==\s+\S|include::|\[tabs\]|\/\/\s*end::single-source)/;
+
+/** Index of the intro anchor, or the end of the file when there is none. */
+function introOf (lines) {
+  const at = lines.findIndex((l) => INTRO_START.test(l));
+  return at === -1 ? lines.length : at;
+}
+
+/** Extract the lines of one tag region from a partial. */
+function tagRegion (partial, tag) {
+  const lines = partial.split('\n');
+  const open = lines.findIndex((l) => new RegExp(`^//\\s*tag::${tag}\\[\\]\\s*$`).test(l));
+  if (open === -1) return null;
+  const close = lines.findIndex((l, i) => i > open && new RegExp(`^//\\s*end::${tag}\\[\\]\\s*$`).test(l));
+  if (close === -1) return null;
+  return lines.slice(open + 1, close);
+}
+
+/** Lines that carry content: blanks and AsciiDoc comments do not count. */
+function contentLines (lines) {
+  return lines.map((l) => l.trim()).filter((l) => l !== '' && !l.startsWith('//'));
+}
+
+/**
+ * True when the partial's body region reproduces every content line of the
+ * page's intro region. This is the guard that keeps the migration from
+ * deleting published prose the generator does not know about.
+ */
+function bodyCoversIntro (introLines, bodyLines) {
+  const body = new Set(contentLines(bodyLines));
+  return contentLines(introLines).every((l) => body.has(l));
+}
+
+/**
+ * Migrate published connector pages onto the regenerated description partial.
+ * @param {object} [options]
+ * @param {boolean} [options.write=false] Apply changes (otherwise dry run).
+ * @param {boolean} [options.skipBody=false] Only rewire page headers.
+ * @param {string} [options.pagesRoot]
+ * @param {string} [options.partialsRoot]
+ * @returns {{headers:string[], bodies:string[], skipped:Array<{page:string, reason:string}>}}
+ */
+function migrateDescriptionsToPartials ({
+  write = false,
+  skipBody = false,
+  pagesRoot,
+  partialsRoot,
+} = {}) {
+  const pages = pagesRoot || path.resolve(process.cwd(), 'modules/components/pages');
+  const partials = partialsRoot || path.resolve(process.cwd(), 'modules/components/partials/descriptions');
+  const results = { headers: [], bodies: [], skipped: [] };
+  if (!fs.existsSync(pages)) return results;
+
+  for (const typeDirName of fs.readdirSync(pages).sort()) {
+    const typeDirPath = path.join(pages, typeDirName);
+    if (!fs.statSync(typeDirPath).isDirectory()) continue;
+    // The pages directory and the partials directory can spell a component
+    // family differently (pages/rate_limits vs the rate-limits data key), so
+    // resolve the partial through the same normalizer the generator writes
+    // with rather than by string equality.
+    const partialTypeDir = normalizeTypeDir(typeDirName);
+
+    for (const file of fs.readdirSync(typeDirPath).sort()) {
+      if (!file.endsWith('.adoc')) continue;
+      const name = file.slice(0, -'.adoc'.length);
+      const key = `${typeDirName}/${name}`;
+      const pagePath = path.join(typeDirPath, file);
+      const partialPath = path.join(partials, partialTypeDir, file);
+
+      if (!fs.existsSync(partialPath)) {
+        results.skipped.push({ page: key, reason: 'no generated description partial' });
+        continue;
+      }
+      const partial = fs.readFileSync(partialPath, 'utf8');
+      const attrs = tagRegion(partial, 'attrs');
+      const body = tagRegion(partial, 'body');
+      if (!attrs || !body || contentLines(body).length === 0) {
+        // A blanked partial (description removed upstream) has both regions
+        // but no content. Wiring a page to it would publish an empty page.
+        results.skipped.push({ page: key, reason: 'description partial is empty' });
+        continue;
+      }
+
+      const includeAttrs = descriptionIncludeLine({ typeDir: partialTypeDir, name }, 'attrs');
+      const includeBody = descriptionIncludeLine({ typeDir: partialTypeDir, name }, 'body');
+
+      let lines = fs.readFileSync(pagePath, 'utf8').split('\n');
+      let changed = false;
+
+      // --- header ---------------------------------------------------------
+      if (!lines.some((l) => l.trim() === includeAttrs)) {
+        const open = lines.findIndex((l) => META_TAG_OPEN.test(l));
+        const close = lines.findIndex((l, i) => i > open && META_TAG_CLOSE.test(l));
+        const attrValue = contentLines(attrs)
+          .find((l) => l.startsWith(':description:'));
+        const pageAttr = lines.slice(0, introOf(lines)).find((l) => l.startsWith(':description:'));
+        if (!attrValue) {
+          // The connector has no summary, so the attrs region sets nothing.
+          // Replacing a hand-written :description: with it would drop the
+          // page's meta description.
+          results.skipped.push({ page: key, reason: 'attrs region sets no :description:' });
+        } else if (pageAttr && flattenToAttributeValue(pageAttr) !== flattenToAttributeValue(attrValue)) {
+          // The page's meta description is not the connector summary, so it
+          // was authored by hand (often tuned for search) and swapping in the
+          // include would silently discard that editorial work. 13 pages in
+          // rp-connect-docs are in this state. A human decides these: either
+          // move the wording upstream into the summary, or leave the page
+          // header alone.
+          results.skipped.push({ page: key, reason: 'hand-authored :description: differs from the summary (needs a human)' });
+        } else if (open !== -1 && close > open) {
+          lines.splice(open + 1, close - open - 1, includeAttrs);
+          results.headers.push(key);
+          changed = true;
+        } else {
+          // Header = everything above the dropdown macro, not "up to the
+          // first blank line": a handful of pages carry a blank line inside
+          // their attribute list, which is how they ended up with two
+          // :description: lines in the first place.
+          const attrLine = lines.findIndex((l) => l.startsWith(':description:'));
+          if (attrLine !== -1 && attrLine < introOf(lines)) {
+            lines.splice(attrLine, 1, '// tag::meta[]', includeAttrs, '// end::meta[]');
+            results.headers.push(key);
+            changed = true;
+          } else {
+            results.skipped.push({ page: key, reason: 'no header :description: to replace' });
+          }
+        }
+      }
+
+      // A handful of pages carry a blank line inside their header, so the
+      // page-header backfill inserted its meta block above a `:description:`
+      // it could not see and the page ended up with two. AsciiDoc takes the
+      // last assignment, so leaving the literal one there would make the
+      // include a no-op and the page would keep publishing frozen text.
+      if (lines.some((l) => l.trim() === includeAttrs)) {
+        for (let i = introOf(lines) - 1; i >= 0; i--) {
+          if (lines[i].startsWith(':description:')) {
+            lines.splice(i, 1);
+            changed = true;
+          }
+        }
+      }
+
+      // --- body -----------------------------------------------------------
+      if (!skipBody && !lines.some((l) => l.trim() === includeBody)) {
+        const start = lines.findIndex((l) => INTRO_START.test(l));
+        if (start === -1) {
+          results.skipped.push({ page: key, reason: 'no component_type_dropdown to anchor the intro' });
+        } else {
+          let end = lines.length;
+          for (let i = start + 1; i < lines.length; i++) {
+            if (INTRO_END.test(lines[i])) { end = i; break; }
+          }
+          const intro = lines.slice(start + 1, end);
+          if (contentLines(intro).length === 0) {
+            results.skipped.push({ page: key, reason: 'intro region is empty' });
+          } else if (!bodyCoversIntro(intro, body)) {
+            results.skipped.push({
+              page: key,
+              reason: 'intro prose is not reproduced by the partial (needs a human)',
+            });
+          } else {
+            // The page's own metadata include is emitted by the partial body
+            // too, so consume it rather than leaving a duplicate behind.
+            let tail = end;
+            const metaInclude = `include::connect:components:partial$metadata/${partialTypeDir}/${name}.adoc[]`;
+            if (lines[tail] && lines[tail].trim() === metaInclude &&
+                contentLines(body).includes(metaInclude)) {
+              tail++;
+            }
+            // Some pages carry the rest of the description BELOW the generated
+            // config listing, so replacing the intro alone would leave the
+            // include and the surviving prose both rendering it. Measured on
+            // caches/redpanda, where an Antora build reported the same xref
+            // twice after the naive rewrite. Anything outside the region that
+            // the partial body also renders means the description is split
+            // across the page, which a human has to untangle.
+            const outside = contentLines(lines.slice(0, start + 1).concat(lines.slice(tail)));
+            const inBody = new Set(contentLines(body));
+            if (outside.some((l) => inBody.has(l))) {
+              results.skipped.push({
+                page: key,
+                reason: 'description prose continues past the config listing (needs a human)',
+              });
+            } else {
+              lines.splice(start + 1, tail - start - 1, '', includeBody, '');
+              results.bodies.push(key);
+              changed = true;
+            }
+          }
+        }
+      }
+
+      if (changed) {
+        console.log(`${write ? 'MIGRATE' : 'WOULD MIGRATE'}: ${key}`);
+        if (write) fs.writeFileSync(pagePath, lines.join('\n'), 'utf8');
+      }
+    }
+  }
+
+  console.log(
+    `\n${write ? 'Migrated' : 'Would migrate'} ${results.headers.length} page header(s) and ` +
+    `${results.bodies.length} page body/bodies; ${results.skipped.length} left alone.`
+  );
+  const byReason = new Map();
+  for (const s of results.skipped) byReason.set(s.reason, (byReason.get(s.reason) || 0) + 1);
+  for (const [reason, count] of [...byReason].sort((a, b) => b[1] - a[1])) {
+    console.log(`  ${count} ${reason}`);
+  }
+  if (!write) console.log('Dry run only. Re-run with --write to apply.');
+
+  return results;
+}
+
+module.exports = { migrateDescriptionsToPartials, tagRegion, bodyCoversIntro };
+
+// Allow direct execution for local development; the supported entry point is
+// `npx doc-tools generate migrate-rpcn-descriptions`.
+if (require.main === module) {
+  migrateDescriptionsToPartials({
+    write: process.argv.includes('--write'),
+    skipBody: process.argv.includes('--skip-body'),
+  });
+}

package/tools/redpanda-connect/normalize-metadata.js

@@ -24,8 +24,10 @@ const FENCE_OPEN = /^(\s*)(```|~~~)(.*)$/;
 const BULLET = /^(\s*[-*]\s+)(.*)$/;
 // AsciiDoc literal/listing block delimiter. Content inside `----` blocks is
 // rendered verbatim, so backticks would show up literally — such regions must
-// pass through untouched (no field-name inline-coding).
-const BLOCK_DELIMITER = /^-{4,}$/;
+// pass through untouched (no field-name inline-coding). Shared with the
+// scanners in metadata-utils rather than redeclared: three copies of this had
+// already drifted into disagreeing about indented delimiters.
+const { BLOCK_DELIMITER } = require('./metadata-utils.js');
 // A metadata field bullet: the leading token is a lowercase snake_case
 // identifier (gcs_key, http_server_verb, header), optionally followed by a
 // parenthetical annotation (for example "(RFC3339)"), and optionally a
@@ -71,12 +73,12 @@ function normalizeMetadataBlock (block) {
     // AsciiDoc literal/listing block: pass the whole `----` … `----` region
     // through verbatim. Field-name lines inside are rendered literally, so they
     // must not be inline-coded (backticks would appear as visible characters).
-    if (BLOCK_DELIMITER.test(lines[i].trim())) {
+    if (BLOCK_DELIMITER.test(lines[i])) {
       out.push(lines[i]);
       let j = i + 1;
       for (; j < lines.length; j++) {
         out.push(lines[j]);
-        if (BLOCK_DELIMITER.test(lines[j].trim())) break;
+        if (BLOCK_DELIMITER.test(lines[j])) break;
       }
       i = j;
       continue;

package/tools/redpanda-connect/pr-summary-formatter.js

@@ -115,6 +115,29 @@ function renderBloblangChanges(data, includeVersion = false) {
   return lines;
 }
 
+/**
+ * Render upstream description-structure findings as a collapsed section of
+ * the PR summary. These are advisory (the fix belongs in the Connect
+ * source), but console.warn output dies in the collapsed workflow log, so
+ * they surface here where the PR reviewer actually looks.
+ * @param {Array<{connector: string, message: string}>} reports
+ * @returns {string[]} lines (empty when there is nothing to report)
+ */
+function renderDescriptionReports(reports) {
+  if (!Array.isArray(reports) || reports.length === 0) return [];
+  const lines = [];
+  lines.push('<details>');
+  lines.push(`<summary>:pencil2: ${reports.length} description(s) need structure fixes upstream in the Connect source</summary>`);
+  lines.push('');
+  for (const r of reports) {
+    lines.push(`- ${r.message}`);
+  }
+  lines.push('');
+  lines.push('</details>');
+  lines.push('');
+  return lines;
+}
+
 /**
  * Generate PR summary for multiple releases
  * @param {object} masterDiff - Master diff with releases array
@@ -153,6 +176,7 @@ function generateMultiVersionPRSummary(masterDiff, binaryAnalysis = null, drafte
   lines.push('<!-- PR_SUMMARY_START -->');
   lines.push('');
   lines.push(...renderLostSectionWarnings(masterDiff?.lostSectionWarnings));
+  lines.push(...renderDescriptionReports(masterDiff?.descriptionReports));
   lines.push('## Redpanda Connect Documentation Update');
   lines.push('');
   lines.push(`**Multi-Release Update:** ${startVersion} → ${endVersion}`);
@@ -559,6 +583,7 @@ function generatePRSummary(diffData, binaryAnalysis = null, draftedConnectors =
   lines.push('<!-- PR_SUMMARY_START -->');
   lines.push('');
   lines.push(...renderLostSectionWarnings(diffData.lostSectionWarnings));
+  lines.push(...renderDescriptionReports(diffData.descriptionReports));
 
   // Detect if this is a master diff
   if (!isMultiVersion && diffData.releases && diffData.totalSummary) {
@@ -1326,9 +1351,35 @@ function printPRSummary(diffData, binaryAnalysis = null, draftedConnectors = nul
   console.log('\n' + summary + '\n');
 }
 
+/**
+ * Merge one generateRpcnConnectorDocs result into the accumulators that feed
+ * this PR summary.
+ *
+ * Every report key the generator returns has to be collected at every call
+ * site, or the summary is quietly incomplete for whichever path forgot one.
+ * The draft call site pushed lostSectionWarnings inline and forgot
+ * descriptionReports, so structure reports for newly drafted connectors --
+ * the ones most likely to need an upstream fix -- never reached the summary.
+ * Doing the merge in one place is what makes that structural rather than
+ * something each call site has to remember.
+ *
+ * @param {object} result generator result
+ * @param {{descriptionReports: Array, lostSectionWarnings: Array}} into accumulators, mutated in place
+ * @returns {{descriptionReports: Array, lostSectionWarnings: Array}} the same accumulators
+ */
+function collectGeneratorReports (result, into) {
+  if (!result) return into;
+  into.descriptionReports.push(...(result.descriptionReports || []));
+  into.lostSectionWarnings.push(...(result.lostSectionWarnings || []));
+  return into;
+}
+
 module.exports = {
+  collectGeneratorReports,
   generatePRSummary,
   generateMultiVersionPRSummary,
   printPRSummary,
+  renderDescriptionReports,
+  renderLostSectionWarnings,
   truncateToSentence
 };