@redpanda-data/docs-extensions-and-macros 5.13.0 → 5.13.2

package/tools/redpanda-connect/generate-rpcn-connector-docs.js

@@ -5,7 +5,9 @@ const path = require('path');
 const handlebars = require('handlebars');
 const yaml = require('yaml');
 const helpers = require('./helpers');
-const { lostMetadataSections, resolvePageTypeDir } = require('./metadata-utils.js');
+const {
+  lostMetadataSections, typeDirFor, flattenToAttributeValue, CONNECTOR_TYPE_DIRS, resolvePageTypeDir,
+} = require('./metadata-utils.js');
 
 // Register each helper under handlebars, verifying that it’s a function
 Object.entries(helpers).forEach(([name, fn]) => {
@@ -22,6 +24,12 @@ const DEFAULT_TEMPLATE = path.resolve(__dirname, './templates/connector.hbs');
 // Default template for the regenerated per-connector metadata partial.
 const DEFAULT_METADATA_TEMPLATE = path.resolve(__dirname, './templates/metadata-partials.hbs');
 
+// Default template for the regenerated per-connector description partial. Like
+// the metadata partial, this is rewritten on every run so summary/description
+// changes upstream flow into already-published pages that include it.
+const DEFAULT_DESCRIPTION_TEMPLATE = path.resolve(__dirname, './templates/descriptions-partials.hbs');
+const { hasStructuralHeadings, hasMarkdownHeadings, firstHeadingDepth, escapePlaceholderBraces, LONG_HEADINGLESS_THRESHOLD } = require('./helpers/renderConnectDescription.js');
+
 // Banner-only content written to a metadata partial when its `== Metadata`
 // section is removed upstream. Kept as an AsciiDoc comment so the file renders
 // nothing while any lingering include directive still resolves cleanly.
@@ -30,6 +38,44 @@ const STALE_METADATA_PARTIAL =
   'The connector description no longer contains a `== Metadata` section, so this partial is intentionally empty. ' +
   'You can remove the include directive that points at this file from the connector page.\n';
 
+// Banner-only content written to a description partial when the connector's
+// description is removed upstream. Same rationale as STALE_METADATA_PARTIAL:
+// deleting the file would break any published page that includes it, so blank
+// it instead and log so a human removes the include line.
+//
+// The empty attrs/body regions are load-bearing. Pages include this file with
+// `[tag=attrs]` and `[tag=body]`, and Asciidoctor logs `tag 'attrs' not found
+// in include file` for every tag a target does not define -- so a banner
+// without the regions turns one blanked partial into two build warnings on
+// every page that includes it, on every build, which is a build failure
+// wherever Asciidoctor warnings are treated as errors. An empty region
+// resolves to nothing and says nothing.
+const STALE_DESCRIPTION_PARTIAL =
+  '// This content is autogenerated. Do not edit manually. ' +
+  'The connector no longer has a description, so this partial is intentionally empty. ' +
+  'You can remove the include directive that points at this file from the connector page.\n' +
+  '\n// tag::attrs[]\n// end::attrs[]\n' +
+  '\n// tag::body[]\n// end::body[]\n';
+
+// Component families with their own reference pages, keyed by the canonical
+// type directory from typeDirFor(). Other data keys -- `config`, the bloblang
+// functions and methods -- have no per-component page that could include a
+// partial, so none is emitted for them.
+const CONNECTOR_PAGE_TYPE_DIRS = new Set(CONNECTOR_TYPE_DIRS);
+
+// Blast-radius guard for the post-generation orphan sweep. The sweep blanks
+// every description partial the current dataset did not claim, so an
+// incomplete dataset (a truncated fetch, a half-succeeded --force-fresh, a
+// hand-made subset file) does not look different from "hundreds of connectors
+// were deleted upstream" -- and the partials are published content. Blanking
+// up to ORPHAN_SWEEP_ALWAYS_ALLOWED files is always fine (that is the size of
+// a plausible upstream deletion); above that the sweep also has to stay under
+// ORPHAN_SWEEP_MAX_FRACTION of the tree, or it refuses to run and reports
+// instead. `pruneOrphanedDescriptions` is the explicit opt-in that overrides
+// the refusal once a human has confirmed the dataset is complete.
+const ORPHAN_SWEEP_ALWAYS_ALLOWED = 10;
+const ORPHAN_SWEEP_MAX_FRACTION = 0.1;
+
 /**
  * Reads a file at `filePath` and registers it as a Handlebars partial called `name`.
  * Throws if the file cannot be read.
@@ -216,6 +262,7 @@ function resolveReferences(obj, root) {
  * @param {string} [options.templateFields] - Path to the fields partial template.
  * @param {string} [options.templateExamples] - Path to the examples partial template.
  * @param {boolean} options.writeFullDrafts - If true, generates full draft documentation; otherwise, generates partials.
+ * @param {boolean} [options.pruneOrphanedDescriptions=false] - Allow the post-generation sweep to blank description partials absent from the dataset even when they exceed the safety threshold. Only pass this for a dataset confirmed complete.
  * @returns {Promise<Object>} An object summarizing the number and paths of generated partials and drafts.
  *
  * @throws {Error} If reading or parsing input files fails, if template rendering fails for a component, or if $ref references cannot be resolved.
@@ -257,8 +304,12 @@ async function generateRpcnConnectorDocs(options) {
     templateFields,
     templateExamples,
     templateMetadata,
+    templateDescription,
     templateBloblang,
     writeFullDrafts,
+    // Explicit opt-in for the destructive description-partial orphan sweep.
+    // Off by default: see ORPHAN_SWEEP_MAX_FRACTION.
+    pruneOrphanedDescriptions = false,
     cgoOnly = [],        // Array of cgo-only connectors from cgo binary inspection
     cloudOnly = [],      // Array of cloud-only connectors from cloud binary inspection
     csvMetadata = []     // Array of CSV metadata with support levels
@@ -358,6 +409,12 @@ async function generateRpcnConnectorDocs(options) {
   const metadataTemplatePath = templateMetadata || DEFAULT_METADATA_TEMPLATE;
   registerPartial('metadata', metadataTemplatePath);
 
+  // Always register the description partial. Like metadata, it is regenerated
+  // on every run so summary/description edits upstream flow into any page that
+  // includes it, instead of being frozen into the page body at first draft.
+  const descriptionTemplatePath = templateDescription || DEFAULT_DESCRIPTION_TEMPLATE;
+  registerPartial('description', descriptionTemplatePath);
+
   // In draft mode, also register the intro partial
   if (writeFullDrafts && templateIntro) {
     registerPartial('intro', templateIntro);
@@ -367,6 +424,7 @@ async function generateRpcnConnectorDocs(options) {
   const fieldsOutRoot   = path.join(outputRoot, 'fields');
   const examplesOutRoot = path.join(outputRoot, 'examples');
   const metadataOutRoot = path.join(outputRoot, 'metadata');
+  const descriptionOutRoot = path.join(outputRoot, 'descriptions');
   // Drafts go to pages/, not partials/
   const componentsRoot  = writeFullDrafts
     ? path.resolve(process.cwd(), 'modules/components/pages')
@@ -377,10 +435,22 @@ async function generateRpcnConnectorDocs(options) {
     fs.mkdirSync(fieldsOutRoot,   { recursive: true });
     fs.mkdirSync(examplesOutRoot, { recursive: true });
     fs.mkdirSync(metadataOutRoot, { recursive: true });
+    fs.mkdirSync(descriptionOutRoot, { recursive: true });
   }
 
+  // Compile the one-line partial wrappers once, not once per component: they
+  // are constants, and the loop runs for every entry in every data key.
+  const renderFieldsPartial = handlebars.compile('{{> fields children=config.children}}');
+  const renderExamplesPartial = handlebars.compile('{{> examples examples=examples}}');
+  const renderMetadataPartial = handlebars.compile('{{> metadata description=description type=type typeDir=typeDir name=name}}');
+  const renderDescriptionPartial = handlebars.compile('{{> description summary=summary version=version description=description type=type typeDir=typeDir name=name}}');
+
   let partialsWritten = 0;
+  const descriptionReports = [];
   const lostSectionWarnings = [];
+  // typeDir/name keys handled this run; used after the loop to blank
+  // description partials orphaned by a connector deleted upstream.
+  const visitedDescriptionPartials = new Set();
   let draftsWritten   = 0;
   const partialFiles  = [];
   const draftFiles    = [];
@@ -395,17 +465,23 @@ async function generateRpcnConnectorDocs(options) {
       // Compute typeDir once for this item (used in templates and file paths)
       const typeDir = type.endsWith('s') ? type : `${type}s`;
       item.typeDir = typeDir;
+      // Canonical directory for the partials that pages address by include
+      // line. `typeDir` keeps the raw data-key spelling because the fields
+      // and examples partials are written under it and published pages
+      // already include them by that path; the metadata and description
+      // partials are addressed through typeDirFor(), so writing them under
+      // anything else leaves the include unresolved. Rate limits are the
+      // live case: data key `rate-limits`, include path `rate_limits`.
+      const partialTypeDir = typeDirFor(item);
 
       // Always generate field and example partials (needed for both standalone and draft modes)
       // Render fields using the registered "fields" partial
-      const fieldsOut = handlebars
-        .compile('{{> fields children=config.children}}')(item);
+      const fieldsOut = renderFieldsPartial(item);
 
       // Render examples only if an examples template was provided
       let examplesOut = '';
       if (examplesTemplatePath) {
-        examplesOut = handlebars
-          .compile('{{> examples examples=examples}}')(item);
+        examplesOut = renderExamplesPartial(item);
       }
 
       if (fieldsOut.trim()) {
@@ -418,7 +494,11 @@ async function generateRpcnConnectorDocs(options) {
         }
       }
 
-      if (examplesOut.trim() && type !== 'bloblang-functions' && type !== 'bloblang-methods') {
+      // Same allowlist as the metadata and description partials below: one
+      // gate, one list. The old denylist named only the two bloblang keys, so
+      // any other non-page data key (config today, anything added upstream
+      // tomorrow) leaked partials no page can include.
+      if (examplesOut.trim() && CONNECTOR_PAGE_TYPE_DIRS.has(partialTypeDir)) {
         const ePath = path.join(examplesOutRoot, type, `${name}.adoc`);
         fs.mkdirSync(path.dirname(ePath), { recursive: true });
         fs.writeFileSync(ePath, examplesOut);
@@ -428,15 +508,17 @@ async function generateRpcnConnectorDocs(options) {
         }
       }
 
-      // Render the metadata partial from the connector's description. Only
-      // written when the description contains a `== Metadata` section.
-      const metadataOut = handlebars
-        .compile('{{> metadata description=description type=type typeDir=typeDir name=name}}')(item);
-
-      if (type !== 'bloblang-functions' && type !== 'bloblang-methods') {
-        // Write under typeDir (not the raw data key) so the file path always
-        // matches the include directive built by metadataIncludeLine().
-        const mPath = path.join(metadataOutRoot, typeDir, `${name}.adoc`);
+      if (CONNECTOR_PAGE_TYPE_DIRS.has(partialTypeDir)) {
+        // Render the metadata partial from the connector's description. Only
+        // written when the description contains a `== Metadata` section.
+        // Rendered inside the guard, not before it: the loop covers every
+        // data key, so rendering first meant computing and discarding a
+        // partial for every bloblang function, method and config entry.
+        const metadataOut = renderMetadataPartial(item);
+        // Write under the canonical type directory (not the raw data key) so
+        // the file path always matches the include directive built by
+        // metadataIncludeLine().
+        const mPath = path.join(metadataOutRoot, partialTypeDir, `${name}.adoc`);
         if (metadataOut.trim()) {
           fs.mkdirSync(path.dirname(mPath), { recursive: true });
           if (fs.existsSync(mPath)) {
@@ -478,6 +560,106 @@ async function generateRpcnConnectorDocs(options) {
         }
       }
 
+      // Render the description partial (summary + version + description, with
+      // the `== Metadata` block replaced by its own partial include). Emitted
+      // for every connector type that has reference pages so a page that
+      // includes it always shows the current summary/description instead of
+      // first-draft text. The summary is brace-escaped here, the same way
+      // renderConnectDescription escapes the description body: the template
+      // emits it raw in both the body and the :description: attribute, where
+      // an unescaped {placeholder} is silently consumed as a missing
+      // attribute reference.
+      if (CONNECTOR_PAGE_TYPE_DIRS.has(partialTypeDir)) {
+        const descriptionItem = typeof item.summary === 'string'
+          ? { ...item, summary: escapePlaceholderBraces(item.summary) }
+          : item;
+        const descriptionOut = renderDescriptionPartial(descriptionItem);
+        visitedDescriptionPartials.add(`${partialTypeDir}/${name}`);
+        const dPath = path.join(descriptionOutRoot, partialTypeDir, `${name}.adoc`);
+        if (descriptionOut.trim()) {
+          fs.mkdirSync(path.dirname(dPath), { recursive: true });
+          if (fs.existsSync(dPath)) {
+            // Same content-loss check the metadata partial ten lines up has
+            // had since #236, for the same reason: regeneration is
+            // authoritative, but a section disappearing from a published
+            // partial is content loss, not cleanup. Now that pages include
+            // these files the risk is real -- a section can only vanish
+            // because it left the upstream description or because a human
+            // added it here -- so warn before overwriting instead of
+            // dropping it without a trace.
+            const lost = lostMetadataSections(fs.readFileSync(dPath, 'utf8'), descriptionOut);
+            if (lost.length) {
+              lostSectionWarnings.push({ partial: path.relative(process.cwd(), dPath), sections: lost });
+              console.warn(
+                `Warning: regenerated description partial ${path.relative(process.cwd(), dPath)} ` +
+                `drops previously published section(s): ${lost.map((h) => `"${h}"`).join(', ')}. ` +
+                'If this content is still wanted, move it to the connector page or restore it ' +
+                'in the upstream description.'
+              );
+            }
+          }
+          fs.writeFileSync(dPath, descriptionOut);
+          // The partial now exists for this connector, so a drafted page can
+          // include its body instead of freezing the summary and description
+          // into the page. Gated on the file actually being written:
+          // component families outside CONNECTOR_PAGE_TYPE_DIRS, and
+          // connectors with neither summary nor description, keep the inline
+          // text rather than getting an include that cannot resolve.
+          item.hasDescriptionPartial = true;
+          // The attrs region only sets :description: when the connector has a
+          // summary, so a page may only replace its header attribute with the
+          // include when there is one. Otherwise the page would lose its meta
+          // description entirely and keep the inline attribute instead.
+          if (typeof item.summary === 'string' && item.summary.trim()) {
+            item.hasDescriptionAttrs = true;
+          }
+          if (!writeFullDrafts) {
+            partialsWritten++;
+            partialFiles.push(path.relative(process.cwd(), dPath));
+          }
+          // Long heading-less descriptions render as a wall of text. The fix
+          // belongs upstream (add == sections in the Connect source), so
+          // surface them here instead of hiding them in the docs build.
+          if (
+            typeof item.description === 'string' &&
+            item.description.length > LONG_HEADINGLESS_THRESHOLD &&
+            !hasStructuralHeadings(item.description)
+          ) {
+            const md = hasMarkdownHeadings(item.description);
+            const msg = md
+              ? `Markdown-style headings: ${partialTypeDir}/${name} structures its description with ## headings, ` +
+                'which render unreliably. Convert them to == sections upstream in the Connect source.'
+              : `Long heading-less description: ${partialTypeDir}/${name} ` +
+                `(${item.description.length} chars). Consider adding == sections upstream in the Connect source.`;
+            descriptionReports.push({ connector: `${partialTypeDir}/${name}`, message: msg });
+            console.warn(msg);
+          }
+          // Headings that start deeper than level one render out of
+          // sequence on the page. Upstream fix, so report it.
+          const headingDepth = typeof item.description === 'string' ? firstHeadingDepth(item.description) : null;
+          if (headingDepth !== null && headingDepth > 2) {
+            const seqMsg =
+              `Out-of-sequence headings: ${partialTypeDir}/${name} starts at level ${headingDepth - 1} ` +
+              '(=== or ### with no parent section). Promote the top headings upstream in the Connect source.';
+            descriptionReports.push({ connector: `${partialTypeDir}/${name}`, message: seqMsg });
+            console.warn(seqMsg);
+          }
+        } else if (fs.existsSync(dPath)) {
+          // The upstream description disappeared, but a previously generated
+          // partial is still on disk and an already published page may include
+          // it. Deleting the file would break that include, so blank it to a
+          // banner-only file (the include still resolves and renders nothing)
+          // and log it so a human knows to remove the include line. Without
+          // this, the stale first-draft text would live on disk forever --
+          // exactly the drift these partials exist to prevent.
+          fs.writeFileSync(dPath, STALE_DESCRIPTION_PARTIAL);
+          console.log(
+            `Description removed upstream; emptied stale partial ${path.relative(process.cwd(), dPath)} ` +
+            `(remove its include directive from the ${type}/${name} page)`
+          );
+        }
+      }
+
       if (writeFullDrafts) {
         if (String(item.status || '').toLowerCase() === 'deprecated') {
           console.log(`Skipping draft for deprecated component: ${type}/${name}`);
@@ -677,6 +859,71 @@ async function generateRpcnConnectorDocs(options) {
     }
   }
 
+  // A connector deleted upstream leaves no dataset entry at all, so the
+  // in-loop stale-partial blanking above never sees it and its description
+  // partial would keep serving deleted-connector text forever, silently.
+  // Sweep the description output root for partials no connector claimed this
+  // run and blank them the same way. Skipped in draft mode: drafts run on a
+  // filtered dataset (missing connectors only), so "not visited" there means
+  // "already documented", not "deleted upstream".
+  // The sweep is destructive against published content, so collect the
+  // candidates first and check the blast radius before writing anything.
+  let orphanSweepSkipped = null;
+  if (!writeFullDrafts && fs.existsSync(descriptionOutRoot)) {
+    const orphans = [];
+    let partialsOnDisk = 0;
+    for (const typeDirName of fs.readdirSync(descriptionOutRoot)) {
+      const typeDirPath = path.join(descriptionOutRoot, typeDirName);
+      if (!fs.statSync(typeDirPath).isDirectory()) continue;
+      for (const file of fs.readdirSync(typeDirPath)) {
+        if (!file.endsWith('.adoc')) continue;
+        partialsOnDisk++;
+        const key = `${typeDirName}/${file.slice(0, -'.adoc'.length)}`;
+        if (visitedDescriptionPartials.has(key)) continue;
+        const orphanPath = path.join(typeDirPath, file);
+        // Already blanked on an earlier run: leave it alone so it is not
+        // re-reported forever.
+        if (fs.readFileSync(orphanPath, 'utf8') === STALE_DESCRIPTION_PARTIAL) continue;
+        orphans.push({ key, orphanPath });
+      }
+    }
+
+    // Over the threshold the far likelier explanation is an incomplete
+    // dataset, not a mass upstream deletion, so refuse and report.
+    const overBlastRadius =
+      orphans.length > ORPHAN_SWEEP_ALWAYS_ALLOWED &&
+      orphans.length > partialsOnDisk * ORPHAN_SWEEP_MAX_FRACTION;
+
+    if (orphans.length && overBlastRadius && !pruneOrphanedDescriptions) {
+      const percent = Math.round((orphans.length / partialsOnDisk) * 100);
+      const msg =
+        `Orphan sweep refused: ${orphans.length} of ${partialsOnDisk} description partial(s) (${percent}%) ` +
+        `are absent from this run's dataset, over the ${Math.round(ORPHAN_SWEEP_MAX_FRACTION * 100)}% safety ` +
+        'threshold. That normally means the dataset is truncated or filtered, not that this many connectors ' +
+        'were deleted upstream, and blanking them would wipe published content. Nothing was blanked. ' +
+        'Confirm the dataset is complete, then re-run with --prune-orphaned-descriptions to sweep. ' +
+        `Affected: ${orphans.slice(0, 10).map((o) => o.key).join(', ')}` +
+        (orphans.length > 10 ? `, and ${orphans.length - 10} more` : '');
+      orphanSweepSkipped = {
+        orphans: orphans.length,
+        partialsOnDisk,
+        connectors: orphans.map((o) => o.key),
+      };
+      descriptionReports.push({ connector: 'descriptions (sweep)', message: msg });
+      console.error(msg);
+    } else {
+      for (const { key, orphanPath } of orphans) {
+        fs.writeFileSync(orphanPath, STALE_DESCRIPTION_PARTIAL);
+        const msg =
+          `Connector removed upstream: emptied orphaned description partial ` +
+          `${path.relative(process.cwd(), orphanPath)} ` +
+          `(remove its include directive and retire the ${key} page)`;
+        descriptionReports.push({ connector: key, message: msg });
+        console.log(msg);
+      }
+    }
+  }
+
   // Self-heal published page headers: pages are one-time drafts, so pages
   // created before the template emitted :description: never got one.
   let descriptionBackfill = { backfilled: [], skippedNoSummary: [] };
@@ -689,12 +936,13 @@ async function generateRpcnConnectorDocs(options) {
     draftsWritten,
     partialFiles,
     draftFiles,
+    descriptionReports,
     lostSectionWarnings,
+    orphanSweepSkipped,
     descriptionBackfill
   };
 }
 
-
 // Connector page families that have per-component pages under
 // modules/components/pages/. Other data keys (config, bloblang-functions,
 // bloblang-methods) have no per-component page to backfill.
@@ -740,25 +988,19 @@ function backfillPageDescriptions (connectorData, { pagesRoot, dryRun = false }
       const lines = content.split('\n');
       const headerEnd = lines.findIndex((l) => l.trim() === '');
       const header = lines.slice(0, headerEnd === -1 ? lines.length : headerEnd);
-      if (header.some((l) => l.startsWith(':description:'))) continue;
+      // A page migrated to the description partial carries the attrs include
+      // instead of a literal :description:. Without this arm the backfill
+      // splices a competing meta block into every migrated page, so the page
+      // ends up with a dead include plus a hand-inserted attribute that
+      // silently wins.
+      if (header.some((l) => l.startsWith(':description:') || /partial\$descriptions\//.test(l))) continue;
 
       // Meta descriptions are read as plain text by search results, link
-      // previews, and assistants, so flatten AsciiDoc markup mechanically:
-      // xrefs and links become their labels, inline code loses its
-      // backticks. No prose is edited.
-      const summary = (item.summary || '')
-        .replace(/(?:xref|link):[^\[\]]+\[([^\]]*)\]/g, '$1')
-        // Bare URL macros (no xref:/link: prefix) and internal xref shorthand
-        // also appear in source summaries and render literally if left in place.
-        .replace(/https?:\/\/[^\s\[\]]+\[([^\]]*)\]/g, '$1')
-        .replace(/<<[^,>]+,([^>]+)>>/g, '$1')
-        .replace(/<<([^>]+)>>/g, '$1')
-        // Strip the "open in new tab" caret that AsciiDoc URL macros carry
-        // inside the link text, e.g. [Open Telemetry collector^].
-        .replace(/\^/g, '')
-        .replace(/`([^`]*)`/g, '$1')
-        .replace(/\s+/g, ' ')
-        .trim();
+      // previews, and assistants, so the markup comes out mechanically. Same
+      // flattener the description partial's attrs region uses, so a page
+      // cannot publish one meta description before migrating to the partial
+      // and a different one after.
+      const summary = flattenToAttributeValue(item.summary);
       if (!summary) {
         results.skippedNoSummary.push(`${typeDir}/${item.name}`);
         continue;

package/tools/redpanda-connect/helpers/descriptionInclude.js

@@ -0,0 +1,24 @@
+'use strict';
+
+const { descriptionIncludeLine } = require('../metadata-utils.js');
+
+/**
+ * Handlebars helper: emit the include directive for a connector's description
+ * partial, optionally for one of its tag regions.
+ *
+ *   {{{descriptionInclude this 'attrs'}}}
+ *   {{{descriptionInclude this 'body'}}}
+ *
+ * Drafted pages use this instead of freezing the summary and description into
+ * the page, so an upstream wording change reaches the published page on the
+ * next generator run. Delegates to descriptionIncludeLine so the path a page
+ * carries is built by the same function that decides where the generator
+ * writes the file.
+ *
+ * @param {object} item connector data with `type`/`typeDir` and `name`
+ * @param {string} [tag] `attrs` or `body`
+ * @returns {string}
+ */
+module.exports = function descriptionInclude (item, tag) {
+  return descriptionIncludeLine(item, typeof tag === 'string' ? tag : undefined);
+};

package/tools/redpanda-connect/helpers/index.js

@@ -12,7 +12,10 @@ module.exports = {
   renderConnectFields:    require('./renderConnectFields.js'),
   renderConnectExamples:  require('./renderConnectExamples.js'),
   renderConnectMetadata:  require('./renderConnectMetadata.js'),
+  renderConnectDescription: require('./renderConnectDescription.js'),
+  summaryAttribute: require('./summaryAttribute.js'),
   descriptionWithMetadataInclude: require('./descriptionWithMetadataInclude.js'),
+  descriptionInclude: require('./descriptionInclude.js'),
   renderLeafField:        require('./renderLeafField.js'),
   renderObjectField:      require('./renderObjectField.js'),
   buildConfigYaml:        require('./buildConfigYaml.js'),

package/tools/redpanda-connect/helpers/renderConnectDescription.js

@@ -0,0 +1,125 @@
+'use strict';
+
+// One annotator for every scanner over connector prose, including the ones in
+// metadata-utils that run before these. See annotateVerbatimLines there for
+// why there is exactly one.
+const { descriptionWithMetadataInclude, annotateVerbatimLines } = require('../metadata-utils.js');
+
+// Length (characters) above which a heading-less description is reported as a
+// candidate for upstream structure. Roughly a screen of prose.
+const LONG_HEADINGLESS_THRESHOLD = 1200;
+
+/**
+ * True when the description contains at least one structural heading that sits
+ * outside a `----` listing block or a markdown ```/~~~ fence.
+ *
+ * Used to report long heading-less descriptions during generation. Embedded
+ * headings are otherwise passed through unchanged: they render as top-level
+ * page sections, which matches every published connector page today. Demoting
+ * them is not an option because the description renders before the page's
+ * first `==` section, so demoted headings produce "section title out of
+ * sequence" errors. Collapsible wrapping was evaluated and rejected: it hides
+ * primary content behind unreliable cross-browser find and deep-link behavior,
+ * and bodies containing their own `====` delimiters cannot be wrapped at all.
+ */
+function hasStructuralHeadings (body) {
+  // AsciiDoc headings only. Markdown-style ## headings deliberately do NOT
+  // count: the descriptions that render worst are exactly the ones whose
+  // only structure is markdown headings, and counting them here exempted
+  // those from the long-description report (they get their own report).
+  return annotateVerbatimLines(body)
+    .some(({ line, verbatim }) => !verbatim && /^={2,}\s+\S/.test(line));
+}
+
+/**
+ * True when the description contains a markdown-style heading (`##`) outside
+ * listing blocks and fences. These are reported for upstream conversion to
+ * `==` (a `#` line inside a fence is a comment in an example, not a heading).
+ */
+function hasMarkdownHeadings (body) {
+  return annotateVerbatimLines(body)
+    .some(({ line, verbatim }) => !verbatim && /^#{2,}\s+\S/.test(line));
+}
+
+/**
+ * Handlebars helper: render a connector's description for the regenerated
+ * description partial.
+ *
+ * The `== Metadata` block is replaced by an include of the metadata partial
+ * (de-duplicating it, since metadata is emitted as its own partial).
+ * Everything else passes through unchanged: the description is the page's
+ * primary content, and structure belongs upstream in the connector source.
+ * The generator reports long heading-less descriptions so they get headings
+ * added there instead of being hidden by the docs build.
+ *
+ * @param {object} item connector data (needs `description`, `type`/`typeDir`, `name`)
+ * @returns {string}
+ */
+module.exports = function renderConnectDescription (item) {
+  const body = descriptionWithMetadataInclude(item);
+  if (!body || !body.trim()) return '';
+  return escapePlaceholderBraces(ensureHeadingSeparation(body.trim()));
+};
+
+/**
+ * Insert the blank line Asciidoctor requires before a section title. A
+ * heading glued to the paragraph above it renders as literal text and
+ * orphans its subsections into "section title out of sequence" warnings
+ * (live case: the protobuf processor's "== Operators"). Listing blocks and
+ * fences are untouched — a glued `#` comment inside a fenced example must
+ * not have a blank line pushed into the example.
+ */
+function ensureHeadingSeparation (body) {
+  const out = [];
+  for (const { line, verbatim } of annotateVerbatimLines(body)) {
+    if (
+      !verbatim &&
+      /^(?:={2,6}|#{2,6})\s+\S/.test(line) &&
+      out.length > 0 &&
+      out[out.length - 1].trim() !== ''
+    ) {
+      out.push('');
+    }
+    out.push(line);
+  }
+  return out.join('\n');
+}
+
+/**
+ * Escape template placeholders like {endpoint} so Asciidoctor keeps them
+ * instead of consuming them as attribute references (which substitutes to
+ * nothing and logs "skipping reference to missing attribute" on the
+ * published page — live today on salesforce_graphql and otlp_http).
+ * Single-backtick spans still apply attribute substitution, so spans are
+ * escaped too; `----` listing blocks and ```/~~~ fences are verbatim and stay
+ * untouched (a `\{` inside a fence would render as a literal backslash).
+ */
+function escapePlaceholderBraces (body) {
+  return annotateVerbatimLines(body).map(({ line, verbatim }) => (
+    verbatim ? line : line.replace(/(?<![\\{])\{([a-z][\w.-]{1,30})\}/g, '\\{$1}')
+  )).join('\n');
+}
+
+/**
+ * Depth of the first structural heading (AsciiDoc `=` or markdown-compat
+ * `#`) outside listing blocks and fences, or null when the body has none.
+ * Descriptions whose first heading is deeper than level one (`===`, `###`)
+ * render "section title out of sequence" on the page, so the generator
+ * reports them as upstream fixes (seen on aws_dynamodb_cdc, iceberg,
+ * protobuf).
+ */
+function firstHeadingDepth (body) {
+  for (const { line, verbatim } of annotateVerbatimLines(body)) {
+    if (verbatim) continue;
+    const m = line.match(/^(={2,6}|#{2,6})\s+\S/);
+    if (m) return m[1].length;
+  }
+  return null;
+}
+
+module.exports.hasStructuralHeadings = hasStructuralHeadings;
+module.exports.hasMarkdownHeadings = hasMarkdownHeadings;
+module.exports.escapePlaceholderBraces = escapePlaceholderBraces;
+module.exports.ensureHeadingSeparation = ensureHeadingSeparation;
+module.exports.firstHeadingDepth = firstHeadingDepth;
+module.exports.LONG_HEADINGLESS_THRESHOLD = LONG_HEADINGLESS_THRESHOLD;

package/tools/redpanda-connect/helpers/summaryAttribute.js

@@ -0,0 +1,22 @@
+'use strict';
+
+const { flattenToAttributeValue } = require('../metadata-utils.js');
+
+/**
+ * Handlebars helper: flatten a connector summary into a single-line, plain-text
+ * value suitable for the `:description:` page attribute (and therefore for the
+ * page's `<meta name="description">`).
+ *
+ * Delegates to the shared flattener so the description partial's attrs region
+ * and `backfillPageDescriptions` publish the same text for the same summary.
+ * Collapsing whitespace alone is not enough: summaries carry xrefs, link
+ * macros and inline code, and an unflattened value ships raw
+ * `xref:guides:bloblang/about.adoc[Bloblang mapping]` markup into search
+ * snippets and link previews (verified in a real Antora build).
+ *
+ * @param {string} summary
+ * @returns {string}
+ */
+module.exports = function summaryAttribute (summary) {
+  return flattenToAttributeValue(summary);
+};