@redpanda-data/docs-extensions-and-macros 5.7.1 → 5.9.0

package/tools/redpanda-connect/generate-rpcn-connector-docs.js

@@ -55,7 +55,10 @@ function mergeOverrides(target, overrides) {
     throw new Error('Target must be a valid object');
   }
 
-  const scalarKeys = ['description', 'type', 'annotated_field', 'version'];
+  // 'summary' earns its place here the hard way: overrides.json has carried
+  // summary overrides (zmq4, ffi) that were silently dropped because the key
+  // fell through every branch below. Summaries feed page meta descriptions.
+  const scalarKeys = ['description', 'summary', 'type', 'annotated_field', 'version'];
 
   for (const key in overrides) {
     // === Handle annotated_options ===
@@ -674,17 +677,120 @@ async function generateRpcnConnectorDocs(options) {
     }
   }
 
+  // Self-heal published page headers: pages are one-time drafts, so pages
+  // created before the template emitted :description: never got one.
+  let descriptionBackfill = { backfilled: [], skippedNoSummary: [] };
+  if (!writeFullDrafts) {
+    descriptionBackfill = backfillPageDescriptions(dataObj);
+  }
+
   return {
     partialsWritten,
     draftsWritten,
     partialFiles,
     draftFiles,
-    lostSectionWarnings
+    lostSectionWarnings,
+    descriptionBackfill
   };
 }
 
+
+// Connector page families that have per-component pages under
+// modules/components/pages/. Other data keys (config, bloblang-functions,
+// bloblang-methods) have no per-component page to backfill.
+const PAGE_TYPE_DIRS = new Set([
+  'inputs', 'outputs', 'processors', 'caches', 'buffers', 'metrics',
+  'tracers', 'scanners', 'rate-limits', 'rate_limits'
+]);
+
+/**
+ * Insert a tagged :description: into existing connector page headers that
+ * lack one, sourced from the component's summary in the connector data.
+ *
+ * Connector pages are one-time first drafts: pages drafted before the
+ * template emitted :description: never got one, and nothing rewrote them,
+ * so hundreds of published pages ship the generic site meta description
+ * even though ~97% of components carry a summary in the source data. This
+ * runs on every generation, so the page set self-heals and stays healed.
+ *
+ * The inserted block matches the connector template's shape: the meta tag
+ * region lets single-source stubs inherit the description as a header
+ * attribute (see the template comment in connector.hbs).
+ *
+ * @param {Object} connectorData - Parsed connector data (typed arrays)
+ * @param {Object} [options]
+ * @param {string} [options.pagesRoot] - Pages directory (default: modules/components/pages under cwd)
+ * @param {boolean} [options.dryRun] - Report without writing
+ * @returns {{backfilled: string[], skippedNoSummary: string[]}}
+ */
+function backfillPageDescriptions (connectorData, { pagesRoot, dryRun = false } = {}) {
+  const root = pagesRoot || path.resolve(process.cwd(), 'modules/components/pages');
+  const results = { backfilled: [], skippedNoSummary: [] };
+  if (!fs.existsSync(root)) return results;
+
+  for (const [dataKey, items] of Object.entries(connectorData)) {
+    if (!PAGE_TYPE_DIRS.has(dataKey) || !Array.isArray(items)) continue;
+    // The data uses 'rate-limits' while the pages directory is
+    // 'rate_limits'. Resolve whichever spelling exists on disk.
+    const typeDir = fs.existsSync(path.join(root, dataKey))
+      ? dataKey
+      : dataKey.replace(/-/g, '_');
+    for (const item of items) {
+      if (!item || !item.name) continue;
+      const pagePath = path.join(root, typeDir, `${item.name}.adoc`);
+      if (!fs.existsSync(pagePath)) continue;
+
+      const content = fs.readFileSync(pagePath, 'utf8');
+      const lines = content.split('\n');
+      const headerEnd = lines.findIndex((l) => l.trim() === '');
+      const header = lines.slice(0, headerEnd === -1 ? lines.length : headerEnd);
+      if (header.some((l) => l.startsWith(':description:'))) continue;
+
+      // Meta descriptions are read as plain text by search results, link
+      // previews, and assistants, so flatten AsciiDoc markup mechanically:
+      // xrefs and links become their labels, inline code loses its
+      // backticks. No prose is edited.
+      const summary = (item.summary || '')
+        .replace(/(?:xref|link):[^\[\]]+\[([^\]]*)\]/g, '$1')
+        // Bare URL macros (no xref:/link: prefix) and internal xref shorthand
+        // also appear in source summaries and render literally if left in place.
+        .replace(/https?:\/\/[^\s\[\]]+\[([^\]]*)\]/g, '$1')
+        .replace(/<<[^,>]+,([^>]+)>>/g, '$1')
+        .replace(/<<([^>]+)>>/g, '$1')
+        // Strip the "open in new tab" caret that AsciiDoc URL macros carry
+        // inside the link text, e.g. [Open Telemetry collector^].
+        .replace(/\^/g, '')
+        .replace(/`([^`]*)`/g, '$1')
+        .replace(/\s+/g, ' ')
+        .trim();
+      if (!summary) {
+        results.skippedNoSummary.push(`${typeDir}/${item.name}`);
+        continue;
+      }
+
+      const insertAt = headerEnd === -1 ? lines.length : headerEnd;
+      lines.splice(insertAt, 0, '// tag::meta[]', `:description: ${summary}`, '// end::meta[]');
+      if (!dryRun) fs.writeFileSync(pagePath, lines.join('\n'), 'utf8');
+      results.backfilled.push(`${typeDir}/${item.name}`);
+    }
+  }
+
+  if (results.backfilled.length) {
+    console.log(`Backfilled :description: into ${results.backfilled.length} connector page header(s) from source summaries`);
+  }
+  if (results.skippedNoSummary.length) {
+    console.warn(
+      `\u26a0 ${results.skippedNoSummary.length} connector page(s) lack both a header description and a source summary: ` +
+      results.skippedNoSummary.join(', ') +
+      '. Author these in the overrides file or upstream.'
+    );
+  }
+  return results;
+}
+
 module.exports = {
   generateRpcnConnectorDocs,
+  backfillPageDescriptions,
   mergeOverrides,
   resolveReferences
 };

package/tools/redpanda-connect/templates/bloblang-functions-overview.hbs

@@ -1,6 +1,8 @@
 = Bloblang Functions
 // tag::single-source[]
+// tag::meta[]
 :description: A list of Bloblang functions
+// end::meta[]
 
 // © 2024 Redpanda Data Inc.
 

package/tools/redpanda-connect/templates/bloblang-methods-overview.hbs

@@ -1,6 +1,8 @@
 = Bloblang Methods
 // tag::single-source[]
+// tag::meta[]
 :description: A list of Bloblang methods
+// end::meta[]
 
 // © 2024 Redpanda Data Inc.
 

package/tools/redpanda-connect/templates/connector.hbs

@@ -3,7 +3,19 @@
 :type: {{type}}
 {{#if support}}:support: {{support}}{{/if}}
 :categories: [{{join categories ", "}}]
-:description: {{#if summary}}{{{summary}}}{{else if description}}{{description}}{{else}}// TODO: Missing description. Add a description in the overrides file or source data.{{/if}}
+{{#if summary}}
+// tag::meta[]
+:description: {{{summary}}}
+// end::meta[]
+{{else if description}}
+// tag::meta[]
+:description: {{description}}
+// end::meta[]
+{{/if}}
+{{!-- No fallback line when neither exists: an emitted TODO would become
+     the attribute VALUE and stubs would inherit it into their meta tags.
+     Missing descriptions are the tracked editorial backlog
+     (cloud-docs .github/description-allowlist.txt). --}}
 
 component_type_dropdown::[]
 

package/tools/rpk-docs/generate-plugin-stubs.js

@@ -23,7 +23,7 @@ const { spawnSync } = require('child_process')
  * authoritative: dashified filenames cannot be reversed unambiguously
  * (rpk-ai-llm-provider could be `llm provider` or `llm-provider`).
  * @param {string} partialsDir - Directory of generated .adoc partials
- * @returns {Array<{file: string, title: string, description: string|undefined}>} Sorted by command path
+ * @returns {Array<{file: string, title: string, description: string|undefined, hasMetaTag: boolean}>} Sorted by command path
  */
 function readPartialTitles(partialsDir) {
   const partials = []
@@ -35,11 +35,18 @@ function readPartialTitles(partialsDir) {
       console.warn(`Warning: no title line in ${file}; skipping`)
       continue
     }
-    // The partial repeats :description: inside its single-source tag, but
-    // Antora resolves page metadata with a header-only parse that never sees
-    // the include — the stub must carry the description in its own header.
+    // Antora resolves page metadata with a header-only parse that never
+    // sees the body include, so the stub must carry the description in its
+    // own header. Partials generated with doc-tools >= 5.4 carry a nested
+    // meta tag region around :description: that stubs include dynamically;
+    // older partials fall back to a literal copy of the header description.
     const description = (content.match(/^:description:[ \t]*(.+)$/m) || [])[1]
-    partials.push({ file, title: match[1].trim(), description: description && description.trim() })
+    partials.push({
+      file,
+      title: match[1].trim(),
+      description: description && description.trim(),
+      hasMetaTag: content.includes('tag::meta['),
+    })
   }
   // Hierarchical order: sort by command words so parents precede children
   partials.sort((a, b) => {
@@ -121,11 +128,23 @@ function inferIncludePrefix(stubDir, plugin) {
  * @param {string} [params.description] - Meta description from the partial header
  * @param {string} params.includePrefix - Antora resource prefix
  * @param {Array<string>} params.attributes - Page attribute lines
+ * @param {boolean} [params.hasMetaTag] - Partial carries a tag::meta[] region (doc-tools >= 5.4)
  * @returns {string}
  */
-function renderStub({ title, file, description, includePrefix, attributes }) {
+function renderStub({ title, file, description, includePrefix, attributes, hasMetaTag }) {
   const lines = [`= ${title}`]
-  if (description) lines.push(`:description: ${description}`)
+  // The description mechanism is always the first header line after the
+  // title: a header include for partials with a meta tag region, or a
+  // literal copy as the fallback for partials generated before 5.4. Antora
+  // resolves page metadata with a header-only parse that stops at the first
+  // blank line, so either form must sit above it — an include below the
+  // blank line is never evaluated for attributes and the page ships the
+  // generic site description.
+  if (hasMetaTag) {
+    lines.push(`include::${includePrefix}${file}[tag=meta]`)
+  } else if (description) {
+    lines.push(`:description: ${description}`)
+  }
   for (const attr of attributes) lines.push(attr)
   lines.push('')
   lines.push(`include::${includePrefix}${file}[tag=single-source]`)
@@ -162,6 +181,7 @@ function reconcileStubs({
   const existingStubs = fs.readdirSync(stubDir).filter(f => f.endsWith('.adoc'))
 
   const created = []
+  const upgraded_ = []
   const deleted = []
   const keptNonStub = []
   const skippedAliasTargets = []
@@ -206,7 +226,34 @@ function reconcileStubs({
     fs.existsSync(stubDir) ? fs.readdirSync(stubDir).filter(f => f.endsWith('.adoc')) : []
   )
   for (const partial of partials) {
-    if (remainingStubs.has(partial.file)) continue
+    if (remainingStubs.has(partial.file)) {
+      // Existing stub: upgrade it in place when the partial has gained the
+      // meta tag region and the stub does not yet include it. The header
+      // include is what lets the stub inherit :description: as page
+      // metadata, so upgrading here makes the consumer repo self-heal on
+      // the next sync instead of needing a hand-written migration PR.
+      if (partial.hasMetaTag && !dryRun) {
+        const stubPath = path.join(stubDir, partial.file)
+        const stubContent = fs.readFileSync(stubPath, 'utf8')
+        if (!stubContent.includes('[tag=meta]')) {
+          const metaInclude = `include::${includePrefix}${partial.file}[tag=meta]`
+          // A later attribute entry overrides an earlier one (verified
+          // empirically against an Antora build), so a literal
+          // :description: below the include would win and leave the
+          // include dead weight. These are generated reference stubs, so
+          // any literal is a static backfill the include supersedes:
+          // strip it. Hand-maintained prose stubs never pass through this
+          // reconciler, so intentionally curated descriptions are safe.
+          let upgraded = stubContent.replace(/^:description: .*\n/m, '')
+          upgraded = upgraded.replace(/^(= .+)$/m, `$1\n${metaInclude}`)
+          if (upgraded !== stubContent) {
+            fs.writeFileSync(stubPath, upgraded, 'utf8')
+            upgraded_.push(partial.file)
+          }
+        }
+      }
+      continue
+    }
     if (aliasClaims.has(partial.file)) {
       skippedAliasTargets.push({ file: partial.file, claimedBy: aliasClaims.get(partial.file) })
       continue
@@ -285,7 +332,7 @@ function reconcileStubs({
     }
   }
 
-  return { created, deleted, keptNonStub, navUpdated, renameCandidates, skippedAliasTargets }
+  return { created, upgraded: upgraded_, deleted, keptNonStub, navUpdated, renameCandidates, skippedAliasTargets }
 }
 
 module.exports = {

package/tools/rpk-docs/templates/command.hbs

@@ -6,7 +6,12 @@
      page at worst. Availability pages are excluded from cloud by not
      generating a stub for them in cloud-docs. --}}
 = {{commandPath}}
+{{!-- Comment lines are legal inside an AsciiDoc header, so the meta tag
+     region can wrap the page's single description. Stub pages inherit it
+     with a header include (see the note at the single-source tag). --}}
+// tag::meta[]
 :description: {{{wrapDescriptionPassthrough shortDesc}}}
+// end::meta[]
 {{#if pageAliases}}
 :page-aliases: {{{pageAliases}}}
 {{/if}}
@@ -23,11 +28,24 @@
 {{/each}}
 
 // tag::single-source[]
-:description: {{{wrapDescriptionPassthrough shortDesc}}}
-{{!-- The description attribute is repeated inside the tagged region so
-     single-sourcing consumers (stub pages that include this tag) inherit
-     the meta description. page-aliases and page-platforms deliberately stay
-     outside the tag: consumers must not inherit alias registrations. --}}
+{{!-- Stub pages inherit this page's description by including the header's
+     meta tag region as a HEADER attribute. A stub that only includes
+     tag=single-source after a blank line never gets it: Antora resolves
+     page metadata with a header-only parse that stops at the stub's first
+     blank line, so the include is never evaluated for attributes and the
+     page ships the generic site description (verified empirically,
+     2026-08-05, including that a headerless include destroys the body).
+     The working stub shape is two includes:
+
+       = Page title
+       include::...page.adoc[tag=meta]        <- header: no blank line above
+                                              <- blank line ends the header
+       include::...page.adoc[tag=single-source]
+
+     Earlier revisions repeated :description: inside this region on the
+     wrong belief that it reached stub metadata by itself; it never did.
+     page-aliases and page-platforms deliberately stay outside both tag
+     regions: consumers must not inherit alias registrations. --}}
 {{!-- Deprecation notice --}}
 {{#if deprecated}}
 [CAUTION]