@redpanda-data/docs-extensions-and-macros 5.54.0 → 5.56.0

package/tools/redpanda-release-notes/generate-release-notes.js

@@ -131,6 +131,118 @@ function insertReleaseSection(pageContent, sectionText) {
   return appendAtEnd();
 }
 
+/**
+ * Reorders the entries within each `=== category` of a curated section so that
+ * entries sharing an `Area::` label sit together. Areas are ordered
+ * ALPHABETICALLY (case-insensitive), so a given area sits in the same place on
+ * every release instead of moving with the source's merge order; entries keep
+ * their source order within an area, so the sort is stable. The label is
+ * repeated per entry (not merged into one heading), so the page format is
+ * unchanged — only the order of entries within a category changes. Running it on
+ * an already-clustered section is a no-op, so it is safe to apply on every
+ * release and on a backfill.
+ *
+ * Only the clean single-line `Area:: description` definition-list form is
+ * reordered. A category whose entries are not all of that form — a phase-1
+ * candidate's plain `*` bullets, or a multi-line entry — is left in its original
+ * order, so this is a safe no-op on anything it does not recognize. A curated
+ * category (one with at least one `Area::` entry) that bails logs a warning, since
+ * it then ships unclustered. Area keys are trimmed for grouping, so `Kafka API`
+ * and `Kafka API ` are one area. Headings and any prose outside a category are
+ * passed through untouched.
+ *
+ * @param {string} sectionText - A rendered release section.
+ * @return {string} The section with each category's entries clustered by area,
+ *   ending in a single newline.
+ */
+function clusterSectionByArea(sectionText) {
+  // The area label is the text up to the first `:: ` (double-colon + space),
+  // which is the AsciiDoc definition-list term separator. A lazy match stops at
+  // that first separator, so a `::` later in the description is left alone. A
+  // leading list marker (`*`, `-`, `.`), at column zero or indented, is rejected
+  // so a plain bullet that happens to carry a `::` — a phase-1 candidate line —
+  // is not mistaken for an area entry; its category then bails and is left
+  // untouched.
+  const ENTRY_RE = /^(?!\s*[*.\-]+\s)(.+?)::\s/;
+
+  // Tokenize into paragraphs: maximal runs of non-blank lines. In this format
+  // every heading and every entry is one such paragraph.
+  const lines = String(sectionText).replace(/\n+$/, '').split('\n');
+  const paragraphs = [];
+  let current = [];
+  for (const line of lines) {
+    if (/^\s*$/.test(line)) {
+      if (current.length) { paragraphs.push(current); current = []; }
+    } else {
+      current.push(line);
+    }
+  }
+  if (current.length) paragraphs.push(current);
+
+  const out = [];
+  let categoryEntries = null; // entry paragraphs of the open `=== category`
+  let categoryBail = false;   // the open category has an unrecognized entry
+  let categoryTitle = '';     // the open category's heading text, for warnings
+
+  const flushCategory = () => {
+    if (!categoryEntries) return;
+    if (categoryBail) {
+      // A phase-1 bullet category is expected to bail. A curated one (it has at
+      // least one `Area::` entry) bailing means some entry spans several lines,
+      // so the whole category ships unclustered — say so rather than fail quietly.
+      if (categoryEntries.some((entry) => ENTRY_RE.test(entry[0]))) {
+        console.warn(`${LOG_TAG} WARN: "${categoryTitle}" not grouped by area: an entry is not a single-line \`Area:: description\` (for example a \`+\` continuation or a code block).`);
+      }
+      for (const entry of categoryEntries) out.push(entry);
+    } else {
+      const order = [];
+      const groups = new Map();
+      for (const entry of categoryEntries) {
+        // Trimmed, so a stray space before `::` does not split one area in two.
+        // Only the grouping key is trimmed; the entry text is emitted verbatim.
+        const key = entry[0].match(ENTRY_RE)[1].trim();
+        if (!groups.has(key)) { groups.set(key, []); order.push(key); }
+        groups.get(key).push(entry);
+      }
+      // Areas alphabetical (case-insensitive), so an area lands in the same
+      // place on every release; entries keep their source order within an area.
+      order.sort((a, b) => a.localeCompare(b, 'en', { sensitivity: 'base' }));
+      for (const key of order) {
+        for (const entry of groups.get(key)) out.push(entry);
+      }
+    }
+    categoryEntries = null;
+    categoryBail = false;
+  };
+
+  for (const para of paragraphs) {
+    const first = para[0];
+    if (/^===\s/.test(first)) {
+      // A category heading closes the previous category and opens a new one.
+      flushCategory();
+      out.push(para);
+      categoryEntries = [];
+      categoryBail = false;
+      categoryTitle = first.replace(/^===\s+/, '').trim();
+    } else if (/^==\s/.test(first)) {
+      // The release heading (or any other top-level heading) closes the open
+      // category; entries do not cross it.
+      flushCategory();
+      out.push(para);
+      categoryEntries = null;
+    } else if (categoryEntries) {
+      categoryEntries.push(para);
+      if (para.length !== 1 || !ENTRY_RE.test(para[0])) categoryBail = true;
+    } else {
+      // Prose outside any category (for example an intro paragraph): untouched.
+      out.push(para);
+    }
+  }
+  flushCategory();
+
+  return out.map((p) => p.join('\n')).join('\n\n') + '\n';
+}
+
 /**
  * Builds a candidate release section from a raw release body.
  *
@@ -221,7 +333,9 @@ function generateReleaseNotes({ body, section, tag, date, pageContent }) {
   let finalSection;
   if (section != null) {
     assertSectionMatchesTag(section, version);
-    finalSection = section;
+    // Deterministically cluster entries by area so the curation step does not
+    // have to, and so a hand-written section is normalized the same way.
+    finalSection = clusterSectionByArea(section);
   } else {
     finalSection = buildReleaseSection({ body, version, date });
   }
@@ -232,6 +346,7 @@ function generateReleaseNotes({ body, section, tag, date, pageContent }) {
 module.exports = {
   generateReleaseNotes,
   buildReleaseSection,
+  clusterSectionByArea,
   assertSectionMatchesTag,
   insertReleaseSection,
   compareVersions,

package/tools/rpk-docs/generate-rpk-docs.js

@@ -11,6 +11,8 @@ Object.entries(helpers).forEach(([name, fn]) => {
     handlebars.registerHelper(name, fn)
   }
 })
+// The provenance marker command.hbs writes under the page header
+handlebars.registerHelper('generatedMarker', require('../../cli-utils/generated-marker').generatedMarker)
 
 // Template paths
 const TEMPLATES_DIR = path.resolve(__dirname, './templates')

package/tools/rpk-docs/generate-x-env-partial.js

@@ -28,17 +28,20 @@ const fs = require('fs')
 const os = require('os')
 const path = require('path')
 const { spawnSync } = require('child_process')
+const { generatedMarker } = require('../../cli-utils/generated-marker')
 
 // A healthy rpk exposes ~30 -X options. Refuse to overwrite the partial
 // with a suspiciously small table (for example, if -X list output ever
 // changes format and only partially parses).
 const MIN_OPTIONS = 20
 
+// The shared marker, so convert-to-markdown reports the partial as generated.
 const GENERATED_BANNER = `// tag::generated[]
-// Generated from rpk's own -X option data. Do not edit manually:
-// regenerate with doc-tools generate rpk-docs (which rewrites this
-// file alongside the rpk command pages) or, for a targeted refresh,
-// doc-tools generate rpk-env-partial.
+${generatedMarker(
+  'rpk-env-partial',
+  "rpk's own -X option data (rpk --print-tree)",
+  'Regenerate with doc-tools generate rpk-docs, which rewrites this file alongside the rpk command pages, or, for a targeted refresh, doc-tools generate rpk-env-partial.'
+)}
 `
 
 /**

package/tools/rpk-docs/helpers/index.js

@@ -426,7 +426,10 @@ function wrapDescriptionPassthrough(str) {
   if (!str) return ''
   // Check if contains backticks (inline code) or other formatting markers
   if (/`/.test(str)) {
-    return `pass:q[${str}]`
+    // pass:q[] skips special-character substitution, so a placeholder such as
+    // <agent> would reach the HTML raw and be dropped as an unknown tag.
+    const escaped = str.replace(/&/g, '&amp;').replace(/</g, '&lt;').replace(/>/g, '&gt;')
+    return `pass:q[${escaped}]`
   }
   return str
 }

package/tools/rpk-docs/templates/command.hbs

@@ -22,7 +22,7 @@
 :page-deprecated: true
 {{/if}}
 
-// This content is autogenerated. Do not edit manually. To customize content, see the writer's guide: https://github.com/redpanda-data/docs/blob/main/docs-data/RPK_OVERRIDES_GUIDE.adoc
+{{{generatedMarker "rpk-docs" "the rpk command help in redpanda-data/streaming-enterprise" "To customize content, see the writer's guide: https://github.com/redpanda-data/docs/blob/main/docs-data/RPK_OVERRIDES_GUIDE.adoc"}}}
 {{#each pageAttributes}}
 :{{@key}}: {{this}}
 {{/each}}
@@ -86,7 +86,7 @@ include::{{{this}}}[]
 ifndef::env-cloud[]
 {{{description}}}
 endif::[]
-{{else if (eq descriptionScope "cloud-only")}}
+{{else if (eq descriptionScope "cloud")}}
 ifdef::env-cloud[]
 {{{description}}}
 endif::[]

package/tools/overrides-audit/adapters/connect.js

@@ -1,144 +0,0 @@
-/**
- * Overrides Audit - Redpanda Connect Adapter (structural)
- *
- * Reads the rp-connect-docs docs-data/overrides.json:
- * - top-level `definitions`: shared field descriptions referenced with
- *   `{"$ref": "#/definitions/<name>"}`
- * - component sections (`inputs`, `outputs`, `processors`, `caches`,
- *   `tracers`, `scanners`, `rate-limits`): ARRAYS of connector entries
- *   `{ name, summary?, description?, config?: { children: [...] },
- *      examples?, version?, ... }` where config children nest recursively
- *   (`{ name, description?, children? , $ref? }`).
- *
- * The prose fields (summary, description, config field descriptions) map to
- * ConfigSpec strings in connect Go source (`.Summary()`, `.Description()`,
- * `NewXField(n).Description(...)`).
- *
- * TODO(quality logic): source comparison for connect is deferred. The
- * extracted side is the connect plugin JSON (rpk connect list --format json
- * or the cached connect-v<ver>.json in rp-connect-docs docs-data), and the
- * comparison must resolve $refs before classifying, then apply the same
- * markup rules as the properties surface (connect descriptions are AsciiDoc
- * inside Go strings, so xref:/glossterm: usage is docs-only there too).
- * Until then, prose fields classify REVIEW with an explicit note and
- * everything else classifies KEEP, so the manifest shape is stable.
- */
-
-'use strict'
-
-const classify = require('../classify')
-const { loadJson } = require('./properties')
-
-const COMPONENT_SECTIONS = [
-  'inputs',
-  'outputs',
-  'processors',
-  'caches',
-  'tracers',
-  'scanners',
-  'rate-limits',
-  'buffers',
-  'metrics'
-]
-
-const TODO_NOTE = 'TODO: connect source comparison not implemented yet (needs the connect plugin JSON and $ref resolution).'
-
-/**
- * Build a REVIEW row for a connect prose field.
- *
- * @param {string} name - Unit name (for example "inputs/amqp_0_9/urls").
- * @param {string} field - Manifest field label.
- * @param {string} text - Override text.
- * @returns {Object} Manifest row.
- */
-function proseRow (name, field, text) {
-  return {
-    name,
-    field,
-    class: classify.CLASSES.REVIEW,
-    content_hash: classify.contentHash(name, text),
-    note: TODO_NOTE
-  }
-}
-
-/**
- * Walk a config children tree, emitting one row per field description.
- *
- * @param {string} prefix - Name prefix ("inputs/amqp_0_9").
- * @param {Object[]} children - Config children array.
- * @param {Object[]} manifest - Output rows (mutated).
- */
-function walkConfigChildren (prefix, children, manifest) {
-  if (!Array.isArray(children)) return
-  for (const child of children) {
-    if (!child || typeof child !== 'object') continue
-    const childName = `${prefix}/${child.name || '(unnamed)'}`
-    if (typeof child.description === 'string') {
-      manifest.push(proseRow(childName, 'config.description', child.description))
-    }
-    if (child.$ref) {
-      manifest.push({
-        name: childName,
-        field: 'config.$ref',
-        class: classify.CLASSES.KEEP,
-        note: `Shared definition reference (${child.$ref}); audit the referenced definition instead.`
-      })
-    }
-    if (child.children) walkConfigChildren(childName, child.children, manifest)
-  }
-}
-
-/**
- * Run the audit for the connect surface (structural pass).
- *
- * @param {Object} args - { overridesPath, extractedPath }.
- * @returns {Object} { surface, manifest, summary }.
- */
-function audit ({ overridesPath, extractedPath }) {
-  const overridesDoc = loadJson(overridesPath, 'connect overrides')
-
-  const manifest = []
-
-  const definitions = overridesDoc.definitions
-  if (definitions && typeof definitions === 'object') {
-    for (const [defName, def] of Object.entries(definitions)) {
-      if (def && typeof def.description === 'string') {
-        manifest.push(proseRow(`definitions/${defName}`, 'description', def.description))
-      }
-    }
-  }
-
-  for (const section of COMPONENT_SECTIONS) {
-    const entries = overridesDoc[section]
-    if (!Array.isArray(entries)) continue
-    for (const entry of entries) {
-      if (!entry || typeof entry !== 'object') continue
-      const entryName = `${section}/${entry.name || '(unnamed)'}`
-      for (const [field, value] of Object.entries(entry)) {
-        if (field === 'name') continue
-        if (field === 'summary' || field === 'description') {
-          manifest.push(proseRow(entryName, field, value))
-        } else if (field === 'config' && value && typeof value === 'object') {
-          walkConfigChildren(entryName, value.children, manifest)
-        } else {
-          manifest.push({
-            name: entryName,
-            field,
-            class: classify.CLASSES.KEEP,
-            note: 'Docs-side connector metadata; stays in the override file by design.'
-          })
-        }
-      }
-    }
-  }
-
-  return {
-    surface: 'connect',
-    overrides_file: overridesPath,
-    extracted_file: extractedPath || null,
-    manifest,
-    summary: classify.summarize(manifest)
-  }
-}
-
-module.exports = { audit, COMPONENT_SECTIONS }