@redpanda-data/docs-extensions-and-macros 5.54.0 → 5.55.0

package/tools/redpanda-connect/pr-summary-formatter.js

@@ -179,7 +179,7 @@ function renderStyleRegressionWarnings(warnings) {
   if (!Array.isArray(warnings) || warnings.length === 0) return [];
   const lines = [];
   lines.push('> [!WARNING]');
-  lines.push('> **This update reintroduces house-style issues that the published docs do not have.** The fix belongs upstream in the Connect source, or in `docs-data/overrides.json`: correcting the generated partial is undone by the next regeneration.');
+  lines.push('> **This update reintroduces house-style issues that the published docs do not have.** The fix belongs upstream in the connect or benthos component spec: correcting the generated partial is undone by the next regeneration.');
   for (const w of warnings) {
     lines.push(`> - \`${w.partial}\` adds: ${w.issues.join(', ')}`);
   }
@@ -323,7 +323,8 @@ function generateMultiVersionPRSummary(masterDiff, binaryAnalysis = null, drafte
       lines.push('|-----------|-------|-------------|');
 
       newFields.forEach(field => {
-        const desc = field.description ? truncateToSentence(field.description, 1).replace(/\|/g, '\\|') : '_No description_';
+        const text = field.shortDescription || field.description;
+        const desc = text ? truncateToSentence(text, 1).replace(/\|/g, '\\|') : '_No description_';
         lines.push(`| \`${field.component}\` | \`${field.field}\` | ${desc} |`);
       });
       lines.push('');
@@ -359,7 +360,8 @@ function generateMultiVersionPRSummary(masterDiff, binaryAnalysis = null, drafte
       lines.push(`**Deprecated Fields (${summary.deprecatedFields}):**`);
       lines.push('');
       deprecatedFields.forEach(field => {
-        const guidance = field.description ? ` — ${truncateToSentence(field.description, 1)}` : '';
+        const text = field.shortDescription || field.description;
+        const guidance = text ? ` — ${truncateToSentence(text, 1)}` : '';
         lines.push(`- \`${field.component}.${field.field}\`${guidance}`);
       });
       lines.push('');
@@ -436,7 +438,7 @@ function generateMultiVersionPRSummary(masterDiff, binaryAnalysis = null, drafte
       allDeprecatedFields.push({
         component: field.component,
         field: field.field,
-        description: field.description,
+        description: field.shortDescription || field.description,
         version: release.toVersion
       });
     });

package/tools/redpanda-connect/report-delta.js

@@ -77,7 +77,8 @@ function generateConnectorDiffJson(oldIndex, newIndex, opts = {}) {
         // span being compared: inputs:oracledb_cdc/snapshot_mode reports
         // 4.99.0 in the 4.98.0 to 4.103.1 diff.
         introducedIn: (rawFieldObj && (rawFieldObj.introducedInVersion || rawFieldObj.version)) || opts.newVersion || null,
-        description: rawFieldObj && rawFieldObj.description
+        description: rawFieldObj && rawFieldObj.description,
+        shortDescription: rawFieldObj && rawFieldObj.short_description
       });
     });
   });
@@ -193,7 +194,8 @@ function generateConnectorDiffJson(oldIndex, newIndex, opts = {}) {
         deprecatedFields.push({
           component: cKey,
           field: fName,
-          description: newFieldObj && newFieldObj.description
+          description: newFieldObj && newFieldObj.description,
+          shortDescription: newFieldObj && newFieldObj.short_description
         });
       }
 
@@ -212,7 +214,8 @@ function generateConnectorDiffJson(oldIndex, newIndex, opts = {}) {
             field: fName,
             oldDefault: oldDefault,
             newDefault: newDefault,
-            description: newFieldObj && newFieldObj.description
+            description: newFieldObj && newFieldObj.description,
+            shortDescription: newFieldObj && newFieldObj.short_description
           });
         }
       }

package/tools/redpanda-connect/rpcn-connector-docs-handler.js

@@ -47,6 +47,21 @@ function capToTwoSentences (description) {
     /\bNo\./gi
   ]
 
+  // A table cell takes one paragraph of plain text. Keep only the text before
+  // the first blank line or list item, so a description such as
+  // "Controls how X is sized:\n\n- `a`: ...\n- `b`: ..." does not leak half a
+  // list into the cell. A lead-in that ends with a colon is left unterminated
+  // so the sentence match below drops it whenever a complete sentence comes
+  // before it ("Configure TCP settings. These controls are useful for:" keeps
+  // only the first sentence). Only a lead-in with nothing before it is kept,
+  // with the colon turned into a period.
+  let colonLeadIn = false
+  const firstBlock = description.split(/\n\s*\n|\n(?=\s*(?:[-*]|\d+\.)\s)/)[0].trim()
+  if (firstBlock && firstBlock !== description.trim() && !/^(?:[=#]|```|----)/.test(firstBlock)) {
+    description = firstBlock
+    colonLeadIn = firstBlock.endsWith(':')
+  }
+
   let normalized = description
   const placeholders = []
 
@@ -65,16 +80,19 @@ function capToTwoSentences (description) {
   })
 
   const sentenceRegex = /[^.!?]+[.!?]+(?:\s|$)/g
-  const sentences = normalized.match(sentenceRegex)
-
-  // If an unprotected mid-token period prevented the leading text from
-  // forming a complete match, merge that dropped prefix into the first
-  // sentence rather than silently discarding it.
-  if (sentences && sentences.length > 0) {
-    const firstIndex = normalized.indexOf(sentences[0])
-    if (firstIndex > 0) {
-      sentences[0] = normalized.slice(0, firstIndex) + sentences[0]
-    }
+
+  // An unprotected mid-token period (`fetch.max.partition.bytes`) stops a
+  // match, and the regex then resumes after it, so the text before that
+  // period never belongs to any match. Merge each such gap into the sentence
+  // that follows it rather than silently discarding it, otherwise the second
+  // "sentence" can be a tail such as "bytes`.". Text after the last match
+  // (an unterminated colon lead-in, for example) is still dropped.
+  let sentences = null
+  let lastEnd = 0
+  for (const m of normalized.matchAll(sentenceRegex)) {
+    if (!sentences) sentences = []
+    sentences.push(normalized.slice(lastEnd, m.index) + m[0])
+    lastEnd = m.index + m[0].length
   }
 
   if (!sentences || sentences.length === 0) {
@@ -82,7 +100,7 @@ function capToTwoSentences (description) {
     placeholders.forEach(({ placeholder, original }) => {
       result = result.replace(placeholder, original)
     })
-    return result
+    return colonLeadIn ? result.replace(/:$/, '.') : result
   }
 
   let maxSentences = 2
@@ -592,7 +610,7 @@ function buildFieldsTable (fields, capFn, opts = {}) {
     if (!byField[key]) {
       byField[key] = {
         field: field.field,
-        description: field.description,
+        description: field.shortDescription || field.description,
         introducedIn: field.introducedIn,
         components: []
       }
@@ -655,7 +673,7 @@ function buildChangedDefaultsTable (changedDefaults, capFn) {
         field: change.field,
         oldDefault: change.oldDefault,
         newDefault: change.newDefault,
-        description: change.description,
+        description: change.shortDescription || change.description,
         components: []
       }
     }
@@ -667,11 +685,15 @@ function buildChangedDefaultsTable (changedDefaults, capFn) {
   section += '|Field |Old default |New default |Description |Affected components\n\n'
 
   for (const [, info] of Object.entries(byFieldAndDefaults)) {
+    // Values render as literal inline code, so interpolations like
+    // ${!uuid_v4()} and characters like * or _ show as typed.
     const formatDefault = (val) => {
       if (val === undefined || val === null) return 'none'
-      if (typeof val === 'string') return val
-      if (typeof val === 'number' || typeof val === 'boolean') return String(val)
-      return JSON.stringify(val)
+      let text
+      if (typeof val === 'string') text = val === '' ? '""' : val
+      else if (typeof val === 'number' || typeof val === 'boolean') text = String(val)
+      else text = JSON.stringify(val)
+      return `\`+${text.replace(/\|/g, '\\|')}+\``
     }
 
     const oldVal = formatDefault(info.oldDefault)

package/tools/redpanda-release-notes/generate-release-notes.js

@@ -131,6 +131,118 @@ function insertReleaseSection(pageContent, sectionText) {
   return appendAtEnd();
 }
 
+/**
+ * Reorders the entries within each `=== category` of a curated section so that
+ * entries sharing an `Area::` label sit together. Areas are ordered
+ * ALPHABETICALLY (case-insensitive), so a given area sits in the same place on
+ * every release instead of moving with the source's merge order; entries keep
+ * their source order within an area, so the sort is stable. The label is
+ * repeated per entry (not merged into one heading), so the page format is
+ * unchanged — only the order of entries within a category changes. Running it on
+ * an already-clustered section is a no-op, so it is safe to apply on every
+ * release and on a backfill.
+ *
+ * Only the clean single-line `Area:: description` definition-list form is
+ * reordered. A category whose entries are not all of that form — a phase-1
+ * candidate's plain `*` bullets, or a multi-line entry — is left in its original
+ * order, so this is a safe no-op on anything it does not recognize. A curated
+ * category (one with at least one `Area::` entry) that bails logs a warning, since
+ * it then ships unclustered. Area keys are trimmed for grouping, so `Kafka API`
+ * and `Kafka API ` are one area. Headings and any prose outside a category are
+ * passed through untouched.
+ *
+ * @param {string} sectionText - A rendered release section.
+ * @return {string} The section with each category's entries clustered by area,
+ *   ending in a single newline.
+ */
+function clusterSectionByArea(sectionText) {
+  // The area label is the text up to the first `:: ` (double-colon + space),
+  // which is the AsciiDoc definition-list term separator. A lazy match stops at
+  // that first separator, so a `::` later in the description is left alone. A
+  // leading list marker (`*`, `-`, `.`), at column zero or indented, is rejected
+  // so a plain bullet that happens to carry a `::` — a phase-1 candidate line —
+  // is not mistaken for an area entry; its category then bails and is left
+  // untouched.
+  const ENTRY_RE = /^(?!\s*[*.\-]+\s)(.+?)::\s/;
+
+  // Tokenize into paragraphs: maximal runs of non-blank lines. In this format
+  // every heading and every entry is one such paragraph.
+  const lines = String(sectionText).replace(/\n+$/, '').split('\n');
+  const paragraphs = [];
+  let current = [];
+  for (const line of lines) {
+    if (/^\s*$/.test(line)) {
+      if (current.length) { paragraphs.push(current); current = []; }
+    } else {
+      current.push(line);
+    }
+  }
+  if (current.length) paragraphs.push(current);
+
+  const out = [];
+  let categoryEntries = null; // entry paragraphs of the open `=== category`
+  let categoryBail = false;   // the open category has an unrecognized entry
+  let categoryTitle = '';     // the open category's heading text, for warnings
+
+  const flushCategory = () => {
+    if (!categoryEntries) return;
+    if (categoryBail) {
+      // A phase-1 bullet category is expected to bail. A curated one (it has at
+      // least one `Area::` entry) bailing means some entry spans several lines,
+      // so the whole category ships unclustered — say so rather than fail quietly.
+      if (categoryEntries.some((entry) => ENTRY_RE.test(entry[0]))) {
+        console.warn(`${LOG_TAG} WARN: "${categoryTitle}" not grouped by area: an entry is not a single-line \`Area:: description\` (for example a \`+\` continuation or a code block).`);
+      }
+      for (const entry of categoryEntries) out.push(entry);
+    } else {
+      const order = [];
+      const groups = new Map();
+      for (const entry of categoryEntries) {
+        // Trimmed, so a stray space before `::` does not split one area in two.
+        // Only the grouping key is trimmed; the entry text is emitted verbatim.
+        const key = entry[0].match(ENTRY_RE)[1].trim();
+        if (!groups.has(key)) { groups.set(key, []); order.push(key); }
+        groups.get(key).push(entry);
+      }
+      // Areas alphabetical (case-insensitive), so an area lands in the same
+      // place on every release; entries keep their source order within an area.
+      order.sort((a, b) => a.localeCompare(b, 'en', { sensitivity: 'base' }));
+      for (const key of order) {
+        for (const entry of groups.get(key)) out.push(entry);
+      }
+    }
+    categoryEntries = null;
+    categoryBail = false;
+  };
+
+  for (const para of paragraphs) {
+    const first = para[0];
+    if (/^===\s/.test(first)) {
+      // A category heading closes the previous category and opens a new one.
+      flushCategory();
+      out.push(para);
+      categoryEntries = [];
+      categoryBail = false;
+      categoryTitle = first.replace(/^===\s+/, '').trim();
+    } else if (/^==\s/.test(first)) {
+      // The release heading (or any other top-level heading) closes the open
+      // category; entries do not cross it.
+      flushCategory();
+      out.push(para);
+      categoryEntries = null;
+    } else if (categoryEntries) {
+      categoryEntries.push(para);
+      if (para.length !== 1 || !ENTRY_RE.test(para[0])) categoryBail = true;
+    } else {
+      // Prose outside any category (for example an intro paragraph): untouched.
+      out.push(para);
+    }
+  }
+  flushCategory();
+
+  return out.map((p) => p.join('\n')).join('\n\n') + '\n';
+}
+
 /**
  * Builds a candidate release section from a raw release body.
  *
@@ -221,7 +333,9 @@ function generateReleaseNotes({ body, section, tag, date, pageContent }) {
   let finalSection;
   if (section != null) {
     assertSectionMatchesTag(section, version);
-    finalSection = section;
+    // Deterministically cluster entries by area so the curation step does not
+    // have to, and so a hand-written section is normalized the same way.
+    finalSection = clusterSectionByArea(section);
   } else {
     finalSection = buildReleaseSection({ body, version, date });
   }
@@ -232,6 +346,7 @@ function generateReleaseNotes({ body, section, tag, date, pageContent }) {
 module.exports = {
   generateReleaseNotes,
   buildReleaseSection,
+  clusterSectionByArea,
   assertSectionMatchesTag,
   insertReleaseSection,
   compareVersions,

package/tools/rpk-docs/templates/command.hbs

@@ -86,7 +86,7 @@ include::{{{this}}}[]
 ifndef::env-cloud[]
 {{{description}}}
 endif::[]
-{{else if (eq descriptionScope "cloud-only")}}
+{{else if (eq descriptionScope "cloud")}}
 ifdef::env-cloud[]
 {{{description}}}
 endif::[]

package/tools/overrides-audit/adapters/connect.js

@@ -1,144 +0,0 @@
-/**
- * Overrides Audit - Redpanda Connect Adapter (structural)
- *
- * Reads the rp-connect-docs docs-data/overrides.json:
- * - top-level `definitions`: shared field descriptions referenced with
- *   `{"$ref": "#/definitions/<name>"}`
- * - component sections (`inputs`, `outputs`, `processors`, `caches`,
- *   `tracers`, `scanners`, `rate-limits`): ARRAYS of connector entries
- *   `{ name, summary?, description?, config?: { children: [...] },
- *      examples?, version?, ... }` where config children nest recursively
- *   (`{ name, description?, children? , $ref? }`).
- *
- * The prose fields (summary, description, config field descriptions) map to
- * ConfigSpec strings in connect Go source (`.Summary()`, `.Description()`,
- * `NewXField(n).Description(...)`).
- *
- * TODO(quality logic): source comparison for connect is deferred. The
- * extracted side is the connect plugin JSON (rpk connect list --format json
- * or the cached connect-v<ver>.json in rp-connect-docs docs-data), and the
- * comparison must resolve $refs before classifying, then apply the same
- * markup rules as the properties surface (connect descriptions are AsciiDoc
- * inside Go strings, so xref:/glossterm: usage is docs-only there too).
- * Until then, prose fields classify REVIEW with an explicit note and
- * everything else classifies KEEP, so the manifest shape is stable.
- */
-
-'use strict'
-
-const classify = require('../classify')
-const { loadJson } = require('./properties')
-
-const COMPONENT_SECTIONS = [
-  'inputs',
-  'outputs',
-  'processors',
-  'caches',
-  'tracers',
-  'scanners',
-  'rate-limits',
-  'buffers',
-  'metrics'
-]
-
-const TODO_NOTE = 'TODO: connect source comparison not implemented yet (needs the connect plugin JSON and $ref resolution).'
-
-/**
- * Build a REVIEW row for a connect prose field.
- *
- * @param {string} name - Unit name (for example "inputs/amqp_0_9/urls").
- * @param {string} field - Manifest field label.
- * @param {string} text - Override text.
- * @returns {Object} Manifest row.
- */
-function proseRow (name, field, text) {
-  return {
-    name,
-    field,
-    class: classify.CLASSES.REVIEW,
-    content_hash: classify.contentHash(name, text),
-    note: TODO_NOTE
-  }
-}
-
-/**
- * Walk a config children tree, emitting one row per field description.
- *
- * @param {string} prefix - Name prefix ("inputs/amqp_0_9").
- * @param {Object[]} children - Config children array.
- * @param {Object[]} manifest - Output rows (mutated).
- */
-function walkConfigChildren (prefix, children, manifest) {
-  if (!Array.isArray(children)) return
-  for (const child of children) {
-    if (!child || typeof child !== 'object') continue
-    const childName = `${prefix}/${child.name || '(unnamed)'}`
-    if (typeof child.description === 'string') {
-      manifest.push(proseRow(childName, 'config.description', child.description))
-    }
-    if (child.$ref) {
-      manifest.push({
-        name: childName,
-        field: 'config.$ref',
-        class: classify.CLASSES.KEEP,
-        note: `Shared definition reference (${child.$ref}); audit the referenced definition instead.`
-      })
-    }
-    if (child.children) walkConfigChildren(childName, child.children, manifest)
-  }
-}
-
-/**
- * Run the audit for the connect surface (structural pass).
- *
- * @param {Object} args - { overridesPath, extractedPath }.
- * @returns {Object} { surface, manifest, summary }.
- */
-function audit ({ overridesPath, extractedPath }) {
-  const overridesDoc = loadJson(overridesPath, 'connect overrides')
-
-  const manifest = []
-
-  const definitions = overridesDoc.definitions
-  if (definitions && typeof definitions === 'object') {
-    for (const [defName, def] of Object.entries(definitions)) {
-      if (def && typeof def.description === 'string') {
-        manifest.push(proseRow(`definitions/${defName}`, 'description', def.description))
-      }
-    }
-  }
-
-  for (const section of COMPONENT_SECTIONS) {
-    const entries = overridesDoc[section]
-    if (!Array.isArray(entries)) continue
-    for (const entry of entries) {
-      if (!entry || typeof entry !== 'object') continue
-      const entryName = `${section}/${entry.name || '(unnamed)'}`
-      for (const [field, value] of Object.entries(entry)) {
-        if (field === 'name') continue
-        if (field === 'summary' || field === 'description') {
-          manifest.push(proseRow(entryName, field, value))
-        } else if (field === 'config' && value && typeof value === 'object') {
-          walkConfigChildren(entryName, value.children, manifest)
-        } else {
-          manifest.push({
-            name: entryName,
-            field,
-            class: classify.CLASSES.KEEP,
-            note: 'Docs-side connector metadata; stays in the override file by design.'
-          })
-        }
-      }
-    }
-  }
-
-  return {
-    surface: 'connect',
-    overrides_file: overridesPath,
-    extracted_file: extractedPath || null,
-    manifest,
-    summary: classify.summarize(manifest)
-  }
-}
-
-module.exports = { audit, COMPONENT_SECTIONS }