@redpanda-data/docs-extensions-and-macros 5.54.0 → 5.56.0

package/tools/metrics/metrics.py

@@ -23,6 +23,16 @@ These output files are stored in a versioned folder under `docs/autogenerated/<v
 
 logging.basicConfig(level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s")
 
+
+def generated_marker(generator, source, hint=""):
+    """Same marker shape as cli-utils/generated-marker.js (no version, so a
+    release does not rewrite every file); convert-to-markdown reads it."""
+    marker = (
+        "// This content is autogenerated. Do not edit manually. "
+        f"Generated by doc-tools ({generator}) from {source.rstrip('.')}."
+    )
+    return marker + (f" {hint.strip()}" if hint.strip() else "")
+
 def fetch_metrics(url):
     """Fetch metrics from the given URL."""
     try:
@@ -123,6 +133,7 @@ def parse_metrics(metrics_text):
 def output_asciidoc(metrics, adoc_file):
     """Output metrics as AsciiDoc."""
     with open(adoc_file, "w") as f:
+        f.write(generated_marker("metrics-docs", "the metrics Redpanda exposes, defined in redpanda-data/streaming-enterprise") + "\n\n")
         for name, data in metrics.items():
             f.write(f"=== {name}\n\n")
             f.write(f"{data['description']}\n\n")

package/tools/overrides-audit/README.adoc

@@ -22,7 +22,7 @@ npx doc-tools overrides audit \
   --format human
 
 # Standalone
-node tools/overrides-audit/index.js --overrides <path> --extracted <path> [--surface properties|rpk|connect] [--format json|human] [--output <file>]
+node tools/overrides-audit/index.js --overrides <path> --extracted <path> [--surface properties|rpk] [--format json|human] [--output <file>]
 ----
 
 IMPORTANT: `--extracted` must be the property extractor's *raw* output (the `--output` file), never the enhanced output (`--enhanced-output`) or the versioned `redpanda-properties-<tag>.json` attachments: those already have overrides applied, so every override would classify as REDUNDANT. The same trap exists for rpk: the versioned `rpk-v<version>.json` files in docs repos are post-override snapshots.
@@ -133,7 +133,6 @@ JSON (default): `{ surface, overrides_file, extracted_file, manifest, summary, c
 
 * `properties` (complete): `property-overrides.json` vs the property extractor's raw JSON.
 * `rpk` (structural): enumerates `rpk-overrides.json` (see `docs-data/rpk-overrides.schema.json`) -- command/flag `description` fields are the upstreamable prose (classified `REVIEW` with a TODO note until source comparison lands; it needs a raw pre-override `--print-tree` snapshot plus the `formatDescription()` pipeline from `tools/rpk-docs`), everything else (`seeAlso`, `pageAliases`, `content`, platform scoping, ...) is `KEEP` by design.
-* `connect` (structural): enumerates the rp-connect-docs `overrides.json` (`definitions` + per-component arrays with nested `config.children`); prose fields classify `REVIEW` with a TODO note until `$ref` resolution and plugin-JSON comparison land.
 
 == Golden run (local checkouts required)
 

package/tools/overrides-audit/adapters/properties.js

@@ -4,7 +4,7 @@
  * Reads docs-data/property-overrides.json and an extracted-properties JSON
  * (the property extractor's raw --output, WITHOUT overrides applied) and
  * classifies every override field. This is the complete reference adapter;
- * rpk and connect adapters follow the same shape.
+ * The rpk adapter follows the same shape.
  */
 
 'use strict'

package/tools/overrides-audit/index.js

@@ -12,7 +12,7 @@
  *   node tools/overrides-audit/index.js \
  *     --overrides docs-data/property-overrides.json \
  *     --extracted tools/property-extractor/gen/properties-output.json \
- *     [--surface properties|rpk|connect] [--format json|human] [--output <file>]
+ *     [--surface properties|rpk] [--format json|human] [--output <file>]
  *
  * Also exposed as `doc-tools overrides audit` and the `audit_overrides`
  * MCP tool. See README.adoc in this directory for the classification rules
@@ -26,8 +26,7 @@ const path = require('path')
 
 const ADAPTERS = {
   properties: () => require('./adapters/properties'),
-  rpk: () => require('./adapters/rpk'),
-  connect: () => require('./adapters/connect')
+  rpk: () => require('./adapters/rpk')
 }
 
 /**
@@ -153,7 +152,7 @@ function main (argv) {
     .description('Classify docs-side override fields against extracted source strings')
     .requiredOption('--overrides <path>', 'Path to the overrides JSON file (for example docs-data/property-overrides.json)')
     .option('--extracted <path>', 'Path to the extracted source JSON (property extractor raw output; required for the properties surface)')
-    .option('--surface <surface>', 'Override surface: properties, rpk, or connect', 'properties')
+    .option('--surface <surface>', 'Override surface: properties or rpk', 'properties')
     .option('--format <format>', 'Output format: json or human', 'json')
     .option('--output <path>', 'Also write the JSON result to this file')
     .action((options) => {

package/tools/property-extractor/generate-handlebars-docs.js

@@ -2,10 +2,18 @@
 // writer already has open. The previous pointer was a Confluence page, and
 // Confluence is retired, so it sent every writer who read this line to a dead
 // end. A repo-relative path cannot rot the same way.
-const AUTOGEN_NOTICE = '// This content is autogenerated. Do not edit manually. To change a description, add a link, or scope text to one docs build, edit docs-data/property-overrides.json (fields are documented in docs-data/property-overrides.schema.json and docs-data/README.adoc) and regenerate.\n';
 'use strict';
 
 const fs = require('fs');
+const { generatedMarker } = require('../../cli-utils/generated-marker');
+
+// The marker's shape is shared by every generator and read by the
+// convert-to-markdown extension, which exports it to the Markdown twins.
+const AUTOGEN_NOTICE = generatedMarker(
+  'property-docs',
+  'the configuration property definitions in redpanda-data/streaming-enterprise',
+  'To change a description, add a link, or scope text to one docs build, edit docs-data/property-overrides.json (fields are documented in docs-data/property-overrides.schema.json and docs-data/README.adoc) and regenerate.'
+) + '\n';
 const bigIntJson = require('../../cli-utils/big-int-json');
 const path = require('path');
 const handlebars = require('handlebars');

package/tools/property-extractor/helpers/applyPropertyLinks.js

@@ -419,6 +419,39 @@ function normalizeIncludes(propName, raw, warnings) {
   return out;
 }
 
+/**
+ * Remove the inline copies of `includes` that the attachment write-back appended.
+ *
+ * generate-handlebars-docs.js writes each include back into the description
+ * on disk, so the attachment's tooltip still renders it, and leaves `includes`
+ * in place. A run that reads that file again (a reused gen/<tag>-properties.json,
+ * or generate-docs run twice without re-extraction) sees the directive in the
+ * description and in `includes`, and the template rendered it twice. Only the
+ * exact shapes the write-back produces are
+ * removed, so an include a human put in the prose is left alone.
+ *
+ * @param {string|undefined} description - Flattened description.
+ * @param {Array<{target: string, cloud_only?: boolean, self_managed_only?: boolean}>} items - Normalized includes.
+ * @returns {string|undefined}
+ */
+function stripInlineIncludes(description, items) {
+  if (typeof description !== 'string' || !description.includes('include::')) return description;
+  let text = description;
+  // The write-back appends in order, so peel from the end.
+  for (const item of [...items].reverse()) {
+    const directive = `include::${item.target}`;
+    let block = directive;
+    if (item.cloud_only) block = `ifdef::env-cloud[]\n${directive}\nendif::[]`;
+    else if (item.self_managed_only) block = `ifndef::env-cloud[]\n${directive}\nendif::[]`;
+    if (text.endsWith(`\n\n${block}`)) {
+      text = text.slice(0, -(block.length + 2));
+    } else if (text === block) {
+      text = '';
+    }
+  }
+  return text;
+}
+
 /**
  * Why the Cloud build cannot reach a link target, or null when it can.
  *
@@ -471,6 +504,7 @@ function applyPropertyLinks(properties) {
     }
     if (prop.includes !== undefined) {
       prop.includes = normalizeIncludes(propName, prop.includes, warnings);
+      prop.description = stripInlineIncludes(prop.description, prop.includes);
     }
     if (Array.isArray(prop.admonitions)) {
       prop.admonitions = prop.admonitions.filter((item) => {
@@ -589,3 +623,4 @@ module.exports.applyLinksToText = applyLinksToText;
 module.exports.renderLink = renderLink;
 module.exports.cloudUnreachable = cloudUnreachable;
 module.exports.normalizeIncludes = normalizeIncludes;
+module.exports.stripInlineIncludes = stripInlineIncludes;

package/tools/redpanda-connect/AUTOMATION.md

@@ -581,7 +581,7 @@ npx doc-tools generate rpcn-connector-docs \
 | `--draft-missing` | Generate full page drafts for new connectors | false |
 | `--update-whats-new` | Update whats-new.adoc with changes | false |
 | `--include-bloblang` | Include Bloblang functions/methods | false |
-| `--overrides <path>` | JSON file with description overrides | `docs-data/overrides.json` |
+| `--overrides <path>` | Optional JSON file with description overrides | None |
 
 ### Examples
 
@@ -766,41 +766,8 @@ Templates are in `tools/redpanda-connect/templates/`:
 - `connector-examples.hbs` - Examples template
 - `connector-full.hbs` - Full page draft template
 
-### Updating Overrides
-Override file: `docs-data/overrides.json` (or `--overrides` flag)
-
-```json
-{
-  "inputs": {
-    "kafka": {
-      "fields": {
-        "addresses": {
-          "description": "Custom description override",
-          "examples": ["localhost:9092"]
-        }
-      }
-    }
-  }
-}
-```
-
-Supports `$ref` syntax for deduplication:
-```json
-{
-  "inputs": {
-    "kafka": {
-      "fields": {
-        "tls": { "$ref": "#/common/tls" }
-      }
-    }
-  },
-  "common": {
-    "tls": {
-      "description": "TLS configuration (reused across components)"
-    }
-  }
-}
-```
+### Fixing connector descriptions
+Field, component, and Bloblang descriptions come from the component specs in `redpanda-data/connect` and `redpanda-data/benthos`. To fix wrong or missing text, open a PR against the spec. The fix ships in the next Connect release.
 
 ## Troubleshooting
 

package/tools/redpanda-connect/README.adoc

@@ -46,11 +46,6 @@ npm install @redpanda-data/docs-extensions-and-macros
 # Generate connector documentation with latest data
 npx doc-tools generate rpcn-connector-docs --fetch-connectors
 
-# Generate with custom overrides
-npx doc-tools generate rpcn-connector-docs \
-  --fetch-connectors \
-  --overrides path/to/overrides.json
-
 # Generate and draft missing connectors
 npx doc-tools generate rpcn-connector-docs \
   --fetch-connectors \
@@ -176,11 +171,6 @@ You can generate connector documentation with various options:
 # Basic generation with fetching latest data
 npx doc-tools generate rpcn-connector-docs --fetch-connectors
 
-# Generate with custom overrides for descriptions/examples
-npx doc-tools generate rpcn-connector-docs \
-  --fetch-connectors \
-  --overrides overrides.json
-
 # Draft full pages for connectors missing documentation
 npx doc-tools generate rpcn-connector-docs \
   --fetch-connectors \
@@ -202,7 +192,7 @@ npx doc-tools generate rpcn-connector-docs \
 * `--fetch-connectors` - Fetch latest data using `rpk connect list`
 * `--connect-version <version>` - Specific Connect version to install and fetch (requires `--fetch-connectors`)
 * `--draft-missing` - Generate full draft pages for undocumented connectors
-* `--overrides <path>` - Apply custom descriptions/examples from JSON
+* `--overrides <path>` - Merge descriptions and examples from an optional JSON file. Nothing is merged unless you pass it
 * `--data-dir <path>` - Directory containing versioned connect JSON files
 * `--old-data <path>` - Specific old version file for diff comparison
 * `--include-bloblang` - Include Bloblang functions/methods
@@ -791,7 +781,9 @@ console.log('<!-- PR_SUMMARY_END -->');
 
 === Override system
 
-Overrides allow customizing extracted documentation:
+Connector field, component, and Bloblang descriptions come from the component specs in https://github.com/redpanda-data/connect[redpanda-data/connect^] and https://github.com/redpanda-data/benthos[redpanda-data/benthos^]. To fix wrong or missing text, open a PR against the spec. The fix ships in the next Connect release.
+
+The generator can also merge an optional JSON file passed with `--overrides` into the extracted data. Nothing is merged unless you pass the flag:
 
 [,json]
 ----
@@ -840,11 +832,9 @@ Overrides support:
 * Connector catalog enrichment
 * Version tracking and change detection
 * Handlebars template rendering
-* Override system for custom documentation
 
 === Current limitations
 
-* *Manual override maintenance*: Custom descriptions must be maintained in overrides.json
 * *Example quality*: Auto-generated examples may not represent best practices
 * *Duplicate YAML keys*: Some examples contain multiple configurations in one block, creating invalid YAML with duplicate top-level keys (for example, multiple `processors:` keys)
 * *CSV catalog dependency*: Relies on manually maintained CSV in Connect repository
@@ -876,25 +866,7 @@ This occurs when multiple separate configurations are concatenated into a single
 
 === Workarounds
 
-For unsupported patterns or issues, use overrides.json:
-
-[,json]
-----
-{
-  "inputs": [
-    {
-      "name": "problematic_connector",
-      "description": "Custom description overriding auto-extracted text",
-      "examples": [
-        {
-          "title": "Corrected example",
-          "config": "input:\n  problematic_connector:\n    field: value"
-        }
-      ]
-    }
-  ]
-}
-----
+To fix an unsupported pattern, a wrong description, or a broken example, edit the component spec in redpanda-data/connect or redpanda-data/benthos. The fix ships in the next Connect release.
 
 == Contributing
 

package/tools/redpanda-connect/generate-rpcn-connector-docs.js

@@ -17,6 +17,9 @@ Object.entries(helpers).forEach(([name, fn]) => {
   }
   handlebars.registerHelper(name, fn);
 });
+// The provenance marker the partial templates write on their first line
+const { generatedMarker } = require('../../cli-utils/generated-marker');
+handlebars.registerHelper('generatedMarker', generatedMarker);
 
 // Default “main” template (connector.hbs) which invokes partials {{> intro}}, {{> fields}}, {{> examples}}
 const DEFAULT_TEMPLATE = path.resolve(__dirname, './templates/connector.hbs');
@@ -34,9 +37,12 @@ const { hasStructuralHeadings, hasMarkdownHeadings, firstHeadingDepth, escapePla
 // section is removed upstream. Kept as an AsciiDoc comment so the file renders
 // nothing while any lingering include directive still resolves cleanly.
 const STALE_METADATA_PARTIAL =
-  '// This content is autogenerated. Do not edit manually. ' +
-  'The connector description no longer contains a `== Metadata` section, so this partial is intentionally empty. ' +
-  'You can remove the include directive that points at this file from the connector page.\n';
+  generatedMarker(
+    'rpcn-connector-docs',
+    'the component spec in redpanda-data/connect',
+    'The connector description no longer contains a `== Metadata` section, so this partial is intentionally empty. ' +
+      'You can remove the include directive that points at this file from the connector page.'
+  ) + '\n';
 
 // Banner-only content written to a description partial when the connector's
 // description is removed upstream. Same rationale as STALE_METADATA_PARTIAL:
@@ -50,12 +56,27 @@ const STALE_METADATA_PARTIAL =
 // every page that includes it, on every build, which is a build failure
 // wherever Asciidoctor warnings are treated as errors. An empty region
 // resolves to nothing and says nothing.
-const STALE_DESCRIPTION_PARTIAL =
-  '// This content is autogenerated. Do not edit manually. ' +
-  'The connector no longer has a description, so this partial is intentionally empty. ' +
-  'You can remove the include directive that points at this file from the connector page.\n' +
+const STALE_DESCRIPTION_REGIONS =
   '\n// tag::meta[]\n// end::meta[]\n' +
   '\n// tag::body[]\n// end::body[]\n';
+const STALE_DESCRIPTION_PARTIAL =
+  generatedMarker(
+    'rpcn-connector-docs',
+    'the component spec in redpanda-data/connect',
+    'The connector no longer has a description, so this partial is intentionally empty. ' +
+      'You can remove the include directive that points at this file from the connector page.'
+  ) + '\n' + STALE_DESCRIPTION_REGIONS;
+// What earlier releases wrote for the same blanked partial. The orphan sweep
+// treats either as already blanked, so a partial emptied before the marker
+// changed is not re-blanked and re-reported. The marker carries no version,
+// so this list grows only when the wording does.
+const STALE_DESCRIPTION_PARTIALS = new Set([
+  STALE_DESCRIPTION_PARTIAL,
+  '// This content is autogenerated. Do not edit manually. ' +
+    'The connector no longer has a description, so this partial is intentionally empty. ' +
+    'You can remove the include directive that points at this file from the connector page.\n' +
+    STALE_DESCRIPTION_REGIONS,
+]);
 
 // Component families with their own reference pages, keyed by the canonical
 // type directory from typeDirFor(). Other data keys -- `config`, the bloblang
@@ -101,9 +122,9 @@ function mergeOverrides(target, overrides) {
     throw new Error('Target must be a valid object');
   }
 
-  // 'summary' earns its place here the hard way: overrides.json has carried
-  // summary overrides (zmq4, ffi) that were silently dropped because the key
-  // fell through every branch below. Summaries feed page meta descriptions.
+  // Scalar keys are replaced outright. 'summary' must be listed: a key that
+  // falls through every branch below is silently dropped, and summaries feed
+  // page meta descriptions.
   const scalarKeys = ['description', 'summary', 'type', 'annotated_field', 'version'];
 
   for (const key in overrides) {
@@ -932,7 +953,7 @@ async function generateRpcnConnectorDocs(options) {
         const orphanPath = path.join(typeDirPath, file);
         // Already blanked on an earlier run: leave it alone so it is not
         // re-reported forever.
-        if (fs.readFileSync(orphanPath, 'utf8') === STALE_DESCRIPTION_PARTIAL) continue;
+        if (STALE_DESCRIPTION_PARTIALS.has(fs.readFileSync(orphanPath, 'utf8'))) continue;
         orphans.push({ key, orphanPath });
       }
     }

package/tools/redpanda-connect/pr-summary-formatter.js

@@ -179,7 +179,7 @@ function renderStyleRegressionWarnings(warnings) {
   if (!Array.isArray(warnings) || warnings.length === 0) return [];
   const lines = [];
   lines.push('> [!WARNING]');
-  lines.push('> **This update reintroduces house-style issues that the published docs do not have.** The fix belongs upstream in the Connect source, or in `docs-data/overrides.json`: correcting the generated partial is undone by the next regeneration.');
+  lines.push('> **This update reintroduces house-style issues that the published docs do not have.** The fix belongs upstream in the connect or benthos component spec: correcting the generated partial is undone by the next regeneration.');
   for (const w of warnings) {
     lines.push(`> - \`${w.partial}\` adds: ${w.issues.join(', ')}`);
   }
@@ -323,7 +323,8 @@ function generateMultiVersionPRSummary(masterDiff, binaryAnalysis = null, drafte
       lines.push('|-----------|-------|-------------|');
 
       newFields.forEach(field => {
-        const desc = field.description ? truncateToSentence(field.description, 1).replace(/\|/g, '\\|') : '_No description_';
+        const text = field.shortDescription || field.description;
+        const desc = text ? truncateToSentence(text, 1).replace(/\|/g, '\\|') : '_No description_';
         lines.push(`| \`${field.component}\` | \`${field.field}\` | ${desc} |`);
       });
       lines.push('');
@@ -359,7 +360,8 @@ function generateMultiVersionPRSummary(masterDiff, binaryAnalysis = null, drafte
       lines.push(`**Deprecated Fields (${summary.deprecatedFields}):**`);
       lines.push('');
       deprecatedFields.forEach(field => {
-        const guidance = field.description ? ` — ${truncateToSentence(field.description, 1)}` : '';
+        const text = field.shortDescription || field.description;
+        const guidance = text ? ` — ${truncateToSentence(text, 1)}` : '';
         lines.push(`- \`${field.component}.${field.field}\`${guidance}`);
       });
       lines.push('');
@@ -436,7 +438,7 @@ function generateMultiVersionPRSummary(masterDiff, binaryAnalysis = null, drafte
       allDeprecatedFields.push({
         component: field.component,
         field: field.field,
-        description: field.description,
+        description: field.shortDescription || field.description,
         version: release.toVersion
       });
     });

package/tools/redpanda-connect/report-delta.js

@@ -77,7 +77,8 @@ function generateConnectorDiffJson(oldIndex, newIndex, opts = {}) {
         // span being compared: inputs:oracledb_cdc/snapshot_mode reports
         // 4.99.0 in the 4.98.0 to 4.103.1 diff.
         introducedIn: (rawFieldObj && (rawFieldObj.introducedInVersion || rawFieldObj.version)) || opts.newVersion || null,
-        description: rawFieldObj && rawFieldObj.description
+        description: rawFieldObj && rawFieldObj.description,
+        shortDescription: rawFieldObj && rawFieldObj.short_description
       });
     });
   });
@@ -193,7 +194,8 @@ function generateConnectorDiffJson(oldIndex, newIndex, opts = {}) {
         deprecatedFields.push({
           component: cKey,
           field: fName,
-          description: newFieldObj && newFieldObj.description
+          description: newFieldObj && newFieldObj.description,
+          shortDescription: newFieldObj && newFieldObj.short_description
         });
       }
 
@@ -212,7 +214,8 @@ function generateConnectorDiffJson(oldIndex, newIndex, opts = {}) {
             field: fName,
             oldDefault: oldDefault,
             newDefault: newDefault,
-            description: newFieldObj && newFieldObj.description
+            description: newFieldObj && newFieldObj.description,
+            shortDescription: newFieldObj && newFieldObj.short_description
           });
         }
       }

package/tools/redpanda-connect/rpcn-connector-docs-handler.js

@@ -30,11 +30,9 @@ function capToTwoSentences (description) {
   }
 
   const abbreviations = [
-    /https?:\/\/[^\s]+?(?=[.!?](?:\s|$)|\s|$)/gi,  // Protect URLs from being split by sentence detection (non-greedy, preserve trailing punctuation)
-    /\bxref:[^\s[\]]+\[[^\]]*\]/g,  // Protect Antora xref macros (e.g. xref:components:processors/try.adoc[`try`]) whose targets contain periods
-    /\b[\w/-]+\.(?:adoc|md|ya?ml|json|html)\b/gi,  // Protect bare doc/config filenames (try.adoc, config.yaml) from splitting
-    /\bv\d+\.\d+(?:\.\d+)?/gi,
-    /\d+\.\d+/g,
+    // A period inside a token (a URL, an xref target, a filename, a version
+    // or a decimal) has no whitespace after it, so it never ends a sentence.
+    // Only abbreviations that are followed by a space need protecting.
     /\be\.g\./gi,
     /\bi\.e\./gi,
     /\betc\./gi,
@@ -47,6 +45,21 @@ function capToTwoSentences (description) {
     /\bNo\./gi
   ]
 
+  // A table cell takes one paragraph of plain text. Keep only the text before
+  // the first blank line or list item, so a description such as
+  // "Controls how X is sized:\n\n- `a`: ...\n- `b`: ..." does not leak half a
+  // list into the cell. A lead-in that ends with a colon is left unterminated
+  // so the sentence match below drops it whenever a complete sentence comes
+  // before it ("Configure TCP settings. These controls are useful for:" keeps
+  // only the first sentence). Only a lead-in with nothing before it is kept,
+  // with the colon turned into a period.
+  let colonLeadIn = false
+  const firstBlock = description.split(/\n\s*\n|\n(?=\s*(?:[-*]|\d+\.)\s)/)[0].trim()
+  if (firstBlock && firstBlock !== description.trim() && !/^(?:[=#]|```|----)/.test(firstBlock)) {
+    description = firstBlock
+    colonLeadIn = firstBlock.endsWith(':')
+  }
+
   let normalized = description
   const placeholders = []
 
@@ -64,25 +77,19 @@ function capToTwoSentences (description) {
     return placeholder
   })
 
-  const sentenceRegex = /[^.!?]+[.!?]+(?:\s|$)/g
-  const sentences = normalized.match(sentenceRegex)
-
-  // If an unprotected mid-token period prevented the leading text from
-  // forming a complete match, merge that dropped prefix into the first
-  // sentence rather than silently discarding it.
-  if (sentences && sentences.length > 0) {
-    const firstIndex = normalized.indexOf(sentences[0])
-    if (firstIndex > 0) {
-      sentences[0] = normalized.slice(0, firstIndex) + sentences[0]
-    }
-  }
+  // A sentence ends at punctuation, optionally followed by a closing quote,
+  // parenthesis or bracket, then whitespace or the end of the text. Matching
+  // lazily up to that boundary keeps a period inside a token, such as the
+  // dotted name partition.assignment.strategy, in its sentence. Text after the
+  // last match (an unterminated colon lead-in, for example) is dropped.
+  const sentences = normalized.match(/[\s\S]+?[.!?]+["')\]`]*(?:\s|$)/g)
 
   if (!sentences || sentences.length === 0) {
     let result = normalized
     placeholders.forEach(({ placeholder, original }) => {
       result = result.replace(placeholder, original)
     })
-    return result
+    return colonLeadIn ? result.replace(/:$/, '.') : result
   }
 
   let maxSentences = 2
@@ -592,7 +599,7 @@ function buildFieldsTable (fields, capFn, opts = {}) {
     if (!byField[key]) {
       byField[key] = {
         field: field.field,
-        description: field.description,
+        description: field.shortDescription || field.description,
         introducedIn: field.introducedIn,
         components: []
       }
@@ -626,7 +633,8 @@ function buildFieldsTable (fields, capFn, opts = {}) {
       })
     }
 
-    const desc = info.description ? capFn(info.description) : '// TODO: Add description'
+    // A | in the description would end the cell and shift the row.
+    const desc = info.description ? capFn(info.description).replace(/\|/g, '\\|') : '// TODO: Add description'
 
     section += `|${fieldName}\n`
     section += `|${desc}\n`
@@ -655,7 +663,7 @@ function buildChangedDefaultsTable (changedDefaults, capFn) {
         field: change.field,
         oldDefault: change.oldDefault,
         newDefault: change.newDefault,
-        description: change.description,
+        description: change.shortDescription || change.description,
         components: []
       }
     }
@@ -667,16 +675,25 @@ function buildChangedDefaultsTable (changedDefaults, capFn) {
   section += '|Field |Old default |New default |Description |Affected components\n\n'
 
   for (const [, info] of Object.entries(byFieldAndDefaults)) {
+    // Values render as literal inline code, so interpolations like
+    // ${!uuid_v4()} and characters like * or _ show as typed.
+    // Values render as literal inline code through pass:c, which, unlike
+    // `+...+`, also keeps values with ++ or with spaces at either end. Strings
+    // that are empty, start or end with whitespace, or contain a newline show
+    // as JSON, so " " and "\n" read as the values they are.
     const formatDefault = (val) => {
       if (val === undefined || val === null) return 'none'
-      if (typeof val === 'string') return val
-      if (typeof val === 'number' || typeof val === 'boolean') return String(val)
-      return JSON.stringify(val)
+      let text
+      if (typeof val === 'string') text = (val === '' || /^\s|\s$|\n/.test(val)) ? JSON.stringify(val) : val
+      else if (typeof val === 'number' || typeof val === 'boolean') text = String(val)
+      else text = JSON.stringify(val)
+      return `\`pass:c[${text.replace(/\]/g, '\\]').replace(/\|/g, '\\|')}]\``
     }
 
     const oldVal = formatDefault(info.oldDefault)
     const newVal = formatDefault(info.newDefault)
-    const desc = info.description ? capFn(info.description) : '// TODO: Add description'
+    // A | in the description would end the cell and shift the row.
+    const desc = info.description ? capFn(info.description).replace(/\|/g, '\\|') : '// TODO: Add description'
 
     const byType = {}
     for (const comp of info.components) {

package/tools/redpanda-connect/templates/bloblang-function.hbs

@@ -1,4 +1,4 @@
-// This content is autogenerated. Do not edit manually. To override descriptions, use the doc-tools CLI with the --overrides option: https://redpandadata.atlassian.net/wiki/spaces/DOC/pages/1247543314/Generate+reference+docs+for+Redpanda+Connect
+{{{generatedMarker "rpcn-connector-docs" "the Bloblang spec in redpanda-data/connect or redpanda-data/benthos" "This file is a doc-tools preview. The published partials are generated in redpanda-data/connect by `task docs`, and their first line names the file that registers the function or method. To change the text, edit the spec in that file."}}}
 
 = {{name}}
 {{#if signature}}

package/tools/redpanda-connect/templates/descriptions-partials.hbs

@@ -1,5 +1,5 @@
 {{#if (or summary description version)}}
-// This content is autogenerated. Do not edit manually. The summary and description come from the connector's definition in the Connect source. To change them, update the connector upstream, or use the doc-tools CLI with the --overrides option: https://redpandadata.atlassian.net/wiki/spaces/DOC/pages/1247543314/Generate+reference+docs+for+Redpanda+Connect
+{{{generatedMarker "rpcn-connector-docs" "the component spec in redpanda-data/connect or redpanda-data/benthos" "The summary and description come from the component spec. This file is a doc-tools preview. The published partials are generated in redpanda-data/connect by `task docs`, and their first line names the file that registers the component. To change the text, edit the component spec in that file."}}}
 //
 // Pages consume this partial in two places so both stay fresh:
 //   header: include::...[tag=meta]  (sets :description: for search snippets and meta tags)

package/tools/redpanda-connect/templates/examples-partials.hbs

@@ -1,5 +1,5 @@
 {{#with examples}}
-// This content is autogenerated. Do not edit manually. To override descriptions, use the doc-tools CLI with the --overrides option: https://redpandadata.atlassian.net/wiki/spaces/DOC/pages/1247543314/Generate+reference+docs+for+Redpanda+Connect
+{{{generatedMarker "rpcn-connector-docs" "the component spec in redpanda-data/connect or redpanda-data/benthos" "This file is a doc-tools preview. The published partials are generated in redpanda-data/connect by `task docs`, and their first line names the file that registers the component. To change the text, edit the component spec in that file."}}}
 
 == Examples
 

package/tools/redpanda-connect/templates/fields-partials.hbs

@@ -1,11 +1,11 @@
 {{#if this.config.children}}
-// This content is autogenerated. Do not edit manually. To override descriptions, use the doc-tools CLI with the --overrides option: https://redpandadata.atlassian.net/wiki/spaces/DOC/pages/1247543314/Generate+reference+docs+for+Redpanda+Connect
+{{{generatedMarker "rpcn-connector-docs" "the component spec in redpanda-data/connect or redpanda-data/benthos" "This file is a doc-tools preview. The published partials are generated in redpanda-data/connect by `task docs`, and their first line names the file that registers the component. To change the text, edit the component spec in that file."}}}
 
 == Fields
 
 {{{renderConnectFields this.config.children}}}
 {{else if this.children}}
-// This content is autogenerated. Do not edit manually. To override descriptions, use the doc-tools CLI with the --overrides option: https://redpandadata.atlassian.net/wiki/spaces/DOC/pages/1247543314/Generate+reference+docs+for+Redpanda+Connect
+{{{generatedMarker "rpcn-connector-docs" "the component spec in redpanda-data/connect or redpanda-data/benthos" "This file is a doc-tools preview. The published partials are generated in redpanda-data/connect by `task docs`, and their first line names the file that registers the component. To change the text, edit the component spec in that file."}}}
 
 == Fields
 

package/tools/redpanda-connect/templates/metadata-partials.hbs

@@ -1,5 +1,5 @@
 {{#if (renderConnectMetadata description)}}
-// This content is autogenerated. Do not edit manually. Metadata fields come from the connector's description in the Connect source. To change them, update the connector description upstream, or use the doc-tools CLI with the --overrides option: https://redpandadata.atlassian.net/wiki/spaces/DOC/pages/1247543314/Generate+reference+docs+for+Redpanda+Connect
+{{{generatedMarker "rpcn-connector-docs" "the component spec in redpanda-data/connect or redpanda-data/benthos" "Metadata fields come from the `== Metadata` section of the component description. This file is a doc-tools preview. The published partials are generated in redpanda-data/connect by `task docs`, and their first line names the file that registers the component. To change the text, edit the component spec in that file."}}}
 
 {{{renderConnectMetadata description}}}
 {{/if}}