@redpanda-data/docs-extensions-and-macros 5.46.0 → 5.47.0

package/tools/overrides-audit/classify.js

@@ -59,6 +59,10 @@ const PRODUCT_ATTR_ALLOWLIST = Object.freeze([])
  */
 const KEEP_BY_DESIGN_FIELDS = Object.freeze([
   'related_topics',
+  'see_also',
+  'admonitions',
+  'links',
+  'includes',
   'category',
   'config_scope',
   'version',
@@ -89,7 +93,59 @@ const META_FIELDS = Object.freeze(['upstream_ref', '_comment'])
  */
 function normalizeText (text) {
   if (typeof text !== 'string') return ''
-  return text.replace(/\s+/g, ' ').trim()
+  // Horizontal whitespace collapses; newlines do not. A blank line is a
+  // paragraph break and a newline before a list marker is a list-item
+  // boundary, and both change what AsciiDoc renders -- the C++ source is one
+  // run-on sentence with no newlines at all, so treating \n the same as a
+  // plain space made an override that turns that wording into a real
+  // bulleted list or separate paragraphs compare byte-equal to source once
+  // "normalized". That classified it REDUNDANT, and the retirement step
+  // deletes an override wholesale, replacing a rendered list with a
+  // run-on paragraph carrying literal `*` characters. Verified against
+  // fetch_read_strategy in the live corpus: 2 rendered list items with the
+  // override, 0 (plus two stray asterisks) once "equal to source" deleted
+  // it.
+  return text
+    .split('\n')
+    .map((line) => line.replace(/[ \t]+/g, ' ').trim())
+    .join('\n')
+    .replace(/\n{3,}/g, '\n\n')
+    .trim()
+}
+
+/**
+ * Separate a description's unconditional prose from its audience-scoped
+ * paragraphs.
+ *
+ * A description is either a plain string or an array of paragraphs, each
+ * optionally prefixed `cloud-only:` / `self-managed-only:`. Only the
+ * unconditional paragraphs are comparable to the source string: a paragraph
+ * written for one docs build is docs-only by definition and must never reach an
+ * upstream candidate, because the C++ doc string has no audience.
+ *
+ * The string form is returned unchanged, so every existing classification is
+ * byte-for-byte what it was before the array form existed.
+ *
+ * @param {string|Array<string>} description - Override description, either form.
+ * @returns {{prose: string, scoped: number, scopedParagraphs: string[]}} The
+ *   unconditional prose, how many paragraphs were scoped away, and their text.
+ *   The text is returned so a row can show a reviewer what it is holding back
+ *   rather than dropping it silently.
+ */
+function unconditionalProse (description) {
+  if (typeof description === 'string') return { prose: description, scoped: 0, scopedParagraphs: [] }
+  if (!Array.isArray(description)) return { prose: '', scoped: 0, scopedParagraphs: [] }
+  const kept = []
+  const scopedParagraphs = []
+  for (const paragraph of description) {
+    if (typeof paragraph !== 'string' || !paragraph.trim()) continue
+    if (/^\s*(?:cloud-only|self-managed-only):/.test(paragraph)) {
+      scopedParagraphs.push(paragraph.trim())
+      continue
+    }
+    kept.push(paragraph.trim())
+  }
+  return { prose: kept.join('\n\n'), scoped: scopedParagraphs.length, scopedParagraphs }
 }
 
 // Docs-only markup detectors. `<<anchor>>` cross-references are included
@@ -115,6 +171,12 @@ const MARKUP_PATTERNS = Object.freeze([
   { id: 'audience-prefix', re: /\b(?:self-managed-only|cloud-only):/ }
 ])
 
+// Pulled out of MARKUP_PATTERNS by id, rather than re-typed, so the two
+// checks that care about audience-scoping specifically -- the guard below,
+// and MARKUP_PATTERNS' own detection -- cannot silently drift apart.
+const AUDIENCE_PREFIX_RX = MARKUP_PATTERNS.find((p) => p.id === 'audience-prefix').re
+const CONDITIONAL_RX = MARKUP_PATTERNS.find((p) => p.id === 'conditional').re
+
 /**
  * Find unescaped AsciiDoc attribute references (`{attr}`) that are not on
  * the product-attr allowlist. Escaped literals (`\{attr}`) do not count.
@@ -264,7 +326,7 @@ function deepEqual (a, b) {
  */
 function row (fields) {
   const base = { name: fields.name, field: fields.field, class: fields.class }
-  for (const key of ['upstream_candidate_text', 'upstream_ref', 'content_hash', 'source_file', 'source_line', 'note']) {
+  for (const key of ['upstream_candidate_text', 'audience_scoped_text', 'upstream_ref', 'content_hash', 'source_file', 'source_line', 'note']) {
     if (fields[key] !== undefined) base[key] = fields[key]
   }
   return base
@@ -281,7 +343,15 @@ function row (fields) {
  */
 function classifyDescription (name, override, sourceProp, opts = {}) {
   const attrAllowlist = opts.attrAllowlist || PRODUCT_ATTR_ALLOWLIST
-  const overrideText = override.description
+  // Audience-scoped paragraphs are filtered out here rather than inside
+  // stripDocsMarkup, which must keep stripping a prefix and retaining the
+  // content: for a see_also item the prefixed value IS the link, and dropping
+  // it there would silently empty every conditional related-topics entry.
+  const {
+    prose: overrideText,
+    scoped: scopedCount,
+    scopedParagraphs: scopedText
+  } = unconditionalProse(override.description)
   const common = {
     name,
     field: 'description',
@@ -290,6 +360,13 @@ function classifyDescription (name, override, sourceProp, opts = {}) {
     source_line: sourceProp && sourceProp.line_start !== undefined ? sourceProp.line_start : undefined
   }
   if (override.upstream_ref !== undefined) common.upstream_ref = override.upstream_ref
+  // Shown, never upstreamed. A scoped paragraph exists precisely so the other
+  // audience does not see it, and the C++ doc string is one string for every
+  // audience, so upstreaming it would both leak it to the wrong readers and
+  // duplicate it for the right ones (the override keeps its copy). Putting the
+  // text on the row lets a reviewer decide to upstream it and delete the
+  // paragraph in one deliberate change, instead of the audit deciding for them.
+  if (scopedText && scopedText.length > 0) common.audience_scoped_text = scopedText
 
   if (!sourceProp) {
     return row({
@@ -299,10 +376,43 @@ function classifyDescription (name, override, sourceProp, opts = {}) {
     })
   }
 
+  // A source property with no line_start has no description to replace. The
+  // topic extractor reports a name constant (topic_properties.h,
+  // handlers/topics/types.h) and fills description with an empty string, so a
+  // naive comparison reads "override differs from source" and proposes
+  // upstreaming prose into a file that has nowhere to put it. Measured against
+  // dev: 45 of 216 candidates, and not one of them in a .cc file, while every
+  // row the upstreaming workflow can actually edit carries a line.
+  //
+  // This is REVIEW rather than KEEP because the right answer is a judgement
+  // call: give the property a real doc string upstream, or accept that docs are
+  // the only place it is described. Never UPSTREAMABLE, which would hand the
+  // upstreaming workflow work it cannot do.
+  if (sourceProp.line_start === undefined) {
+    return row({
+      ...common,
+      class: CLASSES.REVIEW,
+      note:
+        'Source defines this property\'s name but carries no description for it (' +
+        (sourceProp.defined_in || 'unknown file') +
+        '), so there is nothing to upstream into. Decide whether the source should gain one.'
+    })
+  }
+
   const sourceText = normalizeText(sourceProp.description)
   const overrideNormalized = normalizeText(overrideText)
 
   if (overrideNormalized === sourceText) {
+    if (scopedCount > 0) {
+      // Retiring this would delete the audience-scoped paragraphs with it,
+      // because `description` replaces wholesale rather than merging. The
+      // redundant prose cannot be dropped on its own.
+      return row({
+        ...common,
+        class: CLASSES.KEEP,
+        note: `Unconditional prose already matches source, but ${scopedCount} audience-scoped paragraph(s) are docs-only and would be lost with it. Nothing to upstream.`
+      })
+    }
     return row({
       ...common,
       class: CLASSES.REDUNDANT,
@@ -310,8 +420,55 @@ function classifyDescription (name, override, sourceProp, opts = {}) {
     })
   }
 
+  // A STRING description carrying a raw audience prefix or a hand-written
+  // ifdef/ifndef pair was never routed through the array form's
+  // paragraph-level scoping: unconditionalProse only strips a
+  // cloud-only:/self-managed-only: paragraph out of an ARRAY, and returns a
+  // string verbatim. Falling through to stripDocsMarkup below silently
+  // discards the audience information instead of refusing to guess at it: a
+  // raw "cloud-only:" prefix strips to a bare sentence with no trace it was
+  // ever Cloud-specific, and a hand-written ifdef/ifndef pair keeps BOTH
+  // branches' text concatenated as if they were one universal sentence.
+  // Either shape reaching upstream_candidate_text would write
+  // audience-specific or audience-merged prose into a C++ string every
+  // audience reads.
+  if (typeof override.description === 'string' &&
+      (AUDIENCE_PREFIX_RX.test(override.description) || CONDITIONAL_RX.test(override.description))) {
+    return row({
+      ...common,
+      class: CLASSES.REVIEW,
+      note: 'This description is a plain string but contains an audience-scoped prefix or a hand-written conditional. Rewrite it as the array form (one entry per paragraph, prefixed cloud-only:/self-managed-only: where a paragraph is scoped) so the scoped text is never mistaken for universal prose.'
+    })
+  }
+
+  // A description made ENTIRELY of audience-scoped paragraphs has no
+  // unconditional prose, so overrideText is the empty string. Without this
+  // guard it fell through to the SPLIT branch below and published an empty
+  // upstream_candidate_text -- which the upstream workflow selects, because
+  // the SPLIT note is part of its filter, and would then hand the model an
+  // empty string to write into an engineering doc string, blanking it.
+  if (overrideText.length === 0) {
+    return row({
+      ...common,
+      class: CLASSES.REVIEW,
+      note: scopedCount > 0
+        ? `Every paragraph of this description is audience-scoped (${scopedCount}), so there is no unconditional prose to upstream. Decide whether the scoped text belongs in source at all, or whether the property needs unconditional prose written for it.`
+        : 'The override sets an empty description. Decide whether it should be removed or given prose.'
+    })
+  }
+
   const markupKinds = detectDocsMarkup(overrideText, attrAllowlist)
   if (markupKinds.length === 0) {
+    if (scopedCount > 0) {
+      // The prose is upstreamable but the override still has to stay, to carry
+      // the scoped paragraphs -- which is exactly the SPLIT case.
+      return row({
+        ...common,
+        class: CLASSES.KEEP_UNTIL_UPSTREAMED,
+        upstream_candidate_text: overrideText,
+        note: `SPLIT: unconditional prose differs from source and is markup-free, but ${scopedCount} audience-scoped paragraph(s) must stay in the override. Keep it until the prose ships.`
+      })
+    }
     return row({
       ...common,
       class: CLASSES.UPSTREAMABLE,
@@ -321,6 +478,20 @@ function classifyDescription (name, override, sourceProp, opts = {}) {
   }
 
   const stripped = stripDocsMarkup(overrideText)
+  // An include-only (or otherwise entirely-markup) description strips to
+  // nothing. Without this the SPLIT branch below would publish an empty
+  // upstream_candidate_text, which the upstream workflow selects (its
+  // filter matches on class and a SPLIT note, not on the text being
+  // non-empty) and would hand the model an empty string to write into a
+  // source description -- blanking it. The same failure the
+  // all-scoped-paragraphs guard above exists to prevent for the array form.
+  if (stripped.trim().length === 0) {
+    return row({
+      ...common,
+      class: CLASSES.REVIEW,
+      note: `Stripping docs-only markup (${markupKinds.join(', ')}) leaves no prose at all, so there is nothing to upstream. Decide whether the source needs a real description written for it.`
+    })
+  }
   if (normalizeText(stripped) === sourceText) {
     return row({
       ...common,
@@ -530,6 +701,7 @@ function summarize (manifest) {
 module.exports = {
   CLASSES,
   PRODUCT_ATTR_ALLOWLIST,
+  unconditionalProse,
   KEEP_BY_DESIGN_FIELDS,
   TYPO_KEYS,
   normalizeText,

package/tools/property-extractor/README.adoc

@@ -156,11 +156,13 @@ Create a `property-overrides.json` file to customize extracted documentation:
 
 *Override fields:*
 
-* `description` - Replace auto-extracted description with custom text
+* `description` - Replace the auto-extracted description. Either a string, or an array of paragraphs when one of them belongs to a single docs build: an array entry may carry a `cloud-only:` / `self-managed-only:` prefix and the generator wraps it in the matching conditional. Use the array form rather than writing `ifdef::env-cloud[]` into the string. See <<Keeping a description upstreamable>>.
+* `links` - Inline links, keyed by the literal text to find in the prose and valued by the target. See <<Adding links without forking the source prose>>.
+* `includes` - Shared partials to pull into the property's entry, as Antora resource IDs. See <<Pulling in a shared partial>>.
 * `example` - Add usage example in AsciiDoc format
 * `example_file` - Load example from external file
 * `version` - Document when property was introduced
-* `see_also` - Array of related links. Each item is either a plain string (shown in every build) or an object `{content, cloud_only}` / `{content, self_hosted_only}` (shown only in that build, wrapped with `ifdef::env-cloud[]`/`ifndef::env-cloud[]` at generation time). Setting both `cloud_only` and `self_hosted_only` on the same item is rejected by the schema — it would wrap the item in both directives at once, so it renders in neither build.
+* `see_also` - Array of related links. Each item is either a plain string (shown in every build) or an object `{content, cloud_only}` / `{content, self_managed_only}` (shown only in that build, wrapped with `ifdef::env-cloud[]`/`ifndef::env-cloud[]` at generation time). Setting both `cloud_only` and `self_managed_only` on the same item is rejected by the schema — it would wrap the item in both directives at once, so it renders in neither build.
 * `related_topics` - Deprecated predecessor to `see_also`. An array of plain strings, optionally prefixed `cloud-only:` / `self-managed-only:` for the same conditional behavior — that prefix is free text with no validation, which is why `see_also` replaced it. Existing `related_topics` entries keep working; new entries should use `see_also`.
 
 Validate an overrides file against its schema (`docs-data/property-overrides.schema.json`) before generating docs:
@@ -170,6 +172,94 @@ Validate an overrides file against its schema (`docs-data/property-overrides.sch
 npx doc-tools validate property-overrides --overrides docs-data/property-overrides.json
 ----
 
+==== Adding links without forking the source prose
+
+A property description is plain prose, and the links in it are declared separately:
+
+[,json]
+----
+{
+  "properties": {
+    "advertised_rpc_api": {
+      "description": "Address of RPC endpoint published to other cluster members. If not set, the `rpc_server` broker property is used.",
+      "links": {
+        "`rpc_server`": "#rpc_server"
+      }
+    },
+    "recovery_mode_enabled": {
+      "description": "If `true`, start Redpanda in recovery mode, where user partitions are not loaded.",
+      "links": {
+        "recovery mode": "self-managed-only: xref:manage:recovery-mode.adoc"
+      }
+    }
+  }
+}
+----
+
+Each key is the literal text to find in the prose, backticks included, as written. Only the first occurrence is linked, so a property named four times links once. The longest key wins, so `+`cloud_storage_enabled`+` cannot be stolen by `+`cloud_storage`+`.
+
+Three target forms:
+
+`#<property_name>`:: Emits a `prop` macro. The macro resolves the reference page itself and renders plain code in a build that does not publish the target, so a property link needs no conditional and cannot dangle. A name that is not a known property is refused with a warning rather than emitted, because an unverifiable `prop` call publishes as literal macro text.
+`xref:<target>`:: Emits a literal xref with the key as its link text. Property-page targets are canonicalized to `xref:reference:properties/<page>.adoc`.
+`glossterm`:: Wraps the key in a glossary tooltip. The macro takes the term as its target and looks the definition up in the glossary, so the key is the term. Use `glossterm:<Term>` when the prose says something other than the term's own name.
+
+A target may carry a `cloud-only:` / `self-managed-only:` prefix, in which case the containing paragraph is emitted twice: linked in that branch, plain in the other. AsciiDoc preprocessor directives have to start a line, so they cannot bracket an inline span; the paragraph is the smallest thing that can be wrapped. A scoped link that matches inside a delimited block is refused with a warning, since duplicating that paragraph would split the block across both branches.
+
+The generator reports every key that matched no text. A link key that matches nothing is silent link rot: the override looks maintained and the page has no link.
+
+Links are applied at generation time, so they reach the rendered partials but not the published `redpanda-properties-<tag>.json`. That file is the machine-readable data surface, and its descriptions drive hover tooltips, where an inline link is not wanted.
+
+==== Pulling in a shared partial
+
+Some entries end with a shared block: the "internal use only" warning, or the HTTP Proxy ephemeral-credentials breaking-change notice. Those are block-level docs structure, so they go in `includes` rather than into the description string:
+
+[,json]
+----
+{
+  "properties": {
+    "data_transforms_read_buffer_memory_percentage": {
+      "description": "The percentage of available memory in the transform subsystem to use for read buffers.",
+      "includes": ["reference:partial$internal-use-property.adoc"]
+    }
+  }
+}
+----
+
+Each entry is an Antora resource ID, rendered as an `include::` directive immediately after the description, which is where these were previously written by hand. An entry may carry a `cloud-only:` / `self-managed-only:` prefix, and its own `[attrs]` if it needs them; without brackets the generator adds the empty pair the directive requires. A value that does not look like a resource ID is emitted as given, with a warning, because an include with no family segment resolves to nothing at build time and does it silently.
+
+==== Keeping a description upstreamable
+
+`links`, `includes`, `see_also` and `admonitions` all exist so that a description can carry docs-site structure without mixing markup into the sentence. That is not a style preference. It is what makes the override retirable. `doc-tools overrides audit` compares an override's prose against the C++ doc string to decide whether it is redundant, upstreamable, or has to stay. Markup in the sentence makes the two incomparable, so an enriched description could never be retired or sent upstream. A hand-written `ifdef::env-cloud[]` does the same thing, which is why the array form exists.
+
+An audience-scoped paragraph is docs-only by definition, because the source string has no audience, so the audit compares only the unconditional paragraphs and never puts scoped text into an upstream candidate.
+
+==== Conditional admonitions
+
+An `admonitions` entry takes the same `cloud_only` / `self_managed_only` booleans as a `see_also` object, wrapping the whole block:
+
+[,json]
+----
+{
+  "admonitions": [
+    { "type": "note", "text": "Applies to every build." },
+    { "type": "tip", "text": "Cloud readers only.", "cloud_only": true }
+  ]
+}
+----
+
+==== One name per concept
+
+The audience flags are `cloud-only:` / `self-managed-only:` as a string prefix, and `cloud_only` / `self_managed_only` as object booleans. The hyphens and underscores differ because one is prose and the other is a JSON key in a snake_case file, but the *word* is the same in both: self-managed, matching the product name.
+
+`self_hosted_only` was the old spelling of that boolean, so the same concept went by two different words depending on how you wrote it. It still parses, and normalizes to `self_managed_only`, so nothing emits it. `docs-data/rpk-overrides.json` still spells its equivalent `selfHostedOnly` (camelCase, to match that file's own key style) and carries 34 live entries, so it needs its own rename.
+
+==== Conditionals and blank lines
+
+Every conditional the generator emits has a blank line before it and after its `endif::[]`. This is load-bearing. A conditional is resolved by the preprocessor, so one glued to neighbouring prose leaves those lines adjacent and Asciidoctor merges them into a single paragraph. Worse, when the conditional wraps a delimited block it merges them in only one branch, so a self-managed preview can look correct while Cloud loses its paragraph breaks. Nothing errors and no text goes missing, so only a structural assertion on rendered HTML catches it.
+
+The generator warns about any conditional it finds glued, including one an override still writes by hand. `__tests__/tools/property-extractor/conditional-rendering.test.js` pins the measured behaviour and proves both detectors fire; `property-corpus-rendering.test.js` runs the whole live overrides corpus through Asciidoctor in both builds.
+
 This catches malformed `see_also` items and unrecognized override keys — including a typo that would otherwise be silently dropped, since applying an override to an *existing* property has no catch-all pass-through (only overrides that create a new property from scratch do).
 
 === Understanding property changes
@@ -1130,6 +1220,55 @@ npx jest __tests__/tools/topic_property_extractor.test.js
 npm test -- --coverage
 ----
 
+==== The live overrides corpus
+
+`__tests__/tools/property-extractor/property-corpus-rendering.test.js` runs the real
+overrides file, not a fixture, because a fixture passes while the live 438-entry corpus
+breaks. Two files feed it, both mirrors of the `docs` repo:
+
+`__tests__/docs-data/property-overrides.json`:: A byte copy of `docs-data/property-overrides.json`.
+`__tests__/docs-data/property-snapshot.json`:: Every extracted property, reduced to the
+fields the rendering and link layers read. `cloud_supported` is the point: it is what
+makes the "no Cloud link to a property Cloud does not publish" assertion real rather
+than assumed. Descriptions are deliberately absent, because the corpus test drives prose from
+the overrides file, and a full copy would be 695 KB of text that rots.
+
+Keep the overrides copy a faithful mirror. It previously doubled as
+`property-docs-overrides.test.js`'s fixture, someone hand-edited an `admin` entry into
+it that never existed in the `docs` repo, and refreshing it would then have broken that
+test, so it sat three months and 114 entries behind. That test now has its own
+`property-overrides-admin-fixture.json`; put new fixture data there or in the test
+itself, never in the mirror.
+
+Refresh both from a `docs` checkout:
+
+[,bash]
+----
+DOCS=../docs   # path to your docs repo checkout
+
+git -C "$DOCS" show origin/main:docs-data/property-overrides.json \
+  > __tests__/docs-data/property-overrides.json
+
+git -C "$DOCS" show origin/main:modules/reference/attachments/redpanda-properties-v26.2.2.json \
+  | node -e '
+    const fs = require("fs");
+    const src = JSON.parse(fs.readFileSync(0, "utf8"));
+    const keep = ["name","config_scope","type","cloud_supported","cloud_editable",
+                  "cloud_readonly","cloud_byoc_only","is_deprecated","nullable"];
+    const properties = {};
+    for (const [k, v] of Object.entries(src.properties || {})) {
+      properties[k] = Object.fromEntries(keep.filter((f) => f in v).map((f) => [f, v[f]]));
+    }
+    const out = JSON.parse(fs.readFileSync("__tests__/docs-data/property-snapshot.json", "utf8"));
+    out.properties = properties;
+    fs.writeFileSync("__tests__/docs-data/property-snapshot.json", JSON.stringify(out, null, 1));
+  '
+----
+
+The `docs` repo fails its own CI when the overrides mirror drifts, following the
+`sync-schemas --check` pattern; this repo's CI has no `docs` checkout, so the check
+cannot live here.
+
 === Making changes
 
 1. **Before you start:**

package/tools/property-extractor/compare-properties.js

@@ -225,8 +225,18 @@ function compareProperties(oldData, newData, oldVersion, newVersion) {
         });
       }
       
-      // Check for description changes
-      if (oldProp.description !== newProp.description) {
+      // Check for description changes. deepEqual, not !==: an array-form
+      // description (audience-scoped paragraphs) is a fresh array on both
+      // sides of every comparison, since each side comes from parsing its
+      // own attachment JSON, so reference inequality reported EVERY
+      // property carrying one as changed, forever, with identical old and
+      // new text -- permanent noise in the one section of the release PR a
+      // reviewer reads to spot real engineering wording changes. In
+      // practice this attachment is flattened to a string before it ships
+      // (see generate-handlebars-docs.js), so deepEqual and !== agree on a
+      // real release comparison; this guards the case where compare runs
+      // against a pre-flatten JSON, or the flatten step is ever skipped.
+      if (!deepEqual(oldProp.description, newProp.description)) {
         report.changedDescriptions.push({
           name,
           oldDescription: oldProp.description || 'No description',