@redpanda-data/docs-extensions-and-macros 5.55.0 → 5.56.0

package/tools/lint-strings/index.js

@@ -60,6 +60,9 @@ function rulesFor (surface) {
  * @param {Set<string>} [options.reviewedFingerprints] - Diff mode: skip
  *   declarations (and removals) whose fingerprint is in this set, so a PR
  *   review sees each string once across pushes
+ * @param {string[]} [options.reviewerPassages] - Diff mode: passages from
+ *   human review comments. A changed line found in one counts as written by
+ *   a reviewer, and a declaration whose changed lines all are is skipped
  * @param {Function} [options.log] - Progress logger (stderr by default)
  * @returns {Object} { findings, summary, unsupported_surfaces }
  */
@@ -71,6 +74,7 @@ function lintStrings (options) {
     skipRules = [],
     onlyRules = null,
     reviewedFingerprints = null,
+    reviewerPassages = null,
     log = (msg) => process.stderr.write(`${msg}\n`)
   } = options
 
@@ -93,6 +97,8 @@ function lintStrings (options) {
   const reviewed = reviewedFingerprints || new Set()
   const pending = []
   let alreadyReviewed = 0
+  const reviewerWritten = []
+  const headLines = new Map()
   let removal = { rawFiles: [], declarations: [] }
   const skipped = []
   let unverifiable = 0
@@ -138,6 +144,17 @@ function lintStrings (options) {
           alreadyReviewed++
           continue
         }
+        if (reviewerPassages && reviewerPassages.length) {
+          const changed = changedLinesOf(repoPath, decl, files.get(decl.file), headLines)
+          const own = passagesInDeclaration(decl, reviewerPassages)
+          if (changed.length > 0 && changed.every((l) => writtenByReviewer(l.text, own))) {
+            // No wording review, but the published-content check still
+            // sees it: a reviewer's wording can describe a behavior change
+            // that makes another page wrong.
+            reviewerWritten.push(pendingEntry(decl))
+            continue
+          }
+        }
         declarations.push(decl)
         if (decl.meta && decl.meta.unverifiable) unverifiable++
         const detail = surface.reviewContext ? surface.reviewContext(decl) : null
@@ -181,6 +198,8 @@ function lintStrings (options) {
   if (diffBase) {
     merged.declarations = pending
     merged.summary.alreadyReviewed = alreadyReviewed
+    merged.summary.reviewerWritten = reviewerWritten.length
+    merged.summary.reviewerWrittenDeclarations = reviewerWritten
   }
   merged.summary.removedDeclarations = removal.declarations.map(pendingEntry)
   // Declarations whose text has gaps the extractor could not evaluate
@@ -468,6 +487,9 @@ function formatHuman (result) {
   if (summary.alreadyReviewed) {
     lines.push(`Already reviewed on an earlier push (skipped): ${summary.alreadyReviewed}`)
   }
+  if (summary.reviewerWritten) {
+    lines.push(`Text a human reviewer suggested on the PR (skipped): ${summary.reviewerWritten}`)
+  }
   for (const decl of summary.removedDeclarations || []) {
     lines.push(`Removed or renamed: ${decl.name} (${decl.surface}, ${decl.file})`)
   }
@@ -520,6 +542,102 @@ function readFingerprints (file) {
   return new Set(text.split(/[\s,]+/).filter((token) => /^[0-9a-f]{16}$/.test(token)))
 }
 
+// Shorter passages are labels, lead-ins ("Suggest replacing this with:") or
+// fragments that occur in many strings, so matching on them would skip
+// declarations nobody reviewed.
+const MIN_REVIEWER_PASSAGE = 40
+
+function normalizeText (text) {
+  return String(text == null ? '' : text).replace(/\s+/g, ' ').trim()
+}
+
+/**
+ * Passages from human review comments that a declaration can be matched
+ * against: the body of every ```suggestion block, and every prose paragraph
+ * outside a code block. Whitespace is collapsed so a paragraph the author
+ * re-wrapped to the file's line length still matches.
+ */
+function reviewerPassagesFrom (bodies) {
+  const passages = new Set()
+  for (const body of Array.isArray(bodies) ? bodies : []) {
+    if (typeof body !== 'string') continue
+    const chunks = [...body.matchAll(/```suggestion[^\n]*\n([\s\S]*?)```/g)].map((m) => m[1])
+    chunks.push(...body.replace(/```[\s\S]*?(```|$)/g, '\n\n').split(/\n\s*\n/))
+    for (const chunk of chunks) {
+      const text = normalizeText(chunk)
+      if (text.length >= MIN_REVIEWER_PASSAGE) passages.add(text)
+    }
+  }
+  return [...passages]
+}
+
+// A changed line shorter than this ("per topic.", "For example,") occurs in
+// too many paragraphs to say who wrote it, so it never counts as a
+// reviewer's. That errs toward reviewing.
+const MIN_REVIEWER_LINE = 12
+
+/**
+ * The lines this PR changed inside a declaration's span, as HEAD has them.
+ * Lines with no letters or digits (a closing backtick, a blank line) say
+ * nothing about who wrote the string and are left out.
+ */
+function changedLinesOf (repoPath, decl, lineSet, cache) {
+  if (!lineSet) return []
+  if (!cache.has(decl.file)) {
+    let lines = null
+    try {
+      lines = fs.readFileSync(path.join(repoPath, decl.file), 'utf8').split('\n')
+    } catch {}
+    cache.set(decl.file, lines)
+  }
+  const lines = cache.get(decl.file)
+  if (!lines) return []
+  const out = []
+  for (const [start, end] of spansOf(decl)) {
+    if (start == null || end == null) continue
+    for (let n = start; n <= end; n++) {
+      if (!lineSet.has(n) || n > lines.length) continue
+      const text = normalizeText(lines[n - 1])
+      if (/[A-Za-z0-9]/.test(text)) out.push({ line: n, text })
+    }
+  }
+  return out
+}
+
+/**
+ * The passages a declaration contains whole. A reviewer's proposed wording
+ * lands in the string as written, but a comment that quotes the string to
+ * criticize it ("Please change this: ...") carries a lead-in the string
+ * does not have, so it never counts as the reviewer writing the string.
+ */
+function passagesInDeclaration (decl, passages) {
+  const forms = [normalizeText(decl.string), normalizeText(decl.declaration_text)].filter(Boolean)
+  return passages.filter((passage) => forms.some((form) => form.includes(passage)))
+}
+
+/**
+ * True when a human reviewer wrote this line on the PR: it appears whole in
+ * one of their passages. Matching per line, not per string, means an author
+ * who keeps a reviewer's paragraph and rewrites the next one still gets the
+ * new paragraph reviewed (streaming-enterprise PR 461 is the case this
+ * exists for: the review argued with help text applied from a reviewer).
+ */
+function writtenByReviewer (line, passages) {
+  return line.length >= MIN_REVIEWER_LINE && passages.some((passage) => passage.includes(line))
+}
+
+/**
+ * Read a --reviewer-comments file: a JSON array of comment bodies. A missing
+ * or unparseable file yields no passages, so it means "review everything".
+ */
+function readReviewerComments (file) {
+  try {
+    return reviewerPassagesFrom(JSON.parse(fs.readFileSync(file, 'utf8')))
+  } catch {
+    return []
+  }
+}
+
 /**
  * CLI entry point shared by bin/doc-tools.js and direct invocation
  * (node tools/lint-strings --repo <path> ...).
@@ -536,7 +654,8 @@ function runCli (options) {
       diffBase: options.diff || null,
       skipRules: options.skipRules ? String(options.skipRules).split(',').map((s) => s.trim()).filter(Boolean) : [],
       onlyRules: options.onlyRules ? String(options.onlyRules).split(',').map((s) => s.trim()).filter(Boolean) : null,
-      reviewedFingerprints: options.reviewed ? readFingerprints(options.reviewed) : null
+      reviewedFingerprints: options.reviewed ? readFingerprints(options.reviewed) : null,
+      reviewerPassages: options.reviewerComments ? readReviewerComments(options.reviewerComments) : null
     })
   } catch (err) {
     console.error(`Error: ${err.message}`)
@@ -555,7 +674,7 @@ function runCli (options) {
   process.exitCode = options.strict && result.summary.errors > 0 ? 1 : 0
 }
 
-module.exports = { lintStrings, formatHuman, runCli, SURFACES, rulesFor, fingerprint, readFingerprints, pageContext }
+module.exports = { lintStrings, formatHuman, runCli, SURFACES, rulesFor, fingerprint, readFingerprints, reviewerPassagesFrom, readReviewerComments, pageContext }
 
 // Direct usage: node tools/lint-strings --repo <path> [--surface a,b]
 //   [--diff <base>] [--format json|human] [--strict]
@@ -571,10 +690,21 @@ if (require.main === module) {
     else if (arg === '--skip-rules') options.skipRules = args[++i]
     else if (arg === '--only-rules') options.onlyRules = args[++i]
     else if (arg === '--reviewed') options.reviewed = args[++i]
+    else if (arg === '--reviewer-comments') {
+      // A present flag with no operand must not act like an absent one, or
+      // `--reviewer-comments --strict` would read --strict as the file and
+      // quietly turn strict mode off.
+      const operand = args[i + 1]
+      if (!operand || operand.startsWith('-')) {
+        console.error('--reviewer-comments requires a file path')
+        process.exit(2)
+      }
+      options.reviewerComments = args[++i]
+    }
     else if (arg === '--strict') options.strict = true
     else {
       console.error(`Unknown argument: ${arg}`)
-      console.error('Usage: node tools/lint-strings --repo <path> [--surface a,b] [--diff <base>] [--format json|human] [--skip-rules a,b] [--only-rules a,b] [--reviewed <file>] [--strict]')
+      console.error('Usage: node tools/lint-strings --repo <path> [--surface a,b] [--diff <base>] [--format json|human] [--skip-rules a,b] [--only-rules a,b] [--reviewed <file>] [--reviewer-comments <file>] [--strict]')
       process.exit(2)
     }
   }

package/tools/lint-strings/rules/verbatim-asciidoc.js

@@ -105,4 +105,169 @@ const VERBATIM_ASCIIDOC_RULES = [
   }
 ]
 
-module.exports = { VERBATIM_ASCIIDOC_RULES, KNOWN_ATTRIBUTES, MACRO_NAMES, countRawPipes }
+/**
+ * AsciiDoc preprocessor conditionals in doc strings that publish to two
+ * audiences. Connect spec strings are published on the self-managed Connect
+ * docs and on the Redpanda Cloud docs, which set the `env-cloud` attribute,
+ * and each Connect binary renders the same strings as Markdown with the
+ * attribute set only in the Cloud binaries. An author who wraps text whose
+ * behavior differs in Cloud has one convention:
+ *
+ *   - `env-cloud` is the only attribute;
+ *   - block form only: `ifdef::env-cloud[]` or `ifndef::env-cloud[]` alone
+ *     on a line at column 0, the whole paragraph inside, `endif::[]` alone on
+ *     a line, a blank line before the opening directive and after the closing
+ *     one (unless the string starts or ends there);
+ *   - nothing after the closing bracket, not even whitespace: the Markdown
+ *     converter in the Connect binaries only treats a line ending in `]` as
+ *     a directive;
+ *   - never the single-line form `ifndef::env-cloud[text]`;
+ *   - never in a Summary, which renders as one line, or in an example
+ *     summary;
+ *   - never inside a code fence;
+ *   - never `ifndef::env-cloud[]` inside `ifdef::env-cloud[]` or the
+ *     reverse, whose inner text renders for neither audience.
+ *
+ * Links to pages the other site lacks are handled by the renderers, so a
+ * conditional is only for behavior that differs. Not part of
+ * VERBATIM_ASCIIDOC_RULES: a surface that publishes to both audiences opts
+ * in (connect does); the other verbatim surfaces carry audience conditionals
+ * as data, not in the string.
+ */
+const CONDITIONAL_ATTRIBUTE = 'env-cloud'
+const CONDITIONAL_WHY = 'The Cloud docs and the Cloud binaries set env-cloud, and the text inside must read as a whole paragraph in both outputs. See docs/README.md in the connect repo.'
+const CONDITIONAL_TOKEN = /\b(?:ifdef|ifndef|ifeval|endif)::/
+// A directive on its own line: optional indent (reported), kind, attribute
+// list, bracketed text, then optional trailing whitespace (reported).
+const CONDITIONAL_LINE = /^(\s*)(ifdef|ifndef|ifeval|endif)::([^\s[\]]*)\[(.*)\](\s*)$/
+// Listing and literal delimiters, and Markdown fences with an optional
+// language tag (```yaml). Tracked so a directive inside one is reported; the
+// preprocessor still evaluates it there.
+const CODE_FENCE = /^(?:(-{4,}|\.{4,})|(`{3,})([\w.+#-]*))[ \t]*$/
+
+/**
+ * Which summary a declaration is, or null. Summaries render as one line, and
+ * connect's docs test rejects a conditional in any JSON `summary` key, which
+ * includes example summaries.
+ */
+function summaryKind (decl) {
+  const meta = decl.meta || {}
+  if (meta.kind === 'summary' || (meta.kind === 'bloblang' && meta.part === 'summary')) return 'a Summary'
+  if (meta.kind === 'example-summary' || meta.kind === 'bloblang-example') return 'an example summary'
+  return null
+}
+
+/**
+ * Advance the code-fence state over one fence line. An AsciiDoc block closes
+ * only on a delimiter of the same character and length; a Markdown fence
+ * closes on a bare fence at least as long as the opener.
+ */
+function stepFence (fence, match) {
+  const [, adoc, ticks, lang] = match
+  if (fence === null) return adoc ? { char: adoc[0], length: adoc.length } : { char: '`', length: ticks.length }
+  if (adoc) return fence.char === adoc[0] && fence.length === adoc.length ? null : fence
+  return fence.char === '`' && !lang && ticks.length >= fence.length ? null : fence
+}
+
+const CONDITIONAL_RULE = {
+  name: 'conditional',
+  description: 'AsciiDoc conditional that is not a well-formed env-cloud block',
+  severity: 'warning',
+  check: (decl) => {
+    const text = decl.string || ''
+    if (!text || !CONDITIONAL_TOKEN.test(text)) return []
+    // CLI help is not AsciiDoc; the surface marks those declarations.
+    if (decl.convention && decl.convention.verbatim_asciidoc === false) return []
+    const issues = []
+    const say = (what, severity) => issues.push({ message: `${what} ${CONDITIONAL_WHY}`, ...(severity ? { severity } : {}) })
+
+    const summary = summaryKind(decl)
+    if (summary) {
+      say(`Contains an AsciiDoc conditional in ${summary}. A summary renders as one line, so it cannot hold a conditional block; keep the summary the same for both audiences and put the differing paragraph in the Description.`)
+      return issues
+    }
+
+    const lines = text.split('\n')
+    const open = []
+    let fence = null
+    for (let i = 0; i < lines.length; i++) {
+      const line = lines[i]
+      const fenceMatch = line.match(CODE_FENCE)
+      if (fenceMatch) {
+        fence = stepFence(fence, fenceMatch)
+        continue
+      }
+      const m = line.match(CONDITIONAL_LINE)
+      if (!m) {
+        if (CONDITIONAL_TOKEN.test(line)) {
+          say(`"${line.trim().slice(0, 60)}" is not a directive on a line of its own at column 0. Put ifdef::env-cloud[], ifndef::env-cloud[] and endif::[] each on their own line with nothing before them.`)
+        }
+        continue
+      }
+      const [, indent, kind, attribute, body, trailing] = m
+      const label = `${kind}::${attribute}[${body}]`
+      // What is between the brackets decides how Asciidoctor reads the line.
+      const bracket = body === '' ? 'empty' : body.trim() === '' ? 'whitespace' : 'text'
+      if (fence !== null) {
+        // Reported on its own: the form checks below assume prose around
+        // the block, and a fence line is not prose.
+        say(`${label} is inside a code fence. Keep conditionals out of code fences; wrap the whole paragraph or block instead.`)
+        continue
+      }
+      if (indent) {
+        // Asciidoctor and the Connect Markdown converter both read an
+        // indented directive as text, so it neither opens nor closes a block
+        // here either: the never-closed and no-matching-opener errors below
+        // then report the block it leaves unbalanced.
+        say(`${label} is indented. A directive starts at column 0, or AsciiDoc and the Connect binaries treat it as text and it does not open or close a block.`)
+        continue
+      }
+      if (trailing) {
+        say(`${label} has trailing whitespace after the closing bracket. The Connect binaries only treat a line ending in ] as a directive, so CLI readers would see the directive and the text for the other audience. Remove everything after the ].`, 'error')
+      }
+
+      if (kind === 'ifeval') {
+        say(`${label} uses ifeval. env-cloud is the only attribute a doc string may test, with ifdef::env-cloud[] or ifndef::env-cloud[].`)
+        open.push({ label, line: i })
+        continue
+      }
+      if (kind === 'endif') {
+        if (attribute && attribute !== CONDITIONAL_ATTRIBUTE) say(`${label} names "${attribute}". env-cloud is the only attribute; close the block with endif::[].`)
+        if (bracket !== 'empty') {
+          // Asciidoctor rejects an endif with anything in its brackets and
+          // leaves the block open, so it does not close one here either: the
+          // never-closed error below then reports the block it leaves open.
+          say(`${label} has ${bracket === 'text' ? 'text' : 'whitespace'} inside its brackets. Asciidoctor rejects it and leaves the block open, so the text after it is hidden from one audience. Write endif::[] with nothing between the brackets.`, 'error')
+        } else if (open.length === 0) say(`${label} has no matching ifdef::env-cloud[] or ifndef::env-cloud[] before it.`, 'error')
+        else open.pop()
+        if (i < lines.length - 1 && lines[i + 1].trim() !== '') say(`${label} needs a blank line after it, so the paragraph inside ends where the block ends.`)
+        continue
+      }
+      // ifdef / ifndef
+      if (attribute !== CONDITIONAL_ATTRIBUTE) say(`${label} tests "${attribute}". env-cloud is the only attribute a doc string may test.`)
+      if (bracket === 'whitespace') {
+        // Asciidoctor reads any bracket content, whitespace included, as the
+        // single-line form, which opens no block: the text that follows shows
+        // for both audiences. So it is not pushed as an opener, and the endif
+        // that follows reports no matching opener.
+        say(`${label} has whitespace inside its brackets. Asciidoctor then reads it as the single-line form, which opens no block, so the text after it shows for both audiences. Write ${kind}::${attribute}[] with nothing between the brackets.`, 'error')
+      } else if (bracket === 'text') {
+        say(`${label} is the single-line form. Use the block form: the directive alone on its line, the whole paragraph inside, then endif::[] on its own line.`)
+      } else {
+        if (attribute === CONDITIONAL_ATTRIBUTE) {
+          const opposite = kind === 'ifdef' ? 'ifndef' : 'ifdef'
+          const outer = open.find((o) => o.kind === opposite && o.attribute === CONDITIONAL_ATTRIBUTE)
+          if (outer) say(`${label} is inside ${outer.label}, so the text between them renders for neither audience. Close the outer block with endif::[] before opening the other one.`)
+        }
+        open.push({ label, kind, attribute, line: i })
+      }
+      if (i > 0 && lines[i - 1].trim() !== '') say(`${label} needs a blank line before it, so the paragraph inside starts where the block starts.`)
+    }
+    for (const { label } of open) {
+      say(`${label} is never closed with endif::[]. Everything after it is hidden from one audience, or the rest of the page is.`, 'error')
+    }
+    return issues
+  }
+}
+
+module.exports = { VERBATIM_ASCIIDOC_RULES, CONDITIONAL_RULE, KNOWN_ATTRIBUTES, MACRO_NAMES, countRawPipes }

package/tools/lint-strings/surfaces/connect.js

@@ -8,6 +8,7 @@ const { SourceCache } = require('../source-text')
 const { splitTopLevelArgs, findBalancedClose, collectGoFiles } = require('../go-source')
 const { GoIndex, statementEnd, straightLineReturn, stripStrings, renderResult, isResolved, unresolvedParts } = require('../go-index')
 const { parseLiteralFields } = require('./rpk')
+const { CONDITIONAL_RULE } = require('../rules/verbatim-asciidoc')
 
 /**
  * Connect surface: every published string a Redpanda Connect build defines.
@@ -18,6 +19,11 @@ const { parseLiteralFields } = require('./rpk')
  *   the option descriptions of annotated enum fields.
  * - Bloblang plugins (bloblang.PluginSpec): Description, Example summaries,
  *   and parameter descriptions.
+ * - benthos's internal field API (internal/docs): the description argument
+ *   of docs.FieldString, docs.FieldObject, docs.FieldBool and the other
+ *   docs.Field constructors, which document the shared config blocks
+ *   (http_server, tls, logger, batching, ...) that benthos builds outside
+ *   the public service API.
  * - `rpk connect` help (urfave/cli): command Usage, UsageText, ArgsUsage and
  *   Description, and flag Usage.
  *
@@ -87,6 +93,17 @@ const BARE_FIELD_CTORS = new Set([
 
 const SERVICE_PKG = 'github.com/redpanda-data/benthos/v4/public/service'
 const BLOBLANG_PKG = 'github.com/redpanda-data/benthos/v4/public/bloblang'
+const DOCS_PKG = 'github.com/redpanda-data/benthos/v4/internal/docs'
+
+// internal/docs constructors with the signature
+// (name, description string, examples ...any). docs.FieldComponent takes
+// no description and is left out.
+const DOCS_FIELD_CTORS = [
+  'FieldAnything', 'FieldObject', 'FieldString', 'FieldInterpolatedString', 'FieldBloblang',
+  'FieldInt', 'FieldFloat', 'FieldBool', 'FieldURL',
+  'FieldInput', 'FieldProcessor', 'FieldOutput', 'FieldBuffer', 'FieldCache',
+  'FieldRateLimit', 'FieldMetrics', 'FieldTracer', 'FieldScanner'
+]
 const CLI_PKG = /^github\.com\/urfave\/cli(?:\/v\d+)?$/
 
 // Repos whose urfave/cli commands are `rpk connect` help.
@@ -331,6 +348,7 @@ class ConnectScanner {
       file._aliases = {
         service: aliasesFor(file, SERVICE_PKG),
         bloblang: aliasesFor(file, BLOBLANG_PKG),
+        docs: aliasesFor(file, DOCS_PKG),
         cli: aliasesFor(file, CLI_PKG)
       }
     }
@@ -804,6 +822,7 @@ class ConnectScanner {
     const skipped = []
     const { service, bloblang, cli } = this.aliases(file)
     if (service.size > 0 || bloblang.size > 0) this.scanSpecs(file, rel, declarations, skipped)
+    this.scanDocsFields(file, rel, declarations)
     if (cli.size > 0 && this.cli) this.scanCli(file, rel, declarations)
     return { declarations, skipped }
   }
@@ -1055,6 +1074,53 @@ class ConnectScanner {
     this.scanJsFunctions(file, rel, declarations)
   }
 
+  /**
+   * benthos's internal field API: `docs.FieldString(name, description)`
+   * and the other internal/docs constructors. The description is the second
+   * argument, not a `.Description()` call, so scanSpecs never sees it.
+   * Inside internal/docs itself the constructors are unqualified. An empty
+   * description is skipped: internal/docs uses `FieldString("type", "")`
+   * for structural fields that no page documents.
+   */
+  scanDocsFields (file, rel, declarations) {
+    const { docs } = this.aliases(file)
+    const mod = this.index.gomod.module
+    const inPackage = Boolean(mod) && `${mod}/${this.index.rel(file.dir).split(path.sep).join('/')}` === DOCS_PKG
+    if (docs.size === 0 && !inPackage) return
+    const qual = docs.size > 0 ? `(?:${[...docs].map(escapeRe).join('|')})\\s*\\.\\s*` : ''
+    const prefix = inPackage ? (qual ? `(?:${qual})?` : '') : qual
+    const pattern = new RegExp(`(^|[^\\w.])${prefix}(${DOCS_FIELD_CTORS.join('|')})\\s*\\(`, 'g')
+    const masked = file.masked
+    // String contents blanked, so example YAML or prose that mentions a
+    // constructor never reads as a call.
+    const text = stripStrings(masked)
+    let m
+    while ((m = pattern.exec(text)) !== null) {
+      const start = m.index + m[1].length
+      if (/\bfunc\s*$/.test(text.slice(Math.max(0, start - 10), start))) continue
+      const open = m.index + m[0].length - 1
+      const close = file.pairs.get(open)
+      if (close === undefined) continue
+      const args = splitTopLevelArgs(masked.slice(open + 1, close))
+      if (args.length < 2) continue
+      const argStart = (i) => open + 1 + args.slice(0, i).reduce((n, a) => n + a.length + 1, 0)
+      const result = this.evalAt(file, args[1], argStart(1))
+      if (isResolved(result) && renderResult(result).trim() === '') continue
+      const nameResult = this.evalAt(file, args[0], argStart(0))
+      this.pushDecl(declarations, file, rel, {
+        kind: 'field',
+        name: isResolved(nameResult) ? renderResult(nameResult) : null,
+        nameResult,
+        nameExpr: { text: args[0], offset: argStart(0) },
+        result,
+        valueExpr: { text: args[1], offset: argStart(1) },
+        start,
+        end: close,
+        extra: { ctor: `docs.${m[2]}`, ...this.componentMeta(file, start) }
+      })
+    }
+  }
+
   /**
    * javascript processor functions: `registerVMRunnerFunction(name,
    * description).Param(name, type, what)`. The processor renders each one
@@ -1272,7 +1338,7 @@ class ConnectScanner {
           extra: {
             literal: literal === 'Command' ? 'command' : literal,
             // rpk-docs formats CLI help like cobra help; it is not AsciiDoc.
-            skip_rules: ['raw-pipe', 'unknown-attribute', 'broken-macro', 'too-short']
+            skip_rules: ['raw-pipe', 'unknown-attribute', 'broken-macro', 'conditional', 'too-short']
           }
         }))
       }
@@ -1460,8 +1526,6 @@ const MARKUP_PATTERNS = [
   [/\n/, 'a line break']
 ]
 
-const ENV_CONDITIONAL = /\b(?:ifdef|ifndef|ifeval|endif)::/
-
 // Modules of the Redpanda Connect docs component. A generated partial is
 // rendered on the self-managed page and included into Cloud pages, so a
 // link needs an explicit module to resolve in both places.
@@ -1491,15 +1555,11 @@ const RULES = [
       return [{ message: `ShortDescription contains ${found.join(', ')}. Form-based config editors, such as the one in Redpanda Cloud, show it as plain text, so markup renders literally. Write one plain sentence and keep markup in Description().` }]
     }
   },
-  {
-    name: 'connect-env-conditional',
-    description: 'AsciiDoc ifdef/ifndef conditional in a Go spec string',
-    severity: 'warning',
-    check: (decl) => {
-      if (!decl.string || !ENV_CONDITIONAL.test(decl.string)) return []
-      return [{ message: 'Contains an AsciiDoc conditional (ifdef::, ifndef:: or endif::). The docs generator owns every Cloud conditional in the partials, and a hand-written one nests inside them, so the Cloud copy shows or hides the wrong text. Fix the availability data in internal/plugins/info.csv instead, or ask the docs team to carry the sentence on the page.' }]
-    }
-  },
+  // Spec strings publish to the self-managed and the Cloud docs, and the
+  // Cloud binaries render them with env-cloud set, so an env-cloud block
+  // conditional is the supported way to vary a paragraph between the two.
+  // The rule checks the form; see rules/verbatim-asciidoc.js.
+  CONDITIONAL_RULE,
   {
     name: 'connect-xref-module',
     description: 'xref without a module, or to a module the Connect docs do not have',

package/tools/lint-strings/surfaces/crd.js

@@ -405,18 +405,22 @@ const RULES = [
   },
   {
     name: 'go-field-name-first',
-    description: 'Description leads with the Go field name instead of the YAML key',
+    description: 'Description opens with the Go field name or the YAML key instead of what the field does',
     severity: 'warning',
     check: (decl) => {
       const text = (decl.string || '').trim()
       if (!text) return []
       const firstWord = (text.match(/^[A-Za-z0-9_]+/) || [null])[0]
       if (!firstWord) return []
-      if (firstWord === decl.meta.go_name &&
-          decl.meta.go_name.toLowerCase() !== decl.meta.json_name.toLowerCase()) {
-        return [{ message: `Description starts with the Go field name "${decl.meta.go_name}", but users type "${decl.meta.json_name}" in YAML. Describe the json key: "${decl.meta.json_name}" (or start with what the field does).` }]
-      }
-      return []
+      // Case-sensitive on purpose: `RetentionTime is...` on
+      // json:"retentionTime" opens with the Go name, and `retentionTime
+      // is...` opens with the key. Neither says what the field does.
+      const { go_name: goName, json_name: jsonName } = decl.meta
+      let opener = null
+      if (firstWord === goName) opener = `the Go field name "${goName}"`
+      else if (firstWord === jsonName) opener = `the YAML key "${jsonName}"`
+      if (!opener) return []
+      return [{ message: `Description opens with ${opener}. Start with what the field does, for example "Specifies ...". Don't open with the Go field name "${goName}" or the YAML key "${jsonName}".` }]
     }
   }
 ]

package/tools/metrics/metrics.py

@@ -23,6 +23,16 @@ These output files are stored in a versioned folder under `docs/autogenerated/<v
 
 logging.basicConfig(level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s")
 
+
+def generated_marker(generator, source, hint=""):
+    """Same marker shape as cli-utils/generated-marker.js (no version, so a
+    release does not rewrite every file); convert-to-markdown reads it."""
+    marker = (
+        "// This content is autogenerated. Do not edit manually. "
+        f"Generated by doc-tools ({generator}) from {source.rstrip('.')}."
+    )
+    return marker + (f" {hint.strip()}" if hint.strip() else "")
+
 def fetch_metrics(url):
     """Fetch metrics from the given URL."""
     try:
@@ -123,6 +133,7 @@ def parse_metrics(metrics_text):
 def output_asciidoc(metrics, adoc_file):
     """Output metrics as AsciiDoc."""
     with open(adoc_file, "w") as f:
+        f.write(generated_marker("metrics-docs", "the metrics Redpanda exposes, defined in redpanda-data/streaming-enterprise") + "\n\n")
         for name, data in metrics.items():
             f.write(f"=== {name}\n\n")
             f.write(f"{data['description']}\n\n")

package/tools/property-extractor/generate-handlebars-docs.js

@@ -2,10 +2,18 @@
 // writer already has open. The previous pointer was a Confluence page, and
 // Confluence is retired, so it sent every writer who read this line to a dead
 // end. A repo-relative path cannot rot the same way.
-const AUTOGEN_NOTICE = '// This content is autogenerated. Do not edit manually. To change a description, add a link, or scope text to one docs build, edit docs-data/property-overrides.json (fields are documented in docs-data/property-overrides.schema.json and docs-data/README.adoc) and regenerate.\n';
 'use strict';
 
 const fs = require('fs');
+const { generatedMarker } = require('../../cli-utils/generated-marker');
+
+// The marker's shape is shared by every generator and read by the
+// convert-to-markdown extension, which exports it to the Markdown twins.
+const AUTOGEN_NOTICE = generatedMarker(
+  'property-docs',
+  'the configuration property definitions in redpanda-data/streaming-enterprise',
+  'To change a description, add a link, or scope text to one docs build, edit docs-data/property-overrides.json (fields are documented in docs-data/property-overrides.schema.json and docs-data/README.adoc) and regenerate.'
+) + '\n';
 const bigIntJson = require('../../cli-utils/big-int-json');
 const path = require('path');
 const handlebars = require('handlebars');

package/tools/property-extractor/helpers/applyPropertyLinks.js

@@ -419,6 +419,39 @@ function normalizeIncludes(propName, raw, warnings) {
   return out;
 }
 
+/**
+ * Remove the inline copies of `includes` that the attachment write-back appended.
+ *
+ * generate-handlebars-docs.js writes each include back into the description
+ * on disk, so the attachment's tooltip still renders it, and leaves `includes`
+ * in place. A run that reads that file again (a reused gen/<tag>-properties.json,
+ * or generate-docs run twice without re-extraction) sees the directive in the
+ * description and in `includes`, and the template rendered it twice. Only the
+ * exact shapes the write-back produces are
+ * removed, so an include a human put in the prose is left alone.
+ *
+ * @param {string|undefined} description - Flattened description.
+ * @param {Array<{target: string, cloud_only?: boolean, self_managed_only?: boolean}>} items - Normalized includes.
+ * @returns {string|undefined}
+ */
+function stripInlineIncludes(description, items) {
+  if (typeof description !== 'string' || !description.includes('include::')) return description;
+  let text = description;
+  // The write-back appends in order, so peel from the end.
+  for (const item of [...items].reverse()) {
+    const directive = `include::${item.target}`;
+    let block = directive;
+    if (item.cloud_only) block = `ifdef::env-cloud[]\n${directive}\nendif::[]`;
+    else if (item.self_managed_only) block = `ifndef::env-cloud[]\n${directive}\nendif::[]`;
+    if (text.endsWith(`\n\n${block}`)) {
+      text = text.slice(0, -(block.length + 2));
+    } else if (text === block) {
+      text = '';
+    }
+  }
+  return text;
+}
+
 /**
  * Why the Cloud build cannot reach a link target, or null when it can.
  *
@@ -471,6 +504,7 @@ function applyPropertyLinks(properties) {
     }
     if (prop.includes !== undefined) {
       prop.includes = normalizeIncludes(propName, prop.includes, warnings);
+      prop.description = stripInlineIncludes(prop.description, prop.includes);
     }
     if (Array.isArray(prop.admonitions)) {
       prop.admonitions = prop.admonitions.filter((item) => {
@@ -589,3 +623,4 @@ module.exports.applyLinksToText = applyLinksToText;
 module.exports.renderLink = renderLink;
 module.exports.cloudUnreachable = cloudUnreachable;
 module.exports.normalizeIncludes = normalizeIncludes;
+module.exports.stripInlineIncludes = stripInlineIncludes;