@redpanda-data/docs-extensions-and-macros 5.13.5 → 5.17.0

package/tools/overrides-audit/index.js

@@ -0,0 +1,185 @@
+#!/usr/bin/env node
+
+/**
+ * Overrides Audit CLI
+ *
+ * Field-level classification of docs-side override entries against
+ * extracted source strings, powering the override-retirement loop
+ * (retirement: delete REDUNDANT fields on each release regeneration;
+ * upstreaming: draft source PRs from UPSTREAMABLE/SPLIT candidates).
+ *
+ * Usage (standalone):
+ *   node tools/overrides-audit/index.js \
+ *     --overrides docs-data/property-overrides.json \
+ *     --extracted tools/property-extractor/gen/properties-output.json \
+ *     [--surface properties|rpk|connect] [--format json|human] [--output <file>]
+ *
+ * Also exposed as `doc-tools overrides audit` and the `audit_overrides`
+ * MCP tool. See README.adoc in this directory for the classification rules
+ * and the upstream_ref policy.
+ */
+
+'use strict'
+
+const fs = require('fs')
+const path = require('path')
+
+const ADAPTERS = {
+  properties: () => require('./adapters/properties'),
+  rpk: () => require('./adapters/rpk'),
+  connect: () => require('./adapters/connect')
+}
+
+/**
+ * Run the audit and return the result object.
+ *
+ * @param {Object} options - { overrides, extracted, surface, format, output }.
+ * @returns {Object} Audit result ({ surface, manifest, summary, ... }).
+ */
+function runAudit (options) {
+  const surface = options.surface || 'properties'
+  const adapterFactory = ADAPTERS[surface]
+  if (!adapterFactory) {
+    throw new Error(`Unknown surface '${surface}'. Valid surfaces: ${Object.keys(ADAPTERS).join(', ')}`)
+  }
+  if (!options.overrides) {
+    throw new Error('Missing required option --overrides <path>')
+  }
+  let extractedPath = options.extracted ? path.resolve(options.extracted) : undefined
+  if (surface === 'properties' && !extractedPath) {
+    if (!options.repo) {
+      throw new Error('Provide --extracted <path> (raw extractor JSON, no overrides applied) or --repo <path> (a redpanda checkout to extract from)')
+    }
+    // Extract raw source strings ourselves. Never audit against a published
+    // redpanda-properties-<tag>.json: those already have overrides applied,
+    // which classifies every description REDUNDANT.
+    const fs = require('fs')
+    const os = require('os')
+    const { runExtractor } = require('../lint-strings/surfaces/properties')
+    const json = runExtractor(path.resolve(options.repo), (msg) => console.error(msg))
+    extractedPath = path.join(fs.mkdtempSync(path.join(os.tmpdir(), 'overrides-audit-')), 'extracted.json')
+    fs.writeFileSync(extractedPath, JSON.stringify(json))
+  }
+
+  const adapter = adapterFactory()
+  return adapter.audit({
+    overridesPath: path.resolve(options.overrides),
+    extractedPath
+  })
+}
+
+/**
+ * Render the audit result as a human-readable report.
+ *
+ * @param {Object} result - Audit result from runAudit().
+ * @returns {string} Multi-line report.
+ */
+function formatHumanReport (result) {
+  const lines = []
+  const { summary } = result
+  lines.push('='.repeat(64))
+  lines.push(`Overrides audit — surface: ${result.surface}`)
+  lines.push(`Overrides: ${result.overrides_file}`)
+  lines.push(`Extracted: ${result.extracted_file || '(none)'}`)
+  lines.push('='.repeat(64))
+  lines.push('')
+  lines.push(`Classified ${summary.total} override field(s):`)
+  const order = [
+    'REDUNDANT',
+    'UPSTREAMABLE',
+    'KEEP_UNTIL_UPSTREAMED',
+    'UPSTREAMABLE_SLOT',
+    'REDUNDANT_OR_UPSTREAMABLE',
+    'KEEP',
+    'REVIEW'
+  ]
+  for (const cls of order) {
+    if (summary.byClass[cls]) {
+      lines.push(`  ${cls.padEnd(26)} ${summary.byClass[cls]}`)
+    }
+  }
+  lines.push('')
+  lines.push('Per field:')
+  for (const [field, byClass] of Object.entries(summary.byField)) {
+    const parts = Object.entries(byClass).map(([cls, count]) => `${cls}: ${count}`)
+    lines.push(`  ${field.padEnd(20)} ${parts.join(', ')}`)
+  }
+
+  const sections = [
+    ['REDUNDANT', 'Retire in the next auto-docs PR (source already matches)'],
+    ['UPSTREAMABLE', 'Send upstream verbatim'],
+    ['UPSTREAMABLE_SLOT', 'Migrate to a source metadata slot'],
+    ['REDUNDANT_OR_UPSTREAMABLE', 'Needs a human ruling'],
+    ['REVIEW', 'Needs attention (possible source bug, typo, or missing property)']
+  ]
+  for (const [cls, heading] of sections) {
+    const rows = result.manifest.filter((row) => row.class === cls)
+    if (rows.length === 0) continue
+    lines.push('')
+    lines.push(`${cls} — ${heading} (${rows.length}):`)
+    for (const row of rows) {
+      lines.push(`  • ${row.name} [${row.field}]${row.note ? ` — ${row.note}` : ''}`)
+    }
+  }
+
+  if (result.cross_check) {
+    lines.push('')
+    lines.push('Cross-check (compare-properties.js):')
+    lines.push(`  descriptions still masked by overrides: ${result.cross_check.changedDescriptions}`)
+    lines.push(`  empty source descriptions: ${result.cross_check.emptyDescriptions}`)
+    if (result.cross_check.violations.length > 0) {
+      lines.push(`  CLASSIFIER BUG — raw-equal but not REDUNDANT: ${result.cross_check.violations.join(', ')}`)
+    } else {
+      lines.push('  consistency: OK (every raw-equal description classified REDUNDANT)')
+    }
+  }
+
+  lines.push('')
+  lines.push('='.repeat(64))
+  return lines.join('\n')
+}
+
+/**
+ * CLI entry point.
+ *
+ * @param {string[]} argv - process.argv.
+ */
+function main (argv) {
+  const { Command } = require('commander')
+  const program = new Command()
+
+  program
+    .name('overrides-audit')
+    .description('Classify docs-side override fields against extracted source strings')
+    .requiredOption('--overrides <path>', 'Path to the overrides JSON file (for example docs-data/property-overrides.json)')
+    .option('--extracted <path>', 'Path to the extracted source JSON (property extractor raw output; required for the properties surface)')
+    .option('--surface <surface>', 'Override surface: properties, rpk, or connect', 'properties')
+    .option('--format <format>', 'Output format: json or human', 'json')
+    .option('--output <path>', 'Also write the JSON result to this file')
+    .action((options) => {
+      try {
+        const result = runAudit(options)
+        if (options.output) {
+          fs.mkdirSync(path.dirname(path.resolve(options.output)), { recursive: true })
+          fs.writeFileSync(path.resolve(options.output), JSON.stringify(result, null, 2) + '\n')
+          console.error(`JSON result written to ${options.output}`)
+        }
+        if (options.format === 'human') {
+          console.log(formatHumanReport(result))
+        } else {
+          console.log(JSON.stringify(result, null, 2))
+        }
+      } catch (err) {
+        console.error(`Error: ${err.message}`)
+        process.exit(1)
+      }
+    })
+
+  program.parse(argv)
+}
+
+if (require.main === module) {
+  main(process.argv)
+}
+
+module.exports = { runAudit, formatHumanReport }

package/tools/preview-string/index.js

@@ -0,0 +1,245 @@
+'use strict'
+
+const fs = require('fs')
+const path = require('path')
+
+/**
+ * doc-tools preview-string: render ONE doc-string declaration from local
+ * engineering source to the final published snippet, so an engineer can see
+ * what their string becomes on docs.redpanda.com BEFORE it ships.
+ *
+ * - properties: runs the real Python extractor and the real Handlebars
+ *   property template. With --overrides, renders a second "as shipped"
+ *   pane and prints a MASKED-BY-OVERRIDE notice when a docs-repo override
+ *   would replace the source string.
+ * - rpk: runs the string through the real formatDescription() transformer
+ *   from tools/rpk-docs (the killer demo: an ALLCAPS line becomes a "==="
+ *   heading before your eyes).
+ * - metrics/helm/crd/connect: renders the description in the surface's
+ *   output shape with a one-line header.
+ *
+ * Extraction reuses the lint-strings surface modules, so preview and lint
+ * always agree on what the source declares.
+ */
+
+const HR = '-'.repeat(72)
+
+function pane (title, body) {
+  return `${HR}\n${title}\n${HR}\n${body}\n`
+}
+
+/**
+ * Render a preview for one declaration.
+ *
+ * @param {Object} options
+ * @param {string} options.repo - Path to the engineering checkout
+ * @param {string} options.surface - properties|rpk|metrics|helm|crd|connect
+ * @param {string} options.name - Declaration name (property name, rpk
+ *   command or --flag, metric name, helm key path, crd json field, connect
+ *   component/field name)
+ * @param {string} [options.overrides] - Overrides JSON path (properties only)
+ * @param {Function} [options.log]
+ * @returns {string} The rendered preview text
+ */
+function previewString (options) {
+  const { repo, surface, name, overrides = null, log = (msg) => process.stderr.write(`${msg}\n`) } = options
+  if (!repo) throw new Error('preview-string requires --repo <path>')
+  if (!surface) throw new Error('preview-string requires --surface <name>')
+  if (!name) throw new Error('preview-string requires --name <name>')
+  const repoPath = path.resolve(repo)
+  if (!fs.existsSync(repoPath)) throw new Error(`Repo path does not exist: ${repoPath}`)
+
+  if (overrides && surface !== 'properties') {
+    log(`Note: --overrides applies to the properties surface only; ignoring it for ${surface}.`)
+  }
+
+  switch (surface) {
+    case 'properties': return previewProperty(repoPath, name, overrides, log)
+    case 'rpk': return previewRpk(repoPath, name, log)
+    case 'metrics':
+    case 'helm':
+    case 'crd':
+    case 'connect': return previewLintSurface(surface, repoPath, name, log)
+    default: throw new Error(`Unknown surface "${surface}". Supported: properties, rpk, metrics, helm, crd, connect`)
+  }
+}
+
+/* -------------------------------- properties ---------------------------- */
+
+function previewProperty (repo, name, overridesPath, log) {
+  const properties = require('../lint-strings/surfaces/properties')
+  const handlebars = require('handlebars')
+  // Requiring the generator registers its Handlebars helpers (eq, ...) as a
+  // side effect - the same environment a real generation run uses.
+  require('../property-extractor/generate-handlebars-docs')
+
+  const json = properties.runExtractor(repo, log)
+  const prop = (json.properties || {})[name]
+  if (!prop) {
+    const near = Object.keys(json.properties || {}).filter((k) => k.includes(name)).slice(0, 8)
+    throw new Error(`Property "${name}" not found in ${repo}.${near.length > 0 ? ` Close matches: ${near.join(', ')}` : ''}`)
+  }
+
+  const templatePath = path.join(properties.TOOL_ROOT, 'templates', 'property.hbs')
+  const template = handlebars.compile(fs.readFileSync(templatePath, 'utf8'))
+  const asSource = template({ ...prop, name }).trim()
+
+  const spanNote = prop.line_start != null
+    ? `${prop.defined_in || ''}:${prop.line_start}-${prop.line_end != null ? prop.line_end : prop.line_start}`
+    : (prop.defined_in || '(unknown source location)')
+  let out = `Property: ${name}  (${spanNote})\n\n`
+  out += pane('AS SOURCE (rendered from your checkout)', asSource)
+
+  if (overridesPath) {
+    const raw = JSON.parse(fs.readFileSync(path.resolve(overridesPath), 'utf8'))
+    // Accept both { properties: { name: {...} } } (docs-repo
+    // property-overrides.json) and a flat { name: {...} } map.
+    const entry = (raw.properties && raw.properties[name]) || (!raw.properties && raw[name]) || null
+    if (!entry) {
+      out += `\nNo override entry for "${name}" in ${overridesPath}: the source string ships as-is.\n`
+    } else {
+      const shipped = { ...prop, ...entry, name }
+      const asShipped = template(shipped).trim()
+      out += '\n' + pane('AS SHIPPED (override applied)', asShipped)
+      if (asShipped !== asSource) {
+        const maskedFields = Object.keys(entry).filter((k) => JSON.stringify(prop[k]) !== JSON.stringify(entry[k]))
+        out += `\nMASKED-BY-OVERRIDE: the docs repo overrides ${maskedFields.map((f) => `"${f}"`).join(', ') || 'this entry'} for ${name}.\n` +
+          'Changes to the source string will NOT reach docs.redpanda.com until the override field is retired (fix the source string upstream, then delete the override field).\n'
+      } else {
+        out += '\nOverride entry present but the rendered output is identical: the override is redundant and can be retired.\n'
+      }
+    }
+  }
+  return out
+}
+
+/* ----------------------------------- rpk -------------------------------- */
+
+/**
+ * Mirror the real generate-rpk-docs.js pipeline for one string:
+ * - Long: parseDescriptionSections first (ALLCAPS lines become "===" section
+ *   headings, exactly as on the generated command page), then
+ *   formatDescription + ensurePeriod on the main description and
+ *   formatDescription on each section body.
+ * - Short: ensurePeriod(capToTwoSentences(formatDescription(...))) - the
+ *   shortDesc path.
+ * - Flag usage: ensurePeriod(formatDescription(...)) - the flag table path.
+ */
+function publishRpkString (kind, text, helpers) {
+  const { formatDescription, parseDescriptionSections, ensurePeriod, capToTwoSentences } = helpers
+  if (kind === 'long') {
+    const { mainDescription, sections } = parseDescriptionSections(text)
+    let out = ensurePeriod(formatDescription(mainDescription))
+    for (const [section, body] of Object.entries(sections)) {
+      const title = section.charAt(0).toUpperCase() + section.slice(1).toLowerCase()
+      out += `\n\n=== ${title}\n\n${formatDescription(body)}`
+    }
+    return out.trim()
+  }
+  if (kind === 'short') {
+    return ensurePeriod(capToTwoSentences(formatDescription(text, null, { skipTableConversion: true, skipListConversion: true }))).trim()
+  }
+  return ensurePeriod(formatDescription(text)).trim()
+}
+
+function previewRpk (repo, name, log) {
+  const rpk = require('../lint-strings/surfaces/rpk')
+  const helpers = require('../rpk-docs/generate-rpk-docs')
+
+  log(`Scanning rpk sources in ${repo}...`)
+  const decls = rpk.extract({ repo, log })
+  const isFlag = name.startsWith('-')
+  const target = isFlag ? name.replace(/^-+/, '') : name
+  const matches = decls.filter((d) => isFlag
+    ? (d.meta.kind === 'flag' && d.name === target)
+    : (d.meta.kind !== 'flag' && d.name === target))
+  if (matches.length === 0) {
+    throw new Error(`No rpk ${isFlag ? 'flag' : 'command'} named "${target}" found under src/go/rpk/pkg/cli. Use the leaf command token (for example "health") or --flag-name.`)
+  }
+
+  let out = ''
+  for (const d of matches.slice(0, 10)) {
+    const label = d.meta.kind === 'flag' ? `--${d.name} (flag usage)` : `${d.name} (${d.meta.kind})`
+    out += `rpk ${label}  ${d.file}:${d.line_start}-${d.line_end}\n\n`
+    if (d.string == null) {
+      out += 'Source string is built dynamically (not a plain literal); preview unavailable.\n\n'
+      continue
+    }
+    out += pane('AS SOURCE', d.string)
+    out += '\n' + pane('AS PUBLISHED (after formatDescription)', publishRpkString(d.meta.kind, d.string, helpers)) + '\n'
+  }
+  if (matches.length > 10) out += `(${matches.length - 10} more matches not shown)\n`
+  return out
+}
+
+/* -------------------- metrics / helm / crd / connect --------------------- */
+
+const SHAPES = {
+  // How each surface's description lands on the published page.
+  metrics: (d) => `=== ${d.name}\n\n${d.string}`,
+  helm: (d) => `=== ${d.name}\n\n${d.string}${d.meta.default_annotation ? `\n\n*Default:* ${d.meta.default_annotation}` : ''}`,
+  crd: (d) => `| *\`${d.name}\`* +\n(table cell in the CRD reference and \`kubectl explain\` output)\n| ${d.string}`,
+  connect: (d) => d.string // connect descriptions ARE the AsciiDoc page body
+}
+
+function previewLintSurface (surfaceName, repo, name, log) {
+  const { SURFACES } = require('../lint-strings')
+  const surface = SURFACES[surfaceName]
+  log(`Scanning ${surfaceName} sources in ${repo}...`)
+  const decls = surface.extract({ repo, log })
+  const matches = decls.filter((d) => d.name === name ||
+    (surfaceName === 'crd' && `${d.meta.struct}.${d.name}` === name))
+  if (matches.length === 0) {
+    throw new Error(`No ${surfaceName} declaration named "${name}" found in ${repo}.`)
+  }
+
+  let out = ''
+  for (const d of matches.slice(0, 10)) {
+    out += `${surfaceName}: ${d.name}${d.meta.struct ? ` (in ${d.meta.struct})` : ''}  ${d.file}:${d.line_start}-${d.line_end}\n\n`
+    if (d.string == null) {
+      out += 'No description in source; this ships blank.\n\n'
+      continue
+    }
+    out += pane('AS PUBLISHED', SHAPES[surfaceName](d)) + '\n'
+  }
+  if (matches.length > 10) out += `(${matches.length - 10} more matches not shown)\n`
+  return out
+}
+
+/* ----------------------------------- CLI --------------------------------- */
+
+function runCli (options) {
+  try {
+    process.stdout.write(previewString({
+      repo: options.repo,
+      surface: options.surface,
+      name: options.name,
+      overrides: options.overrides || null
+    }))
+  } catch (err) {
+    console.error(`Error: ${err.message}`)
+    process.exit(2)
+  }
+  process.exit(0)
+}
+
+module.exports = { previewString, publishRpkString, runCli }
+
+// Direct usage: node tools/preview-string --repo <path> --surface <s> --name <n> [--overrides <path>]
+if (require.main === module) {
+  const args = process.argv.slice(2)
+  const options = {}
+  for (let i = 0; i < args.length; i++) {
+    const arg = args[i]
+    if (arg === '--repo') options.repo = args[++i]
+    else if (arg === '--surface') options.surface = args[++i]
+    else if (arg === '--name') options.name = args[++i]
+    else if (arg === '--overrides') options.overrides = args[++i]
+    else {
+      console.error(`Unknown argument: ${arg}`)
+      console.error('Usage: node tools/preview-string --repo <path> --surface <s> --name <name> [--overrides <path>]')
+      process.exit(2)
+    }
+  }
+  runCli(options)
+}

package/tools/property-extractor/Makefile

@@ -20,6 +20,11 @@ TOOL_ROOT     := $(MODULE_ROOT)/tools/property-extractor
 TMP_ROOT      := $(TOOL_ROOT)/tmp
 REDPANDA_SRC  := $(TMP_ROOT)/redpanda
 TREESITTER_DIR:= $(TOOL_ROOT)/tree-sitter/tree-sitter-cpp
+# One pin, read from the module that also bootstraps this grammar at run time
+# (tools/lint-strings/surfaces/properties.js ensureTreesitter). Holding the tag
+# in both places meant a bump here would silently leave the linter checking out
+# a different grammar than the generator.
+TREESITTER_TAG:= $(shell node -p "require('$(MODULE_ROOT)/tools/lint-strings/surfaces/properties.js').TREESITTER_TAG")
 VENV          := $(TOOL_ROOT)/tmp/redpanda-property-extractor-venv
 PYTHON        := $(VENV)/bin/python
 TREE_SITTER   := npx tree-sitter
@@ -131,13 +136,13 @@ treesitter:
 	  git clone https://github.com/tree-sitter/tree-sitter-cpp.git "$(TREESITTER_DIR)"; \
 	fi
 	@CURRENT_TAG=$$(git -C "$(TREESITTER_DIR)" describe --tags --exact-match 2>/dev/null || true); \
-	if [ "$$CURRENT_TAG" = "v0.20.5" ] && [ -f "$(TREESITTER_DIR)/src/parser.c" ]; then \
-	  echo "✅ tree-sitter-cpp parser already generated at v0.20.5; skipping regeneration"; \
+	if [ "$$CURRENT_TAG" = "$(TREESITTER_TAG)" ] && [ -f "$(TREESITTER_DIR)/src/parser.c" ]; then \
+	  echo "✅ tree-sitter-cpp parser already generated at $(TREESITTER_TAG); skipping regeneration"; \
 	else \
-	  echo "🔄 Ensuring tree-sitter-cpp is at compatible version v0.20.5…"; \
+	  echo "🔄 Ensuring tree-sitter-cpp is at compatible version $(TREESITTER_TAG)…"; \
 	  cd "$(TREESITTER_DIR)" && \
 	    git fetch --tags -q && \
-	    git checkout -q v0.20.5 && \
+	    git checkout -q $(TREESITTER_TAG) && \
 	    echo "🔧 Generating parser in $(TREESITTER_DIR)…" && \
 	    export CFLAGS="-Wno-unused-but-set-variable" && \
 	    npm install --silent && \

package/tools/property-extractor/parser.py

@@ -362,8 +362,29 @@ def parse_cpp_source(treesitter_parser, cpp_language, source_code):
     parameters = PropertyBag()
     current_field = None
     seen_first_argument = False
+    # Line spans (1-indexed, inclusive) of each full member-initializer entry
+    # (`name(arg, arg, ...)`), keyed by field name. Recorded so downstream
+    # consumers (for example, doc-string linting and PR suggestion blocks) can
+    # anchor a property to its exact source lines. Additive: existing
+    # consumers of the output JSON are unaffected.
+    declaration_spans = {}
 
     for node, label in captures:
+        # The SOURCE_QUERY also captures the whole field_initializer node as
+        # @field. Use it to record the declaration's line span; the entry is
+        # keyed by its field_identifier child so this works regardless of the
+        # order tree-sitter returns same-start captures in.
+        if label == "field" and node.type == "field_initializer":
+            name_node = next(
+                (c for c in node.children if c.type == "field_identifier"), None
+            )
+            if name_node is not None:
+                declaration_spans[name_node.text.decode("utf-8")] = (
+                    node.start_point[0] + 1,
+                    node.end_point[0] + 1,
+                )
+            continue
+
         # Start of a new field initializer
         if label == "field" and node.type == "field_identifier":
             current_field = node.text.decode("utf-8")
@@ -391,6 +412,11 @@ def parse_cpp_source(treesitter_parser, cpp_language, source_code):
             if normalized_param:
                 parameters[current_field]["params"].append(normalized_param)
 
+    for field_name, (line_start, line_end) in declaration_spans.items():
+        if field_name in parameters:
+            parameters[field_name]["line_start"] = line_start
+            parameters[field_name]["line_end"] = line_end
+
     return parameters
 
 

package/tools/property-extractor/property_extractor.py

@@ -963,6 +963,44 @@ def clean_private_fields_from_definitions(definitions):
 
 
 # The definitions.json file contains type definitions that the extractor uses to standardize and centralize type information. After extracting and transforming the properties from the source code, the function merge_properties_and_definitions looks up each property's type in the definitions. If a property's type (or the type of its items, in the case of arrays) matches one of the definitions, the transformer replaces that type with a JSON pointer ( such as #/definitions/<type>) to the corresponding entry in definitions.json. The final JSON output then includes both a properties section (with types now referencing the definitions) and a definitions section, so that consumers of the output can easily resolve the full type information.
+# Fields recorded for tooling only. lint-strings and preview-string read them
+# from the raw --output to anchor a finding to its declaration; nothing on the
+# docs site does. The enhanced JSON is copied verbatim into
+# modules/reference/attachments and downloaded by every browser that hovers a
+# prop: macro, so ~40 bytes per property (~56KB across the real ~1400-property
+# file) would be paid by readers for data no reader uses.
+TOOLING_ONLY_PROPERTY_FIELDS = ("line_start", "line_end")
+
+
+def strip_tooling_only_fields(properties):
+    """
+    Remove tooling-only span fields from properties bound for the PUBLISHED
+    JSON attachment.
+
+    Only the enhanced output is published; the raw --output keeps the spans,
+    because that is the file lint-strings and preview-string consume.
+
+    Args:
+        properties: Dictionary of properties
+
+    Returns:
+        The same dictionary, with the tooling-only fields removed
+    """
+    removed = 0
+    for prop in properties.values():
+        if not isinstance(prop, dict):
+            continue
+        for field in TOOLING_ONLY_PROPERTY_FIELDS:
+            if field in prop:
+                del prop[field]
+                removed += 1
+    if removed:
+        logging.info(
+            f"Stripped {removed} tooling-only span field(s) from the published properties JSON"
+        )
+    return properties
+
+
 def merge_properties_and_definitions(properties, definitions):
     # Do not overwrite the resolved type/default with a reference. Just return the resolved properties and definitions.
     return dict(properties=properties, definitions=definitions)
@@ -3143,6 +3181,9 @@ def main():
     # 7. Clean private fields from definitions (keep JSON output clean)
     filtered_enhanced_definitions = clean_private_fields_from_definitions(filtered_enhanced_definitions)
 
+    # 8. Drop tooling-only spans: this JSON is the published attachment.
+    enhanced_properties = strip_tooling_only_fields(enhanced_properties)
+
     # Generate enhanced properties JSON (with overrides)
     enhanced_properties_and_definitions = merge_properties_and_definitions(
         enhanced_properties, filtered_enhanced_definitions

package/tools/property-extractor/transformers.py

@@ -655,8 +655,10 @@ class BasicInfoTransformer:
     EXPECTED FINAL RESULT:
     {
         "name": "property_name",           # Clean identifier for JSON keys
-        "description": "Human readable...", # Description for documentation  
-        "defined_in": "src/v/config/..."   # Normalized file path
+        "description": "Human readable...", # Description for documentation
+        "defined_in": "src/v/config/...",  # Normalized file path
+        "line_start": 1852,                # First line of the declaration (1-indexed)
+        "line_end": 1858                   # Last line of the declaration (inclusive)
     }
     
     DOWNSTREAM USAGE:
@@ -724,6 +726,17 @@ class BasicInfoTransformer:
         property["defined_in"] = re.sub(
             r"^.*src/", "src/", str(file_pair.implementation)
         )
+
+        # --- Step 4: line span (additive) ---
+        # 1-indexed, inclusive lines of the full member-initializer entry in
+        # the implementation file named by defined_in. Recorded by the parser
+        # (parse_cpp_source) from the tree-sitter field_initializer node.
+        # Spans the whole declaration, not just the description literal, so
+        # diff-anchored tooling can match clang-format-wrapped strings.
+        if info.get("line_start") is not None:
+            property["line_start"] = info["line_start"]
+            property["line_end"] = info.get("line_end", info["line_start"])
+
         return property
 
 @debug_transformer

package/tools/rpk-docs/detect-platform-commands.js

@@ -47,6 +47,7 @@
 
 const fs = require('fs')
 const path = require('path')
+const { maskComments } = require('../lint-strings/go-source')
 
 /** GOOS values recognized in filename suffixes and build tags */
 const KNOWN_GOOS = new Set([
@@ -248,7 +249,13 @@ function commandNameFromConstructor(funcName) {
  * @param {string} content - Go file content
  * @returns {Object<string, {useName: string|null, excluded: boolean}>}
  */
-function parseConstructors(content) {
+function parseConstructors(rawContent) {
+  // Mask comments first. Scanning raw text read a `Use: "..."` out of a
+  // commented-out or example command and reported it as the real one; the
+  // masker blanks comment bodies while preserving offsets, line breaks and
+  // every string literal, so the regexes below see only live code. Shared with
+  // the lint-strings Go surfaces rather than reimplemented here.
+  const content = maskComments(rawContent)
   const constructors = {}
   const funcRe = /^func\s+(\w+)\s*\([^)]*\)\s*\*cobra\.Command\s*\{/gm
   const matches = []

package/tools/rpk-docs/scan-deprecated-commands.js

@@ -7,6 +7,7 @@
 
 const fs = require('fs');
 const path = require('path');
+const { maskComments, findBalancedClose } = require('../lint-strings/go-source');
 
 /**
  * Scan a Go file for cobra command definitions with deprecation/hidden status
@@ -17,31 +18,27 @@ function scanGoFile(filePath) {
   const content = fs.readFileSync(filePath, 'utf8');
 
   // Look for NewCommand function that returns *cobra.Command
-  const newCommandMatch = content.match(/func NewCommand\([^)]*\)\s*\*cobra\.Command\s*\{/);
+  // Mask comments before any scanning. The hand-rolled brace counter this
+  // replaced was comment- and string-blind, so a `{` inside a comment
+  // unbalanced it and truncated the cobra.Command block early, and the field
+  // regexes then read a `Use:` string out of a commented-out example. Both
+  // helpers are string-aware and already shared by the lint-strings Go
+  // surfaces; the masker preserves offsets and every string literal.
+  const masked = maskComments(content);
+
+  const newCommandMatch = masked.match(/func NewCommand\([^)]*\)\s*\*cobra\.Command\s*\{/);
   if (!newCommandMatch) {
     return null;
   }
 
   // Extract the cobra.Command struct definition
-  // Handle nested braces and multiline strings
-  let cmdBlock = '';
-  let braceCount = 0;
-  let startIndex = content.indexOf('&cobra.Command{');
-
-  if (startIndex === -1) {
+  const structIndex = masked.indexOf('&cobra.Command{');
+  if (structIndex === -1) {
     return null;
   }
-
-  startIndex += '&cobra.Command{'.length;
-  for (let i = startIndex; i < content.length; i++) {
-    const char = content[i];
-    if (char === '{') braceCount++;
-    if (char === '}') {
-      if (braceCount === 0) break;
-      braceCount--;
-    }
-    cmdBlock += char;
-  }
+  const openIndex = structIndex + '&cobra.Command'.length;
+  const closeIndex = findBalancedClose(masked, openIndex, '{', '}');
+  const cmdBlock = masked.slice(openIndex + 1, closeIndex === -1 ? masked.length : closeIndex);
 
   // Extract fields - handle multiline strings with backticks or quotes
   const useMatch = cmdBlock.match(/Use:\s*"([^"]+)"/);