@redpanda-data/docs-extensions-and-macros 5.14.0 → 5.22.0

package/tools/property-extractor/property_extractor.py

@@ -963,6 +963,44 @@ def clean_private_fields_from_definitions(definitions):
 
 
 # The definitions.json file contains type definitions that the extractor uses to standardize and centralize type information. After extracting and transforming the properties from the source code, the function merge_properties_and_definitions looks up each property's type in the definitions. If a property's type (or the type of its items, in the case of arrays) matches one of the definitions, the transformer replaces that type with a JSON pointer ( such as #/definitions/<type>) to the corresponding entry in definitions.json. The final JSON output then includes both a properties section (with types now referencing the definitions) and a definitions section, so that consumers of the output can easily resolve the full type information.
+# Fields recorded for tooling only. lint-strings and preview-string read them
+# from the raw --output to anchor a finding to its declaration; nothing on the
+# docs site does. The enhanced JSON is copied verbatim into
+# modules/reference/attachments and downloaded by every browser that hovers a
+# prop: macro, so ~40 bytes per property (~56KB across the real ~1400-property
+# file) would be paid by readers for data no reader uses.
+TOOLING_ONLY_PROPERTY_FIELDS = ("line_start", "line_end")
+
+
+def strip_tooling_only_fields(properties):
+    """
+    Remove tooling-only span fields from properties bound for the PUBLISHED
+    JSON attachment.
+
+    Only the enhanced output is published; the raw --output keeps the spans,
+    because that is the file lint-strings and preview-string consume.
+
+    Args:
+        properties: Dictionary of properties
+
+    Returns:
+        The same dictionary, with the tooling-only fields removed
+    """
+    removed = 0
+    for prop in properties.values():
+        if not isinstance(prop, dict):
+            continue
+        for field in TOOLING_ONLY_PROPERTY_FIELDS:
+            if field in prop:
+                del prop[field]
+                removed += 1
+    if removed:
+        logging.info(
+            f"Stripped {removed} tooling-only span field(s) from the published properties JSON"
+        )
+    return properties
+
+
 def merge_properties_and_definitions(properties, definitions):
     # Do not overwrite the resolved type/default with a reference. Just return the resolved properties and definitions.
     return dict(properties=properties, definitions=definitions)
@@ -1003,7 +1041,7 @@ def apply_property_overrides(properties, overrides, overrides_file_path=None):
     - Otherwise, searches existing properties for an entry whose `"name"` equals the override key and applies the override if found.
     - If no matching property is found, creates a new property from the override and adds it under the override key.
     
-    The function supports overrides that add or replace description, version, example content, default, type, config_scope, related_topics, and other metadata. When examples reference external files, relative paths are resolved relative to overrides_file_path.
+    The function supports overrides that add or replace description, version, example content, default, type, config_scope, see_also (related_topics is the deprecated predecessor, still read for existing overrides), and other metadata. When examples reference external files, relative paths are resolved relative to overrides_file_path.
     
     Parameters:
         properties (dict): Mapping of existing property entries (modified in-place).
@@ -1106,13 +1144,24 @@ def _apply_override_to_existing_property(property_dict, override, overrides_file
     if "config_scope" in override:
         property_dict["config_scope"] = override["config_scope"]
     
-    # Apply related_topics override
+    # Apply related_topics override (deprecated, see see_also)
     if "related_topics" in override:
         if isinstance(override["related_topics"], list):
             property_dict["related_topics"] = override["related_topics"]
         else:
             logger.warning(f"related_topics for property must be an array")
 
+    # Apply see_also override. Structural shape (plain string, or an object
+    # naming exactly one of cloud_only/self_hosted_only) is enforced by
+    # docs-data/property-overrides.schema.json via `doc-tools validate
+    # property-overrides`, not here — this just passes the data through for
+    # seeAlsoView.js to normalize at render time.
+    if "see_also" in override:
+        if isinstance(override["see_also"], list):
+            property_dict["see_also"] = override["see_also"]
+        else:
+            logger.warning(f"see_also for property must be an array")
+
     # Apply exclude_from_docs override
     if "exclude_from_docs" in override:
         property_dict["exclude_from_docs"] = override["exclude_from_docs"]
@@ -1154,19 +1203,26 @@ def _create_property_from_override(prop_name, override, overrides_file_path):
     if example_content:
         new_property["example"] = example_content
     
-    # Add related_topics if specified
+    # Add related_topics if specified (deprecated, see see_also)
     if "related_topics" in override:
         if isinstance(override["related_topics"], list):
             new_property["related_topics"] = override["related_topics"]
         else:
             logger.warning(f"related_topics for property '{prop_name}' must be an array")
-    
+
+    # Add see_also if specified
+    if "see_also" in override:
+        if isinstance(override["see_also"], list):
+            new_property["see_also"] = override["see_also"]
+        else:
+            logger.warning(f"see_also for property '{prop_name}' must be an array")
+
     # Add any other custom fields from override
     for key, value in override.items():
         if key not in ["description", "type", "default", "config_scope", "version",
                        "example", "example_file", "example_yaml", "related_topics",
-                       "is_deprecated", "visibility", "exclude_from_docs", "category",
-                       "accepted_values", "_comment"]:
+                       "see_also", "is_deprecated", "visibility", "exclude_from_docs",
+                       "category", "accepted_values", "_comment"]:
             new_property[key] = value
 
     # Add exclude_from_docs if specified
@@ -3143,6 +3199,9 @@ def main():
     # 7. Clean private fields from definitions (keep JSON output clean)
     filtered_enhanced_definitions = clean_private_fields_from_definitions(filtered_enhanced_definitions)
 
+    # 8. Drop tooling-only spans: this JSON is the published attachment.
+    enhanced_properties = strip_tooling_only_fields(enhanced_properties)
+
     # Generate enhanced properties JSON (with overrides)
     enhanced_properties_and_definitions = merge_properties_and_definitions(
         enhanced_properties, filtered_enhanced_definitions

package/tools/property-extractor/templates/property.hbs

@@ -167,78 +167,66 @@ endif::[]
 |
 {{{example}}}
 {{/if}}
-{{#if related_topics}}
-{{#with (allTopicsConditional related_topics) as |sectionType|}}
+{{#with (seeAlsoView .) as |seeAlso|}}
+{{#if seeAlso.items.length}}
 
-{{#if (eq sectionType "cloud")}}
+{{#if (eq seeAlso.sectionType "cloud")}}
 ifdef::env-cloud[]
 | Related topics
-|{{#if (gt (length ../related_topics) 1)}}{{#each ../related_topics}}
-{{#with (parseRelatedTopic this)}}
-* {{{value}}}
-{{/with}}
+|{{#if (gt seeAlso.items.length 1)}}{{#each seeAlso.items}}
+* {{{content}}}
 {{/each}}
 {{else}}
-{{#each ../related_topics}}
-{{#with (parseRelatedTopic this)}}
-{{{value}}}
-{{/with}}
+{{#each seeAlso.items}}
+{{{content}}}
 {{/each}}
 {{/if}}
 endif::[]
-{{else if (eq sectionType "self-managed")}}
+{{else if (eq seeAlso.sectionType "self-managed")}}
 ifndef::env-cloud[]
 | Related topics
-|{{#if (gt (length ../related_topics) 1)}}{{#each ../related_topics}}
-{{#with (parseRelatedTopic this)}}
-* {{{value}}}
-{{/with}}
+|{{#if (gt seeAlso.items.length 1)}}{{#each seeAlso.items}}
+* {{{content}}}
 {{/each}}
 {{else}}
-{{#each ../related_topics}}
-{{#with (parseRelatedTopic this)}}
-{{{value}}}
-{{/with}}
+{{#each seeAlso.items}}
+{{{content}}}
 {{/each}}
 {{/if}}
 endif::[]
 {{else}}
 | Related topics
-|{{#if (gt (length ../related_topics) 1)}}{{#each ../related_topics}}
-{{#with (parseRelatedTopic this)}}
-{{#if (eq type "cloud")}}
+|{{#if (gt seeAlso.items.length 1)}}{{#each seeAlso.items}}
+{{#if cloudOnly}}
 ifdef::env-cloud[]
-* {{{value}}}
+* {{{content}}}
 endif::[]
-{{else if (eq type "self-managed")}}
+{{else if selfHostedOnly}}
 ifndef::env-cloud[]
-* {{{value}}}
+* {{{content}}}
 endif::[]
 {{else}}
-* {{{value}}}
+* {{{content}}}
 {{/if}}
-{{/with}}
 {{/each}}
 {{else}}
-{{#each ../related_topics}}
-{{#with (parseRelatedTopic this)}}
-{{#if (eq type "cloud")}}
+{{#each seeAlso.items}}
+{{#if cloudOnly}}
 ifdef::env-cloud[]
-{{{value}}}
+{{{content}}}
 endif::[]
-{{else if (eq type "self-managed")}}
+{{else if selfHostedOnly}}
 ifndef::env-cloud[]
-{{{value}}}
+{{{content}}}
 endif::[]
 {{else}}
-{{{value}}}
+{{{content}}}
 {{/if}}
-{{/with}}
 {{/each}}
 {{/if}}
 {{/if}}
-{{/with}}
 {{/if}}
+{{/with}}
 
 |===
 

package/tools/property-extractor/templates/topic-property.hbs

@@ -130,78 +130,66 @@ endif::[]
 |
 {{{example}}}
 {{/if}}
-{{#if related_topics}}
-{{#with (allTopicsConditional related_topics) as |sectionType|}}
+{{#with (seeAlsoView .) as |seeAlso|}}
+{{#if seeAlso.items.length}}
 
-{{#if (eq sectionType "cloud")}}
+{{#if (eq seeAlso.sectionType "cloud")}}
 ifdef::env-cloud[]
 | Related topics
-|{{#if (gt (length ../related_topics) 1)}}{{#each ../related_topics}}
-{{#with (parseRelatedTopic this)}}
-* {{{value}}}
-{{/with}}
+|{{#if (gt seeAlso.items.length 1)}}{{#each seeAlso.items}}
+* {{{content}}}
 {{/each}}
 {{else}}
-{{#each ../related_topics}}
-{{#with (parseRelatedTopic this)}}
-{{{value}}}
-{{/with}}
+{{#each seeAlso.items}}
+{{{content}}}
 {{/each}}
 {{/if}}
 endif::[]
-{{else if (eq sectionType "self-managed")}}
+{{else if (eq seeAlso.sectionType "self-managed")}}
 ifndef::env-cloud[]
 | Related topics
-|{{#if (gt (length ../related_topics) 1)}}{{#each ../related_topics}}
-{{#with (parseRelatedTopic this)}}
-* {{{value}}}
-{{/with}}
+|{{#if (gt seeAlso.items.length 1)}}{{#each seeAlso.items}}
+* {{{content}}}
 {{/each}}
 {{else}}
-{{#each ../related_topics}}
-{{#with (parseRelatedTopic this)}}
-{{{value}}}
-{{/with}}
+{{#each seeAlso.items}}
+{{{content}}}
 {{/each}}
 {{/if}}
 endif::[]
 {{else}}
 | Related topics
-|{{#if (gt (length ../related_topics) 1)}}{{#each ../related_topics}}
-{{#with (parseRelatedTopic this)}}
-{{#if (eq type "cloud")}}
+|{{#if (gt seeAlso.items.length 1)}}{{#each seeAlso.items}}
+{{#if cloudOnly}}
 ifdef::env-cloud[]
-* {{{value}}}
+* {{{content}}}
 endif::[]
-{{else if (eq type "self-managed")}}
+{{else if selfHostedOnly}}
 ifndef::env-cloud[]
-* {{{value}}}
+* {{{content}}}
 endif::[]
 {{else}}
-* {{{value}}}
+* {{{content}}}
 {{/if}}
-{{/with}}
 {{/each}}
 {{else}}
-{{#each ../related_topics}}
-{{#with (parseRelatedTopic this)}}
-{{#if (eq type "cloud")}}
+{{#each seeAlso.items}}
+{{#if cloudOnly}}
 ifdef::env-cloud[]
-{{{value}}}
+{{{content}}}
 endif::[]
-{{else if (eq type "self-managed")}}
+{{else if selfHostedOnly}}
 ifndef::env-cloud[]
-{{{value}}}
+{{{content}}}
 endif::[]
 {{else}}
-{{{value}}}
+{{{content}}}
 {{/if}}
-{{/with}}
 {{/each}}
 {{/if}}
 {{/if}}
-{{/with}}
 {{/if}}
+{{/with}}
 {{#if aliases}}
 
 ifndef::env-cloud[]

package/tools/property-extractor/transformers.py

@@ -655,8 +655,10 @@ class BasicInfoTransformer:
     EXPECTED FINAL RESULT:
     {
         "name": "property_name",           # Clean identifier for JSON keys
-        "description": "Human readable...", # Description for documentation  
-        "defined_in": "src/v/config/..."   # Normalized file path
+        "description": "Human readable...", # Description for documentation
+        "defined_in": "src/v/config/...",  # Normalized file path
+        "line_start": 1852,                # First line of the declaration (1-indexed)
+        "line_end": 1858                   # Last line of the declaration (inclusive)
     }
     
     DOWNSTREAM USAGE:
@@ -724,6 +726,17 @@ class BasicInfoTransformer:
         property["defined_in"] = re.sub(
             r"^.*src/", "src/", str(file_pair.implementation)
         )
+
+        # --- Step 4: line span (additive) ---
+        # 1-indexed, inclusive lines of the full member-initializer entry in
+        # the implementation file named by defined_in. Recorded by the parser
+        # (parse_cpp_source) from the tree-sitter field_initializer node.
+        # Spans the whole declaration, not just the description literal, so
+        # diff-anchored tooling can match clang-format-wrapped strings.
+        if info.get("line_start") is not None:
+            property["line_start"] = info["line_start"]
+            property["line_end"] = info.get("line_end", info["line_start"])
+
         return property
 
 @debug_transformer

package/tools/property-extractor/validate-overrides.js

@@ -0,0 +1,120 @@
+'use strict'
+
+const Ajv2020 = require('ajv/dist/2020')
+const fs = require('fs')
+const path = require('path')
+const { ValidationResult } = require('../../cli-utils/validation-result')
+
+/**
+ * Load and compile the JSON Schema for validation
+ * @param {string} [overridesPath] - Path to the overrides file; the schema is
+ *   looked up as a sibling (same directory) before falling back to the
+ *   package-relative path.
+ * @returns {Function} Compiled schema validator
+ */
+function loadSchemaValidator(overridesPath) {
+  // Primary: look for the schema next to the overrides file (docs repo's docs-data/).
+  if (overridesPath) {
+    const siblingPath = path.join(path.dirname(path.resolve(overridesPath)), 'property-overrides.schema.json')
+    if (fs.existsSync(siblingPath)) {
+      const schema = JSON.parse(fs.readFileSync(siblingPath, 'utf8'))
+      const ajv = new Ajv2020({ allErrors: true, verbose: true })
+      return ajv.compile(schema)
+    }
+  }
+
+  // Fallback: package-relative path (legacy / monorepo usage).
+  const schemaPath = path.resolve(__dirname, '../../docs-data/property-overrides.schema.json')
+
+  if (!fs.existsSync(schemaPath)) {
+    throw new Error(`Schema file not found: ${schemaPath}`)
+  }
+
+  const schema = JSON.parse(fs.readFileSync(schemaPath, 'utf8'))
+  const ajv = new Ajv2020({ allErrors: true, verbose: true })
+  return ajv.compile(schema)
+}
+
+/**
+ * Validate overrides against the JSON Schema
+ * @param {Object} overrides - Overrides object to validate
+ * @param {string} [overridesPath] - Path to the overrides file (used to locate the schema)
+ * @returns {ValidationResult}
+ */
+function validateSchema(overrides, overridesPath) {
+  const result = new ValidationResult()
+
+  try {
+    const validate = loadSchemaValidator(overridesPath)
+    const valid = validate(overrides)
+
+    if (!valid) {
+      for (const error of validate.errors || []) {
+        const instancePath = error.instancePath || '/'
+        result.addError(error.message, instancePath)
+      }
+    }
+  } catch (err) {
+    result.addError(`Schema validation failed: ${err.message}`)
+  }
+
+  return result
+}
+
+/**
+ * Run all validations on a property-overrides object.
+ * @param {Object} overrides - Overrides object to validate
+ * @param {string} [overridesPath] - Path to the overrides file (used to locate the schema)
+ * @returns {ValidationResult}
+ */
+function validateOverrides(overrides, overridesPath = null) {
+  const result = new ValidationResult()
+
+  if (!overrides || typeof overrides !== 'object') {
+    result.addError('Overrides must be a non-null object')
+    return result
+  }
+
+  result.merge(validateSchema(overrides, overridesPath))
+
+  return result
+}
+
+/**
+ * Load and validate a property-overrides.json file.
+ * @param {string} overridesPath - Path to overrides JSON file
+ * @returns {{ overrides: Object|null, validation: ValidationResult }}
+ */
+function loadAndValidateOverrides(overridesPath) {
+  const validation = new ValidationResult()
+
+  if (!overridesPath) {
+    return { overrides: null, validation }
+  }
+
+  if (!fs.existsSync(overridesPath)) {
+    validation.addWarning(`Overrides file not found: ${overridesPath}`)
+    return { overrides: null, validation }
+  }
+
+  let overrides
+  try {
+    const content = fs.readFileSync(overridesPath, 'utf8')
+    overrides = JSON.parse(content)
+  } catch (err) {
+    validation.addError(`Failed to parse overrides JSON: ${err.message}`, overridesPath)
+    return { overrides: null, validation }
+  }
+
+  validation.merge(validateOverrides(overrides, overridesPath))
+
+  return { overrides, validation }
+}
+
+module.exports = {
+  ValidationResult,
+  loadSchemaValidator,
+  validateSchema,
+  validateOverrides,
+  loadAndValidateOverrides,
+}

package/tools/rpk-docs/detect-platform-commands.js

@@ -47,6 +47,7 @@
 
 const fs = require('fs')
 const path = require('path')
+const { maskComments } = require('../lint-strings/go-source')
 
 /** GOOS values recognized in filename suffixes and build tags */
 const KNOWN_GOOS = new Set([
@@ -248,7 +249,13 @@ function commandNameFromConstructor(funcName) {
  * @param {string} content - Go file content
  * @returns {Object<string, {useName: string|null, excluded: boolean}>}
  */
-function parseConstructors(content) {
+function parseConstructors(rawContent) {
+  // Mask comments first. Scanning raw text read a `Use: "..."` out of a
+  // commented-out or example command and reported it as the real one; the
+  // masker blanks comment bodies while preserving offsets, line breaks and
+  // every string literal, so the regexes below see only live code. Shared with
+  // the lint-strings Go surfaces rather than reimplemented here.
+  const content = maskComments(rawContent)
   const constructors = {}
   const funcRe = /^func\s+(\w+)\s*\([^)]*\)\s*\*cobra\.Command\s*\{/gm
   const matches = []

package/tools/rpk-docs/scan-deprecated-commands.js

@@ -7,6 +7,7 @@
 
 const fs = require('fs');
 const path = require('path');
+const { maskComments, findBalancedClose } = require('../lint-strings/go-source');
 
 /**
  * Scan a Go file for cobra command definitions with deprecation/hidden status
@@ -17,31 +18,27 @@ function scanGoFile(filePath) {
   const content = fs.readFileSync(filePath, 'utf8');
 
   // Look for NewCommand function that returns *cobra.Command
-  const newCommandMatch = content.match(/func NewCommand\([^)]*\)\s*\*cobra\.Command\s*\{/);
+  // Mask comments before any scanning. The hand-rolled brace counter this
+  // replaced was comment- and string-blind, so a `{` inside a comment
+  // unbalanced it and truncated the cobra.Command block early, and the field
+  // regexes then read a `Use:` string out of a commented-out example. Both
+  // helpers are string-aware and already shared by the lint-strings Go
+  // surfaces; the masker preserves offsets and every string literal.
+  const masked = maskComments(content);
+
+  const newCommandMatch = masked.match(/func NewCommand\([^)]*\)\s*\*cobra\.Command\s*\{/);
   if (!newCommandMatch) {
     return null;
   }
 
   // Extract the cobra.Command struct definition
-  // Handle nested braces and multiline strings
-  let cmdBlock = '';
-  let braceCount = 0;
-  let startIndex = content.indexOf('&cobra.Command{');
-
-  if (startIndex === -1) {
+  const structIndex = masked.indexOf('&cobra.Command{');
+  if (structIndex === -1) {
     return null;
   }
-
-  startIndex += '&cobra.Command{'.length;
-  for (let i = startIndex; i < content.length; i++) {
-    const char = content[i];
-    if (char === '{') braceCount++;
-    if (char === '}') {
-      if (braceCount === 0) break;
-      braceCount--;
-    }
-    cmdBlock += char;
-  }
+  const openIndex = structIndex + '&cobra.Command'.length;
+  const closeIndex = findBalancedClose(masked, openIndex, '{', '}');
+  const cmdBlock = masked.slice(openIndex + 1, closeIndex === -1 ? masked.length : closeIndex);
 
   // Extract fields - handle multiline strings with backticks or quotes
   const useMatch = cmdBlock.match(/Use:\s*"([^"]+)"/);

package/tools/rpk-docs/validate-overrides.js

@@ -3,6 +3,7 @@
 const Ajv2020 = require('ajv/dist/2020')
 const fs = require('fs')
 const path = require('path')
+const { ValidationResult } = require('../../cli-utils/validation-result')
 
 /**
  * Valid admonition locations for notes, warnings, tips, cautions, importants
@@ -50,51 +51,6 @@ const VALID_SECTION_POSITIONS = [
   'end'
 ]
 
-/**
- * Validation result object
- */
-class ValidationResult {
-  constructor() {
-    this.valid = true
-    this.errors = []
-    this.warnings = []
-  }
-
-  addError(message, context = null) {
-    this.valid = false
-    this.errors.push({ message, context })
-  }
-
-  addWarning(message, context = null) {
-    this.warnings.push({ message, context })
-  }
-
-  merge(other) {
-    if (!other.valid) this.valid = false
-    this.errors.push(...other.errors)
-    this.warnings.push(...other.warnings)
-  }
-
-  format() {
-    const lines = []
-    if (this.errors.length > 0) {
-      lines.push('ERRORS:')
-      for (const err of this.errors) {
-        lines.push(`  ✗ ${err.message}`)
-        if (err.context) lines.push(`    at: ${err.context}`)
-      }
-    }
-    if (this.warnings.length > 0) {
-      lines.push('WARNINGS:')
-      for (const warn of this.warnings) {
-        lines.push(`  ⚠ ${warn.message}`)
-        if (warn.context) lines.push(`    at: ${warn.context}`)
-      }
-    }
-    return lines.join('\n')
-  }
-}
-
 /**
  * Normalize a command path (trim whitespace, collapse multiple spaces)
  * @param {string} cmdPath - Command path to normalize

package/tools/property-extractor/helpers/allTopicsConditional.js

@@ -1,19 +0,0 @@
-// Returns 'cloud' if all topics are cloud-only, 'self-managed' if all are self-managed-only, else 'normal'
-module.exports = function allTopicsConditional(related_topics) {
-  if (!Array.isArray(related_topics) || related_topics.length === 0) return null;
-  let allCloud = true;
-  let allSelfManaged = true;
-  for (const t of related_topics) {
-    if (typeof t !== 'string') {
-      allCloud = false;
-      allSelfManaged = false;
-      break;
-    }
-    const trimmed = t.trim();
-    if (!trimmed.startsWith('cloud-only:')) allCloud = false;
-    if (!trimmed.startsWith('self-managed-only:')) allSelfManaged = false;
-  }
-  if (allCloud) return 'cloud';
-  if (allSelfManaged) return 'self-managed';
-  return 'normal';
-};

package/tools/property-extractor/helpers/parseRelatedTopic.js

@@ -1,13 +0,0 @@
-// Returns an object with type and value for a related topic
-// type: 'cloud', 'self-managed', or 'normal'
-module.exports = function parseRelatedTopic(topic) {
-  if (typeof topic !== 'string') return { type: 'normal', value: topic };
-  const trimmed = topic.trim();
-  if (trimmed.startsWith('cloud-only:')) {
-    return { type: 'cloud', value: trimmed.replace(/^cloud-only:/, '').trim() };
-  }
-  if (trimmed.startsWith('self-managed-only:')) {
-    return { type: 'self-managed', value: trimmed.replace(/^self-managed-only:/, '').trim() };
-  }
-  return { type: 'normal', value: trimmed };
-};