@redpanda-data/docs-extensions-and-macros 5.2.5 → 5.3.1

package/bin/doc-tools.js

@@ -1109,6 +1109,17 @@ automation
   .option('-r, --ref <ref>', 'Git branch or tag to document (e.g., dev, v26.2.0). Clones from GitHub.')
   .option('--from-source <path>', 'Path to local rpk source (src/go/rpk directory)')
   .option('--from-json <path>', 'Regenerate docs from an existing versioned JSON file (skips building)')
+  .option('--plugin <name>', 'Refresh a single rpk plugin\'s docs (ai, connect, k8s, check). Requires --from-json. Installs the plugin, splices its fresh subtree into the snapshot, and re-renders.')
+  .option('--plugin-version <version>', 'Plugin version to install and record (for example, 4.102.0). Defaults to the latest published version.')
+  .option('--plugin-pin <name=version>', 'Pin a plugin version for the installs during full generation (repeatable, for example --plugin-pin k8s=26.3.1-beta.1). Required for pre-GA plugins with no promoted latest version.', (value, pins) => {
+    const eq = value.indexOf('=')
+    if (eq < 1 || eq === value.length - 1) {
+      throw new Error(`Invalid --plugin-pin '${value}': expected <name>=<version>`)
+    }
+    pins[value.slice(0, eq)] = value.slice(eq + 1)
+    return pins
+  }, {})
+  .option('--rpk-bin <path>', 'Path to an existing rpk binary for the plugin refresh (skips download/build)')
   .option('--overrides <path>', 'Path to overrides JSON file', 'docs-data/rpk-overrides.json')
   .option('--diff <oldVersion>', 'Generate diff against previous version')
   .option('--update-whats-new [path]', 'Update what\'s-new file with rpk changes from diff (default: modules/get-started/pages/release-notes/redpanda.adoc)')
@@ -1124,6 +1135,12 @@ automation
     try {
       const { handleRpkDocsGeneration } = require('../tools/rpk-docs/rpk-docs-handler.js')
 
+      if (options.plugin && !options.fromJson) {
+        console.error('Error: --plugin requires --from-json <snapshot>')
+        console.error('A plugin refresh splices the fresh subtree into an existing committed snapshot.')
+        process.exit(1)
+      }
+
       // Handle --update-whats-new with optional path
       let whatsNewPath = null
       if (options.updateWhatsNew !== undefined) {
@@ -1137,6 +1154,10 @@ automation
         ref: options.ref,
         fromSource: options.fromSource,
         fromJson: options.fromJson,
+        plugin: options.plugin,
+        pluginVersion: options.pluginVersion,
+        pluginPins: options.pluginPin,
+        rpkBin: options.rpkBin,
         overrides: options.overrides,
         diff: options.diff,
         updateWhatsNew: whatsNewPath,
@@ -1150,6 +1171,10 @@ automation
       })
 
       if (result.success) {
+        if (result.skipped) {
+          console.log(`\n✓ Skipped: ${result.reason}`)
+          process.exit(0)
+        }
         console.log('\n✓ rpk documentation generated successfully')
 
         // Write PR summary to file if requested (useful for GitHub Actions)
@@ -1175,6 +1200,140 @@ automation
     }
   })
 
+/**
+ * generate rpk-plugin-stubs
+ *
+ * @description
+ * Reconciles a consumer repo's single-source stub pages and nav section
+ * against the rpk plugin partials generated in the docs repo. Run from the
+ * consumer repo root (for example, adp-docs for rpk ai). Creates stubs for
+ * new partials, deletes managed stubs whose partial is gone, rebuilds the
+ * plugin's nav block, and proposes page aliases for likely renames.
+ * Full reconcile, so it is idempotent and heals pre-existing drift.
+ */
+automation
+  .command('rpk-plugin-stubs')
+  .description('Reconcile single-source stub pages and nav against the docs repo\'s rpk plugin partials. Run from the consumer repo root.')
+  .option('--plugin <name>', 'rpk plugin command name', 'ai')
+  .option('--docs-repo <owner/repo>', 'Docs repo that owns the partials', 'redpanda-data/docs')
+  .option('--docs-ref <ref>', 'Branch or tag to read partials from', 'main')
+  .option('--partials-dir <path>', 'Local partials directory (skips cloning the docs repo)')
+  .option('--source-path <path>', 'Path in the docs repo to read from (default: modules/reference/partials/rpk-<plugin>; use modules/reference/pages/rpk/rpk-connect for page-family content)')
+  .option('--stub-dir <path>', 'Stub pages directory in the consumer repo (default: modules/reference/pages/rpk/rpk-<plugin>)')
+  .option('--nav-file <path>', 'Nav file whose plugin block is rebuilt', 'modules/ROOT/nav.adoc')
+  .option('--include-prefix <prefix>', 'Antora resource prefix for stub includes. Default: inferred from an existing stub.')
+  .option('--attribute <line>', 'Page attribute line added to new stubs (repeatable)', (value, acc) => { acc.push(value); return acc }, [])
+  .option('--summary-file <path>', 'Write a markdown summary (for PR bodies)')
+  .option('--dry-run', 'Report what would change without writing')
+  .action(async (options) => {
+    try {
+      const {
+        readPartialTitles, fetchPartialsDir, inferIncludePrefix, reconcileStubs
+      } = require('../tools/rpk-docs/generate-plugin-stubs.js')
+
+      const plugin = options.plugin
+      const stubDir = options.stubDir || `modules/reference/pages/rpk/rpk-${plugin}`
+      const partialsDir = options.partialsDir || fetchPartialsDir({
+        docsRepo: options.docsRepo,
+        docsRef: options.docsRef,
+        plugin,
+        sourcePath: options.sourcePath
+      })
+
+      const includePrefix = options.includePrefix || inferIncludePrefix(stubDir, plugin)
+      if (!includePrefix) {
+        console.error('Error: could not infer the include prefix (no existing stubs). Pass --include-prefix, for example: streaming:reference:partial$rpk-ai/')
+        process.exit(1)
+      }
+
+      const partials = readPartialTitles(partialsDir)
+      console.log(`Reconciling ${partials.length} partial(s) against ${stubDir}`)
+
+      const result = reconcileStubs({
+        partials,
+        stubDir,
+        navFile: options.navFile,
+        plugin,
+        includePrefix,
+        ...(options.attribute.length > 0 ? { attributes: options.attribute } : {}),
+        dryRun: options.dryRun
+      })
+
+      console.log(`  Created: ${result.created.length}, deleted: ${result.deleted.length}, nav updated: ${result.navUpdated}`)
+      for (const f of result.created) console.log(`  + ${f}`)
+      for (const f of result.deleted) console.log(`  - ${f}`)
+      for (const f of result.keptNonStub) console.log(`  ! kept (not a managed stub): ${f}`)
+
+      const lines = []
+      lines.push(`## rpk ${plugin} stub reconciliation`)
+      lines.push('')
+      lines.push(`Reconciled against \`${options.partialsDir ? partialsDir : `${options.docsRepo}@${options.docsRef}`}\`.`)
+      lines.push('')
+      if (result.created.length + result.deleted.length === 0 && !result.navUpdated) {
+        lines.push('No changes: stubs and nav already match the partials.')
+      }
+      if (result.created.length > 0) {
+        lines.push(`### New stubs (${result.created.length})`)
+        lines.push('')
+        result.created.forEach(f => lines.push(`- \`${f}\``))
+        lines.push('')
+      }
+      if (result.deleted.length > 0) {
+        lines.push(`### Deleted stubs (${result.deleted.length})`)
+        lines.push('')
+        result.deleted.forEach(f => lines.push(`- \`${f}\``))
+        lines.push('')
+      }
+      if ((result.skippedAliasTargets || []).length > 0) {
+        lines.push('### Skipped: names claimed as page aliases')
+        lines.push('')
+        lines.push('These partials exist upstream, but a page here already claims the name as a `:page-aliases:` target — creating the stub would make the Antora build fatal. Usually this means a rename alias exists while the upstream partial for the old name has not been cleaned up yet:')
+        lines.push('')
+        for (const t of result.skippedAliasTargets) {
+          lines.push(`- \`${t.file}\` (claimed by \`${t.claimedBy}\`)`)
+        }
+        lines.push('')
+      }
+      const straightDeletions = result.deleted.filter(d => !result.renameCandidates.some(rc => rc.deleted === d))
+      if (straightDeletions.length > 0) {
+        lines.push('### Deletions with no rename partner')
+        lines.push('')
+        lines.push('These pages were removed with no successor detected. Their published URLs will 404 — consider adding a redirect or an alias on a related page:')
+        lines.push('')
+        straightDeletions.forEach(f => lines.push(`- \`${f}\``))
+        lines.push('')
+      }
+      if (result.renameCandidates.length > 0) {
+        lines.push('### Possible renames — reviewer decision needed')
+        lines.push('')
+        lines.push('These deleted/created pairs look like renames. If so, add `:page-aliases:` for the old page name to the new stub so published URLs keep working:')
+        lines.push('')
+        for (const rc of result.renameCandidates) {
+          lines.push(`- \`${rc.deleted}\` → \`${rc.created}\`: add \`:page-aliases: reference:rpk/rpk-${plugin}/${rc.deleted}\` to the new stub`)
+        }
+        lines.push('')
+      }
+      if (result.keptNonStub.length > 0) {
+        lines.push('### Kept (not managed stubs)')
+        lines.push('')
+        lines.push('These pages do not match the managed stub shape, so they were not touched:')
+        lines.push('')
+        result.keptNonStub.forEach(f => lines.push(`- \`${f}\``))
+        lines.push('')
+      }
+      const summary = lines.join('\n')
+      if (options.summaryFile) {
+        fs.writeFileSync(options.summaryFile, summary, 'utf8')
+        console.log(`Summary written to: ${options.summaryFile}`)
+      }
+
+      process.exit(0)
+    } catch (err) {
+      console.error(`Error: ${err.message}`)
+      process.exit(1)
+    }
+  })
+
 /**
  * validate rpk-overrides
  *

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@redpanda-data/docs-extensions-and-macros",
-  "version": "5.2.5",
+  "version": "5.3.1",
   "description": "Antora extensions and macros developed for Redpanda documentation.",
   "keywords": [
     "antora",

package/tools/property-extractor/property_extractor.py

@@ -79,7 +79,7 @@ ENUM_PATTERN = re.compile(r'^[a-zA-Z0-9_:]+::([a-zA-Z0-9_]+)$')  # Match full qu
 CONSTRUCTOR_PATTERN = re.compile(r'([a-zA-Z0-9_:]+)\((.*)\)')
 BRACED_CONSTRUCTOR_PATTERN = re.compile(r'([a-zA-Z0-9_:]+)\{(.*)\}')
 DIGIT_SEPARATOR_PATTERN = re.compile(r"(?<=\d)'(?=\d)")
-FUNCTION_CALL_PATTERN = re.compile(r'([a-zA-Z0-9_:]+)\(\)')
+FUNCTION_CALL_PATTERN = re.compile(r'([a-zA-Z0-9_:.]+)\(\)$')
 CHRONO_PATTERN = re.compile(r'std::chrono::([a-zA-Z]+)\s*\{\s*(\d+)\s*\}')
 CHRONO_PAREN_PATTERN = re.compile(r'(?:std::)?chrono::([a-zA-Z]+)\s*\(\s*([^)]+)\s*\)')
 TIME_UNIT_PATTERN = re.compile(r'(\d+)\s*(min|s|ms|h)')
@@ -159,6 +159,15 @@ class ConstexprCache:
             re.compile(r'const\s+model::ns\s+([a-zA-Z_][a-zA-Z0-9_]*)\s*\(\s*"([^"]+)"\s*\)'),
         ]
 
+        # Member accessors on inline constants used in property defaults,
+        # for example model::schema_registry_internal_tp.topic() -> "_schemas".
+        # Each entry maps a variable declaration to the member name whose call
+        # returns the captured string, cached under "<variable>.<member>".
+        member_accessor_patterns = [
+            # Pattern: inline const model::topic_partition name{model::topic{"value"}, ...}
+            (re.compile(r'inline\s+const\s+(?:model::)?topic_partition\s+([a-zA-Z_][a-zA-Z0-9_]*)\s*\{\s*(?:model::)?topic\{\s*"([^"]+)"\s*\}'), 'topic'),
+        ]
+
         # Legacy specific patterns (kept for compatibility, but general patterns should cover these)
         function_patterns = {}
 
@@ -206,6 +215,17 @@ class ConstexprCache:
                                         qualified_name = f"{namespace}::{func_name}"
                                         self.function_cache[qualified_name] = func_value
 
+                            # Extract member accessors on inline constants
+                            for pattern, member in member_accessor_patterns:
+                                for match in pattern.finditer(content):
+                                    var_name = match.group(1)
+                                    value = match.group(2)
+                                    key = f"{var_name}.{member}"
+                                    namespace = self._extract_namespace_for_function(content, match.start())
+                                    self.function_cache[key] = value
+                                    if namespace:
+                                        self.function_cache[f"{namespace}::{key}"] = value
+
                             # Legacy: Extract function definitions from hardcoded patterns (if any)
                             for func_name, patterns in function_patterns.items():
                                 for pattern in patterns:
@@ -753,7 +773,24 @@ def transform_files_with_properties(files_with_properties):
                 continue
 
             if len(property_definition) > 0:
-                all_properties[name] = property_definition
+                # Key the output by the property's registered name (the string
+                # literal passed to the constructor), which can differ from the
+                # C++ member identifier used as the parser key. For example,
+                # the member `default_topic_replication` registers the name
+                # "default_topic_replications".
+                output_key = property_definition.get("name") or name
+                existing = all_properties.get(output_key)
+                if existing is not None and existing.get("defined_in") == property_definition.get("defined_in"):
+                    # Two members of the same config store register the same
+                    # property name. config_store registers properties with
+                    # emplace(), so the first registration wins at runtime.
+                    # Match that behavior and keep the first definition.
+                    logging.warning(
+                        f"Duplicate property name '{output_key}' (member '{name}') in "
+                        f"{property_definition.get('defined_in')}: keeping the first definition."
+                    )
+                    continue
+                all_properties[output_key] = property_definition
 
     return all_properties
 

package/tools/property-extractor/transformers.py

@@ -639,8 +639,9 @@ class BasicInfoTransformer:
     
     NAME RESOLUTION PRIORITY:
     1. Pre-existing property["name"] (from previous processing)
-    2. info["name_in_file"] (C++ variable name from Tree-sitter)  
-    3. First parameter value if it looks like an identifier
+    2. First string-literal parameter if it looks like an identifier
+       (the name argument passed to the property constructor)
+    3. info["name_in_file"] (C++ member name from Tree-sitter) as fallback
     
     DESCRIPTION EXTRACTION LOGIC:
     1. If first parameter is a string and differs from name → use as description
@@ -673,29 +674,44 @@ class BasicInfoTransformer:
             return property
 
         # --- Step 1: find the "real" start of the property definition ---
-        # Skip lambdas, validators, and non-string literals at the start
-        start_idx = 0
+        # Skip lambdas, validators, and non-string literals at the start.
+        # The parser normalizes string_literal values to unquoted strings, so
+        # select by parameter type rather than by surrounding quotes.
+        start_idx = None
         for i, p in enumerate(params):
-            val = str(p.get("value", ""))
-            typ = p.get("type", "")
             if is_validator_param(p):
                 continue
-            if typ in ("lambda_expression", "qualified_identifier", "unresolved_identifier"):
-                continue
-            if not (val.startswith('"') and val.endswith('"')):
+            if p.get("type") != "string_literal":
                 continue
             # First string literal we hit is the name
             start_idx = i
             break
 
         # --- Step 2: extract name and description robustly ---
-        name = property.get("name") or info.get("name_in_file")
-        if not name and len(params) > start_idx:
-            name = params[start_idx].get("value", "").strip('"')
+        # The property's registered name is the first string-literal constructor
+        # argument, NOT the C++ member identifier. They usually match, but not
+        # always: for example, the member `default_topic_replication` registers
+        # the name "default_topic_replications". info["name_in_file"] (the
+        # member identifier) is only a fallback.
+        name = property.get("name")
+        if not name:
+            for p in params:
+                if p.get("type") != "string_literal":
+                    continue
+                candidate = str(p.get("value", "")).strip().strip('"')
+                # The name literal is a single identifier with no whitespace.
+                # If the first string literal looks like a description instead
+                # (for example, the name was passed as a constant), fall back
+                # to the member identifier below.
+                if candidate and not re.search(r"\s", candidate):
+                    name = candidate
+                break  # only the first string literal can be the name
+        if not name:
+            name = info.get("name_in_file")
         property["name"] = name
 
         desc = None
-        if len(params) > start_idx + 1:
+        if start_idx is not None and len(params) > start_idx + 1:
             v0 = params[start_idx].get("value")
             v1 = params[start_idx + 1].get("value")
             if isinstance(v1, str) and len(v1) > 10 and " " in v1: