@redpanda-data/docs-extensions-and-macros 5.8.0 → 5.9.0

package/tools/enterprise-features/verify.js

@@ -0,0 +1,244 @@
+'use strict'
+
+/* Drift checks between the enterprise features registry (the shared
+ * component's enterprise-features.yml in the docs repo) and the internal
+ * sources of truth. Used by `doc-tools validate enterprise-features`.
+ *
+ * Finding levels:
+ *   error       - registry is internally inconsistent or contradicts a source
+ *   needs-human - a source changed in a way that needs a naming or policy
+ *                 decision (for example, a new core enum value)
+ *   info        - notable but not actionable by itself
+ */
+
+const yaml = require('js-yaml')
+
+const VALID_SCOPES = ['redpanda', 'console', 'connect', 'operator', 'cloud']
+const SOURCE_KINDS = ['core-enum', 'core-property', 'connect-plugin', 'manual']
+
+function finding (level, check, message) {
+  return { level, check, message }
+}
+
+/**
+ * Parse and lint the registry itself.
+ *
+ * @param {string} source - Registry YAML.
+ * @returns {{features: object[], findings: object[]}}
+ */
+function lintRegistry (source) {
+  const findings = []
+  const data = yaml.load(source)
+  if (!data || !Array.isArray(data.features)) {
+    return { features: [], findings: [finding('error', 'registry-lint', "Registry has no 'features' list.")] }
+  }
+  const seen = new Map()
+  for (const feature of data.features) {
+    const name = feature && feature.name
+    if (!name) {
+      findings.push(finding('error', 'registry-lint', 'Entry without a name.'))
+      continue
+    }
+    for (const key of [name, ...(feature.aliases || [])].map((value) => String(value).trim().toLowerCase())) {
+      if (seen.has(key) && seen.get(key) !== name) {
+        findings.push(finding('error', 'registry-lint', `Duplicate name or alias '${key}' shared by '${seen.get(key)}' and '${name}'.`))
+      }
+      seen.set(key, name)
+    }
+    if (!VALID_SCOPES.includes(feature.scope)) {
+      findings.push(finding('error', 'registry-lint', `'${name}' has unknown scope '${feature.scope}'.`))
+    }
+    if (!feature.description || !String(feature.description).trim()) {
+      findings.push(finding('error', 'registry-lint', `'${name}' has no description.`))
+    }
+    if (!feature.source || !feature.source.kind) {
+      findings.push(finding('error', 'registry-lint', `'${name}' has no source pointer.`))
+    } else if (!SOURCE_KINDS.includes(feature.source.kind)) {
+      findings.push(finding('error', 'registry-lint', `'${name}' has unknown source kind '${feature.source.kind}'.`))
+    } else if (!feature.source.value || !String(feature.source.value).trim()) {
+      findings.push(finding('error', 'registry-lint', `'${name}' has an empty source value; manual entries need a justification.`))
+    }
+  }
+  return { features: data.features, findings }
+}
+
+/**
+ * Compare registry core-enum pointers against the actual enum values.
+ * Every enum value needs at least one registry entry (else needs-human),
+ * and every core-enum pointer must be a real enum value (else error).
+ */
+function checkCoreEnum (features, enumValues) {
+  const findings = []
+  const pointers = new Map()
+  for (const feature of features) {
+    if (feature.source && feature.source.kind === 'core-enum') {
+      const value = String(feature.source.value).trim()
+      if (!pointers.has(value)) pointers.set(value, [])
+      pointers.get(value).push(feature.name)
+    }
+  }
+  for (const value of enumValues) {
+    if (!pointers.has(value)) {
+      findings.push(finding('needs-human', 'core-enum', `New or unmapped core enterprise feature '${value}' (license_required_feature) has no registry entry. Decide its approved external name and add an entry.`))
+    }
+  }
+  for (const [value, names] of pointers) {
+    if (!enumValues.includes(value)) {
+      findings.push(finding('error', 'core-enum', `Registry entr${names.length > 1 ? 'ies' : 'y'} ${names.map((n) => `'${n}'`).join(', ')} point${names.length > 1 ? '' : 's'} at core enum value '${value}', which does not exist in enterprise_features.h.`))
+    }
+  }
+  return findings
+}
+
+/**
+ * Check core-property pointers and gating properties against the
+ * config::enterprise<> property names extracted from configuration.h.
+ * Gating properties are checked for existence only when a full property
+ * list is provided.
+ */
+function checkCoreProperties (features, enterpriseProperties, allPropertyNames) {
+  const findings = []
+  for (const feature of features) {
+    if (feature.source && feature.source.kind === 'core-property') {
+      const value = String(feature.source.value).trim()
+      if (!enterpriseProperties.includes(value)) {
+        findings.push(finding('error', 'core-property', `'${feature.name}' points at property '${value}', which is not an enterprise-wrapped property in configuration.h.`))
+      }
+    }
+    if (feature['gating-property'] && allPropertyNames && !allPropertyNames.includes(feature['gating-property'])) {
+      findings.push(finding('error', 'gating-property', `'${feature.name}' names gating property '${feature['gating-property']}', which does not exist in the property reference.`))
+    }
+  }
+  const covered = features
+    .filter((f) => f.source && (f.source.kind === 'core-property' || f.source.kind === 'core-enum'))
+    .flatMap((f) => [String(f.source.value).trim(), f['gating-property']])
+    .filter(Boolean)
+  for (const property of enterpriseProperties) {
+    if (!covered.includes(property)) {
+      findings.push(finding('info', 'core-property', `Enterprise-wrapped property '${property}' is not referenced by any registry entry. Confirm whether it belongs to an existing feature or needs a new entry.`))
+    }
+  }
+  return findings
+}
+
+/**
+ * Compare connect info.csv enterprise plugins against the stale
+ * enterprise-components list in rp-connect-docs antora.yml (when given).
+ */
+function checkConnect (infoCsvPlugins, antoraList) {
+  const findings = []
+  if (!antoraList) return findings
+  for (const plugin of infoCsvPlugins) {
+    if (!antoraList.includes(plugin)) {
+      findings.push(finding('error', 'connect-list', `Plugin '${plugin}' is enterprise in info.csv but missing from the enterprise-components list in rp-connect-docs antora.yml.`))
+    }
+  }
+  for (const plugin of antoraList) {
+    if (!infoCsvPlugins.includes(plugin)) {
+      findings.push(finding('error', 'connect-list', `Plugin '${plugin}' is in the enterprise-components list in rp-connect-docs antora.yml but not enterprise in info.csv.`))
+    }
+  }
+  return findings
+}
+
+/**
+ * Compare the hand-maintained disable-enterprise-features.adoc table with
+ * registry gating properties.
+ */
+function checkDisableTable (features, disableRows) {
+  const findings = []
+  const registryProperties = new Set(features.map((f) => f['gating-property']).filter(Boolean))
+  for (const row of disableRows) {
+    for (const property of row.properties) {
+      if (!registryProperties.has(property)) {
+        findings.push(finding('needs-human', 'disable-table', `disable-enterprise-features.adoc row '${row.feature}' references property '${property}', which is not a gating-property of any registry entry.`))
+      }
+    }
+  }
+  return findings
+}
+
+/**
+ * Render the internal-to-external name mapping partial, so pages can
+ * translate the names printed by `rpk cluster license info`.
+ */
+function buildMappingPartial (features, enumValues) {
+  const byValue = new Map()
+  for (const feature of features) {
+    if (feature.source && feature.source.kind === 'core-enum') {
+      const value = String(feature.source.value).trim()
+      if (!byValue.has(value)) byValue.set(value, [])
+      byValue.get(value).push(feature)
+    }
+  }
+  const lines = [
+    '// Generated by `doc-tools validate enterprise-features --write-mapping`. Do not edit.',
+    '// Maps the internal feature names reported by `rpk cluster license info`',
+    '// (enterprise_features_in_use) to their documented names.',
+    '[cols="1m,2a"]',
+    '|===',
+    '| Reported name | Documented feature',
+    '',
+  ]
+  const order = enumValues.length ? enumValues.filter((value) => byValue.has(value)) : [...byValue.keys()].sort()
+  for (const value of order) {
+    const cell = byValue.get(value)
+      .map((feature) => (feature.xref ? `xref:${feature.xref}[${feature.name}]` : feature.name))
+      .join('\n\n')
+    lines.push(`| ${value}`, `| ${cell}`, '')
+  }
+  lines.push('|===')
+  return lines.join('\n')
+}
+
+/**
+ * Run all checks. Sources that were not provided are skipped with an info
+ * finding, so partial runs stay useful.
+ *
+ * @param {object} sources
+ * @param {string} sources.registryYaml
+ * @param {string} [sources.coreHeader] - enterprise_features.h contents.
+ * @param {string} [sources.configurationHeader] - configuration.h contents.
+ * @param {string} [sources.infoCsv] - connect info.csv contents.
+ * @param {string[]} [sources.antoraEnterpriseComponents]
+ * @param {string} [sources.disablePage] - disable-enterprise-features.adoc contents.
+ * @param {string[]} [sources.allPropertyNames] - full property list for gating checks.
+ * @returns {{findings: object[], features: object[], enumValues: string[]}}
+ */
+function runChecks (sources) {
+  const parsers = require('./parsers')
+  const { features, findings } = lintRegistry(sources.registryYaml)
+  let enumValues = []
+  if (sources.coreHeader) {
+    enumValues = parsers.parseCoreEnum(sources.coreHeader)
+    findings.push(...checkCoreEnum(features, enumValues))
+  } else {
+    findings.push(finding('info', 'core-enum', 'Skipped: no core header provided.'))
+  }
+  if (sources.configurationHeader) {
+    const enterpriseProperties = parsers.parseEnterpriseProperties(sources.configurationHeader)
+    findings.push(...checkCoreProperties(features, enterpriseProperties, sources.allPropertyNames))
+  } else {
+    findings.push(finding('info', 'core-property', 'Skipped: no configuration header provided.'))
+  }
+  if (sources.infoCsv) {
+    const plugins = parsers.parseConnectEnterprisePlugins(sources.infoCsv)
+    findings.push(...checkConnect(plugins, sources.antoraEnterpriseComponents))
+  } else {
+    findings.push(finding('info', 'connect-list', 'Skipped: no connect info.csv provided.'))
+  }
+  if (sources.disablePage) {
+    findings.push(...checkDisableTable(features, parsers.parseDisableTable(sources.disablePage)))
+  }
+  return { findings, features, enumValues }
+}
+
+module.exports = {
+  lintRegistry,
+  checkCoreEnum,
+  checkCoreProperties,
+  checkConnect,
+  checkDisableTable,
+  buildMappingPartial,
+  runChecks,
+}

package/tools/generate-cli-docs.js

@@ -128,7 +128,7 @@ function parseJSDocComments(sourceFile) {
 
   // Regex to match JSDoc comments followed by command definitions
   // Matches both top-level commands and automation.command()
-  const pattern = /\/\*\*\s*([\s\S]*?)\s*\*\/\s*(?:programCli|automation)\s*\.command\(['"]([^'"]+)['"]\)/g;
+  const pattern = /\/\*\*\s*([\s\S]*?)\s*\*\/\s*(?:programCli|automation|validation)\s*\.command\(['"]([^'"]+)['"]\)/g;
 
   let match;
   while ((match = pattern.exec(content)) !== null) {
@@ -316,6 +316,24 @@ IMPORTANT: This documentation is auto-generated. Do not edit manually. Run \`npm
     doc += generateCommandDoc(`generate ${subcmd}`, data, jsdoc, 3);
   });
 
+  // Validate command and its subcommands
+  console.log('  Generating docs for: validate');
+  const validateHelp = getHelpOutput('validate');
+  const validateData = parseHelp(validateHelp);
+  doc += generateCommandDoc('validate', validateData, null, 2);
+
+  const validateSubcommands = [
+    'enterprise-features'
+  ];
+
+  validateSubcommands.forEach(subcmd => {
+    console.log(`    Generating docs for: validate ${subcmd}`);
+    const help = getHelpOutput(`validate ${subcmd}`);
+    const data = parseHelp(help);
+    const jsdoc = jsdocs[subcmd];
+    doc += generateCommandDoc(`validate ${subcmd}`, data, jsdoc, 3);
+  });
+
   // Write to file
   const outputPath = path.join(__dirname, '..', 'CLI_REFERENCE.adoc');
   fs.writeFileSync(outputPath, doc);

package/tools/migrate-property-refs.js

@@ -0,0 +1,206 @@
+'use strict'
+
+/* One-time migration of plain-backtick property mentions to the prop: macro.
+ *
+ * Rewrites `property_name` to prop:property_name[] in AsciiDoc prose when the
+ * name exists in the published property reference JSON AND contains a
+ * separator (_ or .). Separator-free names (admin, brokers, rack, retries,
+ * superusers) are exactly the ambiguous words that motivated opt-in marking,
+ * so they are never converted automatically.
+ *
+ * Conservative by design. A conversion is skipped when the mention is:
+ *   - inside a delimited listing, literal, or comment block (----, ...., ////)
+ *   - inside a fenced code block (```)
+ *   - on a heading, attribute-entry, or block-title line
+ *   - inside the [] payload of another inline macro (unbalanced [ before it)
+ */
+
+const fs = require('fs')
+const path = require('path')
+
+const SKIP_LINE_RX = /^(=+\s|:[-\w]+[!]?:|\.[^.\s])/
+const DELIMITERS = [
+  { rx: /^-{4,}\s*$/, key: 'listing' },
+  { rx: /^\.{4,}\s*$/, key: 'literal' },
+  { rx: /^\/{4,}\s*$/, key: 'comment' },
+  { rx: /^`{3,}/, key: 'fence' },
+]
+
+/**
+ * Collect convertible property names from a published properties JSON.
+ *
+ * @param {object} propertiesJson - Parsed redpanda-properties JSON.
+ * @returns {{convertible: Set<string>, ambiguous: string[]}}
+ */
+function classifyNames (propertiesJson) {
+  const names = Object.keys(propertiesJson.properties || {})
+  const convertible = new Set()
+  const ambiguous = []
+  for (const name of names) {
+    if (name.includes('_') || name.includes('.')) convertible.add(name)
+    else ambiguous.push(name)
+  }
+  return { convertible, ambiguous }
+}
+
+/**
+ * Convert one line of prose. Exported for unit testing.
+ *
+ * @param {string} line
+ * @param {Set<string>} convertible - Names eligible for conversion.
+ * @param {Set<string>} [seen] - Names already marked in the current
+ *   paragraph. Repeat mentions stay as plain backticks: one tooltip per
+ *   property per paragraph is enough.
+ * @returns {{line: string, count: number}}
+ */
+function convertLine (line, convertible, seen) {
+  let count = 0
+  // `name` spans: backtick, name, backtick, not adjacent to more backticks.
+  const result = line.replace(/(^|[^`\\])`([A-Za-z][A-Za-z0-9_.]*)`(?!`)/g, (match, prefix, name, offset) => {
+    if (!convertible.has(name)) return match
+    if (seen && seen.has(name)) return match
+    // Skip mentions inside another macro's [...] payload.
+    const before = line.slice(0, offset + prefix.length)
+    const opens = (before.match(/\[/g) || []).length
+    const closes = (before.match(/\]/g) || []).length
+    if (opens > closes) return match
+    if (seen) seen.add(name)
+    count++
+    return `${prefix}prop:${name}[]`
+  })
+  return { line: result, count }
+}
+
+/**
+ * Convert config_ref macro calls to prop macro calls. The old macro's manual
+ * path argument is dropped (the prop macro discovers the page dynamically),
+ * and its implicit Kubernetes prefixing becomes the explicit, per-property
+ * correct helm-path=auto. Names must exist in the published JSON; unknown
+ * names are left as config_ref and reported. Exported for unit testing.
+ *
+ * @param {string} line
+ * @param {Set<string>} knownNames - Every name in the published JSON.
+ * @returns {{line: string, count: number, skipped: string[]}}
+ */
+function convertConfigRefLine (line, knownNames) {
+  let count = 0
+  const skipped = []
+  const result = line.replace(/config_ref:([^[,]+)(?:,([^[,]*))?(?:,([^[,]*))?\[([^\]]*)\]/g, (match, name, isLink, _path, payload) => {
+    const trimmed = name.trim()
+    if (!knownNames.has(trimmed)) {
+      skipped.push(trimmed)
+      return match
+    }
+    // A payload of the backticked property name is redundant (the prop macro
+    // renders a code element already); anything else becomes a text override.
+    const display = (payload || '').trim().replace(/^`|`$/g, '')
+    const attrs = [
+      ...(isLink === 'true' ? ['link=true'] : []),
+      'helm-path=auto',
+      ...(display && display !== trimmed ? [`text=${display}`] : []),
+    ]
+    count++
+    return `prop:${trimmed}[${attrs.join(',')}]`
+  })
+  return { line: result, count, skipped }
+}
+
+/**
+ * Convert one AsciiDoc document. Exported for unit testing.
+ *
+ * @param {string} content
+ * @param {Set<string>} convertible
+ * @returns {{content: string, count: number}}
+ */
+function convertDocument (content, convertible, options = {}) {
+  const lines = content.split('\n')
+  const open = {}
+  let total = 0
+  const skippedConfigRefs = []
+  // One prop macro per property per paragraph. Blank lines and list items
+  // start a new paragraph; existing prop: calls count as the mention.
+  let paragraphSeen = new Set()
+  const converted = lines.map((line) => {
+    const delimiter = DELIMITERS.find((d) => d.rx.test(line))
+    if (delimiter) {
+      open[delimiter.key] = !open[delimiter.key]
+      paragraphSeen = new Set()
+      return line
+    }
+    if (Object.values(open).some(Boolean)) return line
+    if (line.trim() === '' || /^\s*(?:[*\-.]+|\d+\.)\s/.test(line)) {
+      paragraphSeen = new Set()
+    }
+    if (SKIP_LINE_RX.test(line)) {
+      paragraphSeen = new Set()
+      return line
+    }
+    let current = line
+    if (options.configRefs && options.knownNames) {
+      const refResult = convertConfigRefLine(current, options.knownNames)
+      total += refResult.count
+      skippedConfigRefs.push(...refResult.skipped)
+      current = refResult.line
+    }
+    for (const m of current.matchAll(/prop:([A-Za-z][A-Za-z0-9_.]*)\[/g)) paragraphSeen.add(m[1])
+    const result = convertLine(current, convertible, paragraphSeen)
+    total += result.count
+    return result.line
+  })
+  return { content: converted.join('\n'), count: total, skippedConfigRefs }
+}
+
+/**
+ * Walk a modules tree and migrate every .adoc file.
+ *
+ * @param {object} opts
+ * @param {string} opts.docsDir - Repo root containing modules/.
+ * @param {object} opts.propertiesJson - Parsed properties JSON.
+ * @param {boolean} opts.write - Apply changes (dry run when false).
+ * @param {string[]} [opts.exclude] - Path substrings to skip.
+ * @returns {{files: number, conversions: number, changed: {file: string, count: number}[], ambiguous: string[]}}
+ */
+function migrate ({ docsDir, propertiesJson, write, configRefs = false, exclude = [
+    'modules/reference/partials/properties/',
+    // Autogenerated; hand edits revert on regeneration.
+    'modules/reference/pages/rpk/',
+    'modules/reference/partials/rpk',
+    'modules/reference/pages/k-crd.adoc',
+    '-helm-spec.adoc',
+    '/attachments/',
+    '/examples/',
+    '/test-resources/',
+  ] }) {
+  const { convertible, ambiguous } = classifyNames(propertiesJson)
+  const knownNames = new Set(Object.keys(propertiesJson.properties || {}))
+  const changed = []
+  const skippedConfigRefs = new Set()
+  let conversions = 0
+  let files = 0
+
+  const walk = (dir) => {
+    for (const entry of fs.readdirSync(dir, { withFileTypes: true })) {
+      const full = path.join(dir, entry.name)
+      const relative = path.relative(docsDir, full)
+      if (exclude.some((fragment) => `${relative}/`.includes(fragment) || relative.includes(fragment))) continue
+      if (entry.isDirectory()) {
+        walk(full)
+      } else if (entry.name.endsWith('.adoc')) {
+        files++
+        const original = fs.readFileSync(full, 'utf8')
+        const result = convertDocument(original, convertible, { configRefs, knownNames })
+        for (const name of result.skippedConfigRefs || []) skippedConfigRefs.add(name)
+        if (result.count > 0) {
+          changed.push({ file: relative, count: result.count })
+          conversions += result.count
+          if (write) fs.writeFileSync(full, result.content)
+        }
+      }
+    }
+  }
+
+  walk(path.join(docsDir, 'modules'))
+  return { files, conversions, changed, ambiguous, skippedConfigRefs: [...skippedConfigRefs] }
+}
+
+module.exports = { classifyNames, convertLine, convertConfigRefLine, convertDocument, migrate }

package/tools/property-extractor/generate-handlebars-docs.js

@@ -301,6 +301,26 @@ function generateErrorReports(properties, documentedProperties = []) {
   };
 }
 
+/**
+ * Rewrite legacy config_ref macro calls in a description to prop macro
+ * calls. The prop macro validates names against the published property
+ * JSON, discovers the target page dynamically (config_ref's manual path is
+ * dropped), and powers hover tooltips. A payload that just repeats the
+ * backticked property name is redundant; anything else becomes text=.
+ */
+function convertConfigRefsToProp(text) {
+  if (!text || typeof text !== 'string') return text;
+  return text.replace(/config_ref:([^[,]+)(?:,([^[,]*))?(?:,([^[,]*))?\[([^\]]*)\]/g, (match, name, isLink, _path, payload) => {
+    const trimmed = name.trim();
+    const display = (payload || '').trim().replace(/^`|`$/g, '');
+    const attrs = [
+      ...(isLink === 'true' ? ['link=true'] : []),
+      ...(display && display !== trimmed ? [`text=${display}`] : []),
+    ];
+    return `prop:${trimmed}[${attrs.join(',')}]`;
+  });
+}
+
 /**
  * Main generator
  */
@@ -309,6 +329,12 @@ function generateAllDocs(inputFile, outputDir) {
   // Support both 'properties' (from property_extractor.py) and 'topic_properties' (from topic_property_extractor.py)
   const properties = data.properties || data.topic_properties || {};
 
+  // Generated output must emit the prop macro, never the deprecated
+  // config_ref, regardless of what the input JSON or overrides carry.
+  Object.values(properties).forEach((prop) => {
+    if (prop && prop.description) prop.description = convertConfigRefsToProp(prop.description);
+  });
+
   registerPartials();
 
   let partialsCount = 0;
@@ -361,7 +387,8 @@ function generateAllDocs(inputFile, outputDir) {
 module.exports = {
   generateAllDocs,
   generateDeprecatedDocs,
-  generatePropertyPartials
+  generatePropertyPartials,
+  convertConfigRefsToProp
 };
 
 // CLI

package/tools/property-extractor/property_extractor.py

@@ -1013,6 +1013,36 @@ def apply_property_overrides(properties, overrides, overrides_file_path=None):
     Returns:
         dict: The same properties mapping with overrides applied and any new properties created.
     """
+    def _normalize_config_ref_macros(text):
+        """Rewrite legacy config_ref macro calls to prop macro calls.
+
+        The prop macro validates names against the published property JSON,
+        discovers the target page dynamically (config_ref's manual path is
+        dropped), and powers hover tooltips. A payload that just repeats the
+        backticked property name is redundant; anything else becomes text=.
+        Applied to every description so neither upstream source text nor
+        overrides can reintroduce the deprecated macro into published JSON.
+        """
+        if not isinstance(text, str) or "config_ref:" not in text:
+            return text
+
+        def _replace(match):
+            name = match.group(1).strip()
+            is_link = (match.group(2) or "").strip() == "true"
+            display = (match.group(4) or "").strip().strip("`")
+            attrs = []
+            if is_link:
+                attrs.append("link=true")
+            if display and display != name:
+                attrs.append(f"text={display}")
+            return f"prop:{name}[{','.join(attrs)}]"
+
+        return re.sub(
+            r"config_ref:([^\[,]+)(?:,([^\[,]*))?(?:,([^\[,]*))?\[([^\]]*)\]",
+            _replace,
+            text,
+        )
+
     phantom_stub_entries.clear()
     if overrides and "properties" in overrides:
         for prop, override in overrides["properties"].items():
@@ -1042,6 +1072,10 @@ def apply_property_overrides(properties, overrides, overrides_file_path=None):
                         "name": prop,
                         "config_scope": new_property.get("config_scope"),
                     })
+
+    for property_data in properties.values():
+        if hasattr(property_data, "get") and property_data.get("description"):
+            property_data["description"] = _normalize_config_ref_macros(property_data["description"])
     return properties