@redpanda-data/docs-extensions-and-macros 5.11.1 → 5.12.1

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@redpanda-data/docs-extensions-and-macros",
-  "version": "5.11.1",
+  "version": "5.12.1",
   "description": "Antora extensions and macros developed for Redpanda documentation.",
   "keywords": [
     "antora",
@@ -102,7 +102,8 @@
     "cli-utils",
     "tools",
     "docker-compose",
-    "mcp"
+    "mcp",
+    "docs-data/rpk-overrides.schema.json"
   ],
   "license": "ISC",
   "engines": {
@@ -149,8 +150,8 @@
     "yargs": "^17.7.2"
   },
   "devDependencies": {
-    "@antora/cli": "3.1.4",
-    "@antora/site-generator": "3.1.4",
+    "@antora/cli": "^3.1.15",
+    "@antora/site-generator": "^3.1.15",
     "@asciidoctor/core": "2.2.8",
     "@web/dev-server": "^0.2.5",
     "jest": "^29.7.0",

package/tools/property-extractor/cloud_config.py

@@ -500,10 +500,23 @@ def add_cloud_support_metadata(properties: Dict, cloud_config: CloudConfig) -> D
                 skipped_count += 1
                 continue
             
-            # Only process cluster, broker, and topic properties for cloud support
+            # Only cluster properties get cloud metadata.
+            #
+            # The two sections this data comes from -- customer_managed_configs
+            # and readonly_cluster_config -- describe the Cloud control plane's
+            # view of CLUSTER configuration, and contain nothing else: every one
+            # of their entries resolves to a cluster property, and no topic
+            # property appears anywhere in the install pack.
+            #
+            # Emitting cloud_supported: false for a topic or broker property
+            # therefore asserted something this source cannot know. Downstream
+            # that read as "a Cloud reader cannot set this", which is wrong for
+            # topic properties -- Cloud users set them routinely through the
+            # Kafka API and Console, a surface the control plane does not gate.
+            # Leave the fields absent instead, so absent means "no opinion" and
+            # false keeps its meaning of "the control plane forbids it".
             config_scope = prop_data.get('config_scope', '')
-            if config_scope not in ['cluster', 'broker', 'topic']:
-                # Skip node config properties and others without cloud relevance
+            if config_scope != 'cluster':
                 skipped_count += 1
                 continue
             

package/tools/property-extractor/generate-handlebars-docs.js

@@ -104,6 +104,103 @@ function registerPartials() {
   }
 }
 
+/**
+ * Rewrite property anchors across every description and related-topics entry so
+ * they match the heading IDs Asciidoctor will actually emit.
+ *
+ * Runs once over the whole property map before rendering, because a property's
+ * description routinely links to a sibling property and the anchor shapes differ
+ * by where the text came from. Reports what it corrected: a steady stream of
+ * rewrites is a signal to fix the upstream doc string, not to lean on this pass.
+ *
+ * @param {Object<string, Object>} properties - Property map, mutated in place.
+ * @returns {number} Number of anchors rewritten.
+ */
+function normalizeAnchorsInProperties(properties) {
+  const seen = new Map();
+  let total = 0;
+
+  let xrefTotal = 0;
+  const fix = (value, scope) => {
+    const { text, rewrites } = helpers.normalizePropertyAnchors(value, properties, scope);
+    for (const r of rewrites) {
+      total += 1;
+      seen.set(r.from, r.to);
+    }
+    const canonical = helpers.canonicalizePropertyXrefs(text);
+    xrefTotal += canonical.rewrites;
+    return canonical.text;
+  };
+
+  Object.values(properties).forEach((prop) => {
+    if (!prop || typeof prop !== 'object') return;
+    const scope = prop.config_scope;
+    if (typeof prop.description === 'string') prop.description = fix(prop.description, scope);
+    if (typeof prop.acceptable_values === 'string') prop.acceptable_values = fix(prop.acceptable_values, scope);
+    if (Array.isArray(prop.related_topics)) {
+      // Canonicalizing collapses forms that differed only by prefix, which turns
+      // some entries into exact duplicates of each other. Drop the repeats.
+      const seenTopics = new Set();
+      prop.related_topics = prop.related_topics
+        .map((t) => (typeof t === 'string' ? fix(t, scope) : t))
+        .filter((t) => {
+          if (typeof t !== 'string') return true;
+          const key = t.trim();
+          if (seenTopics.has(key)) return false;
+          seenTopics.add(key);
+          return true;
+        });
+    }
+  });
+
+  if (total > 0) {
+    console.log(`Fixed ${total} property anchor(s) that would have rendered as dead links:`);
+    for (const [from, to] of seen) console.log(`   <<${from}>> -> <<${to}>>`);
+  }
+  if (xrefTotal > 0) {
+    console.log(`Canonicalized ${xrefTotal} property-page xref(s) to reference:properties/…`);
+  }
+  return total;
+}
+
+/**
+ * Record whether a topic property's corresponding cluster property is one a Cloud
+ * customer can actually reach.
+ *
+ * Cloud publishes a subset of cluster properties, so linking a topic property to
+ * its cluster counterpart unconditionally sends Cloud readers to an anchor that
+ * does not exist on their reference page. `cloud_supported` on the cluster
+ * property is the authority here: it is derived from the install pack's
+ * customer_managed_configs and readonly_cluster_config, so true means the Cloud
+ * page publishes it.
+ *
+ * Note what this flag does NOT say. False means the control plane does not expose
+ * the property to customers; it does not distinguish "Redpanda manages the value"
+ * from "the property does not apply to Cloud at all". So the template drops the
+ * link and keeps the name, and asserts nothing about who sets it.
+ *
+ * Both the primary and the alternate mapping are resolved, because a row that
+ * links to two cluster properties needs both to be reachable before it is worth
+ * linking either.
+ *
+ * @param {Object} prop - Topic property to annotate in place.
+ * @param {Object<string, Object>} properties - Full property map, used to look up counterparts.
+ */
+function annotateCorrespondingClusterSupport(prop, properties) {
+  const isSupported = (name) => {
+    if (!name) return false;
+    const target = properties[name];
+    return Boolean(target && target.cloud_supported);
+  };
+
+  const primary = isSupported(prop.corresponding_cluster_property);
+  prop.corresponding_cluster_property_cloud_supported = primary;
+  prop.alternate_cluster_property_cloud_supported = isSupported(prop.alternate_cluster_property);
+  prop.cluster_property_mapping_cloud_supported = prop.alternate_cluster_property
+    ? primary && prop.alternate_cluster_property_cloud_supported
+    : primary;
+}
+
 /**
  * Generate AsciiDoc partial files grouping input properties by scope (cluster, topic, broker, object-storage).
  *
@@ -120,6 +217,8 @@ function registerPartials() {
 function generatePropertyPartials(properties, partialsDir, onRender) {
   console.log(`📝 Generating consolidated property partials in ${partialsDir}…`);
 
+  normalizeAnchorsInProperties(properties);
+
   const propertyTemplate = handlebars.compile(
     fs.readFileSync(getTemplatePath(path.join(__dirname, 'templates', 'property.hbs'), 'TEMPLATE_PROPERTY'), 'utf8')
   );
@@ -151,6 +250,7 @@ function generatePropertyPartials(properties, partialsDir, onRender) {
 
     switch (prop.config_scope) {
       case 'topic':
+        annotateCorrespondingClusterSupport(prop, properties);
         propertyGroups.topic.push(prop);
         break;
       case 'broker':

package/tools/property-extractor/helpers/anchorName.js

@@ -1,5 +1,18 @@
+// Builds the anchor for a property's generated section heading.
+//
+// The headings are plain AsciiDoc (`=== cleanup.policy`), so Asciidoctor derives
+// their IDs itself: lowercase, every run of characters outside [a-z0-9_] replaced
+// with the id separator, which Antora sets to `-`. Underscores are word
+// characters and survive, so `redpanda.cloud_topic.enabled` becomes
+// `redpanda-cloud_topic-enabled`.
+//
+// Reproduce that here rather than inventing a separate scheme: anything else
+// renders as a link to an anchor that does not exist on the page.
 module.exports = function anchorName(name) {
-  const anchor = String(name).replace(/[^a-zA-Z0-9]/g, '').toLowerCase();
+  const anchor = String(name)
+    .toLowerCase()
+    .replace(/[^a-z0-9_]+/g, '-')
+    .replace(/^-+|-+$/g, '');
   if (!anchor) {
     throw new Error(`Invalid property name for anchor generation: "${name}"`);
   }

package/tools/property-extractor/helpers/canonicalizePropertyXrefs.js

@@ -0,0 +1,54 @@
+'use strict';
+
+// The reference pages these xrefs target.
+const PROPERTY_PAGES = [
+  'cluster-properties',
+  'object-storage-properties',
+  'topic-properties',
+  'broker-properties',
+];
+
+// Every prefix seen in the wild for the same target:
+//   xref:cluster-properties.adoc#x            bare filename
+//   xref:./cluster-properties.adoc#x          relative to the current directory
+//   xref:reference:cluster-properties.adoc#x  legacy, when the pages sat at the module root
+//   xref:reference:properties/...#x           canonical
+const PREFIXES = '(?:\\./|reference:properties/|reference:)?';
+const PATTERN = new RegExp(
+  `xref:${PREFIXES}(${PROPERTY_PAGES.join('|')})\\.adoc(?=[#\\[])`,
+  'g'
+);
+
+/**
+ * Rewrite every reference to a property reference page to one canonical resource ID.
+ *
+ * The bare and `reference:`-prefixed forms both resolve to the module root, where
+ * these pages no longer live. Self-managed papers over that with a
+ * `reference:cluster-properties.adoc` page alias, so the broken form renders as a
+ * working link there and as an unresolved xref in cloud-docs, which has no such
+ * alias. The `./` form works but depends on the including page sitting in
+ * `reference/properties/`, which is a constraint on every page that includes a
+ * partial rather than a property of the partial.
+ *
+ * The fully qualified module-and-path form has neither problem: it resolves
+ * inside whichever component includes the partial, from any directory.
+ *
+ * @param {string} text - AsciiDoc source that may contain property-page xrefs.
+ * @returns {{text: string, rewrites: number}}
+ */
+function canonicalizePropertyXrefs(text) {
+  if (typeof text !== 'string' || !text.includes('xref:')) {
+    return { text, rewrites: 0 };
+  }
+  let rewrites = 0;
+  const out = text.replace(PATTERN, (match, page) => {
+    const canonical = `xref:reference:properties/${page}.adoc`;
+    if (match === canonical) return match;
+    rewrites += 1;
+    return canonical;
+  });
+  return { text: out, rewrites };
+}
+
+module.exports = canonicalizePropertyXrefs;
+module.exports.PROPERTY_PAGES = PROPERTY_PAGES;

package/tools/property-extractor/helpers/index.js

@@ -14,6 +14,8 @@ module.exports = {
   renderPropertyExample: require('./renderPropertyExample.js'),
   formatUnits: require('./formatUnits.js'),
   anchorName: require('./anchorName.js'),
+  normalizePropertyAnchors: require('./normalizePropertyAnchors.js'),
+  canonicalizePropertyXrefs: require('./canonicalizePropertyXrefs.js'),
   parseRelatedTopic: require('./parseRelatedTopic.js'),
   allTopicsConditional: require('./allTopicsConditional.js'),
   capitalize: require('./capitalize.js'),

package/tools/property-extractor/helpers/normalizePropertyAnchors.js

@@ -0,0 +1,86 @@
+'use strict';
+
+const anchorName = require('./anchorName.js');
+
+// Strip every separator so `redpanda.storage.mode.impl`, `redpanda-storage-mode-impl`
+// and `redpandastoragemodeimpl` all collapse to one key.
+function squash(value) {
+  return String(value).toLowerCase().replace(/[^a-z0-9]/g, '');
+}
+
+/**
+ * Repair intra-page anchors (`<<anchor,text>>`) that name a property but use the
+ * wrong ID shape.
+ *
+ * Descriptions come from three places -- Redpanda's own doc strings, the
+ * overrides file, and hand-edits -- and each has at some point written a
+ * property anchor by deleting the dots (`<<redpandastoragemodeimpl>>`) or by
+ * hyphenating only part of the name (`<<redpandastorage-mode>>`). Asciidoctor
+ * derives the real heading ID from the heading text, so those render as links to
+ * nothing. Rather than hand-correct each one and wait for the next author to
+ * reintroduce it, resolve any anchor that unambiguously names a known property
+ * back to that property's real anchor.
+ *
+ * Only rewrites when the anchor is not already correct and the match is
+ * unambiguous, so an anchor pointing at a genuine non-property section is left
+ * alone. Names from different scopes can collide once separators are gone --
+ * `retention.bytes` and `retention_bytes` both squash to `retentionbytes` -- so
+ * a collision is resolved in favour of the scope the referring property is in,
+ * which is where a cross-reference nearly always points, and refused outright if
+ * that still leaves two candidates.
+ *
+ * @param {string} text - AsciiDoc source that may contain `<<anchor,label>>` references.
+ * @param {Iterable<string>|Object<string, Object>} properties - Known property names, or the property map.
+ * @param {string} [scope] - config_scope of the property that owns this text.
+ * @returns {{text: string, rewrites: Array<{from: string, to: string}>}}
+ */
+function normalizePropertyAnchors(text, properties, scope) {
+  if (typeof text !== 'string' || !text.includes('<<')) {
+    return { text, rewrites: [] };
+  }
+
+  const isMap = properties && !Array.isArray(properties) && typeof properties === 'object'
+    && !(typeof properties[Symbol.iterator] === 'function');
+  const names = isMap ? Object.keys(properties) : Array.from(properties || []);
+  const scopeOf = (name) => (isMap && properties[name] ? properties[name].config_scope : undefined);
+
+  const bySquashed = new Map();
+  const validAnchors = new Set();
+  for (const name of names) {
+    if (typeof name !== 'string' || !name) continue;
+    let anchor;
+    try {
+      anchor = anchorName(name);
+    } catch {
+      continue;
+    }
+    validAnchors.add(anchor);
+    const key = squash(name);
+    const entry = bySquashed.get(key);
+    if (entry) entry.push({ anchor, scope: scopeOf(name) });
+    else bySquashed.set(key, [{ anchor, scope: scopeOf(name) }]);
+  }
+
+  const resolve = (key) => {
+    const candidates = bySquashed.get(key);
+    if (!candidates || candidates.length === 0) return null;
+    if (candidates.length === 1) return candidates[0].anchor;
+    const sameScope = scope ? candidates.filter((c) => c.scope === scope) : [];
+    return sameScope.length === 1 ? sameScope[0].anchor : null;
+  };
+
+  const rewrites = [];
+  const rewritten = text.replace(/<<([^<>,\]]+?)(,|>>)/g, (match, anchor, tail) => {
+    const trimmed = anchor.trim();
+    if (validAnchors.has(trimmed)) return match;
+    const target = resolve(squash(trimmed));
+    if (!target || target === trimmed) return match;
+    rewrites.push({ from: trimmed, to: target });
+    return `<<${target}${tail}`;
+  });
+
+  return { text: rewritten, rewrites };
+}
+
+module.exports = normalizePropertyAnchors;
+module.exports.squash = squash;

package/tools/property-extractor/templates/topic-property-mappings.hbs

@@ -4,12 +4,21 @@
 
 {{#each topicProperties}}
 | <<{{anchorName name}},`{{name}}`>>
-| {{#if alternate_cluster_property}}xref:./{{cluster_property_doc_file}}#{{corresponding_cluster_property}}[`{{corresponding_cluster_property}}`] or xref:./{{alternate_cluster_property_doc_file}}#{{alternate_cluster_property}}[`{{alternate_cluster_property}}`]{{else}}xref:./{{cluster_property_doc_file}}#{{corresponding_cluster_property}}[`{{corresponding_cluster_property}}`]{{/if}}
+|
+ifdef::env-cloud[]
+{{#if cluster_property_mapping_cloud_supported}}
+{{#if alternate_cluster_property}}xref:reference:properties/{{cluster_property_doc_file}}#{{corresponding_cluster_property}}[`{{corresponding_cluster_property}}`] or xref:reference:properties/{{alternate_cluster_property_doc_file}}#{{alternate_cluster_property}}[`{{alternate_cluster_property}}`]{{else}}xref:reference:properties/{{cluster_property_doc_file}}#{{corresponding_cluster_property}}[`{{corresponding_cluster_property}}`]{{/if}}
+{{else}}
+{{#if alternate_cluster_property}}`{{corresponding_cluster_property}}` or `{{alternate_cluster_property}}`{{else}}`{{corresponding_cluster_property}}`{{/if}}
+{{/if}}
+endif::[]
+ifndef::env-cloud[]
+{{#if alternate_cluster_property}}xref:reference:properties/{{cluster_property_doc_file}}#{{corresponding_cluster_property}}[`{{corresponding_cluster_property}}`] or xref:reference:properties/{{alternate_cluster_property_doc_file}}#{{alternate_cluster_property}}[`{{alternate_cluster_property}}`]{{else}}xref:reference:properties/{{cluster_property_doc_file}}#{{corresponding_cluster_property}}[`{{corresponding_cluster_property}}`]{{/if}}
+endif::[]
 
 {{/each}}
 
 |===
 
-{{!-- Helper to generate anchor names --}}
-{{!-- Usage: anchorName propertyName --}}
-{{!-- Example: cleanup.policy -> cleanuppolicy --}}
+{{!-- Anchors come from the anchorName helper, which reproduces Asciidoctor's --}}
+{{!-- own auto-generated heading ID: cleanup.policy -> cleanup-policy. --}}

package/tools/property-extractor/templates/topic-property.hbs

@@ -77,7 +77,17 @@ endif::[]
 {{#if corresponding_cluster_property}}
 
 | Corresponding cluster property
-| xref:reference:cluster-properties.adoc#{{corresponding_cluster_property}}[{{corresponding_cluster_property}}]
+|
+ifdef::env-cloud[]
+{{#if corresponding_cluster_property_cloud_supported}}
+xref:reference:properties/{{#if cluster_property_doc_file}}{{cluster_property_doc_file}}{{else}}cluster-properties.adoc{{/if}}#{{corresponding_cluster_property}}[`{{corresponding_cluster_property}}`]
+{{else}}
+`{{corresponding_cluster_property}}`
+{{/if}}
+endif::[]
+ifndef::env-cloud[]
+xref:reference:properties/{{#if cluster_property_doc_file}}{{cluster_property_doc_file}}{{else}}cluster-properties.adoc{{/if}}#{{corresponding_cluster_property}}[`{{corresponding_cluster_property}}`]
+endif::[]
 {{/if}}
 {{#if (and minimum maximum)}}
 

package/tools/redpanda-connect/report-delta.js

@@ -422,10 +422,50 @@ function buildComponentMap(indexObj) {
   return map;
 }
 
+/**
+ * Whether to refresh the Connect plugin before reading its version.
+ *
+ * Only ephemeral environments should: the refresh uses "install --force",
+ * which skips the safeguard that "upgrade" applies to a self-managed plugin
+ * (see the comment in getRpkConnectVersion), so on a real machine it can
+ * change which Connect binary the user runs afterwards.
+ *
+ * Defaults to on when CI is set. RPCN_FORCE_CONNECT_INSTALL overrides in
+ * either direction, for forcing a refresh locally or suppressing one in a
+ * CI-like environment that owns its own Connect installation.
+ */
+function shouldRefreshConnectPlugin() {
+  const override = process.env.RPCN_FORCE_CONNECT_INSTALL;
+  if (override !== undefined && override !== '') {
+    return !['0', 'false', 'no', 'off'].includes(override.toLowerCase());
+  }
+  return Boolean(process.env.CI);
+}
+
 function getRpkConnectVersion() {
   try {
-    // Make sure the connect plugin is upgraded first (silent)
-    execSync('rpk connect upgrade', { stdio: 'ignore' });
+    // Ensure the connect plugin is current (silent). Deliberately "install
+    // --force", not "upgrade": upgrade's version comparison parses the
+    // currently-installed version through a regex capped at two digits per
+    // segment (redpanda.VersionFromString in rpk/pkg/redpanda/version.go),
+    // which throws for any Connect version >= 4.100.0 — the same bug class
+    // as CON-529, in a different function that its fix didn't touch. Fixed
+    // upstream on dev (redpanda commit 8fc022e7) but present in all rpk
+    // releases up to at least v26.2.1; keep this workaround until a fixed
+    // rpk release ships. install's "latest" path (the default) skips that
+    // regex entirely, so it isn't exposed to the bug.
+    //
+    // The trade-off is that "install --force" drops a safeguard "upgrade"
+    // keeps: upgrade refuses to touch a plugin whose Managed flag is false
+    // (rpk/pkg/cli/connect/upgrade.go), which is how a Connect installed by
+    // Homebrew or by hand is protected. A forced install instead writes a
+    // managed binary into the plugin dir that rpk prioritises, shadowing
+    // that installation for every later rpk call. So refresh only where the
+    // Connect installation is disposable, and leave a developer's machine
+    // exactly as it is — reporting the version already installed there.
+    if (shouldRefreshConnectPlugin()) {
+      execSync('rpk connect install --force', { stdio: 'ignore' });
+    }
 
     // Now capture the --version output
     const raw = execSync('rpk connect --version', {
@@ -444,7 +484,13 @@ function getRpkConnectVersion() {
     }
     return match[1];
   } catch (err) {
-    throw new Error(`Unable to run "rpk connect --version": ${err.message}`);
+    // Outside CI the plugin is never installed on the caller's behalf, so say
+    // how to get one rather than only reporting that the call failed.
+    const hint = shouldRefreshConnectPlugin()
+      ? ''
+      : ' If Redpanda Connect is not installed, run "rpk connect install" first,' +
+        ' or set RPCN_FORCE_CONNECT_INSTALL=1 to let this command install it.';
+    throw new Error(`Unable to run "rpk connect --version": ${err.message}${hint}`);
   }
 }
 

package/tools/rpk-docs/extract-conditional-content.js

@@ -11,7 +11,9 @@ const { spawnSync } = require('child_process')
 
 const docsDir = process.argv[2] || '../docs'
 const rpkDir = path.join(docsDir, 'modules/reference/pages/rpk')
-const overridesPath = path.join(__dirname, '../../docs-data/rpk-overrides.json')
+// The canonical overrides live in the docs repo (the docsDir this script
+// already takes); this repo ships only the schema.
+const overridesPath = process.argv[3] || path.join(docsDir, 'docs-data/rpk-overrides.json')
 
 // Validate docsDir exists and is a directory
 if (!fs.existsSync(docsDir) || !fs.statSync(docsDir).isDirectory()) {