@redpanda-data/docs-extensions-and-macros 5.46.0 → 5.47.1

package/tools/property-extractor/generate-handlebars-docs.js

@@ -1,4 +1,8 @@
-const AUTOGEN_NOTICE = '// This content is autogenerated. Do not edit manually. To override descriptions, use the doc-tools CLI with the --overrides option: https://redpandadata.atlassian.net/wiki/spaces/DOC/pages/1396244485/Review+Redpanda+configuration+properties\n';
+// Points at the overrides file and its schema, both of which live in the repo a
+// writer already has open. The previous pointer was a Confluence page, and
+// Confluence is retired, so it sent every writer who read this line to a dead
+// end. A repo-relative path cannot rot the same way.
+const AUTOGEN_NOTICE = '// This content is autogenerated. Do not edit manually. To change a description, add a link, or scope text to one docs build, edit docs-data/property-overrides.json (fields are documented in docs-data/property-overrides.schema.json and docs-data/README.adoc) and regenerate.\n';
 'use strict';
 
 const fs = require('fs');
@@ -49,6 +53,35 @@ function isObjectStorageProperty(prop) {
   );
 }
 
+/**
+ * Which reference page a property is written to.
+ *
+ * The one place that decides, so a link's target page cannot disagree with the
+ * page the property actually lands on. generatePropertyPartials uses it to pick
+ * the partial to append to; applyPropertyLinks uses it to tell a same-page
+ * reference from a cross-page one.
+ *
+ * Returns null for a property with no scope, which is not rendered at all.
+ *
+ * @param {Object} prop - The property object.
+ * @returns {('cluster'|'topic'|'broker'|'object-storage'|null)}
+ */
+function propertyPageFor(prop) {
+  if (!prop || !prop.config_scope) return null;
+  switch (prop.config_scope) {
+    case 'topic':
+      return 'topic';
+    case 'broker':
+      return 'broker';
+    case 'cluster':
+      return isObjectStorageProperty(prop) ? 'object-storage' : 'cluster';
+    case 'object-storage':
+      return 'object-storage';
+    default:
+      return null;
+  }
+}
+
 /**
  * Gets template path, checking environment variables for custom paths first
  */
@@ -249,20 +282,16 @@ function generatePropertyPartials(properties, partialsDir, onRender) {
     // This fixes issues where key != name field due to bugs in the source code
     prop.name = key;
 
-    switch (prop.config_scope) {
+    const page = propertyPageFor(prop);
+    switch (page) {
       case 'topic':
         annotateCorrespondingClusterSupport(prop, properties);
         propertyGroups.topic.push(prop);
         break;
       case 'broker':
-        propertyGroups.broker.push(prop);
-        break;
       case 'cluster':
-        if (isObjectStorageProperty(prop)) propertyGroups['object-storage'].push(prop);
-        else propertyGroups.cluster.push(prop);
-        break;
       case 'object-storage':
-        propertyGroups['object-storage'].push(prop);
+        propertyGroups[page].push(prop);
         break;
       default:
         console.warn(`Warning: Unknown config_scope: ${prop.config_scope} for ${prop.name}`);
@@ -410,6 +439,13 @@ function generateErrorReports(properties, documentedProperties = []) {
  * backticked property name is redundant; anything else becomes text=.
  */
 function convertConfigRefsToProp(text) {
+  // An array is the audience-scoped paragraph form of a description, and it
+  // reaches here BEFORE applyPropertyLinks flattens it. Without this the
+  // `typeof text !== 'string'` guard below returned the array untouched, so a
+  // config_ref inside a scoped paragraph survived into the published page as
+  // literal text -- contradicting this pass's own contract of emitting the
+  // prop macro regardless of what the input or the overrides carry.
+  if (Array.isArray(text)) return text.map(convertConfigRefsToProp);
   if (!text || typeof text !== 'string') return text;
   return text.replace(/config_ref:([^[,]+)(?:,([^[,]*))?(?:,([^[,]*))?\[([^\]]*)\]/g, (match, name, isLink, _path, payload) => {
     const trimmed = name.trim();
@@ -440,6 +476,193 @@ function generateAllDocs(inputFile, outputDir) {
     if (prop && prop.description) prop.description = convertConfigRefsToProp(prop.description);
   });
 
+  // The published attachment (modules/reference/attachments/redpanda-properties-
+  // <tag>.json) is a DIFFERENT consumer of this same file than the .adoc
+  // partials generated below, and it was never taught the array-form
+  // description this feature introduces. property_extractor.py writes
+  // inputFile with whatever shape the override declared -- an array, for an
+  // audience-scoped description -- and the Makefile then copies inputFile
+  // to that attachment VERBATIM, before this function ever runs. Two real
+  // readers of that attachment assume `description` is always a string:
+  // extensions/render-property-descriptions.js skips anything else with a
+  // bare `continue` (no warning), so the browser-side fallback in docs-ui's
+  // tooltip formatter receives the raw array and does `String([...])`,
+  // which comma-joins it -- publishing the literal token "cloud-only:" and
+  // the Cloud-only sentence to Self-Managed readers, live on the two
+  // properties in today's corpus that use this shape.
+  //
+  // Fixed by snapshotting a flattened, link-free rendering of every
+  // array-form description (and every includes-bearing one, for the
+  // parallel case below) and writing it back into inputFile once partial
+  // generation finishes, so the Makefile's copy picks up a plain string. No
+  // links: the attachment has never carried them (see the comment on
+  // `enhancedOutputPath` near the end of this function), and Antora's own
+  // AsciiDoc loader resolves the ifdef/ifndef pair per SITE when
+  // render-property-descriptions.js converts it -- cloud-docs sets
+  // env-cloud in its antora.yml, so the existing per-site machinery is what
+  // makes this correct rather than anything new here.
+  //
+  // Includes are folded in for the same reason `includes` regressed a
+  // second surface: on origin/main the include:: directive was written
+  // inline into the description string, so the tooltip's Antora-loader
+  // conversion resolved it into a real admonition (an internal-use warning,
+  // or the HTTP Proxy ephemeral-credentials breaking-change notice on
+  // scram_password, scram_username and sasl_mechanism). Moving it to its
+  // own `includes` field fixed the .adoc partial but orphaned the
+  // attachment, since render-property-descriptions.js never reads
+  // `includes`. Appending it back as an inline directive, exactly the shape
+  // it used to be, restores that without teaching the extension a new
+  // field.
+  const attachmentDescriptions = {};
+  for (const [attachName, attachProp] of Object.entries(properties)) {
+    if (!attachProp) continue;
+    const hasArrayDescription = Array.isArray(attachProp.description);
+    const hasIncludes = attachProp.includes !== undefined;
+    if (!hasArrayDescription && !hasIncludes) continue;
+
+    let text = helpers.applyPropertyLinks.flattenDescription(attachProp.description);
+    if (hasIncludes) {
+      // A scratch warnings array: the real applyPropertyLinks call below
+      // normalizes and warns on the same includes once more, and that is
+      // the run's single copy of each warning. Duplicating the call here
+      // only to read the normalized shape must not double them.
+      const items = helpers.applyPropertyLinks.normalizeIncludes(attachName, attachProp.includes, []);
+      for (const item of items) {
+        const directive = `include::${item.target}`;
+        // Idempotency guard: in the real Makefile flow inputFile is fresh
+        // from property_extractor.py every run, so `text` never already
+        // carries this. It CAN already carry it if generateAllDocs runs
+        // twice against the same file without Python re-writing it in
+        // between (a repeated call in a test, or a re-run of just this
+        // step), because the append below is what makes it a string in
+        // the first place, and `includes` itself is never cleared. Without
+        // this check a second run appended the directive a second time.
+        if (text.includes(directive)) continue;
+        if (item.cloud_only) {
+          text += `\n\nifdef::env-cloud[]\n${directive}\nendif::[]`;
+        } else if (item.self_managed_only) {
+          text += `\n\nifndef::env-cloud[]\n${directive}\nendif::[]`;
+        } else {
+          text += `\n\n${directive}`;
+        }
+      }
+    }
+    attachmentDescriptions[attachName] = text;
+  }
+
+  // Links and audience-scoped text come out of the overrides as data -- a `links`
+  // map and, for a description, an array of optionally scoped paragraphs -- and
+  // become AsciiDoc here. Running before the three renderers means the property
+  // partials, the deprecated partial and the topic mappings partial all see the
+  // same text.
+  //
+  // This is what lets a description stay plain prose. Markup mixed into the
+  // sentence is what the overrides audit cannot compare against the source
+  // string, so an enriched description could never be retired or upstreamed;
+  // keeping the enrichment in its own fields keeps the prose comparable.
+  // Audience scopes that will not take effect, checked BEFORE the link pass:
+  // applyPropertyLinks flattens an array description into one string, after
+  // which a per-paragraph prefix can no longer be attributed to its paragraph.
+  //
+  // This is a louder failure than an unmatched link. An unmatched link key is
+  // simply not applied, so the page reads correctly minus a link. An
+  // unrecognised audience prefix is PUBLISHED: `cloud_only: Cloud clusters
+  // require...` renders as that literal string, prefix and all, in both
+  // builds. A misspelled audience boolean is silent the other way round, with
+  // the value rendering in both builds instead of the one it was scoped to.
+  const scopeProblems = helpers.findAudienceScopeProblems(properties);
+  if (scopeProblems.length > 0) {
+    console.warn(
+      `Warning: ${scopeProblems.length} audience scope(s) will not take effect:`
+    );
+    for (const p of scopeProblems) {
+      console.warn(
+        p.problem === 'unrecognized-prefix'
+          ? `   ${p.property}: ${p.field} starts with "${p.detail}:", which is not a recognized audience prefix, so it will publish as literal text. Use "cloud-only:" or "self-managed-only:".`
+          : `   ${p.property}: ${p.field} carries "${p.detail}", which is not a recognized audience flag, so the value will render in both builds. Use cloud_only or self_managed_only.`
+      );
+    }
+  }
+
+  const linkResult = helpers.applyPropertyLinks(properties);
+  if (linkResult.applied > 0) {
+    console.log(`Applied ${linkResult.applied} declared link(s) from the overrides file.`);
+  }
+  for (const warning of linkResult.warnings) console.warn(`Warning: ${warning}`);
+  if (linkResult.unmatched.length > 0) {
+    // A link key that matches nothing is silent link rot: the override looks
+    // maintained and the page has no link. Name every one.
+    console.warn(
+      `Warning: ${linkResult.unmatched.length} declared link(s) matched no text and were not applied:`
+    );
+    for (const { property, key } of linkResult.unmatched) {
+      console.warn(`   ${property}: links["${key}"]`);
+    }
+  }
+
+  // Count the declarative enrichment so the caller can report what this run
+  // did, rather than a writer having to read a diff of generated AsciiDoc to
+  // find out. Counted here, after the link pass, while the override-derived
+  // fields are still on the property objects.
+  const enrichment = {
+    declaredLinks: 0,
+    appliedLinks: linkResult.applied,
+    unmatchedLinks: linkResult.unmatched,
+    // Everything else the link pass rejected: an empty key, a missing
+    // target, a #target naming no property, a glossterm with no term.
+    // Console-only until now, so it never reached a reviewer.
+    linkWarnings: linkResult.warnings,
+    audienceScopeProblems: scopeProblems,
+    includes: 0,
+    audienceScopedParagraphs: 0
+  };
+  for (const prop of Object.values(properties)) {
+    if (!prop) continue;
+    if (prop.links) enrichment.declaredLinks += Object.keys(prop.links).length;
+    if (Array.isArray(prop.includes)) enrichment.includes += prop.includes.length;
+    // Counted from the emitted directives, not from the paragraph array:
+    // applyPropertyLinks has already flattened an array description into a
+    // string by this point, so the array is gone. Counting the output also
+    // catches a legacy description that carries a hand-written conditional.
+    if (typeof prop.description === 'string') {
+      const conditionals = prop.description.match(/^ifn?def::env-cloud\[\]$/gm);
+      if (conditionals) enrichment.audienceScopedParagraphs += conditionals.length;
+    }
+  }
+
+  // Write the flattened attachment text back into inputFile, so the
+  // Makefile's later `cp` of this same file to
+  // modules/reference/attachments/ carries a plain string rather than the
+  // array/includes shape that broke the tooltip. Best-effort and re-reads
+  // inputFile fresh rather than serializing the already-mutated
+  // `properties` object, so only the specific properties snapshotted above
+  // change on disk -- every other property, and every field on these two,
+  // is byte-for-byte what property_extractor.py wrote.
+  if (Object.keys(attachmentDescriptions).length > 0) {
+    try {
+      const onDisk = bigIntJson.parse(fs.readFileSync(inputFile, 'utf8'));
+      const onDiskProperties = onDisk.properties || onDisk.topic_properties || {};
+      let flattenedCount = 0;
+      for (const [name, text] of Object.entries(attachmentDescriptions)) {
+        if (onDiskProperties[name]) {
+          onDiskProperties[name].description = text;
+          flattenedCount += 1;
+        }
+      }
+      if (flattenedCount > 0) {
+        // indent 4, matching property_extractor.py's json.dumps(indent=4), so
+        // a real regeneration's diff is the description fields that changed,
+        // not a whole-file reindent.
+        fs.writeFileSync(inputFile, bigIntJson.stringify(onDisk, 4) + '\n');
+        console.log(
+          `Flattened ${flattenedCount} audience-scoped/include-bearing description(s) back into ${inputFile} for the published attachment.`
+        );
+      }
+    } catch (err) {
+      console.warn(`Warning: could not write flattened descriptions back into ${inputFile}: ${err.message}`);
+    }
+  }
+
   registerPartials();
 
   let partialsCount = 0;
@@ -479,13 +702,40 @@ function generateAllDocs(inputFile, outputDir) {
     console.log('Ignored:\n   ' + errors.undocumented_properties.join('\n   '));
   }
 
-  return {
+  const summary = {
     totalProperties,
     generatedPartials: partialsCount,
     undocumentedProperties: errors.undocumented_properties,
     deprecatedProperties: deprecatedCount,
-    percentageRendered: pctRendered
+    percentageRendered: pctRendered,
+    enrichment
   };
+
+  // Opt-in machine-readable summary. The re-render workflows set this to a
+  // path outside the checkout and turn the file into the PR body, so a writer
+  // sees what the run did without reading a diff of generated AsciiDoc.
+  //
+  // Opt-in, and never defaulted to outputDir: that directory is inside the
+  // docs checkout, and the re-render workflow commits every change in the
+  // tree, so a default would commit a generated artefact into the repo on
+  // every run. Best-effort, because a summary that cannot be written must not
+  // fail a generation that otherwise succeeded.
+  const summaryPath = process.env.PROPERTY_DOCS_SUMMARY_PATH;
+  if (summaryPath) {
+    try {
+      // bigIntJson.stringify, not JSON.stringify: this file parses property
+      // data with bigIntJson.parse, so a uint64 limit arrives as a real
+      // BigInt. Plain JSON.stringify throws on one, and the throw lands in
+      // the catch below -- a warning and no summary file, which is exactly
+      // the silent degradation this summary exists to remove.
+      fs.writeFileSync(summaryPath, bigIntJson.stringify(summary, 2) + '\n');
+      console.log(`Wrote run summary to ${summaryPath}`);
+    } catch (err) {
+      console.warn(`Warning: could not write ${summaryPath}: ${err.message}`);
+    }
+  }
+
+  return summary;
 }
 
 
@@ -493,7 +743,8 @@ module.exports = {
   generateAllDocs,
   generateDeprecatedDocs,
   generatePropertyPartials,
-  convertConfigRefsToProp
+  convertConfigRefsToProp,
+  propertyPageFor
 };
 
 // CLI