@redpanda-data/docs-extensions-and-macros 5.12.4 → 5.13.1

package/bin/doc-tools.js

@@ -772,9 +772,11 @@ automation
   .option('--template-fields <path>', 'Fields section partial template', path.resolve(__dirname, '../tools/redpanda-connect/templates/fields-partials.hbs'))
   .option('--template-examples <path>', 'Examples section partial template', path.resolve(__dirname, '../tools/redpanda-connect/templates/examples-partials.hbs'))
   .option('--template-metadata <path>', 'Metadata section partial template', path.resolve(__dirname, '../tools/redpanda-connect/templates/metadata-partials.hbs'))
+  .option('--template-description <path>', 'Description section partial template', path.resolve(__dirname, '../tools/redpanda-connect/templates/descriptions-partials.hbs'))
   .option('--template-bloblang <path>', 'Custom Handlebars template for bloblang function/method partials')
   .option('--overrides <path>', 'Optional JSON file with overrides', 'docs-data/overrides.json')
   .option('--include-bloblang', 'Include Bloblang functions and methods in generation')
+  .option('--prune-orphaned-descriptions', 'Allow the description-partial orphan sweep to blank more than 10% of the tree. Only use this when the connector dataset is confirmed complete: an incomplete dataset looks identical to a mass upstream deletion and the sweep blanks published content')
   .option('--cloud-version <version>', 'Cloud binary version (default: auto-detect latest)')
   .option('--cgo-version <version>', 'cgo binary version (default: same as cloud-version)')
   .option('--skip-intermediate', 'Skip intermediate release processing (legacy mode - only compare latest vs last documented)')
@@ -794,6 +796,43 @@ automation
     await handleRpcnConnectorDocs(options)
   })
 
+/**
+ * generate migrate-rpcn-descriptions
+ *
+ * @description
+ * One-time migration that rewires published connector reference pages onto the
+ * regenerated description partial: the header `:description:` becomes
+ * `include::...[tag=attrs]` and the frozen intro prose becomes
+ * `include::...[tag=body]`. Afterwards `generate rpcn-connector-docs` keeps
+ * both fresh on every run.
+ *
+ * @why
+ * Connector pages are one-time first drafts, so their summary, "Introduced in
+ * version" line and description prose never change again even when the
+ * connector does. The body rewire is guarded: a page whose intro carries
+ * anything the partial does not reproduce is reported and left for a human,
+ * because rewriting it would delete published content.
+ *
+ * @example
+ * # Dry run (default): report the pages that would change
+ * npx doc-tools generate migrate-rpcn-descriptions
+ *
+ * # Apply the migration
+ * npx doc-tools generate migrate-rpcn-descriptions --write
+ *
+ * # Rewire page headers only, leaving the body prose alone
+ * npx doc-tools generate migrate-rpcn-descriptions --write --skip-body
+ */
+automation
+  .command('migrate-rpcn-descriptions')
+  .description('One-time migration of frozen connector summaries and description prose onto the regenerated description partial. Dry run unless --write is given.')
+  .option('--write', 'Apply changes (default is a dry run that only reports)')
+  .option('--skip-body', 'Rewire page headers only, leaving the frozen intro prose in the page')
+  .action((options) => {
+    const { migrateDescriptionsToPartials } = require('../tools/redpanda-connect/migrate-descriptions-to-partials.js')
+    migrateDescriptionsToPartials({ write: Boolean(options.write), skipBody: Boolean(options.skipBody) })
+  })
+
 /**
  * generate migrate-rpcn-metadata
  *

package/cli-utils/big-int-json.js

@@ -0,0 +1,85 @@
+'use strict'
+
+/**
+ * JSON parse and stringify that do not destroy integers a JS number cannot hold.
+ *
+ * Redpanda's property data carries the uint64 and int64 limits as the maximum and
+ * minimum of numeric properties: 18446744073709551615, 9223372036854775807 and
+ * -9223372036854775808. The Python extractor writes them exactly, because Python
+ * integers are arbitrary precision. Every stage after that is JavaScript, where
+ * the largest exactly-representable integer is 2^53 - 1, so a plain
+ * `JSON.parse` silently rounds all three -- 18446744073709551615 becomes
+ * 18446744073709552000, which is not the uint64 limit and is not a value the
+ * server accepts.
+ *
+ * That had been happening on every published dataset since April: the reference
+ * pages, the tooltip attachment and the Connect catalog all showed rounded
+ * values, and none of them agreed with the source data.
+ *
+ * `parse` returns those integers as BigInt, so they print exactly and can be
+ * written back unchanged by `stringify`. BigInt rather than a string because it
+ * is unambiguous: a value that was a number in the source can be told apart from
+ * a string that was authored as one, which is what makes the round trip safe.
+ *
+ * Numbers inside the safe range are untouched and stay plain numbers.
+ *
+ * KNOWN LIMITATION: this is a regex over the raw text, not a real JSON scanner,
+ * so it does not track string/escape state. A value that is legitimate JSON
+ * text APPEARING INSIDE an escaped string -- for example a Bloblang example
+ * embedded as an example value, `"{\"delay_for_ns\":110839937000000000}"` --
+ * is indistinguishable from real JSON structure to this regex, which inserts a
+ * quoted sentinel at that position and corrupts the document. This is confirmed
+ * against the real, published Connect component attachment, not a theoretical
+ * risk: parsing it throws "Expected ',' or ']' after array element". Do not
+ * apply this module to Connect/Bloblang-shaped JSON, or to any JSON whose string
+ * values may themselves contain escaped JSON-looking text. It has been checked
+ * against every redpanda-properties-*.json and property-overrides.json this
+ * repo could find (~50 files, every published version) with no false match, and
+ * is applied only to property-shaped data for that reason -- not because the
+ * risk is theoretical elsewhere.
+ */
+
+// An integer literal too long to be safe, in JSON value position: after a colon,
+// an opening bracket or a comma, and followed by a delimiter. A digit run inside
+// a string cannot match, because a string value starts with a quote.
+const UNSAFE_INT_RX = /([:[,]\s*)(-?\d{16,})(?=\s*[,}\]])/g
+// Printable ASCII: a control character is not legal inside a JSON string, so a
+// sentinel containing one produces JSON that will not parse.
+const SENTINEL = '@@bigint:'
+const SENTINEL_RX = new RegExp(`"${SENTINEL}(-?\\d+)"`, 'g')
+
+/** True if the literal needs BigInt to survive a round trip. */
+function isUnsafeInteger (digits) {
+  return !Number.isSafeInteger(Number(digits))
+}
+
+/**
+ * Parse JSON, returning integers outside the safe range as BigInt.
+ *
+ * @param {string} text - JSON text.
+ * @returns {*} The parsed value.
+ */
+function parse (text) {
+  const shielded = String(text).replace(UNSAFE_INT_RX, (match, lead, digits) =>
+    isUnsafeInteger(digits) ? `${lead}"${SENTINEL}${digits}"` : match
+  )
+  return JSON.parse(shielded, (key, value) =>
+    typeof value === 'string' && value.startsWith(SENTINEL) ? BigInt(value.slice(SENTINEL.length)) : value
+  )
+}
+
+/**
+ * Stringify JSON, writing BigInt values back as bare integer literals.
+ *
+ * @param {*} value - Value to serialize.
+ * @param {number|string} [space] - Indentation, as JSON.stringify takes it.
+ * @returns {string} JSON text.
+ */
+function stringify (value, space) {
+  const json = JSON.stringify(value, (key, entry) =>
+    typeof entry === 'bigint' ? `${SENTINEL}${entry.toString()}` : entry, space
+  )
+  return json === undefined ? json : json.replace(SENTINEL_RX, '$1')
+}
+
+module.exports = { parse, stringify, isUnsafeInteger }

package/extensions/render-property-descriptions.js

@@ -0,0 +1,339 @@
+'use strict'
+
+/**
+ * Renders the `description` of every property in a
+ * redpanda-properties-<tag>.json attachment to HTML at build time, with
+ * Antora's own AsciiDoc loader and the site's own AsciiDoc config, and writes
+ * the result to `description_html`.
+ *
+ * Why this exists
+ * ---------------
+ * Property descriptions are authored in AsciiDoc and shipped as JSON, so
+ * something has to render them. The docs UI was doing it in the browser, with a
+ * hand-written parser covering seven constructs: backticks, prop:, config_ref:,
+ * glossterm:, link:, <<anchor>>, and xrefs pre-resolved by a regex extension.
+ *
+ * A whitelist of seven fails open. Everything else -- bold, italics, attribute
+ * references, character replacements, bare URLs, any macro this repo adds next
+ * -- reaches the reader as raw source, and nothing warns. In v26.2.1 that was 31
+ * of 689 properties, including seven that displayed a literal
+ * `include::reference:partial$internal-use-property.adoc[]` where an
+ * internal-use warning or a breaking-change notice should have been.
+ *
+ * Converting here removes the whole class of defect rather than the instances:
+ * Asciidoctor handles all of AsciiDoc, our own macros run as themselves (so
+ * glossterm's positional attributes cannot drift from macros/glossary.js), and
+ * a macro added next year works with no docs-ui change at all.
+ *
+ * Both fields ship. `description` stays because consumers read it -- including
+ * docs-ui versions older than the one that prefers `description_html`, which
+ * deploys separately. The output is minified to pay for the second field.
+ */
+
+const loadAsciiDoc = require('@antora/asciidoc-loader')
+const bigIntJson = require('../cli-utils/big-int-json')
+const { buildPageIndex, propertyAnchor } = require('../macros/prop')
+const { raiseListenerLimit } = require('./util/raise-listener-limit')
+
+const PROPERTIES_JSON_RX = /^redpanda-properties-(v\d+\.\d+\.\d+(?:-[\w.]+)?)\.json$/
+// xref:target[label] as authored in a description. The .adoc extension is
+// optional: one real description writes xref:develop:transactions#tips[...].
+const XREF_RX = /xref:([^[\]\s#]+?)(?:#([^[\]\s]*))?\[([^\]]*)\]/g
+// <<anchor>> and <<anchor,display text>>, as authored in the description. The
+// anchor must start with a letter or underscore: property anchors always do,
+// and without that a C++ shift expression in a code span (`1<<20>>`) was read as
+// a reference, rewritten, and then reported as a broken anchor.
+const INTERNAL_REF_RX = /<<([A-Za-z_][\w.-]*)(?:,\s*([^<>]*?))?>>/g
+
+module.exports.register = function () {
+  raiseListenerLimit(this)
+  const logger = this.getLogger('render-property-descriptions')
+
+  // The anchor index has to be built here, while page sources still exist:
+  // convert-documents.js deletes page.src.contents once conversion finishes
+  // (unless keepSource), and buildPageIndex reads that source to find which page
+  // documents which property. Building it at documentsConverted worked only
+  // because property-page-index happened to warm the same cache earlier, and
+  // without that every <<anchor>> silently degraded to plain text.
+  const anchorIndexes = new Map()
+  this.once('contentClassified', ({ contentCatalog }) => {
+    for (const attachment of propertyAttachments(contentCatalog)) {
+      const properties = readProperties(attachment, logger)
+      if (!properties) continue
+      const key = `${attachment.src.component}@${attachment.src.version || ''}`
+      if (anchorIndexes.has(key)) continue
+      anchorIndexes.set(key, anchorIndex(contentCatalog, attachment, properties, logger))
+    }
+  })
+
+  // Conversion itself waits for documentsConverted: page aliases and the rest of
+  // what page conversion relies on are only fully registered by then. Hooking
+  // earlier left an xref to an aliased page rendering as
+  // class="xref unresolved" while the same xref resolved fine on a real page.
+  // Attachments are still published after this event.
+  this.once('documentsConverted', ({ contentCatalog, siteAsciiDocConfig }) => {
+    const attachments = propertyAttachments(contentCatalog)
+    if (!attachments.length) return
+
+    for (const attachment of attachments) {
+      const where = `${attachment.src.component}@${attachment.src.version || 'unversioned'}`
+      const data = readDataset(attachment, logger)
+      if (!data) continue
+      const properties = data.properties
+
+      // Convert "as" a real page rather than a fabricated one. A hand-built
+      // file.pub is missing fields Antora's resolver needs, and resolving an
+      // xref to another module then throws instead of resolving.
+      const hostPage = findHostPage(contentCatalog, attachment)
+      if (!hostPage) {
+        logger.debug(`${where}: no reference page to convert descriptions as, skipped`)
+        continue
+      }
+      const anchors = anchorIndexes.get(`${attachment.src.component}@${attachment.src.version || ''}`) || new Map()
+
+      // Antora converts each page with its component version's own config
+      // (convert-documents.js builds one per component version and prefers it
+      // over the site config), so attributes declared in antora.yml -- env-cloud
+      // among them, which five descriptions branch on -- only exist there.
+      const componentVersion = contentCatalog.getComponentVersion
+        ? contentCatalog.getComponentVersion(attachment.src.component, attachment.src.version)
+        : undefined
+      const asciidocConfig = (componentVersion && componentVersion.asciidoc) || siteAsciiDocConfig
+
+      let rendered = 0
+      let failed = 0
+      const brokenAnchors = new Set()
+      const unresolved = new Set()
+      const xrefCache = new Map()
+      for (const [name, entry] of Object.entries(properties)) {
+        const description = entry && entry.description
+        if (typeof description !== 'string' || !description.trim()) continue
+        try {
+          const source = resolveInternalRefs(description, anchors, (anchor) => brokenAnchors.add(anchor))
+          const html = render(source, hostPage, contentCatalog, asciidocConfig)
+          if (html == null) continue
+          entry.description_html = html
+          // The attachment is a published download, and `xref:...[]` means
+          // nothing to someone reading the JSON on its own -- resolving it is
+          // why resolve-xrefs-in-attachments existed for these files. This
+          // extension owns that job for property datasets now, so it has to keep
+          // doing it, using Antora's resolver rather than rewriting paths by
+          // hand. Descriptions stay AsciiDoc apart from the resolved anchors,
+          // which is the shape consumers already read.
+          entry.description = resolveXrefs(
+            description, hostPage, contentCatalog, asciidocConfig, xrefCache, (macro) => unresolved.add(macro)
+          )
+          rendered++
+        } catch (error) {
+          // One description that will not convert must not cost the other 680.
+          failed++
+          logger.warn(`${where} ${name}: description could not be converted, so its tooltip falls back to the raw source (${error.message})`)
+        }
+      }
+
+      if (rendered) attachment.contents = Buffer.from(bigIntJson.stringify(data))
+      logger.info(`${where}: rendered ${rendered} property descriptions to HTML${failed ? `, ${failed} failed` : ''}`)
+      if (unresolved.size) {
+        logger.warn(
+          `${where}: ${unresolved.size} xref target(s) in property descriptions could not be resolved, so they stay as raw macros in the published attachment: ` +
+          `${[...unresolved].sort().join(', ')}.`
+        )
+      }
+      if (brokenAnchors.size) {
+        logger.warn(
+          `${where}: ${brokenAnchors.size} <<anchor>> reference(s) in property descriptions name no documented property, so they render as plain text: ` +
+          `${[...brokenAnchors].sort().join(', ')}. Property anchors replace dots with hyphens, so redpanda.storage.mode is <<redpanda-storage-mode>>. ` +
+          'Fix them in the description or in docs-data/property-overrides.json.'
+        )
+      }
+    }
+  })
+}
+
+function basename (file) {
+  return file.src.relative.split('/').pop()
+}
+
+function propertyAttachments (contentCatalog) {
+  return (contentCatalog.findBy({ family: 'attachment' }) || []).filter(
+    (file) => file.src.module === 'reference' && PROPERTIES_JSON_RX.test(basename(file))
+  )
+}
+
+/**
+ * Parse one dataset, or report why it is unusable and return undefined. Left
+ * as-is on failure: the prop macro reports an unusable dataset with the detail,
+ * and failing the whole build over one bad attachment helps nobody.
+ */
+function readDataset (attachment, logger) {
+  const where = `${attachment.src.component}@${attachment.src.version || 'unversioned'}`
+  let data
+  try {
+    data = bigIntJson.parse(attachment.contents.toString())
+  } catch (error) {
+    logger.warn(`${where}: ${basename(attachment)} is not valid JSON, descriptions not rendered (${error.message})`)
+    return undefined
+  }
+  const properties = data && data.properties
+  if (!properties || typeof properties !== 'object' || Array.isArray(properties)) {
+    logger.warn(`${where}: ${basename(attachment)} has no usable 'properties' object, descriptions not rendered`)
+    return undefined
+  }
+  return data
+}
+
+function readProperties (attachment, logger) {
+  const data = readDataset(attachment, logger)
+  return data ? data.properties : undefined
+}
+
+/**
+ * A real page in the attachment's own component and reference module, preferring
+ * one under properties/ so includes and relative xrefs resolve from where the
+ * property docs actually live.
+ */
+function findHostPage (contentCatalog, attachment) {
+  const pages = contentCatalog.findBy({
+    component: attachment.src.component,
+    version: attachment.src.version,
+    family: 'page',
+  }) || []
+  return (
+    pages.find((f) => f.src.module === 'reference' && /^properties\//.test(f.src.relative)) ||
+    pages.find((f) => f.src.module === 'reference') ||
+    undefined
+  )
+}
+
+/**
+ * anchor -> xref target, for the <<anchor>> references that point at another
+ * documented property. Built from the same page index the prop macro links
+ * with, so a description and a prop: call agree about which page documents what.
+ */
+function anchorIndex (contentCatalog, attachment, properties, logger) {
+  const where = `${attachment.src.component}@${attachment.src.version || 'unversioned'}`
+  const index = new Map()
+  let pages
+  try {
+    pages = buildPageIndex(contentCatalog, attachment.src.component, properties, attachment.src.version)
+  } catch (error) {
+    // Returning an empty index silently made a thrown index and an empty one
+    // indistinguishable, and every <<anchor>> then degraded to plain text with
+    // no explanation.
+    logger.warn(`${where}: could not index which page documents each property, so <<anchor>> references in descriptions render as plain text (${error.message})`)
+    return index
+  }
+  if (!pages.size) {
+    logger.warn(`${where}: no reference page was found to document any property, so <<anchor>> references in descriptions render as plain text.`)
+    return index
+  }
+  for (const [name, entry] of pages) {
+    if (!entry || !entry.page) continue
+    index.set(propertyAnchor(name), { page: entry.page, name })
+  }
+  return index
+}
+
+/**
+ * Rewrite <<anchor>> to an xref before conversion.
+ *
+ * Asciidoctor renders <<anchor>> as a same-page fragment link, which is right
+ * on the reference page and wrong in a tooltip: a tooltip is shown on an
+ * arbitrary page, where that fragment does not exist. Pointing it at the page
+ * that documents the property keeps the link working wherever the tooltip
+ * appears, and lets Antora resolve the URL rather than us building one.
+ */
+function resolveInternalRefs (description, anchors, report) {
+  return description.replace(INTERNAL_REF_RX, (match, anchor, display) => {
+    const text = (display || '').trim()
+    const target = anchors.get(anchor)
+    if (target) return `xref:reference:${target.page}.adoc#${anchor}[${attrlistValue(text || `\`${target.name}\``)}]`
+    // The anchor names no documented property, so there is nothing to link to.
+    // Asciidoctor would render a same-page fragment, and a tooltip is shown on
+    // arbitrary pages, so that link would go nowhere and still invite a click --
+    // the same reasoning the prop macro applies to a page it cannot verify.
+    // Render the text and report the anchor so it gets fixed at the source.
+    report(anchor)
+    return text || `\`${anchor}\``
+  })
+}
+
+/**
+ * Display text as a single positional attribute. An unescaped `]` ends the
+ * attribute list, so `<<prop,the flag [beta] option>>` published a truncated
+ * link with ` option]` outside it; and a bare `=` makes Asciidoctor read the
+ * value as a named attribute, which dropped the label entirely and applied part
+ * of it as a CSS role. Escaping the bracket and quoting when there is an equals
+ * sign round-trips every combination of brackets, equals, quotes and commas.
+ */
+function attrlistValue (text) {
+  const escaped = text.replace(/]/g, '\\]')
+  return /=/.test(escaped) ? `"${escaped.replace(/"/g, '\\"')}"` : escaped
+}
+
+/**
+ * Rewrite xref macros in a description to resolved links, so the published
+ * attachment is useful on its own.
+ *
+ * The conversion is done by Antora, one macro at a time, rather than by building
+ * a URL here: that way the page resolution, the fragment, and any id rules from
+ * the site's AsciiDoc config are whatever the site itself would produce, and
+ * cannot drift from the pages the links point at. The previous implementation
+ * rewrote paths with regexes and converted underscores in the fragment to
+ * hyphens -- but property page anchors keep their underscores
+ * (`id="log_cleanup_policy"`), so all 25 of its fragment links in the published
+ * v26.2.1 data were dead.
+ *
+ * Conversions are cached per macro, because the same xref appears in many
+ * descriptions.
+ */
+function resolveXrefs (description, hostPage, contentCatalog, asciidocConfig, cache, report) {
+  return description.replace(XREF_RX, (match) => {
+    if (!cache.has(match)) cache.set(match, convertXref(match, hostPage, contentCatalog, asciidocConfig))
+    const anchor = cache.get(match)
+    if (!anchor) {
+      report(match)
+      // Leave the macro alone rather than emitting a link to nowhere: a reader
+      // of the raw JSON can still see what was meant.
+      return match
+    }
+    return anchor
+  })
+}
+
+/** One xref macro, converted by Antora. Returns undefined if it does not resolve. */
+function convertXref (macro, hostPage, contentCatalog, asciidocConfig) {
+  let html
+  try {
+    html = render(macro, hostPage, contentCatalog, asciidocConfig)
+  } catch (error) {
+    return undefined
+  }
+  if (!html) return undefined
+  const anchor = /<a\b[^>]*>[\s\S]*?<\/a>/.exec(html)
+  // Antora marks a target it could not resolve, and that is not a link worth
+  // publishing into a file people read on its own.
+  if (!anchor || /class="[^"]*\bunresolved\b/.test(anchor[0])) return undefined
+  return anchor[0]
+}
+
+/** Convert one description with the site's converter, as the host page. */
+function render (source, hostPage, contentCatalog, siteAsciiDocConfig) {
+  const file = {
+    contents: Buffer.from(source),
+    mediaType: 'text/asciidoc',
+    src: { ...hostPage.src },
+    pub: { ...hostPage.pub },
+  }
+  // relativizeResourceRefs: false gives site-root-relative URLs. The default
+  // computes them relative to the converting file, which is wrong for a tooltip
+  // shown on some other page.
+  const doc = loadAsciiDoc(file, contentCatalog, { ...siteAsciiDocConfig, relativizeResourceRefs: false })
+  const blocks = doc.getBlocks()
+  if (!blocks.length) return doc.convert() || null
+  // A single paragraph gives inline HTML with no <p> wrapper, which is what a
+  // tooltip wants. Anything richer -- an included admonition, a list, several
+  // paragraphs -- keeps its block markup, and the UI styles it.
+  if (blocks.length === 1 && blocks[0].getContext() === 'paragraph') return blocks[0].getContent()
+  return doc.convert()
+}

package/extensions/resolve-xrefs-in-attachments.js

@@ -20,6 +20,8 @@ const { raiseListenerLimit } = require('./util/raise-listener-limit')
  * - xref:path.adoc[] (empty display uses page title or path)
  */
 
+const PROPERTIES_JSON_RX = /^redpanda-properties-(v\d+\.\d+\.\d+(?:-[\w.]+)?)\.json$/
+
 module.exports.register = function () {
   raiseListenerLimit(this)
   const logger = this.getLogger('resolve-xrefs-in-attachments')
@@ -29,7 +31,15 @@ module.exports.register = function () {
     const allAttachments = contentCatalog.findBy({ family: 'attachment' })
     const jsonAttachments = allAttachments.filter((att) => {
       const path = att.src?.path || att.out?.path || ''
-      return path.endsWith('.json')
+      if (!path.endsWith('.json')) return false
+      // Property datasets are handled by render-property-descriptions, which
+      // converts their descriptions with Asciidoctor instead of rewriting xrefs
+      // by hand. Both running double-processed 53 descriptions in v26.2.1: the
+      // anchor inserted here was escaped to <a href= by the converter. This
+      // extension still serves every other JSON attachment, including the
+      // Connect component catalog, which carries far more xrefs than the
+      // property files do.
+      return !PROPERTIES_JSON_RX.test(path.split('/').pop())
     })
 
     if (!jsonAttachments.length) {
@@ -72,6 +82,19 @@ function processJsonAttachment (attachment, contentCatalog, logger) {
 
   let data
   try {
+    // Deliberately plain JSON.parse, not bigIntJson: this function rewrites
+    // whatever JSON attachment it is handed, including the Connect component
+    // catalog, which contains example config text as an ESCAPED JSON-shaped
+    // string -- "{\"delay_for_ns\":110839937000000000}" -- indistinguishable
+    // by regex from real JSON value position. bigIntJson's value-position regex
+    // matched inside that escaped string and inserted stray quotes, which is not
+    // a rounding error, it is invalid JSON: confirmed to throw
+    // "Expected ',' or ']' after array element" on the real, currently-published
+    // connect-4.103.1.json attachment. A plain round trip still silently rounds
+    // large integers (the pre-existing, understood defect this file has always
+    // had), which is the safer failure mode until a real JSON scanner -- one
+    // that tracks string/escape state rather than pattern-matching text --
+    // replaces the regex here.
     data = JSON.parse(contentStr)
   } catch (err) {
     logger.debug(`Skipping invalid JSON: ${attachment.src?.path}`)

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@redpanda-data/docs-extensions-and-macros",
-  "version": "5.12.4",
+  "version": "5.13.1",
   "description": "Antora extensions and macros developed for Redpanda documentation.",
   "keywords": [
     "antora",
@@ -77,6 +77,7 @@
     "./extensions/convert-llms-to-txt": "./extensions/convert-llms-to-txt.js",
     "./extensions/convert-sitemap-to-markdown": "./extensions/convert-sitemap-to-markdown.js",
     "./extensions/modify-redirects": "./extensions/produce-redirects.js",
+    "./extensions/produce-redirects": "./extensions/produce-redirects.js",
     "./extensions/algolia-indexer/index": "./extensions/algolia-indexer/index.js",
     "./extensions/aggregate-terms": "./extensions/aggregate-terms.js",
     "./extensions/resolve-xrefs-in-attachments": "./extensions/resolve-xrefs-in-attachments.js",
@@ -91,7 +92,8 @@
     "./macros/badge": "./macros/badge.js",
     "./macros/enterprise": "./macros/enterprise.js",
     "./macros/prop": "./macros/prop.js",
-    "./extensions/property-page-index": "./extensions/property-page-index.js"
+    "./extensions/property-page-index": "./extensions/property-page-index.js",
+    "./extensions/render-property-descriptions": "./extensions/render-property-descriptions.js"
   },
   "files": [
     "extensions",
@@ -156,5 +158,8 @@
     "@web/dev-server": "^0.2.5",
     "jest": "^29.7.0",
     "jest-junit": "^16.0.0"
+  },
+  "peerDependencies": {
+    "@antora/asciidoc-loader": "^3.1.0"
   }
 }

package/tools/generate-cli-docs.js

@@ -126,9 +126,19 @@ function parseJSDocComments(sourceFile) {
   const content = fs.readFileSync(sourceFile, 'utf8');
   const comments = {};
 
-  // Regex to match JSDoc comments followed by command definitions
-  // Matches both top-level commands and automation.command()
-  const pattern = /\/\*\*\s*([\s\S]*?)\s*\*\/\s*(?:programCli|automation|validation)\s*\.command\(['"]([^'"]+)['"]\)/g;
+  // Regex to match JSDoc comments followed by command definitions.
+  // Matches both top-level commands and automation.command().
+  //
+  // The comment body is `(?:(?!\*\/)[\s\S])*?`, a tempered match that cannot
+  // cross a closing `*\/`, rather than a plain `[\s\S]*?`. With the plain
+  // version two ADJACENT JSDoc blocks merged into one match: the lazy body ran
+  // past the first block's terminator, because that terminator is followed by
+  // `/**` rather than by `.command(`, and kept going to the second. The
+  // extractor then took the FIRST @description it found, so the second command
+  // was documented with the first command's prose. That is exactly what
+  // happened to `generate migrate-rpcn-descriptions`, which shipped a
+  // regenerated section describing migrate-rpcn-metadata.
+  const pattern = /\/\*\*\s*((?:(?!\*\/)[\s\S])*?)\s*\*\/\s*(?:programCli|automation|validation)\s*\.command\(['"]([^'"]+)['"]\)/g;
 
   let match;
   while ((match = pattern.exec(content)) !== null) {
@@ -306,7 +316,8 @@ IMPORTANT: This documentation is auto-generated. Do not edit manually. Run \`npm
     'crd-spec',
     'bundle-openapi',
     'update-connect-version',
-    'migrate-rpcn-metadata'
+    'migrate-rpcn-metadata',
+    'migrate-rpcn-descriptions'
   ];
 
   generateSubcommands.forEach(subcmd => {

package/tools/property-extractor/compare-properties.js

@@ -15,6 +15,7 @@
  */
 
 const fs = require('fs');
+const bigIntJson = require('../../cli-utils/big-int-json');
 const path = require('path');
 const semver = require('semver');
 
@@ -109,7 +110,10 @@ function formatValue(value) {
     return `[${value.length} items]`;
   }
   if (typeof value === 'object') {
-    return JSON.stringify(value);
+    // bigIntJson, not JSON.stringify: a default/minimum/maximum value that's an
+    // object or array can itself carry an unsafe-integer BigInt from
+    // bigIntJson.parse(), which plain JSON.stringify throws on.
+    return bigIntJson.stringify(value);
   }
   if (typeof value === 'string') {
     return `"${value}"`;
@@ -393,7 +397,11 @@ function generateJsonReport(report, oldVersion, newVersion, outputPath) {
     details: report
   };
   
-  fs.writeFileSync(outputPath, JSON.stringify(jsonReport, null, 2));
+  // bigIntJson, not JSON.stringify: `report.details` carries oldDefault/newDefault
+  // values straight from bigIntJson.parse(), so an unsafe-integer default is a
+  // real BigInt here. Plain JSON.stringify throws on that ("Do not know how to
+  // serialize a BigInt") instead of writing the report.
+  fs.writeFileSync(outputPath, bigIntJson.stringify(jsonReport, 2));
   console.log(`📄 Detailed JSON report saved to: ${outputPath}`);
 }
 
@@ -424,8 +432,8 @@ function comparePropertyFiles(oldFilePath, newFilePath, oldVersion, newVersion,
     console.log(`   Old: ${oldFilePath}`);
     console.log(`   New: ${newFilePath}`);
     
-    const oldData = JSON.parse(fs.readFileSync(oldFilePath, 'utf8'));
-    const newData = JSON.parse(fs.readFileSync(newFilePath, 'utf8'));
+    const oldData = bigIntJson.parse(fs.readFileSync(oldFilePath, 'utf8'));
+    const newData = bigIntJson.parse(fs.readFileSync(newFilePath, 'utf8'));
     
     const report = compareProperties(oldData, newData, oldVersion, newVersion);
 
@@ -447,7 +455,10 @@ function comparePropertyFiles(oldFilePath, newFilePath, oldVersion, newVersion,
       });
 
       newData.properties = newProps;
-      fs.writeFileSync(newFilePath, JSON.stringify(newData, null, 2), 'utf8');
+      // bigIntJson: this rewrites the dataset in place, so a plain stringify
+      // corrupts the uint64/int64 limits on disk -- ahead of, and defeating, the
+      // lossless read in generate-handlebars-docs.js.
+      fs.writeFileSync(newFilePath, bigIntJson.stringify(newData, 2), 'utf8');
       console.log(`\nMerged ${report.removedDeprecatedProperties.length} removed deprecated properties back into ${newVersion} JSON`);
     }