@redpanda-data/docs-extensions-and-macros 5.12.3 → 5.13.0

package/cli-utils/big-int-json.js

@@ -0,0 +1,85 @@
+'use strict'
+
+/**
+ * JSON parse and stringify that do not destroy integers a JS number cannot hold.
+ *
+ * Redpanda's property data carries the uint64 and int64 limits as the maximum and
+ * minimum of numeric properties: 18446744073709551615, 9223372036854775807 and
+ * -9223372036854775808. The Python extractor writes them exactly, because Python
+ * integers are arbitrary precision. Every stage after that is JavaScript, where
+ * the largest exactly-representable integer is 2^53 - 1, so a plain
+ * `JSON.parse` silently rounds all three -- 18446744073709551615 becomes
+ * 18446744073709552000, which is not the uint64 limit and is not a value the
+ * server accepts.
+ *
+ * That had been happening on every published dataset since April: the reference
+ * pages, the tooltip attachment and the Connect catalog all showed rounded
+ * values, and none of them agreed with the source data.
+ *
+ * `parse` returns those integers as BigInt, so they print exactly and can be
+ * written back unchanged by `stringify`. BigInt rather than a string because it
+ * is unambiguous: a value that was a number in the source can be told apart from
+ * a string that was authored as one, which is what makes the round trip safe.
+ *
+ * Numbers inside the safe range are untouched and stay plain numbers.
+ *
+ * KNOWN LIMITATION: this is a regex over the raw text, not a real JSON scanner,
+ * so it does not track string/escape state. A value that is legitimate JSON
+ * text APPEARING INSIDE an escaped string -- for example a Bloblang example
+ * embedded as an example value, `"{\"delay_for_ns\":110839937000000000}"` --
+ * is indistinguishable from real JSON structure to this regex, which inserts a
+ * quoted sentinel at that position and corrupts the document. This is confirmed
+ * against the real, published Connect component attachment, not a theoretical
+ * risk: parsing it throws "Expected ',' or ']' after array element". Do not
+ * apply this module to Connect/Bloblang-shaped JSON, or to any JSON whose string
+ * values may themselves contain escaped JSON-looking text. It has been checked
+ * against every redpanda-properties-*.json and property-overrides.json this
+ * repo could find (~50 files, every published version) with no false match, and
+ * is applied only to property-shaped data for that reason -- not because the
+ * risk is theoretical elsewhere.
+ */
+
+// An integer literal too long to be safe, in JSON value position: after a colon,
+// an opening bracket or a comma, and followed by a delimiter. A digit run inside
+// a string cannot match, because a string value starts with a quote.
+const UNSAFE_INT_RX = /([:[,]\s*)(-?\d{16,})(?=\s*[,}\]])/g
+// Printable ASCII: a control character is not legal inside a JSON string, so a
+// sentinel containing one produces JSON that will not parse.
+const SENTINEL = '@@bigint:'
+const SENTINEL_RX = new RegExp(`"${SENTINEL}(-?\\d+)"`, 'g')
+
+/** True if the literal needs BigInt to survive a round trip. */
+function isUnsafeInteger (digits) {
+  return !Number.isSafeInteger(Number(digits))
+}
+
+/**
+ * Parse JSON, returning integers outside the safe range as BigInt.
+ *
+ * @param {string} text - JSON text.
+ * @returns {*} The parsed value.
+ */
+function parse (text) {
+  const shielded = String(text).replace(UNSAFE_INT_RX, (match, lead, digits) =>
+    isUnsafeInteger(digits) ? `${lead}"${SENTINEL}${digits}"` : match
+  )
+  return JSON.parse(shielded, (key, value) =>
+    typeof value === 'string' && value.startsWith(SENTINEL) ? BigInt(value.slice(SENTINEL.length)) : value
+  )
+}
+
+/**
+ * Stringify JSON, writing BigInt values back as bare integer literals.
+ *
+ * @param {*} value - Value to serialize.
+ * @param {number|string} [space] - Indentation, as JSON.stringify takes it.
+ * @returns {string} JSON text.
+ */
+function stringify (value, space) {
+  const json = JSON.stringify(value, (key, entry) =>
+    typeof entry === 'bigint' ? `${SENTINEL}${entry.toString()}` : entry, space
+  )
+  return json === undefined ? json : json.replace(SENTINEL_RX, '$1')
+}
+
+module.exports = { parse, stringify, isUnsafeInteger }

package/extensions/render-property-descriptions.js

@@ -0,0 +1,339 @@
+'use strict'
+
+/**
+ * Renders the `description` of every property in a
+ * redpanda-properties-<tag>.json attachment to HTML at build time, with
+ * Antora's own AsciiDoc loader and the site's own AsciiDoc config, and writes
+ * the result to `description_html`.
+ *
+ * Why this exists
+ * ---------------
+ * Property descriptions are authored in AsciiDoc and shipped as JSON, so
+ * something has to render them. The docs UI was doing it in the browser, with a
+ * hand-written parser covering seven constructs: backticks, prop:, config_ref:,
+ * glossterm:, link:, <<anchor>>, and xrefs pre-resolved by a regex extension.
+ *
+ * A whitelist of seven fails open. Everything else -- bold, italics, attribute
+ * references, character replacements, bare URLs, any macro this repo adds next
+ * -- reaches the reader as raw source, and nothing warns. In v26.2.1 that was 31
+ * of 689 properties, including seven that displayed a literal
+ * `include::reference:partial$internal-use-property.adoc[]` where an
+ * internal-use warning or a breaking-change notice should have been.
+ *
+ * Converting here removes the whole class of defect rather than the instances:
+ * Asciidoctor handles all of AsciiDoc, our own macros run as themselves (so
+ * glossterm's positional attributes cannot drift from macros/glossary.js), and
+ * a macro added next year works with no docs-ui change at all.
+ *
+ * Both fields ship. `description` stays because consumers read it -- including
+ * docs-ui versions older than the one that prefers `description_html`, which
+ * deploys separately. The output is minified to pay for the second field.
+ */
+
+const loadAsciiDoc = require('@antora/asciidoc-loader')
+const bigIntJson = require('../cli-utils/big-int-json')
+const { buildPageIndex, propertyAnchor } = require('../macros/prop')
+const { raiseListenerLimit } = require('./util/raise-listener-limit')
+
+const PROPERTIES_JSON_RX = /^redpanda-properties-(v\d+\.\d+\.\d+(?:-[\w.]+)?)\.json$/
+// xref:target[label] as authored in a description. The .adoc extension is
+// optional: one real description writes xref:develop:transactions#tips[...].
+const XREF_RX = /xref:([^[\]\s#]+?)(?:#([^[\]\s]*))?\[([^\]]*)\]/g
+// <<anchor>> and <<anchor,display text>>, as authored in the description. The
+// anchor must start with a letter or underscore: property anchors always do,
+// and without that a C++ shift expression in a code span (`1<<20>>`) was read as
+// a reference, rewritten, and then reported as a broken anchor.
+const INTERNAL_REF_RX = /<<([A-Za-z_][\w.-]*)(?:,\s*([^<>]*?))?>>/g
+
+module.exports.register = function () {
+  raiseListenerLimit(this)
+  const logger = this.getLogger('render-property-descriptions')
+
+  // The anchor index has to be built here, while page sources still exist:
+  // convert-documents.js deletes page.src.contents once conversion finishes
+  // (unless keepSource), and buildPageIndex reads that source to find which page
+  // documents which property. Building it at documentsConverted worked only
+  // because property-page-index happened to warm the same cache earlier, and
+  // without that every <<anchor>> silently degraded to plain text.
+  const anchorIndexes = new Map()
+  this.once('contentClassified', ({ contentCatalog }) => {
+    for (const attachment of propertyAttachments(contentCatalog)) {
+      const properties = readProperties(attachment, logger)
+      if (!properties) continue
+      const key = `${attachment.src.component}@${attachment.src.version || ''}`
+      if (anchorIndexes.has(key)) continue
+      anchorIndexes.set(key, anchorIndex(contentCatalog, attachment, properties, logger))
+    }
+  })
+
+  // Conversion itself waits for documentsConverted: page aliases and the rest of
+  // what page conversion relies on are only fully registered by then. Hooking
+  // earlier left an xref to an aliased page rendering as
+  // class="xref unresolved" while the same xref resolved fine on a real page.
+  // Attachments are still published after this event.
+  this.once('documentsConverted', ({ contentCatalog, siteAsciiDocConfig }) => {
+    const attachments = propertyAttachments(contentCatalog)
+    if (!attachments.length) return
+
+    for (const attachment of attachments) {
+      const where = `${attachment.src.component}@${attachment.src.version || 'unversioned'}`
+      const data = readDataset(attachment, logger)
+      if (!data) continue
+      const properties = data.properties
+
+      // Convert "as" a real page rather than a fabricated one. A hand-built
+      // file.pub is missing fields Antora's resolver needs, and resolving an
+      // xref to another module then throws instead of resolving.
+      const hostPage = findHostPage(contentCatalog, attachment)
+      if (!hostPage) {
+        logger.debug(`${where}: no reference page to convert descriptions as, skipped`)
+        continue
+      }
+      const anchors = anchorIndexes.get(`${attachment.src.component}@${attachment.src.version || ''}`) || new Map()
+
+      // Antora converts each page with its component version's own config
+      // (convert-documents.js builds one per component version and prefers it
+      // over the site config), so attributes declared in antora.yml -- env-cloud
+      // among them, which five descriptions branch on -- only exist there.
+      const componentVersion = contentCatalog.getComponentVersion
+        ? contentCatalog.getComponentVersion(attachment.src.component, attachment.src.version)
+        : undefined
+      const asciidocConfig = (componentVersion && componentVersion.asciidoc) || siteAsciiDocConfig
+
+      let rendered = 0
+      let failed = 0
+      const brokenAnchors = new Set()
+      const unresolved = new Set()
+      const xrefCache = new Map()
+      for (const [name, entry] of Object.entries(properties)) {
+        const description = entry && entry.description
+        if (typeof description !== 'string' || !description.trim()) continue
+        try {
+          const source = resolveInternalRefs(description, anchors, (anchor) => brokenAnchors.add(anchor))
+          const html = render(source, hostPage, contentCatalog, asciidocConfig)
+          if (html == null) continue
+          entry.description_html = html
+          // The attachment is a published download, and `xref:...[]` means
+          // nothing to someone reading the JSON on its own -- resolving it is
+          // why resolve-xrefs-in-attachments existed for these files. This
+          // extension owns that job for property datasets now, so it has to keep
+          // doing it, using Antora's resolver rather than rewriting paths by
+          // hand. Descriptions stay AsciiDoc apart from the resolved anchors,
+          // which is the shape consumers already read.
+          entry.description = resolveXrefs(
+            description, hostPage, contentCatalog, asciidocConfig, xrefCache, (macro) => unresolved.add(macro)
+          )
+          rendered++
+        } catch (error) {
+          // One description that will not convert must not cost the other 680.
+          failed++
+          logger.warn(`${where} ${name}: description could not be converted, so its tooltip falls back to the raw source (${error.message})`)
+        }
+      }
+
+      if (rendered) attachment.contents = Buffer.from(bigIntJson.stringify(data))
+      logger.info(`${where}: rendered ${rendered} property descriptions to HTML${failed ? `, ${failed} failed` : ''}`)
+      if (unresolved.size) {
+        logger.warn(
+          `${where}: ${unresolved.size} xref target(s) in property descriptions could not be resolved, so they stay as raw macros in the published attachment: ` +
+          `${[...unresolved].sort().join(', ')}.`
+        )
+      }
+      if (brokenAnchors.size) {
+        logger.warn(
+          `${where}: ${brokenAnchors.size} <<anchor>> reference(s) in property descriptions name no documented property, so they render as plain text: ` +
+          `${[...brokenAnchors].sort().join(', ')}. Property anchors replace dots with hyphens, so redpanda.storage.mode is <<redpanda-storage-mode>>. ` +
+          'Fix them in the description or in docs-data/property-overrides.json.'
+        )
+      }
+    }
+  })
+}
+
+function basename (file) {
+  return file.src.relative.split('/').pop()
+}
+
+function propertyAttachments (contentCatalog) {
+  return (contentCatalog.findBy({ family: 'attachment' }) || []).filter(
+    (file) => file.src.module === 'reference' && PROPERTIES_JSON_RX.test(basename(file))
+  )
+}
+
+/**
+ * Parse one dataset, or report why it is unusable and return undefined. Left
+ * as-is on failure: the prop macro reports an unusable dataset with the detail,
+ * and failing the whole build over one bad attachment helps nobody.
+ */
+function readDataset (attachment, logger) {
+  const where = `${attachment.src.component}@${attachment.src.version || 'unversioned'}`
+  let data
+  try {
+    data = bigIntJson.parse(attachment.contents.toString())
+  } catch (error) {
+    logger.warn(`${where}: ${basename(attachment)} is not valid JSON, descriptions not rendered (${error.message})`)
+    return undefined
+  }
+  const properties = data && data.properties
+  if (!properties || typeof properties !== 'object' || Array.isArray(properties)) {
+    logger.warn(`${where}: ${basename(attachment)} has no usable 'properties' object, descriptions not rendered`)
+    return undefined
+  }
+  return data
+}
+
+function readProperties (attachment, logger) {
+  const data = readDataset(attachment, logger)
+  return data ? data.properties : undefined
+}
+
+/**
+ * A real page in the attachment's own component and reference module, preferring
+ * one under properties/ so includes and relative xrefs resolve from where the
+ * property docs actually live.
+ */
+function findHostPage (contentCatalog, attachment) {
+  const pages = contentCatalog.findBy({
+    component: attachment.src.component,
+    version: attachment.src.version,
+    family: 'page',
+  }) || []
+  return (
+    pages.find((f) => f.src.module === 'reference' && /^properties\//.test(f.src.relative)) ||
+    pages.find((f) => f.src.module === 'reference') ||
+    undefined
+  )
+}
+
+/**
+ * anchor -> xref target, for the <<anchor>> references that point at another
+ * documented property. Built from the same page index the prop macro links
+ * with, so a description and a prop: call agree about which page documents what.
+ */
+function anchorIndex (contentCatalog, attachment, properties, logger) {
+  const where = `${attachment.src.component}@${attachment.src.version || 'unversioned'}`
+  const index = new Map()
+  let pages
+  try {
+    pages = buildPageIndex(contentCatalog, attachment.src.component, properties, attachment.src.version)
+  } catch (error) {
+    // Returning an empty index silently made a thrown index and an empty one
+    // indistinguishable, and every <<anchor>> then degraded to plain text with
+    // no explanation.
+    logger.warn(`${where}: could not index which page documents each property, so <<anchor>> references in descriptions render as plain text (${error.message})`)
+    return index
+  }
+  if (!pages.size) {
+    logger.warn(`${where}: no reference page was found to document any property, so <<anchor>> references in descriptions render as plain text.`)
+    return index
+  }
+  for (const [name, entry] of pages) {
+    if (!entry || !entry.page) continue
+    index.set(propertyAnchor(name), { page: entry.page, name })
+  }
+  return index
+}
+
+/**
+ * Rewrite <<anchor>> to an xref before conversion.
+ *
+ * Asciidoctor renders <<anchor>> as a same-page fragment link, which is right
+ * on the reference page and wrong in a tooltip: a tooltip is shown on an
+ * arbitrary page, where that fragment does not exist. Pointing it at the page
+ * that documents the property keeps the link working wherever the tooltip
+ * appears, and lets Antora resolve the URL rather than us building one.
+ */
+function resolveInternalRefs (description, anchors, report) {
+  return description.replace(INTERNAL_REF_RX, (match, anchor, display) => {
+    const text = (display || '').trim()
+    const target = anchors.get(anchor)
+    if (target) return `xref:reference:${target.page}.adoc#${anchor}[${attrlistValue(text || `\`${target.name}\``)}]`
+    // The anchor names no documented property, so there is nothing to link to.
+    // Asciidoctor would render a same-page fragment, and a tooltip is shown on
+    // arbitrary pages, so that link would go nowhere and still invite a click --
+    // the same reasoning the prop macro applies to a page it cannot verify.
+    // Render the text and report the anchor so it gets fixed at the source.
+    report(anchor)
+    return text || `\`${anchor}\``
+  })
+}
+
+/**
+ * Display text as a single positional attribute. An unescaped `]` ends the
+ * attribute list, so `<<prop,the flag [beta] option>>` published a truncated
+ * link with ` option]` outside it; and a bare `=` makes Asciidoctor read the
+ * value as a named attribute, which dropped the label entirely and applied part
+ * of it as a CSS role. Escaping the bracket and quoting when there is an equals
+ * sign round-trips every combination of brackets, equals, quotes and commas.
+ */
+function attrlistValue (text) {
+  const escaped = text.replace(/]/g, '\\]')
+  return /=/.test(escaped) ? `"${escaped.replace(/"/g, '\\"')}"` : escaped
+}
+
+/**
+ * Rewrite xref macros in a description to resolved links, so the published
+ * attachment is useful on its own.
+ *
+ * The conversion is done by Antora, one macro at a time, rather than by building
+ * a URL here: that way the page resolution, the fragment, and any id rules from
+ * the site's AsciiDoc config are whatever the site itself would produce, and
+ * cannot drift from the pages the links point at. The previous implementation
+ * rewrote paths with regexes and converted underscores in the fragment to
+ * hyphens -- but property page anchors keep their underscores
+ * (`id="log_cleanup_policy"`), so all 25 of its fragment links in the published
+ * v26.2.1 data were dead.
+ *
+ * Conversions are cached per macro, because the same xref appears in many
+ * descriptions.
+ */
+function resolveXrefs (description, hostPage, contentCatalog, asciidocConfig, cache, report) {
+  return description.replace(XREF_RX, (match) => {
+    if (!cache.has(match)) cache.set(match, convertXref(match, hostPage, contentCatalog, asciidocConfig))
+    const anchor = cache.get(match)
+    if (!anchor) {
+      report(match)
+      // Leave the macro alone rather than emitting a link to nowhere: a reader
+      // of the raw JSON can still see what was meant.
+      return match
+    }
+    return anchor
+  })
+}
+
+/** One xref macro, converted by Antora. Returns undefined if it does not resolve. */
+function convertXref (macro, hostPage, contentCatalog, asciidocConfig) {
+  let html
+  try {
+    html = render(macro, hostPage, contentCatalog, asciidocConfig)
+  } catch (error) {
+    return undefined
+  }
+  if (!html) return undefined
+  const anchor = /<a\b[^>]*>[\s\S]*?<\/a>/.exec(html)
+  // Antora marks a target it could not resolve, and that is not a link worth
+  // publishing into a file people read on its own.
+  if (!anchor || /class="[^"]*\bunresolved\b/.test(anchor[0])) return undefined
+  return anchor[0]
+}
+
+/** Convert one description with the site's converter, as the host page. */
+function render (source, hostPage, contentCatalog, siteAsciiDocConfig) {
+  const file = {
+    contents: Buffer.from(source),
+    mediaType: 'text/asciidoc',
+    src: { ...hostPage.src },
+    pub: { ...hostPage.pub },
+  }
+  // relativizeResourceRefs: false gives site-root-relative URLs. The default
+  // computes them relative to the converting file, which is wrong for a tooltip
+  // shown on some other page.
+  const doc = loadAsciiDoc(file, contentCatalog, { ...siteAsciiDocConfig, relativizeResourceRefs: false })
+  const blocks = doc.getBlocks()
+  if (!blocks.length) return doc.convert() || null
+  // A single paragraph gives inline HTML with no <p> wrapper, which is what a
+  // tooltip wants. Anything richer -- an included admonition, a list, several
+  // paragraphs -- keeps its block markup, and the UI styles it.
+  if (blocks.length === 1 && blocks[0].getContext() === 'paragraph') return blocks[0].getContent()
+  return doc.convert()
+}

package/extensions/resolve-xrefs-in-attachments.js

@@ -20,6 +20,8 @@ const { raiseListenerLimit } = require('./util/raise-listener-limit')
  * - xref:path.adoc[] (empty display uses page title or path)
  */
 
+const PROPERTIES_JSON_RX = /^redpanda-properties-(v\d+\.\d+\.\d+(?:-[\w.]+)?)\.json$/
+
 module.exports.register = function () {
   raiseListenerLimit(this)
   const logger = this.getLogger('resolve-xrefs-in-attachments')
@@ -29,7 +31,15 @@ module.exports.register = function () {
     const allAttachments = contentCatalog.findBy({ family: 'attachment' })
     const jsonAttachments = allAttachments.filter((att) => {
       const path = att.src?.path || att.out?.path || ''
-      return path.endsWith('.json')
+      if (!path.endsWith('.json')) return false
+      // Property datasets are handled by render-property-descriptions, which
+      // converts their descriptions with Asciidoctor instead of rewriting xrefs
+      // by hand. Both running double-processed 53 descriptions in v26.2.1: the
+      // anchor inserted here was escaped to <a href= by the converter. This
+      // extension still serves every other JSON attachment, including the
+      // Connect component catalog, which carries far more xrefs than the
+      // property files do.
+      return !PROPERTIES_JSON_RX.test(path.split('/').pop())
     })
 
     if (!jsonAttachments.length) {
@@ -72,6 +82,19 @@ function processJsonAttachment (attachment, contentCatalog, logger) {
 
   let data
   try {
+    // Deliberately plain JSON.parse, not bigIntJson: this function rewrites
+    // whatever JSON attachment it is handed, including the Connect component
+    // catalog, which contains example config text as an ESCAPED JSON-shaped
+    // string -- "{\"delay_for_ns\":110839937000000000}" -- indistinguishable
+    // by regex from real JSON value position. bigIntJson's value-position regex
+    // matched inside that escaped string and inserted stray quotes, which is not
+    // a rounding error, it is invalid JSON: confirmed to throw
+    // "Expected ',' or ']' after array element" on the real, currently-published
+    // connect-4.103.1.json attachment. A plain round trip still silently rounds
+    // large integers (the pre-existing, understood defect this file has always
+    // had), which is the safer failure mode until a real JSON scanner -- one
+    // that tracks string/escape state rather than pattern-matching text --
+    // replaces the regex here.
     data = JSON.parse(contentStr)
   } catch (err) {
     logger.debug(`Skipping invalid JSON: ${attachment.src?.path}`)

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@redpanda-data/docs-extensions-and-macros",
-  "version": "5.12.3",
+  "version": "5.13.0",
   "description": "Antora extensions and macros developed for Redpanda documentation.",
   "keywords": [
     "antora",
@@ -77,6 +77,7 @@
     "./extensions/convert-llms-to-txt": "./extensions/convert-llms-to-txt.js",
     "./extensions/convert-sitemap-to-markdown": "./extensions/convert-sitemap-to-markdown.js",
     "./extensions/modify-redirects": "./extensions/produce-redirects.js",
+    "./extensions/produce-redirects": "./extensions/produce-redirects.js",
     "./extensions/algolia-indexer/index": "./extensions/algolia-indexer/index.js",
     "./extensions/aggregate-terms": "./extensions/aggregate-terms.js",
     "./extensions/resolve-xrefs-in-attachments": "./extensions/resolve-xrefs-in-attachments.js",
@@ -91,7 +92,8 @@
     "./macros/badge": "./macros/badge.js",
     "./macros/enterprise": "./macros/enterprise.js",
     "./macros/prop": "./macros/prop.js",
-    "./extensions/property-page-index": "./extensions/property-page-index.js"
+    "./extensions/property-page-index": "./extensions/property-page-index.js",
+    "./extensions/render-property-descriptions": "./extensions/render-property-descriptions.js"
   },
   "files": [
     "extensions",
@@ -156,5 +158,8 @@
     "@web/dev-server": "^0.2.5",
     "jest": "^29.7.0",
     "jest-junit": "^16.0.0"
+  },
+  "peerDependencies": {
+    "@antora/asciidoc-loader": "^3.1.0"
   }
 }

package/tools/property-extractor/compare-properties.js

@@ -15,6 +15,7 @@
  */
 
 const fs = require('fs');
+const bigIntJson = require('../../cli-utils/big-int-json');
 const path = require('path');
 const semver = require('semver');
 
@@ -109,7 +110,10 @@ function formatValue(value) {
     return `[${value.length} items]`;
   }
   if (typeof value === 'object') {
-    return JSON.stringify(value);
+    // bigIntJson, not JSON.stringify: a default/minimum/maximum value that's an
+    // object or array can itself carry an unsafe-integer BigInt from
+    // bigIntJson.parse(), which plain JSON.stringify throws on.
+    return bigIntJson.stringify(value);
   }
   if (typeof value === 'string') {
     return `"${value}"`;
@@ -393,7 +397,11 @@ function generateJsonReport(report, oldVersion, newVersion, outputPath) {
     details: report
   };
   
-  fs.writeFileSync(outputPath, JSON.stringify(jsonReport, null, 2));
+  // bigIntJson, not JSON.stringify: `report.details` carries oldDefault/newDefault
+  // values straight from bigIntJson.parse(), so an unsafe-integer default is a
+  // real BigInt here. Plain JSON.stringify throws on that ("Do not know how to
+  // serialize a BigInt") instead of writing the report.
+  fs.writeFileSync(outputPath, bigIntJson.stringify(jsonReport, 2));
   console.log(`📄 Detailed JSON report saved to: ${outputPath}`);
 }
 
@@ -424,8 +432,8 @@ function comparePropertyFiles(oldFilePath, newFilePath, oldVersion, newVersion,
     console.log(`   Old: ${oldFilePath}`);
     console.log(`   New: ${newFilePath}`);
     
-    const oldData = JSON.parse(fs.readFileSync(oldFilePath, 'utf8'));
-    const newData = JSON.parse(fs.readFileSync(newFilePath, 'utf8'));
+    const oldData = bigIntJson.parse(fs.readFileSync(oldFilePath, 'utf8'));
+    const newData = bigIntJson.parse(fs.readFileSync(newFilePath, 'utf8'));
     
     const report = compareProperties(oldData, newData, oldVersion, newVersion);
 
@@ -447,7 +455,10 @@ function comparePropertyFiles(oldFilePath, newFilePath, oldVersion, newVersion,
       });
 
       newData.properties = newProps;
-      fs.writeFileSync(newFilePath, JSON.stringify(newData, null, 2), 'utf8');
+      // bigIntJson: this rewrites the dataset in place, so a plain stringify
+      // corrupts the uint64/int64 limits on disk -- ahead of, and defeating, the
+      // lossless read in generate-handlebars-docs.js.
+      fs.writeFileSync(newFilePath, bigIntJson.stringify(newData, 2), 'utf8');
       console.log(`\nMerged ${report.removedDeprecatedProperties.length} removed deprecated properties back into ${newVersion} JSON`);
     }
 

package/tools/property-extractor/generate-handlebars-docs.js

@@ -2,6 +2,7 @@ const AUTOGEN_NOTICE = '// This content is autogenerated. Do not edit manually.
 'use strict';
 
 const fs = require('fs');
+const bigIntJson = require('../../cli-utils/big-int-json');
 const path = require('path');
 const handlebars = require('handlebars');
 const helpers = require('./helpers');
@@ -425,7 +426,11 @@ function convertConfigRefsToProp(text) {
  * Main generator
  */
 function generateAllDocs(inputFile, outputDir) {
-  const data = JSON.parse(fs.readFileSync(inputFile, 'utf8'));
+  // bigIntJson, not JSON.parse: the uint64 and int64 limits carried as property
+  // maxima are beyond what a JS number holds, so a plain parse rounded them and
+  // the generated partials -- and therefore the published reference pages --
+  // showed 18446744073709552000 where the data says 18446744073709551615.
+  const data = bigIntJson.parse(fs.readFileSync(inputFile, 'utf8'));
   // Support both 'properties' (from property_extractor.py) and 'topic_properties' (from topic_property_extractor.py)
   const properties = data.properties || data.topic_properties || {};
 

package/tools/redpanda-connect/connector-binary-analyzer.js

@@ -1,5 +1,5 @@
 const octokit = require('../../cli-utils/octokit-client');
-const { execSync, spawnSync } = require('child_process');
+const { execFileSync, spawnSync } = require('child_process');
 const fs = require('fs');
 const path = require('path');
 const os = require('os');
@@ -16,6 +16,31 @@ const https = require('https');
 const REPO_OWNER = 'redpanda-data';
 const REPO_NAME = 'connect';
 
+// Release asset prefixes per binary flavour. The plain OSS build is the one
+// `rpk connect install` puts on PATH, so fetching that asset directly yields
+// the same connector schema without passing a version through rpk's
+// --connect-version validation.
+const BINARY_ASSET_PREFIXES = {
+  oss: 'redpanda-connect',
+  cloud: 'redpanda-connect-cloud',
+  cgo: 'redpanda-connect-cgo'
+};
+
+/**
+ * Resolve a binary flavour to its release asset prefix.
+ * @param {string} binaryType - 'oss', 'cloud' or 'cgo'
+ * @returns {string} The asset name prefix
+ */
+function resolveAssetPrefix (binaryType) {
+  const prefix = BINARY_ASSET_PREFIXES[binaryType];
+  if (!prefix) {
+    throw new Error(
+      `Unknown binary type "${binaryType}" (expected one of ${Object.keys(BINARY_ASSET_PREFIXES).join(', ')})`
+    );
+  }
+  return prefix;
+}
+
 /**
  * Get platform-specific binary name
  * @returns {Object} Platform and arch info
@@ -46,10 +71,15 @@ function getPlatformInfo() {
 
 /**
  * Get the latest release version from GitHub
- * @param {string} binaryType - Type of binary ('cloud' or 'cgo')
+ * @param {string} binaryType - Type of binary ('oss', 'cloud' or 'cgo')
  * @returns {Promise<string>} Version string (e.g., "4.76.0")
  */
 async function getLatestVersion(binaryType = 'cloud') {
+  // Resolved outside the try so an unknown flavour surfaces its own message
+  // instead of being wrapped as a failure to reach the GitHub API. tools/ is
+  // published, so callers outside this repo see this error.
+  const prefix = resolveAssetPrefix(binaryType);
+
   try {
     const { data: releases } = await octokit.repos.listReleases({
       owner: REPO_OWNER,
@@ -57,12 +87,12 @@ async function getLatestVersion(binaryType = 'cloud') {
       per_page: 20
     });
 
-    const prefix = binaryType === 'cloud' ? 'redpanda-connect-cloud' : 'redpanda-connect-cgo';
-
-    // Find the latest release with the specified binary type
+    // Find the latest release with the specified binary type. Match the
+    // separator too: every prefix starts with 'redpanda-connect', so a bare
+    // startsWith on the OSS prefix would also match the cloud and cgo assets.
     for (const release of releases) {
       const asset = release.assets.find(a =>
-        a.name.startsWith(prefix) &&
+        a.name.startsWith(`${prefix}_`) &&
         a.name.includes('linux') &&
         a.name.includes('amd64')
       );
@@ -124,9 +154,99 @@ function downloadFile(url, destPath) {
   });
 }
 
+/**
+ * Directory that holds downloaded Connect binaries for the life of the process.
+ *
+ * Binaries are ~320 MB extracted, so they never belong in docs-data: the
+ * consuming repo commits that directory wholesale, and a run killed before its
+ * cleanup would leave a blob over GitHub's 100 MB file limit staged for the
+ * auto-docs PR. One directory for the whole run also lets downloadBinary's
+ * skip-if-present check serve repeat requests: the multi-release loop asks for
+ * every interior version twice (as the new side of one diff pair and the old
+ * side of the next), and a per-call directory made that a second download.
+ *
+ * @returns {string} Path to the process-wide binary cache directory
+ */
+let binaryCacheDir = null;
+function getBinaryCacheDir () {
+  if (binaryCacheDir && fs.existsSync(binaryCacheDir)) return binaryCacheDir;
+
+  binaryCacheDir = fs.mkdtempSync(path.join(os.tmpdir(), 'rpcn-binaries-'));
+  const created = binaryCacheDir;
+  process.on('exit', () => {
+    try {
+      fs.rmSync(created, { recursive: true, force: true });
+    } catch (err) {
+      // Nothing useful to do while exiting; the OS reclaims tmpdir anyway.
+    }
+  });
+  return binaryCacheDir;
+}
+
+// Cap the cache: a backfill can touch dozens of versions, but no more than a
+// handful of binaries are live at once (the two OSS versions of the current
+// diff pair plus the cloud and cgo binaries analysed alongside them). Without
+// a cap, sharing one directory would trade repeat downloads for unbounded disk.
+const BINARY_CACHE_LIMIT = 6;
+const binaryCacheUse = [];
+
+/**
+ * Record a binary as most recently used and evict the oldest beyond the cap.
+ * @param {string} destPath - Path to the binary just downloaded or reused
+ */
+function touchBinaryCache (destPath) {
+  const existing = binaryCacheUse.indexOf(destPath);
+  if (existing !== -1) binaryCacheUse.splice(existing, 1);
+  binaryCacheUse.push(destPath);
+
+  while (binaryCacheUse.length > BINARY_CACHE_LIMIT) {
+    const evicted = binaryCacheUse.shift();
+    try {
+      fs.rmSync(evicted, { force: true });
+    } catch (err) {
+      console.warn(`Warning: could not evict cached binary ${evicted}: ${err.message}`);
+    }
+  }
+}
+
+/**
+ * Extract the Connect binary out of a release tarball.
+ *
+ * Extraction goes into an empty directory of its own because the archive's
+ * inner file name is not the destination name, and the destination directory
+ * holds other flavours and versions whose names also contain
+ * 'redpanda-connect'. Scanning the shared directory instead would pick a
+ * neighbour in readdir order and analyse, say, the cloud binary as the cgo one.
+ *
+ * @param {string} tarballPath - Path to the downloaded .tar.gz
+ * @param {string} destPath - Path the extracted binary should end up at
+ * @returns {string} destPath
+ */
+function extractBinaryFromTarball (tarballPath, destPath) {
+  const extractDir = fs.mkdtempSync(`${destPath}.extract-`);
+  try {
+    execFileSync('tar', ['-xzf', tarballPath, '-C', extractDir], { stdio: 'ignore' });
+
+    const candidates = fs.readdirSync(extractDir)
+      .filter(name => name.includes('redpanda-connect') && !name.endsWith('.tar.gz'))
+      .filter(name => fs.statSync(path.join(extractDir, name)).isFile())
+      .sort();
+
+    if (candidates.length === 0) {
+      throw new Error('Binary not found after extraction');
+    }
+
+    fs.renameSync(path.join(extractDir, candidates[0]), destPath);
+    fs.chmodSync(destPath, 0o755);
+    return destPath;
+  } finally {
+    fs.rmSync(extractDir, { recursive: true, force: true });
+  }
+}
+
 /**
  * Download a binary from GitHub releases
- * @param {string} binaryType - 'cloud' or 'cgo'
+ * @param {string} binaryType - 'oss', 'cloud' or 'cgo'
  * @param {string} version - Version to download (e.g., "4.76.0")
  * @param {string} destDir - Destination directory
  * @returns {Promise<string>} Path to downloaded binary
@@ -136,13 +256,14 @@ async function downloadBinary(binaryType, version, destDir) {
   const platformInfo = binaryType === 'cgo'
     ? { platform: 'linux', arch: 'amd64' }
     : getPlatformInfo();
-  const prefix = binaryType === 'cloud' ? 'redpanda-connect-cloud' : 'redpanda-connect-cgo';
+  const prefix = resolveAssetPrefix(binaryType);
   const binaryName = `${prefix}-${version}-${platformInfo.platform}-${platformInfo.arch}`;
   const destPath = path.join(destDir, binaryName);
 
   // Skip download if already exists
   if (fs.existsSync(destPath)) {
     console.log(`✓ ${binaryType.toUpperCase()} binary already downloaded: ${binaryName}`);
+    touchBinaryCache(destPath);
     return destPath;
   }
 
@@ -170,35 +291,12 @@ async function downloadBinary(binaryType, version, destDir) {
 
     // Extract the binary
     console.log(`Extracting ${binaryType.toUpperCase()} binary...`);
-    execSync(`tar -xzf "${tarballPath}" -C "${destDir}"`, { stdio: 'ignore' });
-
-    // Find the extracted binary
-    const extractedFiles = fs.readdirSync(destDir);
-    let binaryPath = null;
-
-    for (const file of extractedFiles) {
-      const fullPath = path.join(destDir, file);
-      if (fs.statSync(fullPath).isFile() && file.includes('redpanda-connect') && !file.endsWith('.tar.gz')) {
-        binaryPath = fullPath;
-        break;
-      }
-    }
-
-    if (!binaryPath) {
-      throw new Error('Binary not found after extraction');
-    }
-
-    // Rename to standard name if needed
-    if (binaryPath !== destPath) {
-      fs.renameSync(binaryPath, destPath);
-    }
-
-    // Make executable
-    fs.chmodSync(destPath, 0o755);
+    extractBinaryFromTarball(tarballPath, destPath);
 
     // Clean up tarball
     fs.unlinkSync(tarballPath);
 
+    touchBinaryCache(destPath);
     console.log(`Done: Downloaded ${binaryType.toUpperCase()} binary: ${binaryName}`);
     return destPath;
   } catch (error) {
@@ -435,9 +533,10 @@ async function analyzeAllBinaries(ossVersion, cloudVersion = null, dataDir = nul
 
   const docsDataDir = dataDir || path.resolve(process.cwd(), 'docs-data');
 
-  // Use temp directory for binaries (not docs-data)
-  const os = require('os');
-  const binaryDir = fs.mkdtempSync(path.join(os.tmpdir(), 'rpcn-binaries-'));
+  // Use temp directory for binaries (not docs-data). Shared with every other
+  // download in this process, so a multi-release run fetches each cloud and cgo
+  // binary once instead of once per diff pair.
+  const binaryDir = getBinaryCacheDir();
 
   console.log(`📁 Using temp directory for binaries: ${binaryDir}`);
 
@@ -508,13 +607,8 @@ async function analyzeAllBinaries(ossVersion, cloudVersion = null, dataDir = nul
     console.log('⏩ Skipping cgo binary analysis');
   }
 
-  // Clean up temp binaries
-  try {
-    fs.rmSync(binaryDir, { recursive: true, force: true });
-    console.log(`🧹 Cleaned up temp binaries`);
-  } catch (err) {
-    console.warn(`Warning: Could not clean up temp directory: ${err.message}`);
-  }
+  // Binaries stay in the shared cache so the next diff pair can reuse them; the
+  // directory is removed when the process exits (see getBinaryCacheDir).
 
   // IMPORTANT: cgoOnly contains connectors that exist ONLY in the cgo binary
   // and NOT in the regular OSS binary (rpk connect list). These are the
@@ -566,6 +660,11 @@ module.exports = {
   analyzeAllBinaries,
   findCgoOnlyConnectors,
   getLatestVersion,
+  getPlatformInfo,
+  resolveAssetPrefix,
+  getBinaryCacheDir,
+  extractBinaryFromTarball,
+  downloadBinary,
   downloadCloudBinary,
   downloadCgoBinary,
   getConnectorList,

package/tools/redpanda-connect/generate-rpcn-connector-docs.js

@@ -5,7 +5,7 @@ const path = require('path');
 const handlebars = require('handlebars');
 const yaml = require('yaml');
 const helpers = require('./helpers');
-const { lostMetadataSections } = require('./metadata-utils.js');
+const { lostMetadataSections, resolvePageTypeDir } = require('./metadata-utils.js');
 
 // Register each helper under handlebars, verifying that it’s a function
 Object.entries(helpers).forEach(([name, fn]) => {
@@ -730,11 +730,7 @@ function backfillPageDescriptions (connectorData, { pagesRoot, dryRun = false }
 
   for (const [dataKey, items] of Object.entries(connectorData)) {
     if (!PAGE_TYPE_DIRS.has(dataKey) || !Array.isArray(items)) continue;
-    // The data uses 'rate-limits' while the pages directory is
-    // 'rate_limits'. Resolve whichever spelling exists on disk.
-    const typeDir = fs.existsSync(path.join(root, dataKey))
-      ? dataKey
-      : dataKey.replace(/-/g, '_');
+    const typeDir = resolvePageTypeDir(root, dataKey);
     for (const item of items) {
       if (!item || !item.name) continue;
       const pagePath = path.join(root, typeDir, `${item.name}.adoc`);

package/tools/redpanda-connect/metadata-utils.js

@@ -1,5 +1,8 @@
 'use strict';
 
+const fs = require('fs');
+const path = require('path');
+
 /**
  * Utilities for extracting the "== Metadata" section out of a connector's
  * `description` prose so it can be emitted as a regenerated partial (like the
@@ -87,6 +90,31 @@ function typeDirFor (item) {
   return type.endsWith('s') ? type : `${type}s`;
 }
 
+/**
+ * Resolve the on-disk page/partial directory for a connector data key.
+ *
+ * The Connect binary emits the rate-limit family as `rate-limits` while the
+ * directory in rp-connect-docs is `rate_limits`, so the data key and the
+ * directory name disagree for exactly one family. Resolve against what is
+ * actually on disk rather than encoding either spelling: a caller that guesses
+ * `rate-limits` reports every existing rate-limit page as missing and then
+ * drafts a duplicate beside it, in a directory update-nav.js's regex does not
+ * match, re-added on every run.
+ *
+ * @param {string} root directory the type directories live under
+ * @param {string} dataKey the data key, for example `inputs` or `rate-limits`
+ * @returns {string} the directory name that exists, preferring the data key
+ */
+function resolvePageTypeDir (root, dataKey) {
+  if (!root || !dataKey) return dataKey || '';
+  if (fs.existsSync(path.join(root, dataKey))) return dataKey;
+  const underscored = dataKey.replace(/-/g, '_');
+  if (underscored !== dataKey && fs.existsSync(path.join(root, underscored))) return underscored;
+  const hyphenated = dataKey.replace(/_/g, '-');
+  if (hyphenated !== dataKey && fs.existsSync(path.join(root, hyphenated))) return hyphenated;
+  return dataKey;
+}
+
 /**
  * Build the Antora include directive for a connector's metadata partial.
  * @param {object} item connector data with `type`/`typeDir` and `name`
@@ -170,6 +198,7 @@ function lostMetadataSections (oldContent, newContent) {
 
 module.exports = {
   locateMetadata,
+  resolvePageTypeDir,
   extractMetadata,
   typeDirFor,
   metadataIncludeLine,

package/tools/redpanda-connect/rpcn-connector-docs-handler.js

@@ -11,6 +11,7 @@ const { getRpkConnectVersion, printDeltaReport } = require('./report-delta')
 const { discoverIntermediateReleases, findCloudVersionForDate } = require('./github-release-utils')
 const { analyzeAllBinaries } = require('./connector-binary-analyzer.js')
 const parseCSVConnectors = require('./parse-csv-connectors.js')
+const { resolvePageTypeDir } = require('./metadata-utils.js')
 const semver = require('semver')
 
 /**
@@ -106,6 +107,23 @@ function capToTwoSentences (description) {
   return result.trim()
 }
 
+// Component families in a connector index, in the spelling the Connect binary
+// emits. Rate limits are 'rate-limits' there; 'rate_limits' is the docs page
+// directory spelling, which older snapshots also carry as an (always empty)
+// data key, so it is walked when present but never created. Every copy of this
+// list used to be the underscore form only, which meant rate-limit components
+// were never given cloudSupported or requiresCgo flags and any cgo-only or
+// cloud-only rate limit was dropped from the published snapshot.
+const COMPONENT_TYPE_KEYS = ['inputs', 'outputs', 'processors', 'caches', 'rate-limits',
+  'rate_limits', 'buffers', 'metrics', 'scanners', 'tracers']
+
+// 'config' components live in the same index but have no config/fields wrapper.
+const COMPONENT_TYPE_KEYS_WITH_CONFIG = [...COMPONENT_TYPE_KEYS, 'config']
+
+// Keys kept only for snapshots written before the spelling was fixed. Walked
+// if a snapshot has them, never added to one.
+const LEGACY_COMPONENT_TYPE_KEYS = new Set(['rate_limits'])
+
 /**
  * Remove platform metadata fields from connectors
  * @param {Array} connectors - Array of connector objects
@@ -146,11 +164,9 @@ function augmentConnectorData (connectorData, binaryAnalysis) {
   let addedCgoCount = 0
   let addedCloudOnlyCount = 0
 
-  const connectorTypes = ['inputs', 'outputs', 'processors', 'caches', 'rate_limits',
-    'buffers', 'metrics', 'scanners', 'tracers', 'config']
-
-  for (const type of connectorTypes) {
+  for (const type of COMPONENT_TYPE_KEYS_WITH_CONFIG) {
     if (!Array.isArray(augmentedData[type])) {
+      if (LEGACY_COMPONENT_TYPE_KEYS.has(type)) continue
       augmentedData[type] = []
     }
 
@@ -219,10 +235,7 @@ function augmentConnectorData (connectorData, binaryAnalysis) {
  */
 function buildCleanOssData (connectorIndex) {
   const cleanData = JSON.parse(JSON.stringify(connectorIndex))
-  const connectorTypes = ['inputs', 'outputs', 'processors', 'caches', 'rate_limits',
-    'buffers', 'metrics', 'scanners', 'tracers']
-
-  for (const type of connectorTypes) {
+  for (const type of COMPONENT_TYPE_KEYS) {
     if (Array.isArray(cleanData[type])) {
       // Keep only connectors from OSS rpk (have config/fields)
       // Remove augmentation-only connectors (added by previous binary analysis)
@@ -703,11 +716,52 @@ function logCollapsed (label, filesArray, maxToShow = 10) {
 }
 
 
+/**
+ * Why a stored connect-<version>.json cannot stand in for a fresh fetch.
+ *
+ * The handler writes augmentConnectorData() output back over the same file, so
+ * a stored snapshot is not necessarily raw binary output. Reusing an augmented
+ * one would re-augment already-augmented data: the cloud-only and cgo-only
+ * entries injected by the previous run are not in the new analysis, so they
+ * survive with their platform markers reset and a cgo-only connector ends up
+ * published as an ordinary OSS one. The version field is the only provenance
+ * the downstream templates get, so a file that disagrees with its own name
+ * cannot be trusted either.
+ *
+ * @param {Object} data - Parsed contents of connect-<version>.json
+ * @param {string} version - Version the file is supposed to describe
+ * @returns {string|null} Reason it cannot be reused, or null if it can
+ */
+function snapshotReuseBlocker (data, version) {
+  if (!data || typeof data !== 'object' || Array.isArray(data)) {
+    return 'file is not a connector index'
+  }
+
+  if (typeof data.version === 'string' && data.version !== version) {
+    return `file reports version ${data.version}`
+  }
+
+  const augmented = Object.values(data)
+    .filter(Array.isArray)
+    .some(components => components.some(c => c && typeof c === 'object' && (
+      c.cloudSupported !== undefined ||
+      c.requiresCgo !== undefined ||
+      c.cloudOnly !== undefined
+    )))
+
+  if (augmented) {
+    return 'file already carries cloud/cgo augmentation from an earlier run'
+  }
+
+  return null
+}
+
 /**
  * Load or fetch connector data for a specific version
  * @param {string} version - Version to load (e.g., "4.50.0")
  * @param {string} dataDir - Directory where JSON files are stored
  * @param {Object} options - Options for fetching if needed
+ * @param {boolean} [options.forceFresh] - Refetch even when a usable file exists
  * @returns {Promise<Object>} Parsed connector data
  */
 async function loadConnectorDataForVersion(version, dataDir, options = {}) {
@@ -715,51 +769,75 @@ async function loadConnectorDataForVersion(version, dataDir, options = {}) {
 
   // If forceFresh is set, always fetch even if file exists
   // This ensures we have accurate connector lists for diff comparison
+  let reason = options.forceFresh ? 'force refresh requested' : 'file not found';
+
   if (!options.forceFresh && fs.existsSync(dataFile)) {
-    console.log(`✓ Using existing data file: connect-${version}.json`);
-    const data = JSON.parse(fs.readFileSync(dataFile, 'utf8'));
-    return data;
+    let stored = null;
+    let blocker = null;
+    try {
+      stored = JSON.parse(fs.readFileSync(dataFile, 'utf8'));
+      blocker = snapshotReuseBlocker(stored, version);
+    } catch (err) {
+      blocker = `file could not be parsed (${err.message})`;
+    }
+
+    if (!blocker) {
+      console.log(`✓ Using existing data file: connect-${version}.json`);
+      return stored;
+    }
+
+    console.warn(`↻ Ignoring existing connect-${version}.json: ${blocker}`);
+    reason = blocker;
   }
 
   // Fetch fresh data
-  const reason = options.forceFresh ? 'force refresh requested' : 'file not found';
   console.log(`📥 Fetching data for ${version} (${reason})...`);
 
+  // Fetch the OSS release asset for this exact version instead of asking rpk to
+  // install it. rpk's validateVersion (rpk/pkg/cli/connect/install.go) matches
+  // --connect-version against a regex that caps each segment at two digits, so
+  // it rejects every version >= 4.100.0, which is every version worth asking
+  // for. The cap is lifted in rpk v26.1.15 and on dev, but v26.1.14 and the
+  // whole v26.2.x line still carry it, so which versions can be installed
+  // depends on which rpk happens to be on PATH. Fetching the asset removes that
+  // dependency instead of working around one rpk release, so this is the
+  // permanent mechanism rather than something to revert once rpk ships a fix:
+  // the plain OSS asset is the same build rpk would have installed, so the
+  // schema is identical, and this leaves the caller's own Connect installation
+  // untouched instead of forcing a managed plugin over the top of it.
   try {
-    // Try installing that specific version and fetching data
-    console.log(`   Installing Redpanda Connect version ${version}...`);
-    const installResult = spawnSync('rpk', ['connect', 'install', '--connect-version', version, '--force'], {
-      stdio: 'pipe'
-    });
-
-    if (installResult.status !== 0) {
-      throw new Error(`Failed to install Connect version ${version}`);
-    }
+    const { downloadBinary, getConnectorList, getBinaryCacheDir } = require('./connector-binary-analyzer.js');
 
-    // Fetch connector list
-    const tmpFile = path.join(dataDir, `connect-${version}.tmp.json`);
-    const fd = fs.openSync(tmpFile, 'w');
-    const listResult = spawnSync('rpk', ['connect', 'list', '--format', 'json-full'], {
-      stdio: ['ignore', fd, 'pipe']
-    });
-    fs.closeSync(fd);
-
-    if (listResult.status !== 0) {
-      throw new Error(`Failed to fetch connector list for version ${version}`);
+    // Binaries go to the process-wide temp cache, never to dataDir: the
+    // consuming repo commits dataDir, and the extracted binary is ~320 MB.
+    const binaryPath = await downloadBinary('oss', version, getBinaryCacheDir());
+    const parsed = getConnectorList(binaryPath);
+
+    if (!parsed || typeof parsed !== 'object') {
+      throw new Error('connector list was empty or not an object');
     }
 
-    // Parse and validate
-    const rawJson = fs.readFileSync(tmpFile, 'utf8');
-    const parsed = JSON.parse(rawJson);
+    // The asset name embeds the version and the release was fetched by tag, so
+    // a binary that reports a different version means the wrong asset was
+    // downloaded. Fail rather than publish a snapshot under the wrong name.
+    if (typeof parsed.version === 'string' && parsed.version !== version) {
+      throw new Error(`asset reports version ${parsed.version}, not ${version}`);
+    }
 
-    // Move to final location
+    // Write via a temp file so an interrupted run cannot leave a half-written
+    // snapshot that the next run would happily reuse.
+    const tmpFile = path.join(dataDir, `connect-${version}.tmp.json`);
+    fs.writeFileSync(tmpFile, JSON.stringify(parsed, null, 2));
     fs.renameSync(tmpFile, dataFile);
 
     console.log(`✓ Successfully fetched data for version ${version}`);
     return parsed;
   } catch (error) {
     console.error(`❌ Failed to fetch data for version ${version}: ${error.message}`);
-    throw new Error(`Cannot process version ${version} - data unavailable`);
+    // Carry the reason into the thrown error: this message is what reaches the
+    // failure summary and the PR body, where a bare "data unavailable" sends
+    // the reader back to the raw workflow log.
+    throw new Error(`Cannot process version ${version} - data unavailable (${error.message})`);
   }
 }
 
@@ -789,34 +867,47 @@ async function handleRpcnConnectorDocs (options) {
           console.error('Expected format: X.Y.Z (e.g., 4.50.0)')
           process.exit(1)
         }
-        console.log(`Installing Redpanda Connect version ${options.connectVersion}...`)
-        const installResult = spawnSync('rpk', ['connect', 'install', '--connect-version', options.connectVersion, '--force'], {
-          stdio: 'inherit'
-        })
-        if (installResult.status !== 0) {
-          throw new Error(`Failed to install Connect version ${options.connectVersion}`)
-        }
-        console.log(`Done: Installed Redpanda Connect version ${options.connectVersion}`)
         newVersion = options.connectVersion
+        // An explicit version comes from the release asset rather than from
+        // rpk, which rejects anything >= 4.100.0 on the rpk releases this
+        // pipeline runs. See loadConnectorDataForVersion. Reuses an existing
+        // snapshot when it is raw, unaugmented output for this same version,
+        // and leaves the caller's own Connect install alone either way.
+        console.log(`Fetching connector data for Connect ${newVersion}...`)
+        await loadConnectorDataForVersion(newVersion, dataDir)
+        dataFile = path.join(dataDir, `connect-${newVersion}.json`)
+        needsAugmentation = true
+        console.log(`Done: Fetched connector data for version ${newVersion}`)
       } else {
+        // No version asked for, so read whatever rpk has: this is install's
+        // unvalidated "latest" path, which the version regex never sees.
         newVersion = getRpkConnectVersion()
+        console.log(`Fetching connector data from Connect ${newVersion}...`)
+
+        const tmpFile = path.join(dataDir, `connect-${newVersion}.tmp.json`)
+        const finalFile = path.join(dataDir, `connect-${newVersion}.json`)
+
+        const fd = fs.openSync(tmpFile, 'w')
+        spawnSync('rpk', ['connect', 'list', '--format', 'json-full'], { stdio: ['ignore', fd, 'inherit'] })
+        fs.closeSync(fd)
+
+        const rawJson = fs.readFileSync(tmpFile, 'utf8')
+        // Deliberately plain JSON.parse/stringify, not bigIntJson: this is connect
+        // data, which carries example config text as escaped JSON-shaped strings
+        // (e.g. bloblang examples), indistinguishable by regex from real JSON
+        // value position. bigIntJson's value-position regex matches inside such a
+        // string and inserts stray quotes -- confirmed to throw on the real,
+        // currently-published connect attachment, not merely round a number. A
+        // plain round trip still silently rounds int64 defaults past MaxInt64 (the
+        // pre-existing defect this file has always had), which is the safer
+        // failure mode until a real JSON scanner replaces the regex.
+        const parsed = JSON.parse(rawJson)
+        fs.writeFileSync(finalFile, JSON.stringify(parsed, null, 2))
+        fs.unlinkSync(tmpFile)
+        dataFile = finalFile
+        needsAugmentation = true
+        console.log(`Done: Fetched connector data for version ${newVersion}`)
       }
-      console.log(`Fetching connector data from Connect ${newVersion}...`)
-
-      const tmpFile = path.join(dataDir, `connect-${newVersion}.tmp.json`)
-      const finalFile = path.join(dataDir, `connect-${newVersion}.json`)
-
-      const fd = fs.openSync(tmpFile, 'w')
-      const r = spawnSync('rpk', ['connect', 'list', '--format', 'json-full'], { stdio: ['ignore', fd, 'inherit'] })
-      fs.closeSync(fd)
-
-      const rawJson = fs.readFileSync(tmpFile, 'utf8')
-      const parsed = JSON.parse(rawJson)
-      fs.writeFileSync(finalFile, JSON.stringify(parsed, null, 2))
-      fs.unlinkSync(tmpFile)
-      dataFile = finalFile
-      needsAugmentation = true
-      console.log(`Done: Fetched connector data for version ${newVersion}`)
 
       // Fetch info.csv
       try {
@@ -1458,11 +1549,8 @@ async function handleRpcnConnectorDocs (options) {
 
       // Strip CGO/cloud-only connectors and metadata from old data
       oldIndexForDiff = JSON.parse(JSON.stringify(oldIndex))
-      const connectorTypes = ['inputs', 'outputs', 'processors', 'caches', 'rate_limits',
-        'buffers', 'metrics', 'scanners', 'tracers', 'config']
-
       let totalStripped = 0
-      for (const type of connectorTypes) {
+      for (const type of COMPONENT_TYPE_KEYS_WITH_CONFIG) {
         if (Array.isArray(oldIndexForDiff[type])) {
           const originalCount = oldIndexForDiff[type].length
 
@@ -1649,8 +1737,7 @@ async function handleRpcnConnectorDocs (options) {
       const dataObj = JSON.parse(rawData)
 
       const validConnectors = []
-      const types = ['inputs', 'outputs', 'processors', 'caches', 'rate_limits', 'buffers', 'metrics', 'scanners', 'tracers']
-      types.forEach(type => {
+      COMPONENT_TYPE_KEYS.forEach(type => {
         if (Array.isArray(dataObj[type])) {
           dataObj[type].forEach(connector => {
             if (connector.name) {
@@ -1721,8 +1808,7 @@ async function handleRpcnConnectorDocs (options) {
         // Fallback when binary analysis is unavailable:
         // Check all connectors that have cloudSupported flag or assume all non-deprecated are cloud-supported
         console.log('   ℹ️  Binary analysis unavailable - checking all non-deprecated connectors for cloud-docs')
-        const types = ['inputs', 'outputs', 'processors', 'caches', 'rate_limits', 'buffers', 'metrics', 'scanners', 'tracers']
-        types.forEach(type => {
+        COMPONENT_TYPE_KEYS.forEach(type => {
           if (Array.isArray(dataObj[type])) {
             dataObj[type].forEach(connector => {
               // Include if cloudSupported is explicitly true, or if it's null/undefined and not deprecated
@@ -1738,17 +1824,21 @@ async function handleRpcnConnectorDocs (options) {
       }
 
       // Check for missing connector documentation in rp-connect-docs
+      // `type` came from a data key, and the rate-limit family's data key
+      // ('rate-limits') is not its directory name ('rate_limits'). Resolve the
+      // directory against disk per root, or every existing rate-limit page is
+      // reported missing and drafted again beside itself.
       const allMissing = validConnectors.filter(({ name, type, cloudOnly }) => {
-        const relPath = path.join(`${type}s`, `${name}.adoc`)
+        const relPathIn = (root) => path.join(resolvePageTypeDir(root, `${type}s`), `${name}.adoc`)
 
         // For cloud-only connectors, ONLY check the cloud-only directory
         if (cloudOnly) {
-          return !fs.existsSync(path.join(roots.cloudOnly, relPath))
+          return !fs.existsSync(path.join(roots.cloudOnly, relPathIn(roots.cloudOnly)))
         }
 
         // For regular connectors, check pages and partials (not cloud-only)
         const existsInAny = [roots.pages, roots.partials].some(root =>
-          fs.existsSync(path.join(root, relPath))
+          fs.existsSync(path.join(root, relPathIn(root)))
         )
         return !existsInAny
       })
@@ -2081,6 +2171,8 @@ module.exports = {
   capToTwoSentences,
   augmentConnectorData,
   buildCleanOssData,
+  loadConnectorDataForVersion,
+  snapshotReuseBlocker,
   fieldAnchor,
   buildFieldsTable,
   buildChangedDefaultsTable