@redpanda-data/docs-extensions-and-macros 5.37.2 → 5.37.4

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@redpanda-data/docs-extensions-and-macros",
-  "version": "5.37.2",
+  "version": "5.37.4",
   "description": "Antora extensions and macros developed for Redpanda documentation.",
   "keywords": [
     "antora",

package/tools/property-extractor/merge-rp-util.js

@@ -37,7 +37,7 @@ const fs = require('fs')
 const os = require('os')
 const path = require('path')
 const { spawnSync } = require('child_process')
-const { getRpUtilSchema, SCHEMA_FLAGS } = require('./rp-util-fetch')
+const { getRpUtilSchema, schemaExpectation, fetchPublishedSchema, SCHEMA_FLAGS } = require('./rp-util-fetch')
 const bigIntJson = require('../../cli-utils/big-int-json')
 
 function getArg(args, flag) {
@@ -57,13 +57,140 @@ const RELEASE_TAG_RX = /^v?\d+\.\d+\.\d+(-rc\d+)?$/
  * tag (see the header comment), a warning plus the Tree-sitter fallback for
  * everything else.
  */
-function handleMergeUnavailable(tag, enhanced, reason) {
+// How long to wait for a schema that is being built right now, and how often
+// to look. streaming-enterprise's dispatch-docs-updates.yml fires the schema
+// publish and this generation from the same job, and the publish is a
+// ~27-minute Bazel build per scope, so on a fresh release tag the schema
+// reliably does not exist yet when we first ask. The docs runner has neither
+// bazel nor docker, so the from-source fallback cannot cover the gap: without
+// waiting, every RC and GA regeneration fails on a race it is guaranteed to
+// lose. Overridable so a one-off backfill need not sit through it.
+/**
+ * Read a millisecond knob from the environment, or fall back. Number('') is 0
+ * and Number('abc') is NaN, and a 0 or NaN poll interval turns the wait loop
+ * into a busy-poll of the GitHub API for the whole budget. `allowZero` is for
+ * the wait budget, where 0 is the documented way to disable waiting.
+ */
+function parseEnvMs(name, fallback, { allowZero = false } = {}) {
+  const raw = process.env[name]
+  if (raw === undefined || raw === '') return fallback
+  const n = Number(raw)
+  if (Number.isInteger(n) && (n > 0 || (allowZero && n === 0))) return n
+  // Quoted by hand: this file is held to bigIntJson.stringify by a repo-wide
+  // guard test, and a plain string needs no BigInt-aware serializer anyway.
+  console.warn(`Warning: ${name}='${raw}' is not a ${allowZero ? 'non-negative' : 'positive'} integer; using ${fallback}.`)
+  return fallback
+}
+
+const SCHEMA_WAIT_MS = parseEnvMs('RP_UTIL_SCHEMA_WAIT_MS', 45 * 60 * 1000, { allowZero: true })
+const SCHEMA_POLL_MS = parseEnvMs('RP_UTIL_SCHEMA_POLL_MS', 60 * 1000)
+const CLASSIFY_BACKOFF_MS = parseEnvMs('RP_UTIL_CLASSIFY_BACKOFF_MS', 5 * 1000)
+
+const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms))
+
+/**
+ * Poll for a published schema release until it appears or the budget runs out.
+ * @returns {Promise<object|null>} The schemas, or null if none appeared
+ */
+function isPermanentLookupError(err) {
+  const s = Number(err && err.status)
+  return s >= 400 && s < 500 && s !== 404 && s !== 429
+}
+
+/**
+ * schemaExpectation() with a few retries. It is one GitHub contents call, and
+ * a single flaky response used to leave the classification at 'unknown', which
+ * skipped the wait entirely and dropped a release straight into the race this
+ * module exists to avoid. Only exceptions are retried; a real classification,
+ * including 'unknown' from the classifier itself, is returned as-is.
+ */
+async function classifyWithRetry(releaseTag, { attempts = 3, backoffMs = CLASSIFY_BACKOFF_MS, sleepFn = sleep } = {}) {
+  let lastErr
+  for (let i = 1; i <= attempts; i++) {
+    try {
+      return await schemaExpectation(releaseTag)
+    } catch (err) {
+      lastErr = err
+      console.warn(`Warning: could not classify ${releaseTag} (attempt ${i} of ${attempts}): ${err.message}`)
+      if (i < attempts) await sleepFn(backoffMs * i)
+    }
+  }
+  return { expectation: 'unknown', reason: `could not classify ${releaseTag} after ${attempts} attempts: ${lastErr.message}` }
+}
+
+async function waitForPublishedSchema(releaseTag, budgetMs, pollMs, now = Date.now) {
+  const deadline = now() + budgetMs
+  let attempt = 0
+  for (;;) {
+    attempt++
+    let published = null
+    try {
+      published = await fetchPublishedSchema(releaseTag)
+    } catch (err) {
+      // A transient failure is not the same as "not published": log and
+      // keep waiting. A 4xx other than 404 (which fetchPublishedSchema maps
+      // to null) or 429 is permanent: a bad or under-scoped token will not
+      // heal in 45 minutes, so surface it now instead of after the budget.
+      if (isPermanentLookupError(err)) {
+        throw new Error(`Cannot look up the rp_util schema for ${releaseTag}: ${err.message}`)
+      }
+      console.warn(`Warning: schema lookup for ${releaseTag} failed (attempt ${attempt}): ${err.message}`)
+    }
+    if (published) {
+      console.log(`rp_util schema for ${releaseTag} is published (after ${attempt} check(s)).`)
+      return published
+    }
+    const remaining = deadline - now()
+    if (remaining <= 0) return null
+    const nap = Math.min(pollMs, remaining)
+    console.log(
+      `Waiting for the rp_util schema for ${releaseTag} to be published ` +
+      `(${Math.round(remaining / 1000)}s of budget left)...`
+    )
+    await sleep(nap)
+  }
+}
+
+async function handleMergeUnavailable(tag, enhanced, reason, output = enhanced) {
   if (RELEASE_TAG_RX.test(tag)) {
     // Print the canonical v-prefixed tag in the remediation command:
     // streaming-enterprise tags (and therefore schema release names) are
     // always v-prefixed, so telling the operator to publish a v-less name
     // would send them to a workflow run that can never help.
     const releaseTag = tag.startsWith('v') ? tag : `v${tag}`
+
+    // Not every release CAN have a schema, and the ones that cannot do not
+    // need one. streaming-enterprise#63 added the broker-scope dump flags and
+    // the enum_set_property conversions in the same change, so a tag without
+    // the flags also has no enum_set properties for the Tree-sitter pass to
+    // misparse. Failing those releases forever would block every regeneration
+    // of a v26.2.x or older line on a schema that can never be published.
+    // See schemaExpectation() for the evidence.
+    const expectation = await classifyWithRetry(releaseTag)
+
+    if (expectation.expectation === 'unavailable') {
+      console.warn(`Warning: ${reason}`)
+      console.warn(`No rp_util schema can exist for ${releaseTag}: ${expectation.reason}`)
+      console.warn('Keeping the Tree-sitter-only extraction, which is accurate for this release.')
+      markRpUtilMergeUnavailable(enhanced)
+      // generate-docs reads --output right after this returns. When it is
+      // not the same file as --enhanced, the degraded result has to land
+      // there too, or the run exits 0 having left the output stale.
+      if (output && output !== enhanced) fs.copyFileSync(enhanced, output)
+      return
+    }
+
+    if (expectation.expectation === 'inconsistent') {
+      console.error(`Error: ${expectation.reason}`)
+      console.error(
+        'This is the one combination that must never ship: neither the schema ' +
+        'nor the Tree-sitter extraction can describe those properties. Backport ' +
+        'the rp_util schema flags to this branch before regenerating.'
+      )
+      process.exitCode = 1
+      return
+    }
+
     console.error(`Error: ${reason}`)
     console.error(
       `Refusing to publish Tree-sitter-only property data for release ${tag}: ` +
@@ -138,14 +265,46 @@ async function main() {
 
   console.log(`Fetching rp_util schema for ${tag}...`)
   let schemas
+
+  // On a release tag whose rp_util can dump every scope, the schema is either
+  // already published or being built right now, so wait for it instead of
+  // racing the publisher. A tag that can never have one skips the wait and
+  // falls through to the classification in handleMergeUnavailable.
+  if (!sourcePath && RELEASE_TAG_RX.test(tag) && SCHEMA_WAIT_MS > 0) {
+    const releaseTag = tag.startsWith('v') ? tag : `v${tag}`
+    const expectation = await classifyWithRetry(releaseTag)
+    if (expectation.expectation === 'required') {
+      let waited
+      try {
+        waited = await waitForPublishedSchema(releaseTag, SCHEMA_WAIT_MS, SCHEMA_POLL_MS)
+      } catch (err) {
+        // 'required' means degrading is not an option, so a permanent
+        // lookup failure is the run's failure.
+        console.error(`Error: ${err.message}`)
+        process.exitCode = 1
+        return
+      }
+      if (waited) {
+        schemas = { ...waited, sourcePath: null }
+      } else {
+        console.warn(
+          `Warning: no rp_util schema for ${releaseTag} after waiting ` +
+          `${Math.round(SCHEMA_WAIT_MS / 60000)} minutes.`
+        )
+      }
+    }
+  }
+
   try {
     // --source-path builds from an existing local streaming-enterprise
     // checkout instead of fetching a published release or cloning+building
     // fresh -- same option rp-util-fetch.js itself exposes, useful for
     // local iteration against a branch that has no published schema yet.
-    schemas = await getRpUtilSchema(tag, sourcePath ? { sourcePath } : undefined)
+    if (!schemas) {
+      schemas = await getRpUtilSchema(tag, sourcePath ? { sourcePath } : undefined)
+    }
   } catch (err) {
-    handleMergeUnavailable(tag, enhanced, `could not get rp_util schema for ${tag}: ${err.message}`)
+    await handleMergeUnavailable(tag, enhanced, `could not get rp_util schema for ${tag}: ${err.message}`, output)
     return
   }
 
@@ -159,7 +318,7 @@ async function main() {
       }
     }
     if (!anySchema) {
-      handleMergeUnavailable(tag, enhanced, `rp_util schema for ${tag} came back empty.`)
+      await handleMergeUnavailable(tag, enhanced, `rp_util schema for ${tag} came back empty.`, output)
       return
     }
 
@@ -188,10 +347,11 @@ async function main() {
     const result = spawnSync(pythonBin, mergeArgs, { cwd: __dirname, stdio: 'inherit' })
     if (result.error || result.status !== 0) {
       fs.rmSync(tempOutput, { force: true })
-      handleMergeUnavailable(
+      await handleMergeUnavailable(
         tag,
         enhanced,
-        `rp_util merge failed (${result.error ? result.error.message : `exit ${result.status}`}).`
+        `rp_util merge failed (${result.error ? result.error.message : `exit ${result.status}`}).`,
+        output
       )
     } else {
       fs.renameSync(tempOutput, output)
@@ -201,7 +361,16 @@ async function main() {
   }
 }
 
-module.exports = { main, markRpUtilMergeUnavailable }
+module.exports = {
+  main,
+  parseEnvMs,
+  classifyWithRetry,
+  isPermanentLookupError,
+  markRpUtilMergeUnavailable,
+  // exported for testing
+  waitForPublishedSchema,
+  handleMergeUnavailable
+}
 
 if (require.main === module) {
   main()

package/tools/property-extractor/property_extractor.py

@@ -1032,6 +1032,85 @@ def report_phantom_stubs():
     logger.warning("=" * 70)
 
 
+def infer_topic_property_category(name):
+    """Map a topic property's dotted name to a category, for properties with
+    no explicit category override.
+
+    "other" is the fallback for a name that matches no group. It only renders
+    where a page includes tags=category-other: cloud-docs' topic properties
+    page does (cloud-docs#686); docs' topic-properties.adoc and
+    broker-properties.adoc do not yet, so on docs main a property that falls
+    to "other" is present in the partial but appears on no page. The point of
+    the fallback is that such a property is at least reachable by a known tag,
+    rather than untagged and unreachable by any include.
+    """
+    retention = [
+        "cleanup.policy", "compaction.strategy", "delete.retention.ms", "max.compaction.lag.ms",
+        "min.cleanable.dirty.ratio", "min.compaction.lag.ms", "retention.bytes", "retention.ms"
+    ]
+    segment = [
+        "compression.type", "max.message.bytes", "message.timestamp.type", "segment.bytes", "segment.ms"
+    ]
+    performance = [
+        "flush.bytes", "flush.ms", "redpanda.leaders.preference", "replication.factor", "write.caching"
+    ]
+    tiered = [
+        "initial.retention.local.target.bytes", "initial.retention.local.target.ms", "redpanda.remote.delete",
+        "redpanda.remote.read", "redpanda.remote.recovery", "redpanda.remote.write", "retention.local.target.bytes",
+        "retention.local.target.ms"
+    ]
+    remote_replica = ["redpanda.remote.readreplica"]
+    iceberg = [
+        "redpanda.iceberg.delete", "redpanda.iceberg.invalid.record.action", "redpanda.iceberg.mode",
+        "redpanda.iceberg.partition.spec", "redpanda.iceberg.target.lag.ms"
+    ]
+    schema_registry = [
+        "redpanda.key.schema.id.validation", "redpanda.key.subject.name.strategy", "redpanda.value.schema.id.validation",
+        "redpanda.value.subject.name.strategy", "confluent.key.schema.validation", "confluent.key.subject.name.strategy",
+        "confluent.value.schema.validation", "confluent.value.subject.name.strategy"
+    ]
+    if name in retention:
+        return "retention-compaction"
+    if name in segment:
+        return "segment-message"
+    if name in performance:
+        return "performance-cluster"
+    if name in tiered:
+        return "tiered-storage"
+    if name in remote_replica:
+        return "remote-read-replica"
+    if name in iceberg:
+        return "iceberg-integration"
+    if name in schema_registry:
+        return "schema-registry"
+    return "other"
+
+
+def _ensure_category_fallback(properties):
+    """Guarantee every property has a category, so the property.hbs/
+    topic-property.hbs templates' `{{#if category}}` always wraps it in a
+    tag::category-X[] region. A property with no category at all gets no
+    tag region and is silently excluded by every page that assembles its
+    content from specific category-tag includes -- present in the raw
+    generated partial, invisible everywhere it's actually read from.
+
+    Runs after overrides are applied (including override-created phantom
+    stub properties, which previously never received a category at all)
+    so this covers every property regardless of where it came from.
+    """
+    for prop_name, property_data in properties.items():
+        if not hasattr(property_data, "get") or property_data.get("category"):
+            continue
+        if property_data.get("config_scope") == "topic" or property_data.get("is_topic_property"):
+            property_data["category"] = infer_topic_property_category(property_data.get("name", prop_name))
+        else:
+            # No per-name taxonomy exists for cluster/broker properties today
+            # (unlike topic properties' infer_topic_property_category) --
+            # "other" at least guarantees a real, included tag region
+            # instead of silent exclusion.
+            property_data["category"] = "other"
+
+
 def apply_property_overrides(properties, overrides, overrides_file_path=None):
     """
     Apply overrides from an overrides mapping to the extracted properties, mutating and returning the properties dictionary.
@@ -1114,6 +1193,8 @@ def apply_property_overrides(properties, overrides, overrides_file_path=None):
     for property_data in properties.values():
         if hasattr(property_data, "get") and property_data.get("description"):
             property_data["description"] = _normalize_config_ref_macros(property_data["description"])
+
+    _ensure_category_fallback(properties)
     return properties
 
 
@@ -2844,49 +2925,6 @@ def extract_topic_properties(source_path, cluster_properties=None):
             if prop_data.get("is_noop", False):
                 continue
 
-            # Assign category based on property name pattern or mapping
-            def infer_category(name):
-                retention = [
-                    "cleanup.policy", "compaction.strategy", "delete.retention.ms", "max.compaction.lag.ms",
-                    "min.cleanable.dirty.ratio", "min.compaction.lag.ms", "retention.bytes", "retention.ms"
-                ]
-                segment = [
-                    "compression.type", "max.message.bytes", "message.timestamp.type", "segment.bytes", "segment.ms"
-                ]
-                performance = [
-                    "flush.bytes", "flush.ms", "redpanda.leaders.preference", "replication.factor", "write.caching"
-                ]
-                tiered = [
-                    "initial.retention.local.target.bytes", "initial.retention.local.target.ms", "redpanda.remote.delete",
-                    "redpanda.remote.read", "redpanda.remote.recovery", "redpanda.remote.write", "retention.local.target.bytes",
-                    "retention.local.target.ms"
-                ]
-                remote_replica = ["redpanda.remote.readreplica"]
-                iceberg = [
-                    "redpanda.iceberg.delete", "redpanda.iceberg.invalid.record.action", "redpanda.iceberg.mode",
-                    "redpanda.iceberg.partition.spec", "redpanda.iceberg.target.lag.ms"
-                ]
-                schema_registry = [
-                    "redpanda.key.schema.id.validation", "redpanda.key.subject.name.strategy", "redpanda.value.schema.id.validation",
-                    "redpanda.value.subject.name.strategy", "confluent.key.schema.validation", "confluent.key.subject.name.strategy",
-                    "confluent.value.schema.validation", "confluent.value.subject.name.strategy"
-                ]
-                if name in retention:
-                    return "retention-compaction"
-                if name in segment:
-                    return "segment-message"
-                if name in performance:
-                    return "performance-cluster"
-                if name in tiered:
-                    return "tiered-storage"
-                if name in remote_replica:
-                    return "remote-read-replica"
-                if name in iceberg:
-                    return "iceberg-integration"
-                if name in schema_registry:
-                    return "schema-registry"
-                return "other"
-
             converted_properties[prop_name] = {
                 "name": prop_name,
                 "description": prop_data.get("description", ""),
@@ -2900,7 +2938,7 @@ def extract_topic_properties(source_path, cluster_properties=None):
                 "acceptable_values": prop_data.get("acceptable_values", ""),
                 "is_deprecated": False,
                 "is_topic_property": True,
-                "category": infer_category(prop_name)
+                "category": infer_topic_property_category(prop_name)
             }
 
             # Add default values if they exist (inherited from cluster properties)

package/tools/property-extractor/rp-util-fetch.js

@@ -276,6 +276,94 @@ const RELEASE_ASSET_TO_SCHEMA_KEY = {
   'schema-registry-config-schema.json': 'schemaRegistrySchema'
 }
 
+// Files schemaExpectation() reads to decide whether a ref can have a schema.
+// Both were changed by the same PR (streaming-enterprise#63), which is what
+// makes the pairing in schemaExpectation() a property of the source rather
+// than a guess.
+const SE_SOURCE_REPO = 'redpanda-data/streaming-enterprise'
+const RP_UTIL_MAIN_PATH = 'src/v/rp_util/main.cc'
+const CONFIGURATION_H_PATH = 'src/v/config/configuration.h'
+
+/**
+ * Read one file from streaming-enterprise at a ref.
+ * @returns {Promise<string|null>} Contents, or null if it cannot be read
+ */
+async function fetchSourceFile(ref, filePath) {
+  const token = getGitHubApiToken()
+  const headers = { Accept: 'application/vnd.github.raw' }
+  if (token) headers.Authorization = `Bearer ${token}`
+  const resp = await fetch(
+    `https://api.github.com/repos/${SE_SOURCE_REPO}/contents/${filePath}?ref=${encodeURIComponent(ref)}`,
+    { headers }
+  )
+  if (!resp.ok) return null
+  return resp.text()
+}
+
+/**
+ * Whether a ref can produce an rp_util schema, and whether it needs one.
+ *
+ * These two are the same question, because they arrived in the same commit.
+ * streaming-enterprise#63 added both the four broker-scope dump flags to
+ * rp_util and the `enum_set_property` conversions that the Tree-sitter
+ * extractor misparses. So a ref either has both or neither:
+ *
+ *   ref             enum_set_property   broker-scope flags
+ *   v26.2.1                         0                  0/4
+ *   v26.2.2                         0                  0/4
+ *   v26.2.3-rc1                     0                  0/4
+ *   v26.2.x (branch)                3                  4/4
+ *   dev                             3                  4/4
+ *
+ * That is what makes degrading safe for an old tag: a tag with no flags can
+ * never have a schema, and equally has no enum_set properties to misparse, so
+ * the Tree-sitter extraction is correct for it rather than merely tolerable.
+ *
+ * The flags are matched QUOTED. `"config_schema_json"` must not match inside
+ * `"node_config_schema_json"`, and the leading quote is what prevents it.
+ *
+ * @param {string} ref - Tag, branch, or SHA
+ * @returns {Promise<{expectation: string, reason: string}>} expectation is one
+ *   of 'required' (schema needed and obtainable), 'unavailable' (this ref can
+ *   never have one, and does not need one), 'inconsistent' (needs one but
+ *   cannot have one, which must never ship), or 'unknown' (could not tell)
+ */
+async function schemaExpectation(ref) {
+  const mainCc = await fetchSourceFile(ref, RP_UTIL_MAIN_PATH)
+  if (mainCc === null) {
+    return { expectation: 'unknown', reason: `could not read ${RP_UTIL_MAIN_PATH} at ${ref}` }
+  }
+  const missingFlags = SCHEMA_FLAGS
+    .map(({ flag }) => flag.replace(/^--/, ''))
+    .filter((name) => !mainCc.includes(`"${name}"`))
+
+  if (missingFlags.length === 0) {
+    return { expectation: 'required', reason: `rp_util at ${ref} dumps every scope` }
+  }
+
+  // No flags. Confirm the other half of the pairing rather than assuming it:
+  // if this ref somehow carries enum_set properties without the dump flags,
+  // the Tree-sitter fallback would silently drop them and degrading would be
+  // the wrong call.
+  const configurationH = await fetchSourceFile(ref, CONFIGURATION_H_PATH)
+  if (configurationH === null) {
+    return {
+      expectation: 'unknown',
+      reason: `rp_util at ${ref} cannot dump ${missingFlags.join(', ')}, and ${CONFIGURATION_H_PATH} could not be read to confirm it has no enum_set properties`
+    }
+  }
+  if (configurationH.includes('enum_set_property')) {
+    return {
+      expectation: 'inconsistent',
+      reason: `${ref} declares enum_set properties but its rp_util cannot dump ${missingFlags.join(', ')}, so neither the schema nor the Tree-sitter extraction can describe them correctly`
+    }
+  }
+  return {
+    expectation: 'unavailable',
+    reason: `rp_util at ${ref} predates the broker-scope dumps (${missingFlags.join(', ')}) and ${ref} declares no enum_set properties, so the Tree-sitter extraction is correct for it`
+  }
+}
+
 /**
  * Fetch rp_util's schema from a previously published GitHub release
  * (publish-rp-util-schema.yaml's rp-util-schema-<tag> release in this repo)
@@ -296,10 +384,10 @@ async function fetchPublishedSchema(tag) {
   )
   if (releaseResp.status === 404) return null
   if (!releaseResp.ok) {
-    throw new Error(
+    throw Object.assign(new Error(
       `Failed to look up published rp_util schema release for ${tag}: ` +
       `${releaseResp.status} ${releaseResp.statusText}`
-    )
+    ), { status: releaseResp.status })
   }
   const release = await releaseResp.json()
 
@@ -333,9 +421,9 @@ async function fetchPublishedSchema(tag) {
       headers: { ...headers, Accept: 'application/octet-stream' }
     })
     if (!assetResp.ok) {
-      throw new Error(
+      throw Object.assign(new Error(
         `Failed to download published rp_util schema asset ${asset.name}: ${assetResp.status}`
-      )
+      ), { status: assetResp.status })
     }
     let payload
     try {
@@ -437,6 +525,7 @@ async function getRpUtilSchema(ref, options = {}) {
 
 module.exports = {
   getRpUtilSchema,
+  schemaExpectation,
   SCHEMA_FLAGS,
   // exported for testing
   cloneStreamingEnterprise,