@redpanda-data/docs-extensions-and-macros 5.0.4 → 5.1.1

package/extensions/algolia-indexer/generate-index.js

@@ -292,40 +292,95 @@ function generateIndex (playbook, contentCatalog, { indexLatestOnly = false, exc
       ? page.asciidoc.attributes['page-commercial-names'].split(',').map(name => name.trim())
       : []
 
-    // FIXED: keywords now included in index item
-    const indexItem = {
-      title: documentTitle,
-      version: version,
-      text: text,
-      intro: intro,
-      objectID: urlPath + page.pub.url,
-      titles: titles,
-      keywords: keywords,
-      categories: categories,
-      commercialNames: commercialNames,
-      unixTimestamp: unixTimestamp
-    }
+    // Algolia only indexes roughly the first ~290 words of any single record. On
+    // long reference pages (cluster/topic properties, metrics) every property or
+    // metric name lives in the `titles` array, and there can be hundreds of them, so
+    // every name past that window is stored but NOT searchable (DOC-1878). Split the
+    // titles of such pages across multiple records, each small enough to be fully
+    // indexed. Normal pages keep a single record. Chunks after the first get a unique
+    // objectID anchored to their first heading so search results deep-link into the
+    // page; the first chunk keeps the page's base objectID.
+    const baseObjectID = urlPath + page.pub.url
+    const titleChunks = isPropertiesPage ? chunkTitles(titles) : [titles]
+
+    titleChunks.forEach((chunk, chunkIndex) => {
+      const objectID = chunkIndex === 0
+        ? baseObjectID
+        : `${baseObjectID}#${chunk[0].h}`
+
+      // keywords included in index item
+      const indexItem = {
+        title: documentTitle,
+        version: version,
+        text: text,
+        intro: intro,
+        objectID: objectID,
+        // Clean page path (no #fragment) shared by every chunk of a page. The search
+        // UI builds result links from `url` and appends its own matched-heading
+        // anchor, so chunk objectIDs (which carry a #anchor for uniqueness) must not
+        // be used for the href. `url` is also the attribute to dedupe chunks on.
+        url: baseObjectID,
+        titles: chunk,
+        keywords: keywords,
+        categories: categories,
+        commercialNames: commercialNames,
+        unixTimestamp: unixTimestamp
+      }
 
-    if (component.name !== 'labs') {
-      indexItem.product = component.title
-      indexItem.breadcrumbs = breadcrumbs
-      indexItem.type = 'Doc'
-      indexItem._tags = Array.isArray(tag) ? tag : [tag]
-    } else {
-      indexItem.deployment = deployment
-      indexItem.type = 'Lab'
-      indexItem.interactive = false
-      indexItem._tags = Array.isArray(tag) ? tag : [tag]
-    }
+      if (component.name !== 'labs') {
+        indexItem.product = component.title
+        indexItem.breadcrumbs = breadcrumbs
+        indexItem.type = 'Doc'
+        indexItem._tags = Array.isArray(tag) ? tag : [tag]
+      } else {
+        indexItem.deployment = deployment
+        indexItem.type = 'Lab'
+        indexItem.interactive = false
+        indexItem._tags = Array.isArray(tag) ? tag : [tag]
+      }
 
-    algolia[cname][version].push(indexItem)
-    algoliaCount++
+      algolia[cname][version].push(indexItem)
+      algoliaCount++
+    })
   }
 
   logger.info(`Indexed ${algoliaCount} pages`)
   return algolia
 }
 
+/**
+ * Split a titles array into chunks whose combined token count stays within
+ * Algolia's per-record indexing window (~290 words; we leave a safety margin).
+ * Each returned chunk is small enough that every heading it contains is actually
+ * indexed for search, which keeps property/metric names on long reference pages
+ * searchable (DOC-1878).
+ *
+ * @param {Array<{t: string, h: string}>} titles - Section headings for a page.
+ * @param {number} [maxTokens=180] - Approximate token budget per chunk.
+ * @returns {Array<Array<{t: string, h: string}>>} One or more title chunks. An
+ *   empty input yields a single empty chunk so the page still gets a record.
+ */
+function chunkTitles (titles, maxTokens = 180) {
+  if (!titles.length) return [[]]
+  const chunks = []
+  let current = []
+  let tokens = 0
+  for (const entry of titles) {
+    // Algolia tokenizes on separators (whitespace, underscores, dots, etc.), so a
+    // name like `partition_autobalancing_mode` counts as several tokens.
+    const entryTokens = String(entry.t || '').split(/[\s_./:-]+/).filter(Boolean).length || 1
+    if (current.length && tokens + entryTokens > maxTokens) {
+      chunks.push(current)
+      current = []
+      tokens = 0
+    }
+    current.push(entry)
+    tokens += entryTokens
+  }
+  if (current.length) chunks.push(current)
+  return chunks
+}
+
 /**
  * Extract the path from a URL
  * @param {string} url - The URL to extract path from

package/extensions/version-fetcher/set-latest-version.js

@@ -135,97 +135,12 @@ module.exports.register = function ({ config }) {
         }
       });
 
-      // Find latest available Redpanda properties JSON in content catalog
-      // This ensures we use a version that actually exists, even if latest-redpanda-tag
-      // points to a newer release that doesn't have docs/attachments yet
-      // Component name is 'streaming' (or 'ROOT' for backwards compatibility)
-      const propertiesFiles = contentCatalog.findBy({
-        family: 'attachment'
-      }).filter(f => {
-        if (!f?.src?.stem || !f?.src?.component) return false;
-        const isStreamingComponent = f.src.component === 'streaming' || f.src.component === 'ROOT';
-        const isReferenceModule = f.src.module === 'reference';
-        const isPropertiesFile = f.src.stem.startsWith('redpanda-properties-');
-        return isStreamingComponent && isReferenceModule && isPropertiesFile;
-      });
-
-      let availablePropertiesTag = null;
-      if (propertiesFiles.length > 0) {
-        const versions = propertiesFiles
-          .map(f => f.src.stem.replace('redpanda-properties-', ''))
-          // Only accept stable versions (no prerelease suffix like -backup, -rc1, etc.)
-          .filter(v => {
-            const normalized = v.replace(/^v/, '');
-            return semver.valid(normalized) && semver.prerelease(normalized) === null;
-          });
-
-        if (versions.length > 0) {
-          availablePropertiesTag = versions.sort((a, b) =>
-            semver.rcompare(a.replace(/^v/, ''), b.replace(/^v/, ''))
-          )[0];
-
-          // Set on all component versions
-          components.forEach(component => {
-            component.versions.forEach(({ asciidoc }) => {
-              asciidoc.attributes['available-properties-tag'] = availablePropertiesTag;
-            });
-          });
-
-          logger.debug(`Found ${propertiesFiles.length} properties JSON files, latest available: ${availablePropertiesTag}`);
-        }
-      } else {
-        logger.debug('No properties JSON files found in content catalog');
-      }
-
-      // Find latest available Connect JSON in content catalog
-      // Component name is 'connect' (or 'redpanda-connect' for backwards compatibility)
-      const connectFiles = contentCatalog.findBy({
-        family: 'attachment'
-      }).filter(f => {
-        if (!f?.src?.stem || !f?.src?.component) return false;
-        const isConnectComponent = f.src.component === 'connect' || f.src.component === 'redpanda-connect';
-        const isComponentsModule = f.src.module === 'components';
-        const isConnectFile = f.src.stem.startsWith('connect-');
-        return isConnectComponent && isComponentsModule && isConnectFile;
-      });
-
-      let availableConnectVersion = null;
-      if (connectFiles.length > 0) {
-        const versions = connectFiles
-          .map(f => f.src.stem.replace('connect-', ''))
-          // Only accept stable versions (no prerelease suffix)
-          .filter(v => semver.valid(v) && semver.prerelease(v) === null);
-
-        if (versions.length > 0) {
-          availableConnectVersion = versions.sort((a, b) =>
-            semver.rcompare(a, b)
-          )[0];
-
-          // Set on all component versions
-          components.forEach(component => {
-            component.versions.forEach(({ asciidoc }) => {
-              asciidoc.attributes['available-connect-version'] = availableConnectVersion;
-            });
-          });
-
-          logger.debug(`Found ${connectFiles.length} connect JSON files, latest available: ${availableConnectVersion}`);
-        }
-      } else {
-        logger.debug('No connect JSON files found in content catalog');
-      }
-
       logger.info('Updated Redpanda documentation versions successfully:');
       logger.info(`- Redpanda: ${latestVersions.redpanda.latestRedpandaRelease.version}${latestVersions.redpanda.latestRcRelease ? ', beta: ' + latestVersions.redpanda.latestRcRelease.version : ''}`);
       logger.info(`- Connect: ${latestVersions.connect}`);
       logger.info(`- Console: ${latestVersions.console.latestStableRelease}${latestVersions.console.latestBetaRelease ? ', beta: ' + latestVersions.console.latestBetaRelease : ''}`);
       logger.info(`- Operator: ${latestVersions.operator?.latestStableRelease || 'unknown'}${latestVersions.operator?.latestBetaRelease ? ', beta: ' + latestVersions.operator.latestBetaRelease : ''}`);
       logger.info(`- Helm chart: ${latestVersions.helmChart?.latestStableRelease || 'unknown'}${latestVersions.helmChart?.latestBetaRelease ? ', beta: ' + latestVersions.helmChart.latestBetaRelease : ''}`);
-      if (availablePropertiesTag) {
-        logger.info(`- Available properties JSON: ${availablePropertiesTag}`);
-      }
-      if (availableConnectVersion) {
-        logger.info(`- Available Connect JSON: ${availableConnectVersion}`);
-      }
     } catch (error) {
       logger.error(`Error updating versions: ${error}`);
     }
@@ -259,50 +174,4 @@ module.exports.register = function ({ config }) {
       }
     });
   }
-};
-
-// Export helper functions for testing
-module.exports.filterPropertiesFiles = function(files) {
-  return files.filter(f => {
-    if (!f?.src?.stem || !f?.src?.component) return false;
-    const isStreamingComponent = f.src.component === 'streaming' || f.src.component === 'ROOT';
-    const isReferenceModule = f.src.module === 'reference';
-    const isPropertiesFile = f.src.stem.startsWith('redpanda-properties-');
-    return isStreamingComponent && isReferenceModule && isPropertiesFile;
-  });
-};
-
-module.exports.filterConnectFiles = function(files) {
-  return files.filter(f => {
-    if (!f?.src?.stem || !f?.src?.component) return false;
-    const isConnectComponent = f.src.component === 'connect' || f.src.component === 'redpanda-connect';
-    const isComponentsModule = f.src.module === 'components';
-    const isConnectFile = f.src.stem.startsWith('connect-');
-    return isConnectComponent && isComponentsModule && isConnectFile;
-  });
-};
-
-module.exports.extractLatestPropertiesVersion = function(files, semver) {
-  const versions = files
-    .map(f => f.src.stem.replace('redpanda-properties-', ''))
-    .filter(v => {
-      const normalized = v.replace(/^v/, '');
-      return semver.valid(normalized) && semver.prerelease(normalized) === null;
-    });
-
-  if (versions.length === 0) return null;
-
-  return versions.sort((a, b) =>
-    semver.rcompare(a.replace(/^v/, ''), b.replace(/^v/, ''))
-  )[0];
-};
-
-module.exports.extractLatestConnectVersion = function(files, semver) {
-  const versions = files
-    .map(f => f.src.stem.replace('connect-', ''))
-    .filter(v => semver.valid(v) && semver.prerelease(v) === null);
-
-  if (versions.length === 0) return null;
-
-  return versions.sort((a, b) => semver.rcompare(a, b))[0];
 };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@redpanda-data/docs-extensions-and-macros",
-  "version": "5.0.4",
+  "version": "5.1.1",
   "description": "Antora extensions and macros developed for Redpanda documentation.",
   "keywords": [
     "antora",
@@ -109,7 +109,7 @@
     "@octokit/plugin-retry": "^7.1.1",
     "@octokit/rest": "^21.1.1",
     "@redocly/cli": "^2.2.0",
-    "ajv": "^8.12.0",
+    "ajv": "^8.20.0",
     "algoliasearch": "^4.17.0",
     "chalk": "4.1.2",
     "cheerio": "^1.1.2",

package/tools/rpk-docs/DEPRECATION-SCANNING.md

@@ -0,0 +1,192 @@
+# Automatic Deprecation Detection for rpk Commands
+
+## Overview
+
+The rpk documentation automation now automatically scans Go source code to detect commands marked as `Hidden: true` or `Deprecated: "message"` in Cobra command definitions. This ensures deprecated commands are properly documented with warnings even though they don't appear in `rpk --print-tree` output.
+
+## How It Works
+
+### 1. Source Code Scanning
+
+The `scan-deprecated-commands.js` module scans the rpk Go source code looking for:
+
+```go
+cmd := &cobra.Command{
+    Use:        "admin",
+    Hidden:     true,
+    Deprecated: "use `rpk cluster` subcommands; see...",
+}
+```
+
+It extracts:
+- Command name (`Use` field)
+- Hidden status (`Hidden: true`)
+- Deprecation message (`Deprecated: "..."`)
+
+### 2. Automatic Override Generation
+
+When detected, the scanner automatically generates overrides in `rpk-overrides.json`:
+
+```json
+{
+  "commands": {
+    "rpk redpanda admin": {
+      "deprecated": true,
+      "deprecatedInVersion": "v26.x",
+      "replacement": "Use xref:reference:rpk/rpk-cluster/rpk-cluster.adoc[`rpk cluster`] instead.",
+      "deprecatedMessage": "use `rpk cluster` subcommands; see...",
+      "_note": "Hidden: true, found by scanning Go source"
+    }
+  }
+}
+```
+
+### 3. Template Rendering
+
+The Handlebars template (`command.hbs`) automatically renders deprecation warnings:
+
+```handlebars
+{{#if deprecated}}
+[CAUTION]
+====
+This command is deprecated{{#if deprecatedInVersion}} as of {{deprecatedInVersion}}{{/if}}.
+{{#if replacement}} {{{replacement}}}{{/if}}
+====
+{{/if}}
+```
+
+This generates:
+
+```asciidoc
+[CAUTION]
+====
+This command is deprecated as of v26.x. Use xref:reference:rpk/rpk-cluster/rpk-cluster.adoc[`rpk cluster`] instead.
+====
+```
+
+## Integration
+
+The deprecation scanner runs automatically during documentation generation:
+
+1. **After building rpk** - Source code is available
+2. **Before generating docs** - Overrides are updated first
+3. **Writes to overrides.json** - Automatically merges detected deprecations
+
+To trigger manually:
+
+```bash
+# Scan source and update overrides
+node tools/rpk-docs/scan-deprecated-commands.js \
+  /path/to/redpanda/src/go/rpk \
+  docs-data/rpk-overrides.json
+```
+
+## Currently Detected Commands
+
+As of the latest scan (June 8, 2026):
+
+1. `rpk benchmark` - Hidden (internal testing command)
+2. `rpk cloud resourcegroup` - Hidden (internal/unreleased)
+3. `rpk redpanda admin` - **Deprecated** in favor of `rpk cluster`
+4. `rpk redpanda admin brokers` - **Deprecated** → use `rpk cluster brokers`
+5. `rpk redpanda admin config` - **Deprecated** → use `rpk cluster config`
+6. `rpk redpanda admin partitions` - **Deprecated** → use `rpk cluster info --detailed`
+
+## Limitations
+
+### Hidden Commands Not in Tree
+
+Commands with `Hidden: true` don't appear in `rpk --print-tree` output, so:
+- They won't be auto-generated from scratch
+- Existing docs files are preserved
+- Overrides contain metadata for future use
+
+### Subcommand Detection
+
+The scanner finds commands with their own `NewCommand()` function but doesn't detect subcommands added via `cmd.AddCommand()`. For example:
+
+- ✅ Detects: `rpk redpanda admin` (has NewCommand in admin.go)
+- ❌ Misses: `rpk redpanda admin brokers list` (subcommand of brokers)
+
+**Workaround**: Manually add subcommand deprecations to overrides or enhance scanner to follow AddCommand calls.
+
+## File Locations
+
+- **Scanner**: `tools/rpk-docs/scan-deprecated-commands.js`
+- **Integration**: `tools/rpk-docs/rpk-docs-handler.js` (Step 5)
+- **Template**: `tools/rpk-docs/templates/command.hbs`
+- **Overrides**: `docs-data/rpk-overrides.json`
+
+## Future Enhancements
+
+1. **Follow AddCommand chains** - Detect all subcommand deprecations
+2. **Extract deprecation versions** - Parse version info from comments/git history
+3. **Build xref links automatically** - Generate xrefs from deprecation messages
+4. **Scan for removed commands** - Compare against previous versions to detect removals
+
+## Example: rpk redpanda admin
+
+### Source Code
+
+```go
+func NewCommand(fs afero.Fs, p *config.Params) *cobra.Command {
+    cmd := &cobra.Command{
+        Use:        "admin",
+        Short:      "Talk to the Redpanda admin listener",
+        Hidden:     true,
+        Deprecated: "use `rpk cluster` subcommands; see `rpk cluster brokers`, `rpk cluster info --detailed`, `rpk cluster config list --node-id`, and `rpk cluster loggers`",
+    }
+    // ...
+}
+```
+
+### Generated Override
+
+```json
+{
+  "rpk redpanda admin": {
+    "deprecated": true,
+    "replacement": "Use xref:reference:rpk/rpk-cluster/rpk-cluster.adoc[`rpk cluster`] instead.",
+    "deprecatedMessage": "use `rpk cluster` subcommands; see `rpk cluster brokers`, `rpk cluster info --detailed`, `rpk cluster config list --node-id`, and `rpk cluster loggers`",
+    "_note": "Hidden: true, found by scanning Go source"
+  }
+}
+```
+
+### Generated Documentation
+
+```asciidoc
+= rpk redpanda admin
+:unsupported-os: macOS, Windows
+
+include::reference:partial$unsupported-os-rpk.adoc[]
+
+[CAUTION]
+====
+This command is deprecated. Use xref:reference:rpk/rpk-cluster/rpk-cluster.adoc[`rpk cluster`] instead.
+====
+
+Talk to the Redpanda admin listener.
+```
+
+## Testing
+
+Test the scanner:
+
+```bash
+# Test on local source
+node tools/rpk-docs/scan-deprecated-commands.js ~/Documents/redpanda/src/go/rpk
+
+# Test with override merging
+node tools/rpk-docs/scan-deprecated-commands.js \
+  ~/Documents/redpanda/src/go/rpk \
+  docs-data/rpk-overrides.json
+```
+
+Test full automation:
+
+```bash
+cd /path/to/docs
+npx doc-tools generate rpk-docs --ref dev
+# Check for "Scanning source for deprecated/hidden commands..." in output
+```

package/tools/rpk-docs/extract-conditional-content.js

@@ -0,0 +1,169 @@
+#!/usr/bin/env node
+
+/**
+ * Extract conditional blocks (ifdef/ifndef) and includes from original rpk docs
+ * and add them to rpk-overrides.json
+ */
+
+const fs = require('fs')
+const path = require('path')
+const { spawnSync } = require('child_process')
+
+const docsDir = process.argv[2] || '../docs'
+const rpkDir = path.join(docsDir, 'modules/reference/pages/rpk')
+const overridesPath = path.join(__dirname, '../../docs-data/rpk-overrides.json')
+
+// Validate docsDir exists and is a directory
+if (!fs.existsSync(docsDir) || !fs.statSync(docsDir).isDirectory()) {
+  console.error(`ERROR: Invalid docs directory: ${docsDir}`)
+  process.exit(1)
+}
+
+console.log('Extracting conditional content from original rpk docs...')
+console.log(`  Docs directory: ${docsDir}`)
+console.log(`  Overrides file: ${overridesPath}`)
+
+// Get list of modified files using spawnSync to avoid command injection
+const gitDiffResult = spawnSync('git', ['diff', '--name-only', 'modules/reference/pages/rpk/'], {
+  cwd: docsDir,
+  encoding: 'utf8'
+})
+
+if (gitDiffResult.error) {
+  console.error(`ERROR: Failed to run git diff: ${gitDiffResult.error.message}`)
+  process.exit(1)
+}
+
+const modifiedFilesOutput = gitDiffResult.stdout
+
+const modifiedFiles = modifiedFilesOutput
+  .trim()
+  .split('\n')
+  .filter(f => f.endsWith('.adoc'))
+
+console.log(`\nFound ${modifiedFiles.length} modified files`)
+
+// Load existing overrides
+let overrides = {}
+if (fs.existsSync(overridesPath)) {
+  overrides = JSON.parse(fs.readFileSync(overridesPath, 'utf8'))
+}
+
+let extractedCount = 0
+let filesWithContent = 0
+
+for (const relPath of modifiedFiles) {
+  // Get the command path from filename
+  // e.g., "rpk/rpk-cluster/rpk-cluster-config.adoc" → "rpk cluster config"
+  const filename = path.basename(relPath, '.adoc')
+  const commandPath = filename.replace(/-/g, ' ')
+
+  // Get the original content (before changes)
+  try {
+    const gitShowResult = spawnSync('git', ['show', `HEAD:${relPath}`], {
+      cwd: docsDir,
+      encoding: 'utf8'
+    })
+
+    if (gitShowResult.error || gitShowResult.status !== 0) {
+      throw new Error(gitShowResult.stderr || 'git show failed')
+    }
+
+    const originalContent = gitShowResult.stdout
+
+    // Extract conditional blocks and includes
+    const conditionals = []
+    const includes = []
+
+    const lines = originalContent.split('\n')
+    let inConditional = false
+    let conditionalStart = -1
+    let conditionalLines = []
+    let conditionalType = null
+
+    for (let i = 0; i < lines.length; i++) {
+      const line = lines[i]
+
+      // Check for conditional start
+      if (/^(ifdef|ifndef)::/.test(line)) {
+        inConditional = true
+        conditionalStart = i
+        conditionalLines = [line]
+        conditionalType = line
+      } else if (inConditional) {
+        conditionalLines.push(line)
+        if (/^endif::/.test(line)) {
+          // End of conditional block
+          conditionals.push({
+            startLine: conditionalStart,
+            endLine: i,
+            type: conditionalType,
+            content: conditionalLines.join('\n')
+          })
+          inConditional = false
+          conditionalLines = []
+        }
+      }
+
+      // Check for includes
+      if (/^include::/.test(line)) {
+        includes.push({
+          line: i,
+          content: line
+        })
+      }
+    }
+
+    // Add to overrides if we found content
+    if (conditionals.length > 0 || includes.length > 0) {
+      filesWithContent++
+
+      if (!overrides[commandPath]) {
+        overrides[commandPath] = { content: [] }
+      }
+
+      if (!overrides[commandPath].content) {
+        overrides[commandPath].content = []
+      }
+
+      // Add conditionals
+      for (const cond of conditionals) {
+        const lines = cond.content.split('\n')
+        const isCloudOnly = cond.type.includes('ifdef::env-cloud')
+        const isSelfHostedOnly = cond.type.includes('ifndef::env-cloud')
+
+        // Extract the actual content (skip ifdef/endif lines)
+        const innerContent = lines.slice(1, -1).join('\n').trim()
+
+        if (innerContent) {
+          overrides[commandPath].content.push({
+            type: isCloudOnly ? 'cloud-only' : 'self-hosted',
+            position: 'after_description',
+            content: innerContent,
+            note: `Extracted from line ${cond.startLine + 1}`
+          })
+          extractedCount++
+        }
+      }
+
+      // Add includes
+      for (const inc of includes) {
+        overrides[commandPath].content.push({
+          type: 'include',
+          position: 'after_description',
+          path: inc.content,
+          note: `Extracted from line ${inc.line + 1}`
+        })
+        extractedCount++
+      }
+    }
+  } catch (err) {
+    console.warn(`  ⚠ Could not read original ${relPath}: ${err.message}`)
+  }
+}
+
+console.log(`\nExtracted ${extractedCount} content items from ${filesWithContent} files`)
+
+// Save overrides
+fs.writeFileSync(overridesPath, JSON.stringify(overrides, null, 2))
+console.log(`\n✓ Updated ${overridesPath}`)