@redpanda-data/docs-extensions-and-macros 5.50.0 → 5.51.1

package/extension-utils/llms-utils.js

@@ -37,9 +37,14 @@ function escapeRegExp(str) {
  * Regex pattern to match and strip the llms directive from markdown content.
  * Dynamically derived from LLMS_DIRECTIVE_BASE to stay in sync.
  * Matches the blockquote format with optional component-specific suffix.
+ * The directive is written with a root-relative link, but links are made
+ * absolute before the exports strip it, so any prefix before /llms.txt matches.
  */
 const LLMS_DIRECTIVE_REGEX = new RegExp(
-  `^> ${escapeRegExp(LLMS_DIRECTIVE_BASE)}.*$`,
+  `^> ${escapeRegExp(LLMS_DIRECTIVE_BASE).replace(
+    escapeRegExp('(/llms.txt)'),
+    '\\([^)\\s]*/llms\\.txt\\)'
+  )}.*$`,
   'gm'
 );
 
@@ -96,11 +101,343 @@ function componentsWithExports(components, pages) {
   );
 }
 
+/**
+ * Maximum characters per page index file. The Agent-Friendly Documentation
+ * Spec passes an llms.txt index at 50,000 characters or less and recommends
+ * splitting a large index into section-level files that each stay under that
+ * limit. Leave a buffer below 50K.
+ */
+const MAX_INDEX_CHARS = 45000;
+
+/**
+ * Descriptions longer than this are cut at a word boundary so that one verbose
+ * page doesn't force an extra split.
+ */
+const MAX_INDEX_DESCRIPTION_CHARS = 200;
+
+function oneLine(text) {
+  return String(text || '')
+    .replace(/<[^>]+>/g, '')
+    .replace(/\s+/g, ' ')
+    .trim();
+}
+
+function truncateDescription(text) {
+  const clean = oneLine(text);
+  if (clean.length <= MAX_INDEX_DESCRIPTION_CHARS) return clean;
+  const cut = clean.slice(0, MAX_INDEX_DESCRIPTION_CHARS);
+  const lastSpace = cut.lastIndexOf(' ');
+  return `${(lastSpace > 0 ? cut.slice(0, lastSpace) : cut).replace(/[\s.,;:]+$/, '')}...`;
+}
+
+// Square brackets in link text end the link early in markdown.
+function escapeLinkText(text) {
+  return oneLine(text).replace(/([[\]])/g, '\\$1');
+}
+
+/**
+ * A URL made safe as a Markdown link destination. An unmatched `)` ends the
+ * destination early, and a space or `<` ends it outright, so those are
+ * percent-encoded. Encoding keeps the plain `[text](url)` shape llms.txt
+ * readers expect, where an angle-bracket destination would not.
+ */
+function linkDestination(url) {
+  return String(url).replace(/[()<> ]/g, (c) => `%${c.charCodeAt(0).toString(16).toUpperCase()}`);
+}
+
+/**
+ * Longest shared directory of a set of URL paths, always ending in `/`.
+ * `/streaming/26.1/manage/a/` and `/streaming/26.1/deploy/b/` share `/streaming/26.1/`.
+ */
+function commonDirectory(urls) {
+  const dirs = urls.map((url) => url.replace(/[^/]*\/?$/, '').split('/').filter(Boolean));
+  if (!dirs.length) return '/';
+  const shared = [];
+  for (let i = 0; i < dirs[0].length; i++) {
+    const segment = dirs[0][i];
+    if (!dirs.every((d) => d[i] === segment)) break;
+    shared.push(segment);
+  }
+  return shared.length ? `/${shared.join('/')}/` : '/';
+}
+
+/**
+ * URL of the component version's root, derived from how Antora built the page
+ * URL: the version root, then the module name (omitted for ROOT), then the
+ * page's relative path (index pages drop their own segment). Works for both
+ * indexified (`/a/b/`) and `.html` URLs, since the page is one segment either
+ * way, except an index page, whose `.html` URL has an `index.html` segment
+ * that its indexified URL does not.
+ * Returns null when the page lacks the source fields needed.
+ */
+function versionRootOf(page) {
+  const { module: moduleName, relative } = page.src || {};
+  if (!relative || !moduleName) return null;
+  const relSegments = relative.replace(/\.adoc$/, '').split('/').filter(Boolean);
+  const urlSegments = page.pub.url.split('/').filter(Boolean);
+  if (relSegments[relSegments.length - 1] === 'index') {
+    relSegments.pop();
+    // An index page keeps an `index.html` segment of its own when URLs are
+    // not indexified (`/streaming/26.1/index.html`), and has none when they
+    // are (`/streaming/26.1/`).
+    if (urlSegments[urlSegments.length - 1] === 'index.html') urlSegments.pop();
+  }
+  const tailCount = relSegments.length + (moduleName === 'ROOT' ? 0 : 1);
+  if (tailCount > urlSegments.length) return null;
+  const rootSegments = urlSegments.slice(0, urlSegments.length - tailCount);
+  return rootSegments.length ? `/${rootSegments.join('/')}/` : '/';
+}
+
+function mostCommon(values) {
+  const counts = new Map();
+  values.forEach((v) => counts.set(v, (counts.get(v) || 0) + 1));
+  let best = null;
+  counts.forEach((count, value) => {
+    if (best === null || count > counts.get(best)) best = value;
+  });
+  return best;
+}
+
+/**
+ * The directory segment directly under `dir` that contains `url`, or null when
+ * the page sits in `dir` itself. With indexified URLs a page `/a/b/` lives in
+ * `/a/`, so the last segment is the page, not a directory.
+ */
+function childSegment(url, dir) {
+  const rest = url.slice(dir.length).replace(/\/$/, '').replace(/\/index\.html$/, '');
+  const segments = rest.split('/').filter(Boolean);
+  return segments.length > 1 ? segments[0] : null;
+}
+
+function latestDate(dates) {
+  const times = dates.filter(Boolean).map((d) => new Date(d)).filter((d) => !isNaN(d));
+  return times.length ? new Date(Math.max(...times)).toISOString() : null;
+}
+
+function renderIndexFile({ heading, intro, entries }) {
+  let out = `# ${heading}\n\n`;
+  intro.forEach((line) => {
+    out += `> ${line}\n`;
+  });
+  out += `\n## Pages\n\n`;
+  entries.forEach((entry) => {
+    out += `${entry.line}\n`;
+  });
+  return out;
+}
+
+/**
+ * Pack entries into chunks whose rendered size stays under the limit.
+ * Used only when a single directory holds more pages than one file allows.
+ */
+function chunkEntries(entries, overhead, maxChars) {
+  const chunks = [];
+  let current = [];
+  let size = overhead;
+  entries.forEach((entry) => {
+    const lineSize = entry.line.length + 1;
+    if (current.length && size + lineSize > maxChars) {
+      chunks.push(current);
+      current = [];
+      size = overhead;
+    }
+    current.push(entry);
+    size += lineSize;
+  });
+  if (current.length) chunks.push(current);
+  return chunks;
+}
+
+/**
+ * Build page index files for every published component version.
+ *
+ * Each component version gets an index at `<version root>/llms.txt` listing
+ * every page as a markdown link to its .md URL. When a version's index would
+ * exceed `maxChars`, it is split by URL directory, recursively, so each file
+ * stays under the limit. The root llms.txt links to every file this returns,
+ * which keeps every page one hop from the root index.
+ *
+ * @param {Object} options
+ * @param {Array} options.pages - Antora pages that have markdown (page.markdownContents)
+ * @param {Array} options.components - Antora components
+ * @param {string} options.siteUrl - Absolute site URL without a trailing slash
+ * @param {Function} options.toMarkdownUrl - Maps a page's HTML URL to its .md URL
+ * @param {number} [options.maxChars] - Size limit per file
+ * @returns {Array<{path: string, url: string, title: string, component: string,
+ *   version: string, isLatest: boolean, pageCount: number, contents: string}>}
+ */
+function buildPageIndexes({ pages, components, siteUrl, toMarkdownUrl, maxChars = MAX_INDEX_CHARS }) {
+  const componentsByName = new Map((components || []).map((c) => [c.name, c]));
+  const groups = new Map();
+
+  (pages || []).forEach((page) => {
+    const url = page.pub && page.pub.url;
+    if (!url || !page.out || !page.src) return;
+    const key = `${page.src.version}@${page.src.component}`;
+    if (!groups.has(key)) groups.set(key, []);
+    groups.get(key).push(page);
+  });
+
+  const indexes = [];
+
+  groups.forEach((groupPages) => {
+    const { component: componentName, version } = groupPages[0].src;
+    const component = componentsByName.get(componentName);
+    if (!component) return;
+    const componentVersion = (component.versions || []).find((v) => v.version === version) || {};
+    const latest = component.latest || (component.versions || [])[0] || {};
+    const isLatest = latest.version === version;
+    const versioned = (component.versions || []).length > 1;
+    const displayVersion = componentVersion.displayVersion || version;
+    const baseTitle = versioned
+      ? `${component.title} ${displayVersion}`
+      : component.title;
+
+    const entries = groupPages
+      .map((page) => {
+        const title = page.asciidoc && (page.asciidoc.doctitle || page.asciidoc.navtitle);
+        const description = page.asciidoc && page.asciidoc.attributes && page.asciidoc.attributes.description;
+        const mdUrl = `${siteUrl}${toMarkdownUrl(page.pub.url)}`;
+        let line = `- [${escapeLinkText(title || page.src.stem)}](${linkDestination(mdUrl)})`;
+        if (description) line += `: ${truncateDescription(description)}`;
+        const modified = page.asciidoc && page.asciidoc.attributes && page.asciidoc.attributes['page-git-modified-date'];
+        return { url: page.pub.url, line, modified };
+      })
+      .sort((a, b) => a.url.localeCompare(b.url));
+
+    const roots = groupPages.map(versionRootOf).filter(Boolean);
+    const versionRoot = roots.length ? mostCommon(roots) : commonDirectory(entries.map((e) => e.url));
+
+    const intro = (scope) => {
+      const lines = [
+        `Index of ${scope} documentation pages. Each link points to the page's markdown version.`,
+      ];
+      if (versioned && !isLatest) {
+        lines.push(`This is version ${displayVersion}. The latest version is ${latest.displayVersion || latest.version}.`);
+      }
+      lines.push(`For the full documentation index, see ${siteUrl}/llms.txt`);
+      return lines;
+    };
+
+    const makeIndex = (dir, dirEntries, label, part) => {
+      const heading = label ? `${baseTitle}: ${label}` : baseTitle;
+      const title = part > 1 ? `${heading} (part ${part})` : heading;
+      const path = `${dir.replace(/^\//, '')}${part > 1 ? `llms-${part}.txt` : 'llms.txt'}`;
+      return {
+        path,
+        url: `${siteUrl}/${path}`,
+        title,
+        component: componentName,
+        version,
+        isLatest,
+        pageCount: dirEntries.length,
+        lastModified: latestDate(dirEntries.map((e) => e.modified)),
+        contents: renderIndexFile({
+          heading: title,
+          intro: intro(label ? `${baseTitle} ${label}` : baseTitle),
+          entries: dirEntries,
+        }),
+      };
+    };
+
+    const split = (dir, dirEntries) => {
+      const label = dir.slice(versionRoot.length).replace(/\/$/, '');
+      const whole = makeIndex(dir, dirEntries, label, 1);
+      if (whole.contents.length <= maxChars) {
+        indexes.push(whole);
+        return;
+      }
+
+      const children = new Map();
+      dirEntries.forEach((entry) => {
+        const child = childSegment(entry.url, dir);
+        if (child === null) return;
+        if (!children.has(child)) children.set(child, []);
+        children.get(child).push(entry);
+      });
+
+      // Peel off the largest subdirectories into their own indexes until the
+      // rest fits. Small subdirectories stay in this directory's index, so the
+      // split produces as few files as the limit allows.
+      const fits = (list) => makeIndex(dir, list, label, 1).contents.length <= maxChars;
+      const bySize = [...children.entries()].sort((a, b) => b[1].length - a[1].length);
+      const kept = new Set(bySize.map(([child]) => child));
+      const remaining = () => dirEntries.filter((entry) => {
+        const child = childSegment(entry.url, dir);
+        return child === null || kept.has(child);
+      });
+      for (const [child, childEntries] of bySize) {
+        if (fits(remaining())) break;
+        kept.delete(child);
+        split(`${dir}${child}/`, childEntries);
+      }
+
+      const rest = remaining();
+      if (!rest.length) return;
+      // A directory whose own pages still exceed the limit is split into numbered parts.
+      // Header size of the largest possible part, so no chunk overflows once rendered.
+      const overhead = makeIndex(dir, [], label, 99).contents.length;
+      chunkEntries(rest, overhead, maxChars).forEach((chunk, i) => indexes.push(makeIndex(dir, chunk, label, i + 1)));
+    };
+
+    split(versionRoot, entries);
+  });
+
+  // Latest versions first, then older versions newest first, each in URL order.
+  return indexes.sort((a, b) => {
+    if (a.component !== b.component) return a.component.localeCompare(b.component);
+    if (a.isLatest !== b.isLatest) return a.isLatest ? -1 : 1;
+    if (a.version !== b.version) return b.version.localeCompare(a.version, undefined, { numeric: true });
+    return a.path.localeCompare(b.path);
+  });
+}
+
+/**
+ * Replace markdown links to the root llms.txt with their plain link text.
+ * A root index that links to itself is read as a nested index by tools that
+ * walk llms.txt, which pushes every page index one level deeper than intended.
+ */
+function unlinkSelfReferences(content, siteUrl) {
+  const self = new RegExp(`\\[([^\\]]*)\\]\\((?:${escapeRegExp(siteUrl)})?/llms\\.txt\\)`, 'g');
+  return content.replace(self, '$1');
+}
+
+/**
+ * Render the root llms.txt section that links to every page index file.
+ * Every index must be linked from the root so agents, and tools that follow
+ * one level of nested indexes, can reach every page.
+ */
+function renderPageIndexSection(indexes, components) {
+  const titles = new Map((components || []).map((c) => [c.name, c.title]));
+  let out = `## Page indexes\n\n`;
+  out += `Each file below lists pages as links to their markdown versions. Start with the latest version of a product unless you need a specific older version.\n\n`;
+  let currentComponent = null;
+  let olderHeadingShown = false;
+  indexes.forEach((index) => {
+    if (index.component !== currentComponent) {
+      currentComponent = index.component;
+      olderHeadingShown = false;
+      out += `${currentComponent === indexes[0].component ? '' : '\n'}### ${titles.get(index.component) || index.component}\n\n`;
+    }
+    if (!index.isLatest && !olderHeadingShown) {
+      olderHeadingShown = true;
+      out += `\nOlder versions:\n\n`;
+    }
+    out += `- [${escapeLinkText(index.title)}](${linkDestination(index.url)}): ${index.pageCount} ${index.pageCount === 1 ? 'page' : 'pages'}\n`;
+  });
+  return out;
+}
+
 module.exports = {
   LLMS_DIRECTIVE_BASE,
   LLMS_DIRECTIVE_REGEX,
   SOURCE_COMMENT_REGEX,
+  MAX_INDEX_CHARS,
   formatLlmsDirective,
+  linkDestination,
   stripMarkdownMetadata,
   componentsWithExports,
+  buildPageIndexes,
+  renderPageIndexSection,
+  unlinkSelfReferences,
 };

package/extensions/REFERENCE.adoc

@@ -1267,6 +1267,7 @@ The `convert-llms-to-txt` extension generates AI-optimized documentation exports
 - Generates `llms.txt` from a curated `llms.adoc` page in the home component
 - Creates `llms-full.txt` with all documentation pages
 - Generates component-specific exports (e.g., `ROOT-full.txt`, `cloud-full.txt`)
+- Generates a page index (`llms.txt`) for every published component version, linked from the root `llms.txt`
 - Unpublishes the HTML version of llms.adoc
 - Replaces site-url attribute dynamically for preview builds
 
@@ -1290,7 +1291,8 @@ Create `modules/home/pages/llms.adoc` in your docs repository with curated docum
 2. Place `llms.txt` at site root
 3. Generate `llms-full.txt` with all pages
 4. Generate component-specific full.txt files
-5. Unpublish the HTML version
+5. Generate a page index for every published component version and link each one from `llms.txt`
+6. Unpublish the HTML version
 
 === Generated files
 
@@ -1299,6 +1301,16 @@ Create `modules/home/pages/llms.adoc` in your docs repository with curated docum
 - `{site-url}/ROOT-full.txt` - Self-Managed docs only
 - `{site-url}/cloud-full.txt` - Cloud docs only
 - `{site-url}/redpanda-connect-full.txt` - Connect docs only
+- `{site-url}/<component>/<version>/llms.txt` - Page index for one component version
+- `{site-url}/<component>/llms.txt` - Page index for a component without versions, such as `connect/llms.txt`
+
+=== Page indexes
+
+Each page index lists every published page in one component version as a markdown link to the page's `.md` URL, followed by the page description when it has one. Older versions get their own indexes, which say which version they cover and name the latest version.
+
+An index that would exceed 45,000 characters is split by URL directory, for example `streaming/current/manage/llms.txt`. A directory that is still too large on its own is split into numbered parts (`llms.txt`, `llms-2.txt`, and so on). The Agent-Friendly Documentation Spec passes an `llms.txt` at 50,000 characters or less and recommends this kind of split for large sites.
+
+The root `llms.txt` links to every page index in a *Page indexes* section. The index list is never truncated, because an index missing from the root leaves its pages unreachable from `llms.txt`. Page indexes do not link to other page indexes, so every page is one hop from the root.
 
 === Attributes
 

package/extensions/convert-llms-to-txt.js

@@ -3,7 +3,13 @@
 const { raiseListenerLimit } = require('./util/raise-listener-limit')
 
 const { toMarkdownUrl } = require('../extension-utils/url-utils');
-const { stripMarkdownMetadata, componentsWithExports } = require('../extension-utils/llms-utils');
+const {
+  stripMarkdownMetadata,
+  componentsWithExports,
+  buildPageIndexes,
+  renderPageIndexSection,
+  unlinkSelfReferences,
+} = require('../extension-utils/llms-utils');
 
 /**
  * Extracts markdown from llms.adoc page and generates AI-friendly documentation exports.
@@ -18,6 +24,9 @@ const { stripMarkdownMetadata, componentsWithExports } = require('../extension-u
  * 5. Places llms.txt (markdown) at site root
  * 6. Generates llms-full.txt with markdown from latest versions of all components
  * 7. Generates component-specific full.txt files (e.g., redpanda-full.txt, cloud-full.txt)
+ * 8. Generates page index files (<version root>/llms.txt) for every published component
+ *    version, split by URL directory to stay under the size limit, and links each
+ *    one from the root llms.txt so every page is one hop from the root index
  *
  * Must run after convert-to-markdown extension to access page.markdownContents.
  */
@@ -110,6 +119,8 @@ module.exports.register = function () {
           logger.info('Unpublished llms HTML page');
         }
 
+        content = unlinkSelfReferences(content, siteUrl);
+
         // Store cleaned markdown content for adding after llms-full.txt
         llmsPage.llmsTxtContent = content;
 
@@ -142,145 +153,168 @@ module.exports.register = function () {
       return latestVersions.has(pageKey);
     });
 
-    if (!pages.length) {
-      logger.warn('No pages with markdown content found in latest versions, skipping llms-full.txt generation');
+    if (!allPages.length) {
+      logger.warn('No pages with markdown content found, skipping llms.txt generation');
       return;
     }
 
-    logger.info(`Filtered to ${pages.length} pages from ${latestVersions.size} latest component versions (from ${allPages.length} total pages)`);
-
-    let fullContent = `# Redpanda Documentation - Full Markdown Export\n\n`;
-    fullContent += `> This file contains all documentation pages in markdown format for AI agent consumption.\n`;
-    fullContent += `> Generated from ${pages.length} pages on ${new Date().toISOString()}\n`;
-    fullContent += `> Site: ${siteUrl}\n\n`;
-    fullContent += `## About This Export\n\n`;
-    fullContent += `This export includes only the **latest version** of each component's documentation:\n`;
-    components.forEach(component => {
-      const latest = component.latest || component.versions[0];
-      if (latest) {
-        fullContent += `- **${component.title}**: version ${latest.version}\n`;
-      }
-    });
-    fullContent += `\n`;
-    fullContent += `### AI-Friendly Documentation Formats\n\n`;
-    fullContent += `We provide multiple formats optimized for AI consumption:\n\n`;
-    fullContent += `- **${siteUrl}/llms.txt**: Curated overview following the llms.txt standard - start here for a quick introduction\n`;
-    fullContent += `- **${siteUrl}/llms-full.txt**: Complete documentation export (this file) - comprehensive reference with all pages\n`;
-    fullContent += `- **Component-specific exports**: Focused documentation for individual products:\n`;
-    // Only advertise exports for components that actually have pages — a
-    // `<name>-full.txt` is generated per component below, so listing corpus-less
-    // landing/utility components (e.g. data-platform, self-managed, search) would
-    // point the AI index at a 404.
-    componentsWithExports(components, pages).forEach(component => {
-      fullContent += `  - \`${siteUrl}/${component.name}-full.txt\`: ${component.title}\n`;
-    });
-    fullContent += `- **Individual markdown pages**: Each HTML page has a corresponding .md file (e.g., \`/docs/page.html\` → \`/docs/page.md\`)\n\n`;
-    fullContent += `### Accessing Versioned Content\n\n`;
-    fullContent += `For components with versioned documentation (like Redpanda Self-Managed), older versions can be accessed by replacing the version segment in the URL:\n`;
-    fullContent += `- Latest: \`${siteUrl}/current/page-path\`\n`;
-    fullContent += `- Specific version: \`${siteUrl}/24.3/page-path\`, \`${siteUrl}/25.1/page-path\`, etc.\n\n`;
-    fullContent += `Available versioned components: ${components.filter(c => c.versions.length > 1).map(c => c.name).join(', ')}\n\n`;
-    fullContent += `---\n\n`;
-
-    // Sort pages by URL for consistent ordering
-    pages.sort((a, b) => {
-      const urlA = a.pub?.url || '';
-      const urlB = b.pub?.url || '';
-      return urlA.localeCompare(urlB);
-    });
-
-    pages.forEach((page, index) => {
-      const mdUrl = page.pub?.url ? toMarkdownUrl(page.pub.url) : '';
-      const pageUrl = mdUrl ? `${siteUrl}${mdUrl}` : 'unknown';
-      const pageTitle = page.asciidoc?.doctitle || page.src?.stem || 'Untitled';
-
-      fullContent += `# Page ${index + 1}: ${pageTitle}\n\n`;
-      fullContent += `**URL**: ${pageUrl}\n\n`;
-      fullContent += `---\n\n`;
-      // Strip metadata (directive, source comments) from page content
-      fullContent += stripMarkdownMetadata(page.markdownContents);
-      fullContent += `\n\n---\n\n`;
-    });
-
-    // Add llms-full.txt to site root
-    siteCatalog.addFile({
-      contents: Buffer.from(fullContent, 'utf8'),
-      out: { path: 'llms-full.txt' },
-    });
-    logger.info(`Generated llms-full.txt with ${pages.length} pages`);
-
-    // Generate component-specific full.txt files
-    logger.info('Generating component-specific full.txt files...');
+    // The full exports cover the latest versions only, but page indexes cover
+    // every published version, so an empty latest set skips only the exports.
     const componentGroups = new Map();
-
-    // Group pages by component
-    pages.forEach(page => {
-      const componentName = page.src.component;
-      if (!componentGroups.has(componentName)) {
-        componentGroups.set(componentName, []);
-      }
-      componentGroups.get(componentName).push(page);
-    });
-
-    // Generate a full.txt file for each component
-    componentGroups.forEach((componentPages, componentName) => {
-      const component = components.find(c => c.name === componentName);
-      if (!component) return;
-
-      const latest = component.latest || component.versions[0];
-      if (!latest) return;
+    if (!pages.length) {
+      logger.warn('No pages with markdown content found in latest versions, skipping llms-full.txt generation');
+    } else {
+      logger.info(`Filtered to ${pages.length} pages from ${latestVersions.size} latest component versions (from ${allPages.length} total pages)`);
+
+      let fullContent = `# Redpanda Documentation - Full Markdown Export\n\n`;
+      fullContent += `> This file contains all documentation pages in markdown format for AI agent consumption.\n`;
+      fullContent += `> Generated from ${pages.length} pages on ${new Date().toISOString()}\n`;
+      fullContent += `> Site: ${siteUrl}\n\n`;
+      fullContent += `## About This Export\n\n`;
+      fullContent += `This export includes only the **latest version** of each component's documentation:\n`;
+      components.forEach(component => {
+        const latest = component.latest || component.versions[0];
+        if (latest) {
+          fullContent += `- **${component.title}**: version ${latest.version}\n`;
+        }
+      });
+      fullContent += `\n`;
+      fullContent += `### AI-Friendly Documentation Formats\n\n`;
+      fullContent += `We provide multiple formats optimized for AI consumption:\n\n`;
+      fullContent += `- **${siteUrl}/llms.txt**: Curated overview following the llms.txt standard - start here for a quick introduction\n`;
+      fullContent += `- **${siteUrl}/llms-full.txt**: Complete documentation export (this file) - comprehensive reference with all pages\n`;
+      fullContent += `- **Component-specific exports**: Focused documentation for individual products:\n`;
+      // Only advertise exports for components that actually have pages — a
+      // `<name>-full.txt` is generated per component below, so listing corpus-less
+      // landing/utility components (e.g. data-platform, self-managed, search) would
+      // point the AI index at a 404.
+      componentsWithExports(components, pages).forEach(component => {
+        fullContent += `  - \`${siteUrl}/${component.name}-full.txt\`: ${component.title}\n`;
+      });
+      fullContent += `- **Individual markdown pages**: Each HTML page has a corresponding .md file (e.g., \`/docs/page.html\` → \`/docs/page.md\`)\n\n`;
+      fullContent += `### Accessing Versioned Content\n\n`;
+      fullContent += `For components with versioned documentation (like Redpanda Self-Managed), older versions can be accessed by replacing the version segment in the URL:\n`;
+      fullContent += `- Latest: \`${siteUrl}/current/page-path\`\n`;
+      fullContent += `- Specific version: \`${siteUrl}/24.3/page-path\`, \`${siteUrl}/25.1/page-path\`, etc.\n\n`;
+      fullContent += `Available versioned components: ${components.filter(c => c.versions.length > 1).map(c => c.name).join(', ')}\n\n`;
+      fullContent += `---\n\n`;
 
       // Sort pages by URL for consistent ordering
-      componentPages.sort((a, b) => {
+      pages.sort((a, b) => {
         const urlA = a.pub?.url || '';
         const urlB = b.pub?.url || '';
         return urlA.localeCompare(urlB);
       });
 
-      let componentContent = `# ${component.title} - Full Markdown Export\n\n`;
-      componentContent += `> This file contains all ${component.title} documentation pages in markdown format for AI agent consumption.\n`;
-      componentContent += `> Generated from ${componentPages.length} pages on ${new Date().toISOString()}\n`;
-      componentContent += `> Component: ${component.name} | Version: ${latest.version}\n`;
-      componentContent += `> Site: ${siteUrl}\n\n`;
-      componentContent += `## About This Export\n\n`;
-      componentContent += `This export includes the **latest version** (${latest.version}) of the ${component.title} documentation.\n\n`;
-      componentContent += `### AI-Friendly Documentation Formats\n\n`;
-      componentContent += `We provide multiple formats optimized for AI consumption:\n\n`;
-      componentContent += `- **${siteUrl}/llms.txt**: Curated overview of all Redpanda documentation\n`;
-      componentContent += `- **${siteUrl}/llms-full.txt**: Complete documentation export with all components\n`;
-      componentContent += `- **${siteUrl}/${componentName}-full.txt**: This file - ${component.title} documentation only\n`;
-      componentContent += `- **Individual markdown pages**: Each HTML page has a corresponding .md file\n\n`;
-
-      if (component.versions.length > 1) {
-        componentContent += `### Accessing Older Versions\n\n`;
-        componentContent += `This component has versioned documentation. Older versions can be accessed by replacing the version segment in the URL:\n`;
-        componentContent += `- Latest: \`${siteUrl}/current/page-path\`\n`;
-        componentContent += `- Specific version: \`${siteUrl}/24.3/page-path\`, \`${siteUrl}/25.1/page-path\`, etc.\n\n`;
-      }
-
-      componentContent += `---\n\n`;
-
-      // Add all pages
-      componentPages.forEach((page, index) => {
+      pages.forEach((page, index) => {
         const mdUrl = page.pub?.url ? toMarkdownUrl(page.pub.url) : '';
         const pageUrl = mdUrl ? `${siteUrl}${mdUrl}` : 'unknown';
         const pageTitle = page.asciidoc?.doctitle || page.src?.stem || 'Untitled';
 
-        componentContent += `# Page ${index + 1}: ${pageTitle}\n\n`;
-        componentContent += `**URL**: ${pageUrl}\n\n`;
-        componentContent += `---\n\n`;
+        fullContent += `# Page ${index + 1}: ${pageTitle}\n\n`;
+        fullContent += `**URL**: ${pageUrl}\n\n`;
+        fullContent += `---\n\n`;
         // Strip metadata (directive, source comments) from page content
-        componentContent += stripMarkdownMetadata(page.markdownContents);
-        componentContent += `\n\n---\n\n`;
+        fullContent += stripMarkdownMetadata(page.markdownContents);
+        fullContent += `\n\n---\n\n`;
       });
 
-      // Add component-specific full.txt file to site root
+      // Add llms-full.txt to site root
       siteCatalog.addFile({
-        contents: Buffer.from(componentContent, 'utf8'),
-        out: { path: `${componentName}-full.txt` },
+        contents: Buffer.from(fullContent, 'utf8'),
+        out: { path: 'llms-full.txt' },
+      });
+      logger.info(`Generated llms-full.txt with ${pages.length} pages`);
+
+      // Generate component-specific full.txt files
+      logger.info('Generating component-specific full.txt files...');
+
+      // Group pages by component
+      pages.forEach(page => {
+        const componentName = page.src.component;
+        if (!componentGroups.has(componentName)) {
+          componentGroups.set(componentName, []);
+        }
+        componentGroups.get(componentName).push(page);
       });
-      logger.info(`Generated ${componentName}-full.txt with ${componentPages.length} pages`);
+
+      // Generate a full.txt file for each component
+      componentGroups.forEach((componentPages, componentName) => {
+        const component = components.find(c => c.name === componentName);
+        if (!component) return;
+
+        const latest = component.latest || component.versions[0];
+        if (!latest) return;
+
+        // Sort pages by URL for consistent ordering
+        componentPages.sort((a, b) => {
+          const urlA = a.pub?.url || '';
+          const urlB = b.pub?.url || '';
+          return urlA.localeCompare(urlB);
+        });
+
+        let componentContent = `# ${component.title} - Full Markdown Export\n\n`;
+        componentContent += `> This file contains all ${component.title} documentation pages in markdown format for AI agent consumption.\n`;
+        componentContent += `> Generated from ${componentPages.length} pages on ${new Date().toISOString()}\n`;
+        componentContent += `> Component: ${component.name} | Version: ${latest.version}\n`;
+        componentContent += `> Site: ${siteUrl}\n\n`;
+        componentContent += `## About This Export\n\n`;
+        componentContent += `This export includes the **latest version** (${latest.version}) of the ${component.title} documentation.\n\n`;
+        componentContent += `### AI-Friendly Documentation Formats\n\n`;
+        componentContent += `We provide multiple formats optimized for AI consumption:\n\n`;
+        componentContent += `- **${siteUrl}/llms.txt**: Curated overview of all Redpanda documentation\n`;
+        componentContent += `- **${siteUrl}/llms-full.txt**: Complete documentation export with all components\n`;
+        componentContent += `- **${siteUrl}/${componentName}-full.txt**: This file - ${component.title} documentation only\n`;
+        componentContent += `- **Individual markdown pages**: Each HTML page has a corresponding .md file\n\n`;
+
+        if (component.versions.length > 1) {
+          componentContent += `### Accessing Older Versions\n\n`;
+          componentContent += `This component has versioned documentation. Older versions can be accessed by replacing the version segment in the URL:\n`;
+          componentContent += `- Latest: \`${siteUrl}/current/page-path\`\n`;
+          componentContent += `- Specific version: \`${siteUrl}/24.3/page-path\`, \`${siteUrl}/25.1/page-path\`, etc.\n\n`;
+        }
+
+        componentContent += `---\n\n`;
+
+        // Add all pages
+        componentPages.forEach((page, index) => {
+          const mdUrl = page.pub?.url ? toMarkdownUrl(page.pub.url) : '';
+          const pageUrl = mdUrl ? `${siteUrl}${mdUrl}` : 'unknown';
+          const pageTitle = page.asciidoc?.doctitle || page.src?.stem || 'Untitled';
+
+          componentContent += `# Page ${index + 1}: ${pageTitle}\n\n`;
+          componentContent += `**URL**: ${pageUrl}\n\n`;
+          componentContent += `---\n\n`;
+          // Strip metadata (directive, source comments) from page content
+          componentContent += stripMarkdownMetadata(page.markdownContents);
+          componentContent += `\n\n---\n\n`;
+        });
+
+        // Add component-specific full.txt file to site root
+        siteCatalog.addFile({
+          contents: Buffer.from(componentContent, 'utf8'),
+          out: { path: `${componentName}-full.txt` },
+        });
+        logger.info(`Generated ${componentName}-full.txt with ${componentPages.length} pages`);
+      });
+    }
+
+    // Generate page index files for every published version (not only the latest),
+    // so the llms.txt tree lists every page the sitemap lists.
+    const pageIndexes = buildPageIndexes({
+      pages: allPages,
+      components,
+      siteUrl,
+      toMarkdownUrl,
     });
+    pageIndexes.forEach((index) => {
+      siteCatalog.addFile({
+        contents: Buffer.from(index.contents, 'utf8'),
+        out: { path: index.path },
+      });
+    });
+    const indexedPageCount = pageIndexes.reduce((n, index) => n + index.pageCount, 0);
+    logger.info(`Generated ${pageIndexes.length} page index files covering ${indexedPageCount} pages`);
 
     // Add llms.txt to site root (using content extracted earlier)
     if (llmsPage && llmsPage.llmsTxtContent) {
@@ -297,6 +331,15 @@ module.exports.register = function () {
         llmsTxtContent = truncateAtNewline(llmsTxtContent, MAX_LLMS_TXT_CHARS - 100) + '\n\n[Content truncated due to size limits]';
       }
 
+      // The page index list is never truncated: an index missing from the root
+      // leaves its pages unreachable from llms.txt.
+      if (pageIndexes.length) {
+        llmsTxtContent = llmsTxtContent + '\n\n' + renderPageIndexSection(pageIndexes, components);
+        if (llmsTxtContent.length >= MAX_LLMS_TXT_CHARS) {
+          logger.warn(`llms.txt with page index list is ${llmsTxtContent.length} chars, over the ${MAX_LLMS_TXT_CHARS} char target. Shorten the llms page or raise the per-index size limit.`);
+        }
+      }
+
       // Generate navigation section with component sitemaps and key sections
       const navSection = generateNavigationSection(siteUrl, contentCatalog, components);
 
@@ -350,6 +393,10 @@ module.exports.register = function () {
           }
         });
 
+        pageIndexes.forEach((index) => {
+          if (index.lastModified) gitDates.set(index.path, index.lastModified);
+        });
+
         addToSitemap(contentCatalog, siteCatalog, siteUrl, gitDates, logger);
       } catch (err) {
         logger.warn(`Failed to add llms.txt to sitemap: ${err.message}`);
@@ -398,7 +445,8 @@ function addToSitemap(contentCatalog, siteCatalog, siteUrl, gitDates, logger) {
       const filename = file.out.path;
       return filename === 'llms.txt' ||
              filename === 'llms-full.txt' ||
-             filename.endsWith('-full.txt');
+             filename.endsWith('-full.txt') ||
+             /(^|\/)llms(-\d+)?\.txt$/.test(filename);
     })
     .map(file => file.out.path)
     .sort(); // Sort for consistent ordering

package/mcp/CLI_INTERFACE.adoc

@@ -379,7 +379,7 @@ Test categories:
 If a breaking change to the CLI is necessary:
 
 . Update the CLI with the breaking change
-. Increment the major version in package.json
+. Mark the pull request as breaking with `!` after its type, for example `feat(cli)!: rename --tag to --ref`, so release-please cuts a major release
 . Update the MCP server to handle both old and new interfaces (if possible)
 . Update the CLI contract tests to verify the new interface
 . Update this document with the new contract

package/mcp/DEVELOPMENT.adoc

@@ -487,17 +487,13 @@ All CLI commands have:
 === Before releasing
 
 . Run all tests: `npm run test:mcp`
-. Update version in `package.json`
 . Update link:USER_GUIDE.adoc[USER_GUIDE.adoc] with any changes
 . Test manually with Claude Code
 . Check that setup command works: `npx doc-tools setup-mcp`
 
 === Publishing
 
-. Commit all changes
-. Tag the release: `git tag -a v1.2.3 -m "Release 1.2.3"`
-. Push with tags: `git push --tags`
-. Publish to npm: `npm publish`
+Do not change the version, tag, or run `npm publish` by hand. Give your pull request a conventional-commit title (`feat(mcp): ...` or `fix(mcp): ...`). After it merges, release-please adds it to the open release pull request, and merging that pull request tags and publishes the release. See the Releases section of link:../CONTRIBUTING.adoc[CONTRIBUTING.adoc].
 
 === After releasing
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@redpanda-data/docs-extensions-and-macros",
-  "version": "5.50.0",
+  "version": "5.51.1",
   "description": "Antora extensions and macros developed for Redpanda documentation.",
   "keywords": [
     "antora",

package/tools/rpk-docs/generate-rpk-docs.js

@@ -2848,14 +2848,20 @@ async function generateRpkDocs(options = {}) {
     // Build subcommands with correct xref paths
     // Filter out excluded and asPartial subcommands — excluded have no file,
     // asPartial ones live in the partials directory with no linkable xref.
+    // The exception is a parent that is itself asPartial: its partial is
+    // included by a stub in the component that publishes the whole subtree
+    // (rpk sql in cloud-docs, rpk ai in adp-docs), so relative xrefs to its
+    // children resolve there, the same way rpk cloud partials link.
     // rpk cloud and rpk security secret rows stay in the table: their pages
     // are routed to the cloud partials directory and published by
     // cloud-docs, so the xref below links across to the cloud component
     // instead of dropping the row (which hid the subcommand entirely).
+    const parentIsPartial = shouldUsePartialDir(resolvedOverrides, commandPath)
     const subcommands = (command.commands || [])
       .filter(sub => {
         const subPath = `${commandPath} ${sub.name}`
-        return !shouldExcludeCommand(resolvedOverrides, subPath) && !shouldUsePartialDir(resolvedOverrides, subPath)
+        return !shouldExcludeCommand(resolvedOverrides, subPath) &&
+          (parentIsPartial || !shouldUsePartialDir(resolvedOverrides, subPath))
       })
       .map(sub => {
         const subPath = `${commandPath} ${sub.name}`

package/tools/rpk-docs/rpk-docs-handler.js

@@ -1137,15 +1137,21 @@ const WHATS_NEW_EXCLUDED_SUBTREES = ['rpk ai']
 
 /**
  * Return a copy of diffData without entries under the excluded subtrees.
+ * Commands marked asPartial in the overrides are dropped too: they render
+ * only as partials single-sourced into another component (for example,
+ * rpk sql for Redpanda SQL in Cloud), so they are not Self-Managed changes.
  * Only the published What's-new block filters; diff reports and PR
  * summaries keep the full picture.
  * @param {Object} diffData - Diff from generateRpkDiff
  * @param {string[]} [excluded] - Command-path prefixes to drop
+ * @param {Object} [overridesData] - rpk overrides; asPartial subtrees are dropped
  * @returns {Object} Filtered copy
  */
-function filterDiffForWhatsNew(diffData, excluded = WHATS_NEW_EXCLUDED_SUBTREES) {
+function filterDiffForWhatsNew(diffData, excluded = WHATS_NEW_EXCLUDED_SUBTREES, overridesData = null) {
+  const resolved = overridesData ? resolveReferences(overridesData, overridesData) : null
   const outside = (cmdPath) => !excluded.some(prefix =>
-    cmdPath === prefix || (typeof cmdPath === 'string' && cmdPath.startsWith(prefix + ' ')))
+    cmdPath === prefix || (typeof cmdPath === 'string' && cmdPath.startsWith(prefix + ' '))) &&
+    !(resolved && typeof cmdPath === 'string' && shouldUsePartialDir(resolved, cmdPath))
   const details = diffData.details || {}
   const filteredDetails = { ...details }
   for (const key of ['newCommands', 'newlyDeprecatedCommands', 'removedCommands', 'descriptionChanges']) {
@@ -1167,7 +1173,28 @@ function updateWhatsNewFile(diffData, whatsNewPath, version, options = {}) {
     ...options
   })
 
+  const startMarker = `// AUTOGEN-RPK-CHANGES ${version} START`
+  const endMarker = `// AUTOGEN-RPK-CHANGES ${version} END`
+  const escapeRe = (s) => s.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')
+
   if (!whatsNewContent) {
+    // A re-run for a version that now has nothing to publish (for example,
+    // every change was filtered out as asPartial) must remove that version's
+    // earlier block, or the page keeps publishing what the filter excluded.
+    if (fs.existsSync(whatsNewPath)) {
+      const existing = fs.readFileSync(whatsNewPath, 'utf8')
+      if (existing.includes(startMarker)) {
+        const blockRe = new RegExp(`\\n*${escapeRe(startMarker)}[\\s\\S]*?${escapeRe(endMarker)}\\n*`)
+        let updated = existing.replace(blockRe, '\n\n')
+        // Drop the section heading too if the block was all it held, so no
+        // empty heading is left on the page.
+        const emptySectionRe = new RegExp(`^${escapeRe(sectionHeading)}[^\\n]*\\n\\s*(?=^== |(?![\\s\\S]))`, 'm')
+        updated = updated.replace(emptySectionRe, '')
+        fs.writeFileSync(whatsNewPath, updated.replace(/\s*$/, '\n'), 'utf8')
+        console.log(`Removed ${version} block from what's-new file (no changes left to publish): ${whatsNewPath}`)
+        return
+      }
+    }
     console.log('No Redpanda CLI changes to add to what\'s new')
     return
   }
@@ -1187,11 +1214,8 @@ function updateWhatsNewFile(diffData, whatsNewPath, version, options = {}) {
   // Redpanda CLI section instead of being dropped. Writers can edit or
   // remove blocks freely; the automation only ever touches content between
   // its own markers for the same version label.
-  const startMarker = `// AUTOGEN-RPK-CHANGES ${version} START`
-  const endMarker = `// AUTOGEN-RPK-CHANGES ${version} END`
   const sectionBody = whatsNewContent.replace(/^== [^\n]*\n+/, '')
   const block = `${startMarker}\n${sectionBody.trimEnd()}\n${endMarker}`
-  const escapeRe = (s) => s.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')
 
   if (existingContent.includes(startMarker)) {
     // Replace this version's existing block (idempotent re-runs)
@@ -2145,7 +2169,7 @@ async function handleRpkDocsGeneration(options = {}) {
 
           // Update what's-new file if requested
           if (whatsNewPath) {
-            updateWhatsNewFile(filterDiffForWhatsNew(diffData), whatsNewPath, rpkVersion, { linkable: makeLinkablePredicate(overridesData), hasSubcommands: makeSubcommandPredicate(tree) })
+            updateWhatsNewFile(filterDiffForWhatsNew(diffData, WHATS_NEW_EXCLUDED_SUBTREES, overridesData), whatsNewPath, rpkVersion, { linkable: makeLinkablePredicate(overridesData), hasSubcommands: makeSubcommandPredicate(tree) })
           }
         } else {
           console.warn(`Warning: Could not load previous version ${diffVersion} for diff`)
@@ -2467,7 +2491,7 @@ async function handleRpkDocsGeneration(options = {}) {
 
         // Update what's-new file if requested
         if (whatsNewPath) {
-          updateWhatsNewFile(filterDiffForWhatsNew(diffData), whatsNewPath, rpkVersion, { linkable: makeLinkablePredicate(overridesData), hasSubcommands: makeSubcommandPredicate(tree) })
+          updateWhatsNewFile(filterDiffForWhatsNew(diffData, WHATS_NEW_EXCLUDED_SUBTREES, overridesData), whatsNewPath, rpkVersion, { linkable: makeLinkablePredicate(overridesData), hasSubcommands: makeSubcommandPredicate(tree) })
         }
       } else {
         console.warn(`Warning: Could not load previous version ${diffVersion} for diff`)