@redpanda-data/docs-extensions-and-macros 4.14.1 → 4.15.0

package/bin/doc-tools.js

@@ -783,7 +783,7 @@ automation
   )
   .option('-t, --tag <tag>', 'Git tag for released content (GA/beta)')
   .option('-b, --branch <branch>', 'Branch name for in-progress content')
-  .option('--diff <oldTag>', 'Also diff autogenerated properties from <oldTag> to current tag/branch')
+  .option('--diff <oldTag>', 'Diff properties against <oldTag> and restore removed deprecated properties. Recommended for accurate output; falls back to latest-redpanda-tag from antora.yml if not specified')
   .option('--overrides <path>', 'Optional JSON file with property description overrides', 'docs-data/property-overrides.json')
   .option('--output-dir <dir>', 'Where to write all generated files', 'modules/reference')
   .option('--cloud-support', 'Add AsciiDoc tags to generated property docs to indicate which ones are supported in Redpanda Cloud. This data is fetched from the cloudv2 repository so requires a GitHub token with repo permissions. Set the token as an environment variable using GITHUB_TOKEN, GH_TOKEN, or REDPANDA_GITHUB_TOKEN', true)
@@ -826,11 +826,17 @@ automation
       }
     }
 
+    if (!oldTag) {
+      console.warn('Warning: No previous version specified (--diff) and no latest-redpanda-tag found in Antora attributes.')
+      console.warn('   Deprecated properties that were removed from source (v26.1+) will not be detected.')
+      console.warn('   For accurate output, specify --diff <previous-tag> or set latest-redpanda-tag in antora.yml.')
+    }
+
     const overridesPath = options.overrides
     const outputDir = options.outputDir
     const cwd = path.resolve(__dirname, '../tools/property-extractor')
 
-    const make = (tag, overrides, templates = {}, outDir = 'modules/reference/') => {
+    const make = (tag, overrides, templates = {}, outDir = 'modules/reference/', { skipPartials = false } = {}) => {
       console.log(`Building property docs for ${tag}…`)
       const args = ['build', `TAG=${tag}`]
       const env = { ...process.env }
@@ -843,7 +849,7 @@ automation
       if (templates.deprecatedProperty) env.TEMPLATE_DEPRECATED_PROPERTY = path.resolve(templates.deprecatedProperty)
       env.OUTPUT_JSON_DIR = path.resolve(outDir, 'attachments')
       env.OUTPUT_AUTOGENERATED_DIR = path.resolve(outDir)
-      if (options.generatePartials) {
+      if (options.generatePartials && !skipPartials) {
         env.GENERATE_PARTIALS = '1'
         env.OUTPUT_PARTIALS_DIR = path.resolve(outDir, options.partialsDir || 'partials')
       }
@@ -864,11 +870,21 @@ automation
     }
 
     const tagsAreSame = oldTag && newTag && oldTag === newTag
-    if (oldTag && !tagsAreSame) {
-      make(oldTag, overridesPath, templates, outputDir)
+    const needsDiff = oldTag && !tagsAreSame
+
+    // Phase 1: Extract JSON from C++ source.
+    // When a diff is needed, skip AsciiDoc generation during extraction so we
+    // can merge removed deprecated properties first and generate only once.
+    if (needsDiff) {
+      make(oldTag, overridesPath, templates, outputDir, { skipPartials: true })
+      make(newTag, overridesPath, templates, outputDir, { skipPartials: true })
+    } else {
+      make(newTag, overridesPath, templates, outputDir)
     }
-    make(newTag, overridesPath, templates, outputDir)
-    if (oldTag && !tagsAreSame) {
+
+    // Phase 2: Compare old vs new and merge removed deprecated properties into
+    // the new JSON so they appear in the generated documentation.
+    if (needsDiff) {
       const diffOutputDir = overridesPath ? path.dirname(path.resolve(overridesPath)) : outputDir
       generatePropertyComparisonReport(oldTag, newTag, diffOutputDir)
 
@@ -888,6 +904,18 @@ automation
       }
 
       cleanupOldDiffs(diffOutputDir)
+
+      // Phase 3: Generate AsciiDoc once from the complete JSON (includes merged deprecated properties)
+      if (options.generatePartials) {
+        const updatedJsonPath = path.resolve(outputDir, 'attachments', `redpanda-properties-${newTag}.json`)
+        if (fs.existsSync(updatedJsonPath)) {
+          process.env.GENERATE_PARTIALS = '1'
+          process.env.OUTPUT_PARTIALS_DIR = path.resolve(outputDir, options.partialsDir || 'partials')
+          const { generateAllDocs } = require('../tools/property-extractor/generate-handlebars-docs')
+          console.log('Generating AsciiDoc from complete property data…')
+          generateAllDocs(updatedJsonPath, path.resolve(outputDir))
+        }
+      }
     }
 
     if (!options.diff && !tagsAreSame) {

package/extensions/convert-llms-to-txt.js

@@ -0,0 +1,182 @@
+'use strict';
+
+/**
+ * Extracts markdown from llms.adoc page and generates llms.txt and llms-full.txt.
+ *
+ * This extension:
+ * 1. Adds site-url attribute to home component:
+ *    - In preview builds (PREVIEW=true): Uses DEPLOY_PRIME_URL
+ *    - In production builds: Uses playbook.site.url
+ * 2. Finds llms page in home component (after AsciiDoc processing)
+ * 3. Gets the markdown content from page.markdownContents (set by convert-to-markdown extension)
+ * 4. Unpublishes the HTML page
+ * 5. Places llms.txt (markdown) at site root
+ * 6. Generates llms-full.txt with markdown from latest versions
+ *
+ * Must run after convert-to-markdown extension to access page.markdownContents.
+ */
+module.exports.register = function () {
+  const logger = this.getLogger('convert-llms-to-txt-extension');
+  let siteUrl = '';
+
+  // Add site-url attribute to home component
+  this.on('playbookBuilt', ({ playbook }) => {
+    // In preview builds, always use the deploy preview URL
+    if (process.env.PREVIEW === 'true' && process.env.DEPLOY_PRIME_URL) {
+      siteUrl = process.env.DEPLOY_PRIME_URL;
+      logger.info(`Using deploy preview URL: ${siteUrl}`);
+    } else {
+      siteUrl = playbook.site?.url || 'https://docs.redpanda.com';
+      logger.info(`Using site URL: ${siteUrl}`);
+    }
+  });
+
+  this.on('contentClassified', ({ contentCatalog }) => {
+    // Add site-url attribute to home component
+    const homeComponent = contentCatalog.getComponents().find(c => c.name === 'home');
+    if (homeComponent && homeComponent.versions) {
+      homeComponent.versions.forEach(version => {
+        if (!version.asciidoc) version.asciidoc = {};
+        if (!version.asciidoc.attributes) version.asciidoc.attributes = {};
+        version.asciidoc.attributes['site-url'] = siteUrl;
+        logger.debug(`Added site-url attribute to home component: ${siteUrl}`);
+      });
+    }
+  });
+
+  // Run after pagesComposed so convert-to-markdown has already run
+  this.on('beforePublish', ({ contentCatalog, siteCatalog }) => {
+    // Find llms.adoc page in home component (after markdown conversion)
+    const llmsPage = contentCatalog.findBy({
+      component: 'home',
+      family: 'page',
+    }).find(page => page.src.stem === 'llms');
+
+    if (!llmsPage) {
+      logger.warn('No llms page found, skipping llms.txt generation');
+    } else {
+      logger.info(`Found llms page: ${llmsPage.src.path}`);
+      logger.info(`Has markdownContents: ${!!llmsPage.markdownContents}`);
+      logger.info(`Has out: ${!!llmsPage.out}`);
+      try {
+        // The convert-to-markdown extension has already processed this page
+        // and stored the markdown in page.markdownContents
+        if (!llmsPage.markdownContents) {
+          throw new Error('No markdown content found on llms page. Ensure convert-to-markdown extension runs before this extension.');
+        }
+
+        let content = llmsPage.markdownContents.toString('utf8');
+        logger.info(`Extracted ${content.length} bytes of markdown content`);
+
+        // Strip HTML comments added by convert-to-markdown extension
+        // These reference the unpublished /home/llms/ URL which doesn't make sense for llms.txt
+        content = content.replace(/^<!--[\s\S]*?-->\s*/gm, '').trim();
+        logger.debug(`Stripped HTML comments, now ${content.length} bytes`);
+
+        // Unpublish the HTML page FIRST (following unpublish-pages pattern)
+        if (llmsPage.out) {
+          if (!siteCatalog.unpublishedPages) siteCatalog.unpublishedPages = [];
+          if (llmsPage.pub?.url) {
+            siteCatalog.unpublishedPages.push(llmsPage.pub.url);
+          }
+          delete llmsPage.out;
+          logger.info('Unpublished llms HTML page');
+        }
+
+        // Store cleaned markdown content for adding after llms-full.txt
+        llmsPage.llmsTxtContent = content;
+
+      } catch (err) {
+        logger.error(`Failed to extract markdown from llms page: ${err.message}`);
+        logger.debug(err.stack);
+      }
+    }
+
+    // Generate llms-full.txt - aggregate markdown from latest version of each component
+    logger.info('Generating llms-full.txt from latest version pages with markdown...');
+
+    // Get all components and identify latest versions
+    const components = contentCatalog.getComponents();
+    const latestVersions = new Set();
+
+    components.forEach(component => {
+      // Find the latest version (non-prerelease if available, otherwise the first version)
+      const latest = component.latest || component.versions[0];
+      if (latest) {
+        latestVersions.add(`${component.name}@${latest.version}`);
+        logger.debug(`Latest version for ${component.name}: ${latest.version}`);
+      }
+    });
+
+    // Filter pages to only include latest versions
+    const allPages = contentCatalog.getPages((p) => p.markdownContents);
+    const pages = allPages.filter(page => {
+      const pageKey = `${page.src.component}@${page.src.version}`;
+      return latestVersions.has(pageKey);
+    });
+
+    if (!pages.length) {
+      logger.warn('No pages with markdown content found in latest versions, skipping llms-full.txt generation');
+      return;
+    }
+
+    logger.info(`Filtered to ${pages.length} pages from ${latestVersions.size} latest component versions (from ${allPages.length} total pages)`);
+
+    let fullContent = `# Redpanda Documentation - Full Markdown Export\n\n`;
+    fullContent += `> This file contains all documentation pages in markdown format for AI agent consumption.\n`;
+    fullContent += `> Generated from ${pages.length} pages on ${new Date().toISOString()}\n`;
+    fullContent += `> Site: ${siteUrl}\n\n`;
+    fullContent += `## About This Export\n\n`;
+    fullContent += `This export includes only the **latest version** of each component's documentation:\n`;
+    components.forEach(component => {
+      const latest = component.latest || component.versions[0];
+      if (latest) {
+        fullContent += `- **${component.title}**: version ${latest.version}\n`;
+      }
+    });
+    fullContent += `\n`;
+    fullContent += `### Accessing Versioned Content\n\n`;
+    fullContent += `For components with versioned documentation (like Redpanda Self-Managed), older versions can be accessed by replacing the version segment in the URL:\n`;
+    fullContent += `- Latest: \`${siteUrl}/current/page-path\`\n`;
+    fullContent += `- Specific version: \`${siteUrl}/24.3/page-path\`, \`${siteUrl}/25.1/page-path\`, etc.\n\n`;
+    fullContent += `Available versioned components: ${components.filter(c => c.versions.length > 1).map(c => c.name).join(', ')}\n\n`;
+    fullContent += `---\n\n`;
+
+    // Sort pages by URL for consistent ordering
+    pages.sort((a, b) => {
+      const urlA = a.pub?.url || '';
+      const urlB = b.pub?.url || '';
+      return urlA.localeCompare(urlB);
+    });
+
+    pages.forEach((page, index) => {
+      const pageUrl = page.pub?.url ? `${siteUrl}${page.pub.url}` : 'unknown';
+      const pageTitle = page.asciidoc?.doctitle || page.src?.stem || 'Untitled';
+
+      fullContent += `# Page ${index + 1}: ${pageTitle}\n\n`;
+      fullContent += `**URL**: ${pageUrl}\n\n`;
+      fullContent += `---\n\n`;
+      fullContent += page.markdownContents.toString('utf8');
+      fullContent += `\n\n---\n\n`;
+    });
+
+    // Add llms-full.txt to site root
+    siteCatalog.addFile({
+      contents: Buffer.from(fullContent, 'utf8'),
+      out: { path: 'llms-full.txt' },
+    });
+    logger.info(`Generated llms-full.txt with ${pages.length} pages`);
+
+    // Add llms.txt to site root (using content extracted earlier)
+    if (llmsPage && llmsPage.llmsTxtContent) {
+      logger.info('Adding llms.txt to site root');
+      siteCatalog.addFile({
+        contents: Buffer.from(llmsPage.llmsTxtContent, 'utf8'),
+        out: { path: 'llms.txt' },
+      });
+      logger.info('Successfully added llms.txt');
+    } else {
+      logger.warn('llms.txt not generated - page not found or no content extracted');
+    }
+  });
+};

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@redpanda-data/docs-extensions-and-macros",
-  "version": "4.14.1",
+  "version": "4.15.0",
   "description": "Antora extensions and macros developed for Redpanda documentation.",
   "keywords": [
     "antora",
@@ -65,6 +65,7 @@
     "./extensions/find-related-docs": "./extensions/find-related-docs.js",
     "./extensions/unpublish-pages": "./extensions/unpublish-pages.js",
     "./extensions/find-related-labs": "./extensions/find-related-labs.js",
+    "./extensions/convert-llms-to-txt": "./extensions/convert-llms-to-txt.js",
     "./extensions/modify-redirects": "./extensions/produce-redirects.js",
     "./extensions/algolia-indexer/index": "./extensions/algolia-indexer/index.js",
     "./extensions/aggregate-terms": "./extensions/aggregate-terms.js",

package/tools/property-extractor/Makefile

@@ -86,7 +86,11 @@ redpanda-git:
 	    git clone -q https://github.com/redpanda-data/redpanda.git "$(REDPANDA_SRC)"; \
 	  fi; \
 	fi; \
-	if git -C "$(REDPANDA_SRC)" rev-parse --verify -q "$(TAG)" >/dev/null; then \
+	if git -C "$(REDPANDA_SRC)" rev-parse --verify -q "origin/$(TAG)" >/dev/null 2>&1; then \
+	  echo "🔖 Checking out remote branch 'origin/$(TAG)'"; \
+	  git -C "$(REDPANDA_SRC)" checkout -q "$(TAG)" 2>/dev/null || git -C "$(REDPANDA_SRC)" checkout -q -b "$(TAG)" "origin/$(TAG)"; \
+	  git -C "$(REDPANDA_SRC)" reset --hard "origin/$(TAG)" -q; \
+	elif git -C "$(REDPANDA_SRC)" rev-parse --verify -q "$(TAG)" >/dev/null 2>&1; then \
 	  echo "🔖 Checking out '$(TAG)'"; \
 	  git -C "$(REDPANDA_SRC)" checkout -q "$(TAG)"; \
 	else \

package/tools/property-extractor/compare-properties.js

@@ -9,12 +9,46 @@
  * - Properties with changed descriptions
  * - Properties with changed types
  * - Deprecated properties
+ * - Removed deprecated properties (deprecated in old version, deleted from source in new)
  * - Removed properties
  * - Properties with empty descriptions (excluding deprecated)
  */
 
 const fs = require('fs');
 const path = require('path');
+const semver = require('semver');
+
+/**
+ * The minimum version where Redpanda removes deprecated properties from the
+ * C++ source code instead of keeping them tagged as deprecated_property.
+ */
+const DEPRECATED_REMOVAL_MIN_VERSION = '26.1.0';
+
+/**
+ * Check if the new version deletes deprecated properties from source.
+ *
+ * Before v26.1, deprecated properties were kept in the C++ source tagged as
+ * deprecated_property. Starting from v26.1, they are deleted entirely. When
+ * a property exists in the old version but is absent in the new version, and
+ * the new version is >= v26.1, we treat it as a deprecation (the property was
+ * removed because it was deprecated).
+ *
+ * Returns false only when the new version is explicitly below v26.1.
+ * For unparseable versions (e.g. "dev", "main"), assumes the latest behavior.
+ *
+ * @param {string} newVersion - The new version string.
+ * @returns {boolean} True if the new version deletes deprecated properties.
+ */
+function isCrossDeprecatedRemovalBoundary(newVersion) {
+  // Only skip if the new version is explicitly below the boundary
+  const newCleaned = semver.coerce(newVersion);
+  if (newCleaned && semver.lt(newCleaned, DEPRECATED_REMOVAL_MIN_VERSION)) {
+    return false;
+  }
+  // New version is either >= 26.1 or unparseable (e.g. "dev", "main").
+  // In both cases, assume the new version has the removal behavior.
+  return true;
+}
 
 /**
  * Recursively compares two values for structural deep equality.
@@ -134,12 +168,15 @@ function compareProperties(oldData, newData, oldVersion, newVersion) {
   const oldProps = extractProperties(oldData);
   const newProps = extractProperties(newData);
 
+  const removesDeprecated = isCrossDeprecatedRemovalBoundary(newVersion);
+
   const report = {
     newProperties: [],
     changedDefaults: [],
     changedDescriptions: [],
     changedTypes: [],
     deprecatedProperties: [],
+    removedDeprecatedProperties: [],
     removedProperties: [],
     emptyDescriptions: []
   };
@@ -206,11 +243,22 @@ function compareProperties(oldData, newData, oldVersion, newVersion) {
       // Check both the is_experimental_property field and development_ prefix
       const isExperimental = oldProp.is_experimental_property || name.startsWith('development_');
       if (!isExperimental) {
-        report.removedProperties.push({
-          name,
-          type: oldProp.type,
-          description: oldProp.description || 'No description'
-        });
+        // In v26.1+, deprecated properties are deleted from source instead of
+        // being tagged. If a property existed before and is gone now, it was
+        // deprecated.
+        if (removesDeprecated) {
+          report.removedDeprecatedProperties.push({
+            name,
+            type: oldProp.type,
+            description: oldProp.description || 'No description'
+          });
+        } else {
+          report.removedProperties.push({
+            name,
+            type: oldProp.type,
+            description: oldProp.description || 'No description'
+          });
+        }
       }
     }
   }
@@ -288,6 +336,14 @@ function generateConsoleReport(report, oldVersion, newVersion) {
     });
   }
   
+  if (report.removedDeprecatedProperties.length > 0) {
+    console.log(`\n➤ Removed deprecated properties (${report.removedDeprecatedProperties.length}):`);
+    console.log(`   These were deprecated in ${oldVersion} and removed from source in ${newVersion}.`);
+    report.removedDeprecatedProperties.forEach(prop => {
+      console.log(`   • ${prop.name} (${prop.type})`);
+    });
+  }
+
   if (report.removedProperties.length > 0) {
     console.log(`\n➤ Removed properties (${report.removedProperties.length}):`);
     report.removedProperties.forEach(prop => {
@@ -329,6 +385,7 @@ function generateJsonReport(report, oldVersion, newVersion, outputPath) {
       changedDescriptions: report.changedDescriptions.length,
       changedTypes: report.changedTypes.length,
       deprecatedProperties: report.deprecatedProperties.length,
+      removedDeprecatedProperties: report.removedDeprecatedProperties.length,
       removedProperties: report.removedProperties.length,
       emptyDescriptions: report.emptyDescriptions.length
     },
@@ -370,17 +427,39 @@ function comparePropertyFiles(oldFilePath, newFilePath, oldVersion, newVersion,
     const newData = JSON.parse(fs.readFileSync(newFilePath, 'utf8'));
     
     const report = compareProperties(oldData, newData, oldVersion, newVersion);
-    
+
+    // Merge removed deprecated properties back into the new JSON so they
+    // remain in generated documentation with is_deprecated + removed_deprecated flags
+    if (report.removedDeprecatedProperties.length > 0) {
+      const oldProps = extractProperties(oldData);
+      const newProps = extractProperties(newData);
+
+      report.removedDeprecatedProperties.forEach(({ name }) => {
+        if (oldProps[name] && !newProps[name]) {
+          newProps[name] = {
+            ...oldProps[name],
+            is_deprecated: true,
+            removed_deprecated: true,
+            visibility: 'deprecated'
+          };
+        }
+      });
+
+      newData.properties = newProps;
+      fs.writeFileSync(newFilePath, JSON.stringify(newData, null, 2), 'utf8');
+      console.log(`\nMerged ${report.removedDeprecatedProperties.length} removed deprecated properties back into ${newVersion} JSON`);
+    }
+
     // Generate console report
     generateConsoleReport(report, oldVersion, newVersion);
-    
+
     // Generate JSON report if output directory provided
     if (outputDir) {
       fs.mkdirSync(outputDir, { recursive: true });
       const jsonReportPath = path.join(outputDir, filename);
       generateJsonReport(report, oldVersion, newVersion, jsonReportPath);
     }
-    
+
     return report;
   } catch (error) {
     console.error(`Error: Error comparing properties: ${error.message}`);