@redpanda-data/docs-extensions-and-macros 4.11.1 → 4.12.1

package/bin/doc-tools.js

@@ -719,6 +719,18 @@ automation
         fs.unlinkSync(tmpFile);
         dataFile = finalFile;
         console.log(`✅ Fetched and saved: ${finalFile}`);
+
+        // Keep only 2 most recent versions in docs-data
+        const dataFiles = fs.readdirSync(dataDir)
+          .filter(f => /^connect-\d+\.\d+\.\d+\.json$/.test(f))
+          .sort();
+
+        while (dataFiles.length > 2) {
+          const oldestFile = dataFiles.shift();
+          const oldestPath = path.join(dataDir, oldestFile);
+          fs.unlinkSync(oldestPath);
+          console.log(`🧹 Deleted old version from docs-data: ${oldestFile}`);
+        }
       } catch (err) {
         console.error(`❌ Failed to fetch connectors: ${err.message}`);
         process.exit(1);
@@ -845,6 +857,45 @@ automation
     }
 
     const newIndex = JSON.parse(fs.readFileSync(dataFile, 'utf8'));
+
+    // Publish merged version with overrides to modules/components/attachments
+    if (options.overrides && fs.existsSync(options.overrides)) {
+      try {
+        const { mergeOverrides, resolveReferences } = require('../tools/redpanda-connect/generate-rpcn-connector-docs.js');
+
+        // Create a copy of newIndex to merge overrides into
+        const mergedData = JSON.parse(JSON.stringify(newIndex));
+
+        // Read and apply overrides
+        const ovRaw = fs.readFileSync(options.overrides, 'utf8');
+        const ovObj = JSON.parse(ovRaw);
+        const resolvedOverrides = resolveReferences(ovObj, ovObj);
+        mergeOverrides(mergedData, resolvedOverrides);
+
+        // Publish to modules/components/attachments
+        const attachmentsRoot = path.resolve(process.cwd(), 'modules/components/attachments');
+        fs.mkdirSync(attachmentsRoot, { recursive: true });
+
+        // Delete older versions from modules/components/attachments
+        const existingFiles = fs.readdirSync(attachmentsRoot)
+          .filter(f => /^connect-\d+\.\d+\.\d+\.json$/.test(f))
+          .sort();
+
+        for (const oldFile of existingFiles) {
+          const oldFilePath = path.join(attachmentsRoot, oldFile);
+          fs.unlinkSync(oldFilePath);
+          console.log(`🧹 Deleted old version: ${oldFile}`);
+        }
+
+        // Save merged version to modules/components/attachments
+        const destFile = path.join(attachmentsRoot, `connect-${newVersion}.json`);
+        fs.writeFileSync(destFile, JSON.stringify(mergedData, null, 2), 'utf8');
+        console.log(`✅ Published merged version to: ${path.relative(process.cwd(), destFile)}`);
+      } catch (err) {
+        console.error(`❌ Failed to publish merged version: ${err.message}`);
+      }
+    }
+
     printDeltaReport(oldIndex, newIndex);
 
     // Generate JSON diff file for whats-new.adoc
@@ -895,6 +946,93 @@ automation
 
     // Optionally update whats-new.adoc
     if (options.updateWhatsNew) {
+      // Helper function to cap description to two sentences
+      const capToTwoSentences = (description) => {
+        if (!description) return '';
+
+        // Helper to check if text contains problematic content
+        const hasProblematicContent = (text) => {
+          return /```[\s\S]*?```/.test(text) ||  // code blocks
+                 /`[^`]+`/.test(text) ||          // inline code
+                 /^[=#]+\s+.+$/m.test(text) ||    // headings
+                 /\n/.test(text);                 // newlines
+        };
+
+        // Step 1: Replace common abbreviations and ellipses with placeholders
+        const abbreviations = [
+          /\bv\d+\.\d+(?:\.\d+)?/gi, // version numbers like v4.12 or v4.12.0 (must come before decimal)
+          /\d+\.\d+/g,               // decimal numbers
+          /\be\.g\./gi,              // e.g.
+          /\bi\.e\./gi,              // i.e.
+          /\betc\./gi,               // etc.
+          /\bvs\./gi,                // vs.
+          /\bDr\./gi,                // Dr.
+          /\bMr\./gi,                // Mr.
+          /\bMs\./gi,                // Ms.
+          /\bMrs\./gi,               // Mrs.
+          /\bSt\./gi,                // St.
+          /\bNo\./gi                 // No.
+        ];
+
+        let normalized = description;
+        const placeholders = [];
+
+        // Replace abbreviations with placeholders
+        abbreviations.forEach((abbrevRegex, idx) => {
+          normalized = normalized.replace(abbrevRegex, (match) => {
+            const placeholder = `__ABBREV${idx}_${placeholders.length}__`;
+            placeholders.push({ placeholder, original: match });
+            return placeholder;
+          });
+        });
+
+        // Replace ellipses (three or more dots) with placeholder
+        normalized = normalized.replace(/\.{3,}/g, (match) => {
+          const placeholder = `__ELLIPSIS_${placeholders.length}__`;
+          placeholders.push({ placeholder, original: match });
+          return placeholder;
+        });
+
+        // Step 2: Split sentences using the regex
+        const sentenceRegex = /[^.!?]+[.!?]+(?:\s|$)/g;
+        const sentences = normalized.match(sentenceRegex);
+
+        if (!sentences || sentences.length === 0) {
+          // Restore placeholders and return original
+          let result = normalized;
+          placeholders.forEach(({ placeholder, original }) => {
+            result = result.replace(placeholder, original);
+          });
+          return result;
+        }
+
+        // Step 3: Determine how many sentences to include
+        let maxSentences = 2;
+
+        // If we have at least 2 sentences, check if the second one has problematic content
+        if (sentences.length >= 2) {
+          // Restore placeholders in second sentence to check original content
+          let secondSentence = sentences[1];
+          placeholders.forEach(({ placeholder, original }) => {
+            secondSentence = secondSentence.replace(new RegExp(placeholder, 'g'), original);
+          });
+
+          // If second sentence has problematic content, only take first sentence
+          if (hasProblematicContent(secondSentence)) {
+            maxSentences = 1;
+          }
+        }
+
+        let result = sentences.slice(0, maxSentences).join('');
+
+        // Step 4: Restore placeholders back to original text
+        placeholders.forEach(({ placeholder, original }) => {
+          result = result.replace(new RegExp(placeholder, 'g'), original);
+        });
+
+        return result.trim();
+      };
+
       try {
         const whatsNewPath = path.join(findRepoRoot(), 'modules/get-started/pages/whats-new.adoc');
         if (!fs.existsSync(whatsNewPath)) {
@@ -956,7 +1094,7 @@ automation
             for (const comp of comps) {
               section += `** xref:components:${type}/${comp.name}.adoc[\`${comp.name}\`]`;
               if (comp.status) section += ` (${comp.status})`;
-              if (comp.description) section += `: ${comp.description}`;
+              if (comp.description) section += `: ${capToTwoSentences(comp.description)}`;
               section += '\n';
             }
           }
@@ -977,6 +1115,60 @@ automation
               description: field.description || '',
             });
           }
+          for (const [type, fields] of Object.entries(fieldsByType)) {
+            section += `* ${type.charAt(0).toUpperCase() + type.slice(1)}:\n`;
+            // Group by component name
+            const byComp = {};
+            for (const f of fields) {
+              if (!byComp[f.compName]) byComp[f.compName] = [];
+              byComp[f.compName].push(f);
+            }
+            for (const [comp, compFields] of Object.entries(byComp)) {
+              section += `** xref:components:${type}/${comp}.adoc[\`${comp}\`]:`;
+              section += '\n';
+              for (const f of compFields) {
+                section += `*** xref:components:${type}/${comp}.adoc#${f.field}[\`${f.field}\`]`;
+                if (f.description) section += `: ${capToTwoSentences(f.description)}`;
+                section += '\n';
+              }
+            }
+          }
+        }
+
+        // Deprecated components
+        if (diff.details.deprecatedComponents && diff.details.deprecatedComponents.length) {
+          section += '\n=== Deprecations\n\n';
+          section += 'The following components are now deprecated:\n\n';
+          // Group by type
+          const byType = {};
+          for (const comp of diff.details.deprecatedComponents) {
+            if (!byType[comp.type]) byType[comp.type] = [];
+            byType[comp.type].push(comp);
+          }
+          for (const [type, comps] of Object.entries(byType)) {
+            section += `* ${type.charAt(0).toUpperCase() + type.slice(1)}:\n`;
+            for (const comp of comps) {
+              section += `** xref:components:${type}/${comp.name}.adoc[\`${comp.name}\`]\n`;
+            }
+          }
+        }
+
+        // Deprecated fields
+        if (diff.details.deprecatedFields && diff.details.deprecatedFields.length) {
+          if (!diff.details.deprecatedComponents || diff.details.deprecatedComponents.length === 0) {
+            section += '\n=== Deprecations\n\n';
+          }
+          section += '\nThe following fields are now deprecated:\n\n';
+          // Group deprecated fields by component type
+          const fieldsByType = {};
+          for (const field of diff.details.deprecatedFields) {
+            const [type, compName] = field.component.split(':');
+            if (!fieldsByType[type]) fieldsByType[type] = [];
+            fieldsByType[type].push({
+              compName,
+              field: field.field
+            });
+          }
           for (const [type, fields] of Object.entries(fieldsByType)) {
             section += `* ${type.charAt(0).toUpperCase() + type.slice(1)} components\n`;
             // Group by component name
@@ -986,23 +1178,20 @@ automation
               byComp[f.compName].push(f);
             }
             for (const [comp, compFields] of Object.entries(byComp)) {
-              section += `** xref:components:${type}/${comp}.adoc['${comp}']`;
+              section += `** xref:components:${type}/${comp}.adoc[\`${comp}\`]`;
               if (compFields.length === 1) {
                 const f = compFields[0];
-                section += `: xref:components:${type}/${comp}.adoc#${f.field}['${f.field}']`;
-                if (f.description) section += ` - ${f.description}`;
-                section += '\n';
+                section += `: xref:components:${type}/${comp}.adoc#${f.field}[\`${f.field}\`]\n`;
               } else {
                 section += '\n';
                 for (const f of compFields) {
-                  section += `*** xref:components:${type}/${comp}.adoc#${f.field}['${f.field}']`;
-                  if (f.description) section += ` - ${f.description}`;
-                  section += '\n';
+                  section += `*** xref:components:${type}/${comp}.adoc#${f.field}[\`${f.field}\`]\n`;
                 }
               }
             }
           }
         }
+
         let updated;
         if (startIdx !== -1) {
           // Replace the existing section
@@ -1148,8 +1337,10 @@ automation
 
     // If we used Antora's latest-redpanda-tag for diff, update it to the new tag
     if (!options.diff && !tagsAreSame) {
-      setAntoraValue('asciidoc.attributes.latest-redpanda-tag', newTag);
-      console.log(`✅ Updated Antora latest-redpanda-tag to: ${newTag}`);
+      const success = setAntoraValue('asciidoc.attributes.latest-redpanda-tag', newTag);
+      if (success) {
+        console.log(`✅ Updated Antora latest-redpanda-tag to: ${newTag}`);
+      }
     }
 
     process.exit(0);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@redpanda-data/docs-extensions-and-macros",
-  "version": "4.11.1",
+  "version": "4.12.1",
   "description": "Antora extensions and macros developed for Redpanda documentation.",
   "keywords": [
     "antora",
@@ -22,6 +22,8 @@
     "build": "antora --to-dir docs --fetch local-antora-playbook.yml",
     "serve": "wds --node-resolve --open preview/test/ --watch --root-dir docs",
     "test": "jest",
+    "test:python": "./__tests__/tools/property-extractor/setup-and-test.sh",
+    "test:all": "npm run test && npm run test:python",
     "bundle:admin": "doc-tools generate bundle-openapi --surface admin",
     "bundle:connect": "doc-tools generate bundle-openapi --surface connect",
     "bundle:both": "doc-tools generate bundle-openapi --surface both"

package/tools/property-extractor/COMPUTED_CONSTANTS.md

@@ -0,0 +1,173 @@
+# Computed C++ Constants Resolution
+
+## Overview
+
+Some C++ constants in Redpanda are defined with complex compile-time expressions that cannot be easily parsed by the property-extractor. These constants need to be pre-computed and mapped to their actual values.
+
+## Problem Statement
+
+Properties like `log_message_timestamp_before_max_ms` use constants like `max_serializable_ms` as their default values. These constants are defined with complex expressions:
+
+```cpp
+// From src/v/serde/rw/chrono.h:20
+inline constexpr auto max_serializable_ms
+  = std::chrono::duration_cast<std::chrono::milliseconds>(
+    std::chrono::nanoseconds::max());
+```
+
+Without resolution, the extracted schema would show:
+```json
+{
+  "log_message_timestamp_before_max_ms": {
+    "default": "max_serializable_ms"  // ❌ String instead of numeric value
+  }
+}
+```
+
+## Solution
+
+### 1. COMPUTED_CONSTANTS Dictionary
+
+Added a dictionary in `transformers.py` that maps constant names to their computed values:
+
+```python
+COMPUTED_CONSTANTS = {
+    # From src/v/serde/rw/chrono.h:20
+    # inline constexpr auto max_serializable_ms = std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::nanoseconds::max());
+    # Calculation: std::numeric_limits<int64_t>::max() / 1,000,000 = 9223372036854775807 / 1000000 = 9223372036854 ms
+    "max_serializable_ms": 9223372036854,  # ~292 years in milliseconds
+}
+```
+
+### 2. FriendlyDefaultTransformer Enhancement
+
+Updated the `FriendlyDefaultTransformer` to check the `COMPUTED_CONSTANTS` dictionary before falling back to string normalization:
+
+```python
+# ------------------------------------------------------------------
+# Computed C++ constants (max_serializable_ms, etc.)
+# ------------------------------------------------------------------
+if d in COMPUTED_CONSTANTS:
+    property["default"] = COMPUTED_CONSTANTS[d]
+    return property
+```
+
+### 3. Test Coverage
+
+Added comprehensive test in `tests/test_known_values.py`:
+
+```python
+def test_max_serializable_ms_constant_resolution(self):
+    """Test that max_serializable_ms constant is resolved to actual numeric value"""
+    info = create_complete_property_info(
+        name="log_message_timestamp_before_max_ms",
+        description="Maximum timestamp difference for record validation",
+        declaration="property<std::chrono::milliseconds> log_message_timestamp_before_max_ms;",
+        metadata="meta{.needs_restart = needs_restart::no, .visibility = visibility::user}",
+        default_value="max_serializable_ms"
+    )
+
+    property = apply_transformer_pipeline(info)
+
+    self.assertEqual(property["name"], "log_message_timestamp_before_max_ms")
+    self.assertEqual(property["type"], "integer")
+    # max_serializable_ms = std::numeric_limits<int64_t>::max() / 1,000,000 = 9223372036854 ms
+    self.assertEqual(property["default"], 9223372036854)
+    self.assertFalse(property["needs_restart"])
+    self.assertEqual(property["visibility"], "user")
+```
+
+## Calculation Details
+
+### max_serializable_ms
+
+**Definition Location:** `src/v/serde/rw/chrono.h:20`
+
+**C++ Expression:**
+```cpp
+std::chrono::duration_cast<std::chrono::milliseconds>(
+  std::chrono::nanoseconds::max()
+)
+```
+
+**Calculation:**
+1. `std::chrono::nanoseconds::max()` = `std::numeric_limits<int64_t>::max()` = `9223372036854775807` nanoseconds
+2. Convert to milliseconds (truncating division): `9223372036854775807 / 1000000` = `9223372036854` milliseconds
+3. This is approximately **292.47 years**
+
+**Verification:**
+```python
+import sys
+
+# std::chrono::nanoseconds uses int64_t for rep
+max_int64 = 9223372036854775807
+
+# Convert nanoseconds to milliseconds (duration_cast truncates)
+max_ms = max_int64 // 1000000
+
+print(f'max_serializable_ms = {max_ms} ms')
+print(f'Which is approximately {max_ms / (1000 * 60 * 60 * 24 * 365):.2f} years')
+```
+
+Output:
+```
+max_serializable_ms = 9223372036854 ms
+Which is approximately 292.47 years
+```
+
+## Result
+
+After the fix, the extracted schema correctly shows:
+```json
+{
+  "log_message_timestamp_before_max_ms": {
+    "default": 9223372036854,  // ✅ Correct numeric value
+    "type": "integer"
+  }
+}
+```
+
+## Adding New Computed Constants
+
+To add support for new computed constants:
+
+1. **Find the definition** in the Redpanda source code
+2. **Compute the value** - either manually or with a Python/C++ test
+3. **Add to COMPUTED_CONSTANTS** dictionary in `transformers.py`:
+   ```python
+   COMPUTED_CONSTANTS = {
+       "existing_constant": 12345,
+       "new_constant_name": computed_value,  # Add comment with source location
+   }
+   ```
+4. **Add a test** in `tests/test_known_values.py` to verify resolution
+5. **Document the calculation** with comments including:
+   - Source file location
+   - C++ expression
+   - Calculation steps
+   - Human-readable interpretation
+
+## Test Coverage
+
+All 53 tests pass, including the new test for `max_serializable_ms` resolution:
+
+```bash
+cd tools/property-extractor
+python -m pytest tests/ -v --tb=short
+# ================================================= 53 passed =================================================
+```
+
+## Benefits
+
+✅ **Accurate defaults** - Properties show actual numeric values instead of symbolic names
+✅ **Type safety** - Numeric properties have numeric defaults, not strings
+✅ **Documentation quality** - Users see real values they can use
+✅ **Maintainability** - Centralized mapping makes updates easy
+✅ **Test coverage** - Ensures constants resolve correctly
+
+## Related Files
+
+- `tools/property-extractor/transformers.py` - COMPUTED_CONSTANTS dictionary and resolution logic
+- `tools/property-extractor/tests/test_known_values.py` - Test for constant resolution
+- `src/v/serde/rw/chrono.h` - Source definition of max_serializable_ms
+- `tools/property-extractor/CI_INTEGRATION.md` - Updated test count documentation

package/tools/property-extractor/Makefile

@@ -119,7 +119,18 @@ generate-docs: node-deps
 	fi
 	@echo "📄 Copying properties JSON files to $(OUTPUT_JSON_DIR)…"
 	@if [ -f "$(TOOL_ROOT)/gen/$(TAG)-properties.json" ]; then \
-	  cp "$(TOOL_ROOT)/gen/$(TAG)-properties.json" "$(OUTPUT_JSON_DIR)/"; \
+	  echo "   Source: $(TOOL_ROOT)/gen/$(TAG)-properties.json"; \
+	  echo "   Target: $(OUTPUT_JSON_DIR)/$(TAG)-properties.json"; \
+	  cp "$(TOOL_ROOT)/gen/$(TAG)-properties.json" "$(OUTPUT_JSON_DIR)/" && \
+	  echo "   ✅ Successfully copied $(TAG)-properties.json" || \
+	  { echo "   ❌ Failed to copy $(TAG)-properties.json"; exit 1; }; \
+	  if [ -f "$(OUTPUT_JSON_DIR)/$(TAG)-properties.json" ]; then \
+	    echo "   📊 Verification: Target file exists and has $$(wc -c < "$(OUTPUT_JSON_DIR)/$(TAG)-properties.json") bytes"; \
+	  else \
+	    echo "   ❌ Verification failed: Target file does not exist"; exit 1; \
+	  fi; \
+	else \
+	  echo "   ⚠️  Source file $(TOOL_ROOT)/gen/$(TAG)-properties.json not found, skipping copy"; \
 	fi
 	@echo "✅ Docs generated at $(OUTPUT_AUTOGENERATED_DIR)"