@redpanda-data/docs-extensions-and-macros 5.57.0 → 5.58.0

package/tools/property-extractor/README.adoc

@@ -1223,7 +1223,7 @@ npm test -- --coverage
 ==== The live overrides corpus
 
 `__tests__/tools/property-extractor/property-corpus-rendering.test.js` runs the real
-overrides file, not a fixture, because a fixture passes while the live 438-entry corpus
+overrides file, not a fixture, because a fixture passes while the live overrides corpus
 breaks. Two files feed it, both mirrors of the `docs` repo:
 
 `__tests__/docs-data/property-overrides.json`:: A byte copy of `docs-data/property-overrides.json`.
@@ -1245,7 +1245,9 @@ test, so it sat three months and 114 entries behind. That test now has its own
 `property-overrides-admin-fixture.json`; put new fixture data there or in the test
 itself, never in the mirror.
 
-Refresh both from a `docs` checkout:
+Every Monday, `.github/workflows/property-corpus-drift.yml` compares both files against `docs` main. When they have drifted, it runs the recipe below and opens or updates one pull request from `bot/refresh-property-corpus`. If that PR's tests fail, the refresh has exposed a real bug: push the fix to the same branch, and while the PR is open the job stops updating the branch once its latest commit isn't the bot's.
+
+To refresh by hand, for example between weekly runs, run the same recipe from a `docs` checkout:
 
 [,bash]
 ----

package/tools/redpanda-connect/AUTOMATION.md

@@ -169,7 +169,8 @@ For each version pair `(oldVersion → newVersion)`:
     "removedFields": 2,       // Breaking changes!
     "deprecatedComponents": 0,
     "deprecatedFields": 1,
-    "changedDefaults": 0
+    "changedDefaults": 0,     // A default changed or was removed (breaking)
+    "documentedDefaults": 0   // A field with no default now states one (informational)
   },
   "details": {
     "newComponents": [
@@ -385,7 +386,8 @@ Changes between two consecutive versions:
     "newFields": [...],
     "removedFields": [...],
     "deprecatedFields": [...],
-    "changedDefaults": [...]
+    "changedDefaults": [...],
+    "documentedDefaults": [...]
   },
   "binaryAnalysis": {
     "versions": {

package/tools/redpanda-connect/multi-version-summary.js

@@ -63,7 +63,8 @@ function createMasterDiff(intermediateResults, finalDiffPath, outputPath) {
     removedFields: releases.reduce((sum, r) => sum + (r.summary.removedFields || 0), 0),
     deprecatedComponents: releases.reduce((sum, r) => sum + (r.summary.deprecatedComponents || 0), 0),
     deprecatedFields: releases.reduce((sum, r) => sum + (r.summary.deprecatedFields || 0), 0),
-    changedDefaults: releases.reduce((sum, r) => sum + (r.summary.changedDefaults || 0), 0)
+    changedDefaults: releases.reduce((sum, r) => sum + (r.summary.changedDefaults || 0), 0),
+    documentedDefaults: releases.reduce((sum, r) => sum + (r.summary.documentedDefaults || 0), 0)
   };
 
   const masterDiff = {

package/tools/redpanda-connect/pr-summary-formatter.js

@@ -3,6 +3,16 @@
  * Outputs a console-parseable format for GitHub Actions
  */
 
+/**
+ * Format a field default for the Markdown PR summary. A field with no default
+ * is `undefined`; a declared null default (Default(nil)) is a value.
+ * @param {*} value - The default, or undefined when the field has none
+ * @returns {string} Inline code, or `_no default_`
+ */
+function formatDefault(value) {
+  return value === undefined ? '_no default_' : `\`${JSON.stringify(value)}\``;
+}
+
 /**
  * Render Bloblang changes section for PR summary
  * @param {object} data - Bloblang change data
@@ -238,6 +248,9 @@ function generateMultiVersionPRSummary(masterDiff, binaryAnalysis = null, drafte
   if (total.changedDefaults > 0) {
     lines.push(`- **${total.changedDefaults}** changed default values`);
   }
+  if (total.documentedDefaults > 0) {
+    lines.push(`- **${total.documentedDefaults}** newly stated default values`);
+  }
   if (total.newBloblangMethods > 0) {
     lines.push(`- **${total.newBloblangMethods}** new Bloblang methods`);
   }
@@ -375,12 +388,25 @@ function generateMultiVersionPRSummary(masterDiff, binaryAnalysis = null, drafte
       lines.push('| Component | Field | Old Default | New Default |');
       lines.push('|-----------|-------|-------------|-------------|');
       changedDefaults.forEach(change => {
-        const oldVal = change.oldDefault !== undefined ? `\`${JSON.stringify(change.oldDefault)}\`` : '_none_';
-        const newVal = change.newDefault !== undefined ? `\`${JSON.stringify(change.newDefault)}\`` : '_none_';
+        const oldVal = formatDefault(change.oldDefault);
+        const newVal = formatDefault(change.newDefault);
         lines.push(`| \`${change.component}\` | \`${change.field}\` | ${oldVal} | ${newVal} |`);
       });
       lines.push('');
     }
+
+    // Fields that gained a stated default
+    const documentedDefaults = details.documentedDefaults || [];
+    if (documentedDefaults.length > 0) {
+      lines.push(`**Newly Stated Defaults (${documentedDefaults.length}):**`);
+      lines.push('');
+      lines.push("_These fields now state a default. They usually behaved this way already, so they are not listed as changes in What's new. Check that none of them changed behavior._");
+      lines.push('');
+      documentedDefaults.forEach(change => {
+        lines.push(`- \`${change.component}.${change.field}\`: ${formatDefault(change.newDefault)}`);
+      });
+      lines.push('');
+    }
   }
 
   // Writer action items (aggregate)
@@ -558,9 +584,9 @@ function generateMultiVersionPRSummary(masterDiff, binaryAnalysis = null, drafte
     lines.push('**Review Changed Defaults for Breaking Changes:**');
     lines.push('');
     allChangedDefaults.forEach(change => {
-      const oldVal = change.oldDefault !== undefined ? JSON.stringify(change.oldDefault) : 'none';
-      const newVal = change.newDefault !== undefined ? JSON.stringify(change.newDefault) : 'none';
-      lines.push(`- [ ] \`${change.component}.${change.field}\`: \`${oldVal}\` → \`${newVal}\` — changed in **${change.version}**`);
+      const oldVal = formatDefault(change.oldDefault);
+      const newVal = formatDefault(change.newDefault);
+      lines.push(`- [ ] \`${change.component}.${change.field}\`: ${oldVal} → ${newVal} — changed in **${change.version}**`);
     });
     lines.push('');
   }
@@ -717,6 +743,10 @@ function generatePRSummary(diffData, binaryAnalysis = null, draftedConnectors =
     lines.push(`- **${stats.changedDefaults}** default value change${stats.changedDefaults !== 1 ? 's' : ''}`);
   }
 
+  if (stats.documentedDefaults > 0) {
+    lines.push(`- **${stats.documentedDefaults}** newly stated default value${stats.documentedDefaults !== 1 ? 's' : ''} (no behavior change expected)`);
+  }
+
   lines.push('');
 
   // Writer Reminder for Commercial Names
@@ -1233,14 +1263,25 @@ function generatePRSummary(diffData, binaryAnalysis = null, draftedConnectors =
       const [type, name] = component.split(':');
       lines.push(`**${type}/${name}:**`);
       changes.forEach(c => {
-        const oldStr = JSON.stringify(c.oldDefault);
-        const newStr = JSON.stringify(c.newDefault);
-        lines.push(`- \`${c.field}\`: ${oldStr} → ${newStr}`);
+        lines.push(`- \`${c.field}\`: ${formatDefault(c.oldDefault)} → ${formatDefault(c.newDefault)}`);
       });
       lines.push('');
     });
   }
 
+  // Newly stated defaults
+  const documentedDefaults = diffData.details.documentedDefaults || [];
+  if (documentedDefaults.length > 0) {
+    lines.push('#### Newly Stated Defaults');
+    lines.push('');
+    lines.push("_These fields now state a default. They usually behaved this way already, so they are not listed as changes in What's new. Check that none of them changed behavior._");
+    lines.push('');
+    documentedDefaults.forEach(c => {
+      lines.push(`- \`${c.component}.${c.field}\`: ${formatDefault(c.newDefault)}`);
+    });
+    lines.push('');
+  }
+
   // New Bloblang Methods
   if (stats.newBloblangMethods > 0 && diffData.details.newBloblangMethods) {
     lines.push('#### New Bloblang Methods');

package/tools/redpanda-connect/report-delta.js

@@ -175,6 +175,10 @@ function generateConnectorDiffJson(oldIndex, newIndex, opts = {}) {
   const deprecatedFields = [];
   // Changed default values
   const changedDefaults = [];
+  // Fields that had no default and now state one. The field behaved the same
+  // way before, the spec just didn't say so, so these are not default changes
+  // and stay out of What's new and the breaking-change warning.
+  const documentedDefaults = [];
 
   Object.keys(newMap).forEach(cKey => {
     if (!(cKey in oldMap)) return;
@@ -209,14 +213,16 @@ function generateConnectorDiffJson(oldIndex, newIndex, opts = {}) {
         const newDefaultStr = JSON.stringify(newDefault);
 
         if (oldDefaultStr !== newDefaultStr) {
-          changedDefaults.push({
+          const entry = {
             component: cKey,
             field: fName,
             oldDefault: oldDefault,
             newDefault: newDefault,
             description: newFieldObj && newFieldObj.description,
             shortDescription: newFieldObj && newFieldObj.short_description
-          });
+          };
+          if (oldDefault === undefined) documentedDefaults.push(entry);
+          else changedDefaults.push(entry);
         }
       }
     });
@@ -280,6 +286,7 @@ function generateConnectorDiffJson(oldIndex, newIndex, opts = {}) {
       deprecatedComponents: deprecatedComponents.length,
       deprecatedFields: deprecatedFields.length,
       changedDefaults: changedDefaults.length,
+      documentedDefaults: documentedDefaults.length,
       platformTransitions: platformTransitions.length,
       newBloblangMethods: newBloblangMethods.length,
       removedBloblangMethods: removedBloblangMethods.length,
@@ -296,6 +303,7 @@ function generateConnectorDiffJson(oldIndex, newIndex, opts = {}) {
       deprecatedComponents,
       deprecatedFields,
       changedDefaults,
+      documentedDefaults,
       platformTransitions,
       newBloblangMethods,
       removedBloblangMethods,
@@ -646,7 +654,8 @@ function printDeltaReport(oldIndex, newIndex, opts = {}) {
         const oldDefaultStr = JSON.stringify(oldDefault);
         const newDefaultStr = JSON.stringify(newDefault);
 
-        if (oldDefaultStr !== newDefaultStr) {
+        // A field that had no default and now states one is not a change.
+        if (oldDefaultStr !== newDefaultStr && oldDefault !== undefined) {
           changedDefaultsList.push({
             component: cKey,
             field: fName,
@@ -719,8 +728,8 @@ function printDeltaReport(oldIndex, newIndex, opts = {}) {
     console.log('➤ Changed default values:');
     changedDefaultsList.forEach(entry => {
       const { component, field, oldDefault, newDefault } = entry;
-      const oldStr = JSON.stringify(oldDefault);
-      const newStr = JSON.stringify(newDefault);
+      const oldStr = oldDefault === undefined ? 'no default' : JSON.stringify(oldDefault);
+      const newStr = newDefault === undefined ? 'no default' : JSON.stringify(newDefault);
       console.log(`   • ${component} → ${field}`);
       console.log(`     Old: ${oldStr}`);
       console.log(`     New: ${newStr}`);

package/tools/redpanda-connect/rpcn-connector-docs-handler.js

@@ -657,7 +657,8 @@ function buildChangedDefaultsTable (changedDefaults, capFn) {
   const byFieldAndDefaults = {}
   for (const change of changedDefaults) {
     const [type, compName] = change.component.split(':')
-    const compositeKey = `${change.field}|${String(change.oldDefault)}|${String(change.newDefault)}`
+    // JSON keeps null apart from "null" and one object default apart from another.
+    const compositeKey = JSON.stringify([change.field, change.oldDefault, change.newDefault])
     if (!byFieldAndDefaults[compositeKey]) {
       byFieldAndDefaults[compositeKey] = {
         field: change.field,
@@ -675,23 +676,31 @@ function buildChangedDefaultsTable (changedDefaults, capFn) {
   section += '|Field |Old default |New default |Description |Affected components\n\n'
 
   for (const [, info] of Object.entries(byFieldAndDefaults)) {
-    // Values render as literal inline code, so interpolations like
-    // ${!uuid_v4()} and characters like * or _ show as typed.
     // Values render as literal inline code through pass:c, which, unlike
     // `+...+`, also keeps values with ++ or with spaces at either end. Strings
     // that are empty, start or end with whitespace, or contain a newline show
     // as JSON, so " " and "\n" read as the values they are.
-    const formatDefault = (val) => {
-      if (val === undefined || val === null) return 'none'
+    const formatDefault = (val, asJson = false) => {
+      // 'none' would read as a value, and is one for some fields (compression).
+      if (val === undefined) return '_no default_'
       let text
-      if (typeof val === 'string') text = (val === '' || /^\s|\s$|\n/.test(val)) ? JSON.stringify(val) : val
+      if (asJson) text = JSON.stringify(val)
+      // A null default is declared (Default(nil)), so it renders as a value.
+      else if (val === null) text = 'null'
+      else if (typeof val === 'string') text = (val === '' || /^\s|\s$|\n/.test(val)) ? JSON.stringify(val) : val
       else if (typeof val === 'number' || typeof val === 'boolean') text = String(val)
       else text = JSON.stringify(val)
       return `\`pass:c[${text.replace(/\]/g, '\\]').replace(/\|/g, '\\|')}]\``
     }
 
-    const oldVal = formatDefault(info.oldDefault)
-    const newVal = formatDefault(info.newDefault)
+    let oldVal = formatDefault(info.oldDefault)
+    let newVal = formatDefault(info.newDefault)
+    // A type change such as "5" to 5 or "null" to null would read as no
+    // change, so show both sides as JSON.
+    if (oldVal === newVal) {
+      oldVal = formatDefault(info.oldDefault, true)
+      newVal = formatDefault(info.newDefault, true)
+    }
     // A | in the description would end the cell and shift the row.
     const desc = info.description ? capFn(info.description).replace(/\|/g, '\\|') : '// TODO: Add description'