@redpanda-data/docs-extensions-and-macros 5.12.2 → 5.12.3

package/extensions/generate-fields-only-pages.js

@@ -11,6 +11,7 @@ const handlebars = require('handlebars')
 
 // Import and register Redpanda Connect helpers
 const helpers = require('../tools/redpanda-connect/helpers')
+const { flattenConnectFields } = require('../tools/redpanda-connect/helpers/flattenConnectFields')
 Object.entries(helpers).forEach(([name, fn]) => {
   if (typeof fn === 'function') {
     handlebars.registerHelper(name, fn)
@@ -25,18 +26,11 @@ handlebars.registerHelper('renderConnectFieldsTable', function (children) {
 
   const rows = []
 
-  function collectFields (fieldsList, pathPrefix = '') {
-    if (!Array.isArray(fieldsList)) return
-
-    fieldsList.forEach(field => {
-      if (field.is_deprecated || !field.name) return
-
-      const isArray = field.kind === 'array'
-      const nameWithArray = (typeof field.name === 'string' && isArray && !field.name.endsWith('[]'))
-        ? `${field.name}[]`
-        : field.name
-      const currentPath = pathPrefix ? `${pathPrefix}.${nameWithArray}` : nameWithArray
-
+  // The dotted paths, the `[]` array marker and the deprecated-field skipping
+  // all come from the shared walker, so this table, the reference-page
+  // headings and the connector delta report cannot drift apart.
+  flattenConnectFields(children, { arrayMarker: true, skipDeprecated: true })
+    .forEach(({ path: currentPath, field }) => {
       // Normalize type
       let displayType
       const isArrayTitle = typeof field.name === 'string' && field.name.endsWith('[]')
@@ -88,15 +82,7 @@ handlebars.registerHelper('renderConnectFieldsTable', function (children) {
         default: defaultValue || '-',
         description: desc || '-'
       })
-
-      // Recurse for children
-      if (field.children && Array.isArray(field.children) && field.children.length > 0) {
-        collectFields(field.children, currentPath)
-      }
     })
-  }
-
-  collectFields(children, '')
 
   if (rows.length === 0) return 'No configuration fields available.\n\n'
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@redpanda-data/docs-extensions-and-macros",
-  "version": "5.12.2",
+  "version": "5.12.3",
   "description": "Antora extensions and macros developed for Redpanda documentation.",
   "keywords": [
     "antora",

package/tools/redpanda-connect/helpers/flattenConnectFields.js

@@ -0,0 +1,73 @@
+'use strict';
+
+/**
+ * Shared walker for Redpanda Connect field trees.
+ *
+ * This is a plain module, not a Handlebars helper, so it is deliberately not
+ * exported from helpers/index.js. It exists because the same recursion was
+ * written four times (the reference-page renderer, the fields-only table
+ * extension and the connector delta report), and the copies disagreed about
+ * what a field path looks like. The renderer is the authority: the path this
+ * module produces must match the `=== ` heading renderConnectFields emits,
+ * because the what's-new tables link to those headings.
+ */
+
+/**
+ * Field name as it appears in the rendered docs: an array-of-object field
+ * carries an `[]` marker so the path is a valid config path.
+ * @param {object} field - Field definition
+ * @returns {string} Display name for the field
+ */
+function connectFieldName (field) {
+  const name = field && field.name;
+  if (typeof name === 'string' && field.kind === 'array' && !name.endsWith('[]')) {
+    return `${name}[]`;
+  }
+  return name;
+}
+
+/**
+ * Walk a field tree depth-first and return one entry per field, in the order
+ * the fields appear in the source data.
+ *
+ * A field nested under an existing group (for example
+ * `sqs.zero_key_warn_interval` under aws_s3's pre-existing `sqs` group) is
+ * only visible to a set-based diff if the tree is flattened first: a map of
+ * top-level names alone treats that as no change at all, because `sqs` itself
+ * is not new.
+ *
+ * Nameless nodes are skipped along with their subtree, matching
+ * renderConnectFields: a node with no name gets no heading, so nothing under
+ * it is addressable in the published docs.
+ *
+ * @param {Array} children - Field definitions (each may itself have `children`)
+ * @param {object} [opts] - { arrayMarker, skipDeprecated, prefix }
+ * @param {boolean} [opts.arrayMarker=false] - Append `[]` to array-of-object names
+ * @param {boolean} [opts.skipDeprecated=false] - Skip deprecated fields and their subtrees
+ * @param {string} [opts.prefix=''] - Dotted path of the parent, if any
+ * @returns {Array<{path: string, name: string, parentPath: (string|null), field: object}>}
+ */
+function flattenConnectFields (children, opts = {}) {
+  const { arrayMarker = false, skipDeprecated = false, prefix = '' } = opts;
+  const result = [];
+
+  const walk = (list, currentPrefix, parentPath) => {
+    if (!Array.isArray(list)) return;
+    list.forEach(field => {
+      if (!field || !field.name) return;
+      if (skipDeprecated && field.is_deprecated) return;
+
+      const name = arrayMarker ? connectFieldName(field) : field.name;
+      const path = currentPrefix ? `${currentPrefix}.${name}` : `${name}`;
+
+      result.push({ path, name, parentPath, field });
+      walk(field.children, path, path);
+    });
+  };
+
+  walk(children, prefix, prefix || null);
+
+  return result;
+}
+
+module.exports = { flattenConnectFields, connectFieldName };

package/tools/redpanda-connect/helpers/renderConnectFields.js

@@ -1,6 +1,7 @@
 const yaml = require('yaml');
 const renderYamlList = require('./renderYamlList');
 const handlebars = require('handlebars');
+const { connectFieldName } = require('./flattenConnectFields');
 
 /**
  * Renders the children of a configuration object into AsciiDoc.
@@ -56,11 +57,10 @@ module.exports = function renderConnectFields(children, prefix = '') {
     }
 
     let block = '';
-    const isArray = child.kind === 'array';
-    // Only append [] if kind is array and name does not already end with []
-    const nameWithArray = (typeof child.name === 'string' && isArray && !child.name.endsWith('[]'))
-      ? `${child.name}[]`
-      : child.name;
+    // Only append [] if kind is array and name does not already end with [].
+    // Shared with the delta report and the fields-only table so all three
+    // producers agree on what a field path looks like.
+    const nameWithArray = connectFieldName(child);
     const currentPath = prefix
       ? `${prefix}.${nameWithArray}`
       : `${nameWithArray}`;

package/tools/redpanda-connect/report-delta.js

@@ -1,4 +1,5 @@
 const { execSync } = require('child_process');
+const { flattenConnectFields } = require('./helpers/flattenConnectFields');
 
 /**
  * Generate a JSON diff report between two connector index objects.
@@ -54,22 +55,29 @@ function generateConnectorDiffJson(oldIndex, newIndex, opts = {}) {
     if (!(cKey in oldMap)) return;
     const oldFields = new Set(oldMap[cKey].fields || []);
     const newFieldsArr = newMap[cKey].fields || [];
-    newFieldsArr.forEach(fName => {
-      if (!oldFields.has(fName)) {
-        const [type, compName] = cKey.split(':');
-        let rawFieldObj = null;
-        if (type === 'config') {
-          rawFieldObj = (newMap[cKey].raw.children || []).find(f => f.name === fName);
-        } else {
-          rawFieldObj = (newMap[cKey].raw.config?.children || []).find(f => f.name === fName);
-        }
-        newFields.push({
-          component: cKey,
-          field: fName,
-          introducedIn: rawFieldObj && (rawFieldObj.introducedInVersion || rawFieldObj.version),
-          description: rawFieldObj && rawFieldObj.description
-        });
-      }
+    const added = collapseToTopmost(
+      newFieldsArr.filter(fName => !oldFields.has(fName)),
+      newMap[cKey].parents
+    );
+    added.forEach(fName => {
+      const rawFieldObj = newMap[cKey].fieldMap.get(fName) || null;
+      // A field that arrives already deprecated -- or sits under a
+      // deprecated ancestor group -- gets no heading on the reference page,
+      // so advertising it under "New field support" would link to a fragment
+      // that never exists.
+      if (isDeprecatedOrDescendsFromDeprecated(fName, newMap[cKey].fieldMap, newMap[cKey].parents)) return;
+      newFields.push({
+        component: cKey,
+        field: fName,
+        // Most fields carry no version of their own, so fall back to the
+        // version this diff is comparing against: the field is new as of
+        // exactly that release. Some fields do carry `version` (268 entries
+        // in connect-4.105.0), and that wins, because it can predate the
+        // span being compared: inputs:oracledb_cdc/snapshot_mode reports
+        // 4.99.0 in the 4.98.0 to 4.103.1 diff.
+        introducedIn: (rawFieldObj && (rawFieldObj.introducedInVersion || rawFieldObj.version)) || opts.newVersion || null,
+        description: rawFieldObj && rawFieldObj.description
+      });
     });
   });
 
@@ -79,13 +87,15 @@ function generateConnectorDiffJson(oldIndex, newIndex, opts = {}) {
     if (!(cKey in newMap)) return;
     const newFieldsSet = new Set(newMap[cKey].fields || []);
     const oldFieldsArr = oldMap[cKey].fields || [];
-    oldFieldsArr.forEach(fName => {
-      if (!newFieldsSet.has(fName)) {
-        removedFields.push({
-          component: cKey,
-          field: fName
-        });
-      }
+    const removed = collapseToTopmost(
+      oldFieldsArr.filter(fName => !newFieldsSet.has(fName)),
+      oldMap[cKey].parents
+    );
+    removed.forEach(fName => {
+      removedFields.push({
+        component: cKey,
+        field: fName
+      });
     });
   });
 
@@ -172,26 +182,11 @@ function generateConnectorDiffJson(oldIndex, newIndex, opts = {}) {
     // Check fields that exist in both versions
     const commonFields = oldFieldsArr.filter(f => newFieldsArr.includes(f));
     commonFields.forEach(fName => {
-      const [type, compName] = cKey.split(':');
+      const oldFieldObj = oldMap[cKey].fieldMap.get(fName) || null;
+      const newFieldObj = newMap[cKey].fieldMap.get(fName) || null;
 
-      // Get old field object
-      let oldFieldObj = null;
-      if (type === 'config') {
-        oldFieldObj = (oldMap[cKey].raw.children || []).find(f => f.name === fName);
-      } else {
-        oldFieldObj = (oldMap[cKey].raw.config?.children || []).find(f => f.name === fName);
-      }
-
-      // Get new field object
-      let newFieldObj = null;
-      if (type === 'config') {
-        newFieldObj = (newMap[cKey].raw.children || []).find(f => f.name === fName);
-      } else {
-        newFieldObj = (newMap[cKey].raw.config?.children || []).find(f => f.name === fName);
-      }
-
-      const oldDeprecated = oldFieldObj && (oldFieldObj.is_deprecated === true || oldFieldObj.deprecated === true || (oldFieldObj.status || '').toLowerCase() === 'deprecated');
-      const newDeprecated = newFieldObj && (newFieldObj.is_deprecated === true || newFieldObj.deprecated === true || (newFieldObj.status || '').toLowerCase() === 'deprecated');
+      const oldDeprecated = isDeprecatedField(oldFieldObj);
+      const newDeprecated = isDeprecatedField(newFieldObj);
 
       if (!oldDeprecated && newDeprecated) {
         deprecatedFields.push({
@@ -380,6 +375,67 @@ function discoverComponentKeys(obj) {
   return Object.keys(obj).filter(key => Array.isArray(obj[key]));
 }
 
+/**
+ * Whether a field is marked deprecated.
+ *
+ * Three markers have been used by the source data over time. Only
+ * `is_deprecated` appears in current releases (verified across connect-4.63.0,
+ * 4.104.0 and 4.105.0: 85, 75 and 75 fields, zero uses of the other two), and
+ * it is the one renderConnectFields keys on when it decides not to render a
+ * heading for a field.
+ *
+ * @param {object} field - Field definition, or null
+ * @returns {boolean} True when the field is deprecated
+ */
+function isDeprecatedField(field) {
+  if (!field) return false;
+  return field.is_deprecated === true ||
+    field.deprecated === true ||
+    (field.status || '').toLowerCase() === 'deprecated';
+}
+
+/**
+ * True if `fName` or any of its ancestor fields is deprecated.
+ *
+ * renderConnectFields never renders a field under a deprecated group (the
+ * whole group is hidden), so a leaf-only check reports a field added under an
+ * existing-but-deprecated group in "New field support" even though it never
+ * appears on the page. Walking the parent chain via `parents` is what
+ * catches that.
+ */
+function isDeprecatedOrDescendsFromDeprecated(fName, fieldMap, parents) {
+  let current = fName;
+  while (current) {
+    if (isDeprecatedField(fieldMap.get(current))) return true;
+    current = parents && parents.get(current);
+  }
+  return false;
+}
+
+/**
+ * Drop paths whose parent changed in the same way, keeping only the topmost
+ * field of each changed subtree.
+ *
+ * Adding or removing one group adds or removes every field under it, and the
+ * flattened field list contains all of them. Reporting each descendant turns
+ * a single change into one row per node: removing `batching` from
+ * outputs:redpanda_migrator would list `batching` plus batching.count,
+ * .byte_size, .period, .check and .processors, and the group row already says
+ * all of that. Only the fields added to or removed from a group that still
+ * exists are news.
+ *
+ * @param {Array<string>} changedPaths - Paths that are new (or removed)
+ * @param {Map<string, (string|null)>} parents - Path to parent path
+ * @returns {Array<string>} The topmost changed path of each subtree, in order
+ */
+function collapseToTopmost(changedPaths, parents) {
+  const changed = new Set(changedPaths);
+  return changedPaths.filter(path => {
+    const parent = parents && parents.get(path);
+    return !(parent && changed.has(parent));
+  });
+}
+
 function buildComponentMap(indexObj) {
   const map = {};
   const types = discoverComponentKeys(indexObj);
@@ -402,7 +458,14 @@ function buildComponentMap(indexObj) {
         }
       }
 
-      const fieldNames = childArray.map(f => f.name);
+      // arrayMarker keeps the paths identical to the headings
+      // renderConnectFields emits, so a field name in the what's-new table is
+      // a config path a reader can paste into YAML (`sasl[].aws.tcp`, not the
+      // invalid `sasl.aws.tcp`).
+      const flattened = flattenConnectFields(childArray, { arrayMarker: true });
+      const fieldNames = flattened.map(f => f.path);
+      const fieldMap = new Map(flattened.map(f => [f.path, f.field]));
+      const parents = new Map(flattened.map(f => [f.path, f.parentPath]));
 
       // Preserve platform metadata for accurate diff comparison
       const metadata = {
@@ -414,6 +477,8 @@ function buildComponentMap(indexObj) {
       map[lookupKey] = {
         raw: component,
         fields: fieldNames,
+        fieldMap: fieldMap,
+        parents: parents,
         metadata: metadata
       };
     });
@@ -499,8 +564,11 @@ function getRpkConnectVersion() {
  *  • which connectors/components are brand-new
  *  • which new fields appeared under existing connectors (including “config” entries)
  *  • for each new component/field, if the raw object contains “version” or “introducedInVersion” or “requiresVersion” metadata, print it
+ * @param {object} oldIndex - Previous version connector index
+ * @param {object} newIndex - Current version connector index
+ * @param {object} [opts] - { newVersion } used as the introducedIn fallback
  */
-function printDeltaReport(oldIndex, newIndex) {
+function printDeltaReport(oldIndex, newIndex, opts = {}) {
   const oldMap = buildComponentMap(oldIndex);
   const newMap = buildComponentMap(newIndex);
 
@@ -513,27 +581,24 @@ function printDeltaReport(oldIndex, newIndex) {
     if (!(cKey in oldMap)) return; // skip brand-new components here
     const oldFields = new Set(oldMap[cKey].fields || []);
     const newFieldsArr = newMap[cKey].fields || [];
-    newFieldsArr.forEach(fName => {
-      if (!oldFields.has(fName)) {
-        // fetch raw field metadata if available
-        const [type, compName] = cKey.split(':');
-        let rawFieldObj = null;
-        if (type === 'config') {
-          rawFieldObj = (newMap[cKey].raw.children || []).find(f => f.name === fName);
-        } else {
-          rawFieldObj = (newMap[cKey].raw.config?.children || []).find(f => f.name === fName);
-        }
-
-        let introducedIn = rawFieldObj && (rawFieldObj.introducedInVersion || rawFieldObj.version);
-        let requiresVer = rawFieldObj && rawFieldObj.requiresVersion;
-
-        newFields.push({
-          component: cKey,
-          field: fName,
-          introducedIn,
-          requiresVersion: requiresVer,
-        });
-      }
+    const added = collapseToTopmost(
+      newFieldsArr.filter(fName => !oldFields.has(fName)),
+      newMap[cKey].parents
+    );
+    added.forEach(fName => {
+      const rawFieldObj = newMap[cKey].fieldMap.get(fName) || null;
+      if (isDeprecatedField(rawFieldObj)) return;
+      // Same fallback as generateConnectorDiffJson, so the console report and
+      // the published table cannot disagree about the same input.
+      const introducedIn = (rawFieldObj && (rawFieldObj.introducedInVersion || rawFieldObj.version)) || opts.newVersion || null;
+      const requiresVer = rawFieldObj && rawFieldObj.requiresVersion;
+
+      newFields.push({
+        component: cKey,
+        field: fName,
+        introducedIn,
+        requiresVersion: requiresVer,
+      });
     });
   });
 
@@ -560,21 +625,10 @@ function printDeltaReport(oldIndex, newIndex) {
     const commonFields = oldFieldsArr.filter(f => newFieldsArr.includes(f));
 
     commonFields.forEach(fName => {
-      const [type, compName] = cKey.split(':');
-      let oldFieldObj = null;
-      if (type === 'config') {
-        oldFieldObj = (oldMap[cKey].raw.children || []).find(f => f.name === fName);
-      } else {
-        oldFieldObj = (oldMap[cKey].raw.config?.children || []).find(f => f.name === fName);
-      }
-      let newFieldObj = null;
-      if (type === 'config') {
-        newFieldObj = (newMap[cKey].raw.children || []).find(f => f.name === fName);
-      } else {
-        newFieldObj = (newMap[cKey].raw.config?.children || []).find(f => f.name === fName);
-      }
-      const oldDeprecated = oldFieldObj && (oldFieldObj.is_deprecated === true || oldFieldObj.deprecated === true || (oldFieldObj.status || '').toLowerCase() === 'deprecated');
-      const newDeprecated = newFieldObj && (newFieldObj.is_deprecated === true || newFieldObj.deprecated === true || (newFieldObj.status || '').toLowerCase() === 'deprecated');
+      const oldFieldObj = oldMap[cKey].fieldMap.get(fName) || null;
+      const newFieldObj = newMap[cKey].fieldMap.get(fName) || null;
+      const oldDeprecated = isDeprecatedField(oldFieldObj);
+      const newDeprecated = isDeprecatedField(newFieldObj);
       if (!oldDeprecated && newDeprecated) {
         deprecatedFieldsList.push({ component: cKey, field: fName });
       }

package/tools/redpanda-connect/rpcn-connector-docs-handler.js

@@ -413,7 +413,7 @@ function updateWhatsNew ({ dataDir, oldVersion, newVersion, binaryAnalysis }) {
       if (regularFields.length > 0) {
         section += '\n=== New field support\n\n'
         section += 'This release adds support for the following new fields:\n\n'
-        section += buildFieldsTable(regularFields, capToTwoSentences)
+        section += buildFieldsTable(regularFields, capToTwoSentences, { showIntroducedIn: true })
       }
     }
 
@@ -553,24 +553,35 @@ function fieldAnchor (fieldName) {
  * @param {Function} capFn - Caption function
  * @returns {string} AsciiDoc table
  */
-function buildFieldsTable (fields, capFn) {
+function buildFieldsTable (fields, capFn, opts = {}) {
+  const { showIntroducedIn = false } = opts
   const byField = {}
   for (const field of fields) {
     const [type, compName] = field.component.split(':')
-    if (!byField[field.field]) {
-      byField[field.field] = {
+    // Two components can gain the same field path in different releases
+    // (`urls` arrived in 3.58.0 for one and 4.23.0 for another). Merging those
+    // into a single row would publish the first version for both, so when the
+    // version is on show it is part of the row's identity.
+    const key = showIntroducedIn
+      ? `${field.field}\u0000${field.introducedIn || ''}`
+      : field.field
+    if (!byField[key]) {
+      byField[key] = {
+        field: field.field,
         description: field.description,
+        introducedIn: field.introducedIn,
         components: []
       }
     }
-    byField[field.field].components.push({ type, name: compName })
+    byField[key].components.push({ type, name: compName })
   }
 
-  let section = '[cols="1m,3,2a"]\n'
+  let section = showIntroducedIn ? '[cols="1m,3,2a,1m"]\n' : '[cols="1m,3,2a"]\n'
   section += '|===\n'
-  section += '|Field |Description |Affected components\n\n'
+  section += showIntroducedIn ? '|Field |Description |Affected components |Introduced in\n\n' : '|Field |Description |Affected components\n\n'
 
-  for (const [fieldName, info] of Object.entries(byField)) {
+  for (const info of Object.values(byField)) {
+    const fieldName = info.field
     const byType = {}
     for (const comp of info.components) {
       if (!byType[comp.type]) byType[comp.type] = []
@@ -595,7 +606,9 @@ function buildFieldsTable (fields, capFn) {
 
     section += `|${fieldName}\n`
     section += `|${desc}\n`
-    section += `|${componentList}\n\n`
+    section += `|${componentList}\n`
+    if (showIntroducedIn) section += `|${info.introducedIn || '-'}\n`
+    section += '\n'
   }
 
   section += '|===\n\n'
@@ -1475,7 +1488,7 @@ async function handleRpcnConnectorDocs (options) {
       }
     }
 
-    printDeltaReport(oldIndexForDiff, newIndex)
+    printDeltaReport(oldIndexForDiff, newIndex, { newVersion })
 
     const { generateConnectorDiffJson } = require('./report-delta.js')
     diffJson = generateConnectorDiffJson(