@redpanda-data/docs-extensions-and-macros 5.51.2 → 5.53.0

package/tools/lint-strings/surfaces/rpk.js

@@ -389,6 +389,7 @@ module.exports = {
   extract,
   scanFile,
   identity,
+  parseLiteralFields,
   rules: RULES,
   // Shorts and flag usages are one-liners by convention; the generic
   // too-short prose rule would flag nearly every conforming declaration.

package/tools/redpanda-connect/generate-rpcn-connector-docs.js

@@ -334,6 +334,11 @@ async function generateRpcnConnectorDocs(options) {
     templateDescription,
     templateBloblang,
     writeFullDrafts,
+    // False when the connect repo provides the generated partials, config
+    // snippets, and Bloblang reference (see the modify-connect-tag-playbook
+    // extension). Drafts are still written, and still include the description
+    // partial the connect repo publishes for them.
+    writePartials = true,
     // Explicit opt-in for the destructive description-partial orphan sweep.
     // Off by default: see ORPHAN_SWEEP_MAX_FRACTION.
     pruneOrphanedDescriptions = false,
@@ -458,7 +463,7 @@ async function generateRpcnConnectorDocs(options) {
     : path.join(outputRoot, 'components');
   const configExamplesRoot = path.resolve(process.cwd(), 'modules/components/examples');
 
-  if (!writeFullDrafts) {
+  if (!writeFullDrafts && writePartials) {
     fs.mkdirSync(fieldsOutRoot,   { recursive: true });
     fs.mkdirSync(examplesOutRoot, { recursive: true });
     fs.mkdirSync(metadataOutRoot, { recursive: true });
@@ -512,7 +517,7 @@ async function generateRpcnConnectorDocs(options) {
         examplesOut = renderExamplesPartial(item);
       }
 
-      if (fieldsOut.trim()) {
+      if (writePartials && fieldsOut.trim()) {
         const fPath = path.join(fieldsOutRoot, type, `${name}.adoc`);
         fs.mkdirSync(path.dirname(fPath), { recursive: true });
         if (fs.existsSync(fPath)) {
@@ -532,7 +537,7 @@ async function generateRpcnConnectorDocs(options) {
       // gate, one list. The old denylist named only the two bloblang keys, so
       // any other non-page data key (config today, anything added upstream
       // tomorrow) leaked partials no page can include.
-      if (examplesOut.trim() && CONNECTOR_PAGE_TYPE_DIRS.has(partialTypeDir)) {
+      if (writePartials && examplesOut.trim() && CONNECTOR_PAGE_TYPE_DIRS.has(partialTypeDir)) {
         const ePath = path.join(examplesOutRoot, type, `${name}.adoc`);
         fs.mkdirSync(path.dirname(ePath), { recursive: true });
         fs.writeFileSync(ePath, examplesOut);
@@ -542,7 +547,7 @@ async function generateRpcnConnectorDocs(options) {
         }
       }
 
-      if (CONNECTOR_PAGE_TYPE_DIRS.has(partialTypeDir)) {
+      if (writePartials && CONNECTOR_PAGE_TYPE_DIRS.has(partialTypeDir)) {
         // Render the metadata partial from the connector's description. Only
         // written when the description contains a `== Metadata` section.
         // Rendered inside the guard, not before it: the loop covers every
@@ -611,8 +616,8 @@ async function generateRpcnConnectorDocs(options) {
         visitedDescriptionPartials.add(`${partialTypeDir}/${name}`);
         const dPath = path.join(descriptionOutRoot, partialTypeDir, `${name}.adoc`);
         if (descriptionOut.trim()) {
-          fs.mkdirSync(path.dirname(dPath), { recursive: true });
-          if (fs.existsSync(dPath)) {
+          if (writePartials) fs.mkdirSync(path.dirname(dPath), { recursive: true });
+          if (writePartials && fs.existsSync(dPath)) {
             // Same content-loss check the metadata partial ten lines up has
             // had since #236, for the same reason: regeneration is
             // authoritative, but a section disappearing from a published
@@ -637,7 +642,7 @@ async function generateRpcnConnectorDocs(options) {
               );
             }
           }
-          fs.writeFileSync(dPath, descriptionOut);
+          if (writePartials) fs.writeFileSync(dPath, descriptionOut);
           // The partial now exists for this connector, so a drafted page can
           // include its body instead of freezing the summary and description
           // into the page. Gated on the file actually being written:
@@ -652,7 +657,7 @@ async function generateRpcnConnectorDocs(options) {
           if (typeof item.summary === 'string' && item.summary.trim()) {
             item.hasDescriptionMeta = true;
           }
-          if (!writeFullDrafts) {
+          if (!writeFullDrafts && writePartials) {
             partialsWritten++;
             partialFiles.push(path.relative(process.cwd(), dPath));
           }
@@ -683,7 +688,7 @@ async function generateRpcnConnectorDocs(options) {
             descriptionReports.push({ connector: `${partialTypeDir}/${name}`, message: seqMsg });
             console.warn(seqMsg);
           }
-        } else if (fs.existsSync(dPath)) {
+        } else if (writePartials && fs.existsSync(dPath)) {
           // The upstream description disappeared, but a previously generated
           // partial is still on disk and an already published page may include
           // it. Deleting the file would break that include, so blank it to a
@@ -732,6 +737,11 @@ async function generateRpcnConnectorDocs(options) {
 
         // typeDir is already computed at the beginning of the loop
 
+        // When the connect repo publishes the partials, the draft includes
+        // its availability, footnotes, and catalog data instead of freezing
+        // categories and the cgo requirement into the page.
+        item.connectPartials = !writePartials;
+
         let content;
         try {
           content = compiledTemplate(item);
@@ -767,7 +777,7 @@ async function generateRpcnConnectorDocs(options) {
   }
 
   // Bloblang function/method partials (only if includeBloblang is true)
-  if (options.includeBloblang) {
+  if (writePartials && options.includeBloblang) {
     for (const { key, folder } of bloblangTypes) {
       const items = dataObj[key];
       if (!Array.isArray(items)) continue;
@@ -877,7 +887,7 @@ async function generateRpcnConnectorDocs(options) {
   // Common/Advanced config snippet YAMLs in modules/components/examples
   const commonConfig = helpers.commonConfig;
   const advancedConfig = helpers.advancedConfig;
-  for (const [type, items] of Object.entries(dataObj)) {
+  for (const [type, items] of writePartials ? Object.entries(dataObj) : []) {
     if (!Array.isArray(items)) continue;
     for (const item of items) {
       if (!item.name || !item.config || !Array.isArray(item.config.children)) continue;
@@ -908,7 +918,7 @@ async function generateRpcnConnectorDocs(options) {
   // The sweep is destructive against published content, so collect the
   // candidates first and check the blast radius before writing anything.
   let orphanSweepSkipped = null;
-  if (!writeFullDrafts && fs.existsSync(descriptionOutRoot)) {
+  if (!writeFullDrafts && writePartials && fs.existsSync(descriptionOutRoot)) {
     const orphans = [];
     let partialsOnDisk = 0;
     for (const typeDirName of fs.readdirSync(descriptionOutRoot)) {
@@ -966,7 +976,7 @@ async function generateRpcnConnectorDocs(options) {
   // Self-heal published page headers: pages are one-time drafts, so pages
   // created before the template emitted :description: never got one.
   let descriptionBackfill = { backfilled: [], skippedNoSummary: [] };
-  if (!writeFullDrafts) {
+  if (!writeFullDrafts && writePartials) {
     descriptionBackfill = backfillPageDescriptions(dataObj);
   }
 

package/tools/redpanda-connect/report-delta.js

@@ -24,6 +24,7 @@ function generateConnectorDiffJson(oldIndex, newIndex, opts = {}) {
       type,
       status: raw.status || '',
       version: raw.version || raw.introducedInVersion || '',
+      summary: raw.summary || '',
       description: raw.description || '',
       requiresCgo: metadata.requiresCgo || false,
       cloudOnly: metadata.cloudOnly || false,

package/tools/redpanda-connect/rpcn-connector-docs-handler.js

@@ -399,7 +399,7 @@ function updateWhatsNew ({ dataDir, oldVersion, newVersion, binaryAnalysis }) {
       for (const comp of regularComponents) {
         const typeLabel = comp.type.charAt(0).toUpperCase() + comp.type.slice(1)
         const statusLabel = comp.status || '-'
-        let desc = comp.summary || (comp.description ? capToTwoSentences(comp.description) : '// TODO: Add description')
+        let desc = comp.summary ? capToTwoSentences(comp.summary) : (comp.description ? capToTwoSentences(comp.description) : '// TODO: Add description')
 
         if (comp.requiresCgo) {
           const cgoNote = '\nNOTE: Requires a cgo-enabled binary. See the xref:install:index.adoc[installation guides] for details.'
@@ -1243,8 +1243,10 @@ async function handleRpcnConnectorDocs (options) {
   // Main Processing: Handle the latest version (final iteration)
   // ========================================================================
 
-  console.log('Generating connector partials...')
-  let partialsWritten, partialFiles
+  // options.partials is false with --no-partials: the connect repo publishes
+  // the generated partials, so this run only updates release-level content.
+  const writePartials = options.partials !== false
+  let partialsWritten = 0, partialFiles = []
   const descriptionReports = []
   const lostSectionWarnings = []
   const styleRegressionWarnings = []
@@ -1256,7 +1258,10 @@ async function handleRpcnConnectorDocs (options) {
   const { collectGeneratorReports } = require('./pr-summary-formatter.js')
   const collect = (result) => collectGeneratorReports(result, { descriptionReports, lostSectionWarnings, styleRegressionWarnings })
 
-  try {
+  if (!writePartials) {
+    console.log('Skipping connector partials (--no-partials): the connect repo provides them.')
+  } else try {
+    console.log('Generating connector partials...')
     const result = await generateRpcnConnectorDocs({
       data: dataFile,
       overrides: options.overrides,
@@ -1511,20 +1516,20 @@ async function handleRpcnConnectorDocs (options) {
     }
   }
 
-  // Publish merged version to attachments
+  // Publish the connector data to attachments. The Bloblang playground and
+  // other site tools read this file, so it is published on every run, with
+  // overrides merged in only when an overrides file still exists.
   // IMPORTANT: This must run AFTER binary analysis and augmentation to include CGO-only connectors
-  if (options.overrides && fs.existsSync(options.overrides)) {
+  {
     try {
-      const { mergeOverrides, resolveReferences } = require('./generate-rpcn-connector-docs.js')
-
       // Use the augmented newIndex which now includes CGO-only and cloud-only connectors
       const mergedData = JSON.parse(JSON.stringify(newIndex))
 
-      const ovRaw = fs.readFileSync(options.overrides, 'utf8')
-      const ovObj = JSON.parse(ovRaw)
-      const resolvedOverrides = resolveReferences(ovObj, ovObj)
-
-      mergeOverrides(mergedData, resolvedOverrides)
+      if (options.overrides && fs.existsSync(options.overrides)) {
+        const { mergeOverrides, resolveReferences } = require('./generate-rpcn-connector-docs.js')
+        const ovObj = JSON.parse(fs.readFileSync(options.overrides, 'utf8'))
+        mergeOverrides(mergedData, resolveReferences(ovObj, ovObj))
+      }
 
       const attachmentsRoot = path.resolve(process.cwd(), 'modules/components/attachments')
       fs.mkdirSync(attachmentsRoot, { recursive: true })
@@ -2066,6 +2071,7 @@ async function handleRpcnConnectorDocs (options) {
           templateDescription: options.templateDescription,
           templateIntro: options.templateIntro,
           writeFullDrafts: true,
+          writePartials,
           cgoOnly: binaryAnalysis?.cgoOnly || [],
           cloudOnly: binaryAnalysis?.comparison?.cloudOnly || [],
           csvMetadata

package/tools/redpanda-connect/templates/connector.hbs

@@ -1,8 +1,14 @@
 = {{{name}}}
 // tag::single-source[]
-:type: {{type}}
-{{#if support}}:support: {{support}}{{/if}}
+{{!-- Drafts are written once and never regenerated, so they hold no
+     reference facts that change with the source. The type comes from the
+     page directory, the support level and Cloud availability from the
+     catalog data, and, when the connect repo publishes the partials
+     (connectPartials), the categories from its catalog.json and the status
+     and cgo flags from the description partial's meta region. --}}
+{{#unless connectPartials}}
 :categories: [{{join categories ", "}}]
+{{/unless}}
 {{#if hasDescriptionMeta}}
 // tag::meta[]
 {{{descriptionInclude this "meta"}}}
@@ -27,6 +33,10 @@
      (cloud-docs .github/description-allowlist.txt). --}}
 
 component_type_dropdown::[]
+{{#if connectPartials}}
+
+include::connect:components:partial$availability/{{typeDir}}/{{name}}.adoc[opts=optional]
+{{/if}}
 
 {{#if hasDescriptionPartial}}
 {{{descriptionInclude this "body"}}}
@@ -58,6 +68,7 @@ include::components:example$advanced/{{typeDir}}/{{name}}.yaml[]
 ======
 {{/if}}
 
+{{#unless connectPartials}}
 {{#if requiresCgo}}
 == Requirements
 
@@ -71,6 +82,7 @@ You can either xref:install:prebuilt-binary.adoc[download a prebuilt cgo-enabled
 {{/if}}
 
 {{/if}}
+{{/unless}}
 {{#if (or this.config.children children)}}
 include::connect:components:partial$fields/{{typeDir}}/{{name}}.adoc[]
 {{/if}}
@@ -78,6 +90,12 @@ include::connect:components:partial$fields/{{typeDir}}/{{name}}.adoc[]
 {{#if this.examples}}
 include::connect:components:partial$examples/{{typeDir}}/{{name}}.adoc[]
 {{/if}}
+{{#if connectPartials}}
+{{#if hasDescriptionPartial}}
+
+{{{descriptionInclude this "footnotes"}}}
+{{/if}}
+{{/if}}
 
 
 // end::single-source[]

package/tools/redpanda-release-notes/generate-release-notes.js

@@ -0,0 +1,242 @@
+// Orchestrate release-notes generation: parse the body, render a candidate
+// section, and insert it into the target page under the mechanical guards
+// (GA-only, floor, idempotency). Pure and IO-free: callers pass the page
+// content in and get the updated content back; the doc-tools action does the
+// file read/write, and the workflow fetches the release body.
+
+const { parseReleaseBody } = require('./parse-release-body');
+const { renderReleaseSection, normalizeVersion } = require('./render-release-notes');
+
+const LOG_TAG = '[release-notes]';
+
+/**
+ * Compares two `X.Y.Z` versions numerically.
+ *
+ * @param {string} a - A version (optionally v-prefixed).
+ * @param {string} b - A version (optionally v-prefixed).
+ * @return {number} -1 if a < b, 0 if equal, 1 if a > b.
+ */
+function compareVersions(a, b) {
+  const pa = normalizeVersion(a).split('.').map(Number);
+  const pb = normalizeVersion(b).split('.').map(Number);
+  for (let i = 0; i < 3; i++) {
+    const x = pa[i] || 0;
+    const y = pb[i] || 0;
+    if (x < y) return -1;
+    if (x > y) return 1;
+  }
+  return 0;
+}
+
+/**
+ * Reports whether a tag is a GA release. Release notes are added for GA tags
+ * only; a prerelease (for example `v26.2.3-rc1`) gets no section.
+ *
+ * @param {string} tag - The release tag.
+ * @return {boolean} True if the tag is a bare `vX.Y.Z` with no prerelease suffix.
+ */
+function isGaTag(tag) {
+  return /^v?\d+\.\d+\.\d+$/.test(String(tag || '').trim());
+}
+
+/**
+ * Reads the `:earliest-tracked-version:` floor attribute from the page.
+ *
+ * @param {string} pageContent - The target page content.
+ * @return {string|null} The bare floor version, or null if the attribute is absent.
+ */
+function readFloor(pageContent) {
+  const m = String(pageContent).match(/^:earliest-tracked-version:\s*(.+?)\s*$/m);
+  return m ? normalizeVersion(m[1]) : null;
+}
+
+/**
+ * Reports whether the page already has a section for this version, so a re-run
+ * for an already-published tag is a no-op rather than a duplicate.
+ *
+ * @param {string} pageContent - The target page content.
+ * @param {string} version - The release version (optionally v-prefixed).
+ * @return {boolean} True if a `== vX.Y.Z (` heading is already present.
+ */
+function hasVersionSection(pageContent, version) {
+  const v = normalizeVersion(version).replace(/[.]/g, '\\.');
+  return new RegExp(`^==\\s+v${v}\\s*\\(`, 'm').test(String(pageContent));
+}
+
+/**
+ * Inserts a rendered section into the page in descending-version order: before
+ * the first existing `== vX.Y.Z` heading OLDER than the section, so a later
+ * backfill of an older release lands below the newer ones already present
+ * rather than jumping to the top. When the section is older than every existing
+ * release, it is inserted AFTER the last version section — before the next
+ * top-level heading following it (the older-versions footer or any other), else
+ * at the end — so a footerless page stays ordered. When there are no version
+ * sections yet, it goes before the older-versions footer, else the first
+ * top-level heading, else the end. The section's own version is read with
+ * multiline matching, so a leading blank line or comment does not hide it.
+ *
+ * @param {string} pageContent - The target page content.
+ * @param {string} sectionText - The rendered section (ending in a newline).
+ * @return {string} The page content with the section inserted.
+ */
+function insertReleaseSection(pageContent, sectionText) {
+  const lines = String(pageContent).split('\n');
+  const sectionLines = sectionText.replace(/\n+$/, '').split('\n');
+  const versionHeading = /^==\s+v(\d+\.\d+\.\d+)\s*\(/;
+  // Multiline: the curated section may open with a blank line or an AsciiDoc
+  // comment, so the heading is not guaranteed to be at character zero.
+  const newVer = (sectionText.match(/^==\s+v(\d+\.\d+\.\d+)\s*\(/m) || [])[1];
+
+  const insertAt = (i) => {
+    // The anchor heading is already preceded by a blank line, so the section
+    // plus one blank separator keeps exactly one blank line on each side.
+    lines.splice(i, 0, ...sectionLines, '');
+    return lines.join('\n');
+  };
+  const appendAtEnd = () => {
+    const tail = lines[lines.length - 1] === '' ? [] : [''];
+    return [...lines, ...tail, ...sectionLines, ''].join('\n');
+  };
+
+  // 1) Descending order: before the first existing section older than this one.
+  //    A new newest release lands at the top this way too.
+  if (newVer) {
+    for (let i = 0; i < lines.length; i++) {
+      const m = lines[i].match(versionHeading);
+      if (m && compareVersions(newVer, m[1]) > 0) return insertAt(i);
+    }
+  }
+
+  // 2) Older than every existing section (or version unknown): sit AFTER the
+  //    last version section — before the next top-level heading that follows it
+  //    (the older-versions footer or any other), else at the end. This keeps a
+  //    footerless page ordered correctly instead of jumping to the top.
+  let lastVersionIdx = -1;
+  for (let i = 0; i < lines.length; i++) {
+    if (versionHeading.test(lines[i])) lastVersionIdx = i;
+  }
+  if (lastVersionIdx !== -1) {
+    for (let i = lastVersionIdx + 1; i < lines.length; i++) {
+      if (/^==\s/.test(lines[i])) return insertAt(i);
+    }
+    return appendAtEnd();
+  }
+
+  // 3) No version sections at all: before the older-versions footer, else
+  //    before the first top-level heading, else at the end.
+  const footerIdx = lines.findIndex((l) => /^==\s+Release notes for older versions/i.test(l));
+  if (footerIdx !== -1) return insertAt(footerIdx);
+  const firstHeadingIdx = lines.findIndex((l) => /^==\s/.test(l));
+  if (firstHeadingIdx !== -1) return insertAt(firstHeadingIdx);
+  return appendAtEnd();
+}
+
+/**
+ * Builds a candidate release section from a raw release body.
+ *
+ * @param {Object} options
+ * @param {string} options.body - The raw rpchangelog release body markdown.
+ * @param {string} options.version - The release version (`v26.2.3` or `26.2.3`).
+ * @param {string} options.date - The release date as `YYYY-MM-DD`.
+ * @return {string} The rendered AsciiDoc section.
+ */
+function buildReleaseSection({ body, version, date }) {
+  const { sections } = parseReleaseBody(body);
+  return renderReleaseSection({ version, date, sections });
+}
+
+/**
+ * Asserts that an externally-produced (curated) section carries exactly one
+ * `== vX.Y.Z (date)` heading whose version matches the requested tag. Phase 2
+ * inserts the section verbatim, and the idempotency guard keys off the tag, so
+ * a heading that disagrees with the tag would defeat idempotency and let
+ * reruns insert duplicates. This is malformed input, so it throws (fail loud)
+ * rather than returning a skip.
+ *
+ * @param {string} section - The curated section text.
+ * @param {string} version - The bare `X.Y.Z` version the tag resolved to.
+ * @throws {Error} If the section has zero or several release headings, or one
+ *   whose version does not match.
+ */
+function assertSectionMatchesTag(section, version) {
+  // Count heading-ish lines loosely (any `== v<digit>…`) so a single MALFORMED
+  // heading fails as malformed rather than slipping through as "found 0".
+  const headingish = String(section).match(/^==\s+v\d.*$/gm) || [];
+  if (headingish.length !== 1) {
+    throw new Error(`Curated section must contain exactly one "== vX.Y.Z (YYYY-MM-DD)" heading; found ${headingish.length}.`);
+  }
+  // Validate the complete heading: version, a YYYY-MM-DD date, a closing paren,
+  // and nothing trailing.
+  const m = headingish[0].match(/^==\s+v(\d+\.\d+\.\d+)\s+\(\d{4}-\d{2}-\d{2}\)\s*$/);
+  if (!m) {
+    throw new Error(`Malformed release heading ${JSON.stringify(headingish[0])}; expected "== vX.Y.Z (YYYY-MM-DD)".`);
+  }
+  if (m[1] !== version) {
+    throw new Error(`Curated section heading (v${m[1]}) does not match the requested tag (v${version}).`);
+  }
+}
+
+/**
+ * Generates the updated page content for a release, applying the GA, floor, and
+ * idempotency guards. Never throws for a guard miss: returns a `skipped` status
+ * so the workflow can no-op cleanly.
+ *
+ * Two phases, because the LLM curation step sits between them:
+ * - Phase 1 (pass `body`): build the de-noised candidate section from the raw
+ *   release body. This candidate is the curation step's input.
+ * - Phase 2 (pass `section`): insert the curated section the skill produced.
+ * Exactly one of `body` or `section` is supplied. The guards apply in both
+ * phases, since they depend on the tag and the page, not on the entries.
+ *
+ * @param {Object} options
+ * @param {string} [options.body] - Raw rpchangelog release body (phase 1).
+ * @param {string} [options.section] - A pre-built/curated section to insert (phase 2).
+ * @param {string} options.tag - The release tag (`v26.2.3`).
+ * @param {string} [options.date] - The authored release date as `YYYY-MM-DD` (phase 1 only).
+ * @param {string} options.pageContent - The current target page content.
+ * @return {{status: 'ok'|'skipped', reason?: string, section?: string, content?: string}}
+ */
+function generateReleaseNotes({ body, section, tag, date, pageContent }) {
+  if (body == null && section == null) {
+    throw new Error('Provide either body (phase 1) or section (phase 2).');
+  }
+  if (body != null && section != null) {
+    throw new Error('Provide only one of body or section, not both.');
+  }
+
+  if (!isGaTag(tag)) {
+    return { status: 'skipped', reason: `not a GA tag: ${tag}` };
+  }
+  const version = normalizeVersion(tag);
+
+  const floor = readFloor(pageContent);
+  if (floor && compareVersions(version, floor) <= 0) {
+    return { status: 'skipped', reason: `version ${version} is not newer than the floor ${floor}` };
+  }
+
+  if (hasVersionSection(pageContent, version)) {
+    return { status: 'skipped', reason: `page already has a section for v${version}` };
+  }
+
+  let finalSection;
+  if (section != null) {
+    assertSectionMatchesTag(section, version);
+    finalSection = section;
+  } else {
+    finalSection = buildReleaseSection({ body, version, date });
+  }
+  const content = insertReleaseSection(pageContent, finalSection);
+  return { status: 'ok', section: finalSection, content };
+}
+
+module.exports = {
+  generateReleaseNotes,
+  buildReleaseSection,
+  assertSectionMatchesTag,
+  insertReleaseSection,
+  compareVersions,
+  isGaTag,
+  readFloor,
+  hasVersionSection,
+  LOG_TAG,
+};