@redpanda-data/docs-extensions-and-macros 5.51.1 → 5.52.0

package/tools/lint-strings/surfaces/rpk.js

@@ -389,6 +389,7 @@ module.exports = {
   extract,
   scanFile,
   identity,
+  parseLiteralFields,
   rules: RULES,
   // Shorts and flag usages are one-liners by convention; the generic
   // too-short prose rule would flag nearly every conforming declaration.

package/tools/redpanda-connect/generate-rpcn-connector-docs.js

@@ -334,6 +334,11 @@ async function generateRpcnConnectorDocs(options) {
     templateDescription,
     templateBloblang,
     writeFullDrafts,
+    // False when the connect repo provides the generated partials, config
+    // snippets, and Bloblang reference (see the modify-connect-tag-playbook
+    // extension). Drafts are still written, and still include the description
+    // partial the connect repo publishes for them.
+    writePartials = true,
     // Explicit opt-in for the destructive description-partial orphan sweep.
     // Off by default: see ORPHAN_SWEEP_MAX_FRACTION.
     pruneOrphanedDescriptions = false,
@@ -458,7 +463,7 @@ async function generateRpcnConnectorDocs(options) {
     : path.join(outputRoot, 'components');
   const configExamplesRoot = path.resolve(process.cwd(), 'modules/components/examples');
 
-  if (!writeFullDrafts) {
+  if (!writeFullDrafts && writePartials) {
     fs.mkdirSync(fieldsOutRoot,   { recursive: true });
     fs.mkdirSync(examplesOutRoot, { recursive: true });
     fs.mkdirSync(metadataOutRoot, { recursive: true });
@@ -512,7 +517,7 @@ async function generateRpcnConnectorDocs(options) {
         examplesOut = renderExamplesPartial(item);
       }
 
-      if (fieldsOut.trim()) {
+      if (writePartials && fieldsOut.trim()) {
         const fPath = path.join(fieldsOutRoot, type, `${name}.adoc`);
         fs.mkdirSync(path.dirname(fPath), { recursive: true });
         if (fs.existsSync(fPath)) {
@@ -532,7 +537,7 @@ async function generateRpcnConnectorDocs(options) {
       // gate, one list. The old denylist named only the two bloblang keys, so
       // any other non-page data key (config today, anything added upstream
       // tomorrow) leaked partials no page can include.
-      if (examplesOut.trim() && CONNECTOR_PAGE_TYPE_DIRS.has(partialTypeDir)) {
+      if (writePartials && examplesOut.trim() && CONNECTOR_PAGE_TYPE_DIRS.has(partialTypeDir)) {
         const ePath = path.join(examplesOutRoot, type, `${name}.adoc`);
         fs.mkdirSync(path.dirname(ePath), { recursive: true });
         fs.writeFileSync(ePath, examplesOut);
@@ -542,7 +547,7 @@ async function generateRpcnConnectorDocs(options) {
         }
       }
 
-      if (CONNECTOR_PAGE_TYPE_DIRS.has(partialTypeDir)) {
+      if (writePartials && CONNECTOR_PAGE_TYPE_DIRS.has(partialTypeDir)) {
         // Render the metadata partial from the connector's description. Only
         // written when the description contains a `== Metadata` section.
         // Rendered inside the guard, not before it: the loop covers every
@@ -611,8 +616,8 @@ async function generateRpcnConnectorDocs(options) {
         visitedDescriptionPartials.add(`${partialTypeDir}/${name}`);
         const dPath = path.join(descriptionOutRoot, partialTypeDir, `${name}.adoc`);
         if (descriptionOut.trim()) {
-          fs.mkdirSync(path.dirname(dPath), { recursive: true });
-          if (fs.existsSync(dPath)) {
+          if (writePartials) fs.mkdirSync(path.dirname(dPath), { recursive: true });
+          if (writePartials && fs.existsSync(dPath)) {
             // Same content-loss check the metadata partial ten lines up has
             // had since #236, for the same reason: regeneration is
             // authoritative, but a section disappearing from a published
@@ -637,7 +642,7 @@ async function generateRpcnConnectorDocs(options) {
               );
             }
           }
-          fs.writeFileSync(dPath, descriptionOut);
+          if (writePartials) fs.writeFileSync(dPath, descriptionOut);
           // The partial now exists for this connector, so a drafted page can
           // include its body instead of freezing the summary and description
           // into the page. Gated on the file actually being written:
@@ -652,7 +657,7 @@ async function generateRpcnConnectorDocs(options) {
           if (typeof item.summary === 'string' && item.summary.trim()) {
             item.hasDescriptionMeta = true;
           }
-          if (!writeFullDrafts) {
+          if (!writeFullDrafts && writePartials) {
             partialsWritten++;
             partialFiles.push(path.relative(process.cwd(), dPath));
           }
@@ -683,7 +688,7 @@ async function generateRpcnConnectorDocs(options) {
             descriptionReports.push({ connector: `${partialTypeDir}/${name}`, message: seqMsg });
             console.warn(seqMsg);
           }
-        } else if (fs.existsSync(dPath)) {
+        } else if (writePartials && fs.existsSync(dPath)) {
           // The upstream description disappeared, but a previously generated
           // partial is still on disk and an already published page may include
           // it. Deleting the file would break that include, so blank it to a
@@ -732,6 +737,11 @@ async function generateRpcnConnectorDocs(options) {
 
         // typeDir is already computed at the beginning of the loop
 
+        // When the connect repo publishes the partials, the draft includes
+        // its availability, footnotes, and catalog data instead of freezing
+        // categories and the cgo requirement into the page.
+        item.connectPartials = !writePartials;
+
         let content;
         try {
           content = compiledTemplate(item);
@@ -767,7 +777,7 @@ async function generateRpcnConnectorDocs(options) {
   }
 
   // Bloblang function/method partials (only if includeBloblang is true)
-  if (options.includeBloblang) {
+  if (writePartials && options.includeBloblang) {
     for (const { key, folder } of bloblangTypes) {
       const items = dataObj[key];
       if (!Array.isArray(items)) continue;
@@ -877,7 +887,7 @@ async function generateRpcnConnectorDocs(options) {
   // Common/Advanced config snippet YAMLs in modules/components/examples
   const commonConfig = helpers.commonConfig;
   const advancedConfig = helpers.advancedConfig;
-  for (const [type, items] of Object.entries(dataObj)) {
+  for (const [type, items] of writePartials ? Object.entries(dataObj) : []) {
     if (!Array.isArray(items)) continue;
     for (const item of items) {
       if (!item.name || !item.config || !Array.isArray(item.config.children)) continue;
@@ -908,7 +918,7 @@ async function generateRpcnConnectorDocs(options) {
   // The sweep is destructive against published content, so collect the
   // candidates first and check the blast radius before writing anything.
   let orphanSweepSkipped = null;
-  if (!writeFullDrafts && fs.existsSync(descriptionOutRoot)) {
+  if (!writeFullDrafts && writePartials && fs.existsSync(descriptionOutRoot)) {
     const orphans = [];
     let partialsOnDisk = 0;
     for (const typeDirName of fs.readdirSync(descriptionOutRoot)) {
@@ -966,7 +976,7 @@ async function generateRpcnConnectorDocs(options) {
   // Self-heal published page headers: pages are one-time drafts, so pages
   // created before the template emitted :description: never got one.
   let descriptionBackfill = { backfilled: [], skippedNoSummary: [] };
-  if (!writeFullDrafts) {
+  if (!writeFullDrafts && writePartials) {
     descriptionBackfill = backfillPageDescriptions(dataObj);
   }
 

package/tools/redpanda-connect/report-delta.js

@@ -24,6 +24,7 @@ function generateConnectorDiffJson(oldIndex, newIndex, opts = {}) {
       type,
       status: raw.status || '',
       version: raw.version || raw.introducedInVersion || '',
+      summary: raw.summary || '',
       description: raw.description || '',
       requiresCgo: metadata.requiresCgo || false,
       cloudOnly: metadata.cloudOnly || false,

package/tools/redpanda-connect/rpcn-connector-docs-handler.js

@@ -399,7 +399,7 @@ function updateWhatsNew ({ dataDir, oldVersion, newVersion, binaryAnalysis }) {
       for (const comp of regularComponents) {
         const typeLabel = comp.type.charAt(0).toUpperCase() + comp.type.slice(1)
         const statusLabel = comp.status || '-'
-        let desc = comp.summary || (comp.description ? capToTwoSentences(comp.description) : '// TODO: Add description')
+        let desc = comp.summary ? capToTwoSentences(comp.summary) : (comp.description ? capToTwoSentences(comp.description) : '// TODO: Add description')
 
         if (comp.requiresCgo) {
           const cgoNote = '\nNOTE: Requires a cgo-enabled binary. See the xref:install:index.adoc[installation guides] for details.'
@@ -1243,8 +1243,10 @@ async function handleRpcnConnectorDocs (options) {
   // Main Processing: Handle the latest version (final iteration)
   // ========================================================================
 
-  console.log('Generating connector partials...')
-  let partialsWritten, partialFiles
+  // options.partials is false with --no-partials: the connect repo publishes
+  // the generated partials, so this run only updates release-level content.
+  const writePartials = options.partials !== false
+  let partialsWritten = 0, partialFiles = []
   const descriptionReports = []
   const lostSectionWarnings = []
   const styleRegressionWarnings = []
@@ -1256,7 +1258,10 @@ async function handleRpcnConnectorDocs (options) {
   const { collectGeneratorReports } = require('./pr-summary-formatter.js')
   const collect = (result) => collectGeneratorReports(result, { descriptionReports, lostSectionWarnings, styleRegressionWarnings })
 
-  try {
+  if (!writePartials) {
+    console.log('Skipping connector partials (--no-partials): the connect repo provides them.')
+  } else try {
+    console.log('Generating connector partials...')
     const result = await generateRpcnConnectorDocs({
       data: dataFile,
       overrides: options.overrides,
@@ -1511,20 +1516,20 @@ async function handleRpcnConnectorDocs (options) {
     }
   }
 
-  // Publish merged version to attachments
+  // Publish the connector data to attachments. The Bloblang playground and
+  // other site tools read this file, so it is published on every run, with
+  // overrides merged in only when an overrides file still exists.
   // IMPORTANT: This must run AFTER binary analysis and augmentation to include CGO-only connectors
-  if (options.overrides && fs.existsSync(options.overrides)) {
+  {
     try {
-      const { mergeOverrides, resolveReferences } = require('./generate-rpcn-connector-docs.js')
-
       // Use the augmented newIndex which now includes CGO-only and cloud-only connectors
       const mergedData = JSON.parse(JSON.stringify(newIndex))
 
-      const ovRaw = fs.readFileSync(options.overrides, 'utf8')
-      const ovObj = JSON.parse(ovRaw)
-      const resolvedOverrides = resolveReferences(ovObj, ovObj)
-
-      mergeOverrides(mergedData, resolvedOverrides)
+      if (options.overrides && fs.existsSync(options.overrides)) {
+        const { mergeOverrides, resolveReferences } = require('./generate-rpcn-connector-docs.js')
+        const ovObj = JSON.parse(fs.readFileSync(options.overrides, 'utf8'))
+        mergeOverrides(mergedData, resolveReferences(ovObj, ovObj))
+      }
 
       const attachmentsRoot = path.resolve(process.cwd(), 'modules/components/attachments')
       fs.mkdirSync(attachmentsRoot, { recursive: true })
@@ -2066,6 +2071,7 @@ async function handleRpcnConnectorDocs (options) {
           templateDescription: options.templateDescription,
           templateIntro: options.templateIntro,
           writeFullDrafts: true,
+          writePartials,
           cgoOnly: binaryAnalysis?.cgoOnly || [],
           cloudOnly: binaryAnalysis?.comparison?.cloudOnly || [],
           csvMetadata

package/tools/redpanda-connect/templates/connector.hbs

@@ -1,8 +1,14 @@
 = {{{name}}}
 // tag::single-source[]
-:type: {{type}}
-{{#if support}}:support: {{support}}{{/if}}
+{{!-- Drafts are written once and never regenerated, so they hold no
+     reference facts that change with the source. The type comes from the
+     page directory, the support level and Cloud availability from the
+     catalog data, and, when the connect repo publishes the partials
+     (connectPartials), the categories from its catalog.json and the status
+     and cgo flags from the description partial's meta region. --}}
+{{#unless connectPartials}}
 :categories: [{{join categories ", "}}]
+{{/unless}}
 {{#if hasDescriptionMeta}}
 // tag::meta[]
 {{{descriptionInclude this "meta"}}}
@@ -27,6 +33,10 @@
      (cloud-docs .github/description-allowlist.txt). --}}
 
 component_type_dropdown::[]
+{{#if connectPartials}}
+
+include::connect:components:partial$availability/{{typeDir}}/{{name}}.adoc[opts=optional]
+{{/if}}
 
 {{#if hasDescriptionPartial}}
 {{{descriptionInclude this "body"}}}
@@ -58,6 +68,7 @@ include::components:example$advanced/{{typeDir}}/{{name}}.yaml[]
 ======
 {{/if}}
 
+{{#unless connectPartials}}
 {{#if requiresCgo}}
 == Requirements
 
@@ -71,6 +82,7 @@ You can either xref:install:prebuilt-binary.adoc[download a prebuilt cgo-enabled
 {{/if}}
 
 {{/if}}
+{{/unless}}
 {{#if (or this.config.children children)}}
 include::connect:components:partial$fields/{{typeDir}}/{{name}}.adoc[]
 {{/if}}
@@ -78,6 +90,12 @@ include::connect:components:partial$fields/{{typeDir}}/{{name}}.adoc[]
 {{#if this.examples}}
 include::connect:components:partial$examples/{{typeDir}}/{{name}}.adoc[]
 {{/if}}
+{{#if connectPartials}}
+{{#if hasDescriptionPartial}}
+
+{{{descriptionInclude this "footnotes"}}}
+{{/if}}
+{{/if}}
 
 
 // end::single-source[]

package/tools/rpk-docs/helpers/index.js

@@ -305,9 +305,9 @@ function isPluginCommand(commandPath, pluginVersions) {
 }
 
 /**
- * Convert ALL CAPS section name to title case for display
+ * Convert ALL CAPS section name to sentence case for display
  * @param {string} name - ALL CAPS section name
- * @returns {string} Title case name
+ * @returns {string} Sentence case name
  */
 function sectionTitle(name) {
   if (!name) return ''
@@ -315,20 +315,43 @@ function sectionTitle(name) {
   // Acronyms that should remain ALL CAPS
   const acronyms = new Set([
     'id', 'ip', 'api', 'url', 'uri', 'cpu', 'gpu', 'ram', 'io',
-    'tls', 'ssl', 'mtls', 'sasl', 'oauth', 'oidc', 'jwt',
+    'tls', 'ssl', 'mtls', 'sasl', 'sso', 'oauth', 'oidc', 'jwt',
     'json', 'yaml', 'xml', 'csv', 'http', 'https', 'grpc', 'rpc',
-    'aws', 'gcp', 'azure', 's3', 'eos'
+    'aws', 'gcp', 's3', 'eos'
   ])
 
-  // Product names that should be title cased
+  // Proper nouns that keep their initial capital
   const properNouns = new Set([
-    'redpanda', 'kafka', 'kubernetes', 'schema', 'registry'
+    'redpanda', 'kafka', 'kubernetes', 'azure'
   ])
 
+  // Multi-word proper nouns, capitalized only as the full phrase
+  // ("SCHEMA REGISTRY" -> "Schema Registry", but "SCHEMA FORMAT" -> "Schema format")
+  const properPhrases = [['schema', 'registry']]
+
+  const capitalize = word => word.charAt(0).toUpperCase() + word.slice(1)
+
   // Split on spaces/hyphens but keep & as its own token
   // "PRODUCER ID & EPOCH" -> ["producer", "id", "&", "epoch"]
   const words = name.toLowerCase().split(/[\s-]+/)
 
+  const inPhrase = new Set()
+  for (const phrase of properPhrases) {
+    for (let i = 0; i + phrase.length <= words.length; i++) {
+      if (phrase.every((part, j) => words[i + j] === part)) {
+        phrase.forEach((_, j) => inPhrase.add(i + j))
+      }
+    }
+  }
+
+  // Sentence case: capitalize the first word, acronyms, and proper nouns;
+  // lowercase everything else
+  const formatPart = (part, isFirst, index) => {
+    if (acronyms.has(part)) return part.toUpperCase()
+    if (isFirst || properNouns.has(part) || inPhrase.has(index)) return capitalize(part)
+    return part
+  }
+
   return words.map((word, index) => {
     // Preserve ampersand as-is
     if (word === '&') {
@@ -337,38 +360,12 @@ function sectionTitle(name) {
 
     // Handle words with slashes (e.g., "enabled/disabled")
     if (word.includes('/')) {
-      const parts = word.split('/')
-      return parts.map((part, partIndex) => {
-        if (acronyms.has(part)) {
-          return part.toUpperCase()
-        }
-        if (index === 0 && partIndex === 0) {
-          return part.charAt(0).toUpperCase() + part.slice(1)
-        }
-        if (properNouns.has(part)) {
-          return part.charAt(0).toUpperCase() + part.slice(1)
-        }
-        return part
-      }).join('/')
-    }
-
-    // Acronyms: ALL CAPS
-    if (acronyms.has(word)) {
-      return word.toUpperCase()
-    }
-
-    // First word: always capitalize
-    if (index === 0) {
-      return word.charAt(0).toUpperCase() + word.slice(1)
-    }
-
-    // Proper nouns: capitalize
-    if (properNouns.has(word)) {
-      return word.charAt(0).toUpperCase() + word.slice(1)
+      return word.split('/')
+        .map((part, partIndex) => formatPart(part, index === 0 && partIndex === 0, index))
+        .join('/')
     }
 
-    // All other words in title: capitalize (standard title case)
-    return word.charAt(0).toUpperCase() + word.slice(1)
+    return formatPart(word, index === 0, index)
   }).join(' ')
 }