@redpanda-data/docs-extensions-and-macros 5.12.4 → 5.13.1

package/tools/redpanda-connect/rpcn-connector-docs-handler.js

@@ -892,6 +892,15 @@ async function handleRpcnConnectorDocs (options) {
         fs.closeSync(fd)
 
         const rawJson = fs.readFileSync(tmpFile, 'utf8')
+        // Deliberately plain JSON.parse/stringify, not bigIntJson: this is connect
+        // data, which carries example config text as escaped JSON-shaped strings
+        // (e.g. bloblang examples), indistinguishable by regex from real JSON
+        // value position. bigIntJson's value-position regex matches inside such a
+        // string and inserts stray quotes -- confirmed to throw on the real,
+        // currently-published connect attachment, not merely round a number. A
+        // plain round trip still silently rounds int64 defaults past MaxInt64 (the
+        // pre-existing defect this file has always had), which is the safer
+        // failure mode until a real JSON scanner replaces the regex.
         const parsed = JSON.parse(rawJson)
         fs.writeFileSync(finalFile, JSON.stringify(parsed, null, 2))
         fs.unlinkSync(tmpFile)
@@ -1225,7 +1234,15 @@ async function handleRpcnConnectorDocs (options) {
 
   console.log('Generating connector partials...')
   let partialsWritten, partialFiles
+  const descriptionReports = []
   const lostSectionWarnings = []
+  // Both generator call sites (partials and drafts) feed the same PR summary,
+  // so they collect through one function. Pushed inline, the draft call site
+  // silently dropped descriptionReports for months: the structure reports for
+  // newly drafted connectors, the ones most likely to need an upstream fix,
+  // never reached the summary.
+  const { collectGeneratorReports } = require('./pr-summary-formatter.js')
+  const collect = (result) => collectGeneratorReports(result, { descriptionReports, lostSectionWarnings })
 
   try {
     const result = await generateRpcnConnectorDocs({
@@ -1236,14 +1253,16 @@ async function handleRpcnConnectorDocs (options) {
       templateFields: options.templateFields,
       templateExamples: options.templateExamples,
       templateMetadata: options.templateMetadata,
+      templateDescription: options.templateDescription,
       templateBloblang: options.templateBloblang,
       writeFullDrafts: false,
+      pruneOrphanedDescriptions: !!options.pruneOrphanedDescriptions,
       includeBloblang: !!options.includeBloblang,
       csvMetadata
     })
     partialsWritten = result.partialsWritten
     partialFiles = result.partialFiles
-    lostSectionWarnings.push(...(result.lostSectionWarnings || []))
+    collect(result)
   } catch (err) {
     console.error(`Error: Failed to generate partials: ${err.message}`)
     process.exit(1)
@@ -1874,6 +1893,12 @@ async function handleRpcnConnectorDocs (options) {
 
           // Check each cloud-supported connector
           // Filter to only check actual connector/component types that need individual pages
+          // NOT the shared CONNECTOR_TYPE_DIRS: this list deliberately omits
+          // rate_limits, per the comment below. rate_limits does have
+          // per-component pages, so the omission looks like a bug and rate
+          // limits are never checked for a cloud-docs stub -- but changing it
+          // adds cloud-docs findings in a path with no test coverage, so it
+          // needs its own ticket rather than riding along here.
           const connectorTypes = ['inputs', 'outputs', 'processors', 'caches', 'buffers', 'scanners', 'metrics', 'tracers']
 
           for (const connectorKey of cloudSupportedSet) {
@@ -2026,6 +2051,7 @@ async function handleRpcnConnectorDocs (options) {
           templateFields: options.templateFields,
           templateExamples: options.templateExamples,
           templateMetadata: options.templateMetadata,
+          templateDescription: options.templateDescription,
           templateIntro: options.templateIntro,
           writeFullDrafts: true,
           cgoOnly: binaryAnalysis?.cgoOnly || [],
@@ -2036,7 +2062,7 @@ async function handleRpcnConnectorDocs (options) {
         fs.unlinkSync(tempDataPath)
         draftsWritten = draftResult.draftsWritten
         draftFiles = draftResult.draftFiles
-        lostSectionWarnings.push(...(draftResult.lostSectionWarnings || []))
+        collect(draftResult)
       }
     } catch (err) {
       console.error(`Error: Could not draft missing: ${err.message}`)
@@ -2083,13 +2109,26 @@ async function handleRpcnConnectorDocs (options) {
 
   // Generate PR summary
   try {
-    const { printPRSummary } = require('./pr-summary-formatter.js')
+    const { printPRSummary, renderLostSectionWarnings, renderDescriptionReports } = require('./pr-summary-formatter.js')
     // Use master diff if available, otherwise use single diff. Content-loss
-    // warnings ride the diff object so they lead the PR summary body instead
-    // of dying in the collapsed workflow log.
+    // warnings and structure reports ride the diff object so they land in the
+    // PR summary body instead of dying in the collapsed workflow log.
     const summaryDiff = masterDiff || diffJson
-    if (lostSectionWarnings.length) summaryDiff.lostSectionWarnings = lostSectionWarnings
-    printPRSummary(summaryDiff, binaryAnalysis, draftFiles, masterDiff ? true : false)
+    if (summaryDiff) {
+      if (lostSectionWarnings.length) summaryDiff.lostSectionWarnings = lostSectionWarnings
+      if (descriptionReports.length) summaryDiff.descriptionReports = descriptionReports
+      printPRSummary(summaryDiff, binaryAnalysis, draftFiles, masterDiff ? true : false)
+    } else if (lostSectionWarnings.length || descriptionReports.length) {
+      // No diff to summarize (no prior version, or versions match), but this
+      // run still produced content-loss warnings or description-structure
+      // reports that a reviewer needs to see -- print them on their own
+      // instead of losing them because there was nothing to diff against.
+      const lines = [
+        ...renderLostSectionWarnings(lostSectionWarnings),
+        ...renderDescriptionReports(descriptionReports)
+      ]
+      console.log('\n' + lines.join('\n') + '\n')
+    }
   } catch (err) {
     console.error(`Warning: Failed to generate PR summary: ${err.message}`)
   }

package/tools/redpanda-connect/templates/connector.hbs

@@ -3,23 +3,36 @@
 :type: {{type}}
 {{#if support}}:support: {{support}}{{/if}}
 :categories: [{{join categories ", "}}]
-{{#if summary}}
+{{#if hasDescriptionAttrs}}
 // tag::meta[]
-:description: {{{summary}}}
+{{{descriptionInclude this "attrs"}}}
+// end::meta[]
+{{else if summary}}
+// tag::meta[]
+:description: {{{summaryAttribute summary}}}
 // end::meta[]
 {{else if description}}
 // tag::meta[]
-:description: {{description}}
+:description: {{{summaryAttribute description}}}
 // end::meta[]
 {{/if}}
-{{!-- No fallback line when neither exists: an emitted TODO would become
+{{!-- The attrs include stays inside the meta tag region so single-source
+     stubs keep inheriting :description: as a header attribute, and the
+     value now refreshes with the connector instead of being frozen at
+     first draft. Component families with no description partial keep the
+     inline attribute.
+     No fallback line when neither exists: an emitted TODO would become
      the attribute VALUE and stubs would inherit it into their meta tags.
      Missing descriptions are the tracked editorial backlog
      (cloud-docs .github/description-allowlist.txt). --}}
 
 component_type_dropdown::[]
 
+{{#if hasDescriptionPartial}}
+{{{descriptionInclude this "body"}}}
+{{else}}
 {{> intro}}
+{{/if}}
 
 {{#if (or this.config.children children)}}
 [tabs]

package/tools/redpanda-connect/templates/descriptions-partials.hbs

@@ -0,0 +1,21 @@
+{{#if (or summary description version)}}
+// This content is autogenerated. Do not edit manually. The summary and description come from the connector's definition in the Connect source. To change them, update the connector upstream, or use the doc-tools CLI with the --overrides option: https://redpandadata.atlassian.net/wiki/spaces/DOC/pages/1247543314/Generate+reference+docs+for+Redpanda+Connect
+//
+// Pages consume this partial in two places so both stay fresh:
+//   header: include::...[tag=attrs]  (sets :description: for search snippets and meta tags)
+//   body:   include::...[tag=body]   (renders the summary and description)
+
+// tag::attrs[]
+{{#if summary}}:description: {{{summaryAttribute summary}}}
+{{/if}}// end::attrs[]
+
+// tag::body[]
+{{#if summary}}{{{summary}}}
+
+{{/if}}{{#if version}}ifndef::env-cloud[]
+Introduced in version {{version}}.
+endif::[]
+
+{{/if}}{{{renderConnectDescription this}}}
+// end::body[]
+{{/if}}

package/tools/redpanda-connect/templates/intro.hbs

@@ -7,7 +7,7 @@ endif::[]
 {{/if}}
 
 {{#if description}}
-{{{descriptionWithMetadataInclude this}}}
+{{{renderConnectDescription this}}}
 {{else}}
 // TODO: Missing description. Add a description in the overrides file or source data.
 {{/if}}