@redpanda-data/docs-extensions-and-macros 5.54.0 → 5.56.0

package/extensions/convert-to-markdown.js

@@ -8,6 +8,98 @@ const TurndownService = require('turndown')
 const turndownPluginGfm = require('turndown-plugin-gfm')
 const { gfm } = turndownPluginGfm
 
+// Every doc-tools generator writes one comment line near the top of each file
+// it produces, in this shape:
+//
+//   // This content is autogenerated. Do not edit manually. Generated by doc-tools (<generator>) from <source>. <how to change it>
+//
+// (cli-utils/generated-marker.js builds it.) Some files written before the
+// version was dropped carry one after "doc-tools". Older files carry only the
+// first two sentences. The twin's front matter reports what the marker says, so a
+// reader (or an agent) can tell generated text from hand-written text on the
+// same page and knows where the fix goes. The comment must open with the
+// generated phrase, which every marker in the docs repos does ("This content
+// is autogenerated", "AUTO-GENERATED FILE", "Generated by `doc-tools ...`"),
+// so a prose comment such as "// Table generated by the build" is not one.
+// "Generated by" counts only when doc-tools follows, so a hand-written
+// "// Generated by running rpk cluster info, then edited" is not one either.
+const GENERATED_MARKER_SCAN_LINES = 15
+const GENERATED_MARKER_RE = /^\/\/\s*((?:this content is auto-?generated\b|auto-?generated\b|generated by\s+`?doc-tools\b).*)$/i
+// "doc-tools (property-docs)", or with a version token: "doc-tools 5.55.0
+// (rpk-docs)". A token that is not a version (the Python helpers once wrote
+// "unknown") still yields the generator.
+const MARKER_TOOL_RE = /doc-tools(?:\s+([^\s()]+))?\s*\(([^)]+)\)/i
+// The source is named only right after the tool, or after "autogenerated"
+// (connect's own banner), so a "from" later in a legacy banner's prose is not
+// read as one.
+const MARKER_SOURCE_RE = /(?:doc-tools(?:\s+[^\s()]+)?\s*\([^)]+\)|auto-?generated)\s+from\s+(.+?)(?:\.(?=\s|$)|$)/i
+const INCLUDE_TARGET_RE = /^include::([^[\s{]+)\[/gm
+
+/**
+ * Read the generated-content marker from the first lines of an AsciiDoc file.
+ * @param {Buffer|string} contents - The file's AsciiDoc source
+ * @returns {{note: string, generator: (string|null), version: (string|null), source: (string|null)}|null}
+ */
+function readGeneratedMarker (contents) {
+  if (!contents) return null
+  const text = Buffer.isBuffer(contents) ? contents.toString('utf8') : String(contents)
+  const lines = text.split(/\r?\n/, GENERATED_MARKER_SCAN_LINES)
+  for (const line of lines) {
+    const m = line.match(GENERATED_MARKER_RE)
+    if (!m) continue
+    const note = m[1].trim()
+    const tool = note.match(MARKER_TOOL_RE)
+    const generator = tool ? tool[2].trim() : null
+    const token = tool && tool[1] ? tool[1].replace(/^v(?=\d)/i, '') : null
+    const version = token && /^\d/.test(token) ? token : null
+    const source = (note.match(MARKER_SOURCE_RE) || [])[1] || null
+    return { note, generator, version, source: source ? source.trim() : null }
+  }
+  return null
+}
+
+/**
+ * List the include targets in an AsciiDoc source. Targets that carry an
+ * attribute reference cannot be resolved here and are skipped.
+ * @param {Buffer|string} contents - The file's AsciiDoc source
+ * @returns {string[]} Unique include targets, in order of first appearance
+ */
+function findIncludeTargets (contents) {
+  if (!contents) return []
+  const text = Buffer.isBuffer(contents) ? contents.toString('utf8') : String(contents)
+  const targets = []
+  for (const m of text.matchAll(INCLUDE_TARGET_RE)) {
+    if (!targets.includes(m[1])) targets.push(m[1])
+  }
+  return targets
+}
+
+/**
+ * Work out which text on a page is generated: the page itself, and any
+ * include whose file carries the marker. Runs while the catalog still holds
+ * AsciiDoc, before conversion.
+ * @param {Object} page - Antora page file (src, contents)
+ * @param {Object} contentCatalog - Antora content catalog
+ * @returns {{page: (Object|null), includes: Object[]}}
+ */
+function collectGeneratedProvenance (page, contentCatalog) {
+  const own = readGeneratedMarker(page.contents)
+  const includes = []
+  for (const target of findIncludeTargets(page.contents)) {
+    let resource
+    try {
+      resource = contentCatalog.resolveResource(target, page.src, 'partial', ['partial', 'page', 'example'])
+    } catch (err) {
+      resource = null
+    }
+    if (!resource || !resource.contents) continue
+    const marker = readGeneratedMarker(resource.contents)
+    if (!marker) continue
+    includes.push({ include: target, generator: marker.generator, version: marker.version, source: marker.source, note: marker.note })
+  }
+  return { page: own, includes }
+}
+
 /**
  * Decode HTML entities in a string
  * @param {string} str - String with potential HTML entities
@@ -70,6 +162,17 @@ function generateFrontmatter(page) {
     'page-eol-date',
     'page-git-created-date',
     'page-git-modified-date',
+    // Generated-content provenance, from the markers generators write
+    'page-generated-content',
+    'page-autogenerated',
+    'page-generated-by',
+    'page-generated-from',
+    'page-generated-sources',
+    // Release that introduced the page's feature, and the product that
+    // release belongs to. Generated Connect reference pages set them on
+    // self-managed pages; the UI shows them as a "Since" badge.
+    'page-introduced-in',
+    'page-introduced-in-product',
     // Component-specific version attributes (from antora.yml)
     'latest-redpanda-tag', // Redpanda docker tag (e.g., v25.3.5)
     'latest-console-tag',
@@ -107,6 +210,29 @@ function generateFrontmatter(page) {
       return
     }
 
+    // Provenance travels as a JSON string in the attribute; the twin wants a list
+    if (key === 'page-generated-sources') {
+      try {
+        frontmatter[key] = JSON.parse(value)
+      } catch (err) {
+        frontmatter[key] = value
+      }
+      return
+    }
+    if (key === 'page-autogenerated') {
+      frontmatter[key] = value === '' || value === true || value === 'true'
+      return
+    }
+    if (key === 'page-introduced-in' || key === 'page-introduced-in-product') {
+      // Not boolean flags: an empty value (an unset attribute in a template)
+      // means there is nothing to record.
+      const text = String(value).trim()
+      if (!text) return
+      // A bare X.Y.Z, as the UI badge shows it.
+      frontmatter[key] = key === 'page-introduced-in' ? text.replace(/^v/i, '').replace(/\.+$/, '') : text
+      return
+    }
+
     // Skip empty attributes (AsciiDoc boolean flags)
     if (value === '') {
       // Special handling for version fields - use actual version from page source
@@ -294,8 +420,12 @@ function createEnterpriseFeatureRules () {
 }
 
 module.exports.createEnterpriseFeatureRules = createEnterpriseFeatureRules
+module.exports.generateFrontmatter = generateFrontmatter
 module.exports.formatStatusMarker = formatStatusMarker
 module.exports.ENTERPRISE_MARKER = ENTERPRISE_MARKER
+module.exports.readGeneratedMarker = readGeneratedMarker
+module.exports.findIncludeTargets = findIncludeTargets
+module.exports.collectGeneratedProvenance = collectGeneratedProvenance
 
 module.exports.register = function () {
   raiseListenerLimit(this)
@@ -556,6 +686,27 @@ module.exports.register = function () {
     return outerTurndown
   }
 
+  // Record which text on each page is generated, while the catalog still
+  // holds AsciiDoc: the page's own marker, and the marker on every include it
+  // resolves to. An extension that generates pages at build time sets
+  // page.generatedProvenance itself, and that wins.
+  this.on('contentClassified', ({ contentCatalog }) => {
+    const pages = contentCatalog.findBy({ family: 'page' })
+    let marked = 0
+    for (const page of pages) {
+      if (page.generatedProvenance) {
+        marked++
+        continue
+      }
+      const provenance = collectGeneratedProvenance(page, contentCatalog)
+      if (provenance.page || provenance.includes.length) {
+        page.generatedProvenance = provenance
+        marked++
+      }
+    }
+    logger.info(`Recorded generated-content provenance on ${marked} of ${pages.length} pages`)
+  })
+
   // Add marker attribute before UI rendering so templates can detect markdown availability
   this.on('documentsConverted', ({ contentCatalog }) => {
     const pages = contentCatalog.findBy({ family: 'page' })
@@ -568,6 +719,31 @@ module.exports.register = function () {
 
       // Add marker that UI templates can check
       page.asciidoc.attributes['page-has-markdown'] = ''
+
+      // Provenance as page attributes, so the twin's front matter and the UI
+      // templates can both read it. page-generated-content is set on every
+      // page, so "none" is a statement and not an absence.
+      const provenance = page.generatedProvenance
+      let generatedContent = 'none'
+      if (provenance?.page) {
+        generatedContent = 'all'
+        page.asciidoc.attributes['page-autogenerated'] = 'true'
+        const { generator, version, source } = provenance.page
+        if (generator) {
+          page.asciidoc.attributes['page-generated-by'] = `doc-tools${version ? ` ${version}` : ''} (${generator})`
+        }
+        // Only a named source: a legacy banner has none, and its prose is
+        // not one.
+        if (source) page.asciidoc.attributes['page-generated-from'] = source
+      } else if (provenance?.includes?.length) {
+        generatedContent = 'partial'
+      }
+      if (provenance?.includes?.length) {
+        page.asciidoc.attributes['page-generated-sources'] = JSON.stringify(
+          provenance.includes.map(({ include, generator, version, source }) => ({ include, generator, version, source }))
+        )
+      }
+      page.asciidoc.attributes['page-generated-content'] = generatedContent
     })
   })
 

package/extensions/generate-fields-only-pages.js

@@ -8,6 +8,7 @@ const os = require('os')
 
 
 const handlebars = require('handlebars')
+const { generatedMarker, DOC_TOOLS_VERSION } = require('../cli-utils/generated-marker')
 
 // Import and register Redpanda Connect helpers
 const helpers = require('../tools/redpanda-connect/helpers')
@@ -269,6 +270,19 @@ module.exports.register = function ({ config }) {
           // Mark as field-only page (used by convert-to-markdown and add-llms-directive to skip directive)
           file.isFieldOnlyPage = true
 
+          // The whole page is generated here at build time, so say so the way
+          // a file marker would; convert-to-markdown exports it as
+          // page-autogenerated and page-generated-from in the twin.
+          file.generatedProvenance = {
+            page: {
+              generator: 'fields-only-pages',
+              version: DOC_TOOLS_VERSION,
+              source: 'the component spec in redpanda-data/connect',
+              note: generatedMarker('fields-only-pages', 'the component spec in redpanda-data/connect').replace(/^\/\/\s*/, ''),
+            },
+            includes: [],
+          }
+
           pagesGenerated++
         } catch (err) {
           logger.error(`Failed to generate field-only page for ${type}/${item.name}: ${err.message}`)

package/extensions/modify-connect-tag-playbook.js

@@ -75,6 +75,16 @@ const LOCAL_DIR_ENV = 'REDPANDA_CONNECT_DOCS_DIR'
 // the probe fallback tries, when the latest release has no asset yet.
 const FALLBACK_RELEASE_LIMIT = 10
 const FALLBACK_PROBE_LIMIT = 3
+// Dating pages by release (only when add-git-dates is in the playbook): the
+// page size of the releases listing, the first release that has the asset
+// (no older release can be compared), and the most older releases the walk
+// downloads per build.
+const DATE_RELEASE_PAGE_SIZE = 100
+const FIRST_ASSET_RELEASE = 'v4.113.0'
+const DATE_RELEASE_LIMIT = 10
+// How deep the walk follows includes inside the asset's own partials, such as
+// bloblang/methods.adoc including every bloblang-methods/* partial.
+const NESTED_INCLUDE_DEPTH = 3
 
 // HTTPS, SSH (ssh://), and scp-style (git@host:owner/repo) URLs are all
 // remote content sources in Antora.
@@ -284,6 +294,163 @@ async function findOlderReleaseWithAsset (latestTag, url, logger) {
   return null
 }
 
+// The resource keys (`partial$<relative>`, `example$<relative>`) of generated
+// files that differ between two releases' assets, or are new in `files`.
+function changedReferenceKeys (files, previousFiles) {
+  const before = new Map()
+  for (const f of previousFiles) {
+    const r = asset.toResource(f.path)
+    if (r && !before.has(`${r.family}$${r.relative}`)) before.set(`${r.family}$${r.relative}`, f.contents)
+  }
+  const changed = new Set()
+  const seen = new Set()
+  for (const f of files) {
+    const r = asset.toResource(f.path)
+    if (!r || r.family === 'attachment') continue
+    const key = `${r.family}$${r.relative}`
+    // addConnectDocs retains the first copy of a resource in an asset.
+    if (seen.has(key)) continue
+    seen.add(key)
+    const old = before.get(key)
+    if (old === undefined || Buffer.compare(Buffer.from(old), Buffer.from(f.contents)) !== 0) changed.add(key)
+  }
+  return changed
+}
+
+// Includes of the connect components module in a page: `partial$x` and
+// `example$x` keys. In the connect component's own components module the
+// module may be implicit; elsewhere only fully qualified includes count. With
+// a content catalog, only files this asset added count, and includes inside
+// those files are followed too (up to NESTED_INCLUDE_DEPTH levels), so a page
+// that includes an aggregate partial is dated by the partials it pulls in.
+const INCLUDE_RX = /^include::([^[\s]+)\[/gm
+function includeKey (target, src) {
+  const own = src.component === COMPONENT && src.module === asset.MODULE
+  const q = target.match(/^connect:components:((?:partial|example)\$.+)$/) ||
+    (own ? target.match(/^(?:components:)?((?:partial|example)\$.+)$/) : null)
+  return q ? q[1] : null
+}
+function connectIncludes (page, contentCatalog, origin) {
+  const keys = []
+  const seen = new Set()
+  const visit = (file, depth) => {
+    for (const m of file.contents.toString('utf8').matchAll(INCLUDE_RX)) {
+      const key = includeKey(m[1], file.src)
+      if (!key) continue
+      if (!contentCatalog) {
+        keys.push(key)
+        continue
+      }
+      const included = contentCatalog.resolveResource(m[1], file.src)
+      // Only files inserted from this asset can acquire a release date.
+      if (!included || included.src.origin !== origin || seen.has(included)) continue
+      seen.add(included)
+      keys.push(key)
+      if (depth < NESTED_INCLUDE_DEPTH) visit(included, depth + 1)
+    }
+  }
+  visit(page, 1)
+  return keys
+}
+
+// Marks pages with the newest change date of their included asset references.
+// Catalog resolution excludes retained duplicates and respects each version.
+// Cloud stubs inherit the date of the connect page they actually include.
+function markReferenceChanges (contentCatalog, dates, origin) {
+  if (!dates.size) return 0
+  const marked = new Set()
+  const pages = contentCatalog.getPages()
+  const mark = (page, date) => {
+    if (!date) return
+    if (!page.connectReferenceModified || date > page.connectReferenceModified) page.connectReferenceModified = date
+    marked.add(page)
+  }
+  for (const page of pages) {
+    for (const key of connectIncludes(page, contentCatalog, origin)) mark(page, dates.get(key))
+  }
+  for (const page of pages) {
+    for (const m of page.contents.toString('utf8').matchAll(INCLUDE_RX)) {
+      if (!/^connect:([^:$]+):([^$]+\.adoc)$/.test(m[1])) continue
+      const included = contentCatalog.resolvePage(m[1], page.src)
+      if (marked.has(included)) mark(page, included.connectReferenceModified)
+    }
+  }
+  return marked.size
+}
+
+// The date of the newest release that changed each of `keys`, as a Map of
+// key to YYYY-MM-DD. It compares the release with older releases that have
+// the asset, newest first, so a file left unchanged by the current release
+// keeps the date of the release that last changed it. Bounded so the cost
+// does not grow with every connect release: the listing stops paging at the
+// first release older than FIRST_ASSET_RELEASE, and the walk downloads at
+// most DATE_RELEASE_LIMIT older releases. A file still unchanged at the end
+// of the walk gets no date. A failed listing, download, or read only costs
+// dates: it logs a warning, and every file not dated yet keeps its git date.
+async function referenceReleaseDates (tag, files, keys, logger) {
+  const dates = new Map()
+  if (!keys.size) return dates
+  const floor = stableVersion(FIRST_ASSET_RELEASE)
+  const releases = []
+  try {
+    const github = await createGitHub()
+    for (let page = 1; ; page++) {
+      const batch = await listConnectReleases(github, OWNER, REPO, DATE_RELEASE_PAGE_SIZE, page)
+      releases.push(...batch)
+      if (batch.length < DATE_RELEASE_PAGE_SIZE) break
+      // Newest first: past the first release older than the asset, no
+      // release can have it.
+      if (batch.some((r) => r && stableVersion(r.tag_name) && compareVersions(stableVersion(r.tag_name), floor) < 0)) break
+    }
+  } catch (error) {
+    logger.warn(`Couldn't list Redpanda Connect releases (${error.message}), so Connect pages keep their git modified dates`)
+    return dates
+  }
+  let newer = releases.find((r) => r && r.tag_name === tag)
+  if (!newer || !newer.published_at) return dates
+  const latest = stableVersion(tag)
+  const older = releases
+    .filter((r) => r && !r.draft && !r.prerelease && stableVersion(r.tag_name) && latest && compareVersions(stableVersion(r.tag_name), latest) < 0)
+    .filter((r) => compareVersions(stableVersion(r.tag_name), floor) >= 0)
+    .filter((r) => (r.assets || []).some((a) => a && a.name === asset.ASSET_NAME))
+    .sort((a, b) => compareVersions(stableVersion(b.tag_name), stableVersion(a.tag_name)))
+    .slice(0, DATE_RELEASE_LIMIT)
+  const pending = new Set(keys)
+  for (const release of older) {
+    let previousFiles
+    try {
+      const archive = await downloadRelease(release.tag_name, logger)
+      if (!archive) throw new Error(`the release lists a ${asset.ASSET_NAME} asset but its download returns 404`)
+      previousFiles = await asset.readTarGz(archive)
+    } catch (error) {
+      // Skipping the release would credit its changes to the newer release,
+      // so stop here: dates found so far are exact, the rest keep git dates.
+      logger.warn(`Couldn't read the Redpanda Connect ${release.tag_name} reference docs to date pages (${error.message}), so ${pending.size} generated files keep their pages' git modified dates`)
+      break
+    }
+    const changed = changedReferenceKeys(files, previousFiles)
+    for (const key of pending) {
+      if (!changed.has(key)) continue
+      if (newer.published_at) dates.set(key, String(newer.published_at).slice(0, 10))
+      pending.delete(key)
+    }
+    if (!pending.size) break
+    files = previousFiles
+    newer = release
+  }
+  return dates
+}
+
+// True when the playbook registers add-git-dates, the only reader of the
+// release dates. Without it, the dating step is skipped.
+function usesGitDates (playbook) {
+  const extensions = (playbook && playbook.antora && playbook.antora.extensions) || []
+  return [].concat(extensions).some((ext) => {
+    const id = typeof ext === 'string' ? ext : ext && ext.enabled !== false ? ext.require || ext.id : null
+    return /(^|[/\\])add-git-dates(\.js)?$/.test(String(id || ''))
+  })
+}
+
 // Adds the generated files to every version of the connect component.
 // Mutates the content catalog and returns what it added, replaced, and
 // skipped. A file another source already provides is skipped, unless
@@ -332,6 +499,8 @@ module.exports.register = function ({ config }) {
   // Set in contextStarted: true when the playbook still lists a connect
   // content source with explicit refs.
   let gitSourceMode = false
+  // Set in contextStarted: true when add-git-dates is in the playbook.
+  let dateReferencePages = false
 
   this.on('contextStarted', ({ playbook }) => {
     // Clear a ref left over from an earlier build in the same process.
@@ -345,6 +514,7 @@ module.exports.register = function ({ config }) {
       )
     }
     gitSourceMode = hasConnectSource(playbook)
+    dateReferencePages = usesGitDates(playbook)
     if (gitSourceMode && (process.env[LOCAL_DIR_ENV] || '').trim()) {
       logger.warn(`${LOCAL_DIR_ENV} is ignored because the playbook lists the connect repo as a content source`)
     }
@@ -422,7 +592,7 @@ module.exports.register = function ({ config }) {
       from = `the ${tag} ${asset.ASSET_NAME} release asset`
     }
     if (!files.some((f) => asset.toResource(f.path))) {
-      throw new Error(`${from} has no files under modules/components/partials or modules/components/examples`)
+      throw new Error(`${from} has no files under modules/components/partials, modules/components/examples, or modules/components/attachments`)
     }
     const r = addConnectDocs(contentCatalog, files, origin, { replace: !!localDir })
     const versions = r.versions.filter(Boolean)
@@ -432,9 +602,17 @@ module.exports.register = function ({ config }) {
       (localDir
         ? `replaced ${r.replaced} provided by another source; skipped ${r.outsideGenerated}`
         : `skipped ${r.providedElsewhere} already provided by another source and ${r.outsideGenerated}`) +
-      ' outside the generated partials and examples'
+      ' outside the generated partials, examples, and attachments'
     )
+    // Resolve only included files supplied by this asset. Retained copies
+    // from another source keep their git dates, even if the asset differs.
+    if (!localDir && dateReferencePages) {
+      const keys = new Set(contentCatalog.getPages().flatMap((page) => connectIncludes(page, contentCatalog, origin)))
+      const dates = await referenceReleaseDates(origin.tag, files, keys, logger)
+      const count = markReferenceChanges(contentCatalog, dates, origin)
+      logger.info(`Redpanda Connect ${origin.tag}: resolved last change dates for ${dates.size} included generated files; ${count} pages use those release dates when later than their last commit`)
+    }
   })
 }
 
-module.exports._internal = { isRemote, isUpstreamConnect, redact, isConnectSource, isConnectOrigin, toTag, highestStableTag, stableTags, findOlderReleaseWithAsset, resolveLatestTag, FALLBACK_RELEASE_LIMIT, FALLBACK_PROBE_LIMIT, wantsLatest, removeLatestConnectSources, filterConnectContent, hasConnectSource, addConnectDocs, LOCAL_DIR_ENV }
+module.exports._internal = { isRemote, isUpstreamConnect, redact, isConnectSource, isConnectOrigin, toTag, highestStableTag, stableTags, findOlderReleaseWithAsset, resolveLatestTag, FALLBACK_RELEASE_LIMIT, FALLBACK_PROBE_LIMIT, wantsLatest, removeLatestConnectSources, filterConnectContent, hasConnectSource, addConnectDocs, LOCAL_DIR_ENV, changedReferenceKeys, connectIncludes, markReferenceChanges, usesGitDates, FIRST_ASSET_RELEASE, DATE_RELEASE_LIMIT }

package/extensions/resolve-xrefs-in-attachments.js

@@ -1,6 +1,7 @@
 'use strict'
 
 const { raiseListenerLimit } = require('./util/raise-listener-limit')
+const { escapeHtml } = require('../extension-utils/html-utils')
 
 /**
  * Resolves AsciiDoc xrefs in JSON attachment files to HTML links.
@@ -240,15 +241,3 @@ function resolveXrefsInString (text, contentCatalog, context, logger) {
     }
   })
 }
-
-/**
- * Escape HTML special characters to prevent XSS.
- */
-function escapeHtml (text) {
-  if (!text) return ''
-  return String(text)
-    .replace(/&/g, '&')
-    .replace(/</g, '&lt;')
-    .replace(/>/g, '&gt;')
-    .replace(/"/g, '&quot;')
-}