@redpanda-data/docs-extensions-and-macros 5.53.0 → 5.54.0

package/bin/doc-tools.js

@@ -2930,6 +2930,133 @@ programCli
     runCli(options)
   })
 
+/**
+ * check-build-log
+ *
+ * @description
+ * Check an Antora JSON log (`antora --log-format json`): exit 1 when any
+ * record is at level error or fatal, and, with --min-pages and --site-dir,
+ * when fewer HTML pages were written than expected. Prints a Markdown
+ * summary (counts by level, then the error and warning messages grouped and
+ * deduplicated, with file and line) for $GITHUB_STEP_SUMMARY. Warnings never
+ * fail the check. With --blocking-sources, only errors from those
+ * repositories (or with no source) fail it, so a broken page in another
+ * content source of the playbook is reported without blocking.
+ *
+ * @why
+ * Antora's --log-failure-level fails the build but explains nothing in a job
+ * summary, and cannot catch a build that exits 0 having written no files,
+ * which is what Antora does on an unsupported Node version. The Connect docs
+ * PR check runs Antora with --log-failure-level=fatal so this command, not
+ * Antora, decides.
+ *
+ * @example
+ * npx antora --log-format json --log-failure-level=fatal antora-playbook.yml > build.ndjson
+ * npx doc-tools check-build-log build.ndjson --site-dir build/site/connect --min-pages 400 >> "$GITHUB_STEP_SUMMARY"
+ *
+ * # Block only on errors from these repositories; list the rest
+ * npx doc-tools check-build-log build.ndjson --blocking-sources redpanda-data/rp-connect-docs,redpanda-data/connect
+ */
+programCli
+  .command('check-build-log')
+  .description('Check an Antora JSON build log: fail on error or fatal records or too few pages, print a Markdown summary')
+  .argument('<log>', 'Antora log written with --log-format json (one JSON record per line)')
+  .option('--site-dir <dir>', 'Output directory to count HTML pages in (with --min-pages)')
+  .option('--min-pages <n>', 'Fail when fewer than <n> HTML pages were written under --site-dir')
+  .option('--blocking-sources <repos>', 'Comma-separated owner/name repositories whose errors block (default: all). Errors with no source always block; others are listed only')
+  .option('--format <format>', 'Output format: markdown or json', 'markdown')
+  .option('--output <path>', 'Also write the output to this file')
+  .action((log, options) => {
+    const { runCli } = require('../tools/connect-docs/check-build-log')
+    runCli(log, options)
+  })
+
+/**
+ * check-rendered-html
+ *
+ * @description
+ * Scan the rendered HTML of one or more Antora component paths for AsciiDoc
+ * that did not convert: literal backticks outside code, literal `xref:` or
+ * `include::` text, leftover {page-*} or {env-*} attribute references, `|===`
+ * table markup, Asciidoctor's "Unresolved include directive" text, links with
+ * the `unresolved` class, and empty sections (a heading followed directly by a
+ * heading of the same or higher level). Also checks links: a fragment with no
+ * matching id on its target page (broken-anchor), and a relative or
+ * root-relative link under a link root that is not in the build
+ * (broken-link). External links are ignored; links outside every link root,
+ * or into a root the build does not contain, are counted as unchecked. Only
+ * the page body is scanned. Reports findings per page and exits 0 unless
+ * --strict.
+ *
+ * With --changed-pages, findings are split into those on the listed pages and
+ * the rest, and --strict exits 1 only when a listed page has findings, so
+ * defects already on the base never fail a PR.
+ *
+ * @why
+ * These defects reach readers without a single line in the Antora log: two
+ * backticks for an empty enum option, or a backtick glued to the next word,
+ * render as literal backticks on the published page.
+ *
+ * @example
+ * npx doc-tools check-rendered-html build/site --component connect >> "$GITHUB_STEP_SUMMARY"
+ *
+ * # Fail on any finding
+ * npx doc-tools check-rendered-html build/site --strict
+ *
+ * # Self-managed and Cloud Connect pages; fail only on the pages a PR changed
+ * npx doc-tools check-rendered-html build/site --component connect,cloud-data-platform/develop/connect \
+ *   --changed-pages changed-pages.txt --strict
+ */
+programCli
+  .command('check-rendered-html')
+  .description('Scan rendered Antora HTML for unconverted AsciiDoc (literal backticks, xref:, include::, attributes, tables, empty sections) and broken anchors and links')
+  .argument('<site-dir>', 'Antora output directory (the --to-dir of the build)')
+  .option('--component <path>', 'Component output path to scan under <site-dir>; repeat it or give a comma list, for example connect,cloud-data-platform/develop/connect (default: connect)', (value, previous = []) => previous.concat(value))
+  .option('--link-root <path>', 'Site path whose linked pages must exist; repeat it or give a comma list. Scanned components are always included. Links outside every root, or into a root not in the build, are counted as unchecked (default: connect,cloud-data-platform/develop/connect)', (value, previous = []) => previous.concat(value))
+  .option('--pages <file>', 'Only scan the pages listed in <file>, one path relative to <site-dir> per line')
+  .option('--changed-pages <file>', 'Pages a PR changed: one path relative to <site-dir> per line (the sitePath from connect-docs-diff) or connect-docs-diff --format json output. All pages are still scanned; findings are split into changed and other pages, and --strict fails only on changed pages')
+  .option('--format <format>', 'Output format: markdown or json', 'markdown')
+  .option('--output <path>', 'Also write the output to this file')
+  .option('--strict', 'Exit 1 when any finding exists, or with --changed-pages when a changed page has findings (default: always exit 0)')
+  .action((siteDir, options) => {
+    const { runCli } = require('../tools/connect-docs/check-rendered-html')
+    runCli(siteDir, options)
+  })
+
+/**
+ * connect-docs-diff
+ *
+ * @description
+ * Compare two trees written by connect's docs generator, for example a PR's
+ * merge base and its head, and map each added, removed, or changed partial or
+ * example to the docs.redpanda.com page that includes it. Prints a Markdown
+ * table of changed pages with links, then a unified diff per file in
+ * collapsible blocks, capped in size. A tree root can be connect's docs/
+ * directory, its modules/ directory, or the components/ directory.
+ *
+ * @why
+ * Writers review Go string literals today and never see the page. The
+ * generated output is the published change: connect strings, benthos strings
+ * from a go.mod bump, and generator changes all show up here.
+ *
+ * @example
+ * npx doc-tools connect-docs-diff base/modules head/modules >> "$GITHUB_STEP_SUMMARY"
+ * npx doc-tools connect-docs-diff base/modules head/modules --format json
+ */
+programCli
+  .command('connect-docs-diff')
+  .description('Diff two generated Connect docs trees and map each changed partial or example to its published page')
+  .argument('<base-dir>', 'Generated tree of the base (merge base) commit')
+  .argument('<head-dir>', 'Generated tree of the head commit')
+  .option('--format <format>', 'Output format: markdown or json', 'markdown')
+  .option('--site-url <url>', 'Docs site the page links point at', 'https://docs.redpanda.com')
+  .option('--max-bytes <n>', 'Cap on the Markdown output size; diffs past it are counted, not shown', '60000')
+  .option('--output <path>', 'Also write the output to this file')
+  .action((baseDir, headDir, options) => {
+    const { runCli } = require('../tools/connect-docs/diff-generated')
+    runCli(baseDir, headDir, options)
+  })
+
 /**
  * preview-string
  *

package/extensions/REFERENCE.adoc

@@ -404,39 +404,75 @@ You can optionally display pages only if they match the component and version of
 
 == Redpanda Connect tag modifier
 
-This extension sources the Redpanda Connect reference partials (field reference, examples, metadata, descriptions, config snippets, and the Bloblang reference) from the `docs/` directory of the connect repository, at the latest Connect release.
+This extension adds the Redpanda Connect generated reference docs (field reference, examples, metadata, descriptions, config snippets, the Bloblang reference, and `platforms/catalog.json`) to the `connect` component that rp-connect-docs provides, from a Connect release.
 
-Add the connect repository as a content source next to rp-connect-docs:
+Every Connect release publishes the generated tree as a release asset named `redpanda-connect-docs.tar.gz`, at `\https://github.com/redpanda-data/connect/releases/download/<tag>/redpanda-connect-docs.tar.gz`. The playbook does not list the connect repository as a content source. If it still lists the upstream repository with `tags: latest`, the extension removes that source. See <<Playbooks that list the connect repository>>.
+
+At build time, when the playbook has the `connect` component, the extension:
+
+. Picks the release: the latest stable `vX.Y.Z` release by default, so the site never shows fields from unreleased code on connect's `main` branch. It asks the GitHub API for the latest release and falls back to the highest stable `vX.Y.Z` tag from `git ls-remote`. If neither works, the build stops with an error. The `tag` option picks a specific release instead.
+. Downloads that release's `redpanda-connect-docs.tar.gz` asset. With a GitHub token it downloads through the releases API, which keeps working if the connect repository becomes private. Without one, or if GitHub rejects the token, it uses the public download URL.
+. Adds `modules/components/partials/**` as partials and `modules/components/examples/**` as examples of the `components` module, in every version of the `connect` component. A file that another source, such as rp-connect-docs, already provides is skipped and counted in the log, so rp-connect-docs can keep committing its own copies until it stops.
+
+A release from before the asset existed has none, and the download returns 404. A new release can also be missing it for a while: Connect publishes the release first and attaches the asset 15 to 20 minutes later in a separate job, which can fail.
+
+So when the latest release has no asset, the extension falls back to the newest older stable `vX.Y.Z` release that has one, and logs a warning that names both, such as `Redpanda Connect v4.114.0 has no redpanda-connect-docs.tar.gz asset yet; using v4.113.0`. It reads the asset lists of the 10 most recent releases from the GitHub releases API, so it downloads only the release it picks. If the API is unavailable, it tries the downloads of the next 3 lower stable tags from `git ls-remote` instead. A release picked with the `tag` option never falls back: if it has no asset, the build stops with an error that names the release.
+
+When no release it checks has the asset, the extension logs this and adds nothing. The `generate-rp-connect-info` extension then stops the build if connector pages are missing any generated set they include: the `fields/`, `descriptions/`, or `availability/` partials, or the `common/` or `advanced/` examples. This also catches an incomplete asset, which would otherwise publish pages with unresolved includes. Any other failure, such as a network error or a corrupt archive, stops the build, because building without the reference would publish connector pages with no fields.
+
+The extension shares the release it used, including a fallback release, so `generate-rp-connect-info` reads `catalog.json` and `info.csv` for the same release as the reference docs. It adds its files before any other `contentClassified` listener runs, wherever it is listed in the playbook.
+
+=== Preview unreleased Connect changes
+
+To build with reference docs that are not released yet, generate them in a connect checkout and point `REDPANDA_CONNECT_DOCS_DIR` at the result:
+
+[,bash]
+----
+export REDPANDA_CONNECT_DOCS_DIR=/path/to/connect/docs
+npx antora --fetch local-antora-playbook.yml
+----
+
+The value is a directory that contains `modules/` (a connect checkout's `docs/` directory after running its docs generator, or the checkout root), or a local copy of a `redpanda-connect-docs.tar.gz` file. It wins over the `tag` option and the latest release, and nothing is downloaded. Its files also replace any copy another source provides, such as the copies rp-connect-docs commits, and the log counts how many were replaced. Set `tag` as well to choose the release that `generate-rp-connect-info` reads `info.csv` from.
+
+=== Playbooks that list the connect repository
+
+Before the release asset, playbooks listed the connect repository as a content source:
 
 ```yaml
 content:
   sources:
   - url: https://github.com/redpanda-data/connect
-    tags: latest # replaced with the latest release tag at build time
+    tags: latest
     start_path: docs
 ```
 
-At build time the extension:
+Connect releases no longer include `docs/antora.yml`, so Antora cannot aggregate that source. When the upstream `redpanda-data/connect` repository is listed with `tags: latest` as its only ref, the extension removes the source from the playbook before Antora fetches content, logs a warning that it is no longer needed, and uses the release asset. Remove the source from the playbook to silence the warning.
 
-* Pins a source that sets `tags: latest` to the latest Connect release tag and clears its branches (a source with any other refs, such as a PR branch for a preview, is left as configured), so the site never shows fields from unreleased code on connect's `main` branch. It asks the GitHub API for the latest release and falls back to the highest stable `vX.Y.Z` tag from `git ls-remote`. If neither works, the build stops with an error.
-* Keeps only the `modules/components/partials/` and `modules/components/examples/` files of the `connect` component from that source. Any file that another source, such as rp-connect-docs, already provides is skipped, and so is content from older Connect tags that name their component `redpanda-connect`. Antora fails the build on a duplicate page or partial, so you can add the source to a playbook before or after rp-connect-docs stops committing its own generated copies.
+A connect source with explicit refs is kept, which is how you preview a connect branch: a branch, a specific tag, a fork, or a local clone whose directory is named `connect`. In that case the extension downloads no asset, and:
 
-A local clone whose directory is named `connect` is filtered the same way but not pinned, so a local playbook can use `branches: HEAD` to test uncommitted changes.
+* Only the `modules/components/partials/` and `modules/components/examples/` files of the `connect` component are kept from that source. Any file that another source already provides is skipped, and so is content from older Connect tags that name their component `redpanda-connect`.
+* `REDPANDA_CONNECT_DOCS_DIR` and the `tag` option are ignored.
+* The ref must have `docs/antora.yml` and the generated docs. If the generated sets are missing, the `generate-rp-connect-info` error names the connect source.
 
 === Environment variables
 
-- `REDPANDA_GITHUB_TOKEN` (optional): A GitHub token for the release lookup. Without it, the extension sends unauthenticated API requests, which GitHub can rate limit. The `git ls-remote` fallback needs no token.
+- `REDPANDA_CONNECT_DOCS_DIR` (optional): A local directory, or a local `redpanda-connect-docs.tar.gz` file, to read the reference docs from instead of a release. See <<Preview unreleased Connect changes>>.
+- `REDPANDA_GITHUB_TOKEN` (optional): A GitHub token for the latest release lookup, the fallback release listing, the asset download, and the `git ls-remote` fallback. The extension also reads `ACTIONS_BOT_TOKEN`, `GITHUB_TOKEN`, `VBOT_GITHUB_API_TOKEN`, `GH_TOKEN`, and a github.com entry in `GIT_CREDENTIALS`, in that order. Without any GitHub token, the extension sends unauthenticated requests, which GitHub can rate limit, and which only work while the connect repository is public.
 
 === Configuration options
 
-There are no configurable options for this extension.
+`tag` (optional):: A Connect release tag, such as `v4.113.0`, to use instead of the latest stable release. A missing `v` prefix is added. If that release has no asset, the build stops: an explicit tag never falls back to an older release. `latest` is rejected, because the latest release is the default without `tag`. Ignored when the playbook keeps a connect content source with explicit refs.
 
 === Registration
 
+Register this extension together with `generate-rp-connect-info`:
+
 ```yaml
 antora:
   extensions:
     - require: '@redpanda-data/docs-extensions-and-macros/extensions/modify-connect-tag-playbook'
+      # tag: v4.113.0 # optional: a specific release instead of the latest
+    - require: '@redpanda-data/docs-extensions-and-macros/extensions/generate-rp-connect-info'
 ```
 
 == Version fetcher

package/extensions/generate-rp-connect-info.js

@@ -13,29 +13,81 @@ const DEFAULTS = {
   githubRepo: 'connect'
 }
 
-// Connector pages include the generated field reference from the connect repo
-// (see the modify-connect-tag-playbook extension). If a playbook has the
-// Connect pages but none of those partials, every connector page would publish
-// without its fields, and Antora only logs the missing includes. Fail the
-// build instead so the playbook gets fixed before anything is published.
-function assertConnectReferencePresent (contentCatalog) {
+// Generated sets that connector pages include, by Antora family. Each one
+// must be present, or pages publish with unresolved includes.
+// The generated partial and example directories that connector pages include,
+// read from the pages themselves, so the guard asks for exactly what the pages
+// in this build use (rp-connect-docs pages gain availability includes, for
+// example, when they move to the generated docs).
+const PARTIAL_INCLUDE = /include::connect:components:partial\$([a-z0-9_-]+)\//g
+const EXAMPLE_INCLUDE = /include::(?:connect:)?components:example\$([a-z0-9_-]+)\//g
+
+function includedDirs (pages) {
+  const partial = new Set()
+  const example = new Set()
+  for (const page of pages) {
+    const text = page.contents ? page.contents.toString() : ''
+    for (const m of text.matchAll(PARTIAL_INCLUDE)) partial.add(m[1])
+    for (const m of text.matchAll(EXAMPLE_INCLUDE)) example.add(m[1])
+  }
+  // Name the core sets first, in a fixed order, so the error reads the same
+  // way every time.
+  const order = ['fields', 'descriptions', 'availability', 'metadata', 'examples', 'common', 'advanced']
+  const rank = (d) => (order.includes(d) ? order.indexOf(d) : order.length)
+  const sorted = (set) => [...set].sort((a, b) => rank(a) - rank(b) || a.localeCompare(b))
+  return { partial: sorted(partial), example: sorted(example) }
+}
+
+// Connector pages include the generated field reference, description meta,
+// availability, and config examples, which the modify-connect-tag-playbook
+// extension adds from the redpanda-connect-docs.tar.gz asset of a connect
+// release. If a playbook has the Connect pages but any set is missing (an
+// incomplete asset, or a connect source whose ref lacks them), connector
+// pages would publish with unresolved includes, and Antora only logs them.
+// Fail the build instead so the playbook gets fixed before anything is
+// published.
+//
+// `connectSources` are the connect content sources still in the playbook,
+// which means no release asset was downloaded.
+function assertConnectReferencePresent (contentCatalog, { connectSources = [] } = {}) {
   const component = contentCatalog.getComponents().find((c) => c.name === 'connect')
   if (!component) return
-  // Only connector pages (inputs/kafka.adoc and so on) include field
-  // partials. Overview pages at the module root don't.
+  // Only connector pages (inputs/kafka.adoc and so on) include the generated
+  // sets. Overview pages at the module root don't.
   const pages = contentCatalog.findBy({ component: 'connect', module: 'components', family: 'page' })
     .filter((p) => p.src.relative.includes('/'))
   if (!pages.length) return
-  const fields = contentCatalog.findBy({ component: 'connect', module: 'components', family: 'partial' })
-    .filter((f) => f.src.relative.startsWith('fields/'))
-  if (fields.length) return
+  const missingIn = (family, dirs) => {
+    const files = contentCatalog.findBy({ component: 'connect', module: 'components', family })
+    return dirs.filter((dir) => !files.some((f) => f.src.relative.startsWith(`${dir}/`)))
+      .map((dir) => `components:${family}$${dir}/*`)
+  }
+  const wanted = includedDirs(pages)
+  const missing = [...missingIn('partial', wanted.partial), ...missingIn('example', wanted.example)]
+  if (!missing.length) return
+  const sourceHint = connectSources.length
+    ? `The playbook lists a connect content source (${connectSources.join(', ')}), so no release asset was downloaded and these files must come from that source's refs. ` +
+      'Point it at a ref that has the generated docs, or remove it to use the release asset. '
+    : ''
   throw new Error(
-    'The connect component has connector pages but no generated field partials (components:partial$fields/*). ' +
-    'Add the connect repository as a content source (url: https://github.com/redpanda-data/connect, tags: latest, start_path: docs) ' +
-    'and register the modify-connect-tag-playbook extension, or check that the latest Connect release includes docs/modules/components.'
+    `The connect component has connector pages but no generated ${missing.join(' or ')} files. ` +
+    sourceHint +
+    'These come from the redpanda-connect-docs.tar.gz asset of a Redpanda Connect release, which the ' +
+    'modify-connect-tag-playbook extension downloads. Register that extension and check its log: ' +
+    'the release it used (the latest stable release, or the one in its `tag` config) may have no asset or an incomplete one. ' +
+    'Set `tag` to a release that has the asset, or set REDPANDA_CONNECT_DOCS_DIR to a local directory that contains ' +
+    'modules/ (for example a connect checkout\'s docs/ after running its docs generator).'
   )
 }
 
+// The connect content sources in a playbook, with credentials removed.
+function connectSourcesOf (playbook) {
+  const sources = (playbook && playbook.content && playbook.content.sources) || []
+  return sources
+    .filter((s) => catalogUtil.isConnectSource(s.url))
+    .map((s) => String(s.url).replace(/([a-z][a-z+.-]*:\/\/)[^@/\s]*@/gi, '$1'))
+}
+
 module.exports.assertConnectReferencePresent = assertConnectReferencePresent
 
 module.exports.register = function ({ config }) {
@@ -63,8 +115,8 @@ module.exports.register = function ({ config }) {
   let translatedRows = null
 
   // Use 'on' and return the promise so Antora waits for async completion
-  this.on('contentClassified', ({ contentCatalog }) => {
-    assertConnectReferencePresent(contentCatalog)
+  this.on('contentClassified', ({ contentCatalog, playbook }) => {
+    assertConnectReferencePresent(contentCatalog, { connectSources: connectSourcesOf(playbook) })
     return processContent(contentCatalog)
   })
 
@@ -132,8 +184,9 @@ module.exports.register = function ({ config }) {
 
   // Raw catalog rows (info.csv column names) for this build.
   //
-  // The generated partials/platforms/catalog.json from the connect content
-  // source comes first: it is pinned to the same ref as the reference content
+  // The generated partials/platforms/catalog.json from connect (the release
+  // asset, or a connect content source) comes first: it is from the same ref
+  // as the reference content
   // and carries status, categories, and cgo data that info.csv lacks. info.csv
   // still supplies the SQL driver rows, which are not components and so are
   // not in the catalog. Without catalog.json, info.csv supplies every row.
@@ -152,7 +205,8 @@ module.exports.register = function ({ config }) {
     let csvRows = []
     try {
       const csvText = await fetchCSV(localCsvPath, contentCatalog)
-      csvRows = Papa.parse(csvText, { header: true, skipEmptyLines: true }).data
+      // connect's info.csv pads every column with spaces, headers included
+      csvRows = Papa.parse(csvText, { header: true, skipEmptyLines: true, transformHeader: (h) => h.trim() }).data
     } catch (error) {
       if (!catalogRows) throw error
       logger.warn(`Could not fetch info.csv for the SQL driver rows, so the SQL driver support list is empty: ${error.message}`)
@@ -186,8 +240,9 @@ module.exports.register = function ({ config }) {
   }
 
   // The ref to read info.csv from, in order:
-  // 1. the tag modify-connect-tag-playbook resolved for the connect content source
-  // 2. the ref of the connect content source in the content catalog
+  // 1. the tag modify-connect-tag-playbook resolved (for the release asset or
+  //    the connect content source)
+  // 2. the ref of the connect files in the content catalog
   // 3. latest-connect-version in antora.yml in the working directory
   // 4. main, with a warning, because the catalog can then disagree with the
   //    reference content
@@ -201,9 +256,9 @@ module.exports.register = function ({ config }) {
     const normalizedVersion = connectVersion ? String(connectVersion).trim().replace(/^v/, '') : ''
     if (normalizedVersion) return { ...base, ref: `v${normalizedVersion}`, source: 'latest-connect-version in antora.yml' }
     logger.warn(
-      'No connect content source or latest-connect-version found, so info.csv is read from connect main. ' +
-      'Catalog badges can then disagree with the reference content. Add the connect content source and the ' +
-      'modify-connect-tag-playbook extension to the playbook.'
+      'No resolved connect release or latest-connect-version found, so info.csv is read from connect main. ' +
+      'Catalog badges can then disagree with the reference content. Register the modify-connect-tag-playbook ' +
+      'extension, or set its `tag` config when REDPANDA_CONNECT_DOCS_DIR is set.'
     )
     return { ...base, ref: 'main', source: 'fallback' }
   }