@redpanda-data/docs-extensions-and-macros 5.53.0 → 5.55.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (51) hide show
  1. package/bin/doc-tools-mcp.js +4 -5
  2. package/bin/doc-tools.js +166 -2
  3. package/bin/mcp-tools/generated-docs-review.js +26 -9
  4. package/bin/mcp-tools/overrides-audit.js +1 -1
  5. package/bin/mcp-tools/rpcn-docs.js +4 -14
  6. package/cli-utils/format-helm-spec.js +4 -3
  7. package/cli-utils/helm-commented-values.js +88 -18
  8. package/cli-utils/self-managed-docs-branch.js +36 -30
  9. package/extension-utils/html-utils.js +27 -0
  10. package/extensions/REFERENCE.adoc +64 -12
  11. package/extensions/add-git-dates.js +13 -1
  12. package/extensions/aggregate-terms.js +34 -20
  13. package/extensions/generate-rp-connect-info.js +79 -24
  14. package/extensions/modify-connect-tag-playbook.js +498 -75
  15. package/extensions/resolve-xrefs-in-attachments.js +1 -12
  16. package/extensions/url-to-xref.js +3 -0
  17. package/extensions/util/connect-catalog.js +10 -5
  18. package/extensions/util/connect-docs-asset.js +188 -0
  19. package/extensions/util/connect-github.js +16 -0
  20. package/extensions/version-fetcher/fetch-latest-docker-tag.js +35 -13
  21. package/extensions/version-fetcher/get-latest-redpanda-helm-version-from-operator.js +64 -40
  22. package/extensions/version-fetcher/list-connect-releases.js +10 -0
  23. package/extensions/version-fetcher/release-lines.js +64 -0
  24. package/extensions/version-fetcher/set-latest-version.js +73 -3
  25. package/macros/README.adoc +1 -0
  26. package/macros/REFERENCE.adoc +99 -0
  27. package/macros/badge.js +5 -8
  28. package/macros/enterprise.js +2 -5
  29. package/macros/glossary.js +4 -14
  30. package/macros/iceberg-explorer.js +242 -0
  31. package/macros/prop.js +3 -13
  32. package/mcp/USER_GUIDE.adoc +25 -135
  33. package/package.json +4 -2
  34. package/tools/connect-docs/check-build-log.js +311 -0
  35. package/tools/connect-docs/check-rendered-html.js +469 -0
  36. package/tools/connect-docs/ci-playbook.yml +77 -0
  37. package/tools/connect-docs/diff-generated.js +382 -0
  38. package/tools/connect-docs/pr-comment.js +104 -0
  39. package/tools/doc-impact-recording/index.js +190 -0
  40. package/tools/overrides-audit/README.adoc +1 -2
  41. package/tools/overrides-audit/adapters/properties.js +1 -1
  42. package/tools/overrides-audit/index.js +3 -4
  43. package/tools/redpanda-connect/AUTOMATION.md +3 -36
  44. package/tools/redpanda-connect/README.adoc +5 -33
  45. package/tools/redpanda-connect/generate-rpcn-connector-docs.js +3 -3
  46. package/tools/redpanda-connect/pr-summary-formatter.js +6 -4
  47. package/tools/redpanda-connect/report-delta.js +6 -3
  48. package/tools/redpanda-connect/rpcn-connector-docs-handler.js +38 -16
  49. package/tools/redpanda-release-notes/generate-release-notes.js +116 -1
  50. package/tools/rpk-docs/templates/command.hbs +1 -1
  51. package/tools/overrides-audit/adapters/connect.js +0 -144
@@ -0,0 +1,27 @@
1
+ 'use strict'
2
+
3
+ /**
4
+ * Shared HTML utility functions for extensions and macros.
5
+ */
6
+
7
+ /**
8
+ * Escape a value for safe inclusion in HTML text or inside a double-quoted
9
+ * attribute. Escaping `&` first is what keeps the result idempotent-safe for a
10
+ * single pass; escaping `"` is what stops an author-supplied string from ending
11
+ * the attribute it sits in.
12
+ *
13
+ * @param {*} value - Value to escape. `null` and `undefined` become ''.
14
+ * @returns {string} The escaped string.
15
+ */
16
+ function escapeHtml (value) {
17
+ if (value === null || value === undefined) return ''
18
+ return String(value)
19
+ .replace(/&/g, '&')
20
+ .replace(/</g, '&lt;')
21
+ .replace(/>/g, '&gt;')
22
+ .replace(/"/g, '&quot;')
23
+ }
24
+
25
+ module.exports = {
26
+ escapeHtml,
27
+ }
@@ -404,39 +404,76 @@ You can optionally display pages only if they match the component and version of
404
404
 
405
405
  == Redpanda Connect tag modifier
406
406
 
407
- This extension sources the Redpanda Connect reference partials (field reference, examples, metadata, descriptions, config snippets, and the Bloblang reference) from the `docs/` directory of the connect repository, at the latest Connect release.
407
+ This extension adds the Redpanda Connect generated reference docs (field reference, examples, metadata, descriptions, config snippets, the Bloblang reference, and `platforms/catalog.json`) to the `connect` component that rp-connect-docs provides, from a Connect release.
408
408
 
409
- Add the connect repository as a content source next to rp-connect-docs:
409
+ Every Connect release publishes the generated tree as a release asset named `redpanda-connect-docs.tar.gz`, at `\https://github.com/redpanda-data/connect/releases/download/<tag>/redpanda-connect-docs.tar.gz`. The playbook does not list the connect repository as a content source. If it still lists the upstream repository with `tags: latest`, the extension removes that source. See <<Playbooks that list the connect repository>>.
410
+
411
+ At build time, when the playbook has the `connect` component, the extension:
412
+
413
+ . Picks the release: the latest stable `vX.Y.Z` release by default, so the site never shows fields from unreleased code on connect's `main` branch. It asks the GitHub API for the latest release and falls back to the highest stable `vX.Y.Z` tag from `git ls-remote`. If neither works, the build stops with an error. The `tag` option picks a specific release instead.
414
+ . Downloads that release's `redpanda-connect-docs.tar.gz` asset. With a GitHub token it downloads through the releases API, which keeps working if the connect repository becomes private. Without one, or if GitHub rejects the token, it uses the public download URL.
415
+ . Adds `modules/components/partials/**` as partials, `modules/components/examples/**` as examples, and `modules/components/attachments/**` (the release's `connect-<version>.json`, read by the Bloblang playground) as attachments of the `components` module, in every version of the `connect` component. A file that another source, such as rp-connect-docs, already provides is skipped and counted in the log, so rp-connect-docs can keep committing its own copies until it stops.
416
+ . When the playbook also registers `add-git-dates`, dates each page by the Connect release that last changed its reference. For every generated partial or example a page includes, directly or through another generated partial such as `bloblang/methods.adoc`, the extension compares the release with older releases that have the asset, newest first, and takes the publish date of the newest release that changed the file. A page that single-sources such a connect page, such as a Cloud stub, gets the same date. `add-git-dates` uses the date as `page-git-modified-date` when it is later than the page's last commit. The walk stops at v4.113.0, the first release with the asset, and after at most 10 older releases. A file that is unchanged for the whole walk keeps the page's git date. If the releases listing or an older release's download fails, the extension logs a warning and pages keep their git dates, so the build never stops for a date. Without `add-git-dates` in the playbook, the extension skips this step and downloads only the release it uses.
417
+
418
+ A release from before the asset existed has none, and the download returns 404. A new release can also be missing it for a while: Connect publishes the release first and attaches the asset 15 to 20 minutes later in a separate job, which can fail.
419
+
420
+ So when the latest release has no asset, the extension falls back to the newest older stable `vX.Y.Z` release that has one, and logs a warning that names both, such as `Redpanda Connect v4.114.0 has no redpanda-connect-docs.tar.gz asset yet; using v4.113.0`. It reads the asset lists of the 10 most recent releases from the GitHub releases API, so it downloads only the release it picks. If the API is unavailable, it tries the downloads of the next 3 lower stable tags from `git ls-remote` instead. A release picked with the `tag` option never falls back: if it has no asset, the build stops with an error that names the release.
421
+
422
+ When no release it checks has the asset, the extension logs this and adds nothing. The `generate-rp-connect-info` extension then stops the build if connector pages are missing any generated set they include: the `fields/`, `descriptions/`, or `availability/` partials, or the `common/` or `advanced/` examples. This also catches an incomplete asset, which would otherwise publish pages with unresolved includes. Any other failure, such as a network error or a corrupt archive, stops the build, because building without the reference would publish connector pages with no fields.
423
+
424
+ The extension shares the release it used, including a fallback release, so `generate-rp-connect-info` reads `catalog.json` and `info.csv` for the same release as the reference docs. It adds its files before any other `contentClassified` listener runs, wherever it is listed in the playbook.
425
+
426
+ === Preview unreleased Connect changes
427
+
428
+ To build with reference docs that are not released yet, generate them in a connect checkout and point `REDPANDA_CONNECT_DOCS_DIR` at the result:
429
+
430
+ [,bash]
431
+ ----
432
+ export REDPANDA_CONNECT_DOCS_DIR=/path/to/connect/docs
433
+ npx antora --fetch local-antora-playbook.yml
434
+ ----
435
+
436
+ The value is a directory that contains `modules/` (a connect checkout's `docs/` directory after running its docs generator, or the checkout root), or a local copy of a `redpanda-connect-docs.tar.gz` file. It wins over the `tag` option and the latest release, and nothing is downloaded. Its files also replace any copy another source provides, such as the copies rp-connect-docs commits, and the log counts how many were replaced. Set `tag` as well to choose the release that `generate-rp-connect-info` reads `info.csv` from.
437
+
438
+ === Playbooks that list the connect repository
439
+
440
+ Before the release asset, playbooks listed the connect repository as a content source:
410
441
 
411
442
  ```yaml
412
443
  content:
413
444
  sources:
414
445
  - url: https://github.com/redpanda-data/connect
415
- tags: latest # replaced with the latest release tag at build time
446
+ tags: latest
416
447
  start_path: docs
417
448
  ```
418
449
 
419
- At build time the extension:
450
+ Connect releases no longer include `docs/antora.yml`, so Antora cannot aggregate that source. When the upstream `redpanda-data/connect` repository is listed with `tags: latest` as its only ref, the extension removes the source from the playbook before Antora fetches content, logs a warning that it is no longer needed, and uses the release asset. Remove the source from the playbook to silence the warning.
420
451
 
421
- * Pins a source that sets `tags: latest` to the latest Connect release tag and clears its branches (a source with any other refs, such as a PR branch for a preview, is left as configured), so the site never shows fields from unreleased code on connect's `main` branch. It asks the GitHub API for the latest release and falls back to the highest stable `vX.Y.Z` tag from `git ls-remote`. If neither works, the build stops with an error.
422
- * Keeps only the `modules/components/partials/` and `modules/components/examples/` files of the `connect` component from that source. Any file that another source, such as rp-connect-docs, already provides is skipped, and so is content from older Connect tags that name their component `redpanda-connect`. Antora fails the build on a duplicate page or partial, so you can add the source to a playbook before or after rp-connect-docs stops committing its own generated copies.
452
+ A connect source with explicit refs is kept, which is how you preview a connect branch: a branch, a specific tag, a fork, or a local clone whose directory is named `connect`. In that case the extension downloads no asset, and:
423
453
 
424
- A local clone whose directory is named `connect` is filtered the same way but not pinned, so a local playbook can use `branches: HEAD` to test uncommitted changes.
454
+ * Only the `modules/components/partials/` and `modules/components/examples/` files of the `connect` component are kept from that source. Any file that another source already provides is skipped, and so is content from older Connect tags that name their component `redpanda-connect`.
455
+ * `REDPANDA_CONNECT_DOCS_DIR` and the `tag` option are ignored.
456
+ * The ref must have `docs/antora.yml` and the generated docs. If the generated sets are missing, the `generate-rp-connect-info` error names the connect source.
425
457
 
426
458
  === Environment variables
427
459
 
428
- - `REDPANDA_GITHUB_TOKEN` (optional): A GitHub token for the release lookup. Without it, the extension sends unauthenticated API requests, which GitHub can rate limit. The `git ls-remote` fallback needs no token.
460
+ - `REDPANDA_CONNECT_DOCS_DIR` (optional): A local directory, or a local `redpanda-connect-docs.tar.gz` file, to read the reference docs from instead of a release. See <<Preview unreleased Connect changes>>.
461
+ - `REDPANDA_GITHUB_TOKEN` (optional): A GitHub token for the latest release lookup, the fallback release listing, the asset download, and the `git ls-remote` fallback. The extension also reads `ACTIONS_BOT_TOKEN`, `GITHUB_TOKEN`, `VBOT_GITHUB_API_TOKEN`, `GH_TOKEN`, and a github.com entry in `GIT_CREDENTIALS`, in that order. Without any GitHub token, the extension sends unauthenticated requests, which GitHub can rate limit, and which only work while the connect repository is public.
429
462
 
430
463
  === Configuration options
431
464
 
432
- There are no configurable options for this extension.
465
+ `tag` (optional):: A Connect release tag, such as `v4.113.0`, to use instead of the latest stable release. A missing `v` prefix is added. If that release has no asset, the build stops: an explicit tag never falls back to an older release. `latest` is rejected, because the latest release is the default without `tag`. Ignored when the playbook keeps a connect content source with explicit refs.
433
466
 
434
467
  === Registration
435
468
 
469
+ Register this extension together with `generate-rp-connect-info`:
470
+
436
471
  ```yaml
437
472
  antora:
438
473
  extensions:
439
474
  - require: '@redpanda-data/docs-extensions-and-macros/extensions/modify-connect-tag-playbook'
475
+ # tag: v4.113.0 # optional: a specific release instead of the latest
476
+ - require: '@redpanda-data/docs-extensions-and-macros/extensions/generate-rp-connect-info'
440
477
  ```
441
478
 
442
479
  == Version fetcher
@@ -454,8 +491,8 @@ The following attributes are available to all versions of all Antora components:
454
491
  `console-beta-version`:: The latest beta version of Redpanda Console.
455
492
  `operator-beta-version`:: The latest beta version of the Redpanda Operator.
456
493
  `helm-beta-version`:: The latest beta version of the Redpanda Helm chart.
457
- `latest-operator-version`:: The latest release version of the Redpanda Operator. This value keeps its `v` prefix (for example, `v25.1.3`) because pages pass it straight to `helm --version`.
458
- `latest-redpanda-helm-chart-version`:: The latest release version of the Redpanda Helm chart.
494
+ `latest-operator-version`:: The latest release version of the Redpanda Operator for the component version's release line (see <<operator-and-chart-per-version>>). This value keeps its `v` prefix (for example, `v25.1.3`) because pages pass it straight to `helm --version`.
495
+ `latest-redpanda-helm-chart-version`:: The latest release version of the Redpanda Helm chart for the component version's release line (see <<operator-and-chart-per-version>>). This value has no `v` prefix (for example, `25.3.11`).
459
496
 
460
497
  The following attributes are available to the latest version of each Antora component:
461
498
 
@@ -475,6 +512,19 @@ ever moves forwards.
475
512
 
476
513
  A short attribute is only set when a `major.minor` pair can be derived. Release channels that are not versions, such as `nightly` or `latest`, get a `-version` and a `-tag` but no `-version-short`, so a page never renders a fabricated release line.
477
514
 
515
+ [[operator-and-chart-per-version]]
516
+ ==== Operator and Helm chart versions per component version
517
+
518
+ An operator or Helm chart release supports its own Redpanda version and one minor version above and below it, so each component version gets the release of its own line:
519
+
520
+ * A component version named `<major>.<minor>` that is older than the component's latest version gets the newest stable release in that line. For example, version `25.3` gets the newest `v25.3.x` operator and the newest `25.3.x` chart, even when a `26.2.x` release exists.
521
+ * The component's latest version, any prerelease version ahead of it, and unversioned components get the newest stable release overall.
522
+ * If an older line has no stable operator release (for example, lines that shipped only legacy `v2.x.y-<redpanda-version>` operator tags), the newest release does not support it, so both attributes keep the values the component version's `antora.yml` sets. If the line has an operator release but no chart release, only the chart attribute keeps its `antora.yml` value. An attribute that `antora.yml` leaves unset or empty falls back to the newest release overall. Either way, the build logs a warning that names the line.
523
+
524
+ The release lines come from the fetched operator tags, so no version map needs updating when a line ships. The extension reads up to three pages (300 tags) of the operator's Docker Hub tags, newest pushes first.
525
+
526
+ The chart version of a line is the newest stable `charts/redpanda/v<major>.<minor>.*` tag in the redpanda-operator repository, which marks a published chart (what `helm search repo` reports). A release branch's `Chart.yaml` can be ahead of the published chart, so the extension reads it only when the line has no chart tag, trying each path the chart has used in order: `charts/redpanda/chart/Chart.yaml`, then `charts/redpanda/Chart.yaml` (the layout of `release/v25.1.x`).
527
+
478
528
  ==== Seed a fallback in antora.yml
479
529
 
480
530
  Every attribute here depends on a successful GitHub lookup at build time. If a lookup fails, for example because of a rate limit or a missing token, the attribute is never set and any page that references it renders the attribute name in braces.
@@ -842,6 +892,8 @@ The extension accepts the following configuration options:
842
892
 
843
893
  termspath (optional):: Specifies the path to a local directory containing term files (in `.adoc` format). If this path is provided, the extension will attempt to load terms from this directory first. If this path is not provided or no valid terms are found in the specified directory, the extension will fall back to loading terms from the `shared` component.
844
894
 
895
+ glossarypage (optional):: The resource ID of one page that holds the glossary for the whole site, such as `home:ROOT:glossary.adoc`. The extension appends every term to that page and sets every component's `glossary-page` attribute to it, so `glossterm` links from any component point at the one glossary. Any `reference:glossary.adoc` pages that components still have are filled too, so they stay complete until they are removed. If the page does not exist, the extension logs a warning and links each component to its own `reference:glossary.adoc`, as without this option. When the page exists, it takes precedence over a `glossary-page` attribute set in a component's `antora.yml`, so a per-component value only applies if this page is missing.
896
+
845
897
  Term files should follow the following structure:
846
898
 
847
899
  ```asciidoc
@@ -1077,7 +1129,7 @@ This extension adds accurate git-based created and modified dates to documentati
1077
1129
  The extension adds the following attributes to each page:
1078
1130
 
1079
1131
  - `page-git-created-date`: The date when the file was first committed (YYYY-MM-DD format)
1080
- - `page-git-modified-date`: The date when the file was last modified (YYYY-MM-DD format)
1132
+ - `page-git-modified-date`: The date when the file was last modified (YYYY-MM-DD format). For a page with Connect reference content, this is the date of the most recent Connect release that changed that content, when it is later than the page's last commit (see <<Redpanda Connect tag modifier>>).
1081
1133
 
1082
1134
  These attributes are available for use in UI templates (for metadata, display banners, etc.) and are exported to markdown files for AI consumption.
1083
1135
 
@@ -180,6 +180,16 @@ async function getTreeFiles (git, gitdir, oid, prefix, cache, treeCache) {
180
180
  return files
181
181
  }
182
182
 
183
+ // The later of two YYYY-MM-DD dates. modify-connect-tag-playbook sets
184
+ // page.connectReferenceModified to the date of the Connect release that last
185
+ // changed a page's generated reference, which lives in a release asset and so
186
+ // in no git commit of the page's repo.
187
+ function laterDate (a, b) {
188
+ if (!b) return a
189
+ if (!a) return b
190
+ return b > a ? b : a
191
+ }
192
+
183
193
  module.exports.register = function () {
184
194
  raiseListenerLimit(this)
185
195
  const logger = this.getLogger('add-git-dates-extension')
@@ -271,7 +281,7 @@ module.exports.register = function () {
271
281
  const dates = fileDateMap.get(relativeFilePath)
272
282
  if (dates) {
273
283
  page.asciidoc.attributes['page-git-created-date'] = dates.created
274
- page.asciidoc.attributes['page-git-modified-date'] = dates.modified
284
+ page.asciidoc.attributes['page-git-modified-date'] = laterDate(dates.modified, page.connectReferenceModified)
275
285
  processedCount++
276
286
  }
277
287
  }
@@ -288,3 +298,5 @@ module.exports.register = function () {
288
298
  logger.info(`Git dates added: processed=${processedCount}, skipped=${skippedCount}, duration=${duration}ms (${perPage}ms/page)`)
289
299
  })
290
300
  }
301
+
302
+ module.exports._internal = Object.assign(module.exports._internal || {}, { laterDate })
@@ -127,32 +127,46 @@ module.exports.register = function ({ config }) {
127
127
  })
128
128
 
129
129
  .on('contentClassified', ({ siteCatalog, contentCatalog }) => {
130
- const components = contentCatalog.getComponents();
130
+ // Appends every term, by category, to a glossary page.
131
+ function mergeTerms (glossaryPage) {
132
+ let glossaryContent = glossaryPage.contents.toString('utf8');
133
+ Object.keys(siteCatalog.termsByCategory).sort().forEach(category => {
134
+ let categoryContent = `\n\n== ${category}\n`;
135
+ siteCatalog.termsByCategory[category].sort((a, b) => a.name.localeCompare(b.name)).forEach(term => {
136
+ categoryContent += `\n\n${processTermContent(term.content)}`;
137
+ });
138
+ glossaryContent += categoryContent;
139
+ });
140
+ glossaryPage.contents = Buffer.from(glossaryContent, 'utf8');
141
+ }
142
+
131
143
  try {
132
- components.forEach(({ versions }) => {
144
+ // With `glossarypage`, one page holds the glossary for the whole site,
145
+ // and every component's glossterm links point at it. Glossary pages
146
+ // components still have are filled too, so they stay complete until
147
+ // they are removed.
148
+ let sitePage = null;
149
+ if (config.glossarypage) {
150
+ sitePage = contentCatalog.resolvePage(config.glossarypage);
151
+ if (sitePage) {
152
+ mergeTerms(sitePage);
153
+ logger.info(`Merged terms into the site glossary ${config.glossarypage}.`);
154
+ } else {
155
+ logger.warn(`The glossarypage ${config.glossarypage} does not exist, so each component links to its own reference:glossary.adoc instead.`);
156
+ }
157
+ }
158
+
159
+ contentCatalog.getComponents().forEach(({ versions }) => {
133
160
  versions.forEach(({ name: component, version, asciidoc, title }) => {
134
161
  if (component == 'shared') return;
162
+ if (sitePage) asciidoc.attributes['glossary-page'] = config.glossarypage;
135
163
 
136
164
  const glossaryPage = contentCatalog.resolvePage(`${version ? version + '@' : ''}${component}:reference:glossary.adoc`);
137
-
138
- if (glossaryPage) {
139
- asciidoc.attributes['glossary-page'] = 'reference:glossary.adoc';
140
- let glossaryContent = glossaryPage.contents.toString('utf8');
141
-
142
- Object.keys(siteCatalog.termsByCategory).sort().forEach(category => {
143
- let categoryContent = `\n\n== ${category}\n`;
144
-
145
- siteCatalog.termsByCategory[category].sort((a, b) => a.name.localeCompare(b.name)).forEach(term => {
146
- let processedContent = processTermContent(term.content);
147
- categoryContent += `\n\n${processedContent}`;
148
- });
149
-
150
- glossaryContent += categoryContent;
151
- });
152
-
153
- glossaryPage.contents = Buffer.from(glossaryContent, 'utf8');
165
+ if (glossaryPage && glossaryPage !== sitePage) {
166
+ if (!sitePage) asciidoc.attributes['glossary-page'] = 'reference:glossary.adoc';
167
+ mergeTerms(glossaryPage);
154
168
  logger.info(`Merged terms into glossary for ${component} component${version ? ' version ' + version : ''}.`);
155
- } else {
169
+ } else if (!glossaryPage && !sitePage) {
156
170
  logger.info(`Skipping ${title} ${version ? ' version ' + version : ''} - No glossary page (reference:glossary.adoc) found`);
157
171
  }
158
172
  });
@@ -13,29 +13,81 @@ const DEFAULTS = {
13
13
  githubRepo: 'connect'
14
14
  }
15
15
 
16
- // Connector pages include the generated field reference from the connect repo
17
- // (see the modify-connect-tag-playbook extension). If a playbook has the
18
- // Connect pages but none of those partials, every connector page would publish
19
- // without its fields, and Antora only logs the missing includes. Fail the
20
- // build instead so the playbook gets fixed before anything is published.
21
- function assertConnectReferencePresent (contentCatalog) {
16
+ // Generated sets that connector pages include, by Antora family. Each one
17
+ // must be present, or pages publish with unresolved includes.
18
+ // The generated partial and example directories that connector pages include,
19
+ // read from the pages themselves, so the guard asks for exactly what the pages
20
+ // in this build use (rp-connect-docs pages gain availability includes, for
21
+ // example, when they move to the generated docs).
22
+ const PARTIAL_INCLUDE = /include::connect:components:partial\$([a-z0-9_-]+)\//g
23
+ const EXAMPLE_INCLUDE = /include::(?:connect:)?components:example\$([a-z0-9_-]+)\//g
24
+
25
+ function includedDirs (pages) {
26
+ const partial = new Set()
27
+ const example = new Set()
28
+ for (const page of pages) {
29
+ const text = page.contents ? page.contents.toString() : ''
30
+ for (const m of text.matchAll(PARTIAL_INCLUDE)) partial.add(m[1])
31
+ for (const m of text.matchAll(EXAMPLE_INCLUDE)) example.add(m[1])
32
+ }
33
+ // Name the core sets first, in a fixed order, so the error reads the same
34
+ // way every time.
35
+ const order = ['fields', 'descriptions', 'availability', 'metadata', 'examples', 'common', 'advanced']
36
+ const rank = (d) => (order.includes(d) ? order.indexOf(d) : order.length)
37
+ const sorted = (set) => [...set].sort((a, b) => rank(a) - rank(b) || a.localeCompare(b))
38
+ return { partial: sorted(partial), example: sorted(example) }
39
+ }
40
+
41
+ // Connector pages include the generated field reference, description meta,
42
+ // availability, and config examples, which the modify-connect-tag-playbook
43
+ // extension adds from the redpanda-connect-docs.tar.gz asset of a connect
44
+ // release. If a playbook has the Connect pages but any set is missing (an
45
+ // incomplete asset, or a connect source whose ref lacks them), connector
46
+ // pages would publish with unresolved includes, and Antora only logs them.
47
+ // Fail the build instead so the playbook gets fixed before anything is
48
+ // published.
49
+ //
50
+ // `connectSources` are the connect content sources still in the playbook,
51
+ // which means no release asset was downloaded.
52
+ function assertConnectReferencePresent (contentCatalog, { connectSources = [] } = {}) {
22
53
  const component = contentCatalog.getComponents().find((c) => c.name === 'connect')
23
54
  if (!component) return
24
- // Only connector pages (inputs/kafka.adoc and so on) include field
25
- // partials. Overview pages at the module root don't.
55
+ // Only connector pages (inputs/kafka.adoc and so on) include the generated
56
+ // sets. Overview pages at the module root don't.
26
57
  const pages = contentCatalog.findBy({ component: 'connect', module: 'components', family: 'page' })
27
58
  .filter((p) => p.src.relative.includes('/'))
28
59
  if (!pages.length) return
29
- const fields = contentCatalog.findBy({ component: 'connect', module: 'components', family: 'partial' })
30
- .filter((f) => f.src.relative.startsWith('fields/'))
31
- if (fields.length) return
60
+ const missingIn = (family, dirs) => {
61
+ const files = contentCatalog.findBy({ component: 'connect', module: 'components', family })
62
+ return dirs.filter((dir) => !files.some((f) => f.src.relative.startsWith(`${dir}/`)))
63
+ .map((dir) => `components:${family}$${dir}/*`)
64
+ }
65
+ const wanted = includedDirs(pages)
66
+ const missing = [...missingIn('partial', wanted.partial), ...missingIn('example', wanted.example)]
67
+ if (!missing.length) return
68
+ const sourceHint = connectSources.length
69
+ ? `The playbook lists a connect content source (${connectSources.join(', ')}), so no release asset was downloaded and these files must come from that source's refs. ` +
70
+ 'Point it at a ref that has the generated docs, or remove it to use the release asset. '
71
+ : ''
32
72
  throw new Error(
33
- 'The connect component has connector pages but no generated field partials (components:partial$fields/*). ' +
34
- 'Add the connect repository as a content source (url: https://github.com/redpanda-data/connect, tags: latest, start_path: docs) ' +
35
- 'and register the modify-connect-tag-playbook extension, or check that the latest Connect release includes docs/modules/components.'
73
+ `The connect component has connector pages but no generated ${missing.join(' or ')} files. ` +
74
+ sourceHint +
75
+ 'These come from the redpanda-connect-docs.tar.gz asset of a Redpanda Connect release, which the ' +
76
+ 'modify-connect-tag-playbook extension downloads. Register that extension and check its log: ' +
77
+ 'the release it used (the latest stable release, or the one in its `tag` config) may have no asset or an incomplete one. ' +
78
+ 'Set `tag` to a release that has the asset, or set REDPANDA_CONNECT_DOCS_DIR to a local directory that contains ' +
79
+ 'modules/ (for example a connect checkout\'s docs/ after running its docs generator).'
36
80
  )
37
81
  }
38
82
 
83
+ // The connect content sources in a playbook, with credentials removed.
84
+ function connectSourcesOf (playbook) {
85
+ const sources = (playbook && playbook.content && playbook.content.sources) || []
86
+ return sources
87
+ .filter((s) => catalogUtil.isConnectSource(s.url))
88
+ .map((s) => String(s.url).replace(/([a-z][a-z+.-]*:\/\/)[^@/\s]*@/gi, '$1'))
89
+ }
90
+
39
91
  module.exports.assertConnectReferencePresent = assertConnectReferencePresent
40
92
 
41
93
  module.exports.register = function ({ config }) {
@@ -63,8 +115,8 @@ module.exports.register = function ({ config }) {
63
115
  let translatedRows = null
64
116
 
65
117
  // Use 'on' and return the promise so Antora waits for async completion
66
- this.on('contentClassified', ({ contentCatalog }) => {
67
- assertConnectReferencePresent(contentCatalog)
118
+ this.on('contentClassified', ({ contentCatalog, playbook }) => {
119
+ assertConnectReferencePresent(contentCatalog, { connectSources: connectSourcesOf(playbook) })
68
120
  return processContent(contentCatalog)
69
121
  })
70
122
 
@@ -132,8 +184,9 @@ module.exports.register = function ({ config }) {
132
184
 
133
185
  // Raw catalog rows (info.csv column names) for this build.
134
186
  //
135
- // The generated partials/platforms/catalog.json from the connect content
136
- // source comes first: it is pinned to the same ref as the reference content
187
+ // The generated partials/platforms/catalog.json from connect (the release
188
+ // asset, or a connect content source) comes first: it is from the same ref
189
+ // as the reference content
137
190
  // and carries status, categories, and cgo data that info.csv lacks. info.csv
138
191
  // still supplies the SQL driver rows, which are not components and so are
139
192
  // not in the catalog. Without catalog.json, info.csv supplies every row.
@@ -152,7 +205,8 @@ module.exports.register = function ({ config }) {
152
205
  let csvRows = []
153
206
  try {
154
207
  const csvText = await fetchCSV(localCsvPath, contentCatalog)
155
- csvRows = Papa.parse(csvText, { header: true, skipEmptyLines: true }).data
208
+ // connect's info.csv pads every column with spaces, headers included
209
+ csvRows = Papa.parse(csvText, { header: true, skipEmptyLines: true, transformHeader: (h) => h.trim() }).data
156
210
  } catch (error) {
157
211
  if (!catalogRows) throw error
158
212
  logger.warn(`Could not fetch info.csv for the SQL driver rows, so the SQL driver support list is empty: ${error.message}`)
@@ -186,8 +240,9 @@ module.exports.register = function ({ config }) {
186
240
  }
187
241
 
188
242
  // The ref to read info.csv from, in order:
189
- // 1. the tag modify-connect-tag-playbook resolved for the connect content source
190
- // 2. the ref of the connect content source in the content catalog
243
+ // 1. the tag modify-connect-tag-playbook resolved (for the release asset or
244
+ // the connect content source)
245
+ // 2. the ref of the connect files in the content catalog
191
246
  // 3. latest-connect-version in antora.yml in the working directory
192
247
  // 4. main, with a warning, because the catalog can then disagree with the
193
248
  // reference content
@@ -201,9 +256,9 @@ module.exports.register = function ({ config }) {
201
256
  const normalizedVersion = connectVersion ? String(connectVersion).trim().replace(/^v/, '') : ''
202
257
  if (normalizedVersion) return { ...base, ref: `v${normalizedVersion}`, source: 'latest-connect-version in antora.yml' }
203
258
  logger.warn(
204
- 'No connect content source or latest-connect-version found, so info.csv is read from connect main. ' +
205
- 'Catalog badges can then disagree with the reference content. Add the connect content source and the ' +
206
- 'modify-connect-tag-playbook extension to the playbook.'
259
+ 'No resolved connect release or latest-connect-version found, so info.csv is read from connect main. ' +
260
+ 'Catalog badges can then disagree with the reference content. Register the modify-connect-tag-playbook ' +
261
+ 'extension, or set its `tag` config when REDPANDA_CONNECT_DOCS_DIR is set.'
207
262
  )
208
263
  return { ...base, ref: 'main', source: 'fallback' }
209
264
  }