@redpanda-data/docs-extensions-and-macros 5.53.0 → 5.55.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/doc-tools-mcp.js +4 -5
- package/bin/doc-tools.js +166 -2
- package/bin/mcp-tools/generated-docs-review.js +26 -9
- package/bin/mcp-tools/overrides-audit.js +1 -1
- package/bin/mcp-tools/rpcn-docs.js +4 -14
- package/cli-utils/format-helm-spec.js +4 -3
- package/cli-utils/helm-commented-values.js +88 -18
- package/cli-utils/self-managed-docs-branch.js +36 -30
- package/extension-utils/html-utils.js +27 -0
- package/extensions/REFERENCE.adoc +64 -12
- package/extensions/add-git-dates.js +13 -1
- package/extensions/aggregate-terms.js +34 -20
- package/extensions/generate-rp-connect-info.js +79 -24
- package/extensions/modify-connect-tag-playbook.js +498 -75
- package/extensions/resolve-xrefs-in-attachments.js +1 -12
- package/extensions/url-to-xref.js +3 -0
- package/extensions/util/connect-catalog.js +10 -5
- package/extensions/util/connect-docs-asset.js +188 -0
- package/extensions/util/connect-github.js +16 -0
- package/extensions/version-fetcher/fetch-latest-docker-tag.js +35 -13
- package/extensions/version-fetcher/get-latest-redpanda-helm-version-from-operator.js +64 -40
- package/extensions/version-fetcher/list-connect-releases.js +10 -0
- package/extensions/version-fetcher/release-lines.js +64 -0
- package/extensions/version-fetcher/set-latest-version.js +73 -3
- package/macros/README.adoc +1 -0
- package/macros/REFERENCE.adoc +99 -0
- package/macros/badge.js +5 -8
- package/macros/enterprise.js +2 -5
- package/macros/glossary.js +4 -14
- package/macros/iceberg-explorer.js +242 -0
- package/macros/prop.js +3 -13
- package/mcp/USER_GUIDE.adoc +25 -135
- package/package.json +4 -2
- package/tools/connect-docs/check-build-log.js +311 -0
- package/tools/connect-docs/check-rendered-html.js +469 -0
- package/tools/connect-docs/ci-playbook.yml +77 -0
- package/tools/connect-docs/diff-generated.js +382 -0
- package/tools/connect-docs/pr-comment.js +104 -0
- package/tools/doc-impact-recording/index.js +190 -0
- package/tools/overrides-audit/README.adoc +1 -2
- package/tools/overrides-audit/adapters/properties.js +1 -1
- package/tools/overrides-audit/index.js +3 -4
- package/tools/redpanda-connect/AUTOMATION.md +3 -36
- package/tools/redpanda-connect/README.adoc +5 -33
- package/tools/redpanda-connect/generate-rpcn-connector-docs.js +3 -3
- package/tools/redpanda-connect/pr-summary-formatter.js +6 -4
- package/tools/redpanda-connect/report-delta.js +6 -3
- package/tools/redpanda-connect/rpcn-connector-docs-handler.js +38 -16
- package/tools/redpanda-release-notes/generate-release-notes.js +116 -1
- package/tools/rpk-docs/templates/command.hbs +1 -1
- package/tools/overrides-audit/adapters/connect.js +0 -144
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
'use strict'
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* Shared HTML utility functions for extensions and macros.
|
|
5
|
+
*/
|
|
6
|
+
|
|
7
|
+
/**
|
|
8
|
+
* Escape a value for safe inclusion in HTML text or inside a double-quoted
|
|
9
|
+
* attribute. Escaping `&` first is what keeps the result idempotent-safe for a
|
|
10
|
+
* single pass; escaping `"` is what stops an author-supplied string from ending
|
|
11
|
+
* the attribute it sits in.
|
|
12
|
+
*
|
|
13
|
+
* @param {*} value - Value to escape. `null` and `undefined` become ''.
|
|
14
|
+
* @returns {string} The escaped string.
|
|
15
|
+
*/
|
|
16
|
+
function escapeHtml (value) {
|
|
17
|
+
if (value === null || value === undefined) return ''
|
|
18
|
+
return String(value)
|
|
19
|
+
.replace(/&/g, '&')
|
|
20
|
+
.replace(/</g, '<')
|
|
21
|
+
.replace(/>/g, '>')
|
|
22
|
+
.replace(/"/g, '"')
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
module.exports = {
|
|
26
|
+
escapeHtml,
|
|
27
|
+
}
|
|
@@ -404,39 +404,76 @@ You can optionally display pages only if they match the component and version of
|
|
|
404
404
|
|
|
405
405
|
== Redpanda Connect tag modifier
|
|
406
406
|
|
|
407
|
-
This extension
|
|
407
|
+
This extension adds the Redpanda Connect generated reference docs (field reference, examples, metadata, descriptions, config snippets, the Bloblang reference, and `platforms/catalog.json`) to the `connect` component that rp-connect-docs provides, from a Connect release.
|
|
408
408
|
|
|
409
|
-
|
|
409
|
+
Every Connect release publishes the generated tree as a release asset named `redpanda-connect-docs.tar.gz`, at `\https://github.com/redpanda-data/connect/releases/download/<tag>/redpanda-connect-docs.tar.gz`. The playbook does not list the connect repository as a content source. If it still lists the upstream repository with `tags: latest`, the extension removes that source. See <<Playbooks that list the connect repository>>.
|
|
410
|
+
|
|
411
|
+
At build time, when the playbook has the `connect` component, the extension:
|
|
412
|
+
|
|
413
|
+
. Picks the release: the latest stable `vX.Y.Z` release by default, so the site never shows fields from unreleased code on connect's `main` branch. It asks the GitHub API for the latest release and falls back to the highest stable `vX.Y.Z` tag from `git ls-remote`. If neither works, the build stops with an error. The `tag` option picks a specific release instead.
|
|
414
|
+
. Downloads that release's `redpanda-connect-docs.tar.gz` asset. With a GitHub token it downloads through the releases API, which keeps working if the connect repository becomes private. Without one, or if GitHub rejects the token, it uses the public download URL.
|
|
415
|
+
. Adds `modules/components/partials/**` as partials, `modules/components/examples/**` as examples, and `modules/components/attachments/**` (the release's `connect-<version>.json`, read by the Bloblang playground) as attachments of the `components` module, in every version of the `connect` component. A file that another source, such as rp-connect-docs, already provides is skipped and counted in the log, so rp-connect-docs can keep committing its own copies until it stops.
|
|
416
|
+
. When the playbook also registers `add-git-dates`, dates each page by the Connect release that last changed its reference. For every generated partial or example a page includes, directly or through another generated partial such as `bloblang/methods.adoc`, the extension compares the release with older releases that have the asset, newest first, and takes the publish date of the newest release that changed the file. A page that single-sources such a connect page, such as a Cloud stub, gets the same date. `add-git-dates` uses the date as `page-git-modified-date` when it is later than the page's last commit. The walk stops at v4.113.0, the first release with the asset, and after at most 10 older releases. A file that is unchanged for the whole walk keeps the page's git date. If the releases listing or an older release's download fails, the extension logs a warning and pages keep their git dates, so the build never stops for a date. Without `add-git-dates` in the playbook, the extension skips this step and downloads only the release it uses.
|
|
417
|
+
|
|
418
|
+
A release from before the asset existed has none, and the download returns 404. A new release can also be missing it for a while: Connect publishes the release first and attaches the asset 15 to 20 minutes later in a separate job, which can fail.
|
|
419
|
+
|
|
420
|
+
So when the latest release has no asset, the extension falls back to the newest older stable `vX.Y.Z` release that has one, and logs a warning that names both, such as `Redpanda Connect v4.114.0 has no redpanda-connect-docs.tar.gz asset yet; using v4.113.0`. It reads the asset lists of the 10 most recent releases from the GitHub releases API, so it downloads only the release it picks. If the API is unavailable, it tries the downloads of the next 3 lower stable tags from `git ls-remote` instead. A release picked with the `tag` option never falls back: if it has no asset, the build stops with an error that names the release.
|
|
421
|
+
|
|
422
|
+
When no release it checks has the asset, the extension logs this and adds nothing. The `generate-rp-connect-info` extension then stops the build if connector pages are missing any generated set they include: the `fields/`, `descriptions/`, or `availability/` partials, or the `common/` or `advanced/` examples. This also catches an incomplete asset, which would otherwise publish pages with unresolved includes. Any other failure, such as a network error or a corrupt archive, stops the build, because building without the reference would publish connector pages with no fields.
|
|
423
|
+
|
|
424
|
+
The extension shares the release it used, including a fallback release, so `generate-rp-connect-info` reads `catalog.json` and `info.csv` for the same release as the reference docs. It adds its files before any other `contentClassified` listener runs, wherever it is listed in the playbook.
|
|
425
|
+
|
|
426
|
+
=== Preview unreleased Connect changes
|
|
427
|
+
|
|
428
|
+
To build with reference docs that are not released yet, generate them in a connect checkout and point `REDPANDA_CONNECT_DOCS_DIR` at the result:
|
|
429
|
+
|
|
430
|
+
[,bash]
|
|
431
|
+
----
|
|
432
|
+
export REDPANDA_CONNECT_DOCS_DIR=/path/to/connect/docs
|
|
433
|
+
npx antora --fetch local-antora-playbook.yml
|
|
434
|
+
----
|
|
435
|
+
|
|
436
|
+
The value is a directory that contains `modules/` (a connect checkout's `docs/` directory after running its docs generator, or the checkout root), or a local copy of a `redpanda-connect-docs.tar.gz` file. It wins over the `tag` option and the latest release, and nothing is downloaded. Its files also replace any copy another source provides, such as the copies rp-connect-docs commits, and the log counts how many were replaced. Set `tag` as well to choose the release that `generate-rp-connect-info` reads `info.csv` from.
|
|
437
|
+
|
|
438
|
+
=== Playbooks that list the connect repository
|
|
439
|
+
|
|
440
|
+
Before the release asset, playbooks listed the connect repository as a content source:
|
|
410
441
|
|
|
411
442
|
```yaml
|
|
412
443
|
content:
|
|
413
444
|
sources:
|
|
414
445
|
- url: https://github.com/redpanda-data/connect
|
|
415
|
-
tags: latest
|
|
446
|
+
tags: latest
|
|
416
447
|
start_path: docs
|
|
417
448
|
```
|
|
418
449
|
|
|
419
|
-
|
|
450
|
+
Connect releases no longer include `docs/antora.yml`, so Antora cannot aggregate that source. When the upstream `redpanda-data/connect` repository is listed with `tags: latest` as its only ref, the extension removes the source from the playbook before Antora fetches content, logs a warning that it is no longer needed, and uses the release asset. Remove the source from the playbook to silence the warning.
|
|
420
451
|
|
|
421
|
-
|
|
422
|
-
* Keeps only the `modules/components/partials/` and `modules/components/examples/` files of the `connect` component from that source. Any file that another source, such as rp-connect-docs, already provides is skipped, and so is content from older Connect tags that name their component `redpanda-connect`. Antora fails the build on a duplicate page or partial, so you can add the source to a playbook before or after rp-connect-docs stops committing its own generated copies.
|
|
452
|
+
A connect source with explicit refs is kept, which is how you preview a connect branch: a branch, a specific tag, a fork, or a local clone whose directory is named `connect`. In that case the extension downloads no asset, and:
|
|
423
453
|
|
|
424
|
-
|
|
454
|
+
* Only the `modules/components/partials/` and `modules/components/examples/` files of the `connect` component are kept from that source. Any file that another source already provides is skipped, and so is content from older Connect tags that name their component `redpanda-connect`.
|
|
455
|
+
* `REDPANDA_CONNECT_DOCS_DIR` and the `tag` option are ignored.
|
|
456
|
+
* The ref must have `docs/antora.yml` and the generated docs. If the generated sets are missing, the `generate-rp-connect-info` error names the connect source.
|
|
425
457
|
|
|
426
458
|
=== Environment variables
|
|
427
459
|
|
|
428
|
-
- `
|
|
460
|
+
- `REDPANDA_CONNECT_DOCS_DIR` (optional): A local directory, or a local `redpanda-connect-docs.tar.gz` file, to read the reference docs from instead of a release. See <<Preview unreleased Connect changes>>.
|
|
461
|
+
- `REDPANDA_GITHUB_TOKEN` (optional): A GitHub token for the latest release lookup, the fallback release listing, the asset download, and the `git ls-remote` fallback. The extension also reads `ACTIONS_BOT_TOKEN`, `GITHUB_TOKEN`, `VBOT_GITHUB_API_TOKEN`, `GH_TOKEN`, and a github.com entry in `GIT_CREDENTIALS`, in that order. Without any GitHub token, the extension sends unauthenticated requests, which GitHub can rate limit, and which only work while the connect repository is public.
|
|
429
462
|
|
|
430
463
|
=== Configuration options
|
|
431
464
|
|
|
432
|
-
|
|
465
|
+
`tag` (optional):: A Connect release tag, such as `v4.113.0`, to use instead of the latest stable release. A missing `v` prefix is added. If that release has no asset, the build stops: an explicit tag never falls back to an older release. `latest` is rejected, because the latest release is the default without `tag`. Ignored when the playbook keeps a connect content source with explicit refs.
|
|
433
466
|
|
|
434
467
|
=== Registration
|
|
435
468
|
|
|
469
|
+
Register this extension together with `generate-rp-connect-info`:
|
|
470
|
+
|
|
436
471
|
```yaml
|
|
437
472
|
antora:
|
|
438
473
|
extensions:
|
|
439
474
|
- require: '@redpanda-data/docs-extensions-and-macros/extensions/modify-connect-tag-playbook'
|
|
475
|
+
# tag: v4.113.0 # optional: a specific release instead of the latest
|
|
476
|
+
- require: '@redpanda-data/docs-extensions-and-macros/extensions/generate-rp-connect-info'
|
|
440
477
|
```
|
|
441
478
|
|
|
442
479
|
== Version fetcher
|
|
@@ -454,8 +491,8 @@ The following attributes are available to all versions of all Antora components:
|
|
|
454
491
|
`console-beta-version`:: The latest beta version of Redpanda Console.
|
|
455
492
|
`operator-beta-version`:: The latest beta version of the Redpanda Operator.
|
|
456
493
|
`helm-beta-version`:: The latest beta version of the Redpanda Helm chart.
|
|
457
|
-
`latest-operator-version`:: The latest release version of the Redpanda Operator. This value keeps its `v` prefix (for example, `v25.1.3`) because pages pass it straight to `helm --version`.
|
|
458
|
-
`latest-redpanda-helm-chart-version`:: The latest release version of the Redpanda Helm chart.
|
|
494
|
+
`latest-operator-version`:: The latest release version of the Redpanda Operator for the component version's release line (see <<operator-and-chart-per-version>>). This value keeps its `v` prefix (for example, `v25.1.3`) because pages pass it straight to `helm --version`.
|
|
495
|
+
`latest-redpanda-helm-chart-version`:: The latest release version of the Redpanda Helm chart for the component version's release line (see <<operator-and-chart-per-version>>). This value has no `v` prefix (for example, `25.3.11`).
|
|
459
496
|
|
|
460
497
|
The following attributes are available to the latest version of each Antora component:
|
|
461
498
|
|
|
@@ -475,6 +512,19 @@ ever moves forwards.
|
|
|
475
512
|
|
|
476
513
|
A short attribute is only set when a `major.minor` pair can be derived. Release channels that are not versions, such as `nightly` or `latest`, get a `-version` and a `-tag` but no `-version-short`, so a page never renders a fabricated release line.
|
|
477
514
|
|
|
515
|
+
[[operator-and-chart-per-version]]
|
|
516
|
+
==== Operator and Helm chart versions per component version
|
|
517
|
+
|
|
518
|
+
An operator or Helm chart release supports its own Redpanda version and one minor version above and below it, so each component version gets the release of its own line:
|
|
519
|
+
|
|
520
|
+
* A component version named `<major>.<minor>` that is older than the component's latest version gets the newest stable release in that line. For example, version `25.3` gets the newest `v25.3.x` operator and the newest `25.3.x` chart, even when a `26.2.x` release exists.
|
|
521
|
+
* The component's latest version, any prerelease version ahead of it, and unversioned components get the newest stable release overall.
|
|
522
|
+
* If an older line has no stable operator release (for example, lines that shipped only legacy `v2.x.y-<redpanda-version>` operator tags), the newest release does not support it, so both attributes keep the values the component version's `antora.yml` sets. If the line has an operator release but no chart release, only the chart attribute keeps its `antora.yml` value. An attribute that `antora.yml` leaves unset or empty falls back to the newest release overall. Either way, the build logs a warning that names the line.
|
|
523
|
+
|
|
524
|
+
The release lines come from the fetched operator tags, so no version map needs updating when a line ships. The extension reads up to three pages (300 tags) of the operator's Docker Hub tags, newest pushes first.
|
|
525
|
+
|
|
526
|
+
The chart version of a line is the newest stable `charts/redpanda/v<major>.<minor>.*` tag in the redpanda-operator repository, which marks a published chart (what `helm search repo` reports). A release branch's `Chart.yaml` can be ahead of the published chart, so the extension reads it only when the line has no chart tag, trying each path the chart has used in order: `charts/redpanda/chart/Chart.yaml`, then `charts/redpanda/Chart.yaml` (the layout of `release/v25.1.x`).
|
|
527
|
+
|
|
478
528
|
==== Seed a fallback in antora.yml
|
|
479
529
|
|
|
480
530
|
Every attribute here depends on a successful GitHub lookup at build time. If a lookup fails, for example because of a rate limit or a missing token, the attribute is never set and any page that references it renders the attribute name in braces.
|
|
@@ -842,6 +892,8 @@ The extension accepts the following configuration options:
|
|
|
842
892
|
|
|
843
893
|
termspath (optional):: Specifies the path to a local directory containing term files (in `.adoc` format). If this path is provided, the extension will attempt to load terms from this directory first. If this path is not provided or no valid terms are found in the specified directory, the extension will fall back to loading terms from the `shared` component.
|
|
844
894
|
|
|
895
|
+
glossarypage (optional):: The resource ID of one page that holds the glossary for the whole site, such as `home:ROOT:glossary.adoc`. The extension appends every term to that page and sets every component's `glossary-page` attribute to it, so `glossterm` links from any component point at the one glossary. Any `reference:glossary.adoc` pages that components still have are filled too, so they stay complete until they are removed. If the page does not exist, the extension logs a warning and links each component to its own `reference:glossary.adoc`, as without this option. When the page exists, it takes precedence over a `glossary-page` attribute set in a component's `antora.yml`, so a per-component value only applies if this page is missing.
|
|
896
|
+
|
|
845
897
|
Term files should follow the following structure:
|
|
846
898
|
|
|
847
899
|
```asciidoc
|
|
@@ -1077,7 +1129,7 @@ This extension adds accurate git-based created and modified dates to documentati
|
|
|
1077
1129
|
The extension adds the following attributes to each page:
|
|
1078
1130
|
|
|
1079
1131
|
- `page-git-created-date`: The date when the file was first committed (YYYY-MM-DD format)
|
|
1080
|
-
- `page-git-modified-date`: The date when the file was last modified (YYYY-MM-DD format)
|
|
1132
|
+
- `page-git-modified-date`: The date when the file was last modified (YYYY-MM-DD format). For a page with Connect reference content, this is the date of the most recent Connect release that changed that content, when it is later than the page's last commit (see <<Redpanda Connect tag modifier>>).
|
|
1081
1133
|
|
|
1082
1134
|
These attributes are available for use in UI templates (for metadata, display banners, etc.) and are exported to markdown files for AI consumption.
|
|
1083
1135
|
|
|
@@ -180,6 +180,16 @@ async function getTreeFiles (git, gitdir, oid, prefix, cache, treeCache) {
|
|
|
180
180
|
return files
|
|
181
181
|
}
|
|
182
182
|
|
|
183
|
+
// The later of two YYYY-MM-DD dates. modify-connect-tag-playbook sets
|
|
184
|
+
// page.connectReferenceModified to the date of the Connect release that last
|
|
185
|
+
// changed a page's generated reference, which lives in a release asset and so
|
|
186
|
+
// in no git commit of the page's repo.
|
|
187
|
+
function laterDate (a, b) {
|
|
188
|
+
if (!b) return a
|
|
189
|
+
if (!a) return b
|
|
190
|
+
return b > a ? b : a
|
|
191
|
+
}
|
|
192
|
+
|
|
183
193
|
module.exports.register = function () {
|
|
184
194
|
raiseListenerLimit(this)
|
|
185
195
|
const logger = this.getLogger('add-git-dates-extension')
|
|
@@ -271,7 +281,7 @@ module.exports.register = function () {
|
|
|
271
281
|
const dates = fileDateMap.get(relativeFilePath)
|
|
272
282
|
if (dates) {
|
|
273
283
|
page.asciidoc.attributes['page-git-created-date'] = dates.created
|
|
274
|
-
page.asciidoc.attributes['page-git-modified-date'] = dates.modified
|
|
284
|
+
page.asciidoc.attributes['page-git-modified-date'] = laterDate(dates.modified, page.connectReferenceModified)
|
|
275
285
|
processedCount++
|
|
276
286
|
}
|
|
277
287
|
}
|
|
@@ -288,3 +298,5 @@ module.exports.register = function () {
|
|
|
288
298
|
logger.info(`Git dates added: processed=${processedCount}, skipped=${skippedCount}, duration=${duration}ms (${perPage}ms/page)`)
|
|
289
299
|
})
|
|
290
300
|
}
|
|
301
|
+
|
|
302
|
+
module.exports._internal = Object.assign(module.exports._internal || {}, { laterDate })
|
|
@@ -127,32 +127,46 @@ module.exports.register = function ({ config }) {
|
|
|
127
127
|
})
|
|
128
128
|
|
|
129
129
|
.on('contentClassified', ({ siteCatalog, contentCatalog }) => {
|
|
130
|
-
|
|
130
|
+
// Appends every term, by category, to a glossary page.
|
|
131
|
+
function mergeTerms (glossaryPage) {
|
|
132
|
+
let glossaryContent = glossaryPage.contents.toString('utf8');
|
|
133
|
+
Object.keys(siteCatalog.termsByCategory).sort().forEach(category => {
|
|
134
|
+
let categoryContent = `\n\n== ${category}\n`;
|
|
135
|
+
siteCatalog.termsByCategory[category].sort((a, b) => a.name.localeCompare(b.name)).forEach(term => {
|
|
136
|
+
categoryContent += `\n\n${processTermContent(term.content)}`;
|
|
137
|
+
});
|
|
138
|
+
glossaryContent += categoryContent;
|
|
139
|
+
});
|
|
140
|
+
glossaryPage.contents = Buffer.from(glossaryContent, 'utf8');
|
|
141
|
+
}
|
|
142
|
+
|
|
131
143
|
try {
|
|
132
|
-
|
|
144
|
+
// With `glossarypage`, one page holds the glossary for the whole site,
|
|
145
|
+
// and every component's glossterm links point at it. Glossary pages
|
|
146
|
+
// components still have are filled too, so they stay complete until
|
|
147
|
+
// they are removed.
|
|
148
|
+
let sitePage = null;
|
|
149
|
+
if (config.glossarypage) {
|
|
150
|
+
sitePage = contentCatalog.resolvePage(config.glossarypage);
|
|
151
|
+
if (sitePage) {
|
|
152
|
+
mergeTerms(sitePage);
|
|
153
|
+
logger.info(`Merged terms into the site glossary ${config.glossarypage}.`);
|
|
154
|
+
} else {
|
|
155
|
+
logger.warn(`The glossarypage ${config.glossarypage} does not exist, so each component links to its own reference:glossary.adoc instead.`);
|
|
156
|
+
}
|
|
157
|
+
}
|
|
158
|
+
|
|
159
|
+
contentCatalog.getComponents().forEach(({ versions }) => {
|
|
133
160
|
versions.forEach(({ name: component, version, asciidoc, title }) => {
|
|
134
161
|
if (component == 'shared') return;
|
|
162
|
+
if (sitePage) asciidoc.attributes['glossary-page'] = config.glossarypage;
|
|
135
163
|
|
|
136
164
|
const glossaryPage = contentCatalog.resolvePage(`${version ? version + '@' : ''}${component}:reference:glossary.adoc`);
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
let glossaryContent = glossaryPage.contents.toString('utf8');
|
|
141
|
-
|
|
142
|
-
Object.keys(siteCatalog.termsByCategory).sort().forEach(category => {
|
|
143
|
-
let categoryContent = `\n\n== ${category}\n`;
|
|
144
|
-
|
|
145
|
-
siteCatalog.termsByCategory[category].sort((a, b) => a.name.localeCompare(b.name)).forEach(term => {
|
|
146
|
-
let processedContent = processTermContent(term.content);
|
|
147
|
-
categoryContent += `\n\n${processedContent}`;
|
|
148
|
-
});
|
|
149
|
-
|
|
150
|
-
glossaryContent += categoryContent;
|
|
151
|
-
});
|
|
152
|
-
|
|
153
|
-
glossaryPage.contents = Buffer.from(glossaryContent, 'utf8');
|
|
165
|
+
if (glossaryPage && glossaryPage !== sitePage) {
|
|
166
|
+
if (!sitePage) asciidoc.attributes['glossary-page'] = 'reference:glossary.adoc';
|
|
167
|
+
mergeTerms(glossaryPage);
|
|
154
168
|
logger.info(`Merged terms into glossary for ${component} component${version ? ' version ' + version : ''}.`);
|
|
155
|
-
} else {
|
|
169
|
+
} else if (!glossaryPage && !sitePage) {
|
|
156
170
|
logger.info(`Skipping ${title} ${version ? ' version ' + version : ''} - No glossary page (reference:glossary.adoc) found`);
|
|
157
171
|
}
|
|
158
172
|
});
|
|
@@ -13,29 +13,81 @@ const DEFAULTS = {
|
|
|
13
13
|
githubRepo: 'connect'
|
|
14
14
|
}
|
|
15
15
|
|
|
16
|
-
//
|
|
17
|
-
//
|
|
18
|
-
//
|
|
19
|
-
//
|
|
20
|
-
//
|
|
21
|
-
|
|
16
|
+
// Generated sets that connector pages include, by Antora family. Each one
|
|
17
|
+
// must be present, or pages publish with unresolved includes.
|
|
18
|
+
// The generated partial and example directories that connector pages include,
|
|
19
|
+
// read from the pages themselves, so the guard asks for exactly what the pages
|
|
20
|
+
// in this build use (rp-connect-docs pages gain availability includes, for
|
|
21
|
+
// example, when they move to the generated docs).
|
|
22
|
+
const PARTIAL_INCLUDE = /include::connect:components:partial\$([a-z0-9_-]+)\//g
|
|
23
|
+
const EXAMPLE_INCLUDE = /include::(?:connect:)?components:example\$([a-z0-9_-]+)\//g
|
|
24
|
+
|
|
25
|
+
function includedDirs (pages) {
|
|
26
|
+
const partial = new Set()
|
|
27
|
+
const example = new Set()
|
|
28
|
+
for (const page of pages) {
|
|
29
|
+
const text = page.contents ? page.contents.toString() : ''
|
|
30
|
+
for (const m of text.matchAll(PARTIAL_INCLUDE)) partial.add(m[1])
|
|
31
|
+
for (const m of text.matchAll(EXAMPLE_INCLUDE)) example.add(m[1])
|
|
32
|
+
}
|
|
33
|
+
// Name the core sets first, in a fixed order, so the error reads the same
|
|
34
|
+
// way every time.
|
|
35
|
+
const order = ['fields', 'descriptions', 'availability', 'metadata', 'examples', 'common', 'advanced']
|
|
36
|
+
const rank = (d) => (order.includes(d) ? order.indexOf(d) : order.length)
|
|
37
|
+
const sorted = (set) => [...set].sort((a, b) => rank(a) - rank(b) || a.localeCompare(b))
|
|
38
|
+
return { partial: sorted(partial), example: sorted(example) }
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
// Connector pages include the generated field reference, description meta,
|
|
42
|
+
// availability, and config examples, which the modify-connect-tag-playbook
|
|
43
|
+
// extension adds from the redpanda-connect-docs.tar.gz asset of a connect
|
|
44
|
+
// release. If a playbook has the Connect pages but any set is missing (an
|
|
45
|
+
// incomplete asset, or a connect source whose ref lacks them), connector
|
|
46
|
+
// pages would publish with unresolved includes, and Antora only logs them.
|
|
47
|
+
// Fail the build instead so the playbook gets fixed before anything is
|
|
48
|
+
// published.
|
|
49
|
+
//
|
|
50
|
+
// `connectSources` are the connect content sources still in the playbook,
|
|
51
|
+
// which means no release asset was downloaded.
|
|
52
|
+
function assertConnectReferencePresent (contentCatalog, { connectSources = [] } = {}) {
|
|
22
53
|
const component = contentCatalog.getComponents().find((c) => c.name === 'connect')
|
|
23
54
|
if (!component) return
|
|
24
|
-
// Only connector pages (inputs/kafka.adoc and so on) include
|
|
25
|
-
//
|
|
55
|
+
// Only connector pages (inputs/kafka.adoc and so on) include the generated
|
|
56
|
+
// sets. Overview pages at the module root don't.
|
|
26
57
|
const pages = contentCatalog.findBy({ component: 'connect', module: 'components', family: 'page' })
|
|
27
58
|
.filter((p) => p.src.relative.includes('/'))
|
|
28
59
|
if (!pages.length) return
|
|
29
|
-
const
|
|
30
|
-
.
|
|
31
|
-
|
|
60
|
+
const missingIn = (family, dirs) => {
|
|
61
|
+
const files = contentCatalog.findBy({ component: 'connect', module: 'components', family })
|
|
62
|
+
return dirs.filter((dir) => !files.some((f) => f.src.relative.startsWith(`${dir}/`)))
|
|
63
|
+
.map((dir) => `components:${family}$${dir}/*`)
|
|
64
|
+
}
|
|
65
|
+
const wanted = includedDirs(pages)
|
|
66
|
+
const missing = [...missingIn('partial', wanted.partial), ...missingIn('example', wanted.example)]
|
|
67
|
+
if (!missing.length) return
|
|
68
|
+
const sourceHint = connectSources.length
|
|
69
|
+
? `The playbook lists a connect content source (${connectSources.join(', ')}), so no release asset was downloaded and these files must come from that source's refs. ` +
|
|
70
|
+
'Point it at a ref that has the generated docs, or remove it to use the release asset. '
|
|
71
|
+
: ''
|
|
32
72
|
throw new Error(
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
'
|
|
73
|
+
`The connect component has connector pages but no generated ${missing.join(' or ')} files. ` +
|
|
74
|
+
sourceHint +
|
|
75
|
+
'These come from the redpanda-connect-docs.tar.gz asset of a Redpanda Connect release, which the ' +
|
|
76
|
+
'modify-connect-tag-playbook extension downloads. Register that extension and check its log: ' +
|
|
77
|
+
'the release it used (the latest stable release, or the one in its `tag` config) may have no asset or an incomplete one. ' +
|
|
78
|
+
'Set `tag` to a release that has the asset, or set REDPANDA_CONNECT_DOCS_DIR to a local directory that contains ' +
|
|
79
|
+
'modules/ (for example a connect checkout\'s docs/ after running its docs generator).'
|
|
36
80
|
)
|
|
37
81
|
}
|
|
38
82
|
|
|
83
|
+
// The connect content sources in a playbook, with credentials removed.
|
|
84
|
+
function connectSourcesOf (playbook) {
|
|
85
|
+
const sources = (playbook && playbook.content && playbook.content.sources) || []
|
|
86
|
+
return sources
|
|
87
|
+
.filter((s) => catalogUtil.isConnectSource(s.url))
|
|
88
|
+
.map((s) => String(s.url).replace(/([a-z][a-z+.-]*:\/\/)[^@/\s]*@/gi, '$1'))
|
|
89
|
+
}
|
|
90
|
+
|
|
39
91
|
module.exports.assertConnectReferencePresent = assertConnectReferencePresent
|
|
40
92
|
|
|
41
93
|
module.exports.register = function ({ config }) {
|
|
@@ -63,8 +115,8 @@ module.exports.register = function ({ config }) {
|
|
|
63
115
|
let translatedRows = null
|
|
64
116
|
|
|
65
117
|
// Use 'on' and return the promise so Antora waits for async completion
|
|
66
|
-
this.on('contentClassified', ({ contentCatalog }) => {
|
|
67
|
-
assertConnectReferencePresent(contentCatalog)
|
|
118
|
+
this.on('contentClassified', ({ contentCatalog, playbook }) => {
|
|
119
|
+
assertConnectReferencePresent(contentCatalog, { connectSources: connectSourcesOf(playbook) })
|
|
68
120
|
return processContent(contentCatalog)
|
|
69
121
|
})
|
|
70
122
|
|
|
@@ -132,8 +184,9 @@ module.exports.register = function ({ config }) {
|
|
|
132
184
|
|
|
133
185
|
// Raw catalog rows (info.csv column names) for this build.
|
|
134
186
|
//
|
|
135
|
-
// The generated partials/platforms/catalog.json from the
|
|
136
|
-
// source comes first: it is
|
|
187
|
+
// The generated partials/platforms/catalog.json from connect (the release
|
|
188
|
+
// asset, or a connect content source) comes first: it is from the same ref
|
|
189
|
+
// as the reference content
|
|
137
190
|
// and carries status, categories, and cgo data that info.csv lacks. info.csv
|
|
138
191
|
// still supplies the SQL driver rows, which are not components and so are
|
|
139
192
|
// not in the catalog. Without catalog.json, info.csv supplies every row.
|
|
@@ -152,7 +205,8 @@ module.exports.register = function ({ config }) {
|
|
|
152
205
|
let csvRows = []
|
|
153
206
|
try {
|
|
154
207
|
const csvText = await fetchCSV(localCsvPath, contentCatalog)
|
|
155
|
-
|
|
208
|
+
// connect's info.csv pads every column with spaces, headers included
|
|
209
|
+
csvRows = Papa.parse(csvText, { header: true, skipEmptyLines: true, transformHeader: (h) => h.trim() }).data
|
|
156
210
|
} catch (error) {
|
|
157
211
|
if (!catalogRows) throw error
|
|
158
212
|
logger.warn(`Could not fetch info.csv for the SQL driver rows, so the SQL driver support list is empty: ${error.message}`)
|
|
@@ -186,8 +240,9 @@ module.exports.register = function ({ config }) {
|
|
|
186
240
|
}
|
|
187
241
|
|
|
188
242
|
// The ref to read info.csv from, in order:
|
|
189
|
-
// 1. the tag modify-connect-tag-playbook resolved for the
|
|
190
|
-
//
|
|
243
|
+
// 1. the tag modify-connect-tag-playbook resolved (for the release asset or
|
|
244
|
+
// the connect content source)
|
|
245
|
+
// 2. the ref of the connect files in the content catalog
|
|
191
246
|
// 3. latest-connect-version in antora.yml in the working directory
|
|
192
247
|
// 4. main, with a warning, because the catalog can then disagree with the
|
|
193
248
|
// reference content
|
|
@@ -201,9 +256,9 @@ module.exports.register = function ({ config }) {
|
|
|
201
256
|
const normalizedVersion = connectVersion ? String(connectVersion).trim().replace(/^v/, '') : ''
|
|
202
257
|
if (normalizedVersion) return { ...base, ref: `v${normalizedVersion}`, source: 'latest-connect-version in antora.yml' }
|
|
203
258
|
logger.warn(
|
|
204
|
-
'No connect
|
|
205
|
-
'Catalog badges can then disagree with the reference content.
|
|
206
|
-
'
|
|
259
|
+
'No resolved connect release or latest-connect-version found, so info.csv is read from connect main. ' +
|
|
260
|
+
'Catalog badges can then disagree with the reference content. Register the modify-connect-tag-playbook ' +
|
|
261
|
+
'extension, or set its `tag` config when REDPANDA_CONNECT_DOCS_DIR is set.'
|
|
207
262
|
)
|
|
208
263
|
return { ...base, ref: 'main', source: 'fallback' }
|
|
209
264
|
}
|