@redpanda-data/docs-extensions-and-macros 5.54.0 → 5.56.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/doc-tools-mcp.js +4 -5
- package/bin/doc-tools.js +354 -9
- package/bin/mcp-tools/generated-docs-review.js +26 -9
- package/bin/mcp-tools/overrides-audit.js +1 -1
- package/bin/mcp-tools/rpcn-docs.js +4 -14
- package/cli-utils/docs-links-to-xrefs.js +174 -0
- package/cli-utils/format-helm-spec.js +4 -3
- package/cli-utils/generated-marker.js +43 -0
- package/cli-utils/helm-commented-values.js +125 -21
- package/cli-utils/self-managed-docs-branch.js +36 -30
- package/extension-utils/html-utils.js +27 -0
- package/extension-utils/llms-utils.js +6 -2
- package/extension-utils/metadata-candidates.js +304 -0
- package/extension-utils/metadata-check.js +174 -0
- package/extension-utils/metadata-suggest.js +573 -0
- package/extension-utils/taxonomy-categories.js +106 -0
- package/extension-utils/taxonomy.js +566 -0
- package/extensions/REFERENCE.adoc +110 -8
- package/extensions/add-git-dates.js +13 -1
- package/extensions/aggregate-terms.js +34 -20
- package/extensions/algolia-indexer/generate-index.js +13 -1
- package/extensions/algolia-indexer/index.js +4 -2
- package/extensions/convert-sitemap-to-markdown.js +47 -12
- package/extensions/convert-to-markdown.js +176 -0
- package/extensions/generate-fields-only-pages.js +14 -0
- package/extensions/modify-connect-tag-playbook.js +181 -3
- package/extensions/resolve-xrefs-in-attachments.js +1 -12
- package/extensions/url-to-xref.js +92 -22
- package/extensions/util/connect-docs-asset.js +6 -5
- package/extensions/validate-attributes.js +10 -1
- package/extensions/version-fetcher/fetch-latest-docker-tag.js +35 -13
- package/extensions/version-fetcher/get-latest-redpanda-helm-version-from-operator.js +64 -40
- package/extensions/version-fetcher/list-connect-releases.js +3 -3
- package/extensions/version-fetcher/release-lines.js +64 -0
- package/extensions/version-fetcher/set-latest-version.js +73 -3
- package/macros/README.adoc +1 -0
- package/macros/REFERENCE.adoc +99 -0
- package/macros/badge.js +5 -8
- package/macros/enterprise.js +2 -5
- package/macros/glossary.js +4 -14
- package/macros/iceberg-explorer.js +242 -0
- package/macros/prop.js +3 -13
- package/mcp/USER_GUIDE.adoc +25 -135
- package/package.json +5 -2
- package/tools/check-metadata/index.js +523 -0
- package/tools/cloud-regions/cloud-regions-table-adoc.hbs +1 -2
- package/tools/cloud-regions/cloud-regions-table-md.hbs +2 -2
- package/tools/cloud-regions/render-cloud-regions.js +3 -0
- package/tools/dispatch-graph/check-dispatch-graph.js +790 -0
- package/tools/dispatch-graph/classify-runs.js +128 -0
- package/tools/dispatch-graph/parse-workflows.js +499 -0
- package/tools/doc-impact-recording/index.js +190 -0
- package/tools/enterprise-features/detect-gating-changes.js +393 -0
- package/tools/enterprise-features/verify.js +2 -1
- package/tools/gen-rpk-ascii.py +7 -1
- package/tools/lint-strings/fixtures/connect/lint_connect.go +8 -0
- package/tools/lint-strings/fixtures/crd/lint_types.go +4 -1
- package/tools/lint-strings/fixtures/rpk-pr461/delete.base.go +42 -0
- package/tools/lint-strings/fixtures/rpk-pr461/delete.head.go +45 -0
- package/tools/lint-strings/fixtures/rpk-pr461/reviewer-comments.json +1 -0
- package/tools/lint-strings/index.js +133 -3
- package/tools/lint-strings/rules/verbatim-asciidoc.js +166 -1
- package/tools/lint-strings/surfaces/connect.js +72 -12
- package/tools/lint-strings/surfaces/crd.js +10 -6
- package/tools/metrics/metrics.py +11 -0
- package/tools/overrides-audit/README.adoc +1 -2
- package/tools/overrides-audit/adapters/properties.js +1 -1
- package/tools/overrides-audit/index.js +3 -4
- package/tools/property-extractor/generate-handlebars-docs.js +9 -1
- package/tools/property-extractor/helpers/applyPropertyLinks.js +35 -0
- package/tools/redpanda-connect/AUTOMATION.md +3 -36
- package/tools/redpanda-connect/README.adoc +5 -33
- package/tools/redpanda-connect/generate-rpcn-connector-docs.js +32 -11
- package/tools/redpanda-connect/pr-summary-formatter.js +6 -4
- package/tools/redpanda-connect/report-delta.js +6 -3
- package/tools/redpanda-connect/rpcn-connector-docs-handler.js +42 -25
- package/tools/redpanda-connect/templates/bloblang-function.hbs +1 -1
- package/tools/redpanda-connect/templates/descriptions-partials.hbs +1 -1
- package/tools/redpanda-connect/templates/examples-partials.hbs +1 -1
- package/tools/redpanda-connect/templates/fields-partials.hbs +2 -2
- package/tools/redpanda-connect/templates/metadata-partials.hbs +1 -1
- package/tools/redpanda-release-notes/generate-release-notes.js +116 -1
- package/tools/rpk-docs/generate-rpk-docs.js +2 -0
- package/tools/rpk-docs/generate-x-env-partial.js +7 -4
- package/tools/rpk-docs/helpers/index.js +4 -1
- package/tools/rpk-docs/templates/command.hbs +2 -2
- package/tools/overrides-audit/adapters/connect.js +0 -144
|
@@ -35,12 +35,23 @@ const { raiseListenerLimit } = require('./util/raise-listener-limit')
|
|
|
35
35
|
* generator wrapped across a line break. An unlabeled URL becomes xref:...[]
|
|
36
36
|
* so Antora fills in the target page title, except when it carries a
|
|
37
37
|
* fragment, where link text has to be supplied (see linkTextFor).
|
|
38
|
+
* - A URL with a fragment is converted only when the anchor exists on the
|
|
39
|
+
* target page: a section id, an explicit [[id]], [#id] or anchor:id[]
|
|
40
|
+
* anchor, or one of those in a partial the page includes (property
|
|
41
|
+
* headings live in generated partials). A URL whose page resolves but whose
|
|
42
|
+
* anchor does not stays a URL and is reported as a broken anchor, since an
|
|
43
|
+
* xref to it would pass Antora's page check and still land at the top of
|
|
44
|
+
* the page.
|
|
38
45
|
* - Targets in the latest version of a component emit an unversioned xref;
|
|
39
46
|
* targets pinned to an older version emit version@component:module:page.
|
|
40
47
|
* - URLs pointing at a page's former path resolve through the page-aliases the
|
|
41
48
|
* page declares (see buildUrlMap).
|
|
42
49
|
* - URLs carrying a query string are left raw, because an xref cannot express
|
|
43
50
|
* one; they are reported at info level rather than as warnings.
|
|
51
|
+
* - A resolver context may name a localComponent (see
|
|
52
|
+
* cli-utils/docs-links-to-xrefs.js, which converts generated pages before
|
|
53
|
+
* they are committed): targets in that component are written as
|
|
54
|
+
* module:page xrefs, the form the docs repos use for same-component links.
|
|
44
55
|
*/
|
|
45
56
|
|
|
46
57
|
// Rewrites for URL shapes that predate the current docs.redpanda.com site
|
|
@@ -80,6 +91,7 @@ module.exports.register = function ({ config = {} }) {
|
|
|
80
91
|
const unmapped = new Map()
|
|
81
92
|
const withQueryString = new Set()
|
|
82
93
|
const withoutLinkText = new Set()
|
|
94
|
+
const missingAnchor = new Map()
|
|
83
95
|
const files = contentCatalog
|
|
84
96
|
.getPages((page) => page.out)
|
|
85
97
|
.concat(contentCatalog.findBy({ family: 'partial' }))
|
|
@@ -96,11 +108,18 @@ module.exports.register = function ({ config = {} }) {
|
|
|
96
108
|
}
|
|
97
109
|
for (const url of result.withQueryString) withQueryString.add(url)
|
|
98
110
|
for (const url of result.withoutLinkText) withoutLinkText.add(url)
|
|
111
|
+
for (const url of result.missingAnchor) {
|
|
112
|
+
if (!missingAnchor.has(url)) missingAnchor.set(url, new Set())
|
|
113
|
+
missingAnchor.get(url).add(file.path)
|
|
114
|
+
}
|
|
99
115
|
}
|
|
100
116
|
if (logUnconverted) {
|
|
101
117
|
for (const [url, pages] of unmapped) {
|
|
102
118
|
logger.warn(`No published page matches ${url} (found in: ${[...pages].join(', ')})`)
|
|
103
119
|
}
|
|
120
|
+
for (const [url, pages] of missingAnchor) {
|
|
121
|
+
logger.warn(`Anchor not found on the target page of ${url}; left as a URL (found in: ${[...pages].join(', ')})`)
|
|
122
|
+
}
|
|
104
123
|
}
|
|
105
124
|
if (withQueryString.size) {
|
|
106
125
|
logger.info(
|
|
@@ -111,7 +130,7 @@ module.exports.register = function ({ config = {} }) {
|
|
|
111
130
|
if (withoutLinkText.size) {
|
|
112
131
|
logger.info(
|
|
113
132
|
`Left ${withoutLinkText.size} docs URL${withoutLinkText.size === 1 ? '' : 's'} with a fragment as raw ` +
|
|
114
|
-
`link${withoutLinkText.size === 1 ? '' : 's'} (
|
|
133
|
+
`link${withoutLinkText.size === 1 ? '' : 's'} (the target page could not be read to check the fragment): ` +
|
|
115
134
|
`${[...withoutLinkText].join(', ')}`
|
|
116
135
|
)
|
|
117
136
|
}
|
|
@@ -335,31 +354,49 @@ function resolveUrlPath (urlPath, resolverContext) {
|
|
|
335
354
|
return undefined
|
|
336
355
|
}
|
|
337
356
|
|
|
338
|
-
function entryToXref (entry, fragment, label) {
|
|
357
|
+
function entryToXref (entry, fragment, label, localComponent) {
|
|
339
358
|
const versionPrefix = entry.latest || !entry.version ? '' : `${entry.version}@`
|
|
340
|
-
|
|
359
|
+
// Content that is itself part of a component (a page a generator writes into
|
|
360
|
+
// the docs repo) names a target in that component by module and page only,
|
|
361
|
+
// the way the docs repos write xrefs by hand. A version-pinned xref still
|
|
362
|
+
// has to name its component.
|
|
363
|
+
const componentPrefix = !versionPrefix && localComponent && entry.component === localComponent ? '' : `${entry.component}:`
|
|
364
|
+
return `xref:${versionPrefix}${componentPrefix}${entry.module}:${entry.relative}${fragment || ''}[${label || ''}]`
|
|
341
365
|
}
|
|
342
366
|
|
|
343
367
|
// Parsed heading id -> heading text, cached per target file.
|
|
344
368
|
const headingsByFile = new WeakMap()
|
|
345
369
|
const EXPLICIT_ANCHOR_PATTERNS = [
|
|
346
370
|
/^\[\[([\w:.-]+)(?:,.*)?\]\]$/,
|
|
347
|
-
/^\[#([\w
|
|
371
|
+
/^\[#([\w:-]+)(?:[,.%].*)?\]$/,
|
|
348
372
|
/^\[id="?([\w:.-]+)"?(?:,.*)?\]$/,
|
|
349
373
|
]
|
|
350
374
|
const HEADING_RX = /^(={1,6})\s+(\S.*)$/
|
|
375
|
+
const INLINE_ANCHOR_RX = /\[\[([\w:.-]+)(?:,[^\]]*)?\]\]|\banchor:([\w:.-]+)\[/g
|
|
376
|
+
|
|
377
|
+
/**
|
|
378
|
+
* The anchors a page publishes, id -> heading text (undefined for an anchor
|
|
379
|
+
* on something other than a heading), or undefined when the page has no
|
|
380
|
+
* contents to read. Cached per page.
|
|
381
|
+
*/
|
|
382
|
+
function pageAnchors (page, resolveInclude) {
|
|
383
|
+
const contents = page && page.contents && page.contents.toString()
|
|
384
|
+
if (!contents) return undefined
|
|
385
|
+
let anchors = headingsByFile.get(page)
|
|
386
|
+
if (!anchors) headingsByFile.set(page, (anchors = pageHeadingIds(contents, page, resolveInclude)))
|
|
387
|
+
return anchors
|
|
388
|
+
}
|
|
351
389
|
|
|
352
390
|
/**
|
|
353
391
|
* Returns the link text for a converted URL, or undefined when the URL should
|
|
354
|
-
* be left alone.
|
|
392
|
+
* be left alone. Callers check the fragment against pageAnchors first.
|
|
355
393
|
*
|
|
356
394
|
* A labeled URL keeps its label, and an unlabeled URL without a fragment gets
|
|
357
395
|
* empty text so that Antora fills in the target page title. An unlabeled URL
|
|
358
396
|
* *with* a fragment has to be given text explicitly: Antora cannot resolve a
|
|
359
397
|
* section title, so `xref:page.adoc#anchor[]` renders the raw resource id as
|
|
360
398
|
* the link text. Prefer the heading the fragment points at, then the page
|
|
361
|
-
* title
|
|
362
|
-
* which still displays sensibly.
|
|
399
|
+
* title (an anchor on a paragraph or table has no heading of its own).
|
|
363
400
|
*/
|
|
364
401
|
function linkTextFor (label, fragment, page, resolveInclude) {
|
|
365
402
|
// A trailing ^ opens a URL link in a new window; an xref would print it.
|
|
@@ -367,11 +404,19 @@ function linkTextFor (label, fragment, page, resolveInclude) {
|
|
|
367
404
|
label = (label || '').replace(/\^$/, '')
|
|
368
405
|
if (label) return label
|
|
369
406
|
if (!fragment) return ''
|
|
370
|
-
const
|
|
371
|
-
if (!
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
|
|
407
|
+
const anchors = pageAnchors(page, resolveInclude)
|
|
408
|
+
if (!anchors) return undefined
|
|
409
|
+
return anchors.get(fragmentId(fragment)) || documentTitle(page.contents.toString())
|
|
410
|
+
}
|
|
411
|
+
|
|
412
|
+
/** The anchor id of a URL fragment (`#a%2Fb` is `a/b`), without the `#`. */
|
|
413
|
+
function fragmentId (fragment) {
|
|
414
|
+
const raw = fragment.slice(1)
|
|
415
|
+
try {
|
|
416
|
+
return decodeURIComponent(raw)
|
|
417
|
+
} catch {
|
|
418
|
+
return raw
|
|
419
|
+
}
|
|
375
420
|
}
|
|
376
421
|
|
|
377
422
|
// Only the first level of includes is followed, and only this many per page:
|
|
@@ -380,7 +425,7 @@ function linkTextFor (label, fragment, page, resolveInclude) {
|
|
|
380
425
|
const MAX_INCLUDES_SEARCHED = 50
|
|
381
426
|
|
|
382
427
|
/**
|
|
383
|
-
* Collects the
|
|
428
|
+
* Collects the anchor ids of a page plus those of the partials it includes.
|
|
384
429
|
* The reference pages that carry most anchor targets are thin wrappers around
|
|
385
430
|
* generated partials — every property heading on the cluster and broker
|
|
386
431
|
* property pages lives in one — and Antora has not resolved includes yet at
|
|
@@ -407,10 +452,14 @@ function pageHeadingIds (contents, page, resolveInclude) {
|
|
|
407
452
|
}
|
|
408
453
|
|
|
409
454
|
/**
|
|
410
|
-
* Maps every
|
|
411
|
-
* ([[id]], [#id],
|
|
412
|
-
*
|
|
413
|
-
*
|
|
455
|
+
* Maps every anchor id in a page to its heading text: the ids Asciidoctor
|
|
456
|
+
* generates from heading text, explicit block anchors ([[id]], [#id],
|
|
457
|
+
* [id=...], on a heading or any other block) and inline anchors ([[id]] or
|
|
458
|
+
* anchor:id[] inside a line). An anchor that is not on a heading maps to
|
|
459
|
+
* undefined. Id generation is approximated with the docs sites' settings
|
|
460
|
+
* (idprefix empty, idseparator -); a heading whose id comes out differently
|
|
461
|
+
* is reported as a missing anchor and its URL is left as written, which
|
|
462
|
+
* still works.
|
|
414
463
|
*/
|
|
415
464
|
function parseHeadingIds (contents) {
|
|
416
465
|
const ids = new Map()
|
|
@@ -420,13 +469,18 @@ function parseHeadingIds (contents) {
|
|
|
420
469
|
const anchor = EXPLICIT_ANCHOR_PATTERNS.reduce((found, pattern) => found || pattern.exec(line), null)
|
|
421
470
|
if (anchor) {
|
|
422
471
|
pendingId = anchor[1]
|
|
472
|
+
if (!ids.has(pendingId)) ids.set(pendingId, undefined)
|
|
423
473
|
continue
|
|
424
474
|
}
|
|
475
|
+
for (const inline of line.matchAll(INLINE_ANCHOR_RX)) {
|
|
476
|
+
const id = inline[1] || inline[2]
|
|
477
|
+
if (!ids.has(id)) ids.set(id, undefined)
|
|
478
|
+
}
|
|
425
479
|
const heading = HEADING_RX.exec(line)
|
|
426
480
|
if (heading) {
|
|
427
481
|
const text = sanitizeLinkText(heading[2])
|
|
428
482
|
const id = pendingId || autoId(text)
|
|
429
|
-
if (id && text &&
|
|
483
|
+
if (id && text && ids.get(id) === undefined) ids.set(id, text)
|
|
430
484
|
}
|
|
431
485
|
if (line) pendingId = undefined
|
|
432
486
|
}
|
|
@@ -467,8 +521,9 @@ function autoId (text) {
|
|
|
467
521
|
* Rewrites every mappable docs URL in content. Returns the updated content,
|
|
468
522
|
* the number of conversions, the internal URLs that matched no published page,
|
|
469
523
|
* the internal URLs that carry a query string (which an xref cannot express),
|
|
470
|
-
*
|
|
471
|
-
*
|
|
524
|
+
* the internal URLs left alone because their target page could not be read to
|
|
525
|
+
* check the fragment, and the internal URLs whose page resolves but whose
|
|
526
|
+
* anchor is not on it.
|
|
472
527
|
*/
|
|
473
528
|
function convertContent (content, resolverContext) {
|
|
474
529
|
const { hostnames, ignore = [] } = resolverContext
|
|
@@ -478,6 +533,7 @@ function convertContent (content, resolverContext) {
|
|
|
478
533
|
const unmapped = []
|
|
479
534
|
const withQueryString = []
|
|
480
535
|
const withoutLinkText = []
|
|
536
|
+
const missingAnchor = []
|
|
481
537
|
for (const match of scanContentUrls(content)) {
|
|
482
538
|
let url
|
|
483
539
|
try {
|
|
@@ -505,17 +561,31 @@ function convertContent (content, resolverContext) {
|
|
|
505
561
|
unmapped.push(match.url)
|
|
506
562
|
continue
|
|
507
563
|
}
|
|
564
|
+
if (url.hash) {
|
|
565
|
+
const anchors = pageAnchors(entry.page, resolverContext.resolveInclude)
|
|
566
|
+
if (!anchors) {
|
|
567
|
+
withoutLinkText.push(match.url)
|
|
568
|
+
continue
|
|
569
|
+
}
|
|
570
|
+
// The page resolves but the anchor does not: an xref would pass
|
|
571
|
+
// Antora's check and still land at the top of the page. Keep the URL,
|
|
572
|
+
// which the site's broken-anchor checks can still see, and report it.
|
|
573
|
+
if (!anchors.has(fragmentId(url.hash))) {
|
|
574
|
+
missingAnchor.push(match.url)
|
|
575
|
+
continue
|
|
576
|
+
}
|
|
577
|
+
}
|
|
508
578
|
const label = linkTextFor(match.label, url.hash, entry.page, resolverContext.resolveInclude)
|
|
509
579
|
if (label === undefined) {
|
|
510
580
|
withoutLinkText.push(match.url)
|
|
511
581
|
continue
|
|
512
582
|
}
|
|
513
|
-
result += content.slice(cursor, match.start) + entryToXref(entry, url.hash, label)
|
|
583
|
+
result += content.slice(cursor, match.start) + entryToXref(entry, url.hash, label, resolverContext.localComponent)
|
|
514
584
|
cursor = match.end
|
|
515
585
|
converted++
|
|
516
586
|
}
|
|
517
587
|
result += content.slice(cursor)
|
|
518
|
-
return { content: result, converted, unmapped, withQueryString, withoutLinkText }
|
|
588
|
+
return { content: result, converted, unmapped, withQueryString, withoutLinkText, missingAnchor }
|
|
519
589
|
}
|
|
520
590
|
|
|
521
591
|
module.exports.buildUrlMap = buildUrlMap
|
|
@@ -5,8 +5,9 @@
|
|
|
5
5
|
// Every connect release (vX.Y.Z, including prereleases) publishes a
|
|
6
6
|
// redpanda-connect-docs.tar.gz asset whose root is `modules/`: the tree that
|
|
7
7
|
// connect's docs generator writes under its docs/ directory. Only
|
|
8
|
-
// modules/components/partials
|
|
9
|
-
//
|
|
8
|
+
// modules/components/partials/**, modules/components/examples/**, and
|
|
9
|
+
// modules/components/attachments/** (the release's connect-<version>.json)
|
|
10
|
+
// belong to the reference. Releases from before the asset existed answer 404.
|
|
10
11
|
//
|
|
11
12
|
// The same tree can come from a local directory instead (a connect checkout's
|
|
12
13
|
// docs/ after running its generator), or from a local copy of the tarball.
|
|
@@ -19,7 +20,7 @@ const OWNER = 'redpanda-data'
|
|
|
19
20
|
const REPO = 'connect'
|
|
20
21
|
const ASSET_NAME = 'redpanda-connect-docs.tar.gz'
|
|
21
22
|
const MODULE = 'components'
|
|
22
|
-
const FAMILY_DIRS = { partials: 'partial', examples: 'example' }
|
|
23
|
+
const FAMILY_DIRS = { partials: 'partial', examples: 'example', attachments: 'attachment' }
|
|
23
24
|
const DOWNLOAD_ATTEMPTS = 3
|
|
24
25
|
const DOWNLOAD_TIMEOUT_MS = 120000
|
|
25
26
|
|
|
@@ -28,10 +29,10 @@ function assetUrl (tag) {
|
|
|
28
29
|
}
|
|
29
30
|
|
|
30
31
|
// The Antora resource of a path inside the tree, or null for anything outside
|
|
31
|
-
// modules/components/partials and
|
|
32
|
+
// modules/components/partials, examples, and attachments.
|
|
32
33
|
function toResource (entryPath) {
|
|
33
34
|
const p = String(entryPath || '').replace(/\\/g, '/').replace(/^(\.\/)+/, '')
|
|
34
|
-
const m = p.match(/^modules\/components\/(partials|examples)\/(.+)$/)
|
|
35
|
+
const m = p.match(/^modules\/components\/(partials|examples|attachments)\/(.+)$/)
|
|
35
36
|
if (!m || m[2].endsWith('/')) return null
|
|
36
37
|
// A path that climbs out of its directory is not part of the tree.
|
|
37
38
|
if (m[2].split('/').some((segment) => segment === '..' || segment === '')) return null
|
|
@@ -7,6 +7,8 @@
|
|
|
7
7
|
'use strict';
|
|
8
8
|
|
|
9
9
|
const { raiseListenerLimit } = require('./util/raise-listener-limit')
|
|
10
|
+
const { getTaxonomy } = require('../extension-utils/taxonomy')
|
|
11
|
+
const { checkTaxonomy, createTaxonomyReport, logTaxonomyReport } = require('../extension-utils/taxonomy-categories')
|
|
10
12
|
|
|
11
13
|
module.exports.register = function ({ config }) {
|
|
12
14
|
raiseListenerLimit(this)
|
|
@@ -18,15 +20,22 @@ module.exports.register = function ({ config }) {
|
|
|
18
20
|
const validCategories = siteCatalog.attributeFile['page-valid-categories'];
|
|
19
21
|
if (!validCategories) return logger.warn('No page-valid-categories attribute found - skipping attribute validation')
|
|
20
22
|
const categoryMap = createCategoryMap(validCategories);
|
|
23
|
+
// The taxonomy (shared/modules/ROOT/attachments/taxonomy.yml) when the
|
|
24
|
+
// docs repo has it; null keeps the valid-categories-only behavior.
|
|
25
|
+
const taxonomy = getTaxonomy({ contentCatalog, siteCatalog, logger });
|
|
26
|
+
const report = createTaxonomyReport();
|
|
21
27
|
const pages = contentCatalog.findBy({ family: 'page' });
|
|
22
28
|
pages.forEach((page) => {
|
|
23
29
|
let pageCategories = page.asciidoc.attributes['page-categories'];
|
|
24
30
|
if (!pageCategories) return;
|
|
25
31
|
let pageCategoryList = pageCategories.split(',').map(c => c.trim());
|
|
26
|
-
const
|
|
32
|
+
const pageInfo = page.asciidoc.attributes['page-relative-src-path'];
|
|
33
|
+
if (taxonomy) pageCategoryList = checkTaxonomy(pageCategoryList, { taxonomy, categoryMap, page, report });
|
|
34
|
+
const validatedCategories = validateCategories(pageCategoryList, pageInfo, categoryMap, logger);
|
|
27
35
|
page.asciidoc.attributes['page-categories'] = validatedCategories
|
|
28
36
|
processEnvironmentAttributes(page, logger);
|
|
29
37
|
})
|
|
38
|
+
if (taxonomy) logTaxonomyReport(report, logger);
|
|
30
39
|
})
|
|
31
40
|
}
|
|
32
41
|
|
|
@@ -4,30 +4,50 @@
|
|
|
4
4
|
* The function separates tags into stable (tags not including "-beta") and beta (tags including "-beta"),
|
|
5
5
|
* sorts each group by their major.minor version in descending order, and returns the top tag from each.
|
|
6
6
|
*
|
|
7
|
+
* It also returns every stable tag it saw (`stableReleases`), so callers can
|
|
8
|
+
* pick the newest release of an older line, not just the newest overall.
|
|
9
|
+
* Docker Hub lists the most recently pushed tags first, so the first page
|
|
10
|
+
* already holds every line that still gets patches. Pass `maxPages` to reach
|
|
11
|
+
* further back.
|
|
12
|
+
*
|
|
7
13
|
* @param {string} dockerNamespace - The Docker Hub namespace (organization or username)
|
|
8
14
|
* @param {string} dockerRepo - The repository name on Docker Hub
|
|
9
15
|
* @param {Object} logger - Optional Antora logger instance for logging
|
|
10
|
-
* @
|
|
16
|
+
* @param {{ maxPages?: number, fetch?: Function }} options - How many pages of 100 tags to read (default 1), and an optional fetch implementation for tests
|
|
17
|
+
* @returns {Promise<{ latestStableRelease: string|null, latestBetaRelease: string|null, stableReleases: string[] }>}
|
|
11
18
|
*/
|
|
12
|
-
module.exports = async (dockerNamespace, dockerRepo, logger = null) => {
|
|
13
|
-
const
|
|
19
|
+
module.exports = async (dockerNamespace, dockerRepo, logger = null, { maxPages = 1, fetch: fetchImpl } = {}) => {
|
|
20
|
+
const fetch = fetchImpl || (await import('node-fetch')).default;
|
|
14
21
|
|
|
15
22
|
try {
|
|
16
|
-
// Fetch a list of tags from Docker Hub.
|
|
17
|
-
|
|
18
|
-
const
|
|
23
|
+
// Fetch a list of tags from Docker Hub. Only the first page is required:
|
|
24
|
+
// a later page that fails costs older lines, not the latest release.
|
|
25
|
+
const results = [];
|
|
26
|
+
let url = `https://hub.docker.com/v2/repositories/${dockerNamespace}/${dockerRepo}/tags?page_size=100`;
|
|
27
|
+
for (let page = 1; url && page <= maxPages; page++) {
|
|
28
|
+
const response = await fetch(url);
|
|
19
29
|
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
30
|
+
if (!response.ok) {
|
|
31
|
+
const message = `Docker Hub API responded with status ${response.status}`;
|
|
32
|
+
if (page === 1) throw new Error(message);
|
|
33
|
+
if (logger) {
|
|
34
|
+
logger.warn(`${message} for page ${page} of ${dockerRepo} tags; using the first ${results.length} tags.`);
|
|
35
|
+
} else {
|
|
36
|
+
console.warn(`${message} for page ${page} of ${dockerRepo} tags; using the first ${results.length} tags.`);
|
|
37
|
+
}
|
|
38
|
+
break;
|
|
39
|
+
}
|
|
23
40
|
|
|
24
|
-
|
|
41
|
+
const data = await response.json();
|
|
42
|
+
results.push(...(data.results || []));
|
|
43
|
+
url = data.next;
|
|
44
|
+
}
|
|
25
45
|
|
|
26
46
|
// Regex to capture major and minor version numbers (e.g. "v2.3")
|
|
27
47
|
const versionRegex = /^v(\d+)\.(\d+)/;
|
|
28
48
|
|
|
29
49
|
// Filter tags to include only those matching the version pattern.
|
|
30
|
-
let tags =
|
|
50
|
+
let tags = results.filter(tag => versionRegex.test(tag.name));
|
|
31
51
|
|
|
32
52
|
// For specific repositories (e.g. "redpanda-operator"), you might want to filter out certain versions.
|
|
33
53
|
if (dockerRepo === 'redpanda-operator') {
|
|
@@ -61,7 +81,8 @@ module.exports = async (dockerNamespace, dockerRepo, logger = null) => {
|
|
|
61
81
|
|
|
62
82
|
return {
|
|
63
83
|
latestStableRelease: latestStableReleaseVersion || null,
|
|
64
|
-
latestBetaRelease: latestBetaReleaseVersion || null
|
|
84
|
+
latestBetaRelease: latestBetaReleaseVersion || null,
|
|
85
|
+
stableReleases: sortedStable.map(tag => tag.name)
|
|
65
86
|
};
|
|
66
87
|
|
|
67
88
|
} catch (error) {
|
|
@@ -72,7 +93,8 @@ module.exports = async (dockerNamespace, dockerRepo, logger = null) => {
|
|
|
72
93
|
}
|
|
73
94
|
return {
|
|
74
95
|
latestStableRelease: null,
|
|
75
|
-
latestBetaRelease: null
|
|
96
|
+
latestBetaRelease: null,
|
|
97
|
+
stableReleases: []
|
|
76
98
|
};
|
|
77
99
|
}
|
|
78
100
|
};
|
|
@@ -1,7 +1,17 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* Fetches the latest Helm chart version
|
|
3
|
-
*
|
|
4
|
-
*
|
|
2
|
+
* Fetches the latest Helm chart version for an operator release line from the
|
|
3
|
+
* redpanda-operator repository.
|
|
4
|
+
*
|
|
5
|
+
* A stable chart version comes from the newest stable `charts/redpanda/v<major>.<minor>.*`
|
|
6
|
+
* tag, because a tag marks a chart that was actually published (what
|
|
7
|
+
* `helm search repo` reports). A release branch's Chart.yaml can be bumped
|
|
8
|
+
* ahead of the published chart, so it is only the fallback, and the only
|
|
9
|
+
* source for beta versions.
|
|
10
|
+
*
|
|
11
|
+
* The chart has lived at more than one path over time (for example,
|
|
12
|
+
* `charts/redpanda/Chart.yaml` on release/v25.1.x and
|
|
13
|
+
* `charts/redpanda/chart/Chart.yaml` from release/v25.2.x on), so the fallback
|
|
14
|
+
* tries each known path in order.
|
|
5
15
|
*
|
|
6
16
|
* @param {object} github - The GitHub API client.
|
|
7
17
|
* @param {string} owner - The repository owner.
|
|
@@ -13,7 +23,50 @@
|
|
|
13
23
|
*/
|
|
14
24
|
module.exports = async (github, owner, repo, stableDockerTag, betaDockerTag, logger = null) => {
|
|
15
25
|
const yaml = require('js-yaml');
|
|
16
|
-
const
|
|
26
|
+
const semver = require('semver');
|
|
27
|
+
const chartPaths = ['charts/redpanda/chart/Chart.yaml', 'charts/redpanda/Chart.yaml'];
|
|
28
|
+
const warn = (message) => (logger ? logger.warn(message) : console.warn(message));
|
|
29
|
+
const logError = (message) => (logger ? logger.error(message) : console.error(message));
|
|
30
|
+
|
|
31
|
+
// Returns the newest stable published chart version of a release line, or
|
|
32
|
+
// null when the line has no stable chart tag or the lookup fails.
|
|
33
|
+
const getChartVersionFromTags = async (major, minor) => {
|
|
34
|
+
if (!github.git?.listMatchingRefs) return null;
|
|
35
|
+
const tagPrefix = 'charts/redpanda/v';
|
|
36
|
+
const prefix = `${tagPrefix}${major}.${minor}.`;
|
|
37
|
+
try {
|
|
38
|
+
const { data } = await github.git.listMatchingRefs({ owner, repo, ref: `tags/${prefix}` });
|
|
39
|
+
let newest = null;
|
|
40
|
+
for (const { ref } of data || []) {
|
|
41
|
+
const version = semver.valid(String(ref).slice(`refs/tags/${tagPrefix}`.length));
|
|
42
|
+
if (!version || semver.prerelease(version)) continue;
|
|
43
|
+
if (semver.major(version) !== Number(major) || semver.minor(version) !== Number(minor)) continue;
|
|
44
|
+
if (!newest || semver.gt(version, newest)) newest = version;
|
|
45
|
+
}
|
|
46
|
+
return newest;
|
|
47
|
+
} catch (error) {
|
|
48
|
+
warn(`Could not list ${prefix}* tags in ${owner}/${repo}: ${error.message || error}`);
|
|
49
|
+
return null;
|
|
50
|
+
}
|
|
51
|
+
};
|
|
52
|
+
|
|
53
|
+
// Returns the chart version from the first known Chart.yaml path on a
|
|
54
|
+
// branch, or null when none of the paths exist there.
|
|
55
|
+
const getChartVersionFromBranch = async (dockerTag, branchName) => {
|
|
56
|
+
for (const path of chartPaths) {
|
|
57
|
+
try {
|
|
58
|
+
const contentResponse = await github.repos.getContent({ owner, repo, path, ref: branchName });
|
|
59
|
+
const contentDecoded = Buffer.from(contentResponse.data.content, 'base64').toString('utf8');
|
|
60
|
+
return yaml.load(contentDecoded)?.version || null;
|
|
61
|
+
} catch (error) {
|
|
62
|
+
if (error.status === 404) continue;
|
|
63
|
+
logError(`Failed to fetch ${path} from branch ${branchName}: ${error.message || error}`);
|
|
64
|
+
return null;
|
|
65
|
+
}
|
|
66
|
+
}
|
|
67
|
+
warn(`Fetched operator ${dockerTag} from Docker Hub but found no Helm chart for it: no ${chartPaths.join(' or ')} on branch ${branchName} (the branch may not exist yet)`);
|
|
68
|
+
return null;
|
|
69
|
+
};
|
|
17
70
|
|
|
18
71
|
/**
|
|
19
72
|
* Helper function to fetch chart version from a branch derived from a docker tag
|
|
@@ -38,44 +91,15 @@ module.exports = async (github, owner, repo, stableDockerTag, betaDockerTag, log
|
|
|
38
91
|
return null;
|
|
39
92
|
}
|
|
40
93
|
|
|
41
|
-
const major = versionMatch
|
|
42
|
-
const
|
|
43
|
-
|
|
44
|
-
// Convert to release/v<major>.<minor>.x format
|
|
45
|
-
const branchName = `release/v${major}.${minor}.x`;
|
|
46
|
-
|
|
47
|
-
try {
|
|
48
|
-
|
|
49
|
-
// Fetch Chart.yaml from the specified branch
|
|
50
|
-
const contentResponse = await github.repos.getContent({
|
|
51
|
-
owner,
|
|
52
|
-
repo,
|
|
53
|
-
path,
|
|
54
|
-
ref: branchName,
|
|
55
|
-
});
|
|
94
|
+
const [, major, minor] = versionMatch;
|
|
95
|
+
const isBeta = /-beta/.test(dockerTag);
|
|
56
96
|
|
|
57
|
-
|
|
58
|
-
const
|
|
59
|
-
|
|
60
|
-
const version = chartYaml.version || null;
|
|
61
|
-
return version;
|
|
62
|
-
} catch (error) {
|
|
63
|
-
if (error.status === 404) {
|
|
64
|
-
// Branch doesn't exist yet - this is expected for unreleased versions
|
|
65
|
-
if (logger) {
|
|
66
|
-
logger.warn(`Fetched operator ${dockerTag} from Docker Hub but branch ${branchName} not found in GitHub (source may not be released yet)`);
|
|
67
|
-
} else {
|
|
68
|
-
console.warn(`Fetched operator ${dockerTag} from Docker Hub but branch ${branchName} not found in GitHub (source may not be released yet)`);
|
|
69
|
-
}
|
|
70
|
-
} else {
|
|
71
|
-
if (logger) {
|
|
72
|
-
logger.error(`Failed to fetch Chart.yaml for branch ${branchName}:`, error.message);
|
|
73
|
-
} else {
|
|
74
|
-
console.error(`Failed to fetch Chart.yaml for branch ${branchName}:`, error.message);
|
|
75
|
-
}
|
|
76
|
-
}
|
|
77
|
-
return null;
|
|
97
|
+
if (!isBeta) {
|
|
98
|
+
const published = await getChartVersionFromTags(major, minor);
|
|
99
|
+
if (published) return published;
|
|
78
100
|
}
|
|
101
|
+
|
|
102
|
+
return await getChartVersionFromBranch(dockerTag, `release/v${major}.${minor}.x`);
|
|
79
103
|
} catch (error) {
|
|
80
104
|
if (logger) {
|
|
81
105
|
logger.error(`Error processing docker tag ${dockerTag}:`, error.message);
|
|
@@ -1,10 +1,10 @@
|
|
|
1
1
|
'use strict'
|
|
2
2
|
|
|
3
3
|
// The most recent releases of a repository, newest first, as the GitHub API
|
|
4
|
-
// lists them (
|
|
4
|
+
// lists them (the requested page of at most `limit` releases). Each release carries its
|
|
5
5
|
// assets, so a caller can check for an asset without downloading it. Throws
|
|
6
6
|
// when the API is unavailable, so the caller can fall back to git tags.
|
|
7
|
-
module.exports = async (github, owner, repo, limit) => {
|
|
8
|
-
const { data } = await github.rest.repos.listReleases({ owner, repo, per_page: limit })
|
|
7
|
+
module.exports = async (github, owner, repo, limit, page = 1) => {
|
|
8
|
+
const { data } = await github.rest.repos.listReleases({ owner, repo, per_page: limit, page })
|
|
9
9
|
return Array.isArray(data) ? data.slice(0, limit) : []
|
|
10
10
|
}
|
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
// Helpers for pinning an attribute to the release line a component version
|
|
4
|
+
// documents, rather than to the newest release overall. The lines come from
|
|
5
|
+
// the fetched release tags themselves, so a new release line is picked up as
|
|
6
|
+
// soon as it ships and nothing needs a manual version map.
|
|
7
|
+
|
|
8
|
+
const semver = require('semver');
|
|
9
|
+
|
|
10
|
+
/**
|
|
11
|
+
* Returns the newest stable tag of each major.minor release line.
|
|
12
|
+
*
|
|
13
|
+
* Only plain `v<major>.<minor>.<patch>` tags count. Prereleases are skipped,
|
|
14
|
+
* and so are the legacy operator tags such as `v2.3.15-24.3.18`, whose suffix
|
|
15
|
+
* semver reads as a prerelease. Patches are compared as semver, so the result
|
|
16
|
+
* does not depend on the order the registry lists tags in.
|
|
17
|
+
*
|
|
18
|
+
* @param {string[]} tags - Release tags, for example ['v26.1.12', 'v26.1.9'].
|
|
19
|
+
* @returns {Map<string, string>} Release line (for example '26.1') to its newest tag.
|
|
20
|
+
*/
|
|
21
|
+
function latestStablePerLine (tags) {
|
|
22
|
+
const lines = new Map();
|
|
23
|
+
for (const tag of tags || []) {
|
|
24
|
+
if (typeof tag !== 'string' || !tag.startsWith('v')) continue;
|
|
25
|
+
const parsed = semver.parse(tag.slice(1));
|
|
26
|
+
if (!parsed || parsed.prerelease.length || parsed.build.length) continue;
|
|
27
|
+
const line = `${parsed.major}.${parsed.minor}`;
|
|
28
|
+
const current = lines.get(line);
|
|
29
|
+
if (!current || semver.gt(parsed, current.slice(1))) lines.set(line, tag);
|
|
30
|
+
}
|
|
31
|
+
return lines;
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
/**
|
|
35
|
+
* Returns the release line an Antora component version names, or null when
|
|
36
|
+
* the version is not a `major.minor` pair (for example, an unversioned
|
|
37
|
+
* component or a version named after something other than a release).
|
|
38
|
+
*
|
|
39
|
+
* @param {string|null|undefined} version - The component version, for example '25.3'.
|
|
40
|
+
* @returns {string|null}
|
|
41
|
+
*/
|
|
42
|
+
function releaseLineOf (version) {
|
|
43
|
+
const match = /^(\d+)\.(\d+)$/.exec(String(version ?? ''));
|
|
44
|
+
return match ? `${Number(match[1])}.${Number(match[2])}` : null;
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
/**
|
|
48
|
+
* Returns true when a component version documents an older release line than
|
|
49
|
+
* the component's latest version. Only those versions are pinned to their own
|
|
50
|
+
* line. The latest version, prereleases ahead of it, and unversioned
|
|
51
|
+
* components keep the newest release overall.
|
|
52
|
+
*
|
|
53
|
+
* @param {string} version - The component version.
|
|
54
|
+
* @param {string} latestVersion - The component's latest version.
|
|
55
|
+
* @returns {boolean}
|
|
56
|
+
*/
|
|
57
|
+
function isOlderReleaseLine (version, latestVersion) {
|
|
58
|
+
const line = releaseLineOf(version);
|
|
59
|
+
const latestLine = releaseLineOf(latestVersion);
|
|
60
|
+
if (!line || !latestLine) return false;
|
|
61
|
+
return semver.lt(`${line}.0`, `${latestLine}.0`);
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
module.exports = { latestStablePerLine, releaseLineOf, isOlderReleaseLine };
|