@redpanda-data/docs-extensions-and-macros 5.53.0 → 5.55.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/doc-tools-mcp.js +4 -5
- package/bin/doc-tools.js +166 -2
- package/bin/mcp-tools/generated-docs-review.js +26 -9
- package/bin/mcp-tools/overrides-audit.js +1 -1
- package/bin/mcp-tools/rpcn-docs.js +4 -14
- package/cli-utils/format-helm-spec.js +4 -3
- package/cli-utils/helm-commented-values.js +88 -18
- package/cli-utils/self-managed-docs-branch.js +36 -30
- package/extension-utils/html-utils.js +27 -0
- package/extensions/REFERENCE.adoc +64 -12
- package/extensions/add-git-dates.js +13 -1
- package/extensions/aggregate-terms.js +34 -20
- package/extensions/generate-rp-connect-info.js +79 -24
- package/extensions/modify-connect-tag-playbook.js +498 -75
- package/extensions/resolve-xrefs-in-attachments.js +1 -12
- package/extensions/url-to-xref.js +3 -0
- package/extensions/util/connect-catalog.js +10 -5
- package/extensions/util/connect-docs-asset.js +188 -0
- package/extensions/util/connect-github.js +16 -0
- package/extensions/version-fetcher/fetch-latest-docker-tag.js +35 -13
- package/extensions/version-fetcher/get-latest-redpanda-helm-version-from-operator.js +64 -40
- package/extensions/version-fetcher/list-connect-releases.js +10 -0
- package/extensions/version-fetcher/release-lines.js +64 -0
- package/extensions/version-fetcher/set-latest-version.js +73 -3
- package/macros/README.adoc +1 -0
- package/macros/REFERENCE.adoc +99 -0
- package/macros/badge.js +5 -8
- package/macros/enterprise.js +2 -5
- package/macros/glossary.js +4 -14
- package/macros/iceberg-explorer.js +242 -0
- package/macros/prop.js +3 -13
- package/mcp/USER_GUIDE.adoc +25 -135
- package/package.json +4 -2
- package/tools/connect-docs/check-build-log.js +311 -0
- package/tools/connect-docs/check-rendered-html.js +469 -0
- package/tools/connect-docs/ci-playbook.yml +77 -0
- package/tools/connect-docs/diff-generated.js +382 -0
- package/tools/connect-docs/pr-comment.js +104 -0
- package/tools/doc-impact-recording/index.js +190 -0
- package/tools/overrides-audit/README.adoc +1 -2
- package/tools/overrides-audit/adapters/properties.js +1 -1
- package/tools/overrides-audit/index.js +3 -4
- package/tools/redpanda-connect/AUTOMATION.md +3 -36
- package/tools/redpanda-connect/README.adoc +5 -33
- package/tools/redpanda-connect/generate-rpcn-connector-docs.js +3 -3
- package/tools/redpanda-connect/pr-summary-formatter.js +6 -4
- package/tools/redpanda-connect/report-delta.js +6 -3
- package/tools/redpanda-connect/rpcn-connector-docs-handler.js +38 -16
- package/tools/redpanda-release-notes/generate-release-notes.js +116 -1
- package/tools/rpk-docs/templates/command.hbs +1 -1
- package/tools/overrides-audit/adapters/connect.js +0 -144
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
'use strict'
|
|
2
2
|
|
|
3
3
|
const { raiseListenerLimit } = require('./util/raise-listener-limit')
|
|
4
|
+
const { escapeHtml } = require('../extension-utils/html-utils')
|
|
4
5
|
|
|
5
6
|
/**
|
|
6
7
|
* Resolves AsciiDoc xrefs in JSON attachment files to HTML links.
|
|
@@ -240,15 +241,3 @@ function resolveXrefsInString (text, contentCatalog, context, logger) {
|
|
|
240
241
|
}
|
|
241
242
|
})
|
|
242
243
|
}
|
|
243
|
-
|
|
244
|
-
/**
|
|
245
|
-
* Escape HTML special characters to prevent XSS.
|
|
246
|
-
*/
|
|
247
|
-
function escapeHtml (text) {
|
|
248
|
-
if (!text) return ''
|
|
249
|
-
return String(text)
|
|
250
|
-
.replace(/&/g, '&')
|
|
251
|
-
.replace(/</g, '<')
|
|
252
|
-
.replace(/>/g, '>')
|
|
253
|
-
.replace(/"/g, '"')
|
|
254
|
-
}
|
|
@@ -362,6 +362,9 @@ const HEADING_RX = /^(={1,6})\s+(\S.*)$/
|
|
|
362
362
|
* which still displays sensibly.
|
|
363
363
|
*/
|
|
364
364
|
function linkTextFor (label, fragment, page, resolveInclude) {
|
|
365
|
+
// A trailing ^ opens a URL link in a new window; an xref would print it.
|
|
366
|
+
// A label that is only ^ counts as no label.
|
|
367
|
+
label = (label || '').replace(/\^$/, '')
|
|
365
368
|
if (label) return label
|
|
366
369
|
if (!fragment) return ''
|
|
367
370
|
const contents = page && page.contents && page.contents.toString()
|
|
@@ -2,9 +2,11 @@
|
|
|
2
2
|
|
|
3
3
|
// Shared Redpanda Connect catalog helpers.
|
|
4
4
|
//
|
|
5
|
-
// The connect repo publishes its reference content (partials and examples)
|
|
6
|
-
//
|
|
7
|
-
//
|
|
5
|
+
// The connect repo publishes its reference content (partials and examples) as
|
|
6
|
+
// a release asset that modify-connect-tag-playbook adds to the content catalog
|
|
7
|
+
// (or, to preview a connect branch, through a content source with explicit
|
|
8
|
+
// refs). Either way the files carry a connect origin with the ref.
|
|
9
|
+
// Catalog data (support level, deprecation, Cloud availability)
|
|
8
10
|
// must come from the same ref, or the badges and catalog drift from the
|
|
9
11
|
// reference content as soon as connect's main branch changes a flag before a
|
|
10
12
|
// release. This module carries that ref between the extensions and turns the
|
|
@@ -13,6 +15,7 @@
|
|
|
13
15
|
|
|
14
16
|
// Matches the GitHub URL (with or without .git) and a local clone whose
|
|
15
17
|
// directory is named connect, which is how the source looks in local builds.
|
|
18
|
+
// Files added from the release asset carry the GitHub URL.
|
|
16
19
|
function isConnectSource (url) {
|
|
17
20
|
return typeof url === 'string' && /(^|[/:])(redpanda-data\/)?connect(\.git)?\/?$/.test(url)
|
|
18
21
|
}
|
|
@@ -42,7 +45,8 @@ function githubRepoOf (url) {
|
|
|
42
45
|
return m ? { owner: m[1], repo: m[2] } : null
|
|
43
46
|
}
|
|
44
47
|
|
|
45
|
-
// The ref of the connect
|
|
48
|
+
// The ref of the connect files in the content catalog: the release asset tag,
|
|
49
|
+
// or the ref of the connect content source as Antora aggregated it. Only refs
|
|
46
50
|
// that exist on the remote count: a tag, or a branch fetched from the remote.
|
|
47
51
|
// A branch of a local clone may exist only on disk, so it is not used.
|
|
48
52
|
function connectOriginRef (contentCatalog) {
|
|
@@ -64,7 +68,8 @@ function connectOriginRef (contentCatalog) {
|
|
|
64
68
|
const CATALOG_RELATIVE = 'platforms/catalog.json'
|
|
65
69
|
|
|
66
70
|
// The generated catalog.json in the content catalog, preferring the copy
|
|
67
|
-
// from the connect content source over one
|
|
71
|
+
// from connect (the release asset or a connect content source) over one
|
|
72
|
+
// another source provides.
|
|
68
73
|
function findConnectCatalogFile (contentCatalog) {
|
|
69
74
|
if (!contentCatalog || typeof contentCatalog.findBy !== 'function') return null
|
|
70
75
|
const candidates = contentCatalog
|
|
@@ -0,0 +1,188 @@
|
|
|
1
|
+
'use strict'
|
|
2
|
+
|
|
3
|
+
// Reads the Redpanda Connect generated reference docs.
|
|
4
|
+
//
|
|
5
|
+
// Every connect release (vX.Y.Z, including prereleases) publishes a
|
|
6
|
+
// redpanda-connect-docs.tar.gz asset whose root is `modules/`: the tree that
|
|
7
|
+
// connect's docs generator writes under its docs/ directory. Only
|
|
8
|
+
// modules/components/partials/**, modules/components/examples/**, and
|
|
9
|
+
// modules/components/attachments/** (the release's connect-<version>.json)
|
|
10
|
+
// belong to the reference. Releases from before the asset existed answer 404.
|
|
11
|
+
//
|
|
12
|
+
// The same tree can come from a local directory instead (a connect checkout's
|
|
13
|
+
// docs/ after running its generator), or from a local copy of the tarball.
|
|
14
|
+
|
|
15
|
+
const fs = require('fs')
|
|
16
|
+
const path = require('path')
|
|
17
|
+
const zlib = require('zlib')
|
|
18
|
+
|
|
19
|
+
const OWNER = 'redpanda-data'
|
|
20
|
+
const REPO = 'connect'
|
|
21
|
+
const ASSET_NAME = 'redpanda-connect-docs.tar.gz'
|
|
22
|
+
const MODULE = 'components'
|
|
23
|
+
const FAMILY_DIRS = { partials: 'partial', examples: 'example', attachments: 'attachment' }
|
|
24
|
+
const DOWNLOAD_ATTEMPTS = 3
|
|
25
|
+
const DOWNLOAD_TIMEOUT_MS = 120000
|
|
26
|
+
|
|
27
|
+
function assetUrl (tag) {
|
|
28
|
+
return `https://github.com/${OWNER}/${REPO}/releases/download/${encodeURIComponent(tag)}/${ASSET_NAME}`
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
// The Antora resource of a path inside the tree, or null for anything outside
|
|
32
|
+
// modules/components/partials, examples, and attachments.
|
|
33
|
+
function toResource (entryPath) {
|
|
34
|
+
const p = String(entryPath || '').replace(/\\/g, '/').replace(/^(\.\/)+/, '')
|
|
35
|
+
const m = p.match(/^modules\/components\/(partials|examples|attachments)\/(.+)$/)
|
|
36
|
+
if (!m || m[2].endsWith('/')) return null
|
|
37
|
+
// A path that climbs out of its directory is not part of the tree.
|
|
38
|
+
if (m[2].split('/').some((segment) => segment === '..' || segment === '')) return null
|
|
39
|
+
return { family: FAMILY_DIRS[m[1]], relative: m[2], path: `modules/components/${m[1]}/${m[2]}` }
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
// Every regular file in a gzipped tar, as [{ path, contents }]. Throws on a
|
|
43
|
+
// corrupt or truncated archive rather than returning a partial list.
|
|
44
|
+
async function readTarGz (buffer) {
|
|
45
|
+
let tarBuffer
|
|
46
|
+
// The real asset unpacks to a few tens of MB. The cap turns a bad asset
|
|
47
|
+
// into a clear error instead of running the build out of memory.
|
|
48
|
+
const limit = module.exports.maxUnpackedBytes
|
|
49
|
+
try {
|
|
50
|
+
tarBuffer = zlib.gunzipSync(buffer, { maxOutputLength: limit })
|
|
51
|
+
} catch (error) {
|
|
52
|
+
if (error.code === 'ERR_BUFFER_TOO_LARGE') {
|
|
53
|
+
throw new Error(`the archive unpacks to more than ${limit} bytes, which no real reference docs asset does`)
|
|
54
|
+
}
|
|
55
|
+
throw new Error(`not a valid gzip archive: ${error.message}`)
|
|
56
|
+
}
|
|
57
|
+
// A complete tar ends with two zero blocks. Without them the archive was cut
|
|
58
|
+
// short at an entry boundary, which the parser alone does not report.
|
|
59
|
+
const end = tarBuffer.subarray(Math.max(0, tarBuffer.length - 1024))
|
|
60
|
+
if (tarBuffer.length % 512 !== 0 || end.length < 1024 || end.some((b) => b !== 0)) {
|
|
61
|
+
throw new Error('not a valid tar archive: missing the end-of-archive marker, so the archive is truncated')
|
|
62
|
+
}
|
|
63
|
+
const { Parser } = require('tar')
|
|
64
|
+
return new Promise((resolve, reject) => {
|
|
65
|
+
const files = []
|
|
66
|
+
let failure = null
|
|
67
|
+
const parser = new Parser({ strict: true })
|
|
68
|
+
parser.on('entry', (entry) => {
|
|
69
|
+
if (entry.type !== 'File' && entry.type !== 'OldFile' && entry.type !== 'ContiguousFile') {
|
|
70
|
+
entry.resume()
|
|
71
|
+
return
|
|
72
|
+
}
|
|
73
|
+
const chunks = []
|
|
74
|
+
entry.on('data', (chunk) => chunks.push(chunk))
|
|
75
|
+
entry.on('end', () => files.push({ path: entry.path, contents: Buffer.concat(chunks) }))
|
|
76
|
+
})
|
|
77
|
+
const fail = (error) => { if (!failure) failure = error }
|
|
78
|
+
parser.on('warn', (code, message) => fail(new Error(`not a valid tar archive: ${message}`)))
|
|
79
|
+
parser.on('error', (error) => fail(new Error(`not a valid tar archive: ${error.message}`)))
|
|
80
|
+
parser.on('close', () => (failure ? reject(failure) : resolve(files)))
|
|
81
|
+
try {
|
|
82
|
+
parser.end(tarBuffer)
|
|
83
|
+
} catch (error) {
|
|
84
|
+
fail(new Error(`not a valid tar archive: ${error.message}`))
|
|
85
|
+
reject(failure)
|
|
86
|
+
}
|
|
87
|
+
})
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
// One download attempt: { status } on an HTTP error, { body } on success.
|
|
91
|
+
// With a token it goes through the releases API, which works whether or not
|
|
92
|
+
// the connect repo is public. The asset endpoint redirects to a short-lived
|
|
93
|
+
// signed URL, which is fetched without the token.
|
|
94
|
+
async function fetchAssetOnce (tag, { fetchImpl, token }) {
|
|
95
|
+
const timeout = () => AbortSignal.timeout(DOWNLOAD_TIMEOUT_MS)
|
|
96
|
+
const ua = { 'User-Agent': 'Redpanda Docs' }
|
|
97
|
+
if (!token) {
|
|
98
|
+
const response = await fetchImpl(assetUrl(tag), { redirect: 'follow', signal: timeout(), headers: ua })
|
|
99
|
+
// Read the body here, so a connection that drops or times out
|
|
100
|
+
// mid-download is retried like a failed request.
|
|
101
|
+
return response.ok ? { body: Buffer.from(await response.arrayBuffer()) } : { status: response.status, statusText: response.statusText }
|
|
102
|
+
}
|
|
103
|
+
const auth = { ...ua, Authorization: `Bearer ${token}` }
|
|
104
|
+
const api = `https://api.github.com/repos/${OWNER}/${REPO}/releases/tags/${encodeURIComponent(tag)}`
|
|
105
|
+
const release = await fetchImpl(api, { signal: timeout(), headers: { ...auth, Accept: 'application/vnd.github+json' } })
|
|
106
|
+
// A token that is expired or lacks access must not break a build that the
|
|
107
|
+
// public download would serve. GitHub answers 404, not 403, for a repo the
|
|
108
|
+
// token can't see, so a 404 here falls back too, and is flagged so the
|
|
109
|
+
// caller can say why when the public download has no asset either.
|
|
110
|
+
if (release.status === 401 || release.status === 403) return fetchAssetOnce(tag, { fetchImpl, token: null })
|
|
111
|
+
if (release.status === 404) return { ...(await fetchAssetOnce(tag, { fetchImpl, token: null })), tokenCouldNotSee: true }
|
|
112
|
+
if (!release.ok) return { status: release.status, statusText: release.statusText }
|
|
113
|
+
const found = ((await release.json()).assets || []).find((a) => a.name === ASSET_NAME)
|
|
114
|
+
if (!found) return { status: 404, statusText: 'Not Found' }
|
|
115
|
+
let response = await fetchImpl(found.url, { redirect: 'manual', signal: timeout(), headers: { ...auth, Accept: 'application/octet-stream' } })
|
|
116
|
+
if (response.status === 401 || response.status === 403) return fetchAssetOnce(tag, { fetchImpl, token: null })
|
|
117
|
+
if (response.status >= 300 && response.status < 400 && response.headers.get('location')) {
|
|
118
|
+
response = await fetchImpl(response.headers.get('location'), { redirect: 'follow', signal: timeout(), headers: ua })
|
|
119
|
+
}
|
|
120
|
+
return response.ok ? { body: Buffer.from(await response.arrayBuffer()) } : { status: response.status, statusText: response.statusText }
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
// Downloads the asset of a release tag. Returns null when the release has no
|
|
124
|
+
// asset (404). Throws on any other failure, after retrying network errors and
|
|
125
|
+
// server errors. Pass the GitHub token when there is one: an unauthenticated
|
|
126
|
+
// download only works while the connect repo is public.
|
|
127
|
+
async function downloadAsset (tag, { fetchImpl = globalThis.fetch, logger, token = null } = {}) {
|
|
128
|
+
const url = token ? `the ${tag} release asset ${ASSET_NAME} through the GitHub API` : assetUrl(tag)
|
|
129
|
+
let lastError = null
|
|
130
|
+
for (let attempt = 1; attempt <= DOWNLOAD_ATTEMPTS; attempt++) {
|
|
131
|
+
let result
|
|
132
|
+
try {
|
|
133
|
+
result = await fetchAssetOnce(tag, { fetchImpl, token })
|
|
134
|
+
} catch (error) {
|
|
135
|
+
lastError = new Error(`could not download ${url}: ${error.message}`)
|
|
136
|
+
}
|
|
137
|
+
if (result && result.body) return result.body
|
|
138
|
+
if (result) {
|
|
139
|
+
if (result.status === 404) {
|
|
140
|
+
if (result.tokenCouldNotSee && logger) {
|
|
141
|
+
logger.warn(`The GitHub API found no ${tag} release of ${OWNER}/${REPO} with the GitHub token, and the public download has no ${ASSET_NAME} either. If the repo is private, check that the token has read access to it.`)
|
|
142
|
+
}
|
|
143
|
+
return null
|
|
144
|
+
}
|
|
145
|
+
lastError = new Error(`could not download ${url}: HTTP ${result.status} ${result.statusText || ''}`.trim())
|
|
146
|
+
// A client error other than 404 will not change on a retry.
|
|
147
|
+
if (result.status < 500 && result.status !== 429) break
|
|
148
|
+
}
|
|
149
|
+
if (attempt < DOWNLOAD_ATTEMPTS) {
|
|
150
|
+
if (logger) logger.warn(`${lastError.message}; retrying (${attempt}/${DOWNLOAD_ATTEMPTS - 1})`)
|
|
151
|
+
await new Promise((resolve) => setTimeout(resolve, module.exports.retryDelayMs * attempt))
|
|
152
|
+
}
|
|
153
|
+
}
|
|
154
|
+
throw lastError
|
|
155
|
+
}
|
|
156
|
+
|
|
157
|
+
function walk (dir, base, out) {
|
|
158
|
+
for (const entry of fs.readdirSync(dir, { withFileTypes: true })) {
|
|
159
|
+
const abs = path.join(dir, entry.name)
|
|
160
|
+
const rel = base ? `${base}/${entry.name}` : entry.name
|
|
161
|
+
if (entry.isDirectory()) walk(abs, rel, out)
|
|
162
|
+
else if (entry.isFile()) out.push({ path: rel, abspath: abs })
|
|
163
|
+
}
|
|
164
|
+
return out
|
|
165
|
+
}
|
|
166
|
+
|
|
167
|
+
// The files of a local copy of the tree: a directory that contains modules/
|
|
168
|
+
// (or a connect checkout whose docs/ does), or a local .tar.gz of the asset.
|
|
169
|
+
async function readLocal (location) {
|
|
170
|
+
const resolved = path.resolve(location)
|
|
171
|
+
let stat
|
|
172
|
+
try {
|
|
173
|
+
stat = fs.statSync(resolved)
|
|
174
|
+
} catch (error) {
|
|
175
|
+
throw new Error(`${resolved} does not exist`)
|
|
176
|
+
}
|
|
177
|
+
if (stat.isFile()) return readTarGz(fs.readFileSync(resolved))
|
|
178
|
+
let root = resolved
|
|
179
|
+
if (!fs.existsSync(path.join(root, 'modules')) && fs.existsSync(path.join(root, 'docs', 'modules'))) root = path.join(root, 'docs')
|
|
180
|
+
if (!fs.existsSync(path.join(root, 'modules', MODULE))) {
|
|
181
|
+
throw new Error(`${resolved} has no modules/${MODULE} directory`)
|
|
182
|
+
}
|
|
183
|
+
return walk(path.join(root, 'modules', MODULE), `modules/${MODULE}`, [])
|
|
184
|
+
.map((f) => ({ path: f.path, contents: fs.readFileSync(f.abspath) }))
|
|
185
|
+
}
|
|
186
|
+
|
|
187
|
+
// retryDelayMs is a property so tests can shorten the backoff.
|
|
188
|
+
module.exports = { OWNER, REPO, ASSET_NAME, MODULE, retryDelayMs: 1000, maxUnpackedBytes: 512 * 1024 * 1024, assetUrl, toResource, readTarGz, downloadAsset, readLocal }
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
'use strict'
|
|
2
|
+
|
|
3
|
+
// The GitHub API client the Redpanda Connect extensions use for release
|
|
4
|
+
// lookups: authenticated when a token is available, retrying transient
|
|
5
|
+
// failures but not rate limits or missing resources.
|
|
6
|
+
|
|
7
|
+
const { getGitHubApiToken } = require('../../cli-utils/github-token')
|
|
8
|
+
|
|
9
|
+
async function createGitHub () {
|
|
10
|
+
const { Octokit } = await import('@octokit/rest')
|
|
11
|
+
const { retry } = await import('@octokit/plugin-retry')
|
|
12
|
+
const token = getGitHubApiToken()
|
|
13
|
+
return new (Octokit.plugin(retry))({ userAgent: 'Redpanda Docs', auth: token || undefined, retry: { doNotRetry: [401, 403, 404, 429] } })
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
module.exports = { createGitHub }
|
|
@@ -4,30 +4,50 @@
|
|
|
4
4
|
* The function separates tags into stable (tags not including "-beta") and beta (tags including "-beta"),
|
|
5
5
|
* sorts each group by their major.minor version in descending order, and returns the top tag from each.
|
|
6
6
|
*
|
|
7
|
+
* It also returns every stable tag it saw (`stableReleases`), so callers can
|
|
8
|
+
* pick the newest release of an older line, not just the newest overall.
|
|
9
|
+
* Docker Hub lists the most recently pushed tags first, so the first page
|
|
10
|
+
* already holds every line that still gets patches. Pass `maxPages` to reach
|
|
11
|
+
* further back.
|
|
12
|
+
*
|
|
7
13
|
* @param {string} dockerNamespace - The Docker Hub namespace (organization or username)
|
|
8
14
|
* @param {string} dockerRepo - The repository name on Docker Hub
|
|
9
15
|
* @param {Object} logger - Optional Antora logger instance for logging
|
|
10
|
-
* @
|
|
16
|
+
* @param {{ maxPages?: number, fetch?: Function }} options - How many pages of 100 tags to read (default 1), and an optional fetch implementation for tests
|
|
17
|
+
* @returns {Promise<{ latestStableRelease: string|null, latestBetaRelease: string|null, stableReleases: string[] }>}
|
|
11
18
|
*/
|
|
12
|
-
module.exports = async (dockerNamespace, dockerRepo, logger = null) => {
|
|
13
|
-
const
|
|
19
|
+
module.exports = async (dockerNamespace, dockerRepo, logger = null, { maxPages = 1, fetch: fetchImpl } = {}) => {
|
|
20
|
+
const fetch = fetchImpl || (await import('node-fetch')).default;
|
|
14
21
|
|
|
15
22
|
try {
|
|
16
|
-
// Fetch a list of tags from Docker Hub.
|
|
17
|
-
|
|
18
|
-
const
|
|
23
|
+
// Fetch a list of tags from Docker Hub. Only the first page is required:
|
|
24
|
+
// a later page that fails costs older lines, not the latest release.
|
|
25
|
+
const results = [];
|
|
26
|
+
let url = `https://hub.docker.com/v2/repositories/${dockerNamespace}/${dockerRepo}/tags?page_size=100`;
|
|
27
|
+
for (let page = 1; url && page <= maxPages; page++) {
|
|
28
|
+
const response = await fetch(url);
|
|
19
29
|
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
30
|
+
if (!response.ok) {
|
|
31
|
+
const message = `Docker Hub API responded with status ${response.status}`;
|
|
32
|
+
if (page === 1) throw new Error(message);
|
|
33
|
+
if (logger) {
|
|
34
|
+
logger.warn(`${message} for page ${page} of ${dockerRepo} tags; using the first ${results.length} tags.`);
|
|
35
|
+
} else {
|
|
36
|
+
console.warn(`${message} for page ${page} of ${dockerRepo} tags; using the first ${results.length} tags.`);
|
|
37
|
+
}
|
|
38
|
+
break;
|
|
39
|
+
}
|
|
23
40
|
|
|
24
|
-
|
|
41
|
+
const data = await response.json();
|
|
42
|
+
results.push(...(data.results || []));
|
|
43
|
+
url = data.next;
|
|
44
|
+
}
|
|
25
45
|
|
|
26
46
|
// Regex to capture major and minor version numbers (e.g. "v2.3")
|
|
27
47
|
const versionRegex = /^v(\d+)\.(\d+)/;
|
|
28
48
|
|
|
29
49
|
// Filter tags to include only those matching the version pattern.
|
|
30
|
-
let tags =
|
|
50
|
+
let tags = results.filter(tag => versionRegex.test(tag.name));
|
|
31
51
|
|
|
32
52
|
// For specific repositories (e.g. "redpanda-operator"), you might want to filter out certain versions.
|
|
33
53
|
if (dockerRepo === 'redpanda-operator') {
|
|
@@ -61,7 +81,8 @@ module.exports = async (dockerNamespace, dockerRepo, logger = null) => {
|
|
|
61
81
|
|
|
62
82
|
return {
|
|
63
83
|
latestStableRelease: latestStableReleaseVersion || null,
|
|
64
|
-
latestBetaRelease: latestBetaReleaseVersion || null
|
|
84
|
+
latestBetaRelease: latestBetaReleaseVersion || null,
|
|
85
|
+
stableReleases: sortedStable.map(tag => tag.name)
|
|
65
86
|
};
|
|
66
87
|
|
|
67
88
|
} catch (error) {
|
|
@@ -72,7 +93,8 @@ module.exports = async (dockerNamespace, dockerRepo, logger = null) => {
|
|
|
72
93
|
}
|
|
73
94
|
return {
|
|
74
95
|
latestStableRelease: null,
|
|
75
|
-
latestBetaRelease: null
|
|
96
|
+
latestBetaRelease: null,
|
|
97
|
+
stableReleases: []
|
|
76
98
|
};
|
|
77
99
|
}
|
|
78
100
|
};
|
|
@@ -1,7 +1,17 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* Fetches the latest Helm chart version
|
|
3
|
-
*
|
|
4
|
-
*
|
|
2
|
+
* Fetches the latest Helm chart version for an operator release line from the
|
|
3
|
+
* redpanda-operator repository.
|
|
4
|
+
*
|
|
5
|
+
* A stable chart version comes from the newest stable `charts/redpanda/v<major>.<minor>.*`
|
|
6
|
+
* tag, because a tag marks a chart that was actually published (what
|
|
7
|
+
* `helm search repo` reports). A release branch's Chart.yaml can be bumped
|
|
8
|
+
* ahead of the published chart, so it is only the fallback, and the only
|
|
9
|
+
* source for beta versions.
|
|
10
|
+
*
|
|
11
|
+
* The chart has lived at more than one path over time (for example,
|
|
12
|
+
* `charts/redpanda/Chart.yaml` on release/v25.1.x and
|
|
13
|
+
* `charts/redpanda/chart/Chart.yaml` from release/v25.2.x on), so the fallback
|
|
14
|
+
* tries each known path in order.
|
|
5
15
|
*
|
|
6
16
|
* @param {object} github - The GitHub API client.
|
|
7
17
|
* @param {string} owner - The repository owner.
|
|
@@ -13,7 +23,50 @@
|
|
|
13
23
|
*/
|
|
14
24
|
module.exports = async (github, owner, repo, stableDockerTag, betaDockerTag, logger = null) => {
|
|
15
25
|
const yaml = require('js-yaml');
|
|
16
|
-
const
|
|
26
|
+
const semver = require('semver');
|
|
27
|
+
const chartPaths = ['charts/redpanda/chart/Chart.yaml', 'charts/redpanda/Chart.yaml'];
|
|
28
|
+
const warn = (message) => (logger ? logger.warn(message) : console.warn(message));
|
|
29
|
+
const logError = (message) => (logger ? logger.error(message) : console.error(message));
|
|
30
|
+
|
|
31
|
+
// Returns the newest stable published chart version of a release line, or
|
|
32
|
+
// null when the line has no stable chart tag or the lookup fails.
|
|
33
|
+
const getChartVersionFromTags = async (major, minor) => {
|
|
34
|
+
if (!github.git?.listMatchingRefs) return null;
|
|
35
|
+
const tagPrefix = 'charts/redpanda/v';
|
|
36
|
+
const prefix = `${tagPrefix}${major}.${minor}.`;
|
|
37
|
+
try {
|
|
38
|
+
const { data } = await github.git.listMatchingRefs({ owner, repo, ref: `tags/${prefix}` });
|
|
39
|
+
let newest = null;
|
|
40
|
+
for (const { ref } of data || []) {
|
|
41
|
+
const version = semver.valid(String(ref).slice(`refs/tags/${tagPrefix}`.length));
|
|
42
|
+
if (!version || semver.prerelease(version)) continue;
|
|
43
|
+
if (semver.major(version) !== Number(major) || semver.minor(version) !== Number(minor)) continue;
|
|
44
|
+
if (!newest || semver.gt(version, newest)) newest = version;
|
|
45
|
+
}
|
|
46
|
+
return newest;
|
|
47
|
+
} catch (error) {
|
|
48
|
+
warn(`Could not list ${prefix}* tags in ${owner}/${repo}: ${error.message || error}`);
|
|
49
|
+
return null;
|
|
50
|
+
}
|
|
51
|
+
};
|
|
52
|
+
|
|
53
|
+
// Returns the chart version from the first known Chart.yaml path on a
|
|
54
|
+
// branch, or null when none of the paths exist there.
|
|
55
|
+
const getChartVersionFromBranch = async (dockerTag, branchName) => {
|
|
56
|
+
for (const path of chartPaths) {
|
|
57
|
+
try {
|
|
58
|
+
const contentResponse = await github.repos.getContent({ owner, repo, path, ref: branchName });
|
|
59
|
+
const contentDecoded = Buffer.from(contentResponse.data.content, 'base64').toString('utf8');
|
|
60
|
+
return yaml.load(contentDecoded)?.version || null;
|
|
61
|
+
} catch (error) {
|
|
62
|
+
if (error.status === 404) continue;
|
|
63
|
+
logError(`Failed to fetch ${path} from branch ${branchName}: ${error.message || error}`);
|
|
64
|
+
return null;
|
|
65
|
+
}
|
|
66
|
+
}
|
|
67
|
+
warn(`Fetched operator ${dockerTag} from Docker Hub but found no Helm chart for it: no ${chartPaths.join(' or ')} on branch ${branchName} (the branch may not exist yet)`);
|
|
68
|
+
return null;
|
|
69
|
+
};
|
|
17
70
|
|
|
18
71
|
/**
|
|
19
72
|
* Helper function to fetch chart version from a branch derived from a docker tag
|
|
@@ -38,44 +91,15 @@ module.exports = async (github, owner, repo, stableDockerTag, betaDockerTag, log
|
|
|
38
91
|
return null;
|
|
39
92
|
}
|
|
40
93
|
|
|
41
|
-
const major = versionMatch
|
|
42
|
-
const
|
|
43
|
-
|
|
44
|
-
// Convert to release/v<major>.<minor>.x format
|
|
45
|
-
const branchName = `release/v${major}.${minor}.x`;
|
|
46
|
-
|
|
47
|
-
try {
|
|
48
|
-
|
|
49
|
-
// Fetch Chart.yaml from the specified branch
|
|
50
|
-
const contentResponse = await github.repos.getContent({
|
|
51
|
-
owner,
|
|
52
|
-
repo,
|
|
53
|
-
path,
|
|
54
|
-
ref: branchName,
|
|
55
|
-
});
|
|
94
|
+
const [, major, minor] = versionMatch;
|
|
95
|
+
const isBeta = /-beta/.test(dockerTag);
|
|
56
96
|
|
|
57
|
-
|
|
58
|
-
const
|
|
59
|
-
|
|
60
|
-
const version = chartYaml.version || null;
|
|
61
|
-
return version;
|
|
62
|
-
} catch (error) {
|
|
63
|
-
if (error.status === 404) {
|
|
64
|
-
// Branch doesn't exist yet - this is expected for unreleased versions
|
|
65
|
-
if (logger) {
|
|
66
|
-
logger.warn(`Fetched operator ${dockerTag} from Docker Hub but branch ${branchName} not found in GitHub (source may not be released yet)`);
|
|
67
|
-
} else {
|
|
68
|
-
console.warn(`Fetched operator ${dockerTag} from Docker Hub but branch ${branchName} not found in GitHub (source may not be released yet)`);
|
|
69
|
-
}
|
|
70
|
-
} else {
|
|
71
|
-
if (logger) {
|
|
72
|
-
logger.error(`Failed to fetch Chart.yaml for branch ${branchName}:`, error.message);
|
|
73
|
-
} else {
|
|
74
|
-
console.error(`Failed to fetch Chart.yaml for branch ${branchName}:`, error.message);
|
|
75
|
-
}
|
|
76
|
-
}
|
|
77
|
-
return null;
|
|
97
|
+
if (!isBeta) {
|
|
98
|
+
const published = await getChartVersionFromTags(major, minor);
|
|
99
|
+
if (published) return published;
|
|
78
100
|
}
|
|
101
|
+
|
|
102
|
+
return await getChartVersionFromBranch(dockerTag, `release/v${major}.${minor}.x`);
|
|
79
103
|
} catch (error) {
|
|
80
104
|
if (logger) {
|
|
81
105
|
logger.error(`Error processing docker tag ${dockerTag}:`, error.message);
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
'use strict'
|
|
2
|
+
|
|
3
|
+
// The most recent releases of a repository, newest first, as the GitHub API
|
|
4
|
+
// lists them (the requested page of at most `limit` releases). Each release carries its
|
|
5
|
+
// assets, so a caller can check for an asset without downloading it. Throws
|
|
6
|
+
// when the API is unavailable, so the caller can fall back to git tags.
|
|
7
|
+
module.exports = async (github, owner, repo, limit, page = 1) => {
|
|
8
|
+
const { data } = await github.rest.repos.listReleases({ owner, repo, per_page: limit, page })
|
|
9
|
+
return Array.isArray(data) ? data.slice(0, limit) : []
|
|
10
|
+
}
|
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
// Helpers for pinning an attribute to the release line a component version
|
|
4
|
+
// documents, rather than to the newest release overall. The lines come from
|
|
5
|
+
// the fetched release tags themselves, so a new release line is picked up as
|
|
6
|
+
// soon as it ships and nothing needs a manual version map.
|
|
7
|
+
|
|
8
|
+
const semver = require('semver');
|
|
9
|
+
|
|
10
|
+
/**
|
|
11
|
+
* Returns the newest stable tag of each major.minor release line.
|
|
12
|
+
*
|
|
13
|
+
* Only plain `v<major>.<minor>.<patch>` tags count. Prereleases are skipped,
|
|
14
|
+
* and so are the legacy operator tags such as `v2.3.15-24.3.18`, whose suffix
|
|
15
|
+
* semver reads as a prerelease. Patches are compared as semver, so the result
|
|
16
|
+
* does not depend on the order the registry lists tags in.
|
|
17
|
+
*
|
|
18
|
+
* @param {string[]} tags - Release tags, for example ['v26.1.12', 'v26.1.9'].
|
|
19
|
+
* @returns {Map<string, string>} Release line (for example '26.1') to its newest tag.
|
|
20
|
+
*/
|
|
21
|
+
function latestStablePerLine (tags) {
|
|
22
|
+
const lines = new Map();
|
|
23
|
+
for (const tag of tags || []) {
|
|
24
|
+
if (typeof tag !== 'string' || !tag.startsWith('v')) continue;
|
|
25
|
+
const parsed = semver.parse(tag.slice(1));
|
|
26
|
+
if (!parsed || parsed.prerelease.length || parsed.build.length) continue;
|
|
27
|
+
const line = `${parsed.major}.${parsed.minor}`;
|
|
28
|
+
const current = lines.get(line);
|
|
29
|
+
if (!current || semver.gt(parsed, current.slice(1))) lines.set(line, tag);
|
|
30
|
+
}
|
|
31
|
+
return lines;
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
/**
|
|
35
|
+
* Returns the release line an Antora component version names, or null when
|
|
36
|
+
* the version is not a `major.minor` pair (for example, an unversioned
|
|
37
|
+
* component or a version named after something other than a release).
|
|
38
|
+
*
|
|
39
|
+
* @param {string|null|undefined} version - The component version, for example '25.3'.
|
|
40
|
+
* @returns {string|null}
|
|
41
|
+
*/
|
|
42
|
+
function releaseLineOf (version) {
|
|
43
|
+
const match = /^(\d+)\.(\d+)$/.exec(String(version ?? ''));
|
|
44
|
+
return match ? `${Number(match[1])}.${Number(match[2])}` : null;
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
/**
|
|
48
|
+
* Returns true when a component version documents an older release line than
|
|
49
|
+
* the component's latest version. Only those versions are pinned to their own
|
|
50
|
+
* line. The latest version, prereleases ahead of it, and unversioned
|
|
51
|
+
* components keep the newest release overall.
|
|
52
|
+
*
|
|
53
|
+
* @param {string} version - The component version.
|
|
54
|
+
* @param {string} latestVersion - The component's latest version.
|
|
55
|
+
* @returns {boolean}
|
|
56
|
+
*/
|
|
57
|
+
function isOlderReleaseLine (version, latestVersion) {
|
|
58
|
+
const line = releaseLineOf(version);
|
|
59
|
+
const latestLine = releaseLineOf(latestVersion);
|
|
60
|
+
if (!line || !latestLine) return false;
|
|
61
|
+
return semver.lt(`${line}.0`, `${latestLine}.0`);
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
module.exports = { latestStablePerLine, releaseLineOf, isOlderReleaseLine };
|