@redpanda-data/docs-extensions-and-macros 5.52.0 → 5.54.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/doc-tools.js +245 -0
- package/extensions/REFERENCE.adoc +45 -9
- package/extensions/generate-rp-connect-info.js +79 -24
- package/extensions/modify-connect-tag-playbook.js +319 -74
- package/extensions/url-to-xref.js +3 -0
- package/extensions/util/connect-catalog.js +10 -5
- package/extensions/util/connect-docs-asset.js +187 -0
- package/extensions/util/connect-github.js +16 -0
- package/extensions/version-fetcher/list-connect-releases.js +10 -0
- package/package.json +1 -1
- package/tools/connect-docs/check-build-log.js +311 -0
- package/tools/connect-docs/check-rendered-html.js +469 -0
- package/tools/connect-docs/ci-playbook.yml +77 -0
- package/tools/connect-docs/diff-generated.js +382 -0
- package/tools/connect-docs/pr-comment.js +104 -0
- package/tools/redpanda-release-notes/generate-release-notes.js +242 -0
- package/tools/redpanda-release-notes/parse-release-body.js +241 -0
- package/tools/redpanda-release-notes/render-release-notes.js +70 -0
|
@@ -1,38 +1,80 @@
|
|
|
1
1
|
'use strict';
|
|
2
2
|
|
|
3
|
-
// Sources the Redpanda Connect reference
|
|
3
|
+
// Sources the Redpanda Connect generated reference docs (field reference,
|
|
4
|
+
// examples, metadata, descriptions, config snippets, Bloblang reference, and
|
|
5
|
+
// platforms/catalog.json) for the `connect` component.
|
|
4
6
|
//
|
|
5
|
-
//
|
|
7
|
+
// The default: the playbook has no connect content source. Every connect
|
|
8
|
+
// release publishes the generated tree as a redpanda-connect-docs.tar.gz
|
|
9
|
+
// release asset (util/connect-docs-asset). In contentClassified, the extension
|
|
10
|
+
// resolves the release to use, downloads that asset, and adds its partials and
|
|
11
|
+
// examples to the `connect` component that rp-connect-docs provides:
|
|
12
|
+
//
|
|
13
|
+
// - The latest stable vX.Y.Z release by default, so the docs never show fields
|
|
14
|
+
// from unreleased code on connect's main branch.
|
|
15
|
+
// - The `tag` extension config key picks a specific release instead.
|
|
16
|
+
// - The REDPANDA_CONNECT_DOCS_DIR environment variable points at a local copy
|
|
17
|
+
// of the tree (a directory that contains modules/, such as a connect
|
|
18
|
+
// checkout's docs/ after running its generator, or a local copy of the
|
|
19
|
+
// tarball) to preview unreleased connect changes. It wins over everything,
|
|
20
|
+
// and its files replace any copy another source provides.
|
|
21
|
+
//
|
|
22
|
+
// Connect publishes a release first and attaches the asset later, in a
|
|
23
|
+
// separate job that can also fail. So when the latest release has no asset
|
|
24
|
+
// (404), the extension falls back to the newest older stable release that has
|
|
25
|
+
// one, and logs a warning that names both. It checks the asset lists from the
|
|
26
|
+
// GitHub releases API (at most FALLBACK_RELEASE_LIMIT releases), or, without
|
|
27
|
+
// the API, probes the downloads of the next FALLBACK_PROBE_LIMIT lower stable
|
|
28
|
+
// tags from git ls-remote. A release picked with the `tag` config never falls
|
|
29
|
+
// back: a release picked with `tag` that has no asset stops the build. When
|
|
30
|
+
// no release has the asset, nothing is added, and the guard in
|
|
31
|
+
// generate-rp-connect-info stops the build if connector pages then have no
|
|
32
|
+
// generated partials. Any other download or archive failure stops the build
|
|
33
|
+
// here. A file another source already provides (rp-connect-docs still commits
|
|
34
|
+
// its own copies until it migrates) is skipped.
|
|
35
|
+
//
|
|
36
|
+
// Playbooks that still list the upstream connect repo with `tags: latest` as
|
|
37
|
+
// its only ref (the setup from before the release asset):
|
|
6
38
|
//
|
|
7
39
|
// - url: https://github.com/redpanda-data/connect
|
|
8
|
-
// tags: latest
|
|
40
|
+
// tags: latest
|
|
9
41
|
// start_path: docs
|
|
10
42
|
//
|
|
11
|
-
//
|
|
43
|
+
// That source is removed from the playbook in contextStarted, with a warning,
|
|
44
|
+
// and the build continues with the release asset. Connect releases no longer
|
|
45
|
+
// ship docs/antora.yml, so Antora could not aggregate it anyway.
|
|
12
46
|
//
|
|
13
|
-
//
|
|
14
|
-
//
|
|
15
|
-
//
|
|
16
|
-
//
|
|
17
|
-
//
|
|
18
|
-
//
|
|
19
|
-
//
|
|
20
|
-
// after rp-connect-docs stops committing its own generated copies, and it
|
|
21
|
-
// ignores the full pages that older connect tags still carry.
|
|
47
|
+
// A connect source with explicit refs (a branch, a specific tag, a fork, or a
|
|
48
|
+
// local clone named connect) is how people preview a connect branch, so it is
|
|
49
|
+
// kept, and no asset is downloaded. Only the generated
|
|
50
|
+
// modules/components/partials and modules/components/examples trees of the
|
|
51
|
+
// `connect` component are kept from that source, and any file another source
|
|
52
|
+
// already provides is dropped, because Antora fails the build on a duplicate
|
|
53
|
+
// page or partial.
|
|
22
54
|
//
|
|
23
|
-
//
|
|
24
|
-
// generate-rp-connect-info reads info.csv from the same ref
|
|
25
|
-
// content.
|
|
55
|
+
// The release asset tag is shared (util/connect-catalog), so
|
|
56
|
+
// generate-rp-connect-info reads info.csv and catalog.json from the same ref
|
|
57
|
+
// as the reference content.
|
|
26
58
|
|
|
27
59
|
const { raiseListenerLimit } = require('./util/raise-listener-limit')
|
|
28
60
|
const getLatestConnectTag = require('./version-fetcher/get-latest-connect')
|
|
29
|
-
const
|
|
61
|
+
const listConnectReleases = require('./version-fetcher/list-connect-releases')
|
|
62
|
+
const { createGitHub } = require('./util/connect-github')
|
|
30
63
|
const { isConnectSource, isConnectOrigin, githubRepoOf, setResolvedConnectRef } = require('./util/connect-catalog')
|
|
64
|
+
const asset = require('./util/connect-docs-asset')
|
|
65
|
+
const { getGitHubApiToken } = require('../cli-utils/github-token')
|
|
66
|
+
const { gitAuthEnv } = require('../cli-utils/git-credential-env')
|
|
31
67
|
|
|
32
68
|
const OWNER = 'redpanda-data'
|
|
33
69
|
const REPO = 'connect'
|
|
34
70
|
const COMPONENT = 'connect'
|
|
35
71
|
const KEPT_PATHS = ['modules/components/partials/', 'modules/components/examples/']
|
|
72
|
+
const CONNECT_URL = `https://github.com/${OWNER}/${REPO}`
|
|
73
|
+
const LOCAL_DIR_ENV = 'REDPANDA_CONNECT_DOCS_DIR'
|
|
74
|
+
// How many releases the API fallback looks at, and how many lower git tags
|
|
75
|
+
// the probe fallback tries, when the latest release has no asset yet.
|
|
76
|
+
const FALLBACK_RELEASE_LIMIT = 10
|
|
77
|
+
const FALLBACK_PROBE_LIMIT = 3
|
|
36
78
|
|
|
37
79
|
// HTTPS, SSH (ssh://), and scp-style (git@host:owner/repo) URLs are all
|
|
38
80
|
// remote content sources in Antora.
|
|
@@ -54,23 +96,43 @@ function redact (text) {
|
|
|
54
96
|
return String(text || '').replace(/([a-z][a-z+.-]*:\/\/)[^@/\s]*@/gi, '$1')
|
|
55
97
|
}
|
|
56
98
|
|
|
99
|
+
// [major, minor, patch] of a stable vX.Y.Z tag, or null for anything else,
|
|
100
|
+
// such as a prerelease.
|
|
101
|
+
function stableVersion (tag) {
|
|
102
|
+
const m = /^v(\d+)\.(\d+)\.(\d+)$/.exec(tag || '')
|
|
103
|
+
return m ? m.slice(1).map(Number) : null
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
function compareVersions (a, b) {
|
|
107
|
+
return a[0] - b[0] || a[1] - b[1] || a[2] - b[2]
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
// Stable vX.Y.Z tags from `git ls-remote` output, highest first.
|
|
111
|
+
function stableTags (lsRemoteOutput) {
|
|
112
|
+
const tags = new Set()
|
|
113
|
+
for (const line of String(lsRemoteOutput || '').split('\n')) {
|
|
114
|
+
const m = line.match(/refs\/tags\/(v\d+\.\d+\.\d+)$/)
|
|
115
|
+
if (m) tags.add(m[1])
|
|
116
|
+
}
|
|
117
|
+
return [...tags].sort((a, b) => compareVersions(stableVersion(b), stableVersion(a)))
|
|
118
|
+
}
|
|
119
|
+
|
|
57
120
|
// Highest stable vX.Y.Z tag from `git ls-remote` output. Used when the GitHub
|
|
58
121
|
// API is unavailable, for example when an unauthenticated build is rate limited.
|
|
59
122
|
function highestStableTag (lsRemoteOutput) {
|
|
60
|
-
|
|
61
|
-
let best = null
|
|
62
|
-
for (const line of String(lsRemoteOutput || '').split('\n')) {
|
|
63
|
-
const m = line.match(/refs\/tags\/(v(\d+)\.(\d+)\.(\d+))$/)
|
|
64
|
-
if (!m) continue
|
|
65
|
-
const v = m.slice(2, 5).map(Number)
|
|
66
|
-
if (!best || newer(v, best.v) > 0) best = { tag: m[1], v }
|
|
67
|
-
}
|
|
68
|
-
return best && best.tag
|
|
123
|
+
return stableTags(lsRemoteOutput)[0] || null
|
|
69
124
|
}
|
|
70
125
|
|
|
71
|
-
|
|
126
|
+
// With a token, git authenticates through the shared credential helper, so a
|
|
127
|
+
// private connect repo still lists its tags.
|
|
128
|
+
function gitTags (url) {
|
|
72
129
|
const { execFileSync } = require('child_process')
|
|
73
|
-
|
|
130
|
+
const env = gitAuthEnv(getGitHubApiToken())
|
|
131
|
+
return execFileSync('git', ['ls-remote', '--tags', '--refs', url], { encoding: 'utf8', timeout: 60000, env })
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
function latestTagFromGit (url) {
|
|
135
|
+
return highestStableTag(gitTags(url))
|
|
74
136
|
}
|
|
75
137
|
|
|
76
138
|
function toTag (tagName) {
|
|
@@ -78,9 +140,10 @@ function toTag (tagName) {
|
|
|
78
140
|
return tagName.startsWith('v') ? tagName : `v${tagName}`
|
|
79
141
|
}
|
|
80
142
|
|
|
81
|
-
// The upstream connect source
|
|
82
|
-
//
|
|
83
|
-
// to preview a connect change, keeps what the playbook
|
|
143
|
+
// The upstream connect source with `tags: latest` as its only ref: the setup
|
|
144
|
+
// from before the release asset. A source with any other refs, such as a fork
|
|
145
|
+
// or a PR branch used to preview a connect change, keeps what the playbook
|
|
146
|
+
// says.
|
|
84
147
|
function wantsLatest (source) {
|
|
85
148
|
if (!isRemote(source.url) || !isConnectSource(source.url) || !isUpstreamConnect(source.url)) return false
|
|
86
149
|
const tags = Array.isArray(source.tags) ? source.tags : [source.tags]
|
|
@@ -88,20 +151,17 @@ function wantsLatest (source) {
|
|
|
88
151
|
return tags.length === 1 && tags[0] === 'latest' && branches.length === 0
|
|
89
152
|
}
|
|
90
153
|
|
|
91
|
-
//
|
|
92
|
-
//
|
|
93
|
-
|
|
154
|
+
// Removes the upstream connect sources that ask for the latest release, which
|
|
155
|
+
// the release asset replaces. Mutates the playbook's source list in place and
|
|
156
|
+
// returns the removed sources.
|
|
157
|
+
function removeLatestConnectSources (playbook) {
|
|
94
158
|
const sources = playbook && playbook.content && playbook.content.sources
|
|
95
|
-
if (!Array.isArray(sources)
|
|
96
|
-
|
|
97
|
-
for (
|
|
98
|
-
if (
|
|
99
|
-
source.tags = [tag]
|
|
100
|
-
// Without this, Antora's default branch patterns also pull in main.
|
|
101
|
-
source.branches = []
|
|
102
|
-
updated = true
|
|
159
|
+
if (!Array.isArray(sources)) return []
|
|
160
|
+
const removed = []
|
|
161
|
+
for (let i = sources.length - 1; i >= 0; i--) {
|
|
162
|
+
if (wantsLatest(sources[i])) removed.unshift(...sources.splice(i, 1))
|
|
103
163
|
}
|
|
104
|
-
return
|
|
164
|
+
return removed
|
|
105
165
|
}
|
|
106
166
|
|
|
107
167
|
// Removes connect-sourced files that must not be published. Mutates the
|
|
@@ -143,46 +203,155 @@ function filterConnectContent (contentAggregate) {
|
|
|
143
203
|
return report
|
|
144
204
|
}
|
|
145
205
|
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
206
|
+
// True when the playbook lists the connect repo (or a local clone named
|
|
207
|
+
// connect) as a content source, which keeps the git source behavior. Run it
|
|
208
|
+
// after removeLatestConnectSources.
|
|
209
|
+
function hasConnectSource (playbook) {
|
|
210
|
+
const sources = (playbook && playbook.content && playbook.content.sources) || []
|
|
211
|
+
return sources.some((source) => isConnectSource(source.url))
|
|
212
|
+
}
|
|
149
213
|
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
214
|
+
// The latest stable connect release tag: the GitHub API first, then the
|
|
215
|
+
// highest stable vX.Y.Z tag from git ls-remote. Null when both fail.
|
|
216
|
+
async function resolveLatestTag (url, logger) {
|
|
217
|
+
let tag = null
|
|
218
|
+
try {
|
|
219
|
+
tag = toTag(await getLatestConnectTag(await createGitHub(), OWNER, REPO, logger))
|
|
220
|
+
} catch (error) {
|
|
221
|
+
logger.warn(`GitHub API lookup of the latest Redpanda Connect release failed: ${error.message}`)
|
|
222
|
+
}
|
|
223
|
+
if (!tag) {
|
|
157
224
|
try {
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
const token = getGitHubApiToken()
|
|
161
|
-
const github = new (Octokit.plugin(retry))({ userAgent: 'Redpanda Docs', auth: token || undefined, retry: { doNotRetry: [403, 404, 429] } })
|
|
162
|
-
tag = toTag(await getLatestConnectTag(github, OWNER, REPO, logger))
|
|
225
|
+
tag = latestTagFromGit(url)
|
|
226
|
+
if (tag) logger.info(`Resolved the latest Redpanda Connect release from git tags: ${tag}`)
|
|
163
227
|
} catch (error) {
|
|
164
|
-
logger.warn(`
|
|
228
|
+
logger.warn(redact(`git ls-remote of ${url} failed: ${error.message}`))
|
|
165
229
|
}
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
230
|
+
}
|
|
231
|
+
return tag
|
|
232
|
+
}
|
|
233
|
+
|
|
234
|
+
// Downloads the asset of a release. Null on a 404; throws on any other failure.
|
|
235
|
+
async function downloadRelease (tag, logger) {
|
|
236
|
+
try {
|
|
237
|
+
return await asset.downloadAsset(tag, { logger, token: getGitHubApiToken() })
|
|
238
|
+
} catch (error) {
|
|
239
|
+
throw new Error(`Could not download the Redpanda Connect reference docs for ${tag}: ${error.message}`)
|
|
240
|
+
}
|
|
241
|
+
}
|
|
242
|
+
|
|
243
|
+
// The newest stable release older than `latestTag` that has the asset, as
|
|
244
|
+
// { tag, archive }, or null when none has it. The GitHub releases API lists
|
|
245
|
+
// the most recent releases with their assets, so only the chosen release is
|
|
246
|
+
// downloaded. Without the API, it probes the downloads of the next few lower
|
|
247
|
+
// stable tags from git ls-remote instead.
|
|
248
|
+
async function findOlderReleaseWithAsset (latestTag, url, logger) {
|
|
249
|
+
const latest = stableVersion(latestTag)
|
|
250
|
+
const isOlder = (tag) => {
|
|
251
|
+
const v = stableVersion(tag)
|
|
252
|
+
return !!v && (!latest || compareVersions(v, latest) < 0)
|
|
253
|
+
}
|
|
254
|
+
let releases = null
|
|
255
|
+
try {
|
|
256
|
+
releases = await listConnectReleases(await createGitHub(), OWNER, REPO, FALLBACK_RELEASE_LIMIT)
|
|
257
|
+
} catch (error) {
|
|
258
|
+
logger.warn(`GitHub API listing of Redpanda Connect releases failed: ${error.message}; probing the next ${FALLBACK_PROBE_LIMIT} lower release tags from git instead`)
|
|
259
|
+
}
|
|
260
|
+
if (releases) {
|
|
261
|
+
const found = releases
|
|
262
|
+
.filter((r) => r && !r.draft && !r.prerelease && isOlder(r.tag_name))
|
|
263
|
+
.sort((a, b) => compareVersions(stableVersion(b.tag_name), stableVersion(a.tag_name)))
|
|
264
|
+
.find((r) => (r.assets || []).some((a) => a && a.name === asset.ASSET_NAME))
|
|
265
|
+
if (!found) {
|
|
266
|
+
logger.info(`None of the ${releases.length} most recent Redpanda Connect releases has a ${asset.ASSET_NAME} release asset`)
|
|
267
|
+
return null
|
|
268
|
+
}
|
|
269
|
+
const archive = await downloadRelease(found.tag_name, logger)
|
|
270
|
+
return archive ? { tag: found.tag_name, archive } : null
|
|
271
|
+
}
|
|
272
|
+
let tags
|
|
273
|
+
try {
|
|
274
|
+
tags = stableTags(gitTags(url)).filter(isOlder).slice(0, FALLBACK_PROBE_LIMIT)
|
|
275
|
+
} catch (error) {
|
|
276
|
+
logger.warn(redact(`git ls-remote of ${url} failed: ${error.message}`))
|
|
277
|
+
return null
|
|
278
|
+
}
|
|
279
|
+
for (const tag of tags) {
|
|
280
|
+
const archive = await downloadRelease(tag, logger)
|
|
281
|
+
if (archive) return { tag, archive }
|
|
282
|
+
}
|
|
283
|
+
if (tags.length) logger.info(`None of the next ${tags.length} lower Redpanda Connect release tags (${tags.join(', ')}) has a ${asset.ASSET_NAME} release asset`)
|
|
284
|
+
return null
|
|
285
|
+
}
|
|
286
|
+
|
|
287
|
+
// Adds the generated files to every version of the connect component.
|
|
288
|
+
// Mutates the content catalog and returns what it added, replaced, and
|
|
289
|
+
// skipped. A file another source already provides is skipped, unless
|
|
290
|
+
// `replace` is set (a local directory, which previews unreleased changes and
|
|
291
|
+
// so wins over committed copies).
|
|
292
|
+
function addConnectDocs (contentCatalog, files, origin, { replace = false } = {}) {
|
|
293
|
+
const report = { added: 0, replaced: 0, providedElsewhere: 0, outsideGenerated: 0, versions: [] }
|
|
294
|
+
const component = contentCatalog.getComponent(COMPONENT)
|
|
295
|
+
if (!component) return report
|
|
296
|
+
report.versions = component.versions.map((v) => v.version)
|
|
297
|
+
for (const entry of files) {
|
|
298
|
+
const resource = asset.toResource(entry.path)
|
|
299
|
+
if (!resource) {
|
|
300
|
+
report.outsideGenerated++
|
|
301
|
+
continue
|
|
302
|
+
}
|
|
303
|
+
for (const version of report.versions) {
|
|
304
|
+
const src = { component: COMPONENT, version, module: asset.MODULE, family: resource.family, relative: resource.relative }
|
|
305
|
+
const existing = contentCatalog.getById(src)
|
|
306
|
+
if (existing && !replace) {
|
|
307
|
+
report.providedElsewhere++
|
|
308
|
+
continue
|
|
172
309
|
}
|
|
310
|
+
if (existing) contentCatalog.removeFile(existing)
|
|
311
|
+
contentCatalog.addFile({
|
|
312
|
+
path: resource.path,
|
|
313
|
+
contents: Buffer.from(entry.contents),
|
|
314
|
+
src: { ...src, path: resource.path, origin },
|
|
315
|
+
})
|
|
316
|
+
if (existing) report.replaced++
|
|
317
|
+
else report.added++
|
|
173
318
|
}
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
319
|
+
}
|
|
320
|
+
return report
|
|
321
|
+
}
|
|
322
|
+
|
|
323
|
+
module.exports.register = function ({ config }) {
|
|
324
|
+
raiseListenerLimit(this)
|
|
325
|
+
const logger = this.getLogger('modify-connect-tag-playbook-extension')
|
|
326
|
+
const rawTag = config && config.tag != null ? String(config.tag).trim() : ''
|
|
327
|
+
// `latest` is the default; as a tag it would become the nonexistent vlatest.
|
|
328
|
+
if (rawTag.toLowerCase() === 'latest') {
|
|
329
|
+
throw new Error('The `tag` config of modify-connect-tag-playbook is `latest`. The latest stable Redpanda Connect release is the default, so remove `tag`, or set it to a release tag such as v4.113.0.')
|
|
330
|
+
}
|
|
331
|
+
const configuredTag = toTag(rawTag || null)
|
|
332
|
+
// Set in contextStarted: true when the playbook still lists a connect
|
|
333
|
+
// content source with explicit refs.
|
|
334
|
+
let gitSourceMode = false
|
|
335
|
+
|
|
336
|
+
this.on('contextStarted', ({ playbook }) => {
|
|
337
|
+
// Clear a ref left over from an earlier build in the same process.
|
|
338
|
+
setResolvedConnectRef(null)
|
|
339
|
+
const removed = removeLatestConnectSources(playbook)
|
|
340
|
+
if (removed.length) {
|
|
341
|
+
this.updateVariables({ playbook })
|
|
342
|
+
logger.warn(
|
|
343
|
+
`The Redpanda Connect content source (${removed.map((s) => redact(s.url)).join(', ')} with tags: latest) is no longer needed and was removed from the playbook, ` +
|
|
344
|
+
`so the reference docs come from the ${asset.ASSET_NAME} release asset. Remove that source from the playbook.`
|
|
345
|
+
)
|
|
346
|
+
}
|
|
347
|
+
gitSourceMode = hasConnectSource(playbook)
|
|
348
|
+
if (gitSourceMode && (process.env[LOCAL_DIR_ENV] || '').trim()) {
|
|
349
|
+
logger.warn(`${LOCAL_DIR_ENV} is ignored because the playbook lists the connect repo as a content source`)
|
|
178
350
|
}
|
|
179
|
-
pinConnectSource(playbook, tag)
|
|
180
|
-
setResolvedConnectRef(tag)
|
|
181
|
-
this.updateVariables({ playbook })
|
|
182
|
-
logger.info(`Sourcing Redpanda Connect reference content from ${tag}`)
|
|
183
351
|
})
|
|
184
352
|
|
|
185
353
|
this.on('contentAggregated', ({ contentAggregate }) => {
|
|
354
|
+
if (!gitSourceMode) return
|
|
186
355
|
const r = filterConnectContent(contentAggregate)
|
|
187
356
|
if (r.kept + r.outsideGenerated + r.providedElsewhere + r.otherComponents === 0) return
|
|
188
357
|
logger.info(
|
|
@@ -190,6 +359,82 @@ module.exports.register = function () {
|
|
|
190
359
|
`${r.outsideGenerated} outside the generated partials and examples, and ${r.otherComponents} from other components`
|
|
191
360
|
)
|
|
192
361
|
})
|
|
362
|
+
|
|
363
|
+
// Antora runs listeners of an event one at a time, in registration order,
|
|
364
|
+
// which follows the order of the playbook's extension list. Prepending puts
|
|
365
|
+
// this listener first wherever the extension is listed, so the files exist
|
|
366
|
+
// before generate-rp-connect-info's guard and any other contentClassified
|
|
367
|
+
// listener reads the connect partials.
|
|
368
|
+
const subscribe = typeof this.prependListener === 'function' ? this.prependListener : this.on
|
|
369
|
+
subscribe.call(this, 'contentClassified', async ({ contentCatalog }) => {
|
|
370
|
+
if (gitSourceMode) return
|
|
371
|
+
if (!contentCatalog.getComponent(COMPONENT)) return
|
|
372
|
+
const localDir = (process.env[LOCAL_DIR_ENV] || '').trim()
|
|
373
|
+
let files
|
|
374
|
+
let origin
|
|
375
|
+
let from
|
|
376
|
+
if (localDir) {
|
|
377
|
+
setResolvedConnectRef(configuredTag)
|
|
378
|
+
try {
|
|
379
|
+
files = await asset.readLocal(localDir)
|
|
380
|
+
} catch (error) {
|
|
381
|
+
throw new Error(`Could not read the Redpanda Connect reference docs from ${LOCAL_DIR_ENV} (${localDir}): ${error.message}`)
|
|
382
|
+
}
|
|
383
|
+
origin = { type: 'local', url: CONNECT_URL, startPath: 'docs', localDir }
|
|
384
|
+
from = `${LOCAL_DIR_ENV} (${localDir})`
|
|
385
|
+
} else {
|
|
386
|
+
let tag = configuredTag || (await resolveLatestTag(`${CONNECT_URL}.git`, logger))
|
|
387
|
+
if (!tag) {
|
|
388
|
+
throw new Error(
|
|
389
|
+
'Could not resolve the latest Redpanda Connect release, so the connect reference docs cannot be downloaded. ' +
|
|
390
|
+
`Set a GitHub token, check network access, or set the \`tag\` config of this extension or ${LOCAL_DIR_ENV}.`
|
|
391
|
+
)
|
|
392
|
+
}
|
|
393
|
+
setResolvedConnectRef(tag)
|
|
394
|
+
let archive = await downloadRelease(tag, logger)
|
|
395
|
+
// The latest release can be published before its asset is attached, or
|
|
396
|
+
// its asset job can fail. An explicit `tag` means that release only.
|
|
397
|
+
if (!archive && !configuredTag) {
|
|
398
|
+
const older = await findOlderReleaseWithAsset(tag, `${CONNECT_URL}.git`, logger)
|
|
399
|
+
if (older) {
|
|
400
|
+
logger.warn(`Redpanda Connect ${tag} has no ${asset.ASSET_NAME} asset yet; using ${older.tag}`)
|
|
401
|
+
tag = older.tag
|
|
402
|
+
archive = older.archive
|
|
403
|
+
setResolvedConnectRef(tag)
|
|
404
|
+
}
|
|
405
|
+
}
|
|
406
|
+
if (!archive && configuredTag) {
|
|
407
|
+
throw new Error(
|
|
408
|
+
`Redpanda Connect release ${tag} has no ${asset.ASSET_NAME} asset (404). ` +
|
|
409
|
+
`Set the \`tag\` config of modify-connect-tag-playbook to a release that has it, or remove \`tag\` to use the latest release.`
|
|
410
|
+
)
|
|
411
|
+
}
|
|
412
|
+
if (!archive) {
|
|
413
|
+
logger.info(`Redpanda Connect ${tag} has no ${asset.ASSET_NAME} release asset (404), so no generated reference docs were added from it`)
|
|
414
|
+
return
|
|
415
|
+
}
|
|
416
|
+
try {
|
|
417
|
+
files = await asset.readTarGz(archive)
|
|
418
|
+
} catch (error) {
|
|
419
|
+
throw new Error(`The ${asset.ASSET_NAME} release asset of Redpanda Connect ${tag} is corrupt: ${error.message}`)
|
|
420
|
+
}
|
|
421
|
+
origin = { type: 'release-asset', url: CONNECT_URL, startPath: 'docs', reftype: 'tag', refname: tag, tag, asset: asset.assetUrl(tag) }
|
|
422
|
+
from = `the ${tag} ${asset.ASSET_NAME} release asset`
|
|
423
|
+
}
|
|
424
|
+
if (!files.some((f) => asset.toResource(f.path))) {
|
|
425
|
+
throw new Error(`${from} has no files under modules/components/partials or modules/components/examples`)
|
|
426
|
+
}
|
|
427
|
+
const r = addConnectDocs(contentCatalog, files, origin, { replace: !!localDir })
|
|
428
|
+
const versions = r.versions.filter(Boolean)
|
|
429
|
+
logger.info(
|
|
430
|
+
`Redpanda Connect reference docs from ${from}: added ${r.added} files to the ${COMPONENT} component` +
|
|
431
|
+
`${versions.length ? ` (${versions.join(', ')})` : ''}; ` +
|
|
432
|
+
(localDir
|
|
433
|
+
? `replaced ${r.replaced} provided by another source; skipped ${r.outsideGenerated}`
|
|
434
|
+
: `skipped ${r.providedElsewhere} already provided by another source and ${r.outsideGenerated}`) +
|
|
435
|
+
' outside the generated partials and examples'
|
|
436
|
+
)
|
|
437
|
+
})
|
|
193
438
|
}
|
|
194
439
|
|
|
195
|
-
module.exports._internal = { isRemote, isUpstreamConnect, redact, isConnectSource, isConnectOrigin, toTag, highestStableTag, wantsLatest,
|
|
440
|
+
module.exports._internal = { isRemote, isUpstreamConnect, redact, isConnectSource, isConnectOrigin, toTag, highestStableTag, stableTags, findOlderReleaseWithAsset, resolveLatestTag, FALLBACK_RELEASE_LIMIT, FALLBACK_PROBE_LIMIT, wantsLatest, removeLatestConnectSources, filterConnectContent, hasConnectSource, addConnectDocs, LOCAL_DIR_ENV }
|
|
@@ -362,6 +362,9 @@ const HEADING_RX = /^(={1,6})\s+(\S.*)$/
|
|
|
362
362
|
* which still displays sensibly.
|
|
363
363
|
*/
|
|
364
364
|
function linkTextFor (label, fragment, page, resolveInclude) {
|
|
365
|
+
// A trailing ^ opens a URL link in a new window; an xref would print it.
|
|
366
|
+
// A label that is only ^ counts as no label.
|
|
367
|
+
label = (label || '').replace(/\^$/, '')
|
|
365
368
|
if (label) return label
|
|
366
369
|
if (!fragment) return ''
|
|
367
370
|
const contents = page && page.contents && page.contents.toString()
|
|
@@ -2,9 +2,11 @@
|
|
|
2
2
|
|
|
3
3
|
// Shared Redpanda Connect catalog helpers.
|
|
4
4
|
//
|
|
5
|
-
// The connect repo publishes its reference content (partials and examples)
|
|
6
|
-
//
|
|
7
|
-
//
|
|
5
|
+
// The connect repo publishes its reference content (partials and examples) as
|
|
6
|
+
// a release asset that modify-connect-tag-playbook adds to the content catalog
|
|
7
|
+
// (or, to preview a connect branch, through a content source with explicit
|
|
8
|
+
// refs). Either way the files carry a connect origin with the ref.
|
|
9
|
+
// Catalog data (support level, deprecation, Cloud availability)
|
|
8
10
|
// must come from the same ref, or the badges and catalog drift from the
|
|
9
11
|
// reference content as soon as connect's main branch changes a flag before a
|
|
10
12
|
// release. This module carries that ref between the extensions and turns the
|
|
@@ -13,6 +15,7 @@
|
|
|
13
15
|
|
|
14
16
|
// Matches the GitHub URL (with or without .git) and a local clone whose
|
|
15
17
|
// directory is named connect, which is how the source looks in local builds.
|
|
18
|
+
// Files added from the release asset carry the GitHub URL.
|
|
16
19
|
function isConnectSource (url) {
|
|
17
20
|
return typeof url === 'string' && /(^|[/:])(redpanda-data\/)?connect(\.git)?\/?$/.test(url)
|
|
18
21
|
}
|
|
@@ -42,7 +45,8 @@ function githubRepoOf (url) {
|
|
|
42
45
|
return m ? { owner: m[1], repo: m[2] } : null
|
|
43
46
|
}
|
|
44
47
|
|
|
45
|
-
// The ref of the connect
|
|
48
|
+
// The ref of the connect files in the content catalog: the release asset tag,
|
|
49
|
+
// or the ref of the connect content source as Antora aggregated it. Only refs
|
|
46
50
|
// that exist on the remote count: a tag, or a branch fetched from the remote.
|
|
47
51
|
// A branch of a local clone may exist only on disk, so it is not used.
|
|
48
52
|
function connectOriginRef (contentCatalog) {
|
|
@@ -64,7 +68,8 @@ function connectOriginRef (contentCatalog) {
|
|
|
64
68
|
const CATALOG_RELATIVE = 'platforms/catalog.json'
|
|
65
69
|
|
|
66
70
|
// The generated catalog.json in the content catalog, preferring the copy
|
|
67
|
-
// from the connect content source over one
|
|
71
|
+
// from connect (the release asset or a connect content source) over one
|
|
72
|
+
// another source provides.
|
|
68
73
|
function findConnectCatalogFile (contentCatalog) {
|
|
69
74
|
if (!contentCatalog || typeof contentCatalog.findBy !== 'function') return null
|
|
70
75
|
const candidates = contentCatalog
|