@redpanda-data/docs-extensions-and-macros 5.29.0 → 5.30.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/doc-tools.js +52 -19
- package/bin/mcp-tools/versions.js +1 -1
- package/cli-utils/enterprise-sources.js +6 -6
- package/cli-utils/install-test-dependencies.sh +58 -6
- package/extensions/version-fetcher/set-latest-version.js +11 -2
- package/mcp/CLI_INTERFACE.adoc +1 -1
- package/mcp/USER_GUIDE.adoc +4 -4
- package/package.json +1 -1
- package/tools/bundle-openapi.js +28 -10
- package/tools/get-redpanda-version.js +14 -1
- package/tools/property-extractor/Makefile +88 -42
- package/tools/property-extractor/README.adoc +1 -1
- package/tools/property-extractor/cloud_config.py +11 -7
- package/tools/rpk-docs/rpk-docs-handler.js +98 -16
package/bin/doc-tools.js
CHANGED
|
@@ -119,7 +119,12 @@ programCli
|
|
|
119
119
|
*
|
|
120
120
|
* @requirements
|
|
121
121
|
* - Internet connection to access GitHub API
|
|
122
|
-
* - GitHub
|
|
122
|
+
* - A GitHub token with access to redpanda-data/streaming-enterprise, which is
|
|
123
|
+
* private. Redpanda releases are published there now, and the old public repo
|
|
124
|
+
* is frozen, so this command exits rather than report a stale version when it
|
|
125
|
+
* has no token. Resolved from GIT_CREDENTIALS, REDPANDA_GITHUB_TOKEN,
|
|
126
|
+
* ACTIONS_BOT_TOKEN, GITHUB_TOKEN, VBOT_GITHUB_API_TOKEN, or GH_TOKEN, in
|
|
127
|
+
* that priority order
|
|
123
128
|
*/
|
|
124
129
|
programCli
|
|
125
130
|
.command('get-redpanda-version')
|
|
@@ -1057,8 +1062,10 @@ automation
|
|
|
1057
1062
|
* @requirements
|
|
1058
1063
|
* - Python 3.9 or higher
|
|
1059
1064
|
* - Git
|
|
1060
|
-
* -
|
|
1061
|
-
*
|
|
1065
|
+
* - A GitHub token (resolved from GIT_CREDENTIALS, REDPANDA_GITHUB_TOKEN, ACTIONS_BOT_TOKEN, GITHUB_TOKEN, VBOT_GITHUB_API_TOKEN, or GH_TOKEN, in that priority order) with
|
|
1066
|
+
* access to redpanda-data/streaming-enterprise, which is private
|
|
1067
|
+
* - Internet connection to clone the streaming-enterprise repository
|
|
1068
|
+
* - For --cloud-support: GitHub token with repo permissions (same resolution chain)
|
|
1062
1069
|
* - For --cloud-support: Python packages pyyaml and requests
|
|
1063
1070
|
*/
|
|
1064
1071
|
automation
|
|
@@ -1073,7 +1080,8 @@ automation
|
|
|
1073
1080
|
.option('--regenerate-old-baseline', 'Re-extract the --diff tag from source instead of using the committed attachments/redpanda-properties-<oldTag>.json baseline')
|
|
1074
1081
|
.option('--overrides <path>', 'Optional JSON file with property description overrides', 'docs-data/property-overrides.json')
|
|
1075
1082
|
.option('--output-dir <dir>', 'Where to write all generated files', 'modules/reference')
|
|
1076
|
-
.option('--cloud-support', 'Add AsciiDoc tags to generated property docs to indicate which ones are supported in Redpanda Cloud. This data is fetched from the cloudv2 repository so requires a GitHub token with repo permissions.
|
|
1083
|
+
.option('--cloud-support', 'Add AsciiDoc tags to generated property docs to indicate which ones are supported in Redpanda Cloud. This data is fetched from the cloudv2 repository so requires a GitHub token with repo permissions. The token is resolved from GIT_CREDENTIALS, REDPANDA_GITHUB_TOKEN, ACTIONS_BOT_TOKEN, GITHUB_TOKEN, VBOT_GITHUB_API_TOKEN, or GH_TOKEN, in that priority order', true)
|
|
1084
|
+
.option('--no-cloud-support', 'Skip Cloud support tags, and the cloudv2 repository access they need. A GitHub token is still required either way: the extractor clones the private streaming-enterprise repo to read the source')
|
|
1077
1085
|
.option('--template-property <path>', 'Custom Handlebars template for individual property sections')
|
|
1078
1086
|
.option('--template-topic-property <path>', 'Custom Handlebars template for topic property sections')
|
|
1079
1087
|
.option('--template-topic-property-mappings <path>', 'Custom Handlebars template for topic property mappings table')
|
|
@@ -1091,17 +1099,35 @@ automation
|
|
|
1091
1099
|
|
|
1092
1100
|
const newTag = options.tag || options.branch || 'dev'
|
|
1093
1101
|
|
|
1094
|
-
|
|
1095
|
-
|
|
1096
|
-
|
|
1097
|
-
|
|
1098
|
-
|
|
1099
|
-
|
|
1100
|
-
|
|
1101
|
-
|
|
1102
|
-
|
|
1102
|
+
// Resolved once, via the same priority chain every other GitHub-fetching
|
|
1103
|
+
// command in this CLI already uses (cli-utils/github-token.js) --
|
|
1104
|
+
// notably GIT_CREDENTIALS, which is the token Antora/Netlify builds
|
|
1105
|
+
// actually populate, and which the Makefile's own narrower shell
|
|
1106
|
+
// fallback (REDPANDA_GITHUB_TOKEN/GITHUB_TOKEN/GH_TOKEN only) can't see.
|
|
1107
|
+
// Passed through as GH_TOKEN so the Makefile's clone step picks up
|
|
1108
|
+
// whatever this resolved, private-repo access included, without having
|
|
1109
|
+
// to duplicate the GIT_CREDENTIALS parsing logic in shell.
|
|
1110
|
+
const { getGitHubToken } = require('../cli-utils/github-token')
|
|
1111
|
+
const githubToken = getGitHubToken()
|
|
1112
|
+
|
|
1113
|
+
// A token is required whether or not Cloud metadata is: the extractor
|
|
1114
|
+
// clones the private streaming-enterprise repo to read the source either
|
|
1115
|
+
// way. --no-cloud-support only drops the extra cloudv2 read, so it is not
|
|
1116
|
+
// a remedy for having no token, and pointing at it as one sent people
|
|
1117
|
+
// into a run that got as far as the Makefile's clone step and died there.
|
|
1118
|
+
if (!githubToken) {
|
|
1119
|
+
console.error('Error: Generating property docs requires a GitHub token')
|
|
1120
|
+
console.error(' The extractor reads Redpanda source from the private')
|
|
1121
|
+
console.error(' redpanda-data/streaming-enterprise repository.')
|
|
1122
|
+
console.error(' Set GH_TOKEN, REDPANDA_GITHUB_TOKEN, or GITHUB_TOKEN to a token with access.')
|
|
1123
|
+
if (options.cloudSupport) {
|
|
1124
|
+
console.error(' Cloud support tags (on by default) additionally read the cloudv2 repository;')
|
|
1125
|
+
console.error(' --no-cloud-support skips those, but not the token requirement above.')
|
|
1103
1126
|
}
|
|
1104
|
-
|
|
1127
|
+
process.exit(1)
|
|
1128
|
+
}
|
|
1129
|
+
if (options.cloudSupport) {
|
|
1130
|
+
console.log('Done: GitHub token validated for source and cloudv2 access')
|
|
1105
1131
|
}
|
|
1106
1132
|
|
|
1107
1133
|
let oldTag = options.diff
|
|
@@ -1127,6 +1153,7 @@ automation
|
|
|
1127
1153
|
console.log(`Building property docs for ${tag}…`)
|
|
1128
1154
|
const args = ['build', `TAG=${tag}`]
|
|
1129
1155
|
const env = { ...process.env }
|
|
1156
|
+
if (githubToken) env.GH_TOKEN = githubToken
|
|
1130
1157
|
if (overrides) env.OVERRIDES = path.resolve(overrides)
|
|
1131
1158
|
if (options.cloudSupport) env.CLOUD_SUPPORT = '1'
|
|
1132
1159
|
if (templates.property) env.TEMPLATE_PROPERTY = path.resolve(templates.property)
|
|
@@ -1290,8 +1317,9 @@ automation
|
|
|
1290
1317
|
*
|
|
1291
1318
|
* @description
|
|
1292
1319
|
* Generates comprehensive CLI reference documentation for rpk (Redpanda Keeper).
|
|
1293
|
-
* Clones the Redpanda source
|
|
1294
|
-
*
|
|
1320
|
+
* Clones the Redpanda source from streaming-enterprise (private), builds rpk
|
|
1321
|
+
* with Go, and parses `rpk --print-tree` JSON output. Detects Linux-only
|
|
1322
|
+
* commands by analyzing Go build tags in the source code.
|
|
1295
1323
|
*
|
|
1296
1324
|
* Key features:
|
|
1297
1325
|
* - Clones source from GitHub (sparse checkout for speed)
|
|
@@ -1325,6 +1353,9 @@ automation
|
|
|
1325
1353
|
* @requirements
|
|
1326
1354
|
* - Go must be installed (https://go.dev/)
|
|
1327
1355
|
* - Git must be installed (for cloning source)
|
|
1356
|
+
* - A GitHub token (resolved from GIT_CREDENTIALS, REDPANDA_GITHUB_TOKEN, ACTIONS_BOT_TOKEN, GITHUB_TOKEN, VBOT_GITHUB_API_TOKEN, or GH_TOKEN, in that priority order) with
|
|
1357
|
+
* access to redpanda-data/streaming-enterprise, which is private (not
|
|
1358
|
+
* needed when --from-source points at an existing local checkout)
|
|
1328
1359
|
*/
|
|
1329
1360
|
automation
|
|
1330
1361
|
.command('rpk-docs')
|
|
@@ -1875,7 +1906,7 @@ automation
|
|
|
1875
1906
|
*
|
|
1876
1907
|
* @requirements
|
|
1877
1908
|
* - GitHub token with access to redpanda-data/cloudv2-infra repository
|
|
1878
|
-
* - Token
|
|
1909
|
+
* - Token is resolved from GIT_CREDENTIALS, REDPANDA_GITHUB_TOKEN, ACTIONS_BOT_TOKEN, GITHUB_TOKEN, VBOT_GITHUB_API_TOKEN, or GH_TOKEN, in that priority order
|
|
1879
1910
|
* - Internet connection to access GitHub API
|
|
1880
1911
|
*/
|
|
1881
1912
|
automation
|
|
@@ -2169,7 +2200,9 @@ automation
|
|
|
2169
2200
|
* npx doc-tools generate bundle-openapi --tag $VERSION --surface both
|
|
2170
2201
|
*
|
|
2171
2202
|
* @requirements
|
|
2172
|
-
* - Git to clone
|
|
2203
|
+
* - Git to clone the streaming-enterprise repository
|
|
2204
|
+
* - A GitHub token (resolved from GIT_CREDENTIALS, REDPANDA_GITHUB_TOKEN, ACTIONS_BOT_TOKEN, GITHUB_TOKEN, VBOT_GITHUB_API_TOKEN, or GH_TOKEN, in that priority order) with
|
|
2205
|
+
* access to redpanda-data/streaming-enterprise, which is private
|
|
2173
2206
|
* - Buf tool (automatically installed via npm)
|
|
2174
2207
|
* - Redocly CLI or vacuum for OpenAPI bundling (automatically detected)
|
|
2175
2208
|
* - Internet connection to clone repository
|
|
@@ -2180,7 +2213,7 @@ automation
|
|
|
2180
2213
|
.description('Bundle Redpanda OpenAPI fragments for admin and connect APIs. Requires either --tag or --branch.')
|
|
2181
2214
|
.option('-t, --tag <tag>', 'Git tag for released content')
|
|
2182
2215
|
.option('-b, --branch <branch>', 'Branch name for in-progress content')
|
|
2183
|
-
.option('--repo <url>', 'Repository URL', 'https://github.com/redpanda-data/
|
|
2216
|
+
.option('--repo <url>', 'Repository URL. The default is a private repo, so requires a GitHub token (resolved from GIT_CREDENTIALS, REDPANDA_GITHUB_TOKEN, ACTIONS_BOT_TOKEN, GITHUB_TOKEN, VBOT_GITHUB_API_TOKEN, or GH_TOKEN, in that priority order)', 'https://github.com/redpanda-data/streaming-enterprise.git')
|
|
2184
2217
|
.addOption(new Option('-s, --surface <surface>', 'Which API surfaces to bundle').choices(['admin', 'connect', 'both']).makeOptionMandatory())
|
|
2185
2218
|
.option('--out-admin <path>', 'Output path for admin API', 'admin/redpanda-admin-api.yaml')
|
|
2186
2219
|
.option('--out-connect <path>', 'Output path for connect API', 'connect/redpanda-connect-api.yaml')
|
|
@@ -74,7 +74,7 @@ function getRedpandaVersion(args = {}) {
|
|
|
74
74
|
version,
|
|
75
75
|
docker_tag: `docker.redpanda.com/redpandadata/${dockerRepo}:${version}`,
|
|
76
76
|
is_beta: args.beta || false,
|
|
77
|
-
notes_url: `https://github.com/redpanda-data/
|
|
77
|
+
notes_url: `https://github.com/redpanda-data/streaming-enterprise/releases/tag/${version}`,
|
|
78
78
|
_modelRecommendation: 'haiku'
|
|
79
79
|
};
|
|
80
80
|
|
|
@@ -17,13 +17,13 @@ const { getGitHubToken } = require('./github-token')
|
|
|
17
17
|
const RAW = 'https://raw.githubusercontent.com'
|
|
18
18
|
|
|
19
19
|
// Repos known to be private, so a 404 can legitimately mean "needs auth".
|
|
20
|
-
// Public sources (
|
|
21
|
-
//
|
|
22
|
-
//
|
|
23
|
-
// GitHub blip).
|
|
20
|
+
// Public sources (connect info.csv) 404 only when the file is genuinely
|
|
21
|
+
// missing, and suggesting credentials there sends the reader after a
|
|
22
|
+
// problem that does not exist (e.g. during a transient GitHub blip).
|
|
24
23
|
const PRIVATE_REPO_PREFIXES = [
|
|
25
24
|
`${RAW}/redpanda-data/docs/`,
|
|
26
25
|
`${RAW}/redpanda-data/rp-connect-docs/`,
|
|
26
|
+
`${RAW}/redpanda-data/streaming-enterprise/`,
|
|
27
27
|
]
|
|
28
28
|
|
|
29
29
|
const TOKEN_HINT = ' This source is in a private repository: set GIT_CREDENTIALS (or GITHUB_TOKEN / REDPANDA_GITHUB_TOKEN / ACTIONS_BOT_TOKEN) so the fetch can authenticate.'
|
|
@@ -220,8 +220,8 @@ async function loadEnterpriseSources (options, deps = {}) {
|
|
|
220
220
|
const registryYaml = options.registry
|
|
221
221
|
? readLocal(options.registry)
|
|
222
222
|
: await fetchText(`${RAW}/redpanda-data/docs/${options.docsRef}/shared/modules/ROOT/partials/enterprise-features.yml`)
|
|
223
|
-
const coreHeader = await fetchText(`${RAW}/redpanda-data/
|
|
224
|
-
const configurationHeader = await fetchText(`${RAW}/redpanda-data/
|
|
223
|
+
const coreHeader = await fetchText(`${RAW}/redpanda-data/streaming-enterprise/${options.tag}/src/v/features/enterprise_features.h`, 'core enterprise_features.h')
|
|
224
|
+
const configurationHeader = await fetchText(`${RAW}/redpanda-data/streaming-enterprise/${options.tag}/src/v/config/configuration.h`, 'core configuration.h')
|
|
225
225
|
|
|
226
226
|
let connectRef = options.connectRef
|
|
227
227
|
let infoCsv
|
|
@@ -462,21 +462,73 @@ install_rpk() {
|
|
|
462
462
|
;;
|
|
463
463
|
esac
|
|
464
464
|
|
|
465
|
-
#
|
|
465
|
+
# rpk releases now publish to the private redpanda-data/streaming-enterprise,
|
|
466
|
+
# not the public redpanda-data/redpanda, so a token is required (same
|
|
467
|
+
# priority order as cli-utils/github-token.js's getGitHubToken(), minus
|
|
468
|
+
# GIT_CREDENTIALS, which is Antora's own credential store and doesn't
|
|
469
|
+
# apply to this standalone installer script).
|
|
470
|
+
local gh_token="${REDPANDA_GITHUB_TOKEN:-${ACTIONS_BOT_TOKEN:-${GITHUB_TOKEN:-${VBOT_GITHUB_API_TOKEN:-${GH_TOKEN:-}}}}}"
|
|
471
|
+
if [ -z "$gh_token" ]; then
|
|
472
|
+
log_warn "No GitHub token found (checked REDPANDA_GITHUB_TOKEN, ACTIONS_BOT_TOKEN, GITHUB_TOKEN, VBOT_GITHUB_API_TOKEN, GH_TOKEN)."
|
|
473
|
+
log_warn "rpk releases are now published to the private redpanda-data/streaming-enterprise repo and require authentication."
|
|
474
|
+
log_warn "Please install rpk manually:"
|
|
475
|
+
log_warn "https://docs.redpanda.com/current/get-started/rpk-install/"
|
|
476
|
+
return 1
|
|
477
|
+
fi
|
|
478
|
+
|
|
466
479
|
local rpk_filename="rpk-${rpk_os}-${rpk_arch}.zip"
|
|
467
|
-
local
|
|
468
|
-
|
|
469
|
-
log_info "Detected ${os_name} ${arch_name},
|
|
470
|
-
|
|
480
|
+
local releases_api="https://api.github.com/repos/redpanda-data/streaming-enterprise/releases/latest"
|
|
481
|
+
|
|
482
|
+
log_info "Detected ${os_name} ${arch_name}, looking up the latest rpk release..."
|
|
483
|
+
|
|
484
|
+
# The Authorization header reaches curl through a config file on stdin
|
|
485
|
+
# (--config -), never as an -H argument: an argv header is readable by
|
|
486
|
+
# any process on the host (ps, /proc) for as long as the transfer runs.
|
|
487
|
+
local release_json
|
|
488
|
+
if ! release_json=$(curl -fsSL --retry 5 --retry-all-errors \
|
|
489
|
+
--connect-timeout 30 --max-time 60 --retry-max-time 120 \
|
|
490
|
+
--config - \
|
|
491
|
+
-H "Accept: application/vnd.github+json" \
|
|
492
|
+
"$releases_api" <<EOF
|
|
493
|
+
header = "Authorization: token ${gh_token}"
|
|
494
|
+
EOF
|
|
495
|
+
); then
|
|
496
|
+
log_warn "Failed to query the latest release from streaming-enterprise"
|
|
497
|
+
log_warn "Please install rpk manually:"
|
|
498
|
+
log_warn "https://docs.redpanda.com/current/get-started/rpk-install/"
|
|
499
|
+
return 1
|
|
500
|
+
fi
|
|
501
|
+
|
|
502
|
+
# Resolve the asset's API url (not browser_download_url): only the API
|
|
503
|
+
# asset endpoint accepts the Authorization header this private repo needs.
|
|
504
|
+
local rpk_url
|
|
505
|
+
rpk_url=$(printf '%s' "$release_json" | jq -r --arg name "$rpk_filename" '.assets[] | select(.name == $name) | .url')
|
|
506
|
+
if [ -z "$rpk_url" ] || [ "$rpk_url" = "null" ]; then
|
|
507
|
+
log_warn "Latest streaming-enterprise release has no asset named ${rpk_filename}"
|
|
508
|
+
log_warn "Please install rpk manually:"
|
|
509
|
+
log_warn "https://docs.redpanda.com/current/get-started/rpk-install/"
|
|
510
|
+
return 1
|
|
511
|
+
fi
|
|
512
|
+
|
|
513
|
+
log_info "Downloading ${rpk_filename}..."
|
|
514
|
+
|
|
471
515
|
# Try to download and install rpk.
|
|
472
516
|
# Use -f so HTTP errors (e.g. 403/404/429/503) fail instead of writing an
|
|
473
517
|
# error page into the zip, and retry with curl's default exponential
|
|
474
518
|
# backoff (1s, 2s, 4s, ...) to ride out transient GitHub/CDN and network
|
|
475
519
|
# errors, including rate limits that need time to clear. --retry-max-time
|
|
476
520
|
# caps the whole retry window so CI fails fast on persistent errors.
|
|
521
|
+
# Accept: application/octet-stream is required on the API asset url to
|
|
522
|
+
# receive the binary itself instead of its JSON metadata.
|
|
523
|
+
# Same as the release lookup above: the token travels on stdin, not argv.
|
|
477
524
|
if curl -fL --retry 5 --retry-all-errors \
|
|
478
525
|
--connect-timeout 30 --max-time 300 --retry-max-time 600 \
|
|
479
|
-
-
|
|
526
|
+
--config - \
|
|
527
|
+
-H "Accept: application/octet-stream" \
|
|
528
|
+
-o "$rpk_filename" "$rpk_url" <<EOF
|
|
529
|
+
header = "Authorization: token ${gh_token}"
|
|
530
|
+
EOF
|
|
531
|
+
then
|
|
480
532
|
if unzip "$rpk_filename" 2>/dev/null; then
|
|
481
533
|
mkdir -p ~/.local/bin
|
|
482
534
|
if mv rpk ~/.local/bin/ 2>/dev/null; then
|
|
@@ -16,7 +16,12 @@ module.exports.register = function ({ config }) {
|
|
|
16
16
|
const token = getGitHubToken();
|
|
17
17
|
|
|
18
18
|
if (!token) {
|
|
19
|
-
|
|
19
|
+
// Unauthenticated requests still work for the public repos this
|
|
20
|
+
// extension queries (connect, redpanda-operator), but the Redpanda
|
|
21
|
+
// version now comes from the private streaming-enterprise repo, whose
|
|
22
|
+
// API 404s without a token -- the latest-redpanda-* attributes are then
|
|
23
|
+
// left unset and pages fall back to their antora.yml values.
|
|
24
|
+
logger.warn('GitHub token not set (GIT_CREDENTIALS, REDPANDA_GITHUB_TOKEN, GITHUB_TOKEN, or GH_TOKEN). The Redpanda version lookup against the private streaming-enterprise repo will fail and leave latest-redpanda-* attributes unset; other lookups proceed unauthenticated.');
|
|
20
25
|
}
|
|
21
26
|
|
|
22
27
|
this.on('contentClassified', async ({ contentCatalog }) => {
|
|
@@ -41,7 +46,11 @@ module.exports.register = function ({ config }) {
|
|
|
41
46
|
latestOperatorResult,
|
|
42
47
|
latestConnectResult,
|
|
43
48
|
] = await Promise.allSettled([
|
|
44
|
-
|
|
49
|
+
// streaming-enterprise, not the frozen public 'redpanda' repo: the public
|
|
50
|
+
// repo still answers API calls but stopped receiving releases on
|
|
51
|
+
// 2026-08-20, so resolving from it silently pins the docs to the last
|
|
52
|
+
// pre-freeze version forever.
|
|
53
|
+
GetLatestRedpandaVersion(github, owner, 'streaming-enterprise', logger),
|
|
45
54
|
GetLatestDockerTag(dockerNamespace, 'console', logger),
|
|
46
55
|
GetLatestDockerTag(dockerNamespace, 'redpanda-operator', logger),
|
|
47
56
|
GetLatestConnectVersion(github, owner, 'connect', logger),
|
package/mcp/CLI_INTERFACE.adoc
CHANGED
|
@@ -43,7 +43,7 @@ npx doc-tools get-redpanda-version [--beta]
|
|
|
43
43
|
----
|
|
44
44
|
REDPANDA_VERSION=v25.3.1
|
|
45
45
|
REDPANDA_DOCKER_TAG=docker.redpanda.com/redpandadata/redpanda:v25.3.1
|
|
46
|
-
REDPANDA_RELEASE_NOTES=https://github.com/redpanda-data/
|
|
46
|
+
REDPANDA_RELEASE_NOTES=https://github.com/redpanda-data/streaming-enterprise/releases/tag/v25.3.1
|
|
47
47
|
----
|
|
48
48
|
|
|
49
49
|
*Flags:*
|
package/mcp/USER_GUIDE.adoc
CHANGED
|
@@ -571,7 +571,7 @@ NOTE: Stack traces are only included when `NODE_ENV=development` for security re
|
|
|
571
571
|
"version": "v25.3.1",
|
|
572
572
|
"docker_tag": "docker.redpanda.com/redpandadata/redpanda:v25.3.1",
|
|
573
573
|
"is_beta": false,
|
|
574
|
-
"notes_url": "https://github.com/redpanda-data/
|
|
574
|
+
"notes_url": "https://github.com/redpanda-data/streaming-enterprise/releases/tag/v25.3.1"
|
|
575
575
|
}
|
|
576
576
|
----
|
|
577
577
|
|
|
@@ -592,7 +592,7 @@ notes_url:: GitHub release notes URL
|
|
|
592
592
|
"version": "v25.4.1-rc1",
|
|
593
593
|
"docker_tag": "docker.redpanda.com/redpandadata/redpanda-unstable:v25.4.1-rc1",
|
|
594
594
|
"is_beta": true,
|
|
595
|
-
"notes_url": "https://github.com/redpanda-data/
|
|
595
|
+
"notes_url": "https://github.com/redpanda-data/streaming-enterprise/releases/tag/v25.4.1-rc1"
|
|
596
596
|
}
|
|
597
597
|
----
|
|
598
598
|
|
|
@@ -857,7 +857,7 @@ Claude: *uses get_redpanda_version MCP tool*
|
|
|
857
857
|
The latest Redpanda version is v25.3.1, released on January 15, 2024.
|
|
858
858
|
|
|
859
859
|
Docker tag: docker.redpanda.com/redpandadata/redpanda:v25.3.1
|
|
860
|
-
Release notes: https://github.com/redpanda-data/
|
|
860
|
+
Release notes: https://github.com/redpanda-data/streaming-enterprise/releases/tag/v25.3.1
|
|
861
861
|
----
|
|
862
862
|
|
|
863
863
|
----
|
|
@@ -969,7 +969,7 @@ Claude: *uses generate_property_docs MCP tool*
|
|
|
969
969
|
❌ Generation failed: Tag v99.9.9 not found in Redpanda repository
|
|
970
970
|
|
|
971
971
|
Suggestion: Check that the version exists in the Redpanda repository. You can find available versions at:
|
|
972
|
-
https://github.com/redpanda-data/
|
|
972
|
+
https://github.com/redpanda-data/streaming-enterprise/releases
|
|
973
973
|
----
|
|
974
974
|
|
|
975
975
|
=== Example 7: Property documentation updates (with prompt)
|
package/package.json
CHANGED
package/tools/bundle-openapi.js
CHANGED
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
|
|
3
3
|
const fs = require('fs');
|
|
4
4
|
const path = require('path');
|
|
5
|
-
const { execSync, spawnSync } = require('child_process');
|
|
5
|
+
const { execSync, execFileSync, spawnSync } = require('child_process');
|
|
6
6
|
const yaml = require('yaml');
|
|
7
7
|
const { getMajorMinor } = require('../cli-utils/version');
|
|
8
8
|
|
|
@@ -585,7 +585,7 @@ function postProcessBundle(filePath, options, quiet = false) {
|
|
|
585
585
|
* @param {string} [options.output] - Standalone output file path; when provided, used for the single output file.
|
|
586
586
|
* @param {string} [options.outAdmin] - Output path for the admin API when integrating with doc-tools mode.
|
|
587
587
|
* @param {string} [options.outConnect] - Output path for the connect API when integrating with doc-tools mode.
|
|
588
|
-
* @param {string} [options.repo] - Repository URL to clone (defaults to https://github.com/redpanda-data/
|
|
588
|
+
* @param {string} [options.repo] - Repository URL to clone (defaults to https://github.com/redpanda-data/streaming-enterprise.git, a private repo that requires a GitHub token).
|
|
589
589
|
* @param {string} [options.adminMajor] - Admin API major version string used for metadata (for example, 'v2.0.0').
|
|
590
590
|
* @param {boolean} [options.useAdminMajorVersion] - When true and processing the admin surface, use `adminMajor` for the bundle info.version.
|
|
591
591
|
* @param {boolean} [options.quiet=false] - Suppress logging to stdout/stderr when true.
|
|
@@ -649,24 +649,42 @@ async function bundleOpenAPI(options) {
|
|
|
649
649
|
try {
|
|
650
650
|
// Clone repository (only once for all surfaces)
|
|
651
651
|
if (!quiet) {
|
|
652
|
-
console.log('📥 Cloning
|
|
652
|
+
console.log('📥 Cloning streaming-enterprise repository...');
|
|
653
653
|
}
|
|
654
654
|
|
|
655
|
-
const {
|
|
655
|
+
const { getGitHubToken } = require('../cli-utils/github-token');
|
|
656
656
|
|
|
657
|
-
|
|
657
|
+
const repositoryUrl = repo || 'https://github.com/redpanda-data/streaming-enterprise.git';
|
|
658
|
+
const usingDefaultRepo = !repo;
|
|
659
|
+
const token = getGitHubToken();
|
|
658
660
|
|
|
659
|
-
|
|
660
|
-
|
|
661
|
-
|
|
661
|
+
if (usingDefaultRepo && !token) {
|
|
662
|
+
throw new Error('redpanda-data/streaming-enterprise is a private repository: set GH_TOKEN, GITHUB_TOKEN, or REDPANDA_GITHUB_TOKEN so the clone can authenticate.');
|
|
663
|
+
}
|
|
664
|
+
|
|
665
|
+
// Auth travels via a credential helper registered through GIT_CONFIG_*
|
|
666
|
+
// environment variables, never as a git -c argument, so the token
|
|
667
|
+
// cannot appear in this process's argv (visible to anything that can
|
|
668
|
+
// list processes) or get embedded in an exec error's command-line
|
|
669
|
+
// message. It's also never embedded in the remote URL or persisted in
|
|
670
|
+
// the clone's .git/config.
|
|
671
|
+
const gitEnv = { ...process.env };
|
|
672
|
+
if (token && repositoryUrl.includes('github.com')) {
|
|
662
673
|
if (!quiet) {
|
|
663
674
|
console.log('🔑 Using authenticated clone (token provided)');
|
|
664
675
|
}
|
|
676
|
+
gitEnv.BUNDLE_OPENAPI_CLONE_TOKEN = token;
|
|
677
|
+
gitEnv.GIT_CONFIG_COUNT = '2';
|
|
678
|
+
gitEnv.GIT_CONFIG_KEY_0 = 'credential.helper';
|
|
679
|
+
gitEnv.GIT_CONFIG_VALUE_0 = '';
|
|
680
|
+
gitEnv.GIT_CONFIG_KEY_1 = 'credential.https://github.com.helper';
|
|
681
|
+
gitEnv.GIT_CONFIG_VALUE_1 = '!f() { echo "username=x-access-token"; echo "password=$BUNDLE_OPENAPI_CLONE_TOKEN"; }; f';
|
|
665
682
|
}
|
|
666
683
|
|
|
667
684
|
try {
|
|
668
|
-
|
|
685
|
+
execFileSync('git', ['clone', '--depth', '1', '--branch', tag, repositoryUrl, 'redpanda'], {
|
|
669
686
|
cwd: tempDir,
|
|
687
|
+
env: gitEnv,
|
|
670
688
|
stdio: quiet ? 'ignore' : 'inherit',
|
|
671
689
|
timeout: 60000 // 1 minute timeout
|
|
672
690
|
});
|
|
@@ -825,7 +843,7 @@ if (require.main === module) {
|
|
|
825
843
|
.option('-o, --output <path>', 'Output file path (defaults: admin/redpanda-admin-api.yaml or connect/redpanda-connect-api.yaml)')
|
|
826
844
|
.option('--out-admin <path>', 'Output path for admin API', 'admin/redpanda-admin-api.yaml')
|
|
827
845
|
.option('--out-connect <path>', 'Output path for connect API', 'connect/redpanda-connect-api.yaml')
|
|
828
|
-
.option('--repo <url>', 'Repository URL', 'https://github.com/redpanda-data/
|
|
846
|
+
.option('--repo <url>', 'Repository URL. The default is a private repo, so requires a GitHub token (GH_TOKEN, GITHUB_TOKEN, or REDPANDA_GITHUB_TOKEN)', 'https://github.com/redpanda-data/streaming-enterprise.git')
|
|
829
847
|
.option('--admin-major <string>', 'Admin API major version', 'v2.0.0')
|
|
830
848
|
.option('--use-admin-major-version', 'Use admin major version for info.version instead of git tag', false)
|
|
831
849
|
.option('-q, --quiet', 'Suppress output', false)
|
|
@@ -2,6 +2,7 @@
|
|
|
2
2
|
|
|
3
3
|
const GetLatestRedpandaVersion = require('../extensions/version-fetcher/get-latest-redpanda-version.js');
|
|
4
4
|
const { getPrereleaseFromAntora } = require('../cli-utils/antora-utils.js');
|
|
5
|
+
const { getGitHubToken } = require('../cli-utils/github-token');
|
|
5
6
|
|
|
6
7
|
/**
|
|
7
8
|
* Fetches and prints the latest Redpanda version and Docker repository.
|
|
@@ -10,8 +11,20 @@ const { getPrereleaseFromAntora } = require('../cli-utils/antora-utils.js');
|
|
|
10
11
|
* @param {boolean} options.fromAntora - Whether to derive beta flag from antora.yml
|
|
11
12
|
*/
|
|
12
13
|
module.exports = async function getRedpandaVersion({ beta = false, fromAntora = false }) {
|
|
14
|
+
// streaming-enterprise, not the old public redpanda repo: the public repo
|
|
15
|
+
// still answers API requests but froze at its 2026-08-20 state when the
|
|
16
|
+
// source moved, so resolving "latest" from it silently pins to the last
|
|
17
|
+
// pre-freeze release forever. streaming-enterprise is private, and
|
|
18
|
+
// unauthenticated release listings 404 rather than fail loudly, so refuse
|
|
19
|
+
// to guess without a token instead of quietly resolving nothing.
|
|
13
20
|
const owner = 'redpanda-data';
|
|
14
|
-
const repo = '
|
|
21
|
+
const repo = 'streaming-enterprise';
|
|
22
|
+
|
|
23
|
+
if (!getGitHubToken()) {
|
|
24
|
+
console.error(`❌ redpanda-data/${repo} is a private repository.`);
|
|
25
|
+
console.error(' Set GH_TOKEN, REDPANDA_GITHUB_TOKEN, or GITHUB_TOKEN to a token with access.');
|
|
26
|
+
process.exit(1);
|
|
27
|
+
}
|
|
15
28
|
|
|
16
29
|
// Determine whether to treat this as a beta (RC) release
|
|
17
30
|
let useBeta = beta;
|
|
@@ -76,55 +76,101 @@ clean:
|
|
|
76
76
|
@echo "🧹 Cleaning up…"
|
|
77
77
|
@rm -rf $(VENV) $(TMP_ROOT) $(REPO_ROOT)/autogenerated
|
|
78
78
|
|
|
79
|
-
# --- Clone or update Redpanda, checkout TAG or default branch ---
|
|
79
|
+
# --- Clone or update the Redpanda source (now in streaming-enterprise), checkout TAG or default branch ---
|
|
80
80
|
# Clones are shallow: the extractor only reads source files at a single tag,
|
|
81
81
|
# so full history is wasted time and disk.
|
|
82
|
-
# Auth
|
|
83
|
-
#
|
|
84
|
-
#
|
|
85
|
-
#
|
|
82
|
+
# Auth travels via a credential helper registered through GIT_CONFIG_*
|
|
83
|
+
# environment variables, never as a git -c argument (which would put the
|
|
84
|
+
# token in this process's argv, visible to anything that can list processes
|
|
85
|
+
# on the runner) and never embedded in the remote URL. The clone lives
|
|
86
|
+
# inside the installed package dir, so a persisted token would also be
|
|
87
|
+
# readable by anything that can list it -- GIT_CONFIG_* env vars don't get
|
|
88
|
+
# written to disk, so nothing persists in the clone's .git/config either.
|
|
89
|
+
# streaming-enterprise is private, so a token is required -- not just
|
|
90
|
+
# preferred the way it was when this cloned the public redpanda repo. GH_TOKEN
|
|
91
|
+
# is checked first because `doc-tools generate property-docs` pre-resolves it
|
|
92
|
+
# via cli-utils/github-token.js's full priority chain (notably GIT_CREDENTIALS,
|
|
93
|
+
# the token Antora/Netlify builds actually populate) before invoking this
|
|
94
|
+
# Makefile. REDPANDA_GITHUB_TOKEN/GITHUB_TOKEN are the fallback for invoking
|
|
95
|
+
# `make` directly, without going through doc-tools.js.
|
|
96
|
+
# An existing checkout may predate the streaming-enterprise migration and
|
|
97
|
+
# still have origin pointed at the old public redpanda repo, so the
|
|
98
|
+
# existing-directory path pins origin to REPO_URL before fetching. Pinning
|
|
99
|
+
# origin is not enough by itself: the old repo's cached remote-tracking refs
|
|
100
|
+
# and tags stay behind it, and a fetch that fails (expired token, outage,
|
|
101
|
+
# ref that does not exist) used to fall straight through to checking one of
|
|
102
|
+
# those out -- generating docs from stale public source under a tag that
|
|
103
|
+
# source never contained, with nothing in the log to say so. So a fetch has
|
|
104
|
+
# to succeed before any ref is resolved, the catch-all fetch prunes refs and
|
|
105
|
+
# tags the new origin does not have, the ref that gets checked out is the
|
|
106
|
+
# one the successful fetch actually wrote, and a TAG that is neither a tag
|
|
107
|
+
# nor a branch there is an error rather than a quiet swap to the default
|
|
108
|
+
# branch.
|
|
86
109
|
redpanda-git:
|
|
87
110
|
@mkdir -p "$(TMP_ROOT)"
|
|
88
|
-
@
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
else \
|
|
94
|
-
GH_TOKEN=$${REDPANDA_GITHUB_TOKEN:-$${GITHUB_TOKEN:-$${GH_TOKEN}}}; \
|
|
95
|
-
REPO_URL="https://github.com/redpanda-data/redpanda.git"; \
|
|
96
|
-
if [ -n "$$GH_TOKEN" ]; then \
|
|
97
|
-
echo "🔑 Using authenticated clone (token provided)"; \
|
|
98
|
-
AUTH_HEADER="AUTHORIZATION: basic $$(printf '%s' "x-access-token:$$GH_TOKEN" | base64 | tr -d '\n')"; \
|
|
99
|
-
else \
|
|
100
|
-
AUTH_HEADER=""; \
|
|
101
|
-
fi; \
|
|
102
|
-
git -c http.https://github.com/.extraheader="$$AUTH_HEADER" clone -q --depth 1 --branch "$(TAG)" "$$REPO_URL" "$(REDPANDA_SRC)" 2>/dev/null || \
|
|
103
|
-
git -c http.https://github.com/.extraheader="$$AUTH_HEADER" clone -q --depth 1 "$$REPO_URL" "$(REDPANDA_SRC)"; \
|
|
111
|
+
@GH_TOKEN=$${GH_TOKEN:-$${REDPANDA_GITHUB_TOKEN:-$${GITHUB_TOKEN}}}; \
|
|
112
|
+
if [ -z "$$GH_TOKEN" ]; then \
|
|
113
|
+
echo "❌ redpanda-data/streaming-enterprise is a private repository."; \
|
|
114
|
+
echo " Set GH_TOKEN, REDPANDA_GITHUB_TOKEN, or GITHUB_TOKEN to a token with access."; \
|
|
115
|
+
exit 1; \
|
|
104
116
|
fi; \
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
117
|
+
export GIT_MAKEFILE_CLONE_TOKEN="$$GH_TOKEN"; \
|
|
118
|
+
export GIT_CONFIG_COUNT=2; \
|
|
119
|
+
export GIT_CONFIG_KEY_0=credential.helper; \
|
|
120
|
+
export GIT_CONFIG_VALUE_0=; \
|
|
121
|
+
export GIT_CONFIG_KEY_1=credential.https://github.com.helper; \
|
|
122
|
+
export GIT_CONFIG_VALUE_1='!f() { echo "username=x-access-token"; echo "password=$$GIT_MAKEFILE_CLONE_TOKEN"; }; f'; \
|
|
123
|
+
REPO_URL="https://github.com/redpanda-data/streaming-enterprise.git"; \
|
|
124
|
+
echo "🔄 Cloning/updating streaming-enterprise into $(REDPANDA_SRC)…"; \
|
|
125
|
+
FETCH_LOG="$$(mktemp)"; \
|
|
126
|
+
FETCHED=""; \
|
|
127
|
+
if [ -d "$(REDPANDA_SRC)" ]; then \
|
|
128
|
+
git -C "$(REDPANDA_SRC)" remote set-url origin "$$REPO_URL" 2>/dev/null || \
|
|
129
|
+
git -C "$(REDPANDA_SRC)" remote add origin "$$REPO_URL"; \
|
|
130
|
+
if git -C "$(REDPANDA_SRC)" fetch --depth 1 -q origin "+refs/tags/$(TAG):refs/tags/$(TAG)" >>"$$FETCH_LOG" 2>&1; then \
|
|
131
|
+
FETCHED=tag; \
|
|
132
|
+
elif git -C "$(REDPANDA_SRC)" fetch --depth 1 -q origin "+refs/heads/$(TAG):refs/remotes/origin/$(TAG)" >>"$$FETCH_LOG" 2>&1; then \
|
|
133
|
+
FETCHED=branch; \
|
|
134
|
+
elif git -C "$(REDPANDA_SRC)" fetch --all --tags --prune --prune-tags -q >>"$$FETCH_LOG" 2>&1; then \
|
|
135
|
+
FETCHED=probe; \
|
|
136
|
+
fi; \
|
|
112
137
|
else \
|
|
113
|
-
|
|
114
|
-
if
|
|
115
|
-
|
|
116
|
-
git -C "$(REDPANDA_SRC)" checkout -q "$${DEFAULT_BRANCH}"; \
|
|
117
|
-
else \
|
|
118
|
-
DEFAULT_BRANCH=$$(git -C "$(REDPANDA_SRC)" ls-remote --symref origin HEAD | awk '/^ref:/ {sub(/^refs\/heads\//, "", $$2); print $$2; exit}'); \
|
|
119
|
-
if [ -z "$$DEFAULT_BRANCH" ]; then \
|
|
120
|
-
echo "❌ Tag '$(TAG)' not found and the default branch could not be determined"; \
|
|
121
|
-
exit 1; \
|
|
122
|
-
fi; \
|
|
123
|
-
echo "⚠️ Tag '$(TAG)' not found; falling back to '$${DEFAULT_BRANCH}'"; \
|
|
124
|
-
git -C "$(REDPANDA_SRC)" fetch --depth 1 -q origin "$$DEFAULT_BRANCH"; \
|
|
125
|
-
git -C "$(REDPANDA_SRC)" checkout -q FETCH_HEAD; \
|
|
138
|
+
echo "🔑 Using authenticated clone"; \
|
|
139
|
+
if git clone -q --depth 1 --branch "$(TAG)" "$$REPO_URL" "$(REDPANDA_SRC)" >>"$$FETCH_LOG" 2>&1; then \
|
|
140
|
+
FETCHED=probe; \
|
|
126
141
|
fi; \
|
|
127
|
-
fi
|
|
142
|
+
fi; \
|
|
143
|
+
if [ -z "$$FETCHED" ]; then \
|
|
144
|
+
echo "❌ Could not fetch '$(TAG)' from redpanda-data/streaming-enterprise:"; \
|
|
145
|
+
sed 's/^/ /' "$$FETCH_LOG"; \
|
|
146
|
+
echo " Not continuing: anything already in $(REDPANDA_SRC) can predate the"; \
|
|
147
|
+
echo " streaming-enterprise migration, and generating docs from it would"; \
|
|
148
|
+
echo " publish stale public source under this tag. Check that the token is"; \
|
|
149
|
+
echo " valid and has access to the repository, and that '$(TAG)' exists there."; \
|
|
150
|
+
rm -f "$$FETCH_LOG"; \
|
|
151
|
+
exit 1; \
|
|
152
|
+
fi; \
|
|
153
|
+
rm -f "$$FETCH_LOG"; \
|
|
154
|
+
case "$$FETCHED" in \
|
|
155
|
+
tag) REF="refs/tags/$(TAG)";; \
|
|
156
|
+
branch) REF="refs/remotes/origin/$(TAG)";; \
|
|
157
|
+
*) REF=""; \
|
|
158
|
+
for candidate in "refs/tags/$(TAG)" "refs/remotes/origin/$(TAG)"; do \
|
|
159
|
+
if git -C "$(REDPANDA_SRC)" rev-parse --verify -q "$$candidate" >/dev/null 2>&1; then \
|
|
160
|
+
REF="$$candidate"; \
|
|
161
|
+
break; \
|
|
162
|
+
fi; \
|
|
163
|
+
done;; \
|
|
164
|
+
esac; \
|
|
165
|
+
if [ -z "$$REF" ]; then \
|
|
166
|
+
echo "❌ '$(TAG)' is neither a tag nor a branch in redpanda-data/streaming-enterprise."; \
|
|
167
|
+
echo " Not falling back to the default branch: that would publish docs labelled"; \
|
|
168
|
+
echo " '$(TAG)' built from entirely different source."; \
|
|
169
|
+
exit 1; \
|
|
170
|
+
fi; \
|
|
171
|
+
echo "🔖 Checking out '$$REF'"; \
|
|
172
|
+
git -C "$(REDPANDA_SRC)" checkout -q --detach "$$REF"; \
|
|
173
|
+
git -C "$(REDPANDA_SRC)" reset --hard -q "$$REF"
|
|
128
174
|
|
|
129
175
|
# --- Clone Tree-sitter grammar & generate parser ---
|
|
130
176
|
# Skips npm install and parser generation when the generated output already
|
|
@@ -1296,7 +1296,7 @@ node generate-handlebars-docs.js test-input.json test-output
|
|
|
1296
1296
|
|
|
1297
1297
|
== Additional resources
|
|
1298
1298
|
|
|
1299
|
-
* https://github.com/redpanda-data/
|
|
1299
|
+
* https://github.com/redpanda-data/streaming-enterprise[streaming-enterprise Repository] (private; this is where the extractor clones Redpanda's source from)
|
|
1300
1300
|
* https://github.com/redpanda-data/docs-extensions-and-macros[docs-extensions-and-macros Repository]
|
|
1301
1301
|
* https://tree-sitter.github.io/tree-sitter/[Tree-sitter Documentation]
|
|
1302
1302
|
* https://handlebarsjs.com/[Handlebars.js Guide]
|
|
@@ -6,7 +6,7 @@ This module fetches cloud configuration from the cloudv2 repository to determine
|
|
|
6
6
|
which Redpanda properties are supported, editable, or readonly in cloud deployments.
|
|
7
7
|
|
|
8
8
|
Prerequisites:
|
|
9
|
-
- GITHUB_TOKEN environment variable set with appropriate permissions
|
|
9
|
+
- GH_TOKEN, GITHUB_TOKEN, or REDPANDA_GITHUB_TOKEN environment variable set with appropriate permissions
|
|
10
10
|
- Internet connection to access GitHub API
|
|
11
11
|
|
|
12
12
|
Usage:
|
|
@@ -105,14 +105,14 @@ def fetch_cloud_config(github_token: Optional[str] = None) -> CloudConfig:
|
|
|
105
105
|
"""
|
|
106
106
|
Fetch the latest cloud configuration from the redpanda-data/cloudv2 repository and return it as a CloudConfig.
|
|
107
107
|
|
|
108
|
-
This function uses a GitHub personal access token for authentication. If `github_token` is not provided, it will read GITHUB_TOKEN or REDPANDA_GITHUB_TOKEN from the environment. It downloads the most recent versioned YAML from the repository's install-pack directory, validates expected sections (`customer_managed_configs` and `readonly_cluster_config`), and constructs a CloudConfig instance.
|
|
109
|
-
|
|
108
|
+
This function uses a GitHub personal access token for authentication. If `github_token` is not provided, it will read GH_TOKEN, GITHUB_TOKEN, or REDPANDA_GITHUB_TOKEN from the environment. It downloads the most recent versioned YAML from the repository's install-pack directory, validates expected sections (`customer_managed_configs` and `readonly_cluster_config`), and constructs a CloudConfig instance.
|
|
109
|
+
|
|
110
110
|
Parameters:
|
|
111
|
-
github_token (Optional[str]): Personal access token for GitHub API. If omitted, the function will try environment variables GITHUB_TOKEN or REDPANDA_GITHUB_TOKEN.
|
|
112
|
-
|
|
111
|
+
github_token (Optional[str]): Personal access token for GitHub API. If omitted, the function will try environment variables GH_TOKEN, GITHUB_TOKEN, or REDPANDA_GITHUB_TOKEN.
|
|
112
|
+
|
|
113
113
|
Returns:
|
|
114
114
|
CloudConfig: Parsed cloud configuration for the latest available version.
|
|
115
|
-
|
|
115
|
+
|
|
116
116
|
Raises:
|
|
117
117
|
GitHubAuthError: Authentication or access problems with the GitHub API (including 401/403 responses).
|
|
118
118
|
NetworkError: Network connectivity or timeout failures when contacting the GitHub API.
|
|
@@ -120,7 +120,11 @@ def fetch_cloud_config(github_token: Optional[str] = None) -> CloudConfig:
|
|
|
120
120
|
CloudConfigError: Generic configuration error (for example, missing token) or unexpected internal failures.
|
|
121
121
|
"""
|
|
122
122
|
if not github_token:
|
|
123
|
-
|
|
123
|
+
# GH_TOKEN takes priority: `doc-tools generate property-docs` pre-resolves
|
|
124
|
+
# a token via cli-utils/github-token.js's full priority chain (GIT_CREDENTIALS,
|
|
125
|
+
# ACTIONS_BOT_TOKEN, etc.) and passes it down as GH_TOKEN, so this must see
|
|
126
|
+
# that name too -- not just the two env vars a directly-invoked `make` sets.
|
|
127
|
+
github_token = os.environ.get('GH_TOKEN') or os.environ.get('GITHUB_TOKEN') or os.environ.get('REDPANDA_GITHUB_TOKEN')
|
|
124
128
|
|
|
125
129
|
if not github_token:
|
|
126
130
|
error_msg = (
|
|
@@ -241,7 +241,31 @@ function prepareSourceFromRef(sourceRef, sourcePath = null) {
|
|
|
241
241
|
const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'rpk-source-'))
|
|
242
242
|
const repoDir = path.join(tmpDir, 'redpanda')
|
|
243
243
|
|
|
244
|
-
|
|
244
|
+
// streaming-enterprise is private, so a token is required. Auth travels
|
|
245
|
+
// via a credential helper registered through GIT_CONFIG_* environment
|
|
246
|
+
// variables, never as a git -c argument, so the token cannot appear in
|
|
247
|
+
// this process's argv (visible to anything that can list processes) and
|
|
248
|
+
// never embedded in the remote URL, so no token is persisted in the
|
|
249
|
+
// clone's .git/config either.
|
|
250
|
+
const { getGitHubToken } = require('../../cli-utils/github-token')
|
|
251
|
+
const token = getGitHubToken()
|
|
252
|
+
if (!token) {
|
|
253
|
+
throw new Error(
|
|
254
|
+
'redpanda-data/streaming-enterprise is a private repository.\n' +
|
|
255
|
+
'Set GH_TOKEN, GITHUB_TOKEN, or REDPANDA_GITHUB_TOKEN so the clone can authenticate.'
|
|
256
|
+
)
|
|
257
|
+
}
|
|
258
|
+
const gitEnv = {
|
|
259
|
+
...process.env,
|
|
260
|
+
RPK_SOURCE_CLONE_TOKEN: token,
|
|
261
|
+
GIT_CONFIG_COUNT: '2',
|
|
262
|
+
GIT_CONFIG_KEY_0: 'credential.helper',
|
|
263
|
+
GIT_CONFIG_VALUE_0: '',
|
|
264
|
+
GIT_CONFIG_KEY_1: 'credential.https://github.com.helper',
|
|
265
|
+
GIT_CONFIG_VALUE_1: '!f() { echo "username=x-access-token"; echo "password=$RPK_SOURCE_CLONE_TOKEN"; }; f'
|
|
266
|
+
}
|
|
267
|
+
|
|
268
|
+
console.log(`Sparse-cloning streaming-enterprise repo (ref: ${sourceRef}) to ${repoDir}...`)
|
|
245
269
|
|
|
246
270
|
// Clone with sparse checkout
|
|
247
271
|
const cloneResult = spawnSync('git', [
|
|
@@ -250,27 +274,31 @@ function prepareSourceFromRef(sourceRef, sourcePath = null) {
|
|
|
250
274
|
'--filter=blob:none',
|
|
251
275
|
'--sparse',
|
|
252
276
|
'--branch', sourceRef,
|
|
253
|
-
'https://github.com/redpanda-data/
|
|
277
|
+
'https://github.com/redpanda-data/streaming-enterprise.git',
|
|
254
278
|
repoDir
|
|
255
279
|
], {
|
|
256
280
|
encoding: 'utf8',
|
|
257
281
|
timeout: 120000,
|
|
282
|
+
env: gitEnv,
|
|
258
283
|
stdio: ['pipe', 'pipe', 'pipe']
|
|
259
284
|
})
|
|
260
285
|
|
|
261
286
|
if (cloneResult.status !== 0) {
|
|
262
287
|
throw new Error(
|
|
263
|
-
`Failed to clone
|
|
288
|
+
`Failed to clone streaming-enterprise repo with ref '${sourceRef}'.\n` +
|
|
264
289
|
`Make sure the branch or tag exists.\n` +
|
|
265
290
|
`Error: ${cloneResult.stderr}`
|
|
266
291
|
)
|
|
267
292
|
}
|
|
268
293
|
|
|
269
|
-
// Set sparse checkout to only get rpk
|
|
294
|
+
// Set sparse checkout to only get rpk. With --filter=blob:none, the blobs
|
|
295
|
+
// for src/go/rpk were not fetched by the clone above and are pulled lazily
|
|
296
|
+
// here, so this invocation needs the same credential helper.
|
|
270
297
|
const sparseResult = spawnSync('git', ['sparse-checkout', 'set', 'src/go/rpk'], {
|
|
271
298
|
cwd: repoDir,
|
|
272
299
|
encoding: 'utf8',
|
|
273
|
-
timeout: 60000
|
|
300
|
+
timeout: 60000,
|
|
301
|
+
env: gitEnv
|
|
274
302
|
})
|
|
275
303
|
|
|
276
304
|
if (sparseResult.status !== 0) {
|
|
@@ -292,8 +320,8 @@ function fetchRpkTreeFromSource(sourcePath) {
|
|
|
292
320
|
if (!fs.existsSync(sourcePath)) {
|
|
293
321
|
throw new Error(
|
|
294
322
|
`rpk source directory not found: ${sourcePath}\n` +
|
|
295
|
-
'To use --from-source, you need a local checkout of the
|
|
296
|
-
'Clone it with: git clone https://github.com/redpanda-data/
|
|
323
|
+
'To use --from-source, you need a local checkout of the streaming-enterprise repository (private).\n' +
|
|
324
|
+
'Clone it with: git clone https://github.com/redpanda-data/streaming-enterprise.git\n' +
|
|
297
325
|
'Then point to: <repo>/src/go/rpk'
|
|
298
326
|
)
|
|
299
327
|
}
|
|
@@ -393,8 +421,8 @@ function fetchRpkTreeFromLinuxSource(sourcePath, pluginPins = {}) {
|
|
|
393
421
|
if (!fs.existsSync(absoluteSourcePath)) {
|
|
394
422
|
throw new Error(
|
|
395
423
|
`rpk source directory not found: ${absoluteSourcePath}\n` +
|
|
396
|
-
'Expected a checkout of the
|
|
397
|
-
'Clone it with: git clone https://github.com/redpanda-data/
|
|
424
|
+
'Expected a checkout of the streaming-enterprise repository (private).\n' +
|
|
425
|
+
'Clone it with: git clone https://github.com/redpanda-data/streaming-enterprise.git'
|
|
398
426
|
)
|
|
399
427
|
}
|
|
400
428
|
|
|
@@ -1292,16 +1320,69 @@ function downloadRpkRelease(tag, destDir) {
|
|
|
1292
1320
|
return null
|
|
1293
1321
|
}
|
|
1294
1322
|
|
|
1323
|
+
// Releases now publish to streaming-enterprise (private), not the public
|
|
1324
|
+
// redpanda-data/redpanda. This function already has a fallback (the
|
|
1325
|
+
// caller builds from source instead), so a missing token warns and
|
|
1326
|
+
// returns null here rather than throwing the way the clone-only paths do.
|
|
1327
|
+
const { getGitHubToken } = require('../../cli-utils/github-token')
|
|
1328
|
+
const token = getGitHubToken()
|
|
1329
|
+
if (!token) {
|
|
1330
|
+
console.warn('No GitHub token available for the private streaming-enterprise release download; falling back to a source build.')
|
|
1331
|
+
return null
|
|
1332
|
+
}
|
|
1333
|
+
|
|
1295
1334
|
const assetName = `rpk-${osName}-${archName}.zip`
|
|
1296
|
-
const
|
|
1335
|
+
const checksumAsset = `rpk_${tag.replace(/^v/, '')}_checksums.txt`
|
|
1297
1336
|
const zipPath = path.join(destDir, assetName)
|
|
1337
|
+
// The Authorization header reaches curl through a config file on stdin
|
|
1338
|
+
// (--config -), never as an -H argument: an argv header is readable by
|
|
1339
|
+
// any process on the host (ps, /proc) for as long as the transfer runs.
|
|
1340
|
+
const curlAuthConfig = `header = "Authorization: token ${token}"`
|
|
1341
|
+
|
|
1342
|
+
// streaming-enterprise is private, so the release's browser download URL
|
|
1343
|
+
// (releases/download/<tag>/<asset>) 404s even with a valid token: it
|
|
1344
|
+
// redirects to a signed storage URL that requires an authenticated
|
|
1345
|
+
// browser session, not a bearer token. Only the API's per-asset url
|
|
1346
|
+
// (asset.url, not asset.browser_download_url) accepts this Authorization
|
|
1347
|
+
// header, so resolve assets through the release-by-tag API first.
|
|
1348
|
+
const releaseResult = spawnSync('curl', [
|
|
1349
|
+
'-fsSL', '--retry', '5', '--retry-all-errors',
|
|
1350
|
+
'--connect-timeout', '30', '--max-time', '60',
|
|
1351
|
+
'--config', '-',
|
|
1352
|
+
'-H', 'Accept: application/vnd.github+json',
|
|
1353
|
+
`https://api.github.com/repos/redpanda-data/streaming-enterprise/releases/tags/${tag}`
|
|
1354
|
+
], { encoding: 'utf8', input: curlAuthConfig, timeout: 90000 })
|
|
1355
|
+
|
|
1356
|
+
if (releaseResult.status !== 0) {
|
|
1357
|
+
console.warn(`Could not look up release ${tag} from streaming-enterprise (draft or missing release)`)
|
|
1358
|
+
return null
|
|
1359
|
+
}
|
|
1360
|
+
|
|
1361
|
+
let release
|
|
1362
|
+
try {
|
|
1363
|
+
release = JSON.parse(releaseResult.stdout)
|
|
1364
|
+
} catch {
|
|
1365
|
+
console.warn(`Could not parse release metadata for ${tag}`)
|
|
1366
|
+
return null
|
|
1367
|
+
}
|
|
1368
|
+
|
|
1369
|
+
const assetUrl = (release.assets || []).find(a => a.name === assetName)?.url
|
|
1370
|
+
if (!assetUrl) {
|
|
1371
|
+
console.warn(`Release ${tag} has no asset named ${assetName} (draft or missing release asset)`)
|
|
1372
|
+
return null
|
|
1373
|
+
}
|
|
1374
|
+
const checksumUrl = (release.assets || []).find(a => a.name === checksumAsset)?.url
|
|
1298
1375
|
|
|
1299
1376
|
console.log(`Downloading ${assetName} for ${tag}...`)
|
|
1377
|
+
// Accept: application/octet-stream is required on the API asset url to
|
|
1378
|
+
// receive the binary itself instead of its JSON metadata.
|
|
1300
1379
|
const curlResult = spawnSync('curl', [
|
|
1301
1380
|
'-fL', '--retry', '5', '--retry-all-errors',
|
|
1302
1381
|
'--connect-timeout', '30', '--max-time', '300',
|
|
1303
|
-
'
|
|
1304
|
-
|
|
1382
|
+
'--config', '-',
|
|
1383
|
+
'-H', 'Accept: application/octet-stream',
|
|
1384
|
+
'-o', zipPath, assetUrl
|
|
1385
|
+
], { encoding: 'utf8', input: curlAuthConfig, timeout: 360000 })
|
|
1305
1386
|
|
|
1306
1387
|
if (curlResult.status !== 0) {
|
|
1307
1388
|
console.warn(`Could not download rpk release for ${tag} (draft or missing release asset)`)
|
|
@@ -1309,12 +1390,13 @@ function downloadRpkRelease(tag, destDir) {
|
|
|
1309
1390
|
}
|
|
1310
1391
|
|
|
1311
1392
|
// Verify against the release checksum file when it exists
|
|
1312
|
-
const checksumAsset = `rpk_${tag.replace(/^v/, '')}_checksums.txt`
|
|
1313
1393
|
const checksumPath = path.join(destDir, checksumAsset)
|
|
1314
|
-
const checksumResult = spawnSync('curl', [
|
|
1394
|
+
const checksumResult = checksumUrl ? spawnSync('curl', [
|
|
1315
1395
|
'-fsSL', '--retry', '3', '--connect-timeout', '30', '--max-time', '60',
|
|
1316
|
-
'
|
|
1317
|
-
|
|
1396
|
+
'--config', '-',
|
|
1397
|
+
'-H', 'Accept: application/octet-stream',
|
|
1398
|
+
'-o', checksumPath, checksumUrl
|
|
1399
|
+
], { encoding: 'utf8', input: curlAuthConfig, timeout: 90000 }) : { status: 1 }
|
|
1318
1400
|
|
|
1319
1401
|
if (checksumResult.status === 0) {
|
|
1320
1402
|
const expectedLine = fs.readFileSync(checksumPath, 'utf8')
|