@redpanda-data/docs-extensions-and-macros 5.38.10 → 5.41.0

package/bin/doc-tools.js

@@ -76,6 +76,11 @@ programCli
  * - Node.js and npm
  * - Python 3.9 or higher
  * - Docker (for some dependencies)
+ * - Network access to https://rpk.redpanda.com (rpk is downloaded from the
+ *   rpk distribution CDN and checksum-verified; no GitHub token needed).
+ *   Optional: RPK_VERSION=vX.Y.Z pins the rpk version. Without it, the
+ *   newest GA is resolved from streaming-enterprise tags when a GitHub token
+ *   is available, otherwise from latest/ on the CDN.
  */
 programCli
   .command('install-test-dependencies')
@@ -1358,6 +1363,10 @@ automation
  * - A GitHub token (resolved from GIT_CREDENTIALS, REDPANDA_GITHUB_TOKEN, ACTIONS_BOT_TOKEN, GITHUB_TOKEN, VBOT_GITHUB_API_TOKEN, or GH_TOKEN, in that priority order) with
  *   access to redpanda-data/streaming-enterprise, which is private (not
  *   needed when --from-source points at an existing local checkout)
+ * - --plugin refreshes need none of the above when the snapshot's rpk_version
+ *   is a published GA or RC tag: the rpk binary is downloaded from
+ *   https://rpk.redpanda.com and checksum-verified. Go, Git and the token are
+ *   only used as a fallback when the CDN has no build for that tag.
  */
 automation
   .command('rpk-docs')

package/cli-utils/install-test-dependencies.sh

@@ -414,166 +414,65 @@ ensure_dependencies_installed() {
 log_info "Installing/checking dependencies for doc-tools CLI commands..."
 ensure_dependencies_installed
 
-# install_rpk installs Redpanda's rpk CLI into ~/.local/bin by downloading the appropriate release for the current OS and architecture, adding it to PATH for the current and future sessions, and verifying the installation; returns 0 on success and non-zero on failure.
+# install_rpk installs Redpanda's rpk CLI into ~/.local/bin from the rpk
+# distribution CDN (https://rpk.redpanda.com), adds it to PATH for the current
+# and future sessions, and verifies the installation; returns 0 on success and
+# non-zero on failure. Set RPK_VERSION=vX.Y.Z to pin a version. The URL layout,
+# version resolution and checksum verification live in cli-utils/rpk-cdn.js so
+# this script and the rpk docs generator share one implementation.
 install_rpk() {
     if command_exists rpk; then
         log_info "rpk is already installed. Version information:"
         rpk --version
         return 0
     fi
-    
-    log_info "Installing rpk..."
-    
-    # Detect OS and architecture
-    local os_name=$(uname -s)
-    local arch_name=$(uname -m)
-    
-    # Map OS name to rpk release format
-    local rpk_os=""
-    case "$os_name" in
-        "Darwin")
-            rpk_os="darwin"
-            ;;
-        "Linux")
-            rpk_os="linux"
-            ;;
-        *)
-            log_warn "Unsupported operating system: $os_name"
-            log_warn "Please install rpk manually:"
-            log_warn "https://docs.redpanda.com/current/get-started/rpk-install/"
-            return 1
-            ;;
-    esac
-    
-    # Map architecture to rpk release format
-    local rpk_arch=""
-    case "$arch_name" in
-        "x86_64" | "amd64")
-            rpk_arch="amd64"
-            ;;
-        "arm64" | "aarch64")
-            rpk_arch="arm64"
-            ;;
-        *)
-            log_warn "Unsupported architecture: $arch_name"
-            log_warn "Please install rpk manually:"
-            log_warn "https://docs.redpanda.com/current/get-started/rpk-install/"
-            return 1
-            ;;
-    esac
-    
-    # rpk releases now publish to the private redpanda-data/streaming-enterprise,
-    # not the public redpanda-data/redpanda, so a token is required (same
-    # priority order as cli-utils/github-token.js's getGitHubToken(), minus
-    # GIT_CREDENTIALS, which is Antora's own credential store and doesn't
-    # apply to this standalone installer script).
-    local gh_token="${REDPANDA_GITHUB_TOKEN:-${ACTIONS_BOT_TOKEN:-${GITHUB_TOKEN:-${VBOT_GITHUB_API_TOKEN:-${GH_TOKEN:-}}}}}"
-    if [ -z "$gh_token" ]; then
-        log_warn "No GitHub token found (checked REDPANDA_GITHUB_TOKEN, ACTIONS_BOT_TOKEN, GITHUB_TOKEN, VBOT_GITHUB_API_TOKEN, GH_TOKEN)."
-        log_warn "rpk releases are now published to the private redpanda-data/streaming-enterprise repo and require authentication."
-        log_warn "Please install rpk manually:"
-        log_warn "https://docs.redpanda.com/current/get-started/rpk-install/"
-        return 1
-    fi
 
-    local rpk_filename="rpk-${rpk_os}-${rpk_arch}.zip"
-    local releases_api="https://api.github.com/repos/redpanda-data/streaming-enterprise/releases/latest"
+    log_info "Installing rpk..."
 
-    log_info "Detected ${os_name} ${arch_name}, looking up the latest rpk release..."
+    local script_dir
+    script_dir="$(cd "$(dirname "${BASH_SOURCE[0]:-$0}")" && pwd)"
+    local tmp_dir
+    tmp_dir="$(mktemp -d)"
+    local rpk_bin
 
-    # The Authorization header reaches curl through a config file on stdin
-    # (--config -), never as an -H argument: an argv header is readable by
-    # any process on the host (ps, /proc) for as long as the transfer runs.
-    local release_json
-    if ! release_json=$(curl -fsSL --retry 5 --retry-all-errors \
-            --connect-timeout 30 --max-time 60 --retry-max-time 120 \
-            --config - \
-            -H "Accept: application/vnd.github+json" \
-            "$releases_api" <<EOF
-header = "Authorization: token ${gh_token}"
-EOF
-    ); then
-        log_warn "Failed to query the latest release from streaming-enterprise"
-        log_warn "Please install rpk manually:"
+    # node is always present: this script is launched by bin/doc-tools.js.
+    # Without RPK_VERSION the module resolves the latest release itself
+    # (GitHub tags when a token is available, otherwise latest/ on the CDN).
+    if ! rpk_bin=$(node "${script_dir}/rpk-cdn.js" install --dest "${tmp_dir}" ${RPK_VERSION:+--version "${RPK_VERSION}"}); then
+        rm -rf "${tmp_dir}"
+        log_warn "Could not install rpk from https://rpk.redpanda.com (see the message above)."
+        log_warn "Set RPK_VERSION=vX.Y.Z to pin a version, or install rpk manually:"
         log_warn "https://docs.redpanda.com/current/get-started/rpk-install/"
         return 1
     fi
 
-    # Resolve the asset's API url (not browser_download_url): only the API
-    # asset endpoint accepts the Authorization header this private repo needs.
-    local rpk_url
-    rpk_url=$(printf '%s' "$release_json" | jq -r --arg name "$rpk_filename" '.assets[] | select(.name == $name) | .url')
-    if [ -z "$rpk_url" ] || [ "$rpk_url" = "null" ]; then
-        log_warn "Latest streaming-enterprise release has no asset named ${rpk_filename}"
+    mkdir -p ~/.local/bin
+    if ! mv "${rpk_bin}" ~/.local/bin/rpk; then
+        rm -rf "${tmp_dir}"
+        log_warn "Failed to move rpk binary to ~/.local/bin/"
         log_warn "Please install rpk manually:"
         log_warn "https://docs.redpanda.com/current/get-started/rpk-install/"
         return 1
     fi
+    rm -rf "${tmp_dir}"
 
-    log_info "Downloading ${rpk_filename}..."
+    # Add to PATH for current session
+    export PATH=$HOME/.local/bin:$PATH
 
-    # Try to download and install rpk.
-    # Use -f so HTTP errors (e.g. 403/404/429/503) fail instead of writing an
-    # error page into the zip, and retry with curl's default exponential
-    # backoff (1s, 2s, 4s, ...) to ride out transient GitHub/CDN and network
-    # errors, including rate limits that need time to clear. --retry-max-time
-    # caps the whole retry window so CI fails fast on persistent errors.
-    # Accept: application/octet-stream is required on the API asset url to
-    # receive the binary itself instead of its JSON metadata.
-    # Same as the release lookup above: the token travels on stdin, not argv.
-    if curl -fL --retry 5 --retry-all-errors \
-            --connect-timeout 30 --max-time 300 --retry-max-time 600 \
-            --config - \
-            -H "Accept: application/octet-stream" \
-            -o "$rpk_filename" "$rpk_url" <<EOF
-header = "Authorization: token ${gh_token}"
-EOF
-    then
-        if unzip "$rpk_filename" 2>/dev/null; then
-            mkdir -p ~/.local/bin
-            if mv rpk ~/.local/bin/ 2>/dev/null; then
-                rm "$rpk_filename"
-                
-                # Add to PATH for current session
-                export PATH=$HOME/.local/bin:$PATH
-                
-                # Add the target directory to PATH for future sessions
-                if ! grep -q 'export PATH=$HOME/.local/bin:$PATH' ~/.bashrc 2>/dev/null; then
-                    echo 'export PATH=$HOME/.local/bin:$PATH' >> ~/.bashrc
-                fi
-                
-                # Verify installation
-                if command_exists rpk; then
-                    log_info "rpk has been installed successfully. Version information:"
-                    rpk --version
-                    return 0
-                else
-                    log_warn "rpk installation may have failed. Please install manually:"
-                    log_warn "https://docs.redpanda.com/current/get-started/rpk-install/"
-                    return 1
-                fi
-            else
-                log_warn "Failed to move rpk binary to ~/.local/bin/"
-                rm -f "$rpk_filename" rpk 2>/dev/null
-                log_warn "Please install rpk manually:"
-                log_warn "https://docs.redpanda.com/current/get-started/rpk-install/"
-                return 1
-            fi
-        else
-            log_warn "Failed to unzip $rpk_filename (may not exist for ${rpk_os}-${rpk_arch})"
-            rm -f "$rpk_filename" 2>/dev/null
-            log_warn "Please install rpk manually:"
-            log_warn "https://docs.redpanda.com/current/get-started/rpk-install/"
-            return 1
-        fi
-    else
-        log_warn "Failed to download $rpk_url"
-        # A mid-transfer failure can leave a partial file behind even with -f.
-        rm -f "$rpk_filename" 2>/dev/null
-        log_warn "Please install rpk manually:"
-        log_warn "https://docs.redpanda.com/current/get-started/rpk-install/"
-        return 1
+    # Add the target directory to PATH for future sessions
+    if ! grep -q 'export PATH=$HOME/.local/bin:$PATH' ~/.bashrc 2>/dev/null; then
+        echo 'export PATH=$HOME/.local/bin:$PATH' >> ~/.bashrc
     fi
+
+    # Verify installation
+    if command_exists rpk; then
+        log_info "rpk has been installed successfully. Version information:"
+        rpk --version
+        return 0
+    fi
+    log_warn "rpk installation may have failed. Please install manually:"
+    log_warn "https://docs.redpanda.com/current/get-started/rpk-install/"
+    return 1
 }
 
 # Install rpk for rpcn-connector-docs command

package/cli-utils/rpk-cdn.js

@@ -0,0 +1,549 @@
+'use strict'
+
+/**
+ * rpk binary distribution on https://rpk.redpanda.com (DEVPROD-4723).
+ *
+ * rpk release binaries stopped shipping as GitHub Release assets in
+ * September 2026 and are served from an S3-backed CDN instead. This module
+ * is the single definition of that layout for every consumer in this
+ * package (the rpk docs generator and install-test-dependencies.sh):
+ *
+ *   https://rpk.redpanda.com/v<tag>/rpk-<os>-<arch>.zip
+ *   https://rpk.redpanda.com/v<tag>/rpk_<version>_checksums.txt   (sha256sum format)
+ *   https://rpk.redpanda.com/latest/...                            (newest GA, copied at promote)
+ *
+ * Both GA (vX.Y.Z) and RC (vX.Y.Z-rcN) tags are published under v<tag>/.
+ * No authentication is needed. The bucket has no listing, and a missing key
+ * answers HTTP 403 (AccessDenied), not 404, so "is this version published"
+ * is a request for its checksums file and both codes mean "not published".
+ *
+ * Runs as a CLI too (see usage() below) so the bash installer shares this
+ * code instead of carrying its own copy of the URL layout.
+ */
+
+const fs = require('fs')
+const os = require('os')
+const path = require('path')
+const crypto = require('crypto')
+const { spawnSync } = require('child_process')
+
+const RPK_CDN_BASE = 'https://rpk.redpanda.com'
+const RPK_CDN_LATEST_PREFIX = 'latest'
+// GA and RC tags both have a build under v<tag>/
+const RPK_RELEASE_TAG_RE = /^v\d+\.\d+\.\d+(-rc\d+)?$/
+// Only GA tags are ever promoted to latest/
+const RPK_GA_TAG_RE = /^v\d+\.\d+\.\d+$/
+const RPK_OS_BY_PLATFORM = { darwin: 'darwin', linux: 'linux', win32: 'windows' }
+const RPK_ARCH_BY_NODE_ARCH = { arm64: 'arm64', x64: 'amd64' }
+// --platform/--arch are Node's own names (process.platform/process.arch), but
+// the CLI asset spelling ("windows", "amd64") leaks into error messages, so
+// callers reasonably try that spelling too. Accept both.
+const PLATFORM_ALIASES = { windows: 'win32' }
+const ARCH_ALIASES = { amd64: 'x64', x86_64: 'x64', aarch64: 'arm64' }
+// S3 without ListBucket answers 403 for a missing key; a 404 would mean the
+// same thing if the distribution ever changes.
+const NOT_PUBLISHED_HTTP_CODES = new Set([403, 404])
+
+const TAGS_OWNER = 'redpanda-data'
+const TAGS_REPO = 'streaming-enterprise'
+
+/**
+ * Normalise a version to the tag shape the CDN publishes under.
+ * @param {string} version - "26.2.2", "v26.2.2" or "v26.3.0-rc1"
+ * @returns {string|null} "v"-prefixed tag, or null when the shape is not a release tag
+ */
+function normalizeRpkTag(version) {
+  if (typeof version !== 'string') return null
+  const trimmed = version.trim()
+  if (!trimmed) return null
+  const tag = trimmed.startsWith('v') ? trimmed : `v${trimmed}`
+  return RPK_RELEASE_TAG_RE.test(tag) ? tag : null
+}
+
+/**
+ * Release asset name for a platform.
+ * @param {Object} [opts]
+ * @param {string} [opts.platform=process.platform]
+ * @param {string} [opts.arch=process.arch]
+ * @returns {string|null} e.g. "rpk-linux-amd64.zip", or null when unsupported
+ */
+function rpkAssetName({ platform = process.platform, arch = process.arch } = {}) {
+  const osName = RPK_OS_BY_PLATFORM[PLATFORM_ALIASES[platform] || platform]
+  const archName = RPK_ARCH_BY_NODE_ARCH[ARCH_ALIASES[arch] || arch]
+  if (!osName || !archName) return null
+  return `rpk-${osName}-${archName}.zip`
+}
+
+/**
+ * goreleaser checksums file name for a tag.
+ * @param {string} tag - e.g. "v26.2.2"
+ * @returns {string} e.g. "rpk_26.2.2_checksums.txt"
+ */
+function rpkChecksumsName(tag) {
+  return `rpk_${tag.replace(/^v/, '')}_checksums.txt`
+}
+
+/**
+ * Build a CDN URL.
+ * @param {string} prefix - "v26.2.2" or "latest"
+ * @param {string} fileName
+ */
+function rpkCdnUrl(prefix, fileName) {
+  return `${RPK_CDN_BASE}/${prefix}/${fileName}`
+}
+
+/**
+ * All URLs for a tag on the current (or given) platform.
+ * @param {string} tag - normalised tag
+ * @param {Object} [opts] - platform/arch overrides (see rpkAssetName)
+ * @returns {{ zip: string, checksums: string, assetName: string, checksumsName: string }|null}
+ */
+function rpkCdnUrls(tag, opts = {}) {
+  const assetName = rpkAssetName(opts)
+  if (!assetName) return null
+  const checksumsName = rpkChecksumsName(tag)
+  return {
+    assetName,
+    checksumsName,
+    zip: rpkCdnUrl(tag, assetName),
+    checksums: rpkCdnUrl(tag, checksumsName)
+  }
+}
+
+function isNotPublished(httpCode) {
+  return NOT_PUBLISHED_HTTP_CODES.has(httpCode)
+}
+
+function parseHttpCode(stdout) {
+  const trimmed = String(stdout || '').trim()
+  return parseInt(trimmed.slice(-3), 10) || 0
+}
+
+/**
+ * Download a URL to a file with curl.
+ *
+ * The HTTP status travels through -w '%{http_code}' rather than curl's exit
+ * code: with -f a 403 exits 22 on most builds but 56 on others, so callers
+ * gate on the status. --retry alone only retries transient errors (timeouts,
+ * 5xx, 429); --retry-all-errors is deliberately absent so a deterministic
+ * 403 for an unpublished version is not retried for half a minute.
+ *
+ * @param {string} url
+ * @param {string} dest - Output path; removed again on failure
+ * @param {Object} [opts]
+ * @param {number} [opts.maxTime=300] - curl --max-time in seconds
+ * @param {number} [opts.retries=3]
+ * @returns {{ ok: boolean, httpCode: number, status: number|null, stderr: string }}
+ */
+function curlToFile(url, dest, { maxTime = 300, retries = 3 } = {}) {
+  const result = spawnSync('curl', [
+    '-sSfL', '--retry', String(retries), '--retry-max-time', String(maxTime),
+    '--connect-timeout', '30', '--max-time', String(maxTime),
+    '-o', dest, '-w', '%{http_code}', url
+  ], { encoding: 'utf8', timeout: (maxTime + 60) * 1000 })
+
+  const httpCode = parseHttpCode(result.stdout)
+  const ok = result.status === 0 && httpCode === 200
+  if (!ok) {
+    // A mid-transfer failure can leave a partial file behind even with -f
+    fs.rmSync(dest, { force: true })
+  }
+  return { ok, httpCode, status: result.status, stderr: (result.stderr || '').trim() }
+}
+
+/**
+ * HEAD a URL and return its HTTP status (0 when curl itself failed).
+ * @param {string} url
+ * @returns {number}
+ */
+function curlHead(url) {
+  const result = spawnSync('curl', [
+    '-sfIL', '--retry', '3', '--retry-max-time', '30',
+    '--connect-timeout', '15', '--max-time', '30',
+    '-o', os.devNull, '-w', '%{http_code}', url
+  ], { encoding: 'utf8', timeout: 90000 })
+  return parseHttpCode(result.stdout)
+}
+
+/**
+ * Find the expected sha256 for an asset in a sha256sum-format checksums file.
+ * @param {string} text - checksums file content
+ * @param {string} assetName
+ * @returns {string|null} lowercase hex digest, or null when the asset is not listed
+ */
+function parseChecksums(text, assetName) {
+  const line = String(text || '')
+    .split(/\r?\n/)
+    .map(l => l.trim())
+    .find(l => l.endsWith(assetName))
+  if (!line) return null
+  const [digest] = line.split(/\s+/)
+  return digest ? digest.toLowerCase() : null
+}
+
+function sha256File(filePath) {
+  return crypto.createHash('sha256').update(fs.readFileSync(filePath)).digest('hex')
+}
+
+/**
+ * Download and verify the rpk release binary for a tag.
+ *
+ * Returns null whenever the CDN has no complete build for the tag (HTTP
+ * 403/404 on the checksums file or the zip, network failure) so callers can
+ * fall back to a source build. Throws only for a published build that is
+ * wrong: a checksums file without this asset, a digest mismatch, or an
+ * archive that does not extract to an rpk binary.
+ *
+ * @param {string} version - Release tag or bare version (GA or RC)
+ * @param {string} destDir - Directory to download and extract into
+ * @param {Object} [opts]
+ * @param {string} [opts.platform=process.platform]
+ * @param {string} [opts.arch=process.arch]
+ * @param {{ log: Function, warn: Function }} [opts.log=console]
+ * @returns {string|null} Path to the extracted binary, or null when not published
+ */
+function downloadRpkBinary(version, destDir, { platform = process.platform, arch = process.arch, log = console } = {}) {
+  const tag = normalizeRpkTag(version)
+  if (!tag) {
+    log.warn(`'${version}' is not a release tag (expected vX.Y.Z or vX.Y.Z-rcN); no rpk build to download`)
+    return null
+  }
+
+  const urls = rpkCdnUrls(tag, { platform, arch })
+  if (!urls) {
+    log.warn(`No rpk release asset for platform ${platform}/${arch}`)
+    return null
+  }
+
+  // The checksums file first: it is small, it is uploaded together with the
+  // zips, and its absence is the cheapest "not published" signal.
+  const checksumsPath = path.join(destDir, urls.checksumsName)
+  const checksumsResult = curlToFile(urls.checksums, checksumsPath, { maxTime: 60 })
+  if (!checksumsResult.ok) {
+    if (isNotPublished(checksumsResult.httpCode)) {
+      log.warn(`rpk ${tag} is not published on ${RPK_CDN_BASE} (HTTP ${checksumsResult.httpCode} for ${urls.checksums})`)
+      return null
+    }
+    // Anything other than 403/404 is the CDN or the network failing, not a
+    // tag with no build. Returning null here would send the caller into a
+    // private source build over a registry blip, and the installer would exit
+    // 0 having installed something other than what was asked for.
+    throw new Error(`Could not fetch ${urls.checksums} (HTTP ${checksumsResult.httpCode}, curl exit ${checksumsResult.status}${checksumsResult.stderr ? `: ${checksumsResult.stderr}` : ''})`)
+  }
+
+  log.log(`Downloading ${urls.assetName} for ${tag} from ${RPK_CDN_BASE}...`)
+  const zipPath = path.join(destDir, urls.assetName)
+  const zipResult = curlToFile(urls.zip, zipPath, { maxTime: 300 })
+  if (!zipResult.ok) {
+    if (isNotPublished(zipResult.httpCode)) {
+      // Checksums present but zip missing: an upload in progress or a
+      // platform the release skipped. Fall back rather than fail.
+      log.warn(`rpk ${tag} has a checksums file but no ${urls.assetName} on ${RPK_CDN_BASE} (HTTP ${zipResult.httpCode} for ${urls.zip})`)
+      return null
+    }
+    throw new Error(`Could not download ${urls.zip} (HTTP ${zipResult.httpCode}, curl exit ${zipResult.status}${zipResult.stderr ? `: ${zipResult.stderr}` : ''})`)
+  }
+
+  const expected = parseChecksums(fs.readFileSync(checksumsPath, 'utf8'), urls.assetName)
+  if (!expected) {
+    throw new Error(`${urls.checksumsName} for ${tag} lists no entry for ${urls.assetName}; refusing to install an unverified binary`)
+  }
+  const actual = sha256File(zipPath)
+  if (expected !== actual) {
+    throw new Error(
+      `Checksum mismatch for ${urls.assetName} (${tag}):\n` +
+      `  expected ${expected}\n  actual   ${actual}`
+    )
+  }
+  log.log('Checksum verified')
+
+  const unzipResult = spawnSync('unzip', ['-o', zipPath, '-d', destDir], {
+    encoding: 'utf8',
+    timeout: 60000
+  })
+  if (unzipResult.error) {
+    // spawnSync could not start unzip at all (typically ENOENT). That is a
+    // gap in this environment, not a bad archive, so leave the caller its
+    // source-build fallback and say why.
+    log.warn(`Could not run unzip to extract ${urls.assetName}: ${unzipResult.error.message}`)
+    return null
+  }
+  if (unzipResult.status !== 0) {
+    throw new Error(`Failed to extract ${urls.assetName}: ${unzipResult.stderr}`)
+  }
+
+  const binPath = path.join(destDir, 'rpk')
+  if (!fs.existsSync(binPath)) {
+    throw new Error(`Extracted archive did not contain an rpk binary: ${zipPath}`)
+  }
+  fs.chmodSync(binPath, 0o755)
+  return binPath
+}
+
+/**
+ * Newest GA tag on streaming-enterprise that has a build on the CDN.
+ * Tags, not releases: RC tags never had a GitHub Release, and GA releases
+ * stopped being published with the S3 move.
+ */
+async function newestPublishedGaTagFromGitHub(octokit, probe, { platform, arch, log }) {
+  const semver = require('semver')
+  const refs = await octokit.paginate(octokit.rest.git.listMatchingRefs, {
+    owner: TAGS_OWNER,
+    repo: TAGS_REPO,
+    ref: 'tags/v',
+    per_page: 100
+  })
+  const gaTags = refs
+    .map(r => r.ref.replace(/^refs\/tags\//, ''))
+    .filter(t => RPK_GA_TAG_RE.test(t))
+  const versions = semver.rsort(gaTags.map(t => t.slice(1)))
+  // A tag can exist a few minutes before its build lands, so walk down a
+  // handful of candidates instead of trusting the newest tag blindly.
+  for (const v of versions.slice(0, 5)) {
+    const tag = `v${v}`
+    const urls = rpkCdnUrls(tag, { platform, arch })
+    const code = probe(urls.checksums)
+    if (code === 200) return tag
+    log.warn(`Newest tag ${tag} has no build on ${RPK_CDN_BASE} yet (HTTP ${code}); trying the previous GA`)
+  }
+  return null
+}
+
+/**
+ * Identify the version behind latest/ by downloading it and asking the
+ * binary. The CDN has no listing and no version marker, so this is the only
+ * tokenless way to learn what latest/ holds. Once the reported version names
+ * a tag, the already-downloaded zip is hashed against that tag's own
+ * checksums file: latest/ is supposed to be a byte copy of v<tag>/, so a
+ * mismatch means the reported version does not describe these bytes, and the
+ * tag is not trusted. The caller still re-downloads and re-verifies through
+ * downloadRpkBinary before installing anything, so this only gates whether
+ * the resolved tag is trustworthy, not the final install.
+ */
+function versionBehindLatest(probe, { platform, arch, log }) {
+  const assetName = rpkAssetName({ platform, arch })
+  if (!assetName) return null
+  const latestZipUrl = rpkCdnUrl(RPK_CDN_LATEST_PREFIX, assetName)
+  const code = probe(latestZipUrl)
+  if (code !== 200) {
+    log.warn(`${RPK_CDN_LATEST_PREFIX}/ is not populated on ${RPK_CDN_BASE} (HTTP ${code} for ${latestZipUrl})`)
+    return null
+  }
+
+  const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'rpk-latest-'))
+  try {
+    const zipPath = path.join(tmpDir, assetName)
+    const dl = curlToFile(latestZipUrl, zipPath, { maxTime: 300 })
+    if (!dl.ok) {
+      log.warn(`Could not download ${latestZipUrl} (HTTP ${dl.httpCode})`)
+      return null
+    }
+    const unzip = spawnSync('unzip', ['-o', zipPath, '-d', tmpDir], { encoding: 'utf8', timeout: 60000 })
+    const binPath = path.join(tmpDir, 'rpk')
+    if (unzip.status !== 0 || !fs.existsSync(binPath)) {
+      log.warn(`Could not extract ${assetName} from ${RPK_CDN_LATEST_PREFIX}/`)
+      return null
+    }
+    fs.chmodSync(binPath, 0o755)
+    const ver = spawnSync(binPath, ['--version'], { encoding: 'utf8', timeout: 30000 })
+    const match = `${ver.stdout || ''}\n${ver.stderr || ''}`.match(/v\d+\.\d+\.\d+(-rc\d+)?/)
+    if (!match) {
+      log.warn(`Could not read a version from the ${RPK_CDN_LATEST_PREFIX}/ rpk binary`)
+      return null
+    }
+    const tag = match[0]
+
+    const checksumsName = rpkChecksumsName(tag)
+    const checksumsPath = path.join(tmpDir, checksumsName)
+    const checksumsResult = curlToFile(rpkCdnUrl(tag, checksumsName), checksumsPath, { maxTime: 60 })
+    if (!checksumsResult.ok) {
+      log.warn(`Could not verify ${RPK_CDN_LATEST_PREFIX}/ against ${tag}'s checksums (HTTP ${checksumsResult.httpCode}); not trusting it`)
+      return null
+    }
+    const expected = parseChecksums(fs.readFileSync(checksumsPath, 'utf8'), assetName)
+    const actual = sha256File(zipPath)
+    if (!expected || expected !== actual) {
+      log.warn(`${RPK_CDN_LATEST_PREFIX}/ does not match ${tag}'s published checksum for ${assetName}; not trusting it`)
+      return null
+    }
+
+    return tag
+  } finally {
+    fs.rmSync(tmpDir, { recursive: true, force: true })
+  }
+}
+
+/**
+ * Resolve the tag to install when the caller wants "the latest rpk".
+ *
+ * Order:
+ *   1. RPK_VERSION in the environment (pin; invalid shapes throw)
+ *   2. With a GitHub token: newest GA tag on streaming-enterprise whose
+ *      build exists on the CDN (verified before anything runs)
+ *   3. latest/ on the CDN, identified by running the binary once
+ *   4. throw with a message that names the RPK_VERSION escape hatch
+ *
+ * @param {Object} [opts]
+ * @param {Object} [opts.env=process.env]
+ * @param {Object} [opts.octokit] - Octokit instance; defaults to the shared client when a token exists
+ * @param {Function} [opts.probe=curlHead] - url -> http status, injectable for tests
+ * @param {string} [opts.platform], [opts.arch]
+ * @param {{ log: Function, warn: Function }} [opts.log=console]
+ * @returns {Promise<string>} normalised tag
+ */
+async function resolveLatestRpkTag({
+  env = process.env,
+  octokit,
+  probe = curlHead,
+  platform = process.platform,
+  arch = process.arch,
+  log = console
+} = {}) {
+  if (env.RPK_VERSION) {
+    const pinned = normalizeRpkTag(env.RPK_VERSION)
+    if (!pinned) {
+      throw new Error(`RPK_VERSION='${env.RPK_VERSION}' is not a release tag; expected vX.Y.Z or vX.Y.Z-rcN`)
+    }
+    return pinned
+  }
+
+  const { getGitHubApiToken } = require('./github-token')
+  if (getGitHubApiToken()) {
+    const client = octokit || require('./octokit-client')
+    try {
+      const tag = await newestPublishedGaTagFromGitHub(client, probe, { platform, arch, log })
+      if (tag) return tag
+    } catch (err) {
+      log.warn(`Could not list ${TAGS_OWNER}/${TAGS_REPO} tags: ${err.message}`)
+    }
+  }
+
+  const fromLatest = versionBehindLatest(probe, { platform, arch, log })
+  if (fromLatest) return fromLatest
+
+  throw new Error(
+    `Could not resolve the latest rpk release: ${RPK_CDN_LATEST_PREFIX}/ is empty on ${RPK_CDN_BASE} ` +
+    `and no GitHub token is available to list ${TAGS_OWNER}/${TAGS_REPO} tags. ` +
+    'Set RPK_VERSION=vX.Y.Z or provide a token (GH_TOKEN, GITHUB_TOKEN, or REDPANDA_GITHUB_TOKEN).'
+  )
+}
+
+// --- CLI ----------------------------------------------------------------------
+// stdout carries only the result so shell callers can capture it; everything
+// else goes to stderr. Exit 0 ok, 2 not published / unresolvable, 1 unexpected.
+
+const EXIT_NOT_PUBLISHED = 2
+
+function usage() {
+  return [
+    'Usage:',
+    '  node rpk-cdn.js install --dest <dir> [--version <tag>] [--platform <p>] [--arch <a>]',
+    '      Download and verify rpk; prints the binary path. Without --version, resolves the',
+    '      latest release (RPK_VERSION, then GitHub tags with a token, then latest/).',
+    '  node rpk-cdn.js resolve-latest [--platform <p>] [--arch <a>]',
+    '      Prints the tag "install" would use.',
+    '  node rpk-cdn.js url --tag <tag> [--platform <p>] [--arch <a>]',
+    '      Prints the zip and checksums URLs for a tag.',
+    '',
+    '--platform accepts win32/darwin/linux (or windows). --arch accepts x64/arm64 (or amd64, x86_64, aarch64).'
+  ].join('\n')
+}
+
+function parseArgs(argv) {
+  const [command, ...rest] = argv
+  const opts = {}
+  for (let i = 0; i < rest.length; i++) {
+    const arg = rest[i]
+    if (!arg.startsWith('--')) throw new Error(`Unexpected argument '${arg}'\n${usage()}`)
+    const key = arg.slice(2)
+    const value = rest[i + 1]
+    if (value === undefined || value.startsWith('--')) throw new Error(`--${key} needs a value\n${usage()}`)
+    opts[key] = value
+    i++
+  }
+  return { command, opts }
+}
+
+async function main(argv) {
+  const stderrLog = { log: (...a) => console.error(...a), warn: (...a) => console.error(...a) }
+  const { command, opts } = parseArgs(argv)
+  const platformOpts = {
+    platform: opts.platform || process.platform,
+    arch: opts.arch || process.arch,
+    log: stderrLog
+  }
+
+  switch (command) {
+    case 'url': {
+      const tag = normalizeRpkTag(opts.tag || '')
+      if (!tag) {
+        console.error(`'${opts.tag}' is not a release tag; expected vX.Y.Z or vX.Y.Z-rcN`)
+        return EXIT_NOT_PUBLISHED
+      }
+      const urls = rpkCdnUrls(tag, platformOpts)
+      if (!urls) {
+        console.error(`No rpk release asset for platform ${platformOpts.platform}/${platformOpts.arch}`)
+        return EXIT_NOT_PUBLISHED
+      }
+      process.stdout.write(`${urls.zip}\n${urls.checksums}\n`)
+      return 0
+    }
+    case 'resolve-latest': {
+      const tag = await resolveLatestRpkTag(platformOpts)
+      process.stdout.write(`${tag}\n`)
+      return 0
+    }
+    case 'install': {
+      if (!opts.dest) throw new Error(`install requires --dest <dir>\n${usage()}`)
+      const tag = opts.version
+        ? normalizeRpkTag(opts.version)
+        : await resolveLatestRpkTag(platformOpts)
+      if (!tag) {
+        console.error(`'${opts.version}' is not a release tag; expected vX.Y.Z or vX.Y.Z-rcN`)
+        return EXIT_NOT_PUBLISHED
+      }
+      fs.mkdirSync(opts.dest, { recursive: true })
+      const binPath = downloadRpkBinary(tag, opts.dest, platformOpts)
+      if (!binPath) return EXIT_NOT_PUBLISHED
+      process.stdout.write(`${binPath}\n`)
+      return 0
+    }
+    default:
+      throw new Error(`Unknown command '${command || ''}'\n${usage()}`)
+  }
+}
+
+if (require.main === module) {
+  // process.exitCode rather than process.exit(): install-test-dependencies.sh
+  // captures the binary path from stdout through a pipe, and exit() can end
+  // the process before that write drains, handing the shell a truncated path.
+  main(process.argv.slice(2)).then(
+    code => { process.exitCode = code },
+    err => {
+      console.error(err.message)
+      // Unresolvable "latest" is a not-published condition, not a bug
+      process.exitCode = /Could not resolve the latest rpk release/.test(err.message) ? EXIT_NOT_PUBLISHED : 1
+    }
+  )
+}
+
+module.exports = {
+  RPK_CDN_BASE,
+  RPK_CDN_LATEST_PREFIX,
+  RPK_RELEASE_TAG_RE,
+  RPK_GA_TAG_RE,
+  RPK_OS_BY_PLATFORM,
+  RPK_ARCH_BY_NODE_ARCH,
+  normalizeRpkTag,
+  rpkAssetName,
+  rpkChecksumsName,
+  rpkCdnUrl,
+  rpkCdnUrls,
+  isNotPublished,
+  curlToFile,
+  curlHead,
+  parseChecksums,
+  sha256File,
+  downloadRpkBinary,
+  resolveLatestRpkTag
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@redpanda-data/docs-extensions-and-macros",
-  "version": "5.38.10",
+  "version": "5.41.0",
   "description": "Antora extensions and macros developed for Redpanda documentation.",
   "keywords": [
     "antora",

package/tools/rpk-docs/rpk-docs-handler.js

@@ -1,7 +1,6 @@
 'use strict'
 
 const { spawnSync } = require('child_process')
-const crypto = require('crypto')
 const path = require('path')
 const fs = require('fs')
 const os = require('os')
@@ -9,6 +8,7 @@ const semver = require('semver')
 const { findRepoRoot } = require('../../cli-utils/doc-tools-utils')
 const { generateRpkDocs, applyOverridesToTree, resolveReferences, shouldExcludeCommand, shouldUsePartialDir, derivePartialsDir } = require('./generate-rpk-docs')
 const { detectLinuxOnlyFromSource, warnIfDetectionLooksBroken } = require('./detect-platform-commands')
+const { downloadRpkBinary, RPK_RELEASE_TAG_RE, RPK_CDN_BASE } = require('../../cli-utils/rpk-cdn')
 const { generateRpkDiff, printDiffReport, generateWhatsNewSection, flattenToMap } = require('./report-delta')
 const { loadAndValidateOverrides, ValidationResult } = require('./validate-overrides')
 const { validateDirectory, formatResults } = require('./validate-output')
@@ -1306,138 +1306,26 @@ function buildRpkBinary(sourcePath, outPath) {
 }
 
 /**
- * Download an official rpk release binary for the current platform.
- * @param {string} tag - Release tag (e.g., v26.1.12)
+ * Download the official rpk release binary for the current platform from the
+ * rpk distribution CDN (https://rpk.redpanda.com). GA and RC tags are both
+ * published there; no token is needed. See cli-utils/rpk-cdn.js for the
+ * layout and the verification rules.
+ * @param {string} tag - Release tag (e.g., v26.1.12, v26.2.1-rc2)
  * @param {string} destDir - Directory to download and extract into
- * @returns {string|null} Path to the extracted binary, or null if the
- *   release asset is unavailable (caller falls back to a source build)
+ * @returns {string|null} Path to the extracted binary, or null if the CDN has
+ *   no build for the tag (HTTP 403/404; caller falls back to a source build).
+ *   Throws on a checksum mismatch or an archive with no rpk binary.
  */
 function downloadRpkRelease(tag, destDir) {
-  const osName = { darwin: 'darwin', linux: 'linux', win32: 'windows' }[process.platform]
-  const archName = { arm64: 'arm64', x64: 'amd64' }[process.arch]
-  if (!osName || !archName) {
-    console.warn(`No rpk release asset for platform ${process.platform}/${process.arch}`)
-    return null
-  }
-
-  // Releases now publish to streaming-enterprise (private), not the public
-  // redpanda-data/redpanda. This function already has a fallback (the
-  // caller builds from source instead), so a missing token warns and
-  // returns null here rather than throwing the way the clone-only paths do.
-  const { getGitHubApiToken } = require('../../cli-utils/github-token')
-  const token = getGitHubApiToken()
-  if (!token) {
-    console.warn('No GitHub token available for the private streaming-enterprise release download; falling back to a source build.')
-    return null
-  }
-
-  const assetName = `rpk-${osName}-${archName}.zip`
-  const checksumAsset = `rpk_${tag.replace(/^v/, '')}_checksums.txt`
-  const zipPath = path.join(destDir, assetName)
-  // The Authorization header reaches curl through a config file on stdin
-  // (--config -), never as an -H argument: an argv header is readable by
-  // any process on the host (ps, /proc) for as long as the transfer runs.
-  const curlAuthConfig = `header = "Authorization: token ${token}"`
-
-  // streaming-enterprise is private, so the release's browser download URL
-  // (releases/download/<tag>/<asset>) 404s even with a valid token: it
-  // redirects to a signed storage URL that requires an authenticated
-  // browser session, not a bearer token. Only the API's per-asset url
-  // (asset.url, not asset.browser_download_url) accepts this Authorization
-  // header, so resolve assets through the release-by-tag API first.
-  const releaseResult = spawnSync('curl', [
-    '-fsSL', '--retry', '5', '--retry-all-errors',
-    '--connect-timeout', '30', '--max-time', '60',
-    '--config', '-',
-    '-H', 'Accept: application/vnd.github+json',
-    `https://api.github.com/repos/redpanda-data/streaming-enterprise/releases/tags/${tag}`
-  ], { encoding: 'utf8', input: curlAuthConfig, timeout: 90000 })
-
-  if (releaseResult.status !== 0) {
-    console.warn(`Could not look up release ${tag} from streaming-enterprise (draft or missing release)`)
-    return null
-  }
-
-  let release
-  try {
-    release = JSON.parse(releaseResult.stdout)
-  } catch {
-    console.warn(`Could not parse release metadata for ${tag}`)
-    return null
-  }
-
-  const assetUrl = (release.assets || []).find(a => a.name === assetName)?.url
-  if (!assetUrl) {
-    console.warn(`Release ${tag} has no asset named ${assetName} (draft or missing release asset)`)
-    return null
-  }
-  const checksumUrl = (release.assets || []).find(a => a.name === checksumAsset)?.url
-
-  console.log(`Downloading ${assetName} for ${tag}...`)
-  // Accept: application/octet-stream is required on the API asset url to
-  // receive the binary itself instead of its JSON metadata.
-  const curlResult = spawnSync('curl', [
-    '-fL', '--retry', '5', '--retry-all-errors',
-    '--connect-timeout', '30', '--max-time', '300',
-    '--config', '-',
-    '-H', 'Accept: application/octet-stream',
-    '-o', zipPath, assetUrl
-  ], { encoding: 'utf8', input: curlAuthConfig, timeout: 360000 })
-
-  if (curlResult.status !== 0) {
-    console.warn(`Could not download rpk release for ${tag} (draft or missing release asset)`)
-    return null
-  }
-
-  // Verify against the release checksum file when it exists
-  const checksumPath = path.join(destDir, checksumAsset)
-  const checksumResult = checksumUrl ? spawnSync('curl', [
-    '-fsSL', '--retry', '3', '--connect-timeout', '30', '--max-time', '60',
-    '--config', '-',
-    '-H', 'Accept: application/octet-stream',
-    '-o', checksumPath, checksumUrl
-  ], { encoding: 'utf8', input: curlAuthConfig, timeout: 90000 }) : { status: 1 }
-
-  if (checksumResult.status === 0) {
-    const expectedLine = fs.readFileSync(checksumPath, 'utf8')
-      .split('\n')
-      .find(line => line.trim().endsWith(assetName))
-    if (expectedLine) {
-      const expected = expectedLine.trim().split(/\s+/)[0]
-      const actual = crypto.createHash('sha256').update(fs.readFileSync(zipPath)).digest('hex')
-      if (expected !== actual) {
-        throw new Error(
-          `Checksum mismatch for ${assetName} (${tag}):\n` +
-          `  expected ${expected}\n  actual   ${actual}`
-        )
-      }
-      console.log('Checksum verified')
-    }
-  } else {
-    console.warn('No checksum file published for this release; skipping verification')
-  }
-
-  const unzipResult = spawnSync('unzip', ['-o', zipPath, '-d', destDir], {
-    encoding: 'utf8',
-    timeout: 60000
-  })
-  if (unzipResult.status !== 0) {
-    throw new Error(`Failed to extract ${assetName}: ${unzipResult.stderr}`)
-  }
-
-  const binPath = path.join(destDir, 'rpk')
-  if (!fs.existsSync(binPath)) {
-    throw new Error(`Extracted archive did not contain an rpk binary: ${zipPath}`)
-  }
-  fs.chmodSync(binPath, 0o755)
-  return binPath
+  return downloadRpkBinary(tag, destDir)
 }
 
 /**
  * Get an rpk binary matching the given version.
- * Prefers the official release download (published stable releases only;
- * RC releases are drafts, so their assets are not publicly downloadable).
- * Falls back to building from source at the tag.
+ * Prefers the official release download: GA and RC tags are published to
+ * https://rpk.redpanda.com/v<tag>/ with no token needed. Falls back to
+ * building from source at the tag (private clone, token required) when the
+ * CDN has no build; vdev always builds from source.
  * @param {string} rpkVersion - Version tag from the snapshot (e.g., v26.1.12, v26.2.1-rc2)
  * @param {Object} [options]
  * @param {string} [options.rpkBin] - Existing binary to use, skipping download/build
@@ -1456,25 +1344,26 @@ function acquireRpkBinary(rpkVersion, options = {}) {
   }
 
   const tag = rpkVersion.startsWith('v') ? rpkVersion : `v${rpkVersion}`
+
+  if (!RPK_RELEASE_TAG_RE.test(tag) && tag !== 'vdev') {
+    throw new Error(
+      `Cannot acquire an rpk binary for version '${rpkVersion}'.\n` +
+      'The snapshot\'s rpk_version is not a release tag. ' +
+      'Pass --rpk-bin <path> to use a local rpk binary instead.'
+    )
+  }
+
   const workDir = fs.mkdtempSync(path.join(os.tmpdir(), 'rpk-bin-'))
 
-  // Only published (non-draft) releases have downloadable assets
-  if (/^v\d+\.\d+\.\d+$/.test(tag)) {
+  // GA and RC tags both have a build on the CDN; only branches do not
+  if (RPK_RELEASE_TAG_RE.test(tag)) {
     const downloaded = downloadRpkRelease(tag, workDir)
     if (downloaded) {
       return downloaded
     }
     console.log('Falling back to building rpk from source...')
   } else {
-    console.log(`No published release binary for ${tag}; building from source...`)
-  }
-
-  if (!/^v\d+\.\d+\.\d+(-rc\d+)?$/.test(tag) && tag !== 'vdev') {
-    throw new Error(
-      `Cannot acquire an rpk binary for version '${rpkVersion}'.\n` +
-      'The snapshot\'s rpk_version is not a release tag. ' +
-      'Pass --rpk-bin <path> to use a local rpk binary instead.'
-    )
+    console.log(`${tag} has no build on ${RPK_CDN_BASE} (branches build from source)...`)
   }
 
   const sourceRef = tag === 'vdev' ? 'dev' : tag