@redpanda-data/docs-extensions-and-macros 5.23.0 → 5.30.0

package/tools/property-extractor/Makefile

@@ -76,55 +76,101 @@ clean:
 	@echo "🧹 Cleaning up…"
 	@rm -rf $(VENV) $(TMP_ROOT) $(REPO_ROOT)/autogenerated
 
-# --- Clone or update Redpanda, checkout TAG or default branch ---
+# --- Clone or update the Redpanda source (now in streaming-enterprise), checkout TAG or default branch ---
 # Clones are shallow: the extractor only reads source files at a single tag,
 # so full history is wasted time and disk.
-# Auth is passed as a per-invocation HTTP header (git -c before the clone
-# subcommand), never embedded in the remote URL, so no token is persisted in
-# the clone's .git/config. The clone lives inside the installed package dir,
-# so a persisted token would be readable by anything that can list it.
+# Auth travels via a credential helper registered through GIT_CONFIG_*
+# environment variables, never as a git -c argument (which would put the
+# token in this process's argv, visible to anything that can list processes
+# on the runner) and never embedded in the remote URL. The clone lives
+# inside the installed package dir, so a persisted token would also be
+# readable by anything that can list it -- GIT_CONFIG_* env vars don't get
+# written to disk, so nothing persists in the clone's .git/config either.
+# streaming-enterprise is private, so a token is required -- not just
+# preferred the way it was when this cloned the public redpanda repo. GH_TOKEN
+# is checked first because `doc-tools generate property-docs` pre-resolves it
+# via cli-utils/github-token.js's full priority chain (notably GIT_CREDENTIALS,
+# the token Antora/Netlify builds actually populate) before invoking this
+# Makefile. REDPANDA_GITHUB_TOKEN/GITHUB_TOKEN are the fallback for invoking
+# `make` directly, without going through doc-tools.js.
+# An existing checkout may predate the streaming-enterprise migration and
+# still have origin pointed at the old public redpanda repo, so the
+# existing-directory path pins origin to REPO_URL before fetching. Pinning
+# origin is not enough by itself: the old repo's cached remote-tracking refs
+# and tags stay behind it, and a fetch that fails (expired token, outage,
+# ref that does not exist) used to fall straight through to checking one of
+# those out -- generating docs from stale public source under a tag that
+# source never contained, with nothing in the log to say so. So a fetch has
+# to succeed before any ref is resolved, the catch-all fetch prunes refs and
+# tags the new origin does not have, the ref that gets checked out is the
+# one the successful fetch actually wrote, and a TAG that is neither a tag
+# nor a branch there is an error rather than a quiet swap to the default
+# branch.
 redpanda-git:
 	@mkdir -p "$(TMP_ROOT)"
-	@echo "🔄 Cloning/updating Redpanda into $(REDPANDA_SRC)…"
-	@if [ -d "$(REDPANDA_SRC)" ]; then \
-	  git -C "$(REDPANDA_SRC)" fetch --depth 1 -q origin "+refs/tags/$(TAG):refs/tags/$(TAG)" 2>/dev/null || \
-	    git -C "$(REDPANDA_SRC)" fetch --depth 1 -q origin "+refs/heads/$(TAG):refs/remotes/origin/$(TAG)" 2>/dev/null || \
-	    git -C "$(REDPANDA_SRC)" fetch --all --tags -q; \
-	else \
-	  GH_TOKEN=$${REDPANDA_GITHUB_TOKEN:-$${GITHUB_TOKEN:-$${GH_TOKEN}}}; \
-	  REPO_URL="https://github.com/redpanda-data/redpanda.git"; \
-	  if [ -n "$$GH_TOKEN" ]; then \
-	    echo "🔑 Using authenticated clone (token provided)"; \
-	    AUTH_HEADER="AUTHORIZATION: basic $$(printf '%s' "x-access-token:$$GH_TOKEN" | base64 | tr -d '\n')"; \
-	  else \
-	    AUTH_HEADER=""; \
-	  fi; \
-	  git -c http.https://github.com/.extraheader="$$AUTH_HEADER" clone -q --depth 1 --branch "$(TAG)" "$$REPO_URL" "$(REDPANDA_SRC)" 2>/dev/null || \
-	    git -c http.https://github.com/.extraheader="$$AUTH_HEADER" clone -q --depth 1 "$$REPO_URL" "$(REDPANDA_SRC)"; \
+	@GH_TOKEN=$${GH_TOKEN:-$${REDPANDA_GITHUB_TOKEN:-$${GITHUB_TOKEN}}}; \
+	if [ -z "$$GH_TOKEN" ]; then \
+	  echo "❌ redpanda-data/streaming-enterprise is a private repository."; \
+	  echo "   Set GH_TOKEN, REDPANDA_GITHUB_TOKEN, or GITHUB_TOKEN to a token with access."; \
+	  exit 1; \
 	fi; \
-	if git -C "$(REDPANDA_SRC)" rev-parse --verify -q "origin/$(TAG)" >/dev/null 2>&1; then \
-	  echo "🔖 Checking out remote branch 'origin/$(TAG)'"; \
-	  git -C "$(REDPANDA_SRC)" checkout -q "$(TAG)" 2>/dev/null || git -C "$(REDPANDA_SRC)" checkout -q -b "$(TAG)" "origin/$(TAG)"; \
-	  git -C "$(REDPANDA_SRC)" reset --hard "origin/$(TAG)" -q; \
-	elif git -C "$(REDPANDA_SRC)" rev-parse --verify -q "$(TAG)" >/dev/null 2>&1; then \
-	  echo "🔖 Checking out '$(TAG)'"; \
-	  git -C "$(REDPANDA_SRC)" checkout -q "$(TAG)"; \
+	export GIT_MAKEFILE_CLONE_TOKEN="$$GH_TOKEN"; \
+	export GIT_CONFIG_COUNT=2; \
+	export GIT_CONFIG_KEY_0=credential.helper; \
+	export GIT_CONFIG_VALUE_0=; \
+	export GIT_CONFIG_KEY_1=credential.https://github.com.helper; \
+	export GIT_CONFIG_VALUE_1='!f() { echo "username=x-access-token"; echo "password=$$GIT_MAKEFILE_CLONE_TOKEN"; }; f'; \
+	REPO_URL="https://github.com/redpanda-data/streaming-enterprise.git"; \
+	echo "🔄 Cloning/updating streaming-enterprise into $(REDPANDA_SRC)…"; \
+	FETCH_LOG="$$(mktemp)"; \
+	FETCHED=""; \
+	if [ -d "$(REDPANDA_SRC)" ]; then \
+	  git -C "$(REDPANDA_SRC)" remote set-url origin "$$REPO_URL" 2>/dev/null || \
+	    git -C "$(REDPANDA_SRC)" remote add origin "$$REPO_URL"; \
+	  if git -C "$(REDPANDA_SRC)" fetch --depth 1 -q origin "+refs/tags/$(TAG):refs/tags/$(TAG)" >>"$$FETCH_LOG" 2>&1; then \
+	    FETCHED=tag; \
+	  elif git -C "$(REDPANDA_SRC)" fetch --depth 1 -q origin "+refs/heads/$(TAG):refs/remotes/origin/$(TAG)" >>"$$FETCH_LOG" 2>&1; then \
+	    FETCHED=branch; \
+	  elif git -C "$(REDPANDA_SRC)" fetch --all --tags --prune --prune-tags -q >>"$$FETCH_LOG" 2>&1; then \
+	    FETCHED=probe; \
+	  fi; \
 	else \
-	  DEFAULT_BRANCH=$$(git -C "$(REDPANDA_SRC)" symbolic-ref --short refs/remotes/origin/HEAD 2>/dev/null || true); \
-	  if [ -n "$$DEFAULT_BRANCH" ]; then \
-	    echo "⚠️ Tag '$(TAG)' not found; falling back to '$${DEFAULT_BRANCH}'"; \
-	    git -C "$(REDPANDA_SRC)" checkout -q "$${DEFAULT_BRANCH}"; \
-	  else \
-	    DEFAULT_BRANCH=$$(git -C "$(REDPANDA_SRC)" ls-remote --symref origin HEAD | awk '/^ref:/ {sub(/^refs\/heads\//, "", $$2); print $$2; exit}'); \
-	    if [ -z "$$DEFAULT_BRANCH" ]; then \
-	      echo "❌ Tag '$(TAG)' not found and the default branch could not be determined"; \
-	      exit 1; \
-	    fi; \
-	    echo "⚠️ Tag '$(TAG)' not found; falling back to '$${DEFAULT_BRANCH}'"; \
-	    git -C "$(REDPANDA_SRC)" fetch --depth 1 -q origin "$$DEFAULT_BRANCH"; \
-	    git -C "$(REDPANDA_SRC)" checkout -q FETCH_HEAD; \
+	  echo "🔑 Using authenticated clone"; \
+	  if git clone -q --depth 1 --branch "$(TAG)" "$$REPO_URL" "$(REDPANDA_SRC)" >>"$$FETCH_LOG" 2>&1; then \
+	    FETCHED=probe; \
 	  fi; \
-	fi
+	fi; \
+	if [ -z "$$FETCHED" ]; then \
+	  echo "❌ Could not fetch '$(TAG)' from redpanda-data/streaming-enterprise:"; \
+	  sed 's/^/   /' "$$FETCH_LOG"; \
+	  echo "   Not continuing: anything already in $(REDPANDA_SRC) can predate the"; \
+	  echo "   streaming-enterprise migration, and generating docs from it would"; \
+	  echo "   publish stale public source under this tag. Check that the token is"; \
+	  echo "   valid and has access to the repository, and that '$(TAG)' exists there."; \
+	  rm -f "$$FETCH_LOG"; \
+	  exit 1; \
+	fi; \
+	rm -f "$$FETCH_LOG"; \
+	case "$$FETCHED" in \
+	  tag) REF="refs/tags/$(TAG)";; \
+	  branch) REF="refs/remotes/origin/$(TAG)";; \
+	  *) REF=""; \
+	     for candidate in "refs/tags/$(TAG)" "refs/remotes/origin/$(TAG)"; do \
+	       if git -C "$(REDPANDA_SRC)" rev-parse --verify -q "$$candidate" >/dev/null 2>&1; then \
+	         REF="$$candidate"; \
+	         break; \
+	       fi; \
+	     done;; \
+	esac; \
+	if [ -z "$$REF" ]; then \
+	  echo "❌ '$(TAG)' is neither a tag nor a branch in redpanda-data/streaming-enterprise."; \
+	  echo "   Not falling back to the default branch: that would publish docs labelled"; \
+	  echo "   '$(TAG)' built from entirely different source."; \
+	  exit 1; \
+	fi; \
+	echo "🔖 Checking out '$$REF'"; \
+	git -C "$(REDPANDA_SRC)" checkout -q --detach "$$REF"; \
+	git -C "$(REDPANDA_SRC)" reset --hard -q "$$REF"
 
 # --- Clone Tree-sitter grammar & generate parser ---
 # Skips npm install and parser generation when the generated output already

package/tools/property-extractor/README.adoc

@@ -1296,7 +1296,7 @@ node generate-handlebars-docs.js test-input.json test-output
 
 == Additional resources
 
-* https://github.com/redpanda-data/redpanda[Redpanda GitHub Repository]
+* https://github.com/redpanda-data/streaming-enterprise[streaming-enterprise Repository] (private; this is where the extractor clones Redpanda's source from)
 * https://github.com/redpanda-data/docs-extensions-and-macros[docs-extensions-and-macros Repository]
 * https://tree-sitter.github.io/tree-sitter/[Tree-sitter Documentation]
 * https://handlebarsjs.com/[Handlebars.js Guide]

package/tools/property-extractor/cloud_config.py

@@ -6,7 +6,7 @@ This module fetches cloud configuration from the cloudv2 repository to determine
 which Redpanda properties are supported, editable, or readonly in cloud deployments.
 
 Prerequisites:
-    - GITHUB_TOKEN environment variable set with appropriate permissions
+    - GH_TOKEN, GITHUB_TOKEN, or REDPANDA_GITHUB_TOKEN environment variable set with appropriate permissions
     - Internet connection to access GitHub API
     
 Usage:
@@ -105,14 +105,14 @@ def fetch_cloud_config(github_token: Optional[str] = None) -> CloudConfig:
     """
     Fetch the latest cloud configuration from the redpanda-data/cloudv2 repository and return it as a CloudConfig.
     
-    This function uses a GitHub personal access token for authentication. If `github_token` is not provided, it will read GITHUB_TOKEN or REDPANDA_GITHUB_TOKEN from the environment. It downloads the most recent versioned YAML from the repository's install-pack directory, validates expected sections (`customer_managed_configs` and `readonly_cluster_config`), and constructs a CloudConfig instance.
-    
+    This function uses a GitHub personal access token for authentication. If `github_token` is not provided, it will read GH_TOKEN, GITHUB_TOKEN, or REDPANDA_GITHUB_TOKEN from the environment. It downloads the most recent versioned YAML from the repository's install-pack directory, validates expected sections (`customer_managed_configs` and `readonly_cluster_config`), and constructs a CloudConfig instance.
+
     Parameters:
-        github_token (Optional[str]): Personal access token for GitHub API. If omitted, the function will try environment variables GITHUB_TOKEN or REDPANDA_GITHUB_TOKEN.
-    
+        github_token (Optional[str]): Personal access token for GitHub API. If omitted, the function will try environment variables GH_TOKEN, GITHUB_TOKEN, or REDPANDA_GITHUB_TOKEN.
+
     Returns:
         CloudConfig: Parsed cloud configuration for the latest available version.
-    
+
     Raises:
         GitHubAuthError: Authentication or access problems with the GitHub API (including 401/403 responses).
         NetworkError: Network connectivity or timeout failures when contacting the GitHub API.
@@ -120,7 +120,11 @@ def fetch_cloud_config(github_token: Optional[str] = None) -> CloudConfig:
         CloudConfigError: Generic configuration error (for example, missing token) or unexpected internal failures.
     """
     if not github_token:
-        github_token = os.environ.get('GITHUB_TOKEN') or os.environ.get('REDPANDA_GITHUB_TOKEN')
+        # GH_TOKEN takes priority: `doc-tools generate property-docs` pre-resolves
+        # a token via cli-utils/github-token.js's full priority chain (GIT_CREDENTIALS,
+        # ACTIONS_BOT_TOKEN, etc.) and passes it down as GH_TOKEN, so this must see
+        # that name too -- not just the two env vars a directly-invoked `make` sets.
+        github_token = os.environ.get('GH_TOKEN') or os.environ.get('GITHUB_TOKEN') or os.environ.get('REDPANDA_GITHUB_TOKEN')
     
     if not github_token:
         error_msg = (

package/tools/rpk-docs/rpk-docs-handler.js

@@ -241,7 +241,31 @@ function prepareSourceFromRef(sourceRef, sourcePath = null) {
   const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'rpk-source-'))
   const repoDir = path.join(tmpDir, 'redpanda')
 
-  console.log(`Sparse-cloning redpanda repo (ref: ${sourceRef}) to ${repoDir}...`)
+  // streaming-enterprise is private, so a token is required. Auth travels
+  // via a credential helper registered through GIT_CONFIG_* environment
+  // variables, never as a git -c argument, so the token cannot appear in
+  // this process's argv (visible to anything that can list processes) and
+  // never embedded in the remote URL, so no token is persisted in the
+  // clone's .git/config either.
+  const { getGitHubToken } = require('../../cli-utils/github-token')
+  const token = getGitHubToken()
+  if (!token) {
+    throw new Error(
+      'redpanda-data/streaming-enterprise is a private repository.\n' +
+      'Set GH_TOKEN, GITHUB_TOKEN, or REDPANDA_GITHUB_TOKEN so the clone can authenticate.'
+    )
+  }
+  const gitEnv = {
+    ...process.env,
+    RPK_SOURCE_CLONE_TOKEN: token,
+    GIT_CONFIG_COUNT: '2',
+    GIT_CONFIG_KEY_0: 'credential.helper',
+    GIT_CONFIG_VALUE_0: '',
+    GIT_CONFIG_KEY_1: 'credential.https://github.com.helper',
+    GIT_CONFIG_VALUE_1: '!f() { echo "username=x-access-token"; echo "password=$RPK_SOURCE_CLONE_TOKEN"; }; f'
+  }
+
+  console.log(`Sparse-cloning streaming-enterprise repo (ref: ${sourceRef}) to ${repoDir}...`)
 
   // Clone with sparse checkout
   const cloneResult = spawnSync('git', [
@@ -250,27 +274,31 @@ function prepareSourceFromRef(sourceRef, sourcePath = null) {
     '--filter=blob:none',
     '--sparse',
     '--branch', sourceRef,
-    'https://github.com/redpanda-data/redpanda.git',
+    'https://github.com/redpanda-data/streaming-enterprise.git',
     repoDir
   ], {
     encoding: 'utf8',
     timeout: 120000,
+    env: gitEnv,
     stdio: ['pipe', 'pipe', 'pipe']
   })
 
   if (cloneResult.status !== 0) {
     throw new Error(
-      `Failed to clone redpanda repo with ref '${sourceRef}'.\n` +
+      `Failed to clone streaming-enterprise repo with ref '${sourceRef}'.\n` +
       `Make sure the branch or tag exists.\n` +
       `Error: ${cloneResult.stderr}`
     )
   }
 
-  // Set sparse checkout to only get rpk
+  // Set sparse checkout to only get rpk. With --filter=blob:none, the blobs
+  // for src/go/rpk were not fetched by the clone above and are pulled lazily
+  // here, so this invocation needs the same credential helper.
   const sparseResult = spawnSync('git', ['sparse-checkout', 'set', 'src/go/rpk'], {
     cwd: repoDir,
     encoding: 'utf8',
-    timeout: 60000
+    timeout: 60000,
+    env: gitEnv
   })
 
   if (sparseResult.status !== 0) {
@@ -292,8 +320,8 @@ function fetchRpkTreeFromSource(sourcePath) {
   if (!fs.existsSync(sourcePath)) {
     throw new Error(
       `rpk source directory not found: ${sourcePath}\n` +
-      'To use --from-source, you need a local checkout of the redpanda repository.\n' +
-      'Clone it with: git clone https://github.com/redpanda-data/redpanda.git\n' +
+      'To use --from-source, you need a local checkout of the streaming-enterprise repository (private).\n' +
+      'Clone it with: git clone https://github.com/redpanda-data/streaming-enterprise.git\n' +
       'Then point to: <repo>/src/go/rpk'
     )
   }
@@ -393,8 +421,8 @@ function fetchRpkTreeFromLinuxSource(sourcePath, pluginPins = {}) {
   if (!fs.existsSync(absoluteSourcePath)) {
     throw new Error(
       `rpk source directory not found: ${absoluteSourcePath}\n` +
-      'Expected a checkout of the redpanda repository.\n' +
-      'Clone it with: git clone https://github.com/redpanda-data/redpanda.git'
+      'Expected a checkout of the streaming-enterprise repository (private).\n' +
+      'Clone it with: git clone https://github.com/redpanda-data/streaming-enterprise.git'
     )
   }
 
@@ -1292,16 +1320,69 @@ function downloadRpkRelease(tag, destDir) {
     return null
   }
 
+  // Releases now publish to streaming-enterprise (private), not the public
+  // redpanda-data/redpanda. This function already has a fallback (the
+  // caller builds from source instead), so a missing token warns and
+  // returns null here rather than throwing the way the clone-only paths do.
+  const { getGitHubToken } = require('../../cli-utils/github-token')
+  const token = getGitHubToken()
+  if (!token) {
+    console.warn('No GitHub token available for the private streaming-enterprise release download; falling back to a source build.')
+    return null
+  }
+
   const assetName = `rpk-${osName}-${archName}.zip`
-  const baseUrl = `https://github.com/redpanda-data/redpanda/releases/download/${tag}`
+  const checksumAsset = `rpk_${tag.replace(/^v/, '')}_checksums.txt`
   const zipPath = path.join(destDir, assetName)
+  // The Authorization header reaches curl through a config file on stdin
+  // (--config -), never as an -H argument: an argv header is readable by
+  // any process on the host (ps, /proc) for as long as the transfer runs.
+  const curlAuthConfig = `header = "Authorization: token ${token}"`
+
+  // streaming-enterprise is private, so the release's browser download URL
+  // (releases/download/<tag>/<asset>) 404s even with a valid token: it
+  // redirects to a signed storage URL that requires an authenticated
+  // browser session, not a bearer token. Only the API's per-asset url
+  // (asset.url, not asset.browser_download_url) accepts this Authorization
+  // header, so resolve assets through the release-by-tag API first.
+  const releaseResult = spawnSync('curl', [
+    '-fsSL', '--retry', '5', '--retry-all-errors',
+    '--connect-timeout', '30', '--max-time', '60',
+    '--config', '-',
+    '-H', 'Accept: application/vnd.github+json',
+    `https://api.github.com/repos/redpanda-data/streaming-enterprise/releases/tags/${tag}`
+  ], { encoding: 'utf8', input: curlAuthConfig, timeout: 90000 })
+
+  if (releaseResult.status !== 0) {
+    console.warn(`Could not look up release ${tag} from streaming-enterprise (draft or missing release)`)
+    return null
+  }
+
+  let release
+  try {
+    release = JSON.parse(releaseResult.stdout)
+  } catch {
+    console.warn(`Could not parse release metadata for ${tag}`)
+    return null
+  }
+
+  const assetUrl = (release.assets || []).find(a => a.name === assetName)?.url
+  if (!assetUrl) {
+    console.warn(`Release ${tag} has no asset named ${assetName} (draft or missing release asset)`)
+    return null
+  }
+  const checksumUrl = (release.assets || []).find(a => a.name === checksumAsset)?.url
 
   console.log(`Downloading ${assetName} for ${tag}...`)
+  // Accept: application/octet-stream is required on the API asset url to
+  // receive the binary itself instead of its JSON metadata.
   const curlResult = spawnSync('curl', [
     '-fL', '--retry', '5', '--retry-all-errors',
     '--connect-timeout', '30', '--max-time', '300',
-    '-o', zipPath, `${baseUrl}/${assetName}`
-  ], { encoding: 'utf8', timeout: 360000 })
+    '--config', '-',
+    '-H', 'Accept: application/octet-stream',
+    '-o', zipPath, assetUrl
+  ], { encoding: 'utf8', input: curlAuthConfig, timeout: 360000 })
 
   if (curlResult.status !== 0) {
     console.warn(`Could not download rpk release for ${tag} (draft or missing release asset)`)
@@ -1309,12 +1390,13 @@ function downloadRpkRelease(tag, destDir) {
   }
 
   // Verify against the release checksum file when it exists
-  const checksumAsset = `rpk_${tag.replace(/^v/, '')}_checksums.txt`
   const checksumPath = path.join(destDir, checksumAsset)
-  const checksumResult = spawnSync('curl', [
+  const checksumResult = checksumUrl ? spawnSync('curl', [
     '-fsSL', '--retry', '3', '--connect-timeout', '30', '--max-time', '60',
-    '-o', checksumPath, `${baseUrl}/${checksumAsset}`
-  ], { encoding: 'utf8', timeout: 90000 })
+    '--config', '-',
+    '-H', 'Accept: application/octet-stream',
+    '-o', checksumPath, checksumUrl
+  ], { encoding: 'utf8', input: curlAuthConfig, timeout: 90000 }) : { status: 1 }
 
   if (checksumResult.status === 0) {
     const expectedLine = fs.readFileSync(checksumPath, 'utf8')