@redpanda-data/docs-extensions-and-macros 5.53.0 → 5.54.0

package/extensions/util/connect-docs-asset.js

@@ -0,0 +1,187 @@
+'use strict'
+
+// Reads the Redpanda Connect generated reference docs.
+//
+// Every connect release (vX.Y.Z, including prereleases) publishes a
+// redpanda-connect-docs.tar.gz asset whose root is `modules/`: the tree that
+// connect's docs generator writes under its docs/ directory. Only
+// modules/components/partials/** and modules/components/examples/** belong to
+// the reference. Releases from before the asset existed answer 404.
+//
+// The same tree can come from a local directory instead (a connect checkout's
+// docs/ after running its generator), or from a local copy of the tarball.
+
+const fs = require('fs')
+const path = require('path')
+const zlib = require('zlib')
+
+const OWNER = 'redpanda-data'
+const REPO = 'connect'
+const ASSET_NAME = 'redpanda-connect-docs.tar.gz'
+const MODULE = 'components'
+const FAMILY_DIRS = { partials: 'partial', examples: 'example' }
+const DOWNLOAD_ATTEMPTS = 3
+const DOWNLOAD_TIMEOUT_MS = 120000
+
+function assetUrl (tag) {
+  return `https://github.com/${OWNER}/${REPO}/releases/download/${encodeURIComponent(tag)}/${ASSET_NAME}`
+}
+
+// The Antora resource of a path inside the tree, or null for anything outside
+// modules/components/partials and modules/components/examples.
+function toResource (entryPath) {
+  const p = String(entryPath || '').replace(/\\/g, '/').replace(/^(\.\/)+/, '')
+  const m = p.match(/^modules\/components\/(partials|examples)\/(.+)$/)
+  if (!m || m[2].endsWith('/')) return null
+  // A path that climbs out of its directory is not part of the tree.
+  if (m[2].split('/').some((segment) => segment === '..' || segment === '')) return null
+  return { family: FAMILY_DIRS[m[1]], relative: m[2], path: `modules/components/${m[1]}/${m[2]}` }
+}
+
+// Every regular file in a gzipped tar, as [{ path, contents }]. Throws on a
+// corrupt or truncated archive rather than returning a partial list.
+async function readTarGz (buffer) {
+  let tarBuffer
+  // The real asset unpacks to a few tens of MB. The cap turns a bad asset
+  // into a clear error instead of running the build out of memory.
+  const limit = module.exports.maxUnpackedBytes
+  try {
+    tarBuffer = zlib.gunzipSync(buffer, { maxOutputLength: limit })
+  } catch (error) {
+    if (error.code === 'ERR_BUFFER_TOO_LARGE') {
+      throw new Error(`the archive unpacks to more than ${limit} bytes, which no real reference docs asset does`)
+    }
+    throw new Error(`not a valid gzip archive: ${error.message}`)
+  }
+  // A complete tar ends with two zero blocks. Without them the archive was cut
+  // short at an entry boundary, which the parser alone does not report.
+  const end = tarBuffer.subarray(Math.max(0, tarBuffer.length - 1024))
+  if (tarBuffer.length % 512 !== 0 || end.length < 1024 || end.some((b) => b !== 0)) {
+    throw new Error('not a valid tar archive: missing the end-of-archive marker, so the archive is truncated')
+  }
+  const { Parser } = require('tar')
+  return new Promise((resolve, reject) => {
+    const files = []
+    let failure = null
+    const parser = new Parser({ strict: true })
+    parser.on('entry', (entry) => {
+      if (entry.type !== 'File' && entry.type !== 'OldFile' && entry.type !== 'ContiguousFile') {
+        entry.resume()
+        return
+      }
+      const chunks = []
+      entry.on('data', (chunk) => chunks.push(chunk))
+      entry.on('end', () => files.push({ path: entry.path, contents: Buffer.concat(chunks) }))
+    })
+    const fail = (error) => { if (!failure) failure = error }
+    parser.on('warn', (code, message) => fail(new Error(`not a valid tar archive: ${message}`)))
+    parser.on('error', (error) => fail(new Error(`not a valid tar archive: ${error.message}`)))
+    parser.on('close', () => (failure ? reject(failure) : resolve(files)))
+    try {
+      parser.end(tarBuffer)
+    } catch (error) {
+      fail(new Error(`not a valid tar archive: ${error.message}`))
+      reject(failure)
+    }
+  })
+}
+
+// One download attempt: { status } on an HTTP error, { body } on success.
+// With a token it goes through the releases API, which works whether or not
+// the connect repo is public. The asset endpoint redirects to a short-lived
+// signed URL, which is fetched without the token.
+async function fetchAssetOnce (tag, { fetchImpl, token }) {
+  const timeout = () => AbortSignal.timeout(DOWNLOAD_TIMEOUT_MS)
+  const ua = { 'User-Agent': 'Redpanda Docs' }
+  if (!token) {
+    const response = await fetchImpl(assetUrl(tag), { redirect: 'follow', signal: timeout(), headers: ua })
+    // Read the body here, so a connection that drops or times out
+    // mid-download is retried like a failed request.
+    return response.ok ? { body: Buffer.from(await response.arrayBuffer()) } : { status: response.status, statusText: response.statusText }
+  }
+  const auth = { ...ua, Authorization: `Bearer ${token}` }
+  const api = `https://api.github.com/repos/${OWNER}/${REPO}/releases/tags/${encodeURIComponent(tag)}`
+  const release = await fetchImpl(api, { signal: timeout(), headers: { ...auth, Accept: 'application/vnd.github+json' } })
+  // A token that is expired or lacks access must not break a build that the
+  // public download would serve. GitHub answers 404, not 403, for a repo the
+  // token can't see, so a 404 here falls back too, and is flagged so the
+  // caller can say why when the public download has no asset either.
+  if (release.status === 401 || release.status === 403) return fetchAssetOnce(tag, { fetchImpl, token: null })
+  if (release.status === 404) return { ...(await fetchAssetOnce(tag, { fetchImpl, token: null })), tokenCouldNotSee: true }
+  if (!release.ok) return { status: release.status, statusText: release.statusText }
+  const found = ((await release.json()).assets || []).find((a) => a.name === ASSET_NAME)
+  if (!found) return { status: 404, statusText: 'Not Found' }
+  let response = await fetchImpl(found.url, { redirect: 'manual', signal: timeout(), headers: { ...auth, Accept: 'application/octet-stream' } })
+  if (response.status === 401 || response.status === 403) return fetchAssetOnce(tag, { fetchImpl, token: null })
+  if (response.status >= 300 && response.status < 400 && response.headers.get('location')) {
+    response = await fetchImpl(response.headers.get('location'), { redirect: 'follow', signal: timeout(), headers: ua })
+  }
+  return response.ok ? { body: Buffer.from(await response.arrayBuffer()) } : { status: response.status, statusText: response.statusText }
+}
+
+// Downloads the asset of a release tag. Returns null when the release has no
+// asset (404). Throws on any other failure, after retrying network errors and
+// server errors. Pass the GitHub token when there is one: an unauthenticated
+// download only works while the connect repo is public.
+async function downloadAsset (tag, { fetchImpl = globalThis.fetch, logger, token = null } = {}) {
+  const url = token ? `the ${tag} release asset ${ASSET_NAME} through the GitHub API` : assetUrl(tag)
+  let lastError = null
+  for (let attempt = 1; attempt <= DOWNLOAD_ATTEMPTS; attempt++) {
+    let result
+    try {
+      result = await fetchAssetOnce(tag, { fetchImpl, token })
+    } catch (error) {
+      lastError = new Error(`could not download ${url}: ${error.message}`)
+    }
+    if (result && result.body) return result.body
+    if (result) {
+      if (result.status === 404) {
+        if (result.tokenCouldNotSee && logger) {
+          logger.warn(`The GitHub API found no ${tag} release of ${OWNER}/${REPO} with the GitHub token, and the public download has no ${ASSET_NAME} either. If the repo is private, check that the token has read access to it.`)
+        }
+        return null
+      }
+      lastError = new Error(`could not download ${url}: HTTP ${result.status} ${result.statusText || ''}`.trim())
+      // A client error other than 404 will not change on a retry.
+      if (result.status < 500 && result.status !== 429) break
+    }
+    if (attempt < DOWNLOAD_ATTEMPTS) {
+      if (logger) logger.warn(`${lastError.message}; retrying (${attempt}/${DOWNLOAD_ATTEMPTS - 1})`)
+      await new Promise((resolve) => setTimeout(resolve, module.exports.retryDelayMs * attempt))
+    }
+  }
+  throw lastError
+}
+
+function walk (dir, base, out) {
+  for (const entry of fs.readdirSync(dir, { withFileTypes: true })) {
+    const abs = path.join(dir, entry.name)
+    const rel = base ? `${base}/${entry.name}` : entry.name
+    if (entry.isDirectory()) walk(abs, rel, out)
+    else if (entry.isFile()) out.push({ path: rel, abspath: abs })
+  }
+  return out
+}
+
+// The files of a local copy of the tree: a directory that contains modules/
+// (or a connect checkout whose docs/ does), or a local .tar.gz of the asset.
+async function readLocal (location) {
+  const resolved = path.resolve(location)
+  let stat
+  try {
+    stat = fs.statSync(resolved)
+  } catch (error) {
+    throw new Error(`${resolved} does not exist`)
+  }
+  if (stat.isFile()) return readTarGz(fs.readFileSync(resolved))
+  let root = resolved
+  if (!fs.existsSync(path.join(root, 'modules')) && fs.existsSync(path.join(root, 'docs', 'modules'))) root = path.join(root, 'docs')
+  if (!fs.existsSync(path.join(root, 'modules', MODULE))) {
+    throw new Error(`${resolved} has no modules/${MODULE} directory`)
+  }
+  return walk(path.join(root, 'modules', MODULE), `modules/${MODULE}`, [])
+    .map((f) => ({ path: f.path, contents: fs.readFileSync(f.abspath) }))
+}
+
+// retryDelayMs is a property so tests can shorten the backoff.
+module.exports = { OWNER, REPO, ASSET_NAME, MODULE, retryDelayMs: 1000, maxUnpackedBytes: 512 * 1024 * 1024, assetUrl, toResource, readTarGz, downloadAsset, readLocal }

package/extensions/util/connect-github.js

@@ -0,0 +1,16 @@
+'use strict'
+
+// The GitHub API client the Redpanda Connect extensions use for release
+// lookups: authenticated when a token is available, retrying transient
+// failures but not rate limits or missing resources.
+
+const { getGitHubApiToken } = require('../../cli-utils/github-token')
+
+async function createGitHub () {
+  const { Octokit } = await import('@octokit/rest')
+  const { retry } = await import('@octokit/plugin-retry')
+  const token = getGitHubApiToken()
+  return new (Octokit.plugin(retry))({ userAgent: 'Redpanda Docs', auth: token || undefined, retry: { doNotRetry: [401, 403, 404, 429] } })
+}
+
+module.exports = { createGitHub }

package/extensions/version-fetcher/list-connect-releases.js

@@ -0,0 +1,10 @@
+'use strict'
+
+// The most recent releases of a repository, newest first, as the GitHub API
+// lists them (one page of at most `limit` releases). Each release carries its
+// assets, so a caller can check for an asset without downloading it. Throws
+// when the API is unavailable, so the caller can fall back to git tags.
+module.exports = async (github, owner, repo, limit) => {
+  const { data } = await github.rest.repos.listReleases({ owner, repo, per_page: limit })
+  return Array.isArray(data) ? data.slice(0, limit) : []
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@redpanda-data/docs-extensions-and-macros",
-  "version": "5.53.0",
+  "version": "5.54.0",
   "description": "Antora extensions and macros developed for Redpanda documentation.",
   "keywords": [
     "antora",

package/tools/connect-docs/check-build-log.js

@@ -0,0 +1,311 @@
+'use strict'
+
+const fs = require('fs')
+const path = require('path')
+
+/**
+ * doc-tools check-build-log: the blocking gate of the Connect docs PR check.
+ *
+ * Antora's own --log-failure-level makes Antora exit non-zero, but it says
+ * nothing about WHAT failed in a form a job summary can show, and it cannot
+ * catch the build that exits 0 having written no pages at all (Antora on an
+ * unsupported Node version does exactly that). So the workflow runs Antora
+ * with --log-failure-level=fatal and --log-format json, and this command
+ * decides:
+ *
+ *   - any record at level error or fatal fails the check. With
+ *     --blocking-sources, an error fails it only when it comes from one of
+ *     those repositories (its own source or any page in its include chain),
+ *     or has no source at all (an extension error); errors from the other
+ *     sources are listed but do not block. Fatal records always block,
+ *   - with --min-pages and --site-dir, fewer HTML pages than expected fails it,
+ *   - warnings never fail it; they are listed in the summary.
+ *
+ * The summary is Markdown, written to stdout, for $GITHUB_STEP_SUMMARY.
+ */
+
+const LEVELS = ['fatal', 'error', 'warn', 'info', 'debug', 'trace']
+const BLOCKING_LEVELS = new Set(['fatal', 'error'])
+// pino's numeric levels, in case a log was written without level labels.
+const NUMERIC_LEVELS = { 10: 'trace', 20: 'debug', 30: 'info', 40: 'warn', 50: 'error', 60: 'fatal' }
+
+const DEFAULTS = Object.freeze({
+  maxGroups: 50,
+  maxLocations: 5
+})
+
+function normalizeLevel (level) {
+  if (typeof level === 'number') return NUMERIC_LEVELS[level] || String(level)
+  return String(level || 'info').toLowerCase()
+}
+
+/**
+ * A human-readable location for one log record: the source repository and
+ * ref, then the file and line. Local worktree paths are made relative to the
+ * worktree so the same file reads the same on every runner.
+ */
+function relativePath (file, source) {
+  let filePath = (file && file.path) || ''
+  if (filePath && source && source.worktree && path.isAbsolute(filePath)) {
+    const rel = path.relative(source.worktree, filePath)
+    if (!rel.startsWith('..')) filePath = rel
+  }
+  return filePath
+}
+
+/** owner/name of a source URL, without the host or .git. */
+function repoSlug (url) {
+  return String(url || '').replace(/\.git$/, '').replace(/^[a-z]+:\/\/[^/]+\//i, '').replace(/^git@[^:]+:/, '')
+}
+
+function locationOf (record) {
+  const file = record.file || {}
+  const source = record.source || {}
+  if (!file.path && !source.url) return null
+  const filePath = relativePath(file, source)
+  let where = ''
+  if (source.url) {
+    const repo = repoSlug(source.url)
+    where = source.refname ? `${repo}@${source.refname}` : repo
+    if (source.startPath) where += `/${source.startPath}`
+  }
+  let loc = filePath
+  if (loc && file.line) loc += `:${file.line}`
+  // The include chain: the page that pulled the file in, which is the useful
+  // location when the problem is in a shared partial.
+  const via = Array.isArray(record.stack) && record.stack.length
+    ? record.stack.map((s) => {
+      const p = relativePath(s.file, s.source)
+      return p ? `${p}${s.file.line ? `:${s.file.line}` : ''}` : null
+    }).filter(Boolean)
+    : []
+  let out = [where, loc].filter(Boolean).join(': ')
+  if (via.length) out += ` (included from ${via.join(' < ')})`
+  return out || null
+}
+
+function messageOf (record) {
+  if (record.msg) return String(record.msg)
+  if (record.err && record.err.message) return String(record.err.message)
+  if (record.message) return String(record.message)
+  return '(no message)'
+}
+
+/**
+ * Parse an Antora JSON (NDJSON) log. Lines that are not JSON objects (stray
+ * stderr, a progress bar) are counted, not fatal: a log is still checkable
+ * when something else wrote to the same file.
+ */
+function parseLog (text) {
+  const records = []
+  let unparsed = 0
+  for (const line of String(text).split('\n')) {
+    const trimmed = line.trim()
+    if (!trimmed) continue
+    let obj
+    try {
+      obj = JSON.parse(trimmed)
+    } catch {
+      unparsed++
+      continue
+    }
+    if (!obj || typeof obj !== 'object' || Array.isArray(obj)) {
+      unparsed++
+      continue
+    }
+    records.push(obj)
+  }
+  return { records, unparsed }
+}
+
+function countHtmlPages (dir) {
+  let count = 0
+  const stack = [dir]
+  while (stack.length) {
+    const current = stack.pop()
+    let entries
+    try {
+      entries = fs.readdirSync(current, { withFileTypes: true })
+    } catch {
+      continue
+    }
+    for (const entry of entries) {
+      const full = path.join(current, entry.name)
+      if (entry.isDirectory()) stack.push(full)
+      else if (entry.isFile() && entry.name.endsWith('.html')) count++
+    }
+  }
+  return count
+}
+
+/**
+ * Check a parsed log. Returns the counts, the grouped messages, and whether
+ * the check passed.
+ */
+/**
+ * Whether an error-level record blocks. Without a source list, every one
+ * does. With one, a record blocks when it has no source at all (an
+ * extension's own error), or when its source or any source in its include
+ * chain is one of the listed repositories.
+ */
+function isBlocking (record, level, blockingSources) {
+  if (level === 'fatal') return true
+  if (!blockingSources || !blockingSources.length) return true
+  const sources = [record.source, ...(Array.isArray(record.stack) ? record.stack.map((s) => s && s.source) : [])]
+    .filter((s) => s && s.url)
+  if (!sources.length) return true
+  const wanted = new Set(blockingSources.map((s) => repoSlug(s).toLowerCase()))
+  return sources.some((s) => wanted.has(repoSlug(s.url).toLowerCase()))
+}
+
+function checkBuildLog ({ text, siteDir, minPages, blockingSources } = {}) {
+  const { records, unparsed } = parseLog(text)
+  const counts = Object.fromEntries(LEVELS.map((l) => [l, 0]))
+  const groups = new Map()
+  let blocking = 0
+  for (const record of records) {
+    let level = normalizeLevel(record.level)
+    counts[level] = (counts[level] || 0) + 1
+    if (!BLOCKING_LEVELS.has(level) && level !== 'warn') continue
+    if (BLOCKING_LEVELS.has(level)) {
+      if (isBlocking(record, level, blockingSources)) blocking++
+      else level = 'error-elsewhere'
+    }
+    const msg = messageOf(record)
+    const key = `${level}\u0000${msg}`
+    let group = groups.get(key)
+    if (!group) {
+      group = { level, msg, count: 0, locations: [] }
+      groups.set(key, group)
+    }
+    group.count++
+    const loc = locationOf(record)
+    if (loc && !group.locations.includes(loc)) group.locations.push(loc)
+  }
+
+  const failures = []
+  if (blocking > 0) {
+    const scope = blockingSources && blockingSources.length ? ` from ${blockingSources.join(', ')}` : ''
+    failures.push(`${blocking} log ${blocking === 1 ? 'record' : 'records'} at level error or fatal${scope}`)
+  }
+
+  let pages = null
+  if (minPages != null) {
+    if (!siteDir) throw new Error('--min-pages needs --site-dir')
+    pages = fs.existsSync(siteDir) ? countHtmlPages(siteDir) : 0
+    if (pages < minPages) {
+      failures.push(`${pages} HTML ${pages === 1 ? 'page' : 'pages'} written under ${siteDir}, fewer than the required ${minPages}`)
+    }
+  }
+
+  const byLevel = (level) => [...groups.values()]
+    .filter((g) => g.level === level)
+    .sort((a, b) => b.count - a.count || a.msg.localeCompare(b.msg))
+
+  return {
+    ok: failures.length === 0,
+    failures,
+    counts,
+    total: records.length,
+    unparsed,
+    pages,
+    minPages: minPages == null ? null : minPages,
+    blocking,
+    blockingSources: blockingSources && blockingSources.length ? blockingSources : null,
+    groups: {
+      fatal: byLevel('fatal'),
+      error: byLevel('error'),
+      errorElsewhere: byLevel('error-elsewhere'),
+      warn: byLevel('warn')
+    }
+  }
+}
+
+// Markdown table cells and inline code cannot hold a raw pipe or backtick
+// run, and a message can contain both.
+function mdInline (text) {
+  return String(text).replace(/\r?\n/g, ' ').replace(/\|/g, '\\|').replace(/</g, '&lt;')
+}
+
+function formatGroups (title, groups, { maxGroups, maxLocations }) {
+  if (!groups.length) return []
+  const lines = [`### ${title}`, '']
+  for (const g of groups.slice(0, maxGroups)) {
+    lines.push(`- ${g.count > 1 ? `(${g.count}x) ` : ''}${mdInline(g.msg)}`)
+    for (const loc of g.locations.slice(0, maxLocations)) lines.push(`  - ${mdInline(loc)}`)
+    if (g.locations.length > maxLocations) lines.push(`  - and ${g.locations.length - maxLocations} more locations`)
+  }
+  if (groups.length > maxGroups) lines.push(`- and ${groups.length - maxGroups} more distinct messages`)
+  lines.push('')
+  return lines
+}
+
+function formatMarkdown (result, opts = {}) {
+  const { maxGroups, maxLocations } = { ...DEFAULTS, ...opts }
+  const lines = []
+  lines.push(`## Antora build log: ${result.ok ? 'passed' : 'failed'}`, '')
+  if (!result.ok) {
+    for (const f of result.failures) lines.push(`- **Blocking:** ${f}`)
+    lines.push('')
+  }
+  lines.push('| Level | Count |', '|---|---|')
+  for (const level of LEVELS) {
+    if (level === 'debug' || level === 'trace') {
+      if (!result.counts[level]) continue
+    }
+    lines.push(`| ${level} | ${result.counts[level] || 0} |`)
+  }
+  if (result.pages != null) lines.push(`| HTML pages | ${result.pages} (minimum ${result.minPages}) |`)
+  if (result.unparsed) lines.push(`| non-JSON lines | ${result.unparsed} |`)
+  lines.push('')
+  lines.push(...formatGroups('Fatal', result.groups.fatal, { maxGroups, maxLocations }))
+  lines.push(...formatGroups('Errors', result.groups.error, { maxGroups, maxLocations }))
+  if (result.groups.errorElsewhere.length) {
+    lines.push(...formatGroups(`Errors from other sources (do not block; only ${result.blockingSources.join(', ')} block)`, result.groups.errorElsewhere, { maxGroups, maxLocations }))
+  }
+  lines.push(...formatGroups('Warnings (do not block)', result.groups.warn, { maxGroups, maxLocations }))
+  return lines.join('\n')
+}
+
+function intOption (value, flag) {
+  if (value == null) return undefined
+  const n = Number(value)
+  if (!Number.isInteger(n) || n < 0) throw new Error(`${flag} must be a non-negative integer, got ${value}`)
+  return n
+}
+
+function runCli (logFile, options = {}) {
+  let result
+  try {
+    if (!logFile || !fs.existsSync(logFile)) throw new Error(`log file not found: ${logFile}`)
+    result = checkBuildLog({
+      text: fs.readFileSync(logFile, 'utf8'),
+      siteDir: options.siteDir,
+      minPages: intOption(options.minPages, '--min-pages'),
+      blockingSources: options.blockingSources
+        ? String(options.blockingSources).split(',').map((x) => x.trim()).filter(Boolean)
+        : undefined
+    })
+  } catch (err) {
+    console.error(`Error: ${err.message}`)
+    process.exit(2)
+  }
+  const out = options.format === 'json' ? JSON.stringify(result, null, 2) : formatMarkdown(result)
+  console.log(out)
+  if (options.output) {
+    fs.mkdirSync(path.dirname(path.resolve(options.output)), { recursive: true })
+    fs.writeFileSync(path.resolve(options.output), out + '\n')
+  }
+  process.exit(result.ok ? 0 : 1)
+}
+
+module.exports = {
+  DEFAULTS,
+  parseLog,
+  checkBuildLog,
+  countHtmlPages,
+  formatMarkdown,
+  locationOf,
+  isBlocking,
+  runCli
+}