@redpanda-data/docs-extensions-and-macros 5.53.0 → 5.54.0

package/tools/connect-docs/diff-generated.js

@@ -0,0 +1,382 @@
+'use strict'
+
+const fs = require('fs')
+const path = require('path')
+
+/**
+ * doc-tools connect-docs-diff: what a connect PR changes on docs.redpanda.com.
+ *
+ * Compares two trees generated by connect's docs generator (the PR's merge
+ * base and its head) and maps every added, removed, or changed partial or
+ * example to the published page or pages that include it. Reviewing this
+ * output instead of the Go source catches every published change: connect
+ * strings, benthos strings that arrive through a go.mod bump, and generator
+ * changes alike.
+ *
+ * A tree root is any of: a directory that contains modules/components/
+ * (connect's docs/ directory), the modules/ directory itself (the root of the
+ * redpanda-connect-docs.tar.gz asset and of the docs artifact connect
+ * uploads), or the components/ directory.
+ */
+
+const DEFAULTS = Object.freeze({
+  siteUrl: 'https://docs.redpanda.com',
+  // The connect component publishes at /connect/. /redpanda-connect/ is an
+  // older prefix that now redirects there.
+  componentPath: 'connect',
+  maxBytes: 60000,
+  maxFileDiffBytes: 8000,
+  contextLines: 3
+})
+
+// The generator writes some families with a hyphenated type directory
+// (fields/rate-limits, examples/common/rate-limits) and others with an
+// underscore (descriptions/rate_limits). Pages use the underscore form.
+const TYPE_DIRS = new Set(['inputs', 'outputs', 'processors', 'caches', 'buffers', 'metrics', 'tracers', 'scanners', 'rate_limits'])
+
+// fields/config/<name> and the config examples are the reference sections of
+// a few hand-written pages. An explicit map, checked against the include
+// directives in rp-connect-docs; anything not listed is reported as having no
+// single page rather than guessed.
+const CONFIG_PAGES = Object.freeze({
+  http: 'components/http/about/',
+  logger: 'components/logger/about/',
+  redpanda: 'components/redpanda/about/',
+  templates: 'configuration/templating/'
+})
+
+const NO_SINGLE_PAGE = 'no single page'
+// A page row lists this many files, then a count.
+const MAX_FILES_PER_ROW = 10
+
+function resolveComponentsDir (root) {
+  const candidates = [
+    path.join(root, 'modules', 'components'),
+    path.join(root, 'docs', 'modules', 'components'),
+    path.join(root, 'components'),
+    root
+  ]
+  for (const dir of candidates) {
+    if (fs.existsSync(path.join(dir, 'partials')) || fs.existsSync(path.join(dir, 'examples'))) return dir
+  }
+  throw new Error(`${root} is not a generated docs tree: no modules/components, components, or partials/examples directory found`)
+}
+
+/** Files under partials/ and examples/, relative to the components dir. */
+function listTree (componentsDir) {
+  const out = new Map()
+  for (const family of ['partials', 'examples']) {
+    const base = path.join(componentsDir, family)
+    if (!fs.existsSync(base)) continue
+    const stack = [base]
+    while (stack.length) {
+      const current = stack.pop()
+      for (const entry of fs.readdirSync(current, { withFileTypes: true })) {
+        const full = path.join(current, entry.name)
+        if (entry.isDirectory()) stack.push(full)
+        else if (entry.isFile()) out.set(path.relative(componentsDir, full).split(path.sep).join('/'), full)
+      }
+    }
+  }
+  return out
+}
+
+function normalizeType (type) {
+  return type.replace(/-/g, '_')
+}
+
+/**
+ * The published pages for one generated file, relative to the component
+ * root, or [] when the file feeds no single page.
+ * `rel` is relative to the components dir: partials/... or examples/...
+ */
+function pagesFor (rel) {
+  const parts = rel.replace(/\.(adoc|ya?ml|json)$/, '').split('/')
+  const [family, kind, ...rest] = parts
+  if (family === 'partials') {
+    if (kind === 'bloblang-functions' && rest.length === 1) return [{ page: 'guides/bloblang/functions/', anchor: rest[0] }]
+    if (kind === 'bloblang-methods' && rest.length === 1) return [{ page: 'guides/bloblang/methods/', anchor: rest[0] }]
+    if (kind === 'bloblang' && rest.length === 1 && (rest[0] === 'functions' || rest[0] === 'methods')) {
+      return [{ page: `guides/bloblang/${rest[0]}/` }]
+    }
+    if (['fields', 'descriptions', 'metadata', 'examples', 'availability'].includes(kind) && rest.length === 2) {
+      const [type, name] = rest
+      if (type === 'config') return CONFIG_PAGES[name] ? [{ page: CONFIG_PAGES[name] }] : []
+      const t = normalizeType(type)
+      if (TYPE_DIRS.has(t)) return [{ page: `components/${t}/${name}/` }]
+    }
+    return []
+  }
+  if (family === 'examples' && (kind === 'common' || kind === 'advanced') && rest.length === 2) {
+    const [type, name] = rest
+    if (type === 'config') return CONFIG_PAGES[name] ? [{ page: CONFIG_PAGES[name] }] : []
+    const t = normalizeType(type)
+    if (TYPE_DIRS.has(t)) return [{ page: `components/${t}/${name}/` }]
+  }
+  return []
+}
+
+// ---------------------------------------------------------------------------
+// Unified diff (Myers O(ND)), small and dependency-free.
+
+function diffLines (a, b) {
+  const n = a.length
+  const m = b.length
+  const max = n + m
+  const offset = max
+  const v = new Array(2 * max + 2).fill(0)
+  const trace = []
+  outer:
+  for (let d = 0; d <= max; d++) {
+    trace.push(v.slice())
+    for (let k = -d; k <= d; k += 2) {
+      let x
+      if (k === -d || (k !== d && v[offset + k - 1] < v[offset + k + 1])) x = v[offset + k + 1]
+      else x = v[offset + k - 1] + 1
+      let y = x - k
+      while (x < n && y < m && a[x] === b[y]) { x++; y++ }
+      v[offset + k] = x
+      if (x >= n && y >= m) break outer
+    }
+  }
+  // Backtrack into an edit script of { op: ' ' | '-' | '+', line }.
+  const ops = []
+  let x = n
+  let y = m
+  for (let d = trace.length - 1; d >= 0 && (x > 0 || y > 0); d--) {
+    const vd = trace[d]
+    const k = x - y
+    let prevK
+    if (k === -d || (k !== d && vd[offset + k - 1] < vd[offset + k + 1])) prevK = k + 1
+    else prevK = k - 1
+    const prevX = vd[offset + prevK]
+    const prevY = prevX - prevK
+    while (x > prevX && y > prevY) { ops.push({ op: ' ', line: a[x - 1] }); x--; y-- }
+    if (d > 0) {
+      if (x === prevX) { ops.push({ op: '+', line: b[y - 1] }); y-- } else { ops.push({ op: '-', line: a[x - 1] }); x-- }
+    }
+  }
+  return ops.reverse()
+}
+
+function splitLines (text) {
+  if (text === '') return []
+  const lines = text.split('\n')
+  if (lines[lines.length - 1] === '') lines.pop()
+  return lines
+}
+
+function unifiedDiff (oldText, newText, { oldName = 'a', newName = 'b', context = DEFAULTS.contextLines } = {}) {
+  const a = splitLines(oldText)
+  const b = splitLines(newText)
+  const ops = diffLines(a, b)
+  if (!ops.some((o) => o.op !== ' ')) return ''
+  // Group changes into hunks with `context` lines around them.
+  const hunks = []
+  let i = 0
+  let aLine = 0
+  let bLine = 0
+  const positions = ops.map((o) => {
+    const pos = { a: aLine, b: bLine }
+    if (o.op !== '+') aLine++
+    if (o.op !== '-') bLine++
+    return pos
+  })
+  while (i < ops.length) {
+    if (ops[i].op === ' ') { i++; continue }
+    let start = Math.max(0, i - context)
+    let end = i
+    // Extend while the next change is within 2*context of the current one.
+    while (end < ops.length) {
+      if (ops[end].op !== ' ') { end++; continue }
+      let j = end
+      while (j < ops.length && ops[j].op === ' ') j++
+      if (j < ops.length && j - end <= 2 * context) { end = j; continue }
+      end = Math.min(ops.length, end + context)
+      break
+    }
+    hunks.push({ start, end })
+    i = end
+  }
+  const out = [`--- ${oldName}`, `+++ ${newName}`]
+  for (const h of hunks) {
+    const slice = ops.slice(h.start, h.end)
+    const aStart = positions[h.start].a
+    const bStart = positions[h.start].b
+    const aCount = slice.filter((o) => o.op !== '+').length
+    const bCount = slice.filter((o) => o.op !== '-').length
+    out.push(`@@ -${aCount ? aStart + 1 : aStart},${aCount} +${bCount ? bStart + 1 : bStart},${bCount} @@`)
+    for (const o of slice) out.push(`${o.op}${o.line}`)
+  }
+  return out.join('\n') + '\n'
+}
+
+// ---------------------------------------------------------------------------
+
+function diffGenerated ({ baseDir, headDir, siteUrl = DEFAULTS.siteUrl, componentPath = DEFAULTS.componentPath } = {}) {
+  const base = listTree(resolveComponentsDir(baseDir))
+  const head = listTree(resolveComponentsDir(headDir))
+  const all = [...new Set([...base.keys(), ...head.keys()])].sort()
+  const root = `${siteUrl.replace(/\/$/, '')}/${componentPath}/`
+  const files = []
+  for (const rel of all) {
+    const oldFile = base.get(rel)
+    const newFile = head.get(rel)
+    const oldText = oldFile ? fs.readFileSync(oldFile, 'utf8') : ''
+    const newText = newFile ? fs.readFileSync(newFile, 'utf8') : ''
+    let status
+    if (!oldFile) status = 'added'
+    else if (!newFile) status = 'removed'
+    else if (oldText !== newText) status = 'changed'
+    else continue
+    const pages = pagesFor(rel).map((p) => ({
+      url: `${root}${p.page}${p.anchor ? `#${p.anchor}` : ''}`,
+      page: `${root}${p.page}`,
+      // Where Antora writes the page, relative to the output directory.
+      sitePath: `${componentPath}/${p.page}index.html`
+    }))
+    files.push({ path: `modules/components/${rel}`, status, pages, oldText, newText })
+  }
+  // One row per page; files that feed no single page are listed on their own.
+  const byPage = new Map()
+  const unmapped = []
+  for (const f of files) {
+    if (!f.pages.length) { unmapped.push(f.path); continue }
+    for (const p of f.pages) {
+      const entry = byPage.get(p.page) || { url: p.page, sitePath: p.sitePath, anchors: [], files: [] }
+      if (p.url !== p.page && !entry.anchors.includes(p.url)) entry.anchors.push(p.url)
+      entry.files.push({ path: f.path, status: f.status })
+      byPage.set(p.page, entry)
+    }
+  }
+  const pages = [...byPage.values()].sort((a, b) => a.url.localeCompare(b.url))
+  return {
+    summary: {
+      files: files.length,
+      added: files.filter((f) => f.status === 'added').length,
+      removed: files.filter((f) => f.status === 'removed').length,
+      changed: files.filter((f) => f.status === 'changed').length,
+      pages: pages.length,
+      unmapped: unmapped.length
+    },
+    pages,
+    unmapped,
+    files
+  }
+}
+
+function fence (text) {
+  // A fence longer than any backtick run in the text, so a diff of AsciiDoc
+  // with its own fences cannot close the block early.
+  const longest = Math.max(2, ...(text.match(/`+/g) || []).map((s) => s.length))
+  return '`'.repeat(longest + 1)
+}
+
+function formatMarkdown (result, { maxBytes = DEFAULTS.maxBytes, maxFileDiffBytes = DEFAULTS.maxFileDiffBytes } = {}) {
+  const s = result.summary
+  const lines = []
+  lines.push('## Connect reference docs: published changes', '')
+  if (!s.files) {
+    lines.push('This PR does not change the generated reference docs.', '')
+    return lines.join('\n')
+  }
+  lines.push(`${s.files} generated ${s.files === 1 ? 'file' : 'files'} differ from the merge base (${s.changed} changed, ${s.added} added, ${s.removed} removed), on ${s.pages} published ${s.pages === 1 ? 'page' : 'pages'}.`, '')
+  if (result.pages.length) {
+    lines.push('| Page | Changed files |', '|---|---|')
+    for (const p of result.pages) {
+      const label = p.url.replace(/^https?:\/\/[^/]+/, '')
+      const shownFiles = p.files.slice(0, MAX_FILES_PER_ROW).map((f) => {
+        const name = f.path.replace(/^modules\/components\//, '')
+        // Per-entry partials (one Bloblang function or method) link to their
+        // section of the page.
+        const anchor = p.anchors.find((a) => a.endsWith(`#${path.basename(name).replace(/\.[^.]+$/, '')}`))
+        return `${anchor && f.status !== 'removed' ? `[${name}](${anchor})` : name} (${f.status})`
+      })
+      if (p.files.length > MAX_FILES_PER_ROW) shownFiles.push(`and ${p.files.length - MAX_FILES_PER_ROW} more`)
+      lines.push(`| [${label}](${p.url}) | ${shownFiles.join('<br>')} |`)
+    }
+    lines.push('')
+  }
+  if (result.unmapped.length) {
+    lines.push(`Files with ${NO_SINGLE_PAGE} (they feed several pages, or none directly):`, '')
+    for (const f of result.unmapped) lines.push(`- ${f}`)
+    lines.push('')
+  }
+  lines.push('### Diffs', '')
+  let used = Buffer.byteLength(lines.join('\n'))
+  let shown = 0
+  for (const f of result.files) {
+    let diff = unifiedDiff(f.oldText, f.newText, {
+      oldName: f.status === 'added' ? '/dev/null' : `a/${f.path}`,
+      newName: f.status === 'removed' ? '/dev/null' : `b/${f.path}`
+    })
+    let note = ''
+    if (Buffer.byteLength(diff) > maxFileDiffBytes) {
+      diff = diff.slice(0, maxFileDiffBytes)
+      diff = diff.slice(0, diff.lastIndexOf('\n') + 1)
+      note = `\n_Diff truncated at ${maxFileDiffBytes} bytes._\n`
+    }
+    const f2 = fence(diff)
+    const block = [
+      '<details>',
+      `<summary>${f.path} (${f.status})</summary>`,
+      '',
+      `${f2}diff`,
+      diff.replace(/\n$/, ''),
+      f2,
+      note,
+      '</details>',
+      ''
+    ].join('\n')
+    if (used + Buffer.byteLength(block) > maxBytes) break
+    lines.push(block)
+    used += Buffer.byteLength(block) + 1
+    shown++
+  }
+  if (shown < result.files.length) {
+    lines.push(`_${result.files.length - shown} more ${result.files.length - shown === 1 ? 'diff is' : 'diffs are'} not shown: the summary is capped at ${maxBytes} bytes._`, '')
+  }
+  return lines.join('\n')
+}
+
+function toJson (result) {
+  return {
+    summary: result.summary,
+    pages: result.pages,
+    unmapped: result.unmapped,
+    files: result.files.map((f) => ({ path: f.path, status: f.status, pages: f.pages.map((p) => p.url) }))
+  }
+}
+
+function runCli (baseDir, headDir, options = {}) {
+  let result
+  try {
+    for (const [label, dir] of [['base', baseDir], ['head', headDir]]) {
+      if (!dir || !fs.existsSync(dir)) throw new Error(`${label} directory not found: ${dir}`)
+    }
+    result = diffGenerated({ baseDir, headDir, siteUrl: options.siteUrl })
+  } catch (err) {
+    console.error(`Error: ${err.message}`)
+    process.exit(2)
+  }
+  const maxBytes = options.maxBytes != null ? Number(options.maxBytes) : DEFAULTS.maxBytes
+  const out = options.format === 'json' ? JSON.stringify(toJson(result), null, 2) : formatMarkdown(result, { maxBytes })
+  console.log(out)
+  if (options.output) {
+    fs.mkdirSync(path.dirname(path.resolve(options.output)), { recursive: true })
+    fs.writeFileSync(path.resolve(options.output), out + '\n')
+  }
+  process.exit(0)
+}
+
+module.exports = {
+  DEFAULTS,
+  CONFIG_PAGES,
+  resolveComponentsDir,
+  pagesFor,
+  unifiedDiff,
+  diffGenerated,
+  formatMarkdown,
+  toJson,
+  runCli
+}

package/tools/connect-docs/pr-comment.js

@@ -0,0 +1,104 @@
+'use strict'
+
+const fs = require('fs')
+
+/**
+ * The connect-docs-check PR comment: one sticky comment per connect PR that
+ * tells the author and the docs team what the PR changes on the published
+ * Connect docs, whether the docs build passed, and where to review the
+ * rendered pages.
+ *
+ * Usage: node pr-comment.js --build <pass|fail> [--diff diff.json] [--html-findings <n>]
+ *          [--preview <deploy url>] [--run-url <url>] [--sha <sha>]
+ *          [--output comment.md]
+ *
+ * Prints JSON { body, notify }. `notify` is true when the PR changes a
+ * published page, breaks the build, or has rendered-HTML findings on a page
+ * it changes: the cases where the docs team should
+ * look. The workflow posts the comment only when notify is true or a
+ * comment already exists, so PRs that touch no docs get no comment.
+ */
+
+const MARKER = '<!-- connect-docs-check -->'
+const MAX_ROWS = 30
+const PROD = 'https://docs.redpanda.com'
+
+function pagePath (sitePath) {
+  return '/' + String(sitePath).replace(/index\.html$/, '')
+}
+
+function buildComment ({ build, diff, preview, runUrl, sha, htmlFindings = 0 }) {
+  const pages = (diff && diff.pages) || []
+  const summary = (diff && diff.summary) || null
+  const failed = build !== 'pass'
+  const lines = [MARKER, '### Connect docs check', '']
+
+  lines.push(failed
+    ? '**The docs build failed.** This PR breaks the Connect docs build: see the job summary for the errors.'
+    : 'The docs build passed with this PR\'s generated docs.')
+  lines.push('')
+  if (htmlFindings > 0) {
+    lines.push(`**${htmlFindings} rendered-HTML ${htmlFindings === 1 ? 'finding' : 'findings'} on pages this PR changes**, such as broken anchors or unconverted AsciiDoc. They are listed in the job summary.`, '')
+  }
+
+  if (preview) {
+    lines.push(`Preview: ${preview.replace(/\/$/, '')}/connect/home/`, '')
+  }
+
+  if (!diff) {
+    lines.push('The merge base was not generated, so the changed pages are not listed.', '')
+  } else if (!pages.length && !(summary && summary.files)) {
+    lines.push('This PR doesn\'t change any published Connect docs page.', '')
+  } else {
+    const s = summary || {}
+    lines.push(`This PR changes ${pages.length} published ${pages.length === 1 ? 'page' : 'pages'} (${s.files || 0} generated ${s.files === 1 ? 'file' : 'files'}: ${s.changed || 0} changed, ${s.added || 0} added, ${s.removed || 0} removed). Review the rendered pages, not only the Go strings.`, '')
+    if (pages.length) {
+      lines.push(preview ? '| Page | Preview |' : '| Page |', preview ? '|---|---|' : '|---|')
+      for (const p of pages.slice(0, MAX_ROWS)) {
+        const pth = pagePath(p.sitePath)
+        const live = `[${pth}](${PROD}${pth})`
+        lines.push(preview ? `| ${live} | [preview](${preview.replace(/\/$/, '')}${pth}) |` : `| ${live} |`)
+      }
+      if (pages.length > MAX_ROWS) lines.push('', `And ${pages.length - MAX_ROWS} more pages.`)
+      lines.push('')
+    }
+  }
+
+  const tail = []
+  if (runUrl) tail.push(`Per-file diffs and the rendered-HTML findings are in the [job summary](${runUrl}).`)
+  if (sha) tail.push(`Checked at ${String(sha).slice(0, 9)}.`)
+  if (tail.length) lines.push(tail.join(' '))
+
+  return { body: lines.join('\n').trimEnd() + '\n', notify: failed || htmlFindings > 0 || pages.length > 0 }
+}
+
+function parseArgs (argv) {
+  const opts = {}
+  for (let i = 0; i < argv.length; i++) {
+    const m = /^--(build|diff|html-findings|preview|run-url|sha|output)$/.exec(argv[i])
+    if (!m) throw new Error(`unknown argument: ${argv[i]}`)
+    opts[m[1]] = argv[++i]
+  }
+  if (opts.build !== 'pass' && opts.build !== 'fail') throw new Error('--build must be pass or fail')
+  return opts
+}
+
+function runCli (argv) {
+  const opts = parseArgs(argv)
+  const diff = opts.diff && fs.existsSync(opts.diff) ? JSON.parse(fs.readFileSync(opts.diff, 'utf8')) : null
+  const result = buildComment({ build: opts.build, diff, htmlFindings: Number(opts['html-findings'] || 0), preview: opts.preview || '', runUrl: opts['run-url'], sha: opts.sha })
+  if (opts.output) fs.writeFileSync(opts.output, result.body)
+  process.stdout.write(JSON.stringify({ notify: result.notify }) + '\n')
+  return result
+}
+
+if (require.main === module) {
+  try {
+    runCli(process.argv.slice(2))
+  } catch (err) {
+    console.error(`Error: ${err.message}`)
+    process.exit(2)
+  }
+}
+
+module.exports = { MARKER, MAX_ROWS, buildComment, runCli }