@redpanda-data/docs-extensions-and-macros 5.13.5 → 5.17.0

package/tools/lint-strings/index.js

@@ -0,0 +1,229 @@
+'use strict'
+
+const fs = require('fs')
+const path = require('path')
+
+const { runRules, mergeResults } = require('./engine')
+const { getChangedLines, classifyDiff, spanIntersects, SURFACE_ROUTES } = require('./diff')
+const { COMMON_RULES } = require('./rules/common')
+const { VERBATIM_ASCIIDOC_RULES } = require('./rules/verbatim-asciidoc')
+
+/**
+ * doc-tools lint-strings: deterministic linting of user-facing doc strings
+ * embedded in engineering source code (the strings doc-tools publishes
+ * verbatim to docs.redpanda.com).
+ *
+ * Registered surfaces. SEAM: new surface modules plug in here - implement
+ * ./surfaces/<name>.js with the same contract as the existing modules
+ * ({ name, convention, extract({repo, files}), rules, [skipRules] }) and
+ * add it to this registry plus ./diff.js SURFACE_ROUTES for diff-mode path
+ * routing.
+ */
+const SURFACES = {
+  properties: require('./surfaces/properties'),
+  metrics: require('./surfaces/metrics'),
+  rpk: require('./surfaces/rpk'),
+  helm: require('./surfaces/helm'),
+  crd: require('./surfaces/crd'),
+  connect: require('./surfaces/connect')
+}
+
+/**
+ * Build the rule set for a surface: common rules + verbatim-AsciiDoc rules
+ * (for surfaces whose strings ship unescaped) + surface-specific rules.
+ * A surface may opt out of specific auto-applied rules via an optional
+ * skipRules array (for example, rpk skips too-short: one-line Shorts and
+ * flag usages are the convention, not a defect).
+ */
+function rulesFor (surface) {
+  const rules = [...COMMON_RULES]
+  if (surface.convention.verbatim_asciidoc) rules.push(...VERBATIM_ASCIIDOC_RULES)
+  rules.push(...surface.rules)
+  const skip = new Set(surface.skipRules || [])
+  return skip.size > 0 ? rules.filter((rule) => !skip.has(rule.name)) : rules
+}
+
+/**
+ * Run the linter.
+ *
+ * @param {Object} options
+ * @param {string} options.repo - Path to the engineering checkout (required)
+ * @param {string[]} [options.surfaces] - Surface names (default: all registered)
+ * @param {string} [options.diffBase] - Declaration-anchored diff mode: lint
+ *   only declarations whose span intersects lines changed since this ref
+ * @param {string[]} [options.skipRules]
+ * @param {string[]} [options.onlyRules]
+ * @param {Function} [options.log] - Progress logger (stderr by default)
+ * @returns {Object} { findings, summary, unsupported_surfaces }
+ */
+function lintStrings (options) {
+  const {
+    repo,
+    surfaces = null,
+    diffBase = null,
+    skipRules = [],
+    onlyRules = null,
+    log = (msg) => process.stderr.write(`${msg}\n`)
+  } = options
+
+  if (!repo) throw new Error('lint-strings requires --repo <path>')
+  const repoPath = path.resolve(repo)
+  if (!fs.existsSync(repoPath)) throw new Error(`Repo path does not exist: ${repoPath}`)
+
+  const requested = surfaces && surfaces.length > 0 ? surfaces : Object.keys(SURFACES)
+  for (const name of requested) {
+    if (!SURFACES[name]) {
+      const known = SURFACE_ROUTES.some((route) => route.surface === name)
+      throw new Error(known
+        ? `Surface "${name}" is routed but has no extractor registered yet. Registered: ${Object.keys(SURFACES).join(', ')}`
+        : `Unknown surface "${name}". Registered: ${Object.keys(SURFACES).join(', ')}`)
+    }
+  }
+
+  const results = []
+  const unsupportedSurfaces = []
+
+  if (diffBase) {
+    const changed = getChangedLines(repoPath, diffBase)
+    const classified = classifyDiff(changed)
+
+    for (const [surfaceName, files] of Object.entries(classified)) {
+      if (!SURFACES[surfaceName]) {
+        // Routed by path but no extractor registered yet (rpk/helm/crd/connect
+        // until their surface modules land). Report instead of dropping.
+        unsupportedSurfaces.push({ surface: surfaceName, files: [...files.keys()] })
+        continue
+      }
+      if (!requested.includes(surfaceName)) continue
+
+      const surface = SURFACES[surfaceName]
+      log(`[${surfaceName}] ${files.size} changed file(s) in diff; extracting declarations at HEAD...`)
+      const fileSet = new Set(files.keys())
+      const declarations = surface
+        .extract({ repo: repoPath, files: fileSet, log })
+        .filter((decl) => spanIntersects(decl.line_start, decl.line_end, files.get(decl.file)))
+      for (const decl of declarations) decl.in_pr_diff = true
+      results.push(runRules(declarations, rulesFor(surface), { skipRules, onlyRules }))
+    }
+  } else {
+    for (const surfaceName of requested) {
+      const surface = SURFACES[surfaceName]
+      log(`[${surfaceName}] extracting declarations from ${repoPath}...`)
+      const declarations = surface.extract({ repo: repoPath, log })
+      for (const decl of declarations) decl.in_pr_diff = false
+      results.push(runRules(declarations, rulesFor(surface), { skipRules, onlyRules }))
+    }
+  }
+
+  const merged = mergeResults(results)
+  merged.findings.sort((a, b) =>
+    a.surface.localeCompare(b.surface) || a.file.localeCompare(b.file) || (a.line_start || 0) - (b.line_start || 0))
+  merged.unsupported_surfaces = unsupportedSurfaces
+  return merged
+}
+
+/**
+ * Human-readable report (the default --format).
+ */
+function formatHuman (result) {
+  const lines = []
+  const { findings, summary } = result
+
+  for (const finding of findings) {
+    const span = finding.line_start != null ? `:${finding.line_start}-${finding.line_end}` : ''
+    const diffMark = finding.in_pr_diff ? ' [in PR diff]' : ''
+    lines.push(`${finding.file}${span}  ${finding.name || '(unresolved name)'}  (${finding.surface})${diffMark}`)
+    if (finding.string != null) {
+      const preview = finding.string.length > 100 ? `${finding.string.slice(0, 100)}...` : finding.string
+      lines.push(`  string: ${JSON.stringify(preview)}`)
+    }
+    for (const issue of finding.rules) {
+      lines.push(`  ${issue.severity.toUpperCase().padEnd(7)} [${issue.id}] ${issue.message}`)
+    }
+    lines.push('')
+  }
+
+  lines.push('='.repeat(60))
+  lines.push('LINT-STRINGS SUMMARY')
+  lines.push('='.repeat(60))
+  lines.push(`Declarations checked: ${summary.totalDeclarations}`)
+  lines.push(`Declarations flagged: ${summary.flaggedDeclarations}`)
+  lines.push(`Errors: ${summary.errors}  Warnings: ${summary.warnings}  Info: ${summary.info}`)
+  if (Object.keys(summary.byRule).length > 0) {
+    lines.push('\nBy rule:')
+    for (const [rule, counts] of Object.entries(summary.byRule).sort()) {
+      const parts = []
+      if (counts.errors) parts.push(`${counts.errors} errors`)
+      if (counts.warnings) parts.push(`${counts.warnings} warnings`)
+      if (counts.info) parts.push(`${counts.info} info`)
+      lines.push(`  ${rule}: ${parts.join(', ')}`)
+    }
+  }
+  if (Object.keys(summary.bySurface).length > 0) {
+    lines.push('\nBy surface:')
+    for (const [surface, counts] of Object.entries(summary.bySurface).sort()) {
+      lines.push(`  ${surface}: ${counts.flagged}/${counts.declarations} declarations flagged (${counts.errors} errors, ${counts.warnings} warnings, ${counts.info} info)`)
+    }
+  }
+  for (const entry of result.unsupported_surfaces || []) {
+    lines.push(`\nNote: diff touches ${entry.surface} files but that surface has no extractor registered yet: ${entry.files.join(', ')}`)
+  }
+  return lines.join('\n')
+}
+
+/**
+ * CLI entry point shared by bin/doc-tools.js and direct invocation
+ * (node tools/lint-strings --repo <path> ...).
+ *
+ * Exit code contract: always 0 (suggest, never block), unless --strict is
+ * passed AND there is at least one error-severity finding.
+ */
+function runCli (options) {
+  let result
+  try {
+    result = lintStrings({
+      repo: options.repo,
+      surfaces: options.surface ? String(options.surface).split(',').map((s) => s.trim()).filter(Boolean) : null,
+      diffBase: options.diff || null,
+      skipRules: options.skipRules ? String(options.skipRules).split(',').map((s) => s.trim()).filter(Boolean) : [],
+      onlyRules: options.onlyRules ? String(options.onlyRules).split(',').map((s) => s.trim()).filter(Boolean) : null
+    })
+  } catch (err) {
+    console.error(`Error: ${err.message}`)
+    process.exit(2)
+  }
+
+  if (options.format === 'json') {
+    console.log(JSON.stringify(result, null, 2))
+  } else {
+    console.log(formatHuman(result))
+  }
+
+  if (options.strict && result.summary.errors > 0) process.exit(1)
+  process.exit(0)
+}
+
+module.exports = { lintStrings, formatHuman, runCli, SURFACES, rulesFor }
+
+// Direct usage: node tools/lint-strings --repo <path> [--surface a,b]
+//   [--diff <base>] [--format json|human] [--strict]
+if (require.main === module) {
+  const args = process.argv.slice(2)
+  const options = {}
+  for (let i = 0; i < args.length; i++) {
+    const arg = args[i]
+    if (arg === '--repo') options.repo = args[++i]
+    else if (arg === '--surface') options.surface = args[++i]
+    else if (arg === '--diff') options.diff = args[++i]
+    else if (arg === '--format') options.format = args[++i]
+    else if (arg === '--skip-rules') options.skipRules = args[++i]
+    else if (arg === '--only-rules') options.onlyRules = args[++i]
+    else if (arg === '--strict') options.strict = true
+    else {
+      console.error(`Unknown argument: ${arg}`)
+      console.error('Usage: node tools/lint-strings --repo <path> [--surface a,b] [--diff <base>] [--format json|human] [--skip-rules a,b] [--only-rules a,b] [--strict]')
+      process.exit(2)
+    }
+  }
+  runCli(options)
+}

package/tools/lint-strings/rules/common.js

@@ -0,0 +1,148 @@
+'use strict'
+
+/**
+ * Rules that apply to every doc-string surface.
+ *
+ * Rule shape follows tools/rpk-docs/validate-output.js VALIDATION_RULES:
+ * { name, description, severity, check(decl) -> issues[] }.
+ * `decl` is the declaration object documented in ../engine.js.
+ */
+
+/**
+ * Jargon that ships verbatim to docs.redpanda.com but means nothing to users
+ * unless expanded. Matched on word boundaries (underscores count as word
+ * characters, so `archival_meta_stm` does not trip the `stm` entry).
+ */
+const JARGON_TERMS = ['NTP', 'seastar', 'nullopt', 'stm', 'smp']
+
+/**
+ * Tokenize a name or description for tautology comparison: strip `_`/`-`,
+ * case-fold, split on non-alphanumerics.
+ */
+function tokenize (text) {
+  return String(text)
+    .toLowerCase()
+    .split(/[^a-z0-9]+/)
+    .filter(Boolean)
+}
+
+/**
+ * Two tokens "match" when one is a prefix of the other (so `id` matches
+ * `identifier`), with a 2-character minimum to avoid single-letter noise.
+ */
+function tokensMatch (a, b) {
+  const [short, long] = a.length <= b.length ? [a, b] : [b, a]
+  return short.length >= 2 && long.startsWith(short)
+}
+
+/**
+ * True when the normalized description is just the normalized name:
+ * same token count, and every description token pairs with a distinct name
+ * token (any order, prefix-tolerant).
+ */
+function isNameEcho (name, description) {
+  const nameTokens = tokenize(name)
+  const descTokens = tokenize(description.replace(/[.!?]\s*$/, ''))
+  if (nameTokens.length === 0 || descTokens.length !== nameTokens.length) return false
+
+  const used = new Array(nameTokens.length).fill(false)
+  for (const descToken of descTokens) {
+    const idx = nameTokens.findIndex((nameToken, i) => !used[i] && tokensMatch(descToken, nameToken))
+    if (idx === -1) return false
+    used[idx] = true
+  }
+  return true
+}
+
+const COMMON_RULES = [
+  {
+    name: 'empty-description',
+    description: 'Declaration ships with no doc string at all',
+    severity: 'error',
+    check: (decl) => {
+      if (decl.string == null || decl.string.trim() === '') {
+        return [{ message: 'Description is empty or missing. This ships as a blank entry on docs.redpanda.com.' }]
+      }
+      return []
+    }
+  },
+  {
+    name: 'starts-lowercase',
+    description: 'Description starts with a lowercase letter',
+    severity: 'warning',
+    check: (decl) => {
+      const text = (decl.string || '').trim()
+      if (!text) return []
+      // A description may legitimately open with a backticked literal
+      // (for example, "`true` means ..."); only flag bare prose.
+      if (text.startsWith('`')) return []
+      const firstAlpha = text.match(/[A-Za-z]/)
+      if (firstAlpha && firstAlpha.index === 0 && /[a-z]/.test(firstAlpha[0])) {
+        // Mixed-case product spellings (gRPC, iOS, jsonPath) are correct as
+        // written; only all-lowercase first words are capitalization drift.
+        const firstWord = text.split(/\s/, 1)[0]
+        if (/[A-Z]/.test(firstWord)) return []
+        return [{ message: `Description starts with a lowercase letter: "${text.slice(0, 60)}"` }]
+      }
+      return []
+    }
+  },
+  {
+    name: 'name-echo',
+    description: 'Description merely restates the declared name (tautology)',
+    severity: 'warning',
+    check: (decl) => {
+      const text = (decl.string || '').trim()
+      if (!text || !decl.name) return []
+      if (isNameEcho(decl.name, text)) {
+        return [{ message: `Description just restates the name "${decl.name}": "${text}". Say what it does, why, or what happens when it changes.` }]
+      }
+      return []
+    }
+  },
+  {
+    name: 'too-short',
+    description: 'Description is under 20 characters',
+    severity: 'warning',
+    check: (decl) => {
+      const text = (decl.string || '').trim()
+      if (!text) return [] // empty-description owns the empty case
+      if (text.length < 20) {
+        return [{ message: `Description is only ${text.length} characters: "${text}". Too short to explain behavior, units, or impact.` }]
+      }
+      return []
+    }
+  },
+  {
+    name: 'unexpanded-jargon',
+    description: 'Internal jargon that ships to users unexpanded',
+    severity: 'warning',
+    check: (decl) => {
+      const text = decl.string || ''
+      if (!text) return []
+      const issues = []
+      for (const term of JARGON_TERMS) {
+        const pattern = new RegExp(`\\b${term}\\b`, 'i')
+        if (pattern.test(text)) {
+          issues.push({ message: `Contains internal jargon "${term}". Expand or replace it; users do not know this term.` })
+        }
+      }
+      return issues
+    }
+  },
+  {
+    name: 'unbalanced-backticks',
+    description: 'Odd number of backticks ships broken markup',
+    severity: 'error',
+    check: (decl) => {
+      const text = decl.string || ''
+      const count = (text.match(/`/g) || []).length
+      if (count % 2 !== 0) {
+        return [{ message: `Unbalanced backticks (${count} found). The rendered page ships broken monospace markup.` }]
+      }
+      return []
+    }
+  }
+]
+
+module.exports = { COMMON_RULES, JARGON_TERMS, isNameEcho, tokenize }

package/tools/lint-strings/rules/verbatim-asciidoc.js

@@ -0,0 +1,108 @@
+'use strict'
+
+/**
+ * Rules for verbatim-AsciiDoc surfaces (properties, metrics, crd, connect).
+ *
+ * These strings ship to docs.redpanda.com with ZERO escaping: a raw `|`
+ * breaks the containing table cell, an unknown `{attr}` renders literally or
+ * trips attribute-missing handling, and a half-written macro ships as broken
+ * markup. rpk is exempt: its formatDescription() transformer rewrites the
+ * string before publication.
+ */
+
+/**
+ * Product attributes that are legitimately available on rendered pages.
+ * Anything else inside `{...}` is not substituted and ships broken.
+ * Agent-extensible: add attributes here as surfaces grow.
+ */
+const KNOWN_ATTRIBUTES = new Set([
+  'latest-version',
+  'latest-redpanda-version',
+  'latest-redpanda-tag',
+  'latest-console-version',
+  'latest-operator-version',
+  'latest-connect-version',
+  'full-version',
+  'page-version',
+  'empty'
+])
+
+/** AsciiDoc inline macros used in embedded doc strings. */
+const MACRO_NAMES = ['xref', 'glossterm', 'config_ref']
+
+/**
+ * Scan text for unescaped `|` characters outside backtick spans.
+ * Returns the count found.
+ */
+function countRawPipes (text) {
+  let inBacktick = false
+  let count = 0
+  for (let i = 0; i < text.length; i++) {
+    const ch = text[i]
+    if (ch === '`') {
+      inBacktick = !inBacktick
+      continue
+    }
+    if (ch === '|' && !inBacktick && text[i - 1] !== '\\') count++
+  }
+  return count
+}
+
+const VERBATIM_ASCIIDOC_RULES = [
+  {
+    name: 'raw-pipe',
+    description: 'Raw unescaped | outside backticks breaks AsciiDoc tables',
+    severity: 'error',
+    check: (decl) => {
+      const text = decl.string || ''
+      if (!text.includes('|')) return []
+      const count = countRawPipes(text)
+      if (count > 0) {
+        return [{ message: `Contains ${count} raw unescaped "|" outside backticks. This string is rendered verbatim inside AsciiDoc tables; escape it (\\|) or wrap it in backticks.` }]
+      }
+      return []
+    }
+  },
+  {
+    name: 'unknown-attribute',
+    description: '{attr} reference that is not a known product attribute',
+    severity: 'warning',
+    check: (decl) => {
+      const text = decl.string || ''
+      const issues = []
+      const pattern = /\{([a-z][a-z0-9_-]*)\}/g
+      let match
+      while ((match = pattern.exec(text)) !== null) {
+        // A backslash-escaped \{...} is not substituted; authors escape
+        // deliberately (URL path templates, JSON pointers).
+        if (match.index > 0 && text[match.index - 1] === '\\') continue
+        if (!KNOWN_ATTRIBUTES.has(match[1])) {
+          issues.push({ message: `"{${match[1]}}" looks like an AsciiDoc attribute reference but is not a known product attribute. It will not be substituted on the rendered page.` })
+        }
+      }
+      return issues
+    }
+  },
+  {
+    name: 'broken-macro',
+    description: 'xref:/glossterm:/config_ref: macro with unbalanced brackets',
+    severity: 'error',
+    check: (decl) => {
+      const text = decl.string || ''
+      const issues = []
+      const pattern = new RegExp(`\\b(${MACRO_NAMES.join('|')}):`, 'g')
+      let match
+      while ((match = pattern.exec(text)) !== null) {
+        const rest = text.slice(match.index + match[0].length)
+        // Well-formed: <target>[<label>] with the target free of whitespace
+        // and brackets, and the label closed before end of string.
+        if (!/^[^\s[\]]*\[[^\]]*\]/.test(rest)) {
+          issues.push({ message: `Broken ${match[1]}: macro syntax near "${text.slice(match.index, match.index + 50)}". Expected ${match[1]}:target[label] with balanced brackets.` })
+        }
+      }
+      return issues
+    }
+  }
+]
+
+module.exports = { VERBATIM_ASCIIDOC_RULES, KNOWN_ATTRIBUTES, MACRO_NAMES, countRawPipes }

package/tools/lint-strings/source-text.js

@@ -0,0 +1,41 @@
+'use strict'
+
+const fs = require('fs')
+const path = require('path')
+
+/**
+ * Shared helper for surface modules: read the exact source lines of a
+ * declaration span so findings can carry `declaration_text` (the text a PR
+ * suggestion block would replace).
+ */
+class SourceCache {
+  constructor (repo) {
+    this.repo = repo
+    this.files = new Map()
+  }
+
+  lines (file) {
+    if (!this.files.has(file)) {
+      const absPath = path.isAbsolute(file) ? file : path.join(this.repo, file)
+      try {
+        this.files.set(file, fs.readFileSync(absPath, 'utf8').split('\n'))
+      } catch (err) {
+        this.files.set(file, null)
+      }
+    }
+    return this.files.get(file)
+  }
+
+  /**
+   * Exact source text of the inclusive 1-indexed span [lineStart, lineEnd],
+   * or null when the file or span is unavailable.
+   */
+  span (file, lineStart, lineEnd) {
+    if (lineStart == null || lineEnd == null) return null
+    const lines = this.lines(file)
+    if (!lines || lineStart < 1 || lineEnd > lines.length) return null
+    return lines.slice(lineStart - 1, lineEnd).join('\n')
+  }
+}
+
+module.exports = { SourceCache }