@redpanda-data/docs-extensions-and-macros 5.14.0 → 5.17.0

package/tools/lint-strings/rules/common.js

@@ -0,0 +1,148 @@
+'use strict'
+
+/**
+ * Rules that apply to every doc-string surface.
+ *
+ * Rule shape follows tools/rpk-docs/validate-output.js VALIDATION_RULES:
+ * { name, description, severity, check(decl) -> issues[] }.
+ * `decl` is the declaration object documented in ../engine.js.
+ */
+
+/**
+ * Jargon that ships verbatim to docs.redpanda.com but means nothing to users
+ * unless expanded. Matched on word boundaries (underscores count as word
+ * characters, so `archival_meta_stm` does not trip the `stm` entry).
+ */
+const JARGON_TERMS = ['NTP', 'seastar', 'nullopt', 'stm', 'smp']
+
+/**
+ * Tokenize a name or description for tautology comparison: strip `_`/`-`,
+ * case-fold, split on non-alphanumerics.
+ */
+function tokenize (text) {
+  return String(text)
+    .toLowerCase()
+    .split(/[^a-z0-9]+/)
+    .filter(Boolean)
+}
+
+/**
+ * Two tokens "match" when one is a prefix of the other (so `id` matches
+ * `identifier`), with a 2-character minimum to avoid single-letter noise.
+ */
+function tokensMatch (a, b) {
+  const [short, long] = a.length <= b.length ? [a, b] : [b, a]
+  return short.length >= 2 && long.startsWith(short)
+}
+
+/**
+ * True when the normalized description is just the normalized name:
+ * same token count, and every description token pairs with a distinct name
+ * token (any order, prefix-tolerant).
+ */
+function isNameEcho (name, description) {
+  const nameTokens = tokenize(name)
+  const descTokens = tokenize(description.replace(/[.!?]\s*$/, ''))
+  if (nameTokens.length === 0 || descTokens.length !== nameTokens.length) return false
+
+  const used = new Array(nameTokens.length).fill(false)
+  for (const descToken of descTokens) {
+    const idx = nameTokens.findIndex((nameToken, i) => !used[i] && tokensMatch(descToken, nameToken))
+    if (idx === -1) return false
+    used[idx] = true
+  }
+  return true
+}
+
+const COMMON_RULES = [
+  {
+    name: 'empty-description',
+    description: 'Declaration ships with no doc string at all',
+    severity: 'error',
+    check: (decl) => {
+      if (decl.string == null || decl.string.trim() === '') {
+        return [{ message: 'Description is empty or missing. This ships as a blank entry on docs.redpanda.com.' }]
+      }
+      return []
+    }
+  },
+  {
+    name: 'starts-lowercase',
+    description: 'Description starts with a lowercase letter',
+    severity: 'warning',
+    check: (decl) => {
+      const text = (decl.string || '').trim()
+      if (!text) return []
+      // A description may legitimately open with a backticked literal
+      // (for example, "`true` means ..."); only flag bare prose.
+      if (text.startsWith('`')) return []
+      const firstAlpha = text.match(/[A-Za-z]/)
+      if (firstAlpha && firstAlpha.index === 0 && /[a-z]/.test(firstAlpha[0])) {
+        // Mixed-case product spellings (gRPC, iOS, jsonPath) are correct as
+        // written; only all-lowercase first words are capitalization drift.
+        const firstWord = text.split(/\s/, 1)[0]
+        if (/[A-Z]/.test(firstWord)) return []
+        return [{ message: `Description starts with a lowercase letter: "${text.slice(0, 60)}"` }]
+      }
+      return []
+    }
+  },
+  {
+    name: 'name-echo',
+    description: 'Description merely restates the declared name (tautology)',
+    severity: 'warning',
+    check: (decl) => {
+      const text = (decl.string || '').trim()
+      if (!text || !decl.name) return []
+      if (isNameEcho(decl.name, text)) {
+        return [{ message: `Description just restates the name "${decl.name}": "${text}". Say what it does, why, or what happens when it changes.` }]
+      }
+      return []
+    }
+  },
+  {
+    name: 'too-short',
+    description: 'Description is under 20 characters',
+    severity: 'warning',
+    check: (decl) => {
+      const text = (decl.string || '').trim()
+      if (!text) return [] // empty-description owns the empty case
+      if (text.length < 20) {
+        return [{ message: `Description is only ${text.length} characters: "${text}". Too short to explain behavior, units, or impact.` }]
+      }
+      return []
+    }
+  },
+  {
+    name: 'unexpanded-jargon',
+    description: 'Internal jargon that ships to users unexpanded',
+    severity: 'warning',
+    check: (decl) => {
+      const text = decl.string || ''
+      if (!text) return []
+      const issues = []
+      for (const term of JARGON_TERMS) {
+        const pattern = new RegExp(`\\b${term}\\b`, 'i')
+        if (pattern.test(text)) {
+          issues.push({ message: `Contains internal jargon "${term}". Expand or replace it; users do not know this term.` })
+        }
+      }
+      return issues
+    }
+  },
+  {
+    name: 'unbalanced-backticks',
+    description: 'Odd number of backticks ships broken markup',
+    severity: 'error',
+    check: (decl) => {
+      const text = decl.string || ''
+      const count = (text.match(/`/g) || []).length
+      if (count % 2 !== 0) {
+        return [{ message: `Unbalanced backticks (${count} found). The rendered page ships broken monospace markup.` }]
+      }
+      return []
+    }
+  }
+]
+
+module.exports = { COMMON_RULES, JARGON_TERMS, isNameEcho, tokenize }

package/tools/lint-strings/rules/verbatim-asciidoc.js

@@ -0,0 +1,108 @@
+'use strict'
+
+/**
+ * Rules for verbatim-AsciiDoc surfaces (properties, metrics, crd, connect).
+ *
+ * These strings ship to docs.redpanda.com with ZERO escaping: a raw `|`
+ * breaks the containing table cell, an unknown `{attr}` renders literally or
+ * trips attribute-missing handling, and a half-written macro ships as broken
+ * markup. rpk is exempt: its formatDescription() transformer rewrites the
+ * string before publication.
+ */
+
+/**
+ * Product attributes that are legitimately available on rendered pages.
+ * Anything else inside `{...}` is not substituted and ships broken.
+ * Agent-extensible: add attributes here as surfaces grow.
+ */
+const KNOWN_ATTRIBUTES = new Set([
+  'latest-version',
+  'latest-redpanda-version',
+  'latest-redpanda-tag',
+  'latest-console-version',
+  'latest-operator-version',
+  'latest-connect-version',
+  'full-version',
+  'page-version',
+  'empty'
+])
+
+/** AsciiDoc inline macros used in embedded doc strings. */
+const MACRO_NAMES = ['xref', 'glossterm', 'config_ref']
+
+/**
+ * Scan text for unescaped `|` characters outside backtick spans.
+ * Returns the count found.
+ */
+function countRawPipes (text) {
+  let inBacktick = false
+  let count = 0
+  for (let i = 0; i < text.length; i++) {
+    const ch = text[i]
+    if (ch === '`') {
+      inBacktick = !inBacktick
+      continue
+    }
+    if (ch === '|' && !inBacktick && text[i - 1] !== '\\') count++
+  }
+  return count
+}
+
+const VERBATIM_ASCIIDOC_RULES = [
+  {
+    name: 'raw-pipe',
+    description: 'Raw unescaped | outside backticks breaks AsciiDoc tables',
+    severity: 'error',
+    check: (decl) => {
+      const text = decl.string || ''
+      if (!text.includes('|')) return []
+      const count = countRawPipes(text)
+      if (count > 0) {
+        return [{ message: `Contains ${count} raw unescaped "|" outside backticks. This string is rendered verbatim inside AsciiDoc tables; escape it (\\|) or wrap it in backticks.` }]
+      }
+      return []
+    }
+  },
+  {
+    name: 'unknown-attribute',
+    description: '{attr} reference that is not a known product attribute',
+    severity: 'warning',
+    check: (decl) => {
+      const text = decl.string || ''
+      const issues = []
+      const pattern = /\{([a-z][a-z0-9_-]*)\}/g
+      let match
+      while ((match = pattern.exec(text)) !== null) {
+        // A backslash-escaped \{...} is not substituted; authors escape
+        // deliberately (URL path templates, JSON pointers).
+        if (match.index > 0 && text[match.index - 1] === '\\') continue
+        if (!KNOWN_ATTRIBUTES.has(match[1])) {
+          issues.push({ message: `"{${match[1]}}" looks like an AsciiDoc attribute reference but is not a known product attribute. It will not be substituted on the rendered page.` })
+        }
+      }
+      return issues
+    }
+  },
+  {
+    name: 'broken-macro',
+    description: 'xref:/glossterm:/config_ref: macro with unbalanced brackets',
+    severity: 'error',
+    check: (decl) => {
+      const text = decl.string || ''
+      const issues = []
+      const pattern = new RegExp(`\\b(${MACRO_NAMES.join('|')}):`, 'g')
+      let match
+      while ((match = pattern.exec(text)) !== null) {
+        const rest = text.slice(match.index + match[0].length)
+        // Well-formed: <target>[<label>] with the target free of whitespace
+        // and brackets, and the label closed before end of string.
+        if (!/^[^\s[\]]*\[[^\]]*\]/.test(rest)) {
+          issues.push({ message: `Broken ${match[1]}: macro syntax near "${text.slice(match.index, match.index + 50)}". Expected ${match[1]}:target[label] with balanced brackets.` })
+        }
+      }
+      return issues
+    }
+  }
+]
+
+module.exports = { VERBATIM_ASCIIDOC_RULES, KNOWN_ATTRIBUTES, MACRO_NAMES, countRawPipes }

package/tools/lint-strings/source-text.js

@@ -0,0 +1,41 @@
+'use strict'
+
+const fs = require('fs')
+const path = require('path')
+
+/**
+ * Shared helper for surface modules: read the exact source lines of a
+ * declaration span so findings can carry `declaration_text` (the text a PR
+ * suggestion block would replace).
+ */
+class SourceCache {
+  constructor (repo) {
+    this.repo = repo
+    this.files = new Map()
+  }
+
+  lines (file) {
+    if (!this.files.has(file)) {
+      const absPath = path.isAbsolute(file) ? file : path.join(this.repo, file)
+      try {
+        this.files.set(file, fs.readFileSync(absPath, 'utf8').split('\n'))
+      } catch (err) {
+        this.files.set(file, null)
+      }
+    }
+    return this.files.get(file)
+  }
+
+  /**
+   * Exact source text of the inclusive 1-indexed span [lineStart, lineEnd],
+   * or null when the file or span is unavailable.
+   */
+  span (file, lineStart, lineEnd) {
+    if (lineStart == null || lineEnd == null) return null
+    const lines = this.lines(file)
+    if (!lines || lineStart < 1 || lineEnd > lines.length) return null
+    return lines.slice(lineStart - 1, lineEnd).join('\n')
+  }
+}
+
+module.exports = { SourceCache }

package/tools/lint-strings/surfaces/connect.js

@@ -0,0 +1,233 @@
+'use strict'
+
+const fs = require('fs')
+const path = require('path')
+
+const { SourceCache } = require('../source-text')
+const {
+  maskComments,
+  findBalancedClose,
+  splitTopLevelArgs,
+  collectStringConsts,
+  evalStringExpr,
+  lineOf,
+  collectGoFiles
+} = require('../go-source')
+
+/**
+ * Connect surface: service.NewConfigSpec() Summary/Description strings and
+ * service.NewXField(name).Description(...) chains in internal/impl.
+ *
+ * These strings are AsciiDoc page bodies (connect docs are generated from
+ * them directly), so AsciiDoc constructs are LEGITIMATE here: `==` headings,
+ * links, code fences, and the `raw + "`literal`" + raw` backtick
+ * concatenation idiom all appear in healthy descriptions. The scanner
+ * evaluates Go string concatenation (raw strings, interpreted strings, and
+ * same-file constants) so those idioms resolve to the exact shipped text.
+ * Descriptions built from calls or cross-file values are skipped silently -
+ * suggest-only posture means silence beats guessing.
+ *
+ * Because a connect description IS the page body (not a table cell), the
+ * verbatim raw-pipe rule is skipped: a bare `|` is valid AsciiDoc table
+ * syntax in this position.
+ */
+
+const CONVENTION = {
+  case: 'sentence',
+  terminal_period: true,
+  verbatim_asciidoc: true
+}
+
+// Field constructors that ship with NO built-in documentation: a chain on
+// one of these without .Description() publishes an empty field description.
+// Composite helpers (NewTLSToggledField, NewAutoRetryNacksToggleField,
+// NewBatchPolicyField, ...) carry their own docs and are exempt.
+const BARE_FIELD_CTORS = new Set([
+  'NewStringField', 'NewStringListField', 'NewStringMapField',
+  'NewStringEnumField', 'NewStringAnnotatedEnumField',
+  'NewIntField', 'NewIntListField', 'NewIntMapField',
+  'NewFloatField', 'NewBoolField', 'NewDurationField',
+  'NewInterpolatedStringField', 'NewInterpolatedStringListField',
+  'NewInterpolatedStringMapField', 'NewInterpolatedStringEnumField',
+  'NewBloblangField', 'NewAnyField', 'NewAnyListField', 'NewAnyMapField',
+  'NewObjectField', 'NewObjectListField', 'NewURLField'
+])
+
+const FIELD_CTOR = /\bNew([A-Z][A-Za-z0-9]*)Field\s*\(/g
+const CONFIG_SPEC = /\bNewConfigSpec\s*\(\s*\)/g
+const REGISTER_NAME = /\b(?:Must)?Register\w*\(\s*(?:"([^"]+)"|`([^`]+)`)\s*,/
+
+/**
+ * Follow a builder chain starting right after `startIndex` (the index just
+ * past a closing paren): repeated `.Method(args)`. Returns
+ * [{ method, args, openIndex, closeIndex }] and the chain end offset.
+ */
+function readChain (masked, startIndex) {
+  const calls = []
+  let i = startIndex
+  for (;;) {
+    const rest = masked.slice(i)
+    const m = rest.match(/^\s*\.\s*([A-Za-z0-9_]+)\s*\(/)
+    if (!m) break
+    const openIndex = i + m[0].length - 1
+    const closeIndex = findBalancedClose(masked, openIndex)
+    if (closeIndex === -1) break
+    calls.push({
+      method: m[1],
+      args: masked.slice(openIndex + 1, closeIndex),
+      openIndex,
+      closeIndex
+    })
+    i = closeIndex + 1
+  }
+  return { calls, end: i }
+}
+
+/**
+ * Scan one file's content for connect declarations. Exported for tests.
+ */
+function scanFile (content, file) {
+  const declarations = []
+  const masked = maskComments(content)
+  const consts = collectStringConsts(masked)
+
+  // Component name: the registration literal, when the file has exactly one.
+  const registerMatches = [...masked.matchAll(new RegExp(REGISTER_NAME.source, 'g'))]
+  const componentName = registerMatches.length === 1
+    ? (registerMatches[0][1] || registerMatches[0][2])
+    : null
+
+  // ConfigSpec chains -> Summary / Description declarations.
+  CONFIG_SPEC.lastIndex = 0
+  let match
+  while ((match = CONFIG_SPEC.exec(masked)) !== null) {
+    const { calls } = readChain(masked, match.index + match[0].length)
+    for (const call of calls) {
+      if (call.method !== 'Summary' && call.method !== 'Description') continue
+      const result = evalStringExpr(call.args, consts)
+      if (!result.verifiable) continue // built dynamically; skip silently
+      declarations.push({
+        surface: 'connect',
+        name: componentName,
+        file,
+        line_start: lineOf(masked, call.openIndex),
+        line_end: lineOf(masked, call.closeIndex),
+        string: result.value.trim() === '' ? null : result.value.trim(),
+        declaration_text: null,
+        convention: CONVENTION,
+        meta: { kind: call.method.toLowerCase() }
+      })
+    }
+  }
+
+  // Field constructor chains -> field description declarations.
+  FIELD_CTOR.lastIndex = 0
+  while ((match = FIELD_CTOR.exec(masked)) !== null) {
+    const ctor = `New${match[1]}Field`
+    const openIndex = match.index + match[0].length - 1
+    const closeIndex = findBalancedClose(masked, openIndex)
+    if (closeIndex === -1) continue
+
+    const ctorArgs = splitTopLevelArgs(masked.slice(openIndex + 1, closeIndex))
+    const nameResult = ctorArgs.length > 0 ? evalStringExpr(ctorArgs[0], consts) : { verifiable: false, value: null }
+    const fieldName = nameResult.verifiable ? nameResult.value : null
+
+    const { calls, end } = readChain(masked, closeIndex + 1)
+    const descriptionCall = calls.find((c) => c.method === 'Description')
+
+    if (descriptionCall) {
+      const result = evalStringExpr(descriptionCall.args, consts)
+      if (result.verifiable) {
+        declarations.push({
+          surface: 'connect',
+          name: fieldName,
+          file,
+          line_start: lineOf(masked, match.index),
+          line_end: lineOf(masked, descriptionCall.closeIndex),
+          string: result.value.trim() === '' ? null : result.value.trim(),
+          declaration_text: null,
+          convention: CONVENTION,
+          meta: { kind: 'field', ctor }
+        })
+      }
+      // Unverifiable description: skip silently (cross-file constant or call).
+    } else if (BARE_FIELD_CTORS.has(ctor) && !calls.some((c) => c.method === 'Deprecated')) {
+      // Deprecated fields are not part of the user-facing docs contract.
+      // A bare constructor with no Description() anywhere in its chain
+      // ships an empty field description.
+      declarations.push({
+        surface: 'connect',
+        name: fieldName,
+        file,
+        line_start: lineOf(masked, match.index),
+        line_end: lineOf(masked, Math.max(closeIndex, end - 1)),
+        string: null,
+        declaration_text: null,
+        convention: CONVENTION,
+        meta: { kind: 'field', ctor, missing_description: true }
+      })
+    }
+    // Resume INSIDE the constructor args: object fields nest child field
+    // constructors (NewObjectField("x", NewStringField("y")...)).
+    FIELD_CTOR.lastIndex = openIndex + 1
+  }
+
+  return declarations
+}
+
+/**
+ * Extract connect declarations.
+ *
+ * @param {Object} options - { repo, files (Set of repo-relative paths, diff
+ *   mode; when omitted, scans internal/impl), log }
+ */
+function extract ({ repo, files = null }) {
+  const IMPL_ROOT = path.join('internal', 'impl')
+  let fileList
+  if (files) {
+    fileList = [...files].filter((f) => f.endsWith('.go') && !f.endsWith('_test.go'))
+  } else {
+    fileList = collectGoFiles(path.join(repo, IMPL_ROOT)).map((f) => path.join(IMPL_ROOT, f))
+  }
+
+  const cache = new SourceCache(repo)
+  const declarations = []
+  for (const file of fileList) {
+    const absPath = path.isAbsolute(file) ? file : path.join(repo, file)
+    if (!fs.existsSync(absPath)) continue
+    const content = fs.readFileSync(absPath, 'utf8')
+    if (!content.includes('Field(') && !content.includes('NewConfigSpec')) continue
+    for (const decl of scanFile(content, file)) {
+      decl.declaration_text = cache.span(file, decl.line_start, decl.line_end)
+      declarations.push(decl)
+    }
+  }
+  return declarations
+}
+
+/** Surface-specific convention rules. */
+const RULES = [
+  {
+    name: 'missing-field-description',
+    description: 'Config field constructed with no .Description()',
+    severity: 'warning',
+    check: (decl) => {
+      if (decl.meta.kind !== 'field' || !decl.meta.missing_description) return []
+      return [{ message: `Field ${decl.name ? `"${decl.name}" ` : ''}(${decl.meta.ctor}) has no .Description(). It ships as an undocumented field on the connector page.` }]
+    }
+  }
+]
+
+module.exports = {
+  name: 'connect',
+  convention: CONVENTION,
+  extract,
+  scanFile,
+  rules: RULES,
+  // A connect description is the AsciiDoc page body, not a table cell:
+  // bare | is legitimate table syntax there, and missing prose is handled
+  // by the connect-specific missing-field-description warning. Brace
+  // placeholders ({endpoint}, {my-domain}) are the established URL-template
+  // idiom in connect descriptions, so the unknown-attribute check is noise.
+  skipRules: ['raw-pipe', 'empty-description', 'unknown-attribute']
+}