@redpanda-data/docs-extensions-and-macros 5.14.0 → 5.22.0

package/tools/lint-strings/surfaces/rpk.js

@@ -0,0 +1,377 @@
+'use strict'
+
+const fs = require('fs')
+const path = require('path')
+
+const { SourceCache } = require('../source-text')
+const {
+  maskComments,
+  findBalancedClose,
+  splitTopLevelArgs,
+  collectStringConsts,
+  evalStringExpr,
+  lineOf,
+  collectGoFiles
+} = require('../go-source')
+
+/**
+ * rpk surface: cobra.Command Use/Short/Long strings and pflag usage strings
+ * in src/go/rpk/pkg/cli.
+ *
+ * Direct Go source scan (comment-masked regex/paren scanner in the style of
+ * surfaces/metrics.js) so findings carry exact file:line spans:
+ * - cobra.Command composite literals -> Short and Long declarations (Use is
+ *   captured to name the command, not linted as prose),
+ * - .Flags()/.PersistentFlags() registration calls (StringVar, BoolVarP,
+ *   Int32Var, Duration, ...) -> flag name + usage string.
+ *
+ * rpk is NOT a verbatim surface: generate-rpk-docs.js formatDescription()
+ * rewrites these strings (~40 passes: auto-backticking, e.g. -> "for
+ * example,", ALLCAPS -> === headings, ensurePeriod) before publication, so
+ * escaping rules do not apply. The convention rules here catch what the
+ * transformer cannot fix: lowercase starts, terminal periods, name echoes,
+ * and Long-text patterns that change page structure when transformed.
+ */
+
+const CONVENTION = {
+  case: 'sentence',
+  terminal_period: false,
+  verbatim_asciidoc: false,
+  transformer: 'formatDescription'
+}
+
+// Composite literal only: gofmt writes `cobra.Command{` with no space, while
+// a function signature returning *cobra.Command always has a space (or a
+// star) before its body brace - so this never swallows a function body.
+const COMMAND_LITERAL = /\bcobra\.Command\{/g
+// Registration methods are pflag value-type names (String, BoolVarP,
+// Int32Var, DurationP, StringSlice, ...). Accessors (Lookup, Set,
+// GetString, MarkHidden, ...) do not start with a value-type prefix.
+const FLAG_CALL = /\.\s*(?:Flags|PersistentFlags)\s*\(\s*\)\s*\.\s*([A-Za-z0-9]+)\s*\(/g
+const FLAG_METHOD = /^(String|Bool|Int|Uint|Float|Duration|Count|Bytes|IP)[A-Za-z0-9]*$/
+// Flag sets bound to a variable first: `f := cmd.Flags()` ... `f.StringVar(...)`.
+const FLAGSET_ASSIGN = /\b([A-Za-z_][A-Za-z0-9_]*)\s*:?=\s*[\w.]+\.(?:Flags|PersistentFlags)\s*\(\s*\)/g
+// Hidden and deprecated flags never reach the generated docs (cobra help and
+// printtree.go both skip them), so their usage strings are not linted.
+const MARK_UNPUBLISHED = /\.\s*Mark(?:Hidden|Deprecated)\s*\(/g
+
+/**
+ * Parse the top-level fields of a composite literal body.
+ * Returns Map(fieldName -> { start, end }) with offsets relative to body.
+ */
+function parseLiteralFields (body) {
+  const fields = new Map()
+  let depth = 0
+  let i = 0
+  while (i < body.length) {
+    const ch = body[i]
+    if (ch === '"' || ch === '`' || ch === "'") {
+      let j = i + 1
+      if (ch === '`') {
+        while (j < body.length && body[j] !== '`') j++
+      } else {
+        while (j < body.length && body[j] !== ch) j += body[j] === '\\' ? 2 : 1
+      }
+      i = j + 1
+      continue
+    }
+    if (ch === '(' || ch === '{' || ch === '[') { depth++; i++; continue }
+    if (ch === ')' || ch === '}' || ch === ']') { depth--; i++; continue }
+    if (depth === 0 && /[A-Za-z_]/.test(ch) && (i === 0 || /[\s,{]/.test(body[i - 1]))) {
+      let j = i
+      while (j < body.length && /[A-Za-z0-9_]/.test(body[j])) j++
+      // A field key is `Ident:` not followed by another ':' (rules out `::`).
+      if (body[j] === ':' && body[j + 1] !== ':' && body[j + 1] !== '=') {
+        const name = body.slice(i, j)
+        const valueStart = j + 1
+        const valueEnd = findTopLevelComma(body, valueStart)
+        fields.set(name, { start: i, valueStart, end: valueEnd })
+        i = valueEnd + 1
+        continue
+      }
+      i = j
+      continue
+    }
+    i++
+  }
+  return fields
+}
+
+/** Offset of the next top-level comma at/after start (or body end). */
+function findTopLevelComma (body, start) {
+  let depth = 0
+  let i = start
+  while (i < body.length) {
+    const ch = body[i]
+    if (ch === '"' || ch === '`' || ch === "'") {
+      let j = i + 1
+      if (ch === '`') {
+        while (j < body.length && body[j] !== '`') j++
+      } else {
+        while (j < body.length && body[j] !== ch) j += body[j] === '\\' ? 2 : 1
+      }
+      i = j + 1
+      continue
+    }
+    if (ch === '(' || ch === '{' || ch === '[') depth++
+    if (ch === ')' || ch === '}' || ch === ']') depth--
+    if (ch === ',' && depth === 0) return i
+    i++
+  }
+  return body.length
+}
+
+/**
+ * Scan one file's content for rpk doc-string declarations.
+ * Exported for tests.
+ */
+function scanFile (content, file) {
+  const declarations = []
+  const masked = maskComments(content)
+  const consts = collectStringConsts(masked)
+
+  // cobra.Command composite literals -> Short / Long
+  COMMAND_LITERAL.lastIndex = 0
+  let match
+  while ((match = COMMAND_LITERAL.exec(masked)) !== null) {
+    const openIndex = match.index + match[0].length - 1
+    const closeIndex = findBalancedClose(masked, openIndex, '{', '}')
+    if (closeIndex === -1) continue
+    const bodyStart = openIndex + 1
+    const body = masked.slice(bodyStart, closeIndex)
+    const fields = parseLiteralFields(body)
+
+    let commandName = null
+    let use = null
+    if (fields.has('Use')) {
+      const f = fields.get('Use')
+      const result = evalStringExpr(body.slice(f.valueStart, f.end), consts)
+      if (result.verifiable) {
+        use = result.value
+        commandName = use.trim().split(/\s+/)[0] || null
+      }
+    }
+
+    for (const kind of ['Short', 'Long']) {
+      if (!fields.has(kind)) continue
+      const f = fields.get(kind)
+      const result = evalStringExpr(body.slice(f.valueStart, f.end), consts)
+      declarations.push({
+        surface: 'rpk',
+        name: commandName,
+        file,
+        line_start: lineOf(masked, bodyStart + f.start),
+        line_end: lineOf(masked, bodyStart + f.end),
+        string: result.verifiable ? result.value : null,
+        declaration_text: null,
+        convention: CONVENTION,
+        meta: {
+          kind: kind.toLowerCase(),
+          use,
+          unverifiable: !result.verifiable
+        }
+      })
+    }
+    COMMAND_LITERAL.lastIndex = closeIndex + 1
+  }
+
+  // Flag registrations -> flag usage strings. Two shapes: direct chains
+  // (cmd.Flags().BoolVarP(...)) and flag sets bound to a variable first
+  // (f := cmd.Flags(); f.StringVar(...)).
+  const flagCalls = []
+  FLAG_CALL.lastIndex = 0
+  while ((match = FLAG_CALL.exec(masked)) !== null) {
+    flagCalls.push({ index: match.index, method: match[1], openIndex: match.index + match[0].length - 1 })
+  }
+  const flagsetVars = new Set()
+  FLAGSET_ASSIGN.lastIndex = 0
+  while ((match = FLAGSET_ASSIGN.exec(masked)) !== null) flagsetVars.add(match[1])
+  if (flagsetVars.size > 0) {
+    const varCall = new RegExp(`\\b(${[...flagsetVars].join('|')})\\s*\\.\\s*([A-Za-z0-9]+)\\s*\\(`, 'g')
+    while ((match = varCall.exec(masked)) !== null) {
+      flagCalls.push({ index: match.index, method: match[2], openIndex: match.index + match[0].length - 1 })
+    }
+    flagCalls.sort((a, b) => a.index - b.index)
+  }
+
+  // Flags marked hidden or deprecated anywhere in the file are excluded:
+  // they never ship to the generated docs, so their usage strings (often
+  // intentionally empty) are not part of the docs contract.
+  const unpublished = new Set()
+  MARK_UNPUBLISHED.lastIndex = 0
+  while ((match = MARK_UNPUBLISHED.exec(masked)) !== null) {
+    const openIndex = match.index + match[0].length - 1
+    const closeIndex = findBalancedClose(masked, openIndex)
+    if (closeIndex === -1) continue
+    const args = splitTopLevelArgs(masked.slice(openIndex + 1, closeIndex))
+    if (args.length === 0) continue
+    const result = evalStringExpr(args[0], consts)
+    if (result.verifiable) unpublished.add(result.value)
+  }
+
+  for (const call of flagCalls) {
+    if (!FLAG_METHOD.test(call.method)) continue
+    const openIndex = call.openIndex
+    const closeIndex = findBalancedClose(masked, openIndex)
+    if (closeIndex === -1) continue
+    const args = splitTopLevelArgs(masked.slice(openIndex + 1, closeIndex))
+    if (args.length < 2) continue
+
+    // Flag name: the first argument that is a resolvable string (Var forms
+    // put &target first). Shorthand ("w") is the following one-char string.
+    let flagName = null
+    for (const arg of args.slice(0, -1)) {
+      const result = evalStringExpr(arg, consts)
+      if (result.verifiable) {
+        flagName = result.value
+        break
+      }
+    }
+    if (flagName != null && unpublished.has(flagName)) continue
+    const usage = evalStringExpr(args[args.length - 1], consts)
+
+    declarations.push({
+      surface: 'rpk',
+      name: flagName,
+      file,
+      line_start: lineOf(masked, call.index),
+      line_end: lineOf(masked, closeIndex),
+      string: usage.verifiable ? usage.value : null,
+      declaration_text: null,
+      convention: CONVENTION,
+      meta: {
+        kind: 'flag',
+        method: call.method,
+        unverifiable: !usage.verifiable
+      }
+    })
+  }
+
+  return declarations
+}
+
+/**
+ * Extract rpk declarations.
+ *
+ * @param {Object} options - { repo, files (Set of repo-relative paths, diff
+ *   mode; when omitted, scans src/go/rpk/pkg/cli), log }
+ */
+function extract ({ repo, files = null }) {
+  const CLI_ROOT = path.join('src', 'go', 'rpk', 'pkg', 'cli')
+  let fileList
+  if (files) {
+    fileList = [...files].filter((f) => f.endsWith('.go') && !f.endsWith('_test.go'))
+  } else {
+    fileList = collectGoFiles(path.join(repo, CLI_ROOT)).map((f) => path.join(CLI_ROOT, f))
+  }
+
+  const cache = new SourceCache(repo)
+  const declarations = []
+  for (const file of fileList) {
+    const absPath = path.isAbsolute(file) ? file : path.join(repo, file)
+    if (!fs.existsSync(absPath)) continue
+    const content = fs.readFileSync(absPath, 'utf8')
+    if (!content.includes('cobra.Command') && !content.includes('Flags()')) continue
+    for (const decl of scanFile(content, file)) {
+      decl.declaration_text = cache.span(file, decl.line_start, decl.line_end)
+      declarations.push(decl)
+    }
+  }
+  return declarations
+}
+
+/**
+ * Mirror of parseDescriptionSections' section-header heuristic in
+ * tools/rpk-docs/generate-rpk-docs.js: an all-caps line (no double spaces,
+ * not directly under a "$ command" invocation) is promoted to a `===`
+ * heading in the generated docs.
+ */
+const ALLCAPS_LINE = /^([A-Z][A-Z\s\-/&]*[A-Z])$/
+
+function isHeadingLine (lines, i) {
+  const line = lines[i]
+  if (/ {2}/.test(line) || !ALLCAPS_LINE.test(line)) return false
+  for (let p = i - 1; p >= 0; p--) {
+    if (lines[p].trim() === '') break
+    if (/^\$\s/.test(lines[p])) return false
+  }
+  return true
+}
+
+/** Surface-specific convention rules. */
+const RULES = [
+  {
+    name: 'rpk-short-multiline',
+    description: 'Short must be a single line',
+    severity: 'warning',
+    check: (decl) => {
+      if (decl.meta.kind !== 'short') return []
+      const text = (decl.string || '').trim()
+      if (text.includes('\n')) {
+        return [{ message: 'Short spans multiple lines. The rpk convention is a one-line Short; move detail into Long.' }]
+      }
+      return []
+    }
+  },
+  {
+    name: 'rpk-terminal-period',
+    description: 'Short and flag usage strings do not end with a period',
+    severity: 'warning',
+    check: (decl) => {
+      if (decl.meta.kind !== 'short' && decl.meta.kind !== 'flag') return []
+      const text = (decl.string || '').trim()
+      if (!text) return []
+      if (/\.$/.test(text)) {
+        const what = decl.meta.kind === 'flag' ? 'Flag usage' : 'Short'
+        return [{ message: `${what} ends with a period: "...${text.slice(-50)}". The rpk convention is no terminal period; formatDescription adds one where the output needs it.` }]
+      }
+      return []
+    }
+  },
+  {
+    name: 'rpk-long-allcaps-heading',
+    description: 'ALLCAPS line in Long becomes a === heading in the docs',
+    severity: 'info',
+    check: (decl) => {
+      if (decl.meta.kind !== 'long' || !decl.string) return []
+      const issues = []
+      const lines = decl.string.split('\n')
+      for (let i = 0; i < lines.length; i++) {
+        if (isHeadingLine(lines, i)) {
+          issues.push({ message: `"${lines[i].trim()}" is an ALLCAPS line, which formatDescription promotes to a "===" heading on the docs page. Keep it if a section heading is intended; reword it if not.` })
+        }
+      }
+      return issues
+    }
+  },
+  {
+    name: 'rpk-long-block-delimiter',
+    description: 'A standalone ==== or ---- line in Long breaks AsciiDoc blocks',
+    severity: 'warning',
+    check: (decl) => {
+      if (decl.meta.kind !== 'long' || !decl.string) return []
+      const issues = []
+      const lines = decl.string.split('\n')
+      for (let i = 0; i < lines.length; i++) {
+        if (!/^\s*(={4,}|-{4,})\s*$/.test(lines[i])) continue
+        // An underline directly below an ALLCAPS heading is consumed by the
+        // section parser; anywhere else it ships as an unterminated AsciiDoc
+        // block delimiter and breaks the page.
+        if (i > 0 && isHeadingLine(lines, i - 1)) continue
+        issues.push({ message: `Line ${i + 1} of Long is a run of "${lines[i].trim()[0]}" characters. Outside a heading underline, this renders as an AsciiDoc block delimiter and breaks the generated page.` })
+      }
+      return issues
+    }
+  }
+]
+
+module.exports = {
+  name: 'rpk',
+  convention: CONVENTION,
+  extract,
+  scanFile,
+  rules: RULES,
+  // Shorts and flag usages are one-liners by convention; the generic
+  // too-short prose rule would flag nearly every conforming declaration.
+  skipRules: ['too-short']
+}

package/tools/overrides-audit/README.adoc

@@ -0,0 +1,152 @@
+= Overrides Audit
+:toc:
+
+Field-level classification of docs-side override entries against extracted source strings. This tool powers the override-retirement loop: overrides that patch bad source prose are stopgaps that die once the fixed string ships upstream, while structured docs enrichment stays in the override layer by design.
+
+== Why
+
+`docs-data/property-overrides.json` in the docs repo carries hundreds of hand-patched property entries. Each `description` override masks the string engineers wrote in C++ source. When that source string gets fixed (upstreamed), the override becomes dead weight that silently masks all future source improvements. This audit classifies every override *field* so that:
+
+* redundant fields can be deleted automatically on each release regeneration (retirement workflow),
+* upstreamable prose can be drafted into source PRs against the engineering repo (upstreaming workflow),
+* by-design docs enrichment is left alone.
+
+== Usage
+
+[,bash]
+----
+# Through doc-tools
+npx doc-tools overrides audit \
+  --overrides docs-data/property-overrides.json \
+  --extracted tools/property-extractor/gen/properties-output.json \
+  --format human
+
+# Standalone
+node tools/overrides-audit/index.js --overrides <path> --extracted <path> [--surface properties|rpk|connect] [--format json|human] [--output <file>]
+----
+
+IMPORTANT: `--extracted` must be the property extractor's *raw* output (the `--output` file), never the enhanced output (`--enhanced-output`) or the versioned `redpanda-properties-<tag>.json` attachments: those already have overrides applied, so every override would classify as REDUNDANT. The same trap exists for rpk: the versioned `rpk-v<version>.json` files in docs repos are post-override snapshots.
+
+To produce a raw extraction from a local Redpanda checkout:
+
+[,bash]
+----
+cd tools/property-extractor
+make venv treesitter            # one-time setup
+tmp/redpanda-property-extractor-venv/bin/python -W ignore property_extractor.py \
+  --recursive --path <path-to-redpanda-checkout> \
+  --output gen/source-properties.json
+----
+
+== Classification rules (properties surface)
+
+Each field of each override entry gets one manifest row.
+
+[cols="1,1,3"]
+|===
+|Field |Class |Rule
+
+|`description`
+|`REDUNDANT`
+|Override equals the extracted source description after normalization (whitespace collapsed, lines unwrapped). Safe to delete in the next auto-docs PR.
+
+|`description`
+|`UPSTREAMABLE`
+|Differs from source and is free of docs-only markup. `upstream_candidate_text` carries the override text verbatim.
+
+|`description`
+|`KEEP_UNTIL_UPSTREAMED`
+|Differs from source and contains docs-only markup (the SPLIT case). `upstream_candidate_text` carries the markup-stripped prose; the override keeps masking source until that prose ships upstream.
+
+|`description`
+|`KEEP`
+|Contains markup, but the stripped prose already matches source: markup-only enrichment, nothing to upstream.
+
+|`example`
+|`UPSTREAMABLE_SLOT`
+|Property declarations have a `.example` metadata slot in C++ source; migrate the value there.
+
+|`example_file` / `example_yaml`
+|`KEEP`
+|File- and YAML-rendered examples are docs-only rendering with no source slot.
+
+|`accepted_values`
+|`REDUNDANT_OR_UPSTREAMABLE`
+|The extracted property is an enum. The override may merely restate the enum (redundant) or deliberately hide internal-only values (for example `leader_balancer_mode` hides `greedy`). Always needs a human ruling.
+
+|`accepted_values`
+|`KEEP`
+|The extracted property is not an enum; the values are docs-side enrichment.
+
+|`default` / `type`
+|`REDUNDANT`
+|Equal to the extracted value.
+
+|`default` / `type`
+|`REVIEW`
+|Differs from the extracted value: possible source bug. Never auto-delete.
+
+|`related_topics`, `category`, `config_scope`, `version`, `exclude_from_docs`, `descriptionScope`, page attributes
+|`KEEP`
+|Docs-layer structure by design; source strings cannot carry it.
+
+|`acceptable_values` (typo)
+|`REVIEW`
+|Known typo'd key: the extractor only honors `accepted_values`, so this key is silently ignored. Flagged wherever found.
+|===
+
+Any field on a property that is missing from the extracted JSON classifies `REVIEW` (the override may create a doc-only property, the property may have been removed upstream, or the extractor may not see the declaration -- `advertised_kafka_api` and `advertised_rpc_api` are known extractor blind spots).
+
+=== Docs-only markup
+
+A description counts as markup-laden when it contains any of: `xref:`, `glossterm:`, `include::`, `ifdef::`/`ifndef::`/`endif::`, `pass:`, `config_ref:`/`prop:` property-link macros, `<<anchor>>` cross-references (anchors only exist on the generated page), or unescaped `{attr}` attribute references beyond the product-attr allowlist (`PRODUCT_ATTR_ALLOWLIST` in `classify.js`, empty today; escaped `\{attr}` literals are fine).
+
+The stripper converts markup to plain prose: `xref:page[text]` keeps the text, `glossterm:Term[def]` keeps the term, `<<anchor,text>>` keeps the text, `config_ref:`/`prop:` macros become the backticked property name (or their display text), `pass:q[content]` keeps the content, and `include::`/conditional directive lines are dropped.
+
+== The `upstream_ref` policy field
+
+Description overrides may carry an *optional* `upstream_ref` field: a URL or identifier of the engineering-repo PR or issue that upstreams the fix, so every description override is born with a retirement path.
+
+[,json]
+----
+"my_property": {
+  "description": "The corrected description.",
+  "upstream_ref": "https://github.com/redpanda-data/redpanda/pull/12345"
+}
+----
+
+The audit carries `upstream_ref` from the override entry into the manifest row for the description field. The extractor ignores unknown keys on existing properties, so adding `upstream_ref` is safe today; the docs-repo `validate-docs-data` workflow is the intended home for a warn-only check that new or changed description overrides carry it. This tool never modifies the overrides file.
+
+== Output
+
+JSON (default): `{ surface, overrides_file, extracted_file, manifest, summary, cross_check }`.
+
+* `manifest`: one row per classified field: `{ name, field, class, upstream_candidate_text?, upstream_ref?, content_hash?, source_file?, source_line?, note }`.
+** `content_hash` is a stable 16-hex-char digest of `(name, normalized override description)`. The docs-repo upstreaming workflow keeps a manifest of already-PRed hashes, so re-runs skip candidates whose hash is unchanged and re-draft when the override text moved.
+** `source_line` is populated only when the extracted JSON carries `line_start` (pending the extractor line-span patch); today it is absent.
+* `summary`: total, counts per class, and counts per (field, class).
+* `cross_check` (properties only): reuses `tools/property-extractor/compare-properties.js` -- the override descriptions are overlaid on the extracted properties and diffed. Every raw-equal description must have classified `REDUNDANT`; `violations` lists classifier bugs (expected empty).
+
+`--format human` prints the summary plus per-class listings for the actionable classes.
+
+== Surfaces
+
+* `properties` (complete): `property-overrides.json` vs the property extractor's raw JSON.
+* `rpk` (structural): enumerates `rpk-overrides.json` (see `docs-data/rpk-overrides.schema.json`) -- command/flag `description` fields are the upstreamable prose (classified `REVIEW` with a TODO note until source comparison lands; it needs a raw pre-override `--print-tree` snapshot plus the `formatDescription()` pipeline from `tools/rpk-docs`), everything else (`seeAlso`, `pageAliases`, `content`, platform scoping, ...) is `KEEP` by design.
+* `connect` (structural): enumerates the rp-connect-docs `overrides.json` (`definitions` + per-component arrays with nested `config.children`); prose fields classify `REVIEW` with a TODO note until `$ref` resolution and plugin-JSON comparison land.
+
+== Golden run (local checkouts required)
+
+The class distribution against live data depends on local checkouts of the docs repo and Redpanda source, so it is a documented command rather than a CI jest test:
+
+[,bash]
+----
+# 1. Extract raw source properties (see setup above)
+# 2. Audit the live overrides file
+node tools/overrides-audit/index.js \
+  --overrides <docs-repo>/docs-data/property-overrides.json \
+  --extracted tools/property-extractor/gen/source-properties.json \
+  --format human
+----
+
+Sanity expectations for the live 445-entry file: ~343 description fields classified; tens of REDUNDANT (source already matches); a large UPSTREAMABLE plus KEEP_UNTIL_UPSTREAMED (SPLIT) population; exactly one `acceptable_values` typo flag; `cross_check.violations` empty.

package/tools/overrides-audit/adapters/connect.js

@@ -0,0 +1,144 @@
+/**
+ * Overrides Audit - Redpanda Connect Adapter (structural)
+ *
+ * Reads the rp-connect-docs docs-data/overrides.json:
+ * - top-level `definitions`: shared field descriptions referenced with
+ *   `{"$ref": "#/definitions/<name>"}`
+ * - component sections (`inputs`, `outputs`, `processors`, `caches`,
+ *   `tracers`, `scanners`, `rate-limits`): ARRAYS of connector entries
+ *   `{ name, summary?, description?, config?: { children: [...] },
+ *      examples?, version?, ... }` where config children nest recursively
+ *   (`{ name, description?, children? , $ref? }`).
+ *
+ * The prose fields (summary, description, config field descriptions) map to
+ * ConfigSpec strings in connect Go source (`.Summary()`, `.Description()`,
+ * `NewXField(n).Description(...)`).
+ *
+ * TODO(quality logic): source comparison for connect is deferred. The
+ * extracted side is the connect plugin JSON (rpk connect list --format json
+ * or the cached connect-v<ver>.json in rp-connect-docs docs-data), and the
+ * comparison must resolve $refs before classifying, then apply the same
+ * markup rules as the properties surface (connect descriptions are AsciiDoc
+ * inside Go strings, so xref:/glossterm: usage is docs-only there too).
+ * Until then, prose fields classify REVIEW with an explicit note and
+ * everything else classifies KEEP, so the manifest shape is stable.
+ */
+
+'use strict'
+
+const classify = require('../classify')
+const { loadJson } = require('./properties')
+
+const COMPONENT_SECTIONS = [
+  'inputs',
+  'outputs',
+  'processors',
+  'caches',
+  'tracers',
+  'scanners',
+  'rate-limits',
+  'buffers',
+  'metrics'
+]
+
+const TODO_NOTE = 'TODO: connect source comparison not implemented yet (needs the connect plugin JSON and $ref resolution).'
+
+/**
+ * Build a REVIEW row for a connect prose field.
+ *
+ * @param {string} name - Unit name (for example "inputs/amqp_0_9/urls").
+ * @param {string} field - Manifest field label.
+ * @param {string} text - Override text.
+ * @returns {Object} Manifest row.
+ */
+function proseRow (name, field, text) {
+  return {
+    name,
+    field,
+    class: classify.CLASSES.REVIEW,
+    content_hash: classify.contentHash(name, text),
+    note: TODO_NOTE
+  }
+}
+
+/**
+ * Walk a config children tree, emitting one row per field description.
+ *
+ * @param {string} prefix - Name prefix ("inputs/amqp_0_9").
+ * @param {Object[]} children - Config children array.
+ * @param {Object[]} manifest - Output rows (mutated).
+ */
+function walkConfigChildren (prefix, children, manifest) {
+  if (!Array.isArray(children)) return
+  for (const child of children) {
+    if (!child || typeof child !== 'object') continue
+    const childName = `${prefix}/${child.name || '(unnamed)'}`
+    if (typeof child.description === 'string') {
+      manifest.push(proseRow(childName, 'config.description', child.description))
+    }
+    if (child.$ref) {
+      manifest.push({
+        name: childName,
+        field: 'config.$ref',
+        class: classify.CLASSES.KEEP,
+        note: `Shared definition reference (${child.$ref}); audit the referenced definition instead.`
+      })
+    }
+    if (child.children) walkConfigChildren(childName, child.children, manifest)
+  }
+}
+
+/**
+ * Run the audit for the connect surface (structural pass).
+ *
+ * @param {Object} args - { overridesPath, extractedPath }.
+ * @returns {Object} { surface, manifest, summary }.
+ */
+function audit ({ overridesPath, extractedPath }) {
+  const overridesDoc = loadJson(overridesPath, 'connect overrides')
+
+  const manifest = []
+
+  const definitions = overridesDoc.definitions
+  if (definitions && typeof definitions === 'object') {
+    for (const [defName, def] of Object.entries(definitions)) {
+      if (def && typeof def.description === 'string') {
+        manifest.push(proseRow(`definitions/${defName}`, 'description', def.description))
+      }
+    }
+  }
+
+  for (const section of COMPONENT_SECTIONS) {
+    const entries = overridesDoc[section]
+    if (!Array.isArray(entries)) continue
+    for (const entry of entries) {
+      if (!entry || typeof entry !== 'object') continue
+      const entryName = `${section}/${entry.name || '(unnamed)'}`
+      for (const [field, value] of Object.entries(entry)) {
+        if (field === 'name') continue
+        if (field === 'summary' || field === 'description') {
+          manifest.push(proseRow(entryName, field, value))
+        } else if (field === 'config' && value && typeof value === 'object') {
+          walkConfigChildren(entryName, value.children, manifest)
+        } else {
+          manifest.push({
+            name: entryName,
+            field,
+            class: classify.CLASSES.KEEP,
+            note: 'Docs-side connector metadata; stays in the override file by design.'
+          })
+        }
+      }
+    }
+  }
+
+  return {
+    surface: 'connect',
+    overrides_file: overridesPath,
+    extracted_file: extractedPath || null,
+    manifest,
+    summary: classify.summarize(manifest)
+  }
+}
+
+module.exports = { audit, COMPONENT_SECTIONS }