mandrel 2.53.0 → 2.55.0

package/.agents/scripts/lib/audit-to-stories/parse-audit-md.js

@@ -19,11 +19,19 @@ import path from 'node:path';
 
 import { normalizeSeverity } from '../findings/severity.js';
 
-const KEY_LINE = /^\s*-\s*\*\*([^:*]+):\*\*\s*(.*)$/;
+// A lens writes its field bullets either way round — `- **Severity:** High`
+// (colon inside the bold run) or `- **Severity**: High` (colon outside it).
+// Accepting only the first silently dropped the axis of every finding written
+// the second way: it parsed as prose, so the block carried no severity, and the
+// grouping-header rule below then read the whole finding as an organisational
+// heading. Both spellings are the same field.
+const KEY_LINE = /^\s*-\s*\*\*([^:*]+?)\s*(?::\*\*|\*\*\s*:)\s*(.*)$/;
 const HEADING_FINDING = /^(#{3,4})\s+(.+?)\s*$/;
-const SEVERITY_KEY_LINE = /^\s*-\s*\*\*(?:severity|impact)\s*:\*\*/i;
+const SEVERITY_KEY_LINE =
+  /^\s*-\s*\*\*(?:severity|impact)\s*(?::\*\*|\*\*\s*:)/i;
 const TALLY_LINE =
   /severity\s+tally\s*:?\**\s*critical\s+(\d+)\s*\/\s*high\s+(\d+)\s*\/\s*medium\s+(\d+)\s*\/\s*low\s+(\d+)/i;
+const TALLY_LINE_GLOBAL = new RegExp(TALLY_LINE.source, 'gi');
 const HEADING_SECTION = /^##\s+(.+?)\s*$/;
 const PATH_HINT =
   /(?<![\w/])([A-Za-z0-9_./\\@-]+\.(?:js|ts|tsx|jsx|mjs|cjs|md|json|yaml|yml|css|scss|html|py|go|rs|java|kt|rb|sh|ps1|tf|env))(?![\w])/g;
@@ -269,72 +277,186 @@ function carriesSeverity(block) {
 }
 
 /**
- * Resolve `###` headings that are **grouping headers** rather than findings.
+ * Does this block carry any `- **Key:** value` field bullet at all? A finding
+ * block is a field record; a grouping header is a heading with prose (or
+ * nothing) under it. Read together with {@link carriesSeverity} this is what
+ * lets an axis-less heading be recognised as organisational **without** having
+ * to see `####` children under it.
  *
- * Several lenses nest their findings one level deeper — a `###` per dimension
+ * @param {{ bodyLines: string[] }} block
+ * @returns {boolean}
+ */
+function carriesFieldBullet(block) {
+  return block.bodyLines.some((line) => KEY_LINE.test(line));
+}
+
+/**
+ * A block that declares nothing: no severity axis and no field bullets. A
+ * `### Robust` header whose whole body is `_No findings._` is the canonical
+ * case — it used to parse as a severity-less finding with no files and no
+ * recommendation, which an unattended sweep then filed as an empty Story.
+ *
+ * @param {{ bodyLines: string[] }} block
+ * @returns {boolean}
+ */
+function isEmptyBlock(block) {
+  return !carriesSeverity(block) && !carriesFieldBullet(block);
+}
+
+/**
+ * Decide whether a `###` block is a **grouping header** rather than a finding,
+ * reading its **whole subtree** — its own body and its `####` children.
+ *
+ * Several lenses nest their findings one level deeper: a `###` per dimension
  * (`### Perceivable`), each holding `####` finding blocks. Read flat, that
- * report parsed as one severity-less finding per dimension with no files and
- * no recommendation, and `--auto` filed those empties (Story #5144). The rule
- * this applies: a `###` heading that carries no `Severity:`/`Impact:` line and
- * is followed by `####` headings is a grouping header — its `####` children
- * are emitted as findings and the header itself never is.
+ * report parsed as one severity-less finding per dimension, and the unattended
+ * sweep filed those empties. The subtree rule this applies, in order:
+ *
+ *   1. A block whose own body carries the severity axis **or any field
+ *      bullet** is a finding. Its `####` sub-sections fold back into its body
+ *      rather than splitting into phantom findings, so flat reports parse
+ *      exactly as before.
+ *   2. A block whose title leads with a backticked path — the anchor the
+ *      finding-block skeleton mandates — is a finding even when its own body
+ *      is empty, because its fields live under a `#### Evidence`-style
+ *      sub-section. Folding is what stops that sub-section's heading from
+ *      becoming the finding's title.
+ *   3. Otherwise it is a grouping header when its children look like findings:
+ *      any child carrying its own path anchor, or more than one child carrying
+ *      the severity axis. A **single** axis-bearing child under an anchorless
+ *      header is the `#### Evidence` shape again, so it folds.
+ *   4. A block with no children at all and nothing declared is a grouping
+ *      header (or a stray heading) either way — it yields no finding.
  *
- * A `###` heading that DOES carry the axis line keeps the previous behaviour:
- * its `####` sub-sections fold back into its own body rather than splitting
- * into phantom findings, so existing flat reports parse exactly as before.
+ * @param {{ title: string, bodyLines: string[] }} parent
+ * @param {Array<{ title: string, bodyLines: string[] }>} children
+ * @returns {boolean}
+ */
+function isGroupingHeader(parent, children) {
+  if (!isEmptyBlock(parent)) return false;
+  if (TITLE_ANCHOR.test(parent.title)) return false;
+  if (children.length === 0) return true;
+  if (children.some((child) => TITLE_ANCHOR.test(child.title))) return true;
+  return children.filter((child) => carriesSeverity(child)).length > 1;
+}
+
+/**
+ * Fold the flat heading stream into findings: emit a grouping header's
+ * children, fold a finding's sub-sections into its own body, and drop every
+ * block that declares nothing.
  *
  * @param {Array<{ level: number, title: string, bodyLines: string[] }>} blocks
  * @returns {Array<{ level: number, title: string, bodyLines: string[] }>}
  */
 function foldGroupingHeaders(blocks) {
   const out = [];
-  let parent = null;
-  for (const block of blocks) {
-    if (block.level <= 3) {
-      parent = block;
-      out.push(block);
+  const keep = (block) => {
+    if (!isEmptyBlock(block)) out.push(block);
+  };
+
+  let i = 0;
+  while (i < blocks.length) {
+    const block = blocks[i];
+    if (block.level > 3) {
+      // A `####` with no `###` parent above it — read it on its own terms.
+      keep(block);
+      i += 1;
       continue;
     }
-    if (!parent) {
-      out.push(block);
-      continue;
+    const children = [];
+    let j = i + 1;
+    while (j < blocks.length && blocks[j].level > 3) {
+      children.push(blocks[j]);
+      j += 1;
     }
-    if (carriesSeverity(parent)) {
-      parent.bodyLines.push(`#### ${block.title}`, ...block.bodyLines);
-      continue;
+    if (isGroupingHeader(block, children)) {
+      for (const child of children) keep(child);
+    } else {
+      for (const child of children) {
+        block.bodyLines.push(`#### ${child.title}`, ...child.bodyLines);
+      }
+      keep(block);
     }
-    parent.isGroupingHeader = true;
-    out.push(block);
+    i = j;
   }
-  return out.filter((block) => !block.isGroupingHeader);
+  return out;
 }
 
 /**
- * Read the machine-readable severity tally the report envelope mandates in its
- * `## Executive Summary`:
+ * Slice the `## Executive Summary` section out of a report, or `null` when the
+ * report declares none. The tally the cross-check trusts is the one the report
+ * envelope mandates *there*; a `Severity tally:` string anywhere else is prose
+ * quoting the format, not a declaration.
  *
- * ```text
- * Severity tally: Critical 0 / High 2 / Medium 1 / Low 0
- * ```
+ * @param {string} markdown
+ * @returns {{ start: number, end: number }|null} character offsets.
+ */
+function executiveSummaryRange(markdown) {
+  const heading = /^##\s+executive\s+summary\s*$/gim;
+  const opened = heading.exec(markdown);
+  if (!opened) return null;
+  const start = opened.index + opened[0].length;
+  const next = /^##\s+/gm;
+  next.lastIndex = start;
+  const closed = next.exec(markdown);
+  return { start, end: closed ? closed.index : markdown.length };
+}
+
+/**
+ * Read every `Severity tally:` line the report carries, scoped to its
+ * Executive Summary and failing closed on more than one.
  *
- * The line is what lets a consumer cross-check what the lens says it found
+ * The tally is what lets a consumer cross-check what the lens says it found
  * against what the parser actually extracted — a parse that silently drops
- * findings is otherwise indistinguishable from a clean report. `Info` is never
- * counted (the severity scale already excludes it from scheduled work).
+ * findings is otherwise indistinguishable from a clean report. That only holds
+ * while exactly one line claims to be the tally. A report whose prose quotes
+ * the format a second time (a remediation section restating a lens's output,
+ * say) used to have whichever line the regex reached first silently adopted as
+ * the declaration, so the cross-check compared the parse against an arbitrary
+ * one of two numbers. Both are now reported and the report fails.
+ *
+ * `Info` is never counted (the severity scale already excludes it from
+ * scheduled work).
  *
  * @param {string} markdown — full report text.
- * @returns {{ critical: number, high: number, medium: number, low: number }|null}
- *   `null` when the report declares no tally at all.
+ * @returns {{ tally: {critical:number,high:number,medium:number,low:number}|null,
+ *   matches: string[], duplicate: boolean }}
  */
-export function parseSeverityTally(markdown) {
-  if (typeof markdown !== 'string') return null;
-  const match = TALLY_LINE.exec(markdown);
-  if (!match) return null;
+export function readSeverityTally(markdown) {
+  if (typeof markdown !== 'string') {
+    return { tally: null, matches: [], duplicate: false };
+  }
+  const range = executiveSummaryRange(markdown);
+  const matches = [];
+  TALLY_LINE_GLOBAL.lastIndex = 0;
+  for (const hit of markdown.matchAll(TALLY_LINE_GLOBAL)) {
+    // Outside an Executive Summary the whole document is the scope; with one,
+    // a line beyond it is a stray that must not be adopted silently.
+    matches.push({ text: hit[0].trim(), groups: hit, index: hit.index });
+  }
+  const scoped = range
+    ? matches.filter((m) => m.index >= range.start && m.index < range.end)
+    : matches;
+  if (matches.length === 0) {
+    return { tally: null, matches: [], duplicate: false };
+  }
+  if (matches.length > 1) {
+    return {
+      tally: null,
+      matches: matches.map((m) => m.text),
+      duplicate: true,
+    };
+  }
+  const [only] = scoped.length > 0 ? scoped : matches;
   return {
-    critical: Number(match[1]),
-    high: Number(match[2]),
-    medium: Number(match[3]),
-    low: Number(match[4]),
+    tally: {
+      critical: Number(only.groups[1]),
+      high: Number(only.groups[2]),
+      medium: Number(only.groups[3]),
+      low: Number(only.groups[4]),
+    },
+    matches: [only.text],
+    duplicate: false,
   };
 }
 
@@ -446,7 +568,10 @@ export function parseAuditReports(reports, { repoRoot } = {}) {
 }
 
 export const __testing = {
+  carriesFieldBullet,
   carriesSeverity,
+  isEmptyBlock,
+  isGroupingHeader,
   foldGroupingHeaders,
   normaliseSeverity,
   extractFilePaths,

package/.agents/scripts/lib/baselines/merge-envelopes.js

@@ -179,6 +179,56 @@ function mergeStamps(base, ours, theirs) {
   return { merged, conflicts };
 }
 
+/**
+ * The row half of a 3-way merge, shared by the envelope path and the plain
+ * row-baseline path below. Both merge a set of rows keyed by identity against
+ * a common ancestor; they differ only in where the identity comes from and in
+ * what they assemble around the result.
+ *
+ * @param {{
+ *   baseEnv: object,
+ *   ours: object|null|undefined,
+ *   theirs: object|null|undefined,
+ *   rowIdentity: (row: object) => string,
+ * }} params
+ * @returns {{ rows: Array<object>, conflicts: Array<object> }}
+ */
+function mergeRowSets({ baseEnv, ours, theirs, rowIdentity }) {
+  const baseRows = indexRows(baseEnv?.rows, rowIdentity, 'base');
+  const ourRows = indexRows(ours?.rows, rowIdentity, 'ours');
+  const theirRows = indexRows(theirs?.rows, rowIdentity, 'theirs');
+
+  const conflicts = [];
+  const rows = [];
+  const identities = new Set([
+    ...ourRows.keys(),
+    ...theirRows.keys(),
+    ...baseRows.keys(),
+  ]);
+  for (const id of identities) {
+    const b = baseRows.get(id);
+    const o = ourRows.get(id);
+    const t = theirRows.get(id);
+    const pick = choose(b, o, t);
+    if (pick.conflict) {
+      conflicts.push({
+        scope: 'row',
+        identity: id,
+        base: b,
+        ours: o,
+        theirs: t,
+      });
+      // Keep the ours-side value so the row set stays well-formed; the
+      // driver renders the conflict markers around it from this record.
+      if (o !== undefined) rows.push(o);
+      else if (t !== undefined) rows.push(t);
+      continue;
+    }
+    if (pick.value !== undefined) rows.push(pick.value);
+  }
+  return { rows, conflicts };
+}
+
 /**
  * 3-way merge two baseline envelopes against their common ancestor.
  *
@@ -217,38 +267,12 @@ export function mergeEnvelopes({
   const mod = getKindModule(resolvedKind);
   const baseEnv = base && typeof base === 'object' ? base : { rows: [] };
 
-  const baseRows = indexRows(baseEnv.rows, mod.rowIdentity, 'base');
-  const ourRows = indexRows(ours?.rows, mod.rowIdentity, 'ours');
-  const theirRows = indexRows(theirs?.rows, mod.rowIdentity, 'theirs');
-
-  const conflicts = [];
-  const rows = [];
-  const identities = new Set([
-    ...ourRows.keys(),
-    ...theirRows.keys(),
-    ...baseRows.keys(),
-  ]);
-  for (const id of identities) {
-    const b = baseRows.get(id);
-    const o = ourRows.get(id);
-    const t = theirRows.get(id);
-    const pick = choose(b, o, t);
-    if (pick.conflict) {
-      conflicts.push({
-        scope: 'row',
-        identity: id,
-        base: b,
-        ours: o,
-        theirs: t,
-      });
-      // Keep the ours-side value so the row set stays well-formed; the
-      // driver renders the conflict markers around it from this record.
-      if (o !== undefined) rows.push(o);
-      else if (t !== undefined) rows.push(t);
-      continue;
-    }
-    if (pick.value !== undefined) rows.push(pick.value);
-  }
+  const { rows, conflicts } = mergeRowSets({
+    baseEnv,
+    ours,
+    theirs,
+    rowIdentity: mod.rowIdentity,
+  });
 
   const sortedRows = mod.sortRows(rows);
   const { merged: stamps, conflicts: stampConflicts } = mergeStamps(
@@ -270,3 +294,245 @@ export function mergeEnvelopes({
     conflicts: [...stampConflicts, ...conflicts],
   };
 }
+
+/**
+ * Emit the object's keys in a declared order, then anything left over.
+ *
+ * The invariant this serves is "a clean driver merge stays byte-identical to
+ * a regeneration": both generators below write
+ * `JSON.stringify(envelope, null, 2)`, which preserves insertion order, so a
+ * merged file whose keys are in a different order than the generator's is a
+ * spurious diff on every subsequent refresh. Unknown keys are appended rather
+ * than dropped — a merge is never the place to lose a field.
+ *
+ * @param {object} obj
+ * @param {string[]} order
+ * @returns {object}
+ */
+function orderKeys(obj, order) {
+  const out = {};
+  for (const key of order) {
+    if (obj[key] !== undefined) out[key] = obj[key];
+  }
+  for (const key of Object.keys(obj)) {
+    if (out[key] === undefined && obj[key] !== undefined) out[key] = obj[key];
+  }
+  return out;
+}
+
+/**
+ * Derive `cyclomatic`'s rollup from its merged rows — the same arithmetic
+ * `cyclomatic-ceiling.js#buildCyclomaticEnvelope` applies at generation time.
+ *
+ * Deriving rather than merging matters more here than anywhere: the rollup is
+ * three counts over the whole row set, so two branches that each add one row
+ * BOTH write `filesAboveCeiling: n + 1`. A 3-way merge sees two identical
+ * values and resolves them clean — to a number that is wrong by one, with no
+ * conflict and no evidence. That is the silent splice this module exists to
+ * refuse, compressed into a single field.
+ *
+ * @param {Array<object>} rows
+ * @returns {{ '*': { filesAboveCeiling: number, methodsAboveCeiling: number, maxCyclomatic: number } }}
+ */
+function cyclomaticRollup(rows) {
+  let methods = 0;
+  let max = 0;
+  for (const row of rows) {
+    methods += Number(row?.methodsAboveCeiling ?? 0);
+    const rowMax = Number(row?.maxCyclomatic ?? 0);
+    if (rowMax > max) max = rowMax;
+  }
+  return {
+    '*': {
+      filesAboveCeiling: rows.length,
+      methodsAboveCeiling: methods,
+      maxCyclomatic: max,
+    },
+  };
+}
+
+/**
+ * Baselines that are row sets but NOT per-kind envelopes (Story #5277).
+ *
+ * `baselines/*.json` matches more than the eight kernel kinds. Three of the
+ * extras are row sets with a real identity, and before this registry the
+ * driver handed all three straight back to `git merge-file` — so `.gitattributes`
+ * declared them driver-merged while they were still text-merged, with exactly
+ * the two failure modes the driver replaces. `dead-exports*.json` in particular
+ * is a long, uniform list of two-key objects: the shape git splices most
+ * happily and least visibly.
+ *
+ * `rollup: null` is a genuine absence, not an omission — the dead-export
+ * generator writes no rollup key at all, so deriving one would invent a field
+ * the checker never reads and the generator would strip on the next refresh.
+ * `cyclomatic` does carry one, and derives it (see {@link cyclomaticRollup}).
+ *
+ * `keyOrder` mirrors each generator's own insertion order; see
+ * {@link orderKeys}.
+ */
+const PLAIN_BASELINE_KINDS = Object.freeze({
+  cyclomatic: Object.freeze({
+    rowIdentity: (row) => String(row?.file ?? ''),
+    sortRows: (rows) =>
+      [...rows].sort((a, b) => String(a.file).localeCompare(String(b.file))),
+    rollup: cyclomaticRollup,
+    keyOrder: ['$schema', 'generatedAt', 'ceiling', 'rollup', 'rows'],
+  }),
+  'dead-exports': Object.freeze({
+    rowIdentity: (row) => `${row?.file ?? ''}::${row?.symbol ?? ''}`,
+    sortRows: (rows) =>
+      [...rows].sort(
+        (a, b) =>
+          String(a.file).localeCompare(String(b.file)) ||
+          String(a.symbol).localeCompare(String(b.symbol)),
+      ),
+    rollup: null,
+    keyOrder: ['$schema', 'kernelVersion', 'generatedAt', 'mode', 'rows'],
+  }),
+});
+
+/** `dead-exports.json` and `dead-exports-production.json` share a `$schema`. */
+const PLAIN_SCHEMA_KINDS = Object.freeze({
+  'cyclomatic.schema.json': 'cyclomatic',
+  'dead-exports.schema.json': 'dead-exports',
+});
+
+/**
+ * Identify a plain row baseline from its `$schema`, or `null`.
+ *
+ * The production dead-export pass reports as its own kind so the regenerate
+ * remedy can name the right command, even though the two share a schema and
+ * merge identically.
+ *
+ * @param {unknown} envelope
+ * @returns {string|null}
+ */
+export function plainKindFromEnvelope(envelope) {
+  const ref = envelope?.$schema;
+  if (typeof ref !== 'string') return null;
+  const kind = PLAIN_SCHEMA_KINDS[ref.split('/').pop()] ?? null;
+  if (kind === 'dead-exports' && envelope?.mode === 'production') {
+    return 'dead-exports-production';
+  }
+  return kind;
+}
+
+/** The merge spec for a plain kind, collapsing the two dead-export passes. */
+function plainSpec(kind) {
+  return PLAIN_BASELINE_KINDS[
+    kind === 'dead-exports-production' ? 'dead-exports' : kind
+  ];
+}
+
+/**
+ * 3-way merge a plain row baseline by row identity.
+ *
+ * Same contract as {@link mergeEnvelopes} — rows merge by identity, stamps
+ * merge 3-way, `generatedAt` resolves to the later of the two — minus the
+ * kernel protocol these kinds do not participate in.
+ *
+ * @param {{ base: object|null, ours: object, theirs: object, kind?: string }} params
+ * @returns {{ kind: string, envelope: object, conflicts: Array<object> }}
+ */
+export function mergePlainBaseline({ base, ours, theirs, kind } = {}) {
+  const resolvedKind =
+    kind ?? plainKindFromEnvelope(ours) ?? plainKindFromEnvelope(theirs);
+  const spec = resolvedKind ? plainSpec(resolvedKind) : undefined;
+  if (!spec) {
+    throw new Error(
+      'mergePlainBaseline: could not resolve a known plain baseline kind from the envelopes',
+    );
+  }
+  const baseEnv = base && typeof base === 'object' ? base : { rows: [] };
+  const { rows, conflicts } = mergeRowSets({
+    baseEnv,
+    ours,
+    theirs,
+    rowIdentity: spec.rowIdentity,
+  });
+  const sortedRows = spec.sortRows(rows);
+  const { merged: stamps, conflicts: stampConflicts } = mergeStamps(
+    baseEnv,
+    ours ?? {},
+    theirs ?? {},
+  );
+  const envelope = orderKeys(
+    {
+      ...stamps,
+      generatedAt: laterStamp(ours?.generatedAt, theirs?.generatedAt),
+      ...(spec.rollup ? { rollup: spec.rollup(sortedRows) } : {}),
+      rows: sortedRows,
+    },
+    spec.keyOrder,
+  );
+  return {
+    kind: resolvedKind,
+    envelope,
+    conflicts: [...stampConflicts, ...conflicts],
+  };
+}
+
+/** Per-kind regeneration commands; anything unlisted follows the convention. */
+const REGENERATE_COMMANDS = Object.freeze({
+  'dead-exports': 'npm run dead-exports:update',
+  'dead-exports-production': 'npm run dead-exports:update',
+  cyclomatic: 'npm run cyclomatic:update',
+});
+
+/**
+ * The command that re-derives a baseline from the tree.
+ *
+ * A conflicted merge leaves the rollup describing a row set nobody scored:
+ * the driver derives it from rows that still carry conflict markers around
+ * them, and no hand-resolution of those markers updates it. Resolving the
+ * rows is therefore only half the job, and the half operators skip — so the
+ * driver says the other half out loud rather than leaving a plausible-looking
+ * number behind.
+ *
+ * @param {string} kind
+ * @returns {string}
+ */
+export function baselineRegenerateRemedy(kind) {
+  return REGENERATE_COMMANDS[kind] ?? `npm run ${kind}:update`;
+}
+
+/**
+ * Wrap each conflicted envelope-level stamp in git conflict markers.
+ *
+ * Envelope conflicts used to be reported on stderr alone, and the file was
+ * left holding the ours-side value with no marker in it. Git reads the
+ * driver's non-zero exit as "conflicted" and leaves the path unmerged, so the
+ * operator opens a file that looks cleanly merged and has to reconstruct from
+ * a scrollback line which stamp disagreed. A `kernelVersion` double-bump is
+ * exactly the case where the two sides mean different scorers and picking one
+ * silently is wrong.
+ *
+ * Operates on the canonical text the merged projection already produced, so
+ * every unconflicted line stays byte-identical to a clean merge.
+ *
+ * @param {string} text Canonical serialization of the merged envelope.
+ * @param {Array<{ identity: string, ours?: unknown, theirs?: unknown }>} conflicts
+ * @returns {string}
+ */
+export function renderStampConflict(text, conflicts) {
+  let out = text;
+  for (const conflict of conflicts) {
+    const key = conflict?.identity;
+    if (typeof key !== 'string' || key.length === 0) continue;
+    const escaped = key.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
+    const pattern = new RegExp(`^([ \\t]*)"${escaped}"(: .*)$`, 'm');
+    const match = pattern.exec(out);
+    if (!match) continue;
+    const [line, indent, tail] = match;
+    const comma = tail.endsWith(',') ? ',' : '';
+    const render = (value) =>
+      value === undefined
+        ? ''
+        : `${indent}${JSON.stringify(key)}: ${JSON.stringify(value)}${comma}\n`;
+    out = out.replace(
+      line,
+      `<<<<<<< ours\n${render(conflict.ours)}=======\n${render(conflict.theirs)}>>>>>>> theirs`,
+    );
+  }
+  return out;
+}