@codyswann/lisa 2.240.0 → 2.242.0

package/typescript/copy-overwrite/scripts/threshold-ratchet-compare.mjs

@@ -0,0 +1,315 @@
+/**
+ * Threshold ratchet — comparison rules and reporting.
+ *
+ * Pure comparison layer: given a watched file's baseline and current
+ * contents, report every weakening. No filesystem or git access. See
+ * threshold-ratchet-families.mjs for extraction and threshold-ratchet.mjs
+ * for the CLI.
+ */
+import {
+  extractAllowEntries,
+  extractExemptionEntries,
+  extractK6Constraints,
+  extractNumericLeaves,
+  extractRubocopThresholds,
+  extractStrykerConstraints,
+  extractStrykerMutate,
+  familyFor,
+  parseJson,
+} from "./threshold-ratchet-families.mjs";
+
+/** Finding type: a numeric bound or boolean gate moved the weakening way. */
+const TYPE_WEAKENED = "weakened";
+/** Finding type: an exemption entry was added (Tier 3). */
+const TYPE_EXEMPTION_ADDED = "exemption-added";
+/** Family kind for .lisa.config.json (the thresholdRatchet.allow carrier). */
+const KIND_ALLOW_LIST = "allow-list";
+
+/**
+ * @typedef {object} Finding
+ * @property {string} file Repo-relative path of the gate file
+ * @property {string} key Dotted key path within the file
+ * @property {"weakened"|"removed"|"exemption-added"|"file-deleted"|"allow-added"|"unparseable"} type
+ *   Which ratchet rule the change violated
+ * @property {number|string} [base] Baseline value
+ * @property {number|string} [current] Current value
+ * @property {string} message Operator-readable explanation
+ */
+
+/**
+ * Build the "file could not be parsed" finding.
+ * @param {string} relPath Repo-relative path
+ * @returns {Finding} The unparseable-file finding
+ */
+function unparseable(relPath) {
+  return {
+    file: relPath,
+    key: "*",
+    type: "unparseable",
+    message: `${relPath} is no longer valid JSON — a broken gate file disables the gate.`,
+  };
+}
+
+/**
+ * Compare two constraint maps: report removals and direction violations.
+ * @param {string} relPath Repo-relative path the constraints came from
+ * @param {Map<string, { value: number, direction: "min"|"max" }>} base
+ *   Baseline constraints
+ * @param {Map<string, { value: number, direction: "min"|"max" }>} current
+ *   Current constraints
+ * @returns {Finding[]} One finding per removed or weakened constraint
+ */
+export function compareConstraints(relPath, base, current) {
+  const findings = [];
+  for (const [key, baseC] of base) {
+    const currentC = current.get(key);
+    if (!currentC) {
+      findings.push({
+        file: relPath,
+        key,
+        type: "removed",
+        base: baseC.value,
+        message: `${relPath}: ${key} was removed — the tuned floor would silently fall back to a default.`,
+      });
+      continue;
+    }
+    const weakened =
+      baseC.direction === "min"
+        ? currentC.value < baseC.value
+        : currentC.value > baseC.value;
+    if (weakened) {
+      const verb =
+        baseC.direction === "min" ? "may only increase" : "may only decrease";
+      findings.push({
+        file: relPath,
+        key,
+        type: TYPE_WEAKENED,
+        base: baseC.value,
+        current: currentC.value,
+        message: `${relPath}: ${key} changed ${baseC.value} → ${currentC.value} (this value ${verb}).`,
+      });
+    }
+  }
+  return findings;
+}
+
+/**
+ * Compare a stryker.conf.json pair: the break threshold plus mutate-list
+ * exemptions (new negations or removed targets shrink the gate).
+ * @param {string} relPath Repo-relative path
+ * @param {unknown} base Parsed baseline config
+ * @param {unknown} current Parsed current config
+ * @returns {Finding[]} Break-threshold and mutate-scope findings
+ */
+function compareStryker(relPath, base, current) {
+  const findings = compareConstraints(
+    relPath,
+    extractStrykerConstraints(base),
+    extractStrykerConstraints(current)
+  );
+  const baseMutate = extractStrykerMutate(base);
+  const currentMutate = extractStrykerMutate(current);
+  for (const negation of currentMutate.negations) {
+    if (!baseMutate.negations.has(negation)) {
+      findings.push({
+        file: relPath,
+        key: `mutate ${negation}`,
+        type: TYPE_EXEMPTION_ADDED,
+        message: `${relPath}: new mutation-testing exclusion "${negation}" — excluding files from a gate is a weakening.`,
+      });
+    }
+  }
+  for (const positive of baseMutate.positives) {
+    if (!currentMutate.positives.has(positive)) {
+      findings.push({
+        file: relPath,
+        key: `mutate ${positive}`,
+        type: TYPE_EXEMPTION_ADDED,
+        message: `${relPath}: mutation-testing target "${positive}" was removed — shrinking a gate's coverage is a weakening.`,
+      });
+    }
+  }
+  return findings;
+}
+
+/**
+ * Compare a k6 thresholds pair: numeric bounds plus abortOnFail downgrades.
+ * @param {string} relPath Repo-relative path
+ * @param {unknown} base Parsed baseline thresholds
+ * @param {unknown} current Parsed current thresholds
+ * @returns {Finding[]} Bound and abortOnFail findings
+ */
+function compareK6(relPath, base, current) {
+  const baseC = extractK6Constraints(base);
+  const currentC = extractK6Constraints(current);
+  const findings = compareConstraints(relPath, baseC.numeric, currentC.numeric);
+  for (const [key, wasOn] of baseC.booleans) {
+    // k6 defaults abortOnFail to false, so DELETING an explicit `true` is as
+    // much a weakening as flipping it — anything but a current `true` blocks.
+    if (wasOn && currentC.booleans.get(key) !== true) {
+      findings.push({
+        file: relPath,
+        key,
+        type: TYPE_WEAKENED,
+        base: "true",
+        current: "false",
+        message: `${relPath}: ${key} turned off — the gate no longer stops the run on failure.`,
+      });
+    }
+  }
+  return findings;
+}
+
+/**
+ * Compare an exemption-list pair (audit ignore files): report added entries.
+ * @param {string} relPath Repo-relative path
+ * @param {unknown} base Parsed baseline list
+ * @param {unknown} current Parsed current list
+ * @returns {Finding[]} One finding per newly added ignore entry
+ */
+function compareExemptions(relPath, base, current) {
+  const baseEntries = extractExemptionEntries(base);
+  const findings = [];
+  for (const entry of extractExemptionEntries(current)) {
+    if (!baseEntries.has(entry)) {
+      findings.push({
+        file: relPath,
+        key: entry,
+        type: TYPE_EXEMPTION_ADDED,
+        message: `${relPath}: new security-audit ignore entry "${entry}" — ignoring a finding weakens the gate and needs a human's sign-off.`,
+      });
+    }
+  }
+  return findings;
+}
+
+/**
+ * Compare .lisa.config.json allow lists: report added exception entries so a
+ * change can never grant itself an exception.
+ * @param {string} relPath Repo-relative path
+ * @param {unknown} base Parsed baseline config
+ * @param {unknown} current Parsed current config
+ * @returns {Finding[]} One finding per newly added allow entry
+ */
+function compareAllowList(relPath, base, current) {
+  const baseKeys = new Set(
+    extractAllowEntries(base).map(e => `${e.file} ${e.key}`)
+  );
+  const findings = [];
+  for (const entry of extractAllowEntries(current)) {
+    if (!baseKeys.has(`${entry.file} ${entry.key}`)) {
+      findings.push({
+        file: relPath,
+        key: `thresholdRatchet.allow ${entry.file}#${entry.key}`,
+        type: "allow-added",
+        message: `${relPath}: new threshold exception for ${entry.file} → ${entry.key}. Exceptions are a human decision: land this entry in its own human-approved change first, then make the threshold change.`,
+      });
+    }
+  }
+  return findings;
+}
+
+/**
+ * Compare one watched file's baseline and current contents and report every
+ * weakening. Pure: no filesystem or git access.
+ * @param {string} relPath Repo-relative path (forward slashes)
+ * @param {string | null} baselineText Baseline contents (null = file is new)
+ * @param {string | null} currentText Current contents (null = file deleted)
+ * @returns {Finding[]} Every ratchet violation in the change (empty = clean)
+ */
+export function compareFile(relPath, baselineText, currentText) {
+  const family = familyFor(relPath);
+  if (!family || baselineText === null || baselineText === undefined) return [];
+
+  if (currentText === null || currentText === undefined) {
+    if (family.kind === KIND_ALLOW_LIST) return [];
+    return [
+      {
+        file: relPath,
+        key: "*",
+        type: "file-deleted",
+        message: `${relPath} was deleted — deleting a quality gate is a weakening.`,
+      },
+    ];
+  }
+
+  if (family.kind === "rubocop-yaml") {
+    return compareConstraints(
+      relPath,
+      extractRubocopThresholds(baselineText, family.direction),
+      extractRubocopThresholds(currentText, family.direction)
+    );
+  }
+
+  const base = parseJson(baselineText);
+  const current = parseJson(currentText);
+  if (base === undefined) return [];
+  if (current === undefined) {
+    return family.kind === KIND_ALLOW_LIST ? [] : [unparseable(relPath)];
+  }
+  switch (family.kind) {
+    case "json-num":
+      return compareConstraints(
+        relPath,
+        extractNumericLeaves(base, family.direction),
+        extractNumericLeaves(current, family.direction)
+      );
+    case "stryker":
+      return compareStryker(relPath, base, current);
+    case "k6":
+      return compareK6(relPath, base, current);
+    case "exemption-list":
+      return compareExemptions(relPath, base, current);
+    case KIND_ALLOW_LIST:
+      return compareAllowList(relPath, base, current);
+    default:
+      return [];
+  }
+}
+
+/**
+ * Drop findings covered by baseline-side allow entries. `allow-added`
+ * findings are never dropped — an exception cannot approve its own creation.
+ * @param {Finding[]} findings All findings from the change
+ * @param {Array<{ file: string, key: string }>} allowEntries Baseline
+ *   (already-merged) allow list
+ * @returns {{ blocked: Finding[], allowed: Finding[] }} Findings that still
+ *   block vs. findings covered by a recorded exception
+ */
+export function applyAllowList(findings, allowEntries) {
+  const blocked = [];
+  const allowed = [];
+  for (const finding of findings) {
+    const isAllowed =
+      finding.type !== "allow-added" &&
+      allowEntries.some(
+        e =>
+          (finding.file === e.file || finding.file.endsWith(`/${e.file}`)) &&
+          (e.key === "*" || e.key === finding.key)
+      );
+    if (isAllowed) allowed.push(finding);
+    else blocked.push(finding);
+  }
+  return { blocked, allowed };
+}
+
+/**
+ * Render the operator-facing block message.
+ * @param {Finding[]} findings Blocked findings
+ * @returns {string} Multi-line report explaining what weakened and the
+ *   human-approved exception path
+ */
+export function formatReport(findings) {
+  return [
+    "⛔ Quality gate weakened — blocked by the threshold ratchet.",
+    "",
+    ...findings.map(f => `  • ${f.message}`),
+    "",
+    "Quality thresholds are a one-way ratchet: they may tighten but never",
+    "loosen. Fix the code so it meets the current gate instead of lowering the",
+    "gate. If a human decides an exception is genuinely correct, they record it",
+    "in .lisa.config.json under thresholdRatchet.allow (with a reason) in a",
+    "separate human-approved change; this check honors exceptions only after",
+    "they are merged.",
+  ].join("\n");
+}

package/typescript/copy-overwrite/scripts/threshold-ratchet-families.mjs

@@ -0,0 +1,295 @@
+/**
+ * Threshold ratchet — watched file families and value extractors.
+ *
+ * Pure extraction layer: given file contents, produce comparable constraint
+ * maps. No filesystem or git access. See threshold-ratchet.mjs for the CLI
+ * and threshold-ratchet-compare.mjs for the comparison rules.
+ */
+
+/**
+ * File families the ratchet watches. `kind` selects the extractor;
+ * `direction` applies to numeric-leaf kinds ("min" values may only rise,
+ * "max" values may only fall).
+ */
+export const FAMILIES = [
+  {
+    id: "coverage",
+    match: /(^|\/)(vitest|jest)\.thresholds\.json$/,
+    kind: "json-num",
+    direction: "min",
+  },
+  {
+    id: "simplecov",
+    match: /(^|\/)simplecov\.thresholds\.json$/,
+    kind: "json-num",
+    direction: "min",
+  },
+  {
+    id: "e2e",
+    match: /(^|\/)e2e\.thresholds\.json$/,
+    kind: "json-num",
+    direction: "min",
+  },
+  {
+    id: "eslint",
+    match: /(^|\/)eslint\.thresholds\.json$/,
+    kind: "json-num",
+    direction: "max",
+  },
+  {
+    id: "rubocop",
+    match: /(^|\/)rubocop\.thresholds\.yml$/,
+    kind: "rubocop-yaml",
+    direction: "max",
+  },
+  { id: "stryker", match: /(^|\/)stryker\.conf\.json$/, kind: "stryker" },
+  {
+    id: "k6",
+    match: /(^|\/)\.github\/k6\/thresholds\/[^/]+\.json$/,
+    kind: "k6",
+  },
+  {
+    id: "audit-ignore",
+    match: /(^|\/)audit\.ignore\.(config|local)\.json$/,
+    kind: "exemption-list",
+  },
+  {
+    id: "lisa-config",
+    match: /(^|\/)\.lisa\.config\.json$/,
+    kind: "allow-list",
+  },
+];
+
+/**
+ * Find the family a repo-relative path belongs to.
+ * @param {string} relPath Repo-relative path (forward slashes)
+ * @returns {(typeof FAMILIES)[number] | undefined} The matching family, or
+ *   undefined when the path is not a watched gate file
+ */
+export function familyFor(relPath) {
+  return FAMILIES.find(f => f.match.test(relPath));
+}
+
+/**
+ * Safe JSON parse.
+ * @param {string | null | undefined} text JSON text
+ * @returns {unknown | undefined} Parsed value, or undefined on failure
+ */
+export function parseJson(text) {
+  if (typeof text !== "string") return undefined;
+  try {
+    return JSON.parse(text);
+  } catch {
+    return undefined;
+  }
+}
+
+/**
+ * Walk a JSON object and collect numeric leaves as dotted-path constraints.
+ * Keys starting with "_" (e.g. `_comment`) are documentation, not thresholds.
+ * @param {unknown} node Parsed JSON value
+ * @param {"min"|"max"} direction Ratchet direction for every leaf
+ * @param {string} [prefix] Dotted path accumulated so far
+ * @returns {Map<string, { value: number, direction: "min"|"max" }>} Dotted
+ *   path → numeric constraint for every finite numeric leaf
+ */
+export function extractNumericLeaves(node, direction, prefix = "") {
+  const out = new Map();
+  if (node === null || typeof node !== "object" || Array.isArray(node)) {
+    return out;
+  }
+  for (const [key, value] of Object.entries(node)) {
+    if (key.startsWith("_")) continue;
+    const p = prefix ? `${prefix}.${key}` : key;
+    if (typeof value === "number" && Number.isFinite(value)) {
+      out.set(p, { value, direction });
+    } else if (value && typeof value === "object" && !Array.isArray(value)) {
+      for (const [cp, c] of extractNumericLeaves(value, direction, p)) {
+        out.set(cp, c);
+      }
+    }
+  }
+  return out;
+}
+
+/**
+ * Minimal parser for rubocop.thresholds.yml — a two-level document of
+ * `Section:` headers with indented `Key: <number>` scalars (comments and
+ * blank lines ignored). Deliberately NOT a general YAML parser: the file is
+ * Lisa-authored with this exact shape, and hand-parsing keeps the gate
+ * dependency-free with no backtracking-prone regexes.
+ * @param {string} text File contents
+ * @param {"min"|"max"} direction Ratchet direction for every scalar
+ * @returns {Map<string, { value: number, direction: "min"|"max" }>} Dotted
+ *   `Section.Key` path → numeric constraint
+ */
+export function extractRubocopThresholds(text, direction) {
+  const out = new Map();
+  const state = { section: "" };
+  for (const rawLine of text.split("\n")) {
+    const hash = rawLine.indexOf("#");
+    const line = (hash >= 0 ? rawLine.slice(0, hash) : rawLine).trimEnd();
+    if (!line.trim()) continue;
+    const indented = line.startsWith(" ") || line.startsWith("\t");
+    if (!indented && line.endsWith(":")) {
+      state.section = line.slice(0, -1).trim();
+      continue;
+    }
+    const colon = line.indexOf(":");
+    if (!indented || colon < 0 || !state.section) continue;
+    const key = line.slice(0, colon).trim();
+    const rawValue = line.slice(colon + 1).trim();
+    // Number("") is 0, so an empty value (e.g. a nested `Exclude:` list
+    // header) must be skipped, not recorded as a zero threshold.
+    const value = Number(rawValue);
+    if (key && rawValue !== "" && Number.isFinite(value)) {
+      out.set(`${state.section}.${key}`, { value, direction });
+    }
+  }
+  return out;
+}
+
+/**
+ * Extract the gating constraint from stryker.conf.json: only
+ * `thresholds.break` fails a run (`high`/`low` are reporting bands).
+ * @param {unknown} conf Parsed stryker.conf.json
+ * @returns {Map<string, { value: number, direction: "min"|"max" }>} The
+ *   `thresholds.break` constraint when present, otherwise an empty map
+ */
+export function extractStrykerConstraints(conf) {
+  const out = new Map();
+  const breakValue = conf?.thresholds?.break;
+  if (typeof breakValue === "number" && Number.isFinite(breakValue)) {
+    out.set("thresholds.break", { value: breakValue, direction: "min" });
+  }
+  return out;
+}
+
+/**
+ * Split a stryker `mutate` array into positive globs and negations.
+ * @param {unknown} conf Parsed stryker.conf.json
+ * @returns {{ positives: Set<string>, negations: Set<string> }} Globs that
+ *   include files vs. `!`-prefixed globs that exclude them
+ */
+export function extractStrykerMutate(conf) {
+  const positives = new Set();
+  const negations = new Set();
+  const mutate = Array.isArray(conf?.mutate) ? conf.mutate : [];
+  for (const glob of mutate) {
+    if (typeof glob !== "string") continue;
+    if (glob.startsWith("!")) negations.add(glob);
+    else positives.add(glob);
+  }
+  return { positives, negations };
+}
+
+/**
+ * Parse one k6 threshold expression (`p(95)<1000`, `rate>=0.99`, …) into a
+ * ratchet constraint. Upper bounds (`<`, `<=`) may only decrease; lower
+ * bounds (`>`, `>=`) may only increase. Hand-parsed — no regex.
+ * @param {string} expr Threshold expression
+ * @returns {{ value: number, direction: "min"|"max" } | undefined} The bound
+ *   as a constraint, or undefined when the expression has no numeric bound
+ */
+export function parseK6Expression(expr) {
+  for (const op of ["<=", ">=", "<", ">"]) {
+    const idx = expr.indexOf(op);
+    if (idx < 0) continue;
+    const value = Number(expr.slice(idx + op.length).trim());
+    if (!Number.isFinite(value)) return undefined;
+    return { value, direction: op.startsWith("<") ? "max" : "min" };
+  }
+  return undefined;
+}
+
+/**
+ * Extract constraints from a k6 thresholds file. Each metric contributes its
+ * expression bound(s) plus (when present) `abortOnFail` boolean constraints —
+ * turning abortOnFail off makes the gate advisory, which is a weakening.
+ * Handles every documented k6 shape: a bare expression string, a single long
+ * form `{ threshold, abortOnFail }` object, and arrays mixing both.
+ * @param {unknown} conf Parsed k6 thresholds JSON
+ * @returns {{
+ *   numeric: Map<string, { value: number, direction: "min"|"max" }>,
+ *   booleans: Map<string, boolean>,
+ * }} Numeric bounds keyed by `<metric>.threshold[i]` and abortOnFail flags
+ *   keyed by `<metric>.abortOnFail` (`[i]`-suffixed for array items)
+ */
+export function extractK6Constraints(conf) {
+  const numeric = new Map();
+  const booleans = new Map();
+  const thresholds = conf?.thresholds;
+  if (thresholds && typeof thresholds === "object") {
+    for (const [metric, spec] of Object.entries(thresholds)) {
+      const isArray = Array.isArray(spec);
+      const items = isArray ? spec : [spec];
+      items.forEach((item, i) => {
+        const expr =
+          typeof item === "string"
+            ? item
+            : item && typeof item === "object" && !Array.isArray(item)
+              ? item.threshold
+              : undefined;
+        if (typeof expr === "string") {
+          const c = parseK6Expression(expr);
+          if (c) numeric.set(`${metric}.threshold[${i}]`, c);
+        }
+        if (
+          item &&
+          typeof item === "object" &&
+          !Array.isArray(item) &&
+          typeof item.abortOnFail === "boolean"
+        ) {
+          booleans.set(
+            isArray ? `${metric}.abortOnFail[${i}]` : `${metric}.abortOnFail`,
+            item.abortOnFail
+          );
+        }
+      });
+    }
+  }
+  return { numeric, booleans };
+}
+
+/**
+ * Flatten an exemption file (audit ignore list) into a set of entry tokens.
+ * Arrays contribute their string items; objects contribute their keys.
+ * @param {unknown} conf Parsed JSON
+ * @returns {Set<string>} One token per exemption entry
+ */
+export function extractExemptionEntries(conf) {
+  const out = new Set();
+  if (Array.isArray(conf)) {
+    for (const item of conf) {
+      if (typeof item === "string") out.add(item);
+      else if (item && typeof item === "object") out.add(JSON.stringify(item));
+    }
+  } else if (conf && typeof conf === "object") {
+    for (const [key, value] of Object.entries(conf)) {
+      if (Array.isArray(value)) {
+        for (const item of value) {
+          out.add(
+            `${key}:${typeof item === "string" ? item : JSON.stringify(item)}`
+          );
+        }
+      } else {
+        out.add(key);
+      }
+    }
+  }
+  return out;
+}
+
+/**
+ * Extract thresholdRatchet.allow entries from parsed .lisa.config.json.
+ * @param {unknown} config Parsed config (may be undefined)
+ * @returns {Array<{ file: string, key: string, reason?: string }>} The
+ *   well-formed allow entries; malformed entries are dropped
+ */
+export function extractAllowEntries(config) {
+  const raw = config?.thresholdRatchet?.allow;
+  if (!Array.isArray(raw)) return [];
+  return raw.filter(
+    e => e && typeof e.file === "string" && typeof e.key === "string"
+  );
+}