@1aboveio/skills 0.20.2 → 0.20.4

package/skills/compliance/compliance-audit/scripts/reconcile_tracker.py

@@ -0,0 +1,454 @@
+#!/usr/bin/env python3
+"""Reconcile validated findings into the tracker (Step 9E).
+
+Reads ``tracker.md`` and applies findings that are deterministically
+verifiable, so hallucinated coverage cannot survive into the deliverable:
+
+  - From ``validate_tracker.py --json``: citation_grounding "missing clause"
+    warnings.  The cited ``section X`` is not in the evidence file, so the citation
+    is hallucinated.  Bullet flips to ``Not addressed`` with the reason.
+  - From the citation-judge JSON sidecars (Variant C, Step 9C): verdicts of
+    ``Off-target`` and ``Missing`` flip the bullet to ``Not addressed`` with
+    the judge's reason quoted.
+
+Findings that are *not* auto-applied (judge ``Weak``, adversarial-challenger
+recommendations) are surfaced in the reconciliation log for the human
+reviewer to decide on.
+
+After flipping bullets, the Control Effectiveness Rating is recomputed using
+the same rubric as ``validate_tracker.py``.  A line tagging the row as
+"reconciled by Step 9E" is prepended to the Issue Description cell so the
+reviewer can see what changed at a glance; the existing prose is preserved.
+
+Usage:
+    python3 reconcile_tracker.py <tracker.md> --checklist <path> \\
+        [--validation <validate_tracker --json output>] \\
+        [--judge <judge-<group>.json>]... \\
+        [--output <path>] [--log <reconciliation.md>] [--dry-run]
+
+Default ``--output`` is the tracker path itself (in-place rewrite).
+Default ``--log`` is ``reconciliation.md`` next to the tracker.
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import re
+import sys
+from dataclasses import dataclass, field
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Optional
+
+SCRIPT_DIR = Path(__file__).resolve().parent
+sys.path.insert(0, str(SCRIPT_DIR))
+
+from validate_tracker import (  # noqa: E402  (import after sys.path)
+    BULLET_PREFIX_RE,
+    BULLET_SPLIT_RE,
+    Observation,
+    classify_status,
+    derive_expected_rating,
+    normalize_item_id,
+    parse_checklist,
+)
+
+
+VALIDATION_MISSING_CLAUSE_RE = re.compile(
+    r"^\((?P<item>[^)]+)\)\s*clause\s*'(?:§\s*|section\s+)(?P<clause>[^']+?)'\s*not\s+found\s+in\s+(?P<doc>.+?)\s*$",
+    re.IGNORECASE,
+)
+VALIDATION_NO_DOC_RE = re.compile(
+    r"^\((?P<item>[^)]+)\)\s*no\s+evidence\s+file\s+matches\s*'(?P<doc>[^']+)'",
+    re.IGNORECASE,
+)
+
+CONTROL_ID_RE = re.compile(r"^[A-Z][A-Z0-9]*\.C\d+(?:\.\d+)?$")
+TABLE_HEADER_RE = re.compile(r"^\|\s*Control ID\s*\|", re.IGNORECASE)
+SEPARATOR_RE = re.compile(r"^\|[\s\-:|]+\|\s*$")
+BULLET_PREFIX_LITERAL_RE = re.compile(r"^([\s•\-\*]*)Control\s+([^:]+):\s*(.*)$", re.IGNORECASE)
+
+
+@dataclass
+class AutoApplyFinding:
+    """A single finding that should auto-flip a bullet."""
+
+    control_id: str
+    item: str  # canonical item id
+    source: str  # "9B-clause-missing" | "9B-doc-missing" | "9C-off-target" | "9C-missing"
+    reason: str
+
+    def short_label(self) -> str:
+        return {
+            "9B-clause-missing": "9B — cited clause not in evidence",
+            "9B-doc-missing": "9B — cited document not found",
+            "9C-off-target": "9C judge — Off-target",
+            "9C-missing": "9C judge — Missing",
+        }.get(self.source, self.source)
+
+
+@dataclass
+class ReviewerMediated:
+    """A finding that needs human judgement; not auto-applied."""
+
+    control_id: str
+    item: str
+    source: str  # "9C-weak" | "9D-challenge"
+    reason: str
+
+
+@dataclass
+class RowChange:
+    control_id: str
+    flips: list[tuple[str, str]] = field(default_factory=list)  # (item, reason)
+    old_rating: str = ""
+    new_rating: str = ""
+
+
+def load_validation_findings(path: Path) -> tuple[list[AutoApplyFinding], list[str]]:
+    """Extract auto-apply findings from validate_tracker.py --json output.
+
+    Returns (findings, unparsed_messages). Unparsed messages are surfaced in
+    the reconciliation log so the reviewer knows the validator flagged
+    something the reconciler didn't know how to act on.
+    """
+    data = json.loads(path.read_text(encoding="utf-8"))
+    findings: list[AutoApplyFinding] = []
+    unparsed: list[str] = []
+    for entry in data.get("warnings", []):
+        if entry.get("check") != "citation_grounding":
+            continue
+        ctrl = entry.get("control_id") or ""
+        msg = entry.get("message") or ""
+        m = VALIDATION_MISSING_CLAUSE_RE.match(msg)
+        if m:
+            findings.append(
+                AutoApplyFinding(
+                    control_id=ctrl,
+                    item=normalize_item_id(m.group("item")),
+                    source="9B-clause-missing",
+                    reason=f"cited clause section {m.group('clause').strip()} not found in {m.group('doc').strip()}",
+                )
+            )
+            continue
+        m = VALIDATION_NO_DOC_RE.match(msg)
+        if m:
+            findings.append(
+                AutoApplyFinding(
+                    control_id=ctrl,
+                    item=normalize_item_id(m.group("item")),
+                    source="9B-doc-missing",
+                    reason=f"cited document '{m.group('doc')}' not found among evidence files",
+                )
+            )
+            continue
+        unparsed.append(f"[{ctrl or '-'}] {msg}")
+    return findings, unparsed
+
+
+def load_judge_findings(path: Path) -> tuple[list[AutoApplyFinding], list[ReviewerMediated]]:
+    auto: list[AutoApplyFinding] = []
+    mediated: list[ReviewerMediated] = []
+    raw = json.loads(path.read_text(encoding="utf-8"))
+    if not isinstance(raw, list):
+        raise ValueError(f"{path}: expected a JSON array of verdicts")
+    for entry in raw:
+        ctrl = (entry.get("control_id") or "").strip()
+        item_raw = (entry.get("item") or "").strip()
+        verdict = (entry.get("verdict") or "").strip()
+        reason = (entry.get("reason") or "").strip()
+        if not ctrl or not item_raw or not verdict:
+            continue
+        item = normalize_item_id(item_raw)
+        v_low = verdict.lower()
+        if v_low == "off-target":
+            auto.append(AutoApplyFinding(ctrl, item, "9C-off-target", reason or "judge: clause does not address the requirement"))
+        elif v_low == "missing":
+            auto.append(AutoApplyFinding(ctrl, item, "9C-missing", reason or "judge: cited clause not located in evidence"))
+        elif v_low == "weak":
+            mediated.append(ReviewerMediated(ctrl, item, "9C-weak", reason or "judge: clause references topic but does not operationalise"))
+        elif v_low == "confirmed":
+            continue
+        else:
+            mediated.append(ReviewerMediated(ctrl, item, f"9C-{v_low}", reason))
+    return auto, mediated
+
+
+# --- Tracker mutation -----------------------------------------------------
+
+
+def split_bullets(cell: str) -> list[str]:
+    pieces = [c for c in BULLET_SPLIT_RE.split(cell)]
+    return pieces
+
+
+def join_bullets(bullets: list[str]) -> str:
+    return "<br>".join(bullets)
+
+
+def parse_bullet(text: str) -> Optional[tuple[str, str, str, str]]:
+    """Return (leading_whitespace_and_bullet, item_raw, item_canon, body) or None."""
+    m = BULLET_PREFIX_LITERAL_RE.match(text)
+    if not m:
+        return None
+    prefix = m.group(1) or ""
+    item_raw = m.group(2).strip()
+    body = m.group(3).strip()
+    return (prefix, item_raw, normalize_item_id(item_raw), body)
+
+
+def flip_bullet_text(original_bullet: str, reason: str) -> Optional[str]:
+    parsed = parse_bullet(original_bullet)
+    if not parsed:
+        return None
+    prefix, item_raw, _canon, _body = parsed
+    return f"{prefix}Control {item_raw}: Not addressed — {reason}"
+
+
+def parse_table_row(line: str) -> Optional[list[str]]:
+    if not line.startswith("|"):
+        return None
+    cells = [c.strip() for c in line.strip().strip("|").split("|")]
+    if not cells or not cells[0]:
+        return None
+    if not CONTROL_ID_RE.match(cells[0]):
+        return None
+    return cells
+
+
+def render_table_row(cells: list[str]) -> str:
+    return "| " + " | ".join(cells) + " |"
+
+
+def reconcile_row(
+    cells: list[str],
+    findings: list[AutoApplyFinding],
+    control_items: list,
+    change: RowChange,
+) -> list[str]:
+    """Apply auto-apply findings to a single tracker row's cells."""
+    if len(cells) < 6:
+        return cells
+
+    obs_cell = cells[3]
+    issue_cell = cells[4]
+    rating_cell = cells[5]
+
+    bullets = split_bullets(obs_cell)
+    findings_by_item: dict[str, list[AutoApplyFinding]] = {}
+    for f in findings:
+        findings_by_item.setdefault(f.item, []).append(f)
+
+    new_bullets: list[str] = []
+    for raw in bullets:
+        chunk = BULLET_PREFIX_RE.sub("", raw).strip()
+        if not chunk:
+            new_bullets.append(raw)
+            continue
+        parsed = parse_bullet(raw)
+        if not parsed:
+            new_bullets.append(raw)
+            continue
+        prefix, item_raw, item_canon, body = parsed
+        applicable = findings_by_item.get(item_canon)
+        if not applicable:
+            new_bullets.append(raw)
+            continue
+        # Skip if the bullet already says Not addressed (no-op flip).
+        if classify_status(body) == "not_addressed":
+            new_bullets.append(raw)
+            continue
+        reasons = "; ".join(f.reason for f in applicable)
+        flipped = flip_bullet_text(raw, reasons)
+        if flipped is None:
+            new_bullets.append(raw)
+            continue
+        new_bullets.append(flipped)
+        change.flips.append((item_raw, "; ".join(f.short_label() for f in applicable)))
+
+    new_obs = join_bullets(new_bullets)
+
+    # Recompute rating from the new bullet states.
+    observations: list[Observation] = []
+    for raw in new_bullets:
+        parsed = parse_bullet(raw)
+        if not parsed:
+            continue
+        _prefix, item_raw, _canon, body = parsed
+        observations.append(Observation(item_raw, classify_status(body), body))
+
+    new_rating = derive_expected_rating(observations, control_items)
+    change.old_rating = rating_cell
+    change.new_rating = new_rating or rating_cell
+
+    rating_out = new_rating if new_rating else rating_cell
+
+    if change.flips:
+        flipped_summary = "; ".join(
+            f"Control {item} ({label})" for item, label in change.flips
+        )
+        annotation = f"**[Step 9E reconciled]:** {flipped_summary}.<br>"
+        if not issue_cell.startswith("**[Step 9E reconciled]"):
+            issue_cell = annotation + issue_cell
+
+    new_cells = list(cells)
+    new_cells[3] = new_obs
+    new_cells[4] = issue_cell
+    new_cells[5] = rating_out
+    return new_cells
+
+
+def reconcile_tracker(
+    tracker_text: str,
+    auto_findings: list[AutoApplyFinding],
+    controls: dict,
+) -> tuple[str, list[RowChange]]:
+    findings_by_control: dict[str, list[AutoApplyFinding]] = {}
+    for f in auto_findings:
+        findings_by_control.setdefault(f.control_id, []).append(f)
+
+    out_lines: list[str] = []
+    changes: list[RowChange] = []
+    for line in tracker_text.splitlines():
+        cells = parse_table_row(line)
+        if cells is None:
+            out_lines.append(line)
+            continue
+        ctrl_id = cells[0]
+        applicable = findings_by_control.get(ctrl_id, [])
+        control = controls.get(ctrl_id)
+        if not applicable or control is None:
+            out_lines.append(line)
+            continue
+        change = RowChange(control_id=ctrl_id)
+        new_cells = reconcile_row(cells, applicable, control.items, change)
+        if change.flips:
+            changes.append(change)
+            out_lines.append(render_table_row(new_cells))
+        else:
+            out_lines.append(line)
+
+    return "\n".join(out_lines) + ("\n" if tracker_text.endswith("\n") else ""), changes
+
+
+# --- Reconciliation log ----------------------------------------------------
+
+
+def write_log(
+    log_path: Path,
+    tracker_path: Path,
+    output_path: Path,
+    auto_findings: list[AutoApplyFinding],
+    mediated: list[ReviewerMediated],
+    unparsed_validation: list[str],
+    changes: list[RowChange],
+) -> None:
+    now = datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M UTC")
+    lines: list[str] = []
+    lines.append("# Reconciliation Log (Step 9E)")
+    lines.append("")
+    lines.append(f"- **Generated:** {now}")
+    lines.append(f"- **Source tracker:** `{tracker_path}`")
+    lines.append(f"- **Reconciled tracker:** `{output_path}`")
+    lines.append(f"- **Auto-applied findings:** {len(auto_findings)}")
+    lines.append(f"- **Reviewer-mediated findings:** {len(mediated)}")
+    lines.append(f"- **Rows changed:** {len(changes)}")
+    lines.append("")
+
+    lines.append("## Auto-applied")
+    if not changes:
+        lines.append("")
+        lines.append("_None — no auto-apply findings hit any tracker bullet._")
+    else:
+        for change in changes:
+            lines.append("")
+            lines.append(f"### {change.control_id}")
+            lines.append("")
+            lines.append(f"- **Rating:** `{change.old_rating}` → **`{change.new_rating}`**")
+            lines.append("- **Bullets flipped:**")
+            for item, label in change.flips:
+                lines.append(f"  - Control {item}: {label}")
+    lines.append("")
+
+    lines.append("## Pending review (NOT auto-applied)")
+    if not mediated:
+        lines.append("")
+        lines.append("_None._")
+    else:
+        by_ctrl: dict[str, list[ReviewerMediated]] = {}
+        for m in mediated:
+            by_ctrl.setdefault(m.control_id, []).append(m)
+        for ctrl_id, items in sorted(by_ctrl.items()):
+            lines.append("")
+            lines.append(f"### {ctrl_id}")
+            lines.append("")
+            for m in items:
+                lines.append(f"- **Control {m.item}** ({m.source}): {m.reason}")
+    lines.append("")
+
+    if unparsed_validation:
+        lines.append("## Validator messages the reconciler could not parse")
+        lines.append("")
+        lines.append("These were emitted by `validate_tracker.py` but did not match a known auto-apply pattern. Review manually.")
+        lines.append("")
+        for msg in unparsed_validation:
+            lines.append(f"- {msg}")
+        lines.append("")
+
+    log_path.write_text("\n".join(lines), encoding="utf-8")
+
+
+# --- CLI -------------------------------------------------------------------
+
+
+def main() -> int:
+    ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
+    ap.add_argument("tracker", type=Path, help="path to tracker.md")
+    ap.add_argument("--checklist", type=Path, required=True)
+    ap.add_argument("--validation", type=Path, default=None, help="JSON output from validate_tracker.py --json")
+    ap.add_argument("--judge", type=Path, action="append", default=[], help="judge-<group>.json sidecar (may be passed multiple times)")
+    ap.add_argument("--output", type=Path, default=None, help="where to write the reconciled tracker (default: overwrite input)")
+    ap.add_argument("--log", type=Path, default=None, help="reconciliation log path (default: reconciliation.md alongside tracker)")
+    ap.add_argument("--dry-run", action="store_true", help="don't write the reconciled tracker; still write the log")
+    args = ap.parse_args()
+
+    if not args.tracker.exists():
+        print(f"tracker not found: {args.tracker}", file=sys.stderr)
+        return 1
+
+    controls = parse_checklist(args.checklist)
+
+    auto: list[AutoApplyFinding] = []
+    mediated: list[ReviewerMediated] = []
+    unparsed: list[str] = []
+
+    if args.validation:
+        v_auto, v_unparsed = load_validation_findings(args.validation)
+        auto.extend(v_auto)
+        unparsed.extend(v_unparsed)
+
+    for jp in args.judge:
+        j_auto, j_mediated = load_judge_findings(jp)
+        auto.extend(j_auto)
+        mediated.extend(j_mediated)
+
+    tracker_text = args.tracker.read_text(encoding="utf-8")
+    new_text, changes = reconcile_tracker(tracker_text, auto, controls)
+
+    output_path = args.output or args.tracker
+    if not args.dry_run:
+        output_path.write_text(new_text, encoding="utf-8")
+
+    log_path = args.log or args.tracker.with_name("reconciliation.md")
+    write_log(log_path, args.tracker, output_path, auto, mediated, unparsed, changes)
+
+    print(f"auto-applied: {sum(len(c.flips) for c in changes)} bullet(s) across {len(changes)} control(s)", file=sys.stderr)
+    print(f"reviewer-mediated: {len(mediated)} finding(s)", file=sys.stderr)
+    print(f"reconciled tracker: {output_path}", file=sys.stderr)
+    print(f"log: {log_path}", file=sys.stderr)
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

package/skills/compliance/compliance-audit/scripts/test_validate_tracker.py

@@ -0,0 +1,90 @@
+#!/usr/bin/env python3
+"""Tests for validate_tracker.py header/scope parsing.
+
+Runs under pytest (``pytest scripts/test_validate_tracker.py``) or directly
+(``python3 scripts/test_validate_tracker.py``). No third-party dependencies.
+"""
+import importlib.util
+import pathlib
+import tempfile
+
+_SPEC = importlib.util.spec_from_file_location(
+    "validate_tracker", pathlib.Path(__file__).with_name("validate_tracker.py")
+)
+v = importlib.util.module_from_spec(_SPEC)
+_SPEC.loader.exec_module(v)
+
+
+_TRACKER = """# VARS Assessment Tracker
+
+- **Archetypes In Scope:** AACQ (22 controls), ATPA (15 controls)
+
+| Control ID | Control Title | Archetype | Observations | Issue Description | Control Effectiveness Rating | Proposed Remediation | Complete By | Remediation Status |
+|---|---|---|---|---|---|---|---|---|
+| AACQ.C1.1 | T | AACQ | • Control 1: Covered | | Satisfactory | | | |
+"""
+
+_CHECKLIST = """### AACQ.C1.1: T
+- **Severity:** mandatory
+- **Verification:**
+  - [ ] **(1)** thing
+- **Evidence:**
+  - doc
+
+### AHIR.C1.1: Out of scope
+- **Severity:** mandatory
+- **Verification:**
+  - [ ] **(1)** thing
+- **Evidence:**
+  - doc
+"""
+
+
+def _write(tmp, name, content):
+    p = pathlib.Path(tmp) / name
+    p.write_text(content, encoding="utf-8")
+    return p
+
+
+def test_annotated_archetype_header_parses_to_bare_codes():
+    """'AACQ (22 controls)' must yield the code 'AACQ', not the whole token."""
+    phases, archetypes, scope, rows = v.parse_tracker(
+        _write(tempfile.mkdtemp(), "tracker.md", _TRACKER)
+    )
+    assert archetypes == ["AACQ", "ATPA"]
+
+
+def test_derive_scope_resolves_annotated_header():
+    """With cleaned codes, scope includes the annotated archetypes' controls."""
+    controls = {"AACQ.C1.1": None, "ATPA.C1.1": None, "AHIR.C1.1": None}
+    scope = v.derive_scope(controls, ["AACQ", "ATPA"], None, [])
+    assert scope == {"AACQ.C1.1", "ATPA.C1.1"}
+
+
+def test_annotated_header_does_not_flag_in_scope_rows():
+    """End-to-end: an in-scope row is not reported 'outside declared scope'."""
+    tmp = tempfile.mkdtemp()
+    findings = v.validate(
+        _write(tmp, "tracker.md", _TRACKER),
+        _write(tmp, "checklist.md", _CHECKLIST),
+        None,
+    )
+    out_of_scope = [
+        f for f in findings
+        if f["check"] == "row_coverage" and "outside declared scope" in f["message"]
+    ]
+    assert out_of_scope == [], out_of_scope
+
+
+def test_bare_archetype_header_still_works():
+    tracker = _TRACKER.replace("AACQ (22 controls), ATPA (15 controls)", "AACQ, ATPA")
+    _, archetypes, _, _ = v.parse_tracker(_write(tempfile.mkdtemp(), "t.md", tracker))
+    assert archetypes == ["AACQ", "ATPA"]
+
+
+if __name__ == "__main__":
+    fns = [f for n, f in sorted(globals().items()) if n.startswith("test_")]
+    for fn in fns:
+        fn()
+        print(f"ok  {fn.__name__}")
+    print(f"\n{len(fns)} passed")