@1aboveio/skills 0.20.2 → 0.20.4

package/skills/document-utils/reg-parser/scripts/compare_checklist.py

@@ -0,0 +1,228 @@
+#!/usr/bin/env python3
+"""Compare a generated checklist against a test target file.
+
+Targets often live in a different format than the skill's own output
+(e.g. tests/vars-checklist.md is a wide markdown table; reg-parser
+output is heading-form per checklist-schema.md). This script does a
+structural comparison rather than a byte diff:
+
+  * Extract source-native control IDs from both files (regex over the
+    file text — works for headings, table rows, prose).
+  * Report set diffs: missing-from-output, extra-in-output, matched.
+  * From the output only, count items by archetype prefix and severity
+    (target files generally don't carry severity in a uniform place).
+
+Exit code:
+  0  — every target ID is present in the output (extras allowed)
+  1  — at least one target ID is missing in the output
+
+Usage:
+    python3 compare_checklist.py <output-checklist.md> <target-file.md>
+"""
+import argparse
+import re
+import sys
+from collections import Counter
+from pathlib import Path
+
+# Source-native control IDs: <CAT>.<seg>(.<seg>)+
+# CAT is an uppercase token; segments are alphanumeric (preserve source
+# punctuation). Examples: AACQ.C1.1, ATPA.C2.1, AVDC.C2.1.
+ID_RE = re.compile(r"\b([A-Z][A-Z0-9]{2,})\.([A-Za-z0-9]+(?:\.[A-Za-z0-9]+)+)\b")
+# A checklist item heading (used to count by severity in the output).
+ITEM_HEADING_RE = re.compile(
+    r"^###\s+([A-Z][A-Z0-9]*\.[A-Za-z0-9]+(?:\.[A-Za-z0-9]+)+):\s+",
+    re.MULTILINE,
+)
+SEVERITY_RE = re.compile(r"\*\*Severity:\*\*\s*([A-Za-z]+)", re.IGNORECASE)
+VERIFICATION_HEADER_RE = re.compile(r"^-?\s*\*\*Verification:\*\*", re.MULTILINE)
+NEXT_FIELD_RE = re.compile(r"^-?\s*\*\*[A-Z][\w\s/-]*:\*\*")
+CHECKBOX_INDENT_RE = re.compile(r"^(\s*)-\s*\[[ xX]\]\s*(.+)$")
+LETTER_ENUM_PROSE_RE = re.compile(r"\([a-z]\)")
+ROMAN_ENUM_PROSE_RE = re.compile(r"\([ivx]{1,4}\)")
+
+
+def verification_stats(text: str) -> dict[str, dict]:
+    """Per-item Verification structure stats: total checkbox count, max
+    nesting depth, and a flattening flag (top-level bullet with no
+    children but containing 2+ lettered/roman enumerator markers).
+
+    The depth distribution and flattening count are useful as a sanity
+    check for whether an agent run preserved source nesting. A run that
+    produces only 1-checkbox items across the board, or many flattened
+    bullets, is almost certainly mis-parsing source structure.
+    """
+    stats: dict[str, dict] = {}
+    item_matches = list(ITEM_HEADING_RE.finditer(text))
+    for i, m in enumerate(item_matches):
+        item_id = m.group(1)
+        seg_start = m.end()
+        seg_end = item_matches[i + 1].start() if i + 1 < len(item_matches) else len(text)
+        segment = text[seg_start:seg_end]
+        verif_match = VERIFICATION_HEADER_RE.search(segment)
+        if not verif_match:
+            stats[item_id] = {"total": 0, "top_level": 0, "max_depth": 0, "flattened": 0}
+            continue
+        verif_body = segment[verif_match.end():]
+        bullets: list[tuple[int, str]] = []
+        for line in verif_body.splitlines():
+            stripped = line.strip()
+            if not stripped:
+                continue
+            cb = CHECKBOX_INDENT_RE.match(line.rstrip())
+            if cb:
+                bullets.append((len(cb.group(1)), cb.group(2)))
+                continue
+            # Stop at next field
+            if NEXT_FIELD_RE.match(stripped):
+                break
+        if not bullets:
+            stats[item_id] = {"total": 0, "top_level": 0, "max_depth": 0, "flattened": 0}
+            continue
+        min_indent = min(b[0] for b in bullets)
+        depth_levels = sorted({b[0] for b in bullets})
+        max_depth = depth_levels.index(max(depth_levels)) + 1 if depth_levels else 0
+        top_level = sum(1 for b in bullets if b[0] == min_indent)
+        flattened = 0
+        for idx, (indent, t) in enumerate(bullets):
+            if indent != min_indent:
+                continue
+            has_nested = idx + 1 < len(bullets) and bullets[idx + 1][0] > indent
+            if has_nested:
+                continue
+            if len(LETTER_ENUM_PROSE_RE.findall(t)) >= 2 or len(
+                ROMAN_ENUM_PROSE_RE.findall(t)
+            ) >= 2:
+                flattened += 1
+        stats[item_id] = {
+            "total": len(bullets),
+            "top_level": top_level,
+            "max_depth": max_depth,
+            "flattened": flattened,
+        }
+    return stats
+
+
+def extract_ids(text: str) -> set[str]:
+    return {f"{m.group(1)}.{m.group(2)}" for m in ID_RE.finditer(text)}
+
+
+def archetype(item_id: str) -> str:
+    return item_id.split(".", 1)[0]
+
+
+def output_severity_counts(text: str) -> Counter:
+    """Count severities by walking item-headings in the output and
+    grabbing the first Severity line that follows each."""
+    counts: Counter = Counter()
+    matches = list(ITEM_HEADING_RE.finditer(text))
+    for i, m in enumerate(matches):
+        start = m.end()
+        end = matches[i + 1].start() if i + 1 < len(matches) else len(text)
+        segment = text[start:end]
+        sev_match = SEVERITY_RE.search(segment)
+        if sev_match:
+            counts[sev_match.group(1).lower()] += 1
+        else:
+            counts["<none>"] += 1
+    return counts
+
+
+def main() -> int:
+    p = argparse.ArgumentParser(description=__doc__.split("\n\n")[0])
+    p.add_argument("output", help="Generated checklist (skill output)")
+    p.add_argument("target", help="Test target file (ground truth)")
+    args = p.parse_args()
+
+    out_path = Path(args.output)
+    tgt_path = Path(args.target)
+    if not out_path.is_file():
+        print(f"❌ Output file not found: {out_path}", file=sys.stderr)
+        return 2
+    if not tgt_path.is_file():
+        print(f"❌ Target file not found: {tgt_path}", file=sys.stderr)
+        return 2
+
+    out_text = out_path.read_text(encoding="utf-8")
+    tgt_text = tgt_path.read_text(encoding="utf-8")
+
+    out_ids = extract_ids(out_text)
+    tgt_ids = extract_ids(tgt_text)
+
+    missing = sorted(tgt_ids - out_ids)
+    extra = sorted(out_ids - tgt_ids)
+    matched = sorted(out_ids & tgt_ids)
+
+    out_arch = Counter(archetype(i) for i in out_ids)
+    tgt_arch = Counter(archetype(i) for i in tgt_ids)
+    sev = output_severity_counts(out_text)
+
+    print(f"Output: {out_path}")
+    print(f"Target: {tgt_path}")
+    print()
+    print(f"Control IDs — target: {len(tgt_ids)}, output: {len(out_ids)}, matched: {len(matched)}")
+
+    if missing:
+        print(f"\n❌ Missing from output ({len(missing)}):")
+        for i in missing:
+            print(f"   - {i}")
+    else:
+        print("\n✅ Every target control ID is present in the output.")
+
+    if extra:
+        print(f"\n⚠️  Extra in output, not in target ({len(extra)}):")
+        for i in extra:
+            print(f"   + {i}")
+
+    print("\nBy archetype:")
+    archetypes = sorted(set(out_arch) | set(tgt_arch))
+    print(f"   {'archetype':<8} {'target':>7} {'output':>7}")
+    for a in archetypes:
+        print(f"   {a:<8} {tgt_arch.get(a, 0):>7} {out_arch.get(a, 0):>7}")
+
+    print("\nOutput severity counts (from item bodies):")
+    for s, n in sorted(sev.items()):
+        print(f"   {s:<14} {n}")
+
+    # Verification structure stats — useful flattening signal
+    vstats = verification_stats(out_text)
+    if vstats:
+        totals = [v["total"] for v in vstats.values()]
+        depths = [v["max_depth"] for v in vstats.values()]
+        flat_total = sum(v["flattened"] for v in vstats.values())
+        flat_items = [iid for iid, v in vstats.items() if v["flattened"] > 0]
+        single_bullet_items = [iid for iid, v in vstats.items() if v["total"] == 1]
+        depth_dist = Counter(depths)
+
+        print("\nOutput Verification structure:")
+        print(f"   total bullets across {len(vstats)} items: {sum(totals)}")
+        if totals:
+            print(
+                f"   bullets per item: min={min(totals)} max={max(totals)} "
+                f"mean={sum(totals)/len(totals):.1f}"
+            )
+        print("   max-depth distribution:")
+        for d in sorted(depth_dist):
+            print(f"     depth {d}: {depth_dist[d]} item(s)")
+        if flat_total:
+            print(
+                f"\n⚠️  Flattening signals: {flat_total} bullet(s) across "
+                f"{len(flat_items)} item(s) contain 2+ lettered/roman "
+                f"enumerator markers in prose with no nested checkboxes."
+            )
+            for iid in flat_items[:10]:
+                print(f"     - {iid}")
+            if len(flat_items) > 10:
+                print(f"     … and {len(flat_items) - 10} more")
+        if single_bullet_items and len(single_bullet_items) > len(vstats) * 0.5:
+            print(
+                f"\n⚠️  {len(single_bullet_items)}/{len(vstats)} items have only "
+                f"a single Verification bullet — agent may have collapsed "
+                f"multi-step controls into single statements."
+            )
+
+    return 1 if missing else 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

package/skills/document-utils/reg-parser/scripts/dump_checklist.py

@@ -0,0 +1,348 @@
+#!/usr/bin/env python3
+"""Dump a compliance checklist Markdown file to JSON.
+
+The markdown file is the authored source of truth — auditors read and
+review it, authors edit it, GitHub renders it. Downstream consumers
+(compliance-audit, report-writing) want structure, not regex parsers.
+This script emits a JSON representation that mirrors the markdown's
+nested structure verbatim so consumers can iterate items, walk the
+verification tree, and look up severity/evidence without re-parsing.
+
+The output JSON is a build artifact. Treat the markdown as canonical
+and regenerate JSON on every change (CI guard or pre-commit hook).
+
+Usage:
+    python3 dump_checklist.py <checklist.md>                 # to stdout
+    python3 dump_checklist.py <checklist.md> --out <out.json>
+
+Schema (illustrative):
+    {
+      "metadata": {"name": ..., "regulation": ..., "id_scheme": ..., "summary": {...}},
+      "categories": [{"id": "AACQ", "title": "...", "item_ids": [...]}],
+      "items": [
+        {
+          "id": "AACQ.C5.1",
+          "title": "Underwriting Policy",
+          "category": "AACQ",
+          "risk_domain": "...",
+          "risk_sub_domain": "...",
+          "risk_statement": "...",
+          "parent_control": "AACQ.C5",
+          "requirement": "...",
+          "source": "AACQ.C5.1 (Visa VARS, October 2024)",
+          "severity": "mandatory",
+          "verification": [
+            {
+              "prefix": "(1)",
+              "text": "...",
+              "section": "mandatory",      # only on top-level nodes
+              "children": [
+                {"prefix": "(a)", "text": "...", "children": [...]}
+              ]
+            }
+          ],
+          "evidence": ["...", "..."]
+        }
+      ]
+    }
+"""
+
+import argparse
+import json
+import re
+import sys
+from typing import Any
+
+# ---- regexes shared with validate_checklist.py -----------------------------
+
+YAML_BLOCK_RE = re.compile(r"^```yaml\s*$")
+YAML_END_RE = re.compile(r"^```\s*$")
+
+CATEGORY_RES = {
+    "cat": re.compile(r"^##\s+(CAT-\d{2}):\s+(.+)$"),
+    "source-native": re.compile(r"^##\s+([A-Z][A-Z0-9]*):\s+(.+)$"),
+}
+ITEM_RES = {
+    "cat": re.compile(r"^###\s+(CAT-\d{2}-\d{3}):\s+(.+)$"),
+    "source-native": re.compile(
+        r"^###\s+([A-Z][A-Z0-9]*\.[A-Za-z0-9]+(?:\.[A-Za-z0-9]+)+):\s+(.+)$"
+    ),
+}
+
+# Bold field label, e.g. "- **Risk Domain:** 4.1.2 ..."
+FIELD_LINE_RE = re.compile(r"^-\s+\*\*([A-Za-z][\w\s/-]*?):\*\*\s*(.*)$")
+# Continuation of a single-line field (next line starts with "  " but not a bullet/checkbox)
+CHECKBOX_RE = re.compile(r"^(\s*)-\s+\[[ xX]\]\s+(.+)$")
+PLAIN_BULLET_RE = re.compile(r"^(\s*)-\s+(?!\[[ xX]\])(.+)$")
+# Optional bold prefix at the start of a checkbox text: "**(1)** rest" or "**(a)** rest"
+PREFIX_RE = re.compile(r"^\*\*\(([^)]+)\)\*\*\s*(.*)$")
+
+
+# ---- YAML parser (lightweight, only what the schema uses) ------------------
+
+
+def parse_yaml(lines: list[str]) -> tuple[dict[str, Any], int]:
+    """Parse the first ```yaml ... ``` block. Return (meta, end_line_idx)."""
+    meta: dict[str, Any] = {}
+    summary: dict[str, int] = {}
+    in_block = False
+    in_summary = False
+    end = 0
+    for i, line in enumerate(lines):
+        stripped = line.rstrip("\n")
+        if not in_block:
+            if YAML_BLOCK_RE.match(stripped):
+                in_block = True
+            continue
+        if YAML_END_RE.match(stripped):
+            end = i + 1
+            break
+        s = stripped.strip()
+        if not s:
+            continue
+        if s == "summary:":
+            in_summary = True
+            continue
+        if in_summary:
+            sm = re.match(r"^(\w+):\s*(\d+)\s*$", s)
+            if sm:
+                summary[sm.group(1)] = int(sm.group(2))
+                continue
+            # End of summary block
+            in_summary = False
+        m = re.match(r'^(\w+):\s*"?(.*?)"?\s*$', s)
+        if m:
+            meta[m.group(1)] = m.group(2)
+    if summary:
+        meta["summary"] = summary
+    return meta, end
+
+
+# ---- Verification tree builder ---------------------------------------------
+
+
+def build_verification_tree(raw_lines: list[str]) -> list[dict[str, Any]]:
+    """Convert a sequence of checkbox lines (already isolated to the
+    Verification field's body) into a nested tree.
+
+    Each node: {"prefix": "(N)", "text": "rest", "children": [...]}.
+    The prefix is whatever sits inside the leading `**(...)**` bold; if
+    no prefix is present, prefix is None and text is the whole bullet.
+    Indent depth determines parent/child relationships — every level of
+    deeper indent is a child of the most recent shallower node.
+    """
+    # Each entry: (indent, node)
+    stack: list[tuple[int, dict[str, Any]]] = []
+    roots: list[dict[str, Any]] = []
+    for line in raw_lines:
+        m = CHECKBOX_RE.match(line.rstrip())
+        if not m:
+            continue
+        indent = len(m.group(1))
+        body = m.group(2).strip()
+        prefix_m = PREFIX_RE.match(body)
+        if prefix_m:
+            prefix = prefix_m.group(1)
+            text = prefix_m.group(2).strip()
+        else:
+            prefix = None
+            text = body
+        node = {"prefix": prefix, "text": text, "children": []}
+        # Pop stack until we find a parent at strictly smaller indent
+        while stack and stack[-1][0] >= indent:
+            stack.pop()
+        if stack:
+            stack[-1][1]["children"].append(node)
+        else:
+            roots.append(node)
+        stack.append((indent, node))
+    return roots
+
+
+# ---- Item parser -----------------------------------------------------------
+
+
+def parse_item(
+    cid: str,
+    title: str,
+    body_lines: list[str],
+    scheme: str,
+) -> dict[str, Any]:
+    """Parse the body of one ### heading into a structured item."""
+    item: dict[str, Any] = {
+        "id": cid,
+        "title": title.strip(),
+        "category": cid.split(".", 1)[0] if scheme == "source-native" else cid[:6],
+        "risk_domain": None,
+        "risk_sub_domain": None,
+        "risk_statement": None,
+        "parent_control": None,
+        "requirement": None,
+        "source": None,
+        "severity": None,
+        "verification": [],
+        "evidence": [],
+    }
+
+    # State machine: walk lines, dispatching by current field.
+    current_field: str | None = None
+    field_buffer: list[str] = []  # for verification + evidence
+    simple_fields = {
+        "Risk Domain": "risk_domain",
+        "Risk Sub-Domain": "risk_sub_domain",
+        "Risk Statement": "risk_statement",
+        "Parent Control": "parent_control",
+        "Requirement": "requirement",
+        "Source": "source",
+        "Severity": "severity",
+    }
+
+    def flush_buffer() -> None:
+        nonlocal field_buffer
+        if not current_field:
+            field_buffer = []
+            return
+        if current_field == "Verification":
+            tree = build_verification_tree(field_buffer)
+            # Tag each top-level node as mandatory vs recommended so
+            # downstream consumers can disambiguate cleanly when
+            # mandatory and recommended controls share `(1)`/`(2)`
+            # prefixes (per source numbering). For items whose own
+            # severity is "recommended" the whole tree is recommended.
+            sev = item.get("severity") or ""
+            for node in tree:
+                if sev == "recommended":
+                    node["section"] = "recommended"
+                elif (node.get("text") or "").lstrip().startswith("(Recommended)"):
+                    node["section"] = "recommended"
+                else:
+                    node["section"] = "mandatory"
+            item["verification"] = tree
+        elif current_field == "Evidence":
+            evidence: list[str] = []
+            for ln in field_buffer:
+                m = PLAIN_BULLET_RE.match(ln.rstrip())
+                if m:
+                    evidence.append(m.group(2).strip())
+            item["evidence"] = evidence
+        field_buffer = []
+
+    for line in body_lines:
+        m = FIELD_LINE_RE.match(line.rstrip())
+        if m:
+            # Starting a new field — flush the previous list-bearing one
+            flush_buffer()
+            label = m.group(1).strip()
+            rest = m.group(2).strip()
+            if label in simple_fields:
+                key = simple_fields[label]
+                # Keep severity normalized lowercase
+                value: Any = rest.strip()
+                if key == "severity":
+                    value = re.sub(r"[^a-z]", "", value.lower())
+                item[key] = value
+                current_field = None
+            elif label in ("Verification", "Evidence"):
+                current_field = label
+                # `rest` should be empty for these fields, but tolerate inline content
+                if rest:
+                    field_buffer.append(rest)
+            else:
+                current_field = None
+            continue
+        # Continuation under Verification / Evidence
+        if current_field in ("Verification", "Evidence"):
+            field_buffer.append(line)
+    flush_buffer()
+    return item
+
+
+# ---- Top-level walker ------------------------------------------------------
+
+
+def parse_checklist(text: str) -> dict[str, Any]:
+    lines = text.splitlines()
+    meta, yaml_end = parse_yaml(lines)
+    scheme = meta.get("id_scheme", "cat")
+    if scheme not in CATEGORY_RES:
+        raise SystemExit(f"Unknown id_scheme '{scheme}'")
+    cat_re = CATEGORY_RES[scheme]
+    item_re = ITEM_RES[scheme]
+
+    categories: list[dict[str, Any]] = []
+    items: list[dict[str, Any]] = []
+    cat_by_id: dict[str, dict[str, Any]] = {}
+
+    current_item_id: str | None = None
+    current_item_title: str | None = None
+    current_item_lines: list[str] = []
+
+    def flush_item() -> None:
+        nonlocal current_item_id, current_item_title, current_item_lines
+        if current_item_id is None:
+            return
+        parsed = parse_item(
+            current_item_id, current_item_title or "", current_item_lines, scheme
+        )
+        items.append(parsed)
+        cat = cat_by_id.get(parsed["category"])
+        if cat is not None:
+            cat["item_ids"].append(parsed["id"])
+        current_item_id = None
+        current_item_title = None
+        current_item_lines = []
+
+    for line in lines[yaml_end:]:
+        cm = cat_re.match(line.rstrip())
+        if cm:
+            flush_item()
+            cat_id = cm.group(1)
+            cat = {"id": cat_id, "title": cm.group(2).strip(), "item_ids": []}
+            categories.append(cat)
+            cat_by_id[cat_id] = cat
+            continue
+        im = item_re.match(line.rstrip())
+        if im:
+            flush_item()
+            current_item_id = im.group(1)
+            current_item_title = im.group(2)
+            current_item_lines = []
+            continue
+        if current_item_id is not None:
+            current_item_lines.append(line)
+    flush_item()
+
+    return {"metadata": meta, "categories": categories, "items": items}
+
+
+# ---- CLI -------------------------------------------------------------------
+
+
+def main() -> None:
+    p = argparse.ArgumentParser(description=__doc__.split("\n\n")[0])
+    p.add_argument("filepath", help="Path to checklist Markdown file")
+    p.add_argument("--out", help="Output JSON path (default: stdout)")
+    p.add_argument(
+        "--indent",
+        type=int,
+        default=2,
+        help="JSON indent (default: 2; use 0 for minified)",
+    )
+    args = p.parse_args()
+
+    with open(args.filepath, "r", encoding="utf-8") as f:
+        text = f.read()
+    parsed = parse_checklist(text)
+    indent = args.indent if args.indent > 0 else None
+    out = json.dumps(parsed, indent=indent, ensure_ascii=False)
+    if args.out:
+        with open(args.out, "w", encoding="utf-8") as f:
+            f.write(out)
+            f.write("\n")
+    else:
+        sys.stdout.write(out)
+        sys.stdout.write("\n")
+
+
+if __name__ == "__main__":
+    main()