@1aboveio/skills 0.0.0-stage → 1.0.0

package/skills/core/reg-parser/scripts/validate_checklist.py

@@ -0,0 +1,563 @@
+#!/usr/bin/env python3
+"""Validate a compliance checklist Markdown file against the checklist schema.
+
+Usage:
+    python3 validate_checklist.py <checklist.md> [--source <converted-md>]
+
+Without --source, structural checks plus a flattening-detector warning run.
+
+With --source <converted-md>, three additional semantic checks run as warnings:
+  1. Source citations: each item's `Source:` field must have at least
+     one anchor (digit-bearing token) that appears in the source markdown.
+  2. Modal verbs: each Requirement should contain mandatory language
+     (must / shall / required / prohibited / mandatory).
+  3. Verification independence: Verification steps should not be a
+     near-paraphrase of the Requirement (Jaccard overlap > 0.5).
+
+Always-on flattening detectors (source-native scheme):
+  - Marker-based: warns when a top-level Verification bullet contains
+    lettered or roman enumerator markers in prose (e.g. "(a)…(b)…(c)")
+    with no nested checkboxes — the agent likely flattened source
+    sub-bullets that should have been preserved as nested checkboxes
+    for auditor traceability.
+  - Comma-list: warns when a top-level Verification bullet is long
+    (>40 words) and contains many list joiners (4+ commas/semicolons)
+    with no nested children. Catches the case where the agent
+    paraphrased multiple source provisions into a comma-separated
+    summary, dropping the enumerator markers along the way.
+
+Exit code 0 = valid.  Non-zero = errors found (printed to stderr).
+Warnings never cause non-zero exit.
+"""
+
+import argparse
+import re
+import sys
+import os
+
+REQUIRED_METADATA = {"name", "regulation", "version", "scope", "date"}
+REQUIRED_ITEM_FIELDS = {"Requirement", "Source", "Severity", "Verification", "Evidence"}
+
+# Scheme-dependent rules. id_scheme metadata field selects between them;
+# default is "cat".
+SCHEMES = {
+    "cat": {
+        "cat_re": re.compile(r"^##\s+(CAT-\d{2}):\s+.+"),
+        "item_re": re.compile(r"^###\s+(CAT-\d{2}-\d{3}):\s+.+"),
+        "severities": ("critical", "major", "minor"),
+        "cat_hint": "## CAT-XX: Title",
+        "item_hint": "### CAT-XX-NNN: Title",
+    },
+    "source-native": {
+        # Category: uppercase token (letters/digits), e.g. AACQ, ATPA, PCIDSS.
+        "cat_re": re.compile(r"^##\s+([A-Z][A-Z0-9]*):\s+.+"),
+        # Item: <CAT>.<segment>(.<segment>)+ — preserves source punctuation.
+        "item_re": re.compile(
+            r"^###\s+([A-Z][A-Z0-9]*\.[A-Za-z0-9]+(?:\.[A-Za-z0-9]+)+):\s+.+"
+        ),
+        "severities": ("mandatory", "recommended"),
+        "cat_hint": "## <UPPERCASE-TOKEN>: Title",
+        "item_hint": "### <CAT>.<seg>.<seg>: Title",
+    },
+}
+
+YAML_BLOCK_RE = re.compile(r"^```yaml\s*$")
+YAML_END_RE = re.compile(r"^```\s*$")
+DATE_RE = re.compile(r"^\d{4}-\d{2}-\d{2}$")
+
+
+def category_prefix(item_id: str, scheme: str) -> str:
+    """Return the category-ID prefix of an item ID under the given scheme."""
+    if scheme == "cat":
+        return item_id[:6]  # 'CAT-01-001' -> 'CAT-01'
+    # source-native: everything before the first dot
+    return item_id.split(".", 1)[0]
+
+MANDATORY_RE = re.compile(
+    r"\b(must|shall|required|requires|require|prohibited|prohibit|mandatory|"
+    r"obligated|forbidden|cannot|shall not|may not)\b",
+    re.IGNORECASE,
+)
+# Hedges that indicate a non-binding obligation. Present without an
+# overriding mandatory anchor → likely the agent pulled in "should"/"may"
+# guidance and treated it as a requirement.
+WEAK_RE = re.compile(
+    r"\b(should|may|encouraged|recommended|best practice|where possible|"
+    r"as appropriate|if practicable|where practicable)\b",
+    re.IGNORECASE,
+)
+# Anchor token = a substring containing at least one digit, ≥2 chars.
+# Catches: 13(1)(a), 2015/849, 615, 5.2, 76E, AACQ.C2.1.
+ANCHOR_RE = re.compile(r"[\w./()\-]*\d[\w./()\-]*")
+TOKEN_RE = re.compile(r"\b\w+\b")
+STOPWORDS = {
+    "the", "a", "an", "of", "to", "and", "or", "in", "on", "for", "with",
+    "be", "is", "are", "as", "at", "by", "must", "shall", "all", "any",
+    "each", "from", "that", "this", "their", "its", "such", "where", "when",
+    "before", "after", "into", "out", "if", "not", "no", "than", "then",
+    "have", "has", "been", "being", "must", "may",
+}
+JACCARD_THRESHOLD = 0.5
+JACCARD_MIN_TOKENS = 4  # avoid false positives on tiny items
+
+# Flattening detection (source-native only). A top-level Verification
+# bullet that contains 2+ parenthesized lowercase letters or 2+ roman
+# numerals in its text — and has no nested checkboxes under it — is
+# almost certainly a flattened source list (e.g. "Acquirer (a) collects
+# X, (b) verifies Y, (c) reports Z" should have been three nested
+# checkboxes). One match is tolerated (often a citation like section 13(1)(a)).
+LETTER_ENUM_RE = re.compile(r"\([a-z]\)")
+ROMAN_ENUM_RE = re.compile(r"\([ivx]{1,4}\)")
+CHECKBOX_LINE_RE = re.compile(r"^(\s*)-\s*\[[ xX]\]\s*(.+)$")
+FIELD_LABEL_LINE_RE = re.compile(r"^-?\s*\*\*[A-Z][\w\s/-]*:\*\*")
+
+
+def verification_flattening_warnings(
+    raw_lines: list[str],
+    item_re: re.Pattern,
+) -> list[str]:
+    """Detect Verification bullets that flatten source sub-bullets into prose.
+
+    Walks the raw file, tracking which item we're inside and whether we're
+    inside that item's Verification field. For each item, collects the
+    indentation + text of every checkbox bullet, then flags top-level
+    bullets (minimum indent within the item's Verification block) that
+    have no children but contain 2+ lettered/roman enumerator markers in
+    their text — the textbook flattening pattern.
+    """
+    warnings: list[str] = []
+    # item_id -> list of (indent, text)
+    blocks: dict[str, list[tuple[int, str]]] = {}
+    current_item: str | None = None
+    in_verif = False
+
+    for line in raw_lines:
+        stripped = line.strip()
+        m = item_re.match(stripped)
+        if m:
+            current_item = m.group(1)
+            blocks.setdefault(current_item, [])
+            in_verif = False
+            continue
+        if not current_item:
+            continue
+        # Verification field start
+        if stripped.startswith("**Verification:**") or stripped.startswith(
+            "- **Verification:**"
+        ):
+            in_verif = True
+            continue
+        if not in_verif:
+            continue
+        # Field-label line that is NOT a checkbox ends the Verification block
+        cb_match = CHECKBOX_LINE_RE.match(line.rstrip())
+        if not cb_match:
+            if FIELD_LABEL_LINE_RE.match(stripped) and not stripped.startswith(
+                "- [ ]"
+            ) and not stripped.startswith("- [x]"):
+                in_verif = False
+            continue
+        indent = len(cb_match.group(1))
+        text = cb_match.group(2)
+        blocks[current_item].append((indent, text))
+
+    for item_id, bullets in blocks.items():
+        if not bullets:
+            continue
+        unprefixed = sum(1 for _, text in bullets if not re.match(r"^\*\*\([^)]+\)\*\*", text))
+        if unprefixed:
+            warnings.append(
+                f"{item_id}: {unprefixed} Verification checkbox(es) lack a bold source prefix "
+                f"like **(1)** / **(a)** / **(i)** — source-native steps must name the "
+                f"source control item they audit (see checklist-schema.md)."
+            )
+        min_indent = min(b[0] for b in bullets)
+        for idx, (indent, text) in enumerate(bullets):
+            if indent != min_indent:
+                continue
+            has_nested = idx + 1 < len(bullets) and bullets[idx + 1][0] > indent
+            if has_nested:
+                continue
+            letter_hits = len(LETTER_ENUM_RE.findall(text))
+            roman_hits = len(ROMAN_ENUM_RE.findall(text))
+            if letter_hits >= 2 or roman_hits >= 2:
+                kind = "lettered" if letter_hits >= 2 else "roman"
+                warnings.append(
+                    f"{item_id}: Verification bullet contains {kind} enumerator "
+                    f"markers in prose with no nested checkboxes — likely "
+                    f"flattened source sub-bullets. Preserve source nesting depth "
+                    f"(see checklist-schema.md 'Verification step prefixes')."
+                )
+                continue
+            # Comma-list flattening: long bullet packed with joiners but no
+            # children. Typically the agent paraphrased multiple source
+            # provisions into a single summary and stripped the markers.
+            # Strip the bold prefix (e.g. "**(1)**") so it doesn't inflate
+            # the word count for short, prefix-heavy bullets.
+            stripped_text = re.sub(r"^\*\*\([^)]*\)\*\*\s*", "", text)
+            words = stripped_text.split()
+            wc = len(words)
+            joiner_count = stripped_text.count(",") + stripped_text.count(";")
+            if wc > 40 and joiner_count >= 4:
+                warnings.append(
+                    f"{item_id}: Verification bullet is {wc} words with "
+                    f"{joiner_count} comma/semicolon joiners and no nested "
+                    f"checkboxes — possibly multiple source provisions "
+                    f"collapsed into a single summary. Compare against source "
+                    f"and split if each clause is a discrete obligation."
+                )
+    return warnings
+
+
+def parse_yaml_block(lines: list[str]) -> tuple[dict[str, str], dict[str, int], list[str]]:
+    """Extract key-value pairs and summary block from the first YAML fenced block."""
+    errors = []
+    meta: dict[str, str] = {}
+    summary: dict[str, int] = {}
+    in_yaml = False
+    in_summary = False
+    found = False
+
+    for line in lines:
+        stripped = line.strip()
+        if not in_yaml and YAML_BLOCK_RE.match(stripped):
+            in_yaml = True
+            found = True
+            continue
+        if in_yaml:
+            if YAML_END_RE.match(stripped):
+                break
+            # Detect summary block start
+            if stripped == "summary:":
+                in_summary = True
+                continue
+            # Parse summary sub-fields (indented)
+            if in_summary:
+                sm = re.match(r'^(\w+):\s*(\d+)\s*$', stripped)
+                if sm:
+                    summary[sm.group(1)] = int(sm.group(2))
+                    continue
+                else:
+                    in_summary = False  # end of summary block
+            # Simple key: "value" or key: value parsing
+            m = re.match(r'^(\w+):\s*"?(.+?)"?\s*$', stripped)
+            if m:
+                meta[m.group(1)] = m.group(2)
+
+    if not found:
+        errors.append("Missing YAML metadata block (```yaml ... ```)")
+    return meta, summary, errors
+
+
+def tokenize(text: str) -> set[str]:
+    """Lowercase content tokens, stopwords removed, length >= 3."""
+    return {
+        w.lower()
+        for w in TOKEN_RE.findall(text)
+        if len(w) >= 3 and w.lower() not in STOPWORDS
+    }
+
+
+def jaccard(a: set[str], b: set[str]) -> float:
+    if not a and not b:
+        return 0.0
+    return len(a & b) / len(a | b)
+
+
+def fuzzy_present(token: str, haystack_lower: str) -> bool:
+    """Token appears in the haystack as a whole reference, ignoring whitespace.
+
+    "2(1)(a)" matches "section 2 (1)(a)" but "6" does not match "2026" and
+    "1" does not match "2.1" or "10": a leading digit may not follow a digit
+    or dot, and a trailing digit may not be followed by one.
+    """
+    tok = re.sub(r"\s+", "", token.lower()).strip(".,;:")
+    if len(re.sub(r"[^a-z0-9]", "", tok)) < 2 and not tok.isdigit():
+        return False
+    hay = re.sub(r"\s+", "", haystack_lower)
+    lead = r"(?<![\d.])" if tok[:1].isdigit() else r"(?<![a-z0-9])"
+    tail = r"(?![\d])" if tok[-1:].isdigit() else r"(?![a-z0-9])"
+    return re.search(lead + re.escape(tok) + tail, hay) is not None
+
+
+def validate(filepath: str, source_path: str | None = None) -> tuple[list[str], list[str]]:
+    """Return (errors, warnings)."""
+    errors: list[str] = []
+    warnings: list[str] = []
+
+    if not os.path.isfile(filepath):
+        return [f"File not found: {filepath}"], []
+
+    with open(filepath, "r", encoding="utf-8") as f:
+        lines = f.readlines()
+
+    raw_lines = [l.rstrip("\n") for l in lines]
+
+    # --- Metadata ---
+    meta, summary, meta_errors = parse_yaml_block(raw_lines)
+    errors.extend(meta_errors)
+
+    for field in REQUIRED_METADATA:
+        if field not in meta:
+            errors.append(f"Missing required metadata field: {field}")
+
+    if "date" in meta and not DATE_RE.match(meta["date"]):
+        errors.append(f"Invalid date format: '{meta['date']}' (expected YYYY-MM-DD)")
+
+    # Resolve id_scheme; default = cat. Unknown values are an error.
+    scheme_name = meta.get("id_scheme", "cat")
+    if scheme_name not in SCHEMES:
+        errors.append(
+            f"Invalid id_scheme: '{scheme_name}' (must be 'cat' or 'source-native')"
+        )
+        scheme_name = "cat"
+    scheme = SCHEMES[scheme_name]
+    cat_re = scheme["cat_re"]
+    item_re = scheme["item_re"]
+    valid_severities = set(scheme["severities"])
+    summary_count_fields = ("categories", "total_items", *scheme["severities"])
+
+    if not summary:
+        errors.append(
+            "Missing required metadata field: summary "
+            f"({', '.join(summary_count_fields)})"
+        )
+    else:
+        for sf in summary_count_fields:
+            if sf not in summary:
+                errors.append(f"Missing summary field: {sf}")
+
+    # Check extends target exists
+    if "extends" in meta:
+        base_path = meta["extends"]
+        # Resolve relative to the checklist file's directory
+        checklist_dir = os.path.dirname(os.path.abspath(filepath))
+        resolved = os.path.normpath(os.path.join(checklist_dir, base_path))
+        if not os.path.isfile(resolved):
+            errors.append(f"extends target not found: {base_path} (resolved: {resolved})")
+
+    # --- Categories and Items ---
+    categories: set[str] = set()
+    item_ids: list[str] = []
+    current_cat: str | None = None
+    current_item_id: str | None = None
+    item_fields_found: dict[str, set[str]] = {}  # item_id -> set of field names
+    # Per-item captured text for semantic checks
+    item_text: dict[str, dict[str, list[str]]] = {}
+    current_field: str | None = None  # field currently being captured (multiline)
+    verification_checkboxes: dict[str, int] = {}
+
+    field_label_re = re.compile(r"^-\s+\*\*([\w\s\-]+?):\*\*\s*(.*)$")
+
+    for line_num, line in enumerate(raw_lines, 1):
+        stripped = line.strip()
+
+        # Category heading
+        cat_match = cat_re.match(stripped)
+        if cat_match:
+            current_cat = cat_match.group(1)
+            categories.add(current_cat)
+            current_field = None
+            continue
+
+        # Item heading
+        item_match = item_re.match(stripped)
+        if item_match:
+            current_item_id = item_match.group(1)
+            item_cat = category_prefix(current_item_id, scheme_name)
+
+            if item_cat not in categories:
+                errors.append(
+                    f"Line {line_num}: Item {current_item_id} belongs to "
+                    f"undeclared category {item_cat}"
+                )
+
+            if current_item_id in item_fields_found:
+                errors.append(f"Line {line_num}: Duplicate item ID {current_item_id}")
+            else:
+                item_fields_found[current_item_id] = set()
+                item_text[current_item_id] = {
+                    "Requirement": [], "Source": [], "Verification": [],
+                    "Evidence": [], "Severity": [],
+                }
+                item_ids.append(current_item_id)
+            current_field = None
+            continue
+
+        # Detect item fields (bold field labels — including multi-word
+        # optional fields like "Risk Domain", "Risk Sub-Domain")
+        if current_item_id:
+            field_match = field_label_re.match(stripped)
+            if field_match:
+                field_name = field_match.group(1).strip()
+                rest = field_match.group(2)
+                item_fields_found[current_item_id].add(field_name)
+                current_field = field_name if field_name in item_text[current_item_id] else None
+                if current_field and rest:
+                    item_text[current_item_id][current_field].append(rest)
+
+                # Validate severity value
+                if field_name == "Severity":
+                    sev_val = re.sub(r"[^a-z]", "", rest.lower())
+                    if sev_val not in valid_severities:
+                        errors.append(
+                            f"Line {line_num}: Invalid severity '{sev_val}' "
+                            f"in item {current_item_id}. "
+                            f"Must be one of: {', '.join(sorted(valid_severities))}"
+                        )
+                continue
+
+            # Continuation of a multi-line field (sub-bullets, wrapped text)
+            if current_field == "Verification" and re.match(r"^[-*]\s*\[[ xX]\]", stripped):
+                verification_checkboxes[current_item_id] = verification_checkboxes.get(current_item_id, 0) + 1
+            if current_field and current_item_id and stripped:
+                # Strip leading list markers / checkbox so token comparison
+                # works on the prose only.
+                clean = re.sub(r"^[-*]\s*\[[ xX]\]\s*", "", stripped)
+                clean = re.sub(r"^[-*]\s+", "", clean)
+                item_text[current_item_id][current_field].append(clean)
+
+    # Check required fields per item
+    for item_id, fields in item_fields_found.items():
+        missing = REQUIRED_ITEM_FIELDS - fields
+        if missing:
+            errors.append(
+                f"Item {item_id}: Missing required fields: {', '.join(sorted(missing))}"
+            )
+
+    # Warn if no categories or items found
+    if not categories:
+        errors.append(f"No categories found (expected '{scheme['cat_hint']}' headings)")
+    if not item_ids:
+        errors.append(f"No checklist items found (expected '{scheme['item_hint']}' headings)")
+
+    # Cross-validate summary counts against actual content
+    if summary:
+        sev_counts: dict[str, int] = {s: 0 for s in scheme["severities"]}
+        for item_id, fields in item_fields_found.items():
+            in_item = False
+            for line in raw_lines:
+                s = line.strip()
+                if item_re.match(s) and item_id in s:
+                    in_item = True
+                    continue
+                if in_item and item_re.match(s):
+                    break
+                if in_item and "**Severity:**" in s:
+                    val = s.split("**Severity:**")[-1].strip().lower()
+                    val = re.sub(r"[^a-z]", "", val)
+                    if val in sev_counts:
+                        sev_counts[val] += 1
+                    break
+
+        if summary.get("categories", 0) != len(categories):
+            errors.append(f"Summary mismatch: categories={summary['categories']} but found {len(categories)}")
+        if summary.get("total_items", 0) != len(item_ids):
+            errors.append(f"Summary mismatch: total_items={summary['total_items']} but found {len(item_ids)}")
+        for sev in scheme["severities"]:
+            if summary.get(sev, 0) != sev_counts[sev]:
+                errors.append(f"Summary mismatch: {sev}={summary.get(sev, 0)} but found {sev_counts[sev]}")
+
+    # Required fields must carry content, and Verification needs a checkbox.
+    for item_id in item_ids:
+        for field in ("Requirement", "Source", "Verification", "Evidence"):
+            if field in item_fields_found.get(item_id, set()) and not "".join(item_text[item_id][field]).strip():
+                errors.append(f"Item {item_id}: field {field} is empty")
+        if "Verification" in item_fields_found.get(item_id, set()) and not verification_checkboxes.get(item_id):
+            errors.append(f"Item {item_id}: Verification has no '- [ ]' steps")
+
+    # --- Semantic checks (warnings only) ---
+    regulation_anchors = set(ANCHOR_RE.findall(meta.get("regulation", "")))
+    source_text_lower: str | None = None
+    if source_path:
+        if not os.path.isfile(source_path):
+            warnings.append(f"--source file not found: {source_path}")
+        else:
+            with open(source_path, "r", encoding="utf-8") as sf:
+                source_text_lower = sf.read().lower()
+
+    for item_id in item_ids:
+        fields = item_text.get(item_id, {})
+        req = " ".join(fields.get("Requirement", [])).strip()
+        src = " ".join(fields.get("Source", [])).strip()
+        verif = " ".join(fields.get("Verification", [])).strip()
+        sev = " ".join(fields.get("Severity", [])).strip().lower()
+        sev = re.sub(r"[^a-z]", "", sev)
+
+        # Weak-modal check: hedge present, mandatory anchor absent.
+        # Imperative-form requirements ("Collect…", "Verify…") pass.
+        # Skip for items explicitly classified as recommended — by
+        # definition they aren't binding obligations.
+        is_recommended = sev == "recommended"
+        if not is_recommended and req and WEAK_RE.search(req) and not MANDATORY_RE.search(req):
+            warnings.append(
+                f"{item_id}: Requirement uses non-binding language "
+                f"(should/may/recommended) with no must/shall anchor — "
+                f"verify this is actually mandatory in the source"
+            )
+
+        # Verification independence
+        if req and verif:
+            req_tok = tokenize(req)
+            verif_tok = tokenize(verif)
+            if len(req_tok) >= JACCARD_MIN_TOKENS and len(verif_tok) >= JACCARD_MIN_TOKENS:
+                score = jaccard(req_tok, verif_tok)
+                if score > JACCARD_THRESHOLD:
+                    warnings.append(
+                        f"{item_id}: Verification overlaps Requirement "
+                        f"(Jaccard={score:.2f}) — verification should list "
+                        f"observable steps, not paraphrase the requirement"
+                    )
+
+        # Citation existence. Anchors that are only the regulation's own
+        # number (Cap. 615, 2015/849) prove nothing about the clause.
+        if source_text_lower is not None and src:
+            anchors = [a for a in ANCHOR_RE.findall(src) if a not in regulation_anchors]
+            if not anchors:
+                warnings.append(
+                    f"{item_id}: Source cites no clause (only the regulation, or no number) — "
+                    f"add the section/article reference"
+                )
+            else:
+                if not any(fuzzy_present(a, source_text_lower) for a in anchors):
+                    preview = ", ".join(anchors[:3])
+                    warnings.append(
+                        f"{item_id}: Source citation not found in source markdown "
+                        f"(anchors checked: {preview})"
+                    )
+
+    # Flattening detector — only meaningful for source-native, where
+    # nested numbering is a structural commitment. cat scheme uses a
+    # synthetic ID that doesn't promise to mirror source bullets.
+    if scheme_name == "source-native":
+        warnings.extend(verification_flattening_warnings(raw_lines, item_re))
+
+    return errors, warnings
+
+
+def main() -> None:
+    p = argparse.ArgumentParser(description=__doc__.split("\n\n")[0])
+    p.add_argument("filepath", help="Path to checklist Markdown file")
+    p.add_argument(
+        "--source",
+        help="Optional path to converted source Markdown for citation, "
+             "modal-verb, and verification-independence warnings",
+    )
+    args = p.parse_args()
+
+    errors, warnings = validate(args.filepath, args.source)
+
+    for w in warnings:
+        print(f"  ⚠️  {w}", file=sys.stderr)
+
+    if errors:
+        print(f"❌ Validation failed for {args.filepath}:", file=sys.stderr)
+        for err in errors:
+            print(f"  • {err}", file=sys.stderr)
+        sys.exit(1)
+    print(f"✅ {args.filepath} is valid.")
+    sys.exit(0)
+
+
+if __name__ == "__main__":
+    main()