novahiz 0.1.0

package/bundled-skills/spec-driven-workflow/scripts/spec_validator.py

@@ -0,0 +1,461 @@
+#!/usr/bin/env python3
+"""
+Spec Validator - Validates a feature specification for completeness and quality.
+
+Checks that a spec document contains all required sections, uses RFC 2119 keywords
+correctly, has acceptance criteria in Given/When/Then format, and scores overall
+completeness from 0-100.
+
+Sections checked:
+- Context, Functional Requirements, Non-Functional Requirements
+- Acceptance Criteria, Edge Cases, API Contracts, Data Models, Out of Scope
+
+Exit codes: 0 = pass, 1 = warnings, 2 = critical (or --strict with score < 80)
+
+No external dependencies - uses only Python standard library.
+"""
+
+import argparse
+import json
+import re
+import sys
+from pathlib import Path
+from typing import Dict, List, Any, Tuple
+
+
+# Section definitions: (key, display_name, required_header_patterns, weight)
+SECTIONS = [
+    ("context", "Context", [r"^##\s+Context"], 10),
+    ("functional_requirements", "Functional Requirements", [r"^##\s+Functional\s+Requirements"], 15),
+    ("non_functional_requirements", "Non-Functional Requirements", [r"^##\s+Non-Functional\s+Requirements"], 10),
+    ("acceptance_criteria", "Acceptance Criteria", [r"^##\s+Acceptance\s+Criteria"], 20),
+    ("edge_cases", "Edge Cases", [r"^##\s+Edge\s+Cases"], 10),
+    ("api_contracts", "API Contracts", [r"^##\s+API\s+Contracts"], 10),
+    ("data_models", "Data Models", [r"^##\s+Data\s+Models"], 10),
+    ("out_of_scope", "Out of Scope", [r"^##\s+Out\s+of\s+Scope"], 10),
+    ("metadata", "Metadata (Author/Date/Status)", [r"\*\*Author:\*\*", r"\*\*Date:\*\*", r"\*\*Status:\*\*"], 5),
+]
+
+RFC_KEYWORDS = ["MUST", "MUST NOT", "SHOULD", "SHOULD NOT", "MAY"]
+
+# Patterns that indicate placeholder/unfilled content
+PLACEHOLDER_PATTERNS = [
+    r"\[your\s+name\]",
+    r"\[list\s+reviewers\]",
+    r"\[describe\s+",
+    r"\[input/condition\]",
+    r"\[precondition\]",
+    r"\[action\]",
+    r"\[expected\s+result\]",
+    r"\[feature/capability\]",
+    r"\[operation\]",
+    r"\[threshold\]",
+    r"\[UI\s+component\]",
+    r"\[service\]",
+    r"\[percentage\]",
+    r"\[number\]",
+    r"\[METHOD\]",
+    r"\[endpoint\]",
+    r"\[Name\]",
+    r"\[Entity\s+Name\]",
+    r"\[type\]",
+    r"\[constraints\]",
+    r"\[field\]",
+    r"\[reason\]",
+]
+
+
+class SpecValidator:
+    """Validates a spec document for completeness and quality."""
+
+    def __init__(self, content: str, file_path: str = ""):
+        self.content = content
+        self.file_path = file_path
+        self.lines = content.split("\n")
+        self.findings: List[Dict[str, Any]] = []
+        self.section_scores: Dict[str, Dict[str, Any]] = {}
+
+    def validate(self) -> Dict[str, Any]:
+        """Run all validation checks and return results."""
+        self._check_sections_present()
+        self._check_functional_requirements()
+        self._check_acceptance_criteria()
+        self._check_edge_cases()
+        self._check_rfc_keywords()
+        self._check_api_contracts()
+        self._check_data_models()
+        self._check_out_of_scope()
+        self._check_placeholders()
+        self._check_traceability()
+
+        total_score = self._calculate_score()
+
+        return {
+            "file": self.file_path,
+            "score": total_score,
+            "grade": self._score_to_grade(total_score),
+            "sections": self.section_scores,
+            "findings": self.findings,
+            "summary": self._build_summary(total_score),
+        }
+
+    def _add_finding(self, severity: str, section: str, message: str):
+        """Record a validation finding."""
+        self.findings.append({
+            "severity": severity,  # "error", "warning", "info"
+            "section": section,
+            "message": message,
+        })
+
+    def _find_section_content(self, header_pattern: str) -> str:
+        """Extract content between a section header and the next ## header."""
+        in_section = False
+        section_lines = []
+        for line in self.lines:
+            if re.match(header_pattern, line, re.IGNORECASE):
+                in_section = True
+                continue
+            if in_section and re.match(r"^##\s+", line):
+                break
+            if in_section:
+                section_lines.append(line)
+        return "\n".join(section_lines)
+
+    def _check_sections_present(self):
+        """Check that all required sections exist."""
+        for key, name, patterns, weight in SECTIONS:
+            found = False
+            for pattern in patterns:
+                for line in self.lines:
+                    if re.search(pattern, line, re.IGNORECASE):
+                        found = True
+                        break
+                if found:
+                    break
+
+            if found:
+                self.section_scores[key] = {"name": name, "present": True, "score": weight, "max": weight}
+            else:
+                self.section_scores[key] = {"name": name, "present": False, "score": 0, "max": weight}
+                self._add_finding("error", key, f"Missing section: {name}")
+
+    def _check_functional_requirements(self):
+        """Validate functional requirements format and content."""
+        content = self._find_section_content(r"^##\s+Functional\s+Requirements")
+        if not content.strip():
+            return
+
+        fr_pattern = re.compile(r"-\s+FR-(\d+):")
+        matches = fr_pattern.findall(content)
+
+        if not matches:
+            self._add_finding("error", "functional_requirements", "No numbered requirements found (expected FR-N: format)")
+            if "functional_requirements" in self.section_scores:
+                self.section_scores["functional_requirements"]["score"] = max(
+                    0, self.section_scores["functional_requirements"]["score"] - 10
+                )
+            return
+
+        fr_count = len(matches)
+        if fr_count < 3:
+            self._add_finding("warning", "functional_requirements", f"Only {fr_count} requirements found. Most features need 3+.")
+
+        # Check for RFC keywords
+        has_keyword = False
+        for kw in RFC_KEYWORDS:
+            if kw in content:
+                has_keyword = True
+                break
+        if not has_keyword:
+            self._add_finding("warning", "functional_requirements", "No RFC 2119 keywords (MUST/SHOULD/MAY) found.")
+
+    def _check_acceptance_criteria(self):
+        """Validate acceptance criteria use Given/When/Then format."""
+        content = self._find_section_content(r"^##\s+Acceptance\s+Criteria")
+        if not content.strip():
+            return
+
+        ac_pattern = re.compile(r"###\s+AC-(\d+):")
+        matches = ac_pattern.findall(content)
+
+        if not matches:
+            self._add_finding("error", "acceptance_criteria", "No numbered acceptance criteria found (expected ### AC-N: format)")
+            if "acceptance_criteria" in self.section_scores:
+                self.section_scores["acceptance_criteria"]["score"] = max(
+                    0, self.section_scores["acceptance_criteria"]["score"] - 15
+                )
+            return
+
+        ac_count = len(matches)
+
+        # Check Given/When/Then
+        given_count = len(re.findall(r"(?i)\bgiven\b", content))
+        when_count = len(re.findall(r"(?i)\bwhen\b", content))
+        then_count = len(re.findall(r"(?i)\bthen\b", content))
+
+        if given_count < ac_count:
+            self._add_finding("warning", "acceptance_criteria",
+                              f"Found {ac_count} criteria but only {given_count} 'Given' clauses. Each AC needs Given/When/Then.")
+        if when_count < ac_count:
+            self._add_finding("warning", "acceptance_criteria",
+                              f"Found {ac_count} criteria but only {when_count} 'When' clauses.")
+        if then_count < ac_count:
+            self._add_finding("warning", "acceptance_criteria",
+                              f"Found {ac_count} criteria but only {then_count} 'Then' clauses.")
+
+        # Check for FR references
+        fr_refs = re.findall(r"\(FR-\d+", content)
+        if not fr_refs:
+            self._add_finding("warning", "acceptance_criteria",
+                              "No acceptance criteria reference functional requirements (expected (FR-N) in title).")
+
+    def _check_edge_cases(self):
+        """Validate edge cases section."""
+        content = self._find_section_content(r"^##\s+Edge\s+Cases")
+        if not content.strip():
+            return
+
+        ec_pattern = re.compile(r"-\s+EC-(\d+):")
+        matches = ec_pattern.findall(content)
+
+        if not matches:
+            self._add_finding("warning", "edge_cases", "No numbered edge cases found (expected EC-N: format)")
+        elif len(matches) < 3:
+            self._add_finding("warning", "edge_cases", f"Only {len(matches)} edge cases. Consider failure modes for each external dependency.")
+
+    def _check_rfc_keywords(self):
+        """Check RFC 2119 keywords are used consistently (capitalized)."""
+        # Look for lowercase must/should/may that might be intended as RFC keywords
+        context_content = self._find_section_content(r"^##\s+Functional\s+Requirements")
+        context_content += self._find_section_content(r"^##\s+Non-Functional\s+Requirements")
+
+        for kw in ["must", "should", "may"]:
+            # Find lowercase usage in requirement-like sentences
+            pattern = rf"(?:system|service|API|endpoint)\s+{kw}\s+"
+            if re.search(pattern, context_content):
+                self._add_finding("warning", "rfc_keywords",
+                                  f"Found lowercase '{kw}' in requirements. RFC 2119 keywords should be UPPERCASE: {kw.upper()}")
+
+    def _check_api_contracts(self):
+        """Validate API contracts section."""
+        content = self._find_section_content(r"^##\s+API\s+Contracts")
+        if not content.strip():
+            return
+
+        # Check for at least one endpoint definition
+        has_endpoint = bool(re.search(r"(GET|POST|PUT|PATCH|DELETE)\s+/", content))
+        if not has_endpoint:
+            self._add_finding("warning", "api_contracts", "No HTTP method + path found (expected e.g., POST /api/endpoint)")
+
+        # Check for request/response definitions
+        has_interface = bool(re.search(r"interface\s+\w+", content))
+        if not has_interface:
+            self._add_finding("info", "api_contracts", "No TypeScript interfaces found. Consider defining request/response shapes.")
+
+    def _check_data_models(self):
+        """Validate data models section."""
+        content = self._find_section_content(r"^##\s+Data\s+Models")
+        if not content.strip():
+            return
+
+        # Check for table format
+        has_table = bool(re.search(r"\|.*\|.*\|", content))
+        if not has_table:
+            self._add_finding("warning", "data_models", "No table-formatted data models found. Use | Field | Type | Constraints | format.")
+
+    def _check_out_of_scope(self):
+        """Validate out of scope section."""
+        content = self._find_section_content(r"^##\s+Out\s+of\s+Scope")
+        if not content.strip():
+            return
+
+        os_pattern = re.compile(r"-\s+OS-(\d+):")
+        matches = os_pattern.findall(content)
+
+        if not matches:
+            self._add_finding("warning", "out_of_scope", "No numbered exclusions found (expected OS-N: format)")
+        elif len(matches) < 2:
+            self._add_finding("info", "out_of_scope", "Only 1 exclusion listed. Consider what was deliberately left out.")
+
+    def _check_placeholders(self):
+        """Check for unfilled placeholder text."""
+        placeholder_count = 0
+        for pattern in PLACEHOLDER_PATTERNS:
+            matches = re.findall(pattern, self.content, re.IGNORECASE)
+            placeholder_count += len(matches)
+
+        if placeholder_count > 0:
+            self._add_finding("warning", "placeholders",
+                              f"Found {placeholder_count} placeholder(s) that need to be filled in (e.g., [your name], [describe ...]).")
+            # Deduct from overall score proportionally
+            for key in self.section_scores:
+                if self.section_scores[key]["present"]:
+                    deduction = min(3, self.section_scores[key]["score"])
+                    self.section_scores[key]["score"] = max(0, self.section_scores[key]["score"] - deduction)
+
+    def _check_traceability(self):
+        """Check that acceptance criteria reference functional requirements."""
+        ac_content = self._find_section_content(r"^##\s+Acceptance\s+Criteria")
+        fr_content = self._find_section_content(r"^##\s+Functional\s+Requirements")
+
+        if not ac_content.strip() or not fr_content.strip():
+            return
+
+        # Extract FR IDs
+        fr_ids = set(re.findall(r"FR-(\d+)", fr_content))
+        # Extract FR references from AC
+        ac_fr_refs = set(re.findall(r"FR-(\d+)", ac_content))
+
+        unreferenced = fr_ids - ac_fr_refs
+        if unreferenced:
+            unreferenced_list = ", ".join(f"FR-{i}" for i in sorted(unreferenced))
+            self._add_finding("warning", "traceability",
+                              f"Functional requirements without acceptance criteria: {unreferenced_list}")
+
+    def _calculate_score(self) -> int:
+        """Calculate the total completeness score."""
+        total = sum(s["score"] for s in self.section_scores.values())
+        maximum = sum(s["max"] for s in self.section_scores.values())
+
+        if maximum == 0:
+            return 0
+
+        # Apply finding-based deductions
+        error_count = sum(1 for f in self.findings if f["severity"] == "error")
+        warning_count = sum(1 for f in self.findings if f["severity"] == "warning")
+
+        base_score = round((total / maximum) * 100)
+        deduction = (error_count * 5) + (warning_count * 2)
+
+        return max(0, min(100, base_score - deduction))
+
+    @staticmethod
+    def _score_to_grade(score: int) -> str:
+        """Convert score to letter grade."""
+        if score >= 90:
+            return "A"
+        if score >= 80:
+            return "B"
+        if score >= 70:
+            return "C"
+        if score >= 60:
+            return "D"
+        return "F"
+
+    def _build_summary(self, score: int) -> str:
+        """Build human-readable summary."""
+        errors = [f for f in self.findings if f["severity"] == "error"]
+        warnings = [f for f in self.findings if f["severity"] == "warning"]
+        infos = [f for f in self.findings if f["severity"] == "info"]
+
+        lines = [
+            f"Spec Completeness Score: {score}/100 (Grade: {self._score_to_grade(score)})",
+            f"Errors: {len(errors)}, Warnings: {len(warnings)}, Info: {len(infos)}",
+            "",
+        ]
+
+        if errors:
+            lines.append("ERRORS (must fix):")
+            for e in errors:
+                lines.append(f"  [{e['section']}] {e['message']}")
+            lines.append("")
+
+        if warnings:
+            lines.append("WARNINGS (should fix):")
+            for w in warnings:
+                lines.append(f"  [{w['section']}] {w['message']}")
+            lines.append("")
+
+        if infos:
+            lines.append("INFO:")
+            for i in infos:
+                lines.append(f"  [{i['section']}] {i['message']}")
+            lines.append("")
+
+        # Section breakdown
+        lines.append("Section Breakdown:")
+        for key, data in self.section_scores.items():
+            status = "PRESENT" if data["present"] else "MISSING"
+            lines.append(f"  {data['name']}: {data['score']}/{data['max']} ({status})")
+
+        return "\n".join(lines)
+
+
+def format_human(result: Dict[str, Any]) -> str:
+    """Format validation result for human reading."""
+    lines = [
+        "=" * 60,
+        "SPEC VALIDATION REPORT",
+        "=" * 60,
+        "",
+    ]
+    if result["file"]:
+        lines.append(f"File: {result['file']}")
+        lines.append("")
+
+    lines.append(result["summary"])
+
+    return "\n".join(lines)
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        description="Validate a feature specification for completeness and quality.",
+        epilog="Example: python spec_validator.py --file spec.md --strict",
+    )
+    parser.add_argument(
+        "--file",
+        "-f",
+        required=True,
+        help="Path to the spec markdown file",
+    )
+    parser.add_argument(
+        "--strict",
+        action="store_true",
+        help="Exit with code 2 if score is below 80",
+    )
+    parser.add_argument(
+        "--json",
+        action="store_true",
+        dest="json_flag",
+        help="Output results as JSON",
+    )
+
+    args = parser.parse_args()
+
+    file_path = Path(args.file)
+    if not file_path.exists():
+        print(f"Error: File not found: {file_path}", file=sys.stderr)
+        sys.exit(2)
+
+    content = file_path.read_text(encoding="utf-8")
+
+    if not content.strip():
+        print(f"Error: File is empty: {file_path}", file=sys.stderr)
+        sys.exit(2)
+
+    validator = SpecValidator(content, str(file_path))
+    result = validator.validate()
+
+    if args.json_flag:
+        print(json.dumps(result, indent=2))
+    else:
+        print(format_human(result))
+
+    # Determine exit code
+    score = result["score"]
+    has_errors = any(f["severity"] == "error" for f in result["findings"])
+    has_warnings = any(f["severity"] == "warning" for f in result["findings"])
+
+    if args.strict and score < 80:
+        sys.exit(2)
+    elif has_errors:
+        sys.exit(2)
+    elif has_warnings:
+        sys.exit(1)
+    else:
+        sys.exit(0)
+
+
+if __name__ == "__main__":
+    main()