@mcptoolshop/accessibility-suite 0.1.0

package/src/a11y-lint/a11y_lint/scorecard.py

@@ -0,0 +1,244 @@
+"""Scorecard builder for accessibility assessments.
+
+Generates scorecards summarizing accessibility check results.
+
+Philosophy:
+- Grades (A-F) are DERIVED summaries for executive reporting
+- Grades are NEVER primary - CI gates should be based on:
+  - Specific rule failures (especially WCAG-mapped rules)
+  - Error count thresholds
+  - Regressions from baseline
+- Grades compress nuance; always check underlying rule failures
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import Any
+
+from .errors import A11yMessage, Level
+
+
+@dataclass
+class RuleScore:
+    """Score for a single rule."""
+
+    rule: str
+    passed: int = 0
+    warnings: int = 0
+    errors: int = 0
+
+    @property
+    def total(self) -> int:
+        """Total number of checks for this rule."""
+        return self.passed + self.warnings + self.errors
+
+    @property
+    def score(self) -> float:
+        """Score as a percentage (0-100)."""
+        if self.total == 0:
+            return 100.0
+        # Passed = full points, warnings = half points, errors = no points
+        points = self.passed + (self.warnings * 0.5)
+        return (points / self.total) * 100
+
+    @property
+    def grade(self) -> str:
+        """Letter grade based on score."""
+        s = self.score
+        if s >= 90:
+            return "A"
+        elif s >= 80:
+            return "B"
+        elif s >= 70:
+            return "C"
+        elif s >= 60:
+            return "D"
+        else:
+            return "F"
+
+
+@dataclass
+class Scorecard:
+    """Accessibility scorecard summarizing check results."""
+
+    name: str
+    rule_scores: dict[str, RuleScore] = field(default_factory=dict)
+    messages: list[A11yMessage] = field(default_factory=list)
+
+    def add_message(self, message: A11yMessage) -> None:
+        """Add a message to the scorecard."""
+        self.messages.append(message)
+
+        # Get or create rule score
+        rule_name = message.rule or "unknown"
+        if rule_name not in self.rule_scores:
+            self.rule_scores[rule_name] = RuleScore(rule=rule_name)
+
+        score = self.rule_scores[rule_name]
+        if message.level == Level.OK:
+            score.passed += 1
+        elif message.level == Level.WARN:
+            score.warnings += 1
+        elif message.level == Level.ERROR:
+            score.errors += 1
+
+    def add_messages(self, messages: list[A11yMessage]) -> None:
+        """Add multiple messages to the scorecard."""
+        for msg in messages:
+            self.add_message(msg)
+
+    @property
+    def total_passed(self) -> int:
+        """Total number of passed checks."""
+        return sum(s.passed for s in self.rule_scores.values())
+
+    @property
+    def total_warnings(self) -> int:
+        """Total number of warnings."""
+        return sum(s.warnings for s in self.rule_scores.values())
+
+    @property
+    def total_errors(self) -> int:
+        """Total number of errors."""
+        return sum(s.errors for s in self.rule_scores.values())
+
+    @property
+    def total_checks(self) -> int:
+        """Total number of checks performed."""
+        return self.total_passed + self.total_warnings + self.total_errors
+
+    @property
+    def overall_score(self) -> float:
+        """Overall accessibility score (0-100)."""
+        if self.total_checks == 0:
+            return 100.0
+        points = self.total_passed + (self.total_warnings * 0.5)
+        return (points / self.total_checks) * 100
+
+    @property
+    def overall_grade(self) -> str:
+        """Overall letter grade."""
+        s = self.overall_score
+        if s >= 90:
+            return "A"
+        elif s >= 80:
+            return "B"
+        elif s >= 70:
+            return "C"
+        elif s >= 60:
+            return "D"
+        else:
+            return "F"
+
+    @property
+    def is_passing(self) -> bool:
+        """Check if the scorecard represents a passing assessment."""
+        return self.total_errors == 0
+
+    def summary(self) -> str:
+        """Get a text summary of the scorecard."""
+        lines = [
+            f"Accessibility Scorecard: {self.name}",
+            "=" * 40,
+            f"Overall Score: {self.overall_score:.1f}% ({self.overall_grade})",
+            f"Total Checks: {self.total_checks}",
+            f"  Passed: {self.total_passed}",
+            f"  Warnings: {self.total_warnings}",
+            f"  Errors: {self.total_errors}",
+            "",
+            "By Rule:",
+        ]
+
+        for rule_name, score in sorted(self.rule_scores.items()):
+            lines.append(
+                f"  {rule_name}: {score.score:.1f}% ({score.grade}) "
+                f"[{score.passed}P/{score.warnings}W/{score.errors}E]"
+            )
+
+        return "\n".join(lines)
+
+    def to_dict(self) -> dict[str, Any]:
+        """Convert to dictionary for JSON serialization."""
+        return {
+            "name": self.name,
+            "overall_score": round(self.overall_score, 2),
+            "overall_grade": self.overall_grade,
+            "is_passing": self.is_passing,
+            "totals": {
+                "checks": self.total_checks,
+                "passed": self.total_passed,
+                "warnings": self.total_warnings,
+                "errors": self.total_errors,
+            },
+            "rules": {
+                name: {
+                    "score": round(score.score, 2),
+                    "grade": score.grade,
+                    "passed": score.passed,
+                    "warnings": score.warnings,
+                    "errors": score.errors,
+                }
+                for name, score in self.rule_scores.items()
+            },
+            "messages": [msg.to_dict() for msg in self.messages],
+        }
+
+
+class ScorecardBuilder:
+    """Builder for creating scorecards from scan results."""
+
+    def __init__(self, name: str = "CLI Accessibility Assessment") -> None:
+        """Initialize the builder.
+
+        Args:
+            name: Name for the scorecard
+        """
+        self.scorecard = Scorecard(name=name)
+
+    def add_scan_result(self, messages: list[A11yMessage]) -> ScorecardBuilder:
+        """Add scan results to the scorecard.
+
+        Args:
+            messages: Messages from scanning
+
+        Returns:
+            Self for chaining
+        """
+        self.scorecard.add_messages(messages)
+        return self
+
+    def add_ok_check(self, rule: str, code: str, what: str) -> ScorecardBuilder:
+        """Record a passing check.
+
+        Args:
+            rule: Rule name
+            code: Error code
+            what: Description of what was checked
+
+        Returns:
+            Self for chaining
+        """
+        self.scorecard.add_message(A11yMessage.ok(code, what, rule=rule))
+        return self
+
+    def build(self) -> Scorecard:
+        """Build and return the scorecard."""
+        return self.scorecard
+
+
+def create_scorecard(
+    messages: list[A11yMessage], name: str = "CLI Accessibility Assessment"
+) -> Scorecard:
+    """Convenience function to create a scorecard from messages.
+
+    Args:
+        messages: Messages to include
+        name: Name for the scorecard
+
+    Returns:
+        Built scorecard
+    """
+    builder = ScorecardBuilder(name)
+    builder.add_scan_result(messages)
+    return builder.build()

package/src/a11y-lint/a11y_lint/validate.py

@@ -0,0 +1,225 @@
+"""Schema validation for CLI accessibility messages.
+
+Validates that messages conform to the ground truth schema.
+"""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+from typing import Any
+
+import jsonschema
+from jsonschema import Draft202012Validator, ValidationError
+
+from .errors import A11yMessage, Level
+
+# Path to the schema file
+SCHEMA_PATH = Path(__file__).parent / "schemas" / "cli.error.schema.v0.1.json"
+
+
+def load_schema() -> dict[str, Any]:
+    """Load the CLI error schema from disk."""
+    with open(SCHEMA_PATH, "r", encoding="utf-8") as f:
+        return json.load(f)
+
+
+# Cache the schema and validator
+_schema: dict[str, Any] | None = None
+_validator: Draft202012Validator | None = None
+
+
+def get_validator() -> Draft202012Validator:
+    """Get or create the cached schema validator."""
+    global _schema, _validator
+    if _validator is None:
+        _schema = load_schema()
+        _validator = Draft202012Validator(_schema)
+    return _validator
+
+
+def validate_dict(data: dict[str, Any]) -> list[str]:
+    """Validate a dictionary against the CLI error schema.
+
+    Args:
+        data: Dictionary to validate
+
+    Returns:
+        List of validation error messages (empty if valid)
+    """
+    validator = get_validator()
+    errors = []
+
+    for error in validator.iter_errors(data):
+        # Build a human-readable path to the error
+        path = ".".join(str(p) for p in error.path) if error.path else "root"
+        errors.append(f"{path}: {error.message}")
+
+    return errors
+
+
+def validate_message(message: A11yMessage) -> list[str]:
+    """Validate an A11yMessage against the schema.
+
+    Args:
+        message: Message to validate
+
+    Returns:
+        List of validation error messages (empty if valid)
+    """
+    return validate_dict(message.to_dict())
+
+
+def is_valid(data: dict[str, Any] | A11yMessage) -> bool:
+    """Check if data is valid against the schema.
+
+    Args:
+        data: Dictionary or A11yMessage to validate
+
+    Returns:
+        True if valid, False otherwise
+    """
+    if isinstance(data, A11yMessage):
+        data = data.to_dict()
+    return len(validate_dict(data)) == 0
+
+
+def validate_json_file(path: Path | str) -> tuple[list[dict[str, Any]], list[str]]:
+    """Validate a JSON file containing messages.
+
+    The file can contain either a single message object or an array of messages.
+
+    Args:
+        path: Path to the JSON file
+
+    Returns:
+        Tuple of (valid messages, validation errors)
+    """
+    path = Path(path)
+    errors: list[str] = []
+
+    try:
+        with open(path, "r", encoding="utf-8") as f:
+            data = json.load(f)
+    except json.JSONDecodeError as e:
+        return [], [f"Invalid JSON: {e}"]
+    except FileNotFoundError:
+        return [], [f"File not found: {path}"]
+
+    # Handle both single object and array
+    if isinstance(data, dict):
+        messages = [data]
+    elif isinstance(data, list):
+        messages = data
+    else:
+        return [], ["JSON must be an object or array of objects"]
+
+    valid_messages = []
+    for i, msg in enumerate(messages):
+        if not isinstance(msg, dict):
+            errors.append(f"Message {i}: expected object, got {type(msg).__name__}")
+            continue
+
+        msg_errors = validate_dict(msg)
+        if msg_errors:
+            for err in msg_errors:
+                errors.append(f"Message {i}: {err}")
+        else:
+            valid_messages.append(msg)
+
+    return valid_messages, errors
+
+
+def validate_and_convert(data: dict[str, Any]) -> A11yMessage | list[str]:
+    """Validate and convert a dictionary to an A11yMessage.
+
+    Args:
+        data: Dictionary to validate and convert
+
+    Returns:
+        A11yMessage if valid, list of errors otherwise
+    """
+    errors = validate_dict(data)
+    if errors:
+        return errors
+
+    try:
+        return A11yMessage.from_dict(data)
+    except (ValueError, KeyError) as e:
+        return [str(e)]
+
+
+class MessageValidator:
+    """Validator for batches of messages with summary statistics."""
+
+    def __init__(self) -> None:
+        self.valid_count = 0
+        self.invalid_count = 0
+        self.errors: list[tuple[int, list[str]]] = []
+        self.messages: list[A11yMessage] = []
+
+    def validate(self, data: dict[str, Any], index: int = 0) -> bool:
+        """Validate a single message.
+
+        Args:
+            data: Message dictionary to validate
+            index: Index for error reporting
+
+        Returns:
+            True if valid, False otherwise
+        """
+        result = validate_and_convert(data)
+        if isinstance(result, list):
+            self.invalid_count += 1
+            self.errors.append((index, result))
+            return False
+        else:
+            self.valid_count += 1
+            self.messages.append(result)
+            return True
+
+    def validate_batch(self, messages: list[dict[str, Any]]) -> None:
+        """Validate a batch of messages.
+
+        Args:
+            messages: List of message dictionaries to validate
+        """
+        for i, msg in enumerate(messages):
+            self.validate(msg, i)
+
+    @property
+    def is_all_valid(self) -> bool:
+        """Check if all validated messages were valid."""
+        return self.invalid_count == 0
+
+    @property
+    def total_count(self) -> int:
+        """Get total number of validated messages."""
+        return self.valid_count + self.invalid_count
+
+    def summary(self) -> str:
+        """Get a summary of validation results."""
+        total = self.total_count
+        if total == 0:
+            return "No messages validated"
+
+        if self.is_all_valid:
+            return f"All {total} messages valid"
+
+        return (
+            f"Validated {total} messages: "
+            f"{self.valid_count} valid, {self.invalid_count} invalid"
+        )
+
+    def error_report(self) -> str:
+        """Get detailed error report."""
+        if not self.errors:
+            return "No errors"
+
+        lines = [f"Found {len(self.errors)} invalid messages:"]
+        for index, errs in self.errors:
+            lines.append(f"\n  Message {index}:")
+            for err in errs:
+                lines.append(f"    - {err}")
+
+        return "\n".join(lines)

package/src/a11y-lint/pyproject.toml

@@ -0,0 +1,75 @@
+[build-system]
+requires = ["setuptools>=61.0", "wheel"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "a11y-lint"
+version = "0.1.0"
+description = "Accessibility linter for CLI output - validates error messages follow accessible patterns"
+readme = "README.md"
+requires-python = ">=3.10"
+license = {text = "MIT"}
+authors = [
+    { name = "mcp-tool-shop", email = "64996768+mcp-tool-shop@users.noreply.github.com" }
+]
+keywords = [
+    "accessibility",
+    "a11y",
+    "cli",
+    "linter",
+    "validation",
+    "wcag",
+    "low-vision",
+    "error-messages",
+    "testing",
+]
+classifiers = [
+    "Development Status :: 3 - Alpha",
+    "Environment :: Console",
+    "Intended Audience :: Developers",
+    "License :: OSI Approved :: MIT License",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+    "Topic :: Software Development :: Quality Assurance",
+    "Topic :: Software Development :: Testing",
+]
+dependencies = [
+    "jsonschema>=4.20.0",
+    "click>=8.1.0",
+    "rich>=13.0.0",
+]
+
+[project.optional-dependencies]
+dev = [
+    "pytest>=7.4.0",
+    "pytest-cov>=4.1.0",
+    "pyright>=1.1.350",
+]
+
+[project.scripts]
+a11y-lint = "a11y_lint.cli:main"
+
+[project.urls]
+Homepage = "https://github.com/mcp-tool-shop-org/a11y-lint"
+Repository = "https://github.com/mcp-tool-shop-org/a11y-lint"
+Issues = "https://github.com/mcp-tool-shop-org/a11y-lint/issues"
+
+[tool.setuptools.packages.find]
+where = ["."]
+include = ["a11y_lint*"]
+
+[tool.setuptools.package-data]
+a11y_lint = ["schemas/*.json"]
+
+[tool.pytest.ini_options]
+testpaths = ["tests"]
+python_files = ["test_*.py"]
+python_functions = ["test_*"]
+addopts = "-v --tb=short"
+
+[tool.pyright]
+include = ["a11y_lint", "tests"]
+pythonVersion = "3.10"
+typeCheckingMode = "standard"

package/src/a11y-lint/tests/__init__.py

@@ -0,0 +1 @@
+"""Tests for a11y-lint."""