@mcptoolshop/accessibility-suite 0.1.0

package/src/a11y-assist/tests/test_ingest.py

@@ -0,0 +1,311 @@
+"""Tests for the ingest command."""
+
+from __future__ import annotations
+
+import json
+import shutil
+from pathlib import Path
+
+import pytest
+
+from a11y_assist.ingest import (
+    IngestError,
+    build_advisories,
+    canonicalize,
+    group_by_file,
+    group_by_rule,
+    ingest,
+    load_findings,
+    verify_provenance,
+    write_advisories,
+    write_ingest_summary,
+)
+
+
+@pytest.fixture
+def sample_findings(tmp_path: Path) -> Path:
+    """Create a sample findings.json for testing."""
+    findings = {
+        "engine": "a11y-evidence-engine",
+        "version": "0.1.0",
+        "target": {"path": "./test-html"},
+        "summary": {"files_scanned": 2, "errors": 3, "warnings": 1, "info": 0},
+        "findings": [
+            {
+                "finding_id": "finding-0001",
+                "rule_id": "html.img.missing_alt",
+                "severity": "error",
+                "confidence": 0.98,
+                "message": "Image element is missing alt text.",
+                "location": {"file": "index.html", "json_pointer": "/nodes/5"},
+                "evidence_ref": {
+                    "record": "provenance/finding-0001/record.json",
+                    "digest": "provenance/finding-0001/digest.json",
+                    "envelope": "provenance/finding-0001/envelope.json",
+                },
+            },
+            {
+                "finding_id": "finding-0002",
+                "rule_id": "html.img.missing_alt",
+                "severity": "error",
+                "confidence": 0.98,
+                "message": "Image element is missing alt text.",
+                "location": {"file": "index.html", "json_pointer": "/nodes/8"},
+                "evidence_ref": {
+                    "record": "provenance/finding-0002/record.json",
+                    "digest": "provenance/finding-0002/digest.json",
+                    "envelope": "provenance/finding-0002/envelope.json",
+                },
+            },
+            {
+                "finding_id": "finding-0003",
+                "rule_id": "html.form_control.missing_label",
+                "severity": "error",
+                "confidence": 0.95,
+                "message": "Form control is missing an associated label.",
+                "location": {"file": "form.html", "json_pointer": "/nodes/3"},
+                "evidence_ref": {
+                    "record": "provenance/finding-0003/record.json",
+                    "digest": "provenance/finding-0003/digest.json",
+                    "envelope": "provenance/finding-0003/envelope.json",
+                },
+            },
+            {
+                "finding_id": "finding-0004",
+                "rule_id": "html.document.missing_lang",
+                "severity": "warning",
+                "confidence": 1.0,
+                "message": "Document is missing lang attribute.",
+                "location": {"file": "form.html", "json_pointer": "/nodes/0"},
+                "evidence_ref": {
+                    "record": "provenance/finding-0004/record.json",
+                    "digest": "provenance/finding-0004/digest.json",
+                    "envelope": "provenance/finding-0004/envelope.json",
+                },
+            },
+        ],
+    }
+
+    findings_path = tmp_path / "findings.json"
+    findings_path.write_text(json.dumps(findings, indent=2))
+    return findings_path
+
+
+@pytest.fixture
+def findings_with_provenance(tmp_path: Path, sample_findings: Path) -> Path:
+    """Create findings with valid provenance bundles."""
+    # Create provenance directories
+    for i in range(1, 5):
+        prov_dir = tmp_path / f"provenance/finding-000{i}"
+        prov_dir.mkdir(parents=True, exist_ok=True)
+
+        # Evidence content
+        evidence = {
+            "document_ref": "test.html",
+            "pointer": f"/nodes/{i}",
+            "evidence": {"tagName": "img", "attrs": {}},
+        }
+
+        # Compute digest
+        canonical = canonicalize(evidence)
+        import hashlib
+
+        digest_value = hashlib.sha256(canonical.encode("utf-8")).hexdigest()
+
+        # Write record.json
+        record = {
+            "prov.record.v0.1": {
+                "method_id": "engine.extract.evidence.json_pointer",
+                "timestamp": "2026-01-26T00:00:00Z",
+                "inputs": [],
+                "outputs": [{"artifact.v0.1": {"name": "evidence", "content": evidence}}],
+                "agent": {"name": "test", "version": "1.0"},
+            }
+        }
+        (prov_dir / "record.json").write_text(json.dumps(record))
+
+        # Write digest.json
+        digest = {
+            "prov.record.v0.1": {
+                "method_id": "integrity.digest.sha256",
+                "timestamp": "2026-01-26T00:00:00Z",
+                "inputs": [],
+                "outputs": [
+                    {
+                        "artifact.v0.1": {
+                            "name": "digest",
+                            "digest": {"algorithm": "sha256", "value": digest_value},
+                        }
+                    }
+                ],
+                "agent": {"name": "test", "version": "1.0"},
+            }
+        }
+        (prov_dir / "digest.json").write_text(json.dumps(digest))
+
+        # Write envelope.json
+        envelope = {"mcp.envelope.v0.1": {"result": {}, "provenance": {}}}
+        (prov_dir / "envelope.json").write_text(json.dumps(envelope))
+
+    return sample_findings
+
+
+class TestLoadFindings:
+    def test_load_valid_findings(self, sample_findings: Path):
+        data = load_findings(sample_findings)
+        assert data["engine"] == "a11y-evidence-engine"
+        assert len(data["findings"]) == 4
+
+    def test_load_missing_file(self, tmp_path: Path):
+        with pytest.raises(IngestError, match="not found"):
+            load_findings(tmp_path / "nonexistent.json")
+
+    def test_load_invalid_json(self, tmp_path: Path):
+        bad_file = tmp_path / "bad.json"
+        bad_file.write_text("not json")
+        with pytest.raises(IngestError, match="Invalid JSON"):
+            load_findings(bad_file)
+
+    def test_load_missing_required_fields(self, tmp_path: Path):
+        incomplete = tmp_path / "incomplete.json"
+        incomplete.write_text('{"engine": "test"}')
+        with pytest.raises(IngestError, match="Missing required"):
+            load_findings(incomplete)
+
+
+class TestCanonicalize:
+    def test_sorted_keys(self):
+        assert canonicalize({"z": 1, "a": 2}) == '{"a":2,"z":1}'
+
+    def test_nested_objects(self):
+        obj = {"b": {"d": 1, "c": 2}, "a": 1}
+        assert canonicalize(obj) == '{"a":1,"b":{"c":2,"d":1}}'
+
+    def test_arrays_preserve_order(self):
+        assert canonicalize([3, 1, 2]) == "[3,1,2]"
+
+    def test_null_and_booleans(self):
+        assert canonicalize(None) == "null"
+        assert canonicalize(True) == "true"
+        assert canonicalize(False) == "false"
+
+    def test_strings_escaped(self):
+        assert canonicalize('hello "world"') == '"hello \\"world\\""'
+
+
+class TestGroupByRule:
+    def test_groups_correctly(self, sample_findings: Path):
+        data = load_findings(sample_findings)
+        grouped = group_by_rule(data["findings"])
+
+        assert len(grouped) == 3
+        # Should be sorted by count descending
+        assert grouped[0]["rule_id"] == "html.img.missing_alt"
+        assert grouped[0]["count"] == 2
+
+
+class TestGroupByFile:
+    def test_groups_correctly(self, sample_findings: Path):
+        data = load_findings(sample_findings)
+        grouped = group_by_file(data["findings"])
+
+        assert len(grouped) == 2
+        # index.html has 2 errors, form.html has 1 error + 1 warning
+        assert grouped[0]["file"] == "index.html"
+        assert grouped[0]["errors"] == 2
+
+
+class TestBuildAdvisories:
+    def test_builds_advisories(self, sample_findings: Path):
+        data = load_findings(sample_findings)
+        advisories = build_advisories(data["findings"])
+
+        assert len(advisories) == 3
+        # First advisory should be for most common rule
+        assert advisories[0]["rule_id"] == "html.img.missing_alt"
+        assert len(advisories[0]["instances"]) == 2
+        assert advisories[0]["title"] == "Add alt text to images"
+
+
+class TestIngest:
+    def test_basic_ingest(self, sample_findings: Path):
+        result = ingest(sample_findings)
+
+        assert result.source_engine == "a11y-evidence-engine"
+        assert result.source_version == "0.1.0"
+        assert len(result.findings) == 4
+        assert len(result.by_rule) == 3
+
+    def test_filter_by_severity(self, sample_findings: Path):
+        result = ingest(sample_findings, min_severity="error")
+
+        # Should exclude the warning
+        assert len(result.findings) == 3
+
+    def test_provenance_verification_missing_files(self, sample_findings: Path):
+        result = ingest(sample_findings, verify_provenance_flag=True)
+
+        # Should have errors because provenance files don't exist
+        assert not result.provenance_verified
+        assert len(result.provenance_errors) > 0
+
+    def test_provenance_verification_success(self, findings_with_provenance: Path):
+        result = ingest(findings_with_provenance, verify_provenance_flag=True)
+
+        assert result.provenance_verified
+        assert len(result.provenance_errors) == 0
+
+
+class TestWriteOutputs:
+    def test_write_summary(self, sample_findings: Path, tmp_path: Path):
+        result = ingest(sample_findings)
+        out_path = tmp_path / "output" / "ingest-summary.json"
+
+        write_ingest_summary(result, out_path)
+
+        assert out_path.exists()
+        data = json.loads(out_path.read_text())
+        assert data["source_engine"] == "a11y-evidence-engine"
+        assert "by_rule" in data
+
+    def test_write_advisories(self, sample_findings: Path, tmp_path: Path):
+        result = ingest(sample_findings)
+        out_path = tmp_path / "output" / "advisories.json"
+
+        write_advisories(result, out_path)
+
+        assert out_path.exists()
+        data = json.loads(out_path.read_text())
+        assert data["schema"] == "a11y-assist/advisories@v0.1"
+        assert len(data["advisories"]) == 3
+
+
+class TestVerifyProvenance:
+    def test_missing_evidence_ref(self, tmp_path: Path):
+        finding = {"finding_id": "test-001"}
+        success, error = verify_provenance(finding, tmp_path)
+
+        assert not success
+        assert "Missing evidence_ref" in error
+
+    def test_missing_file(self, tmp_path: Path):
+        finding = {
+            "finding_id": "test-001",
+            "evidence_ref": {
+                "record": "provenance/record.json",
+                "digest": "provenance/digest.json",
+                "envelope": "provenance/envelope.json",
+            },
+        }
+        success, error = verify_provenance(finding, tmp_path)
+
+        assert not success
+        assert "not found" in error
+
+    def test_valid_provenance(self, findings_with_provenance: Path):
+        data = load_findings(findings_with_provenance)
+        base_dir = findings_with_provenance.parent
+
+        for finding in data["findings"]:
+            success, error = verify_provenance(finding, base_dir)
+            assert success, f"Failed for {finding['finding_id']}: {error}"

package/src/a11y-assist/tests/test_methods_metadata.py

@@ -0,0 +1,236 @@
+"""Tests for methods metadata (audit-only fields).
+
+These tests verify:
+1. Metadata is populated deterministically
+2. Metadata does not affect rendering output
+3. Evidence anchors are correct
+"""
+
+from pathlib import Path
+
+import pytest
+
+from a11y_assist.from_cli_error import assist_from_cli_error, load_cli_error
+from a11y_assist.methods import (
+    METHOD_GUARD_VALIDATE,
+    METHOD_NORMALIZE_CLI_ERROR,
+    METHOD_NORMALIZE_RAW_TEXT,
+    METHOD_PROFILE_COGNITIVE_LOAD,
+    METHOD_PROFILE_DYSLEXIA,
+    METHOD_PROFILE_LOWVISION,
+    METHOD_PROFILE_PLAIN_LANGUAGE,
+    METHOD_PROFILE_SCREEN_READER,
+    with_method,
+    with_methods,
+)
+from a11y_assist.profiles import (
+    apply_cognitive_load,
+    apply_screen_reader,
+    render_cognitive_load,
+)
+from a11y_assist.render import AssistResult, Evidence, render_assist
+
+FIXTURES_DIR = Path(__file__).parent / "fixtures"
+
+
+class TestCliErrorMetadata:
+    """Tests for metadata from cli.error.v0.1 path."""
+
+    @pytest.fixture
+    def result(self):
+        """Load high-confidence fixture and get AssistResult."""
+        obj = load_cli_error(str(FIXTURES_DIR / "base_inputs" / "cli_error_high.json"))
+        return assist_from_cli_error(obj)
+
+    def test_methods_applied_contains_normalize(self, result):
+        """Result should include normalization method."""
+        assert METHOD_NORMALIZE_CLI_ERROR in result.methods_applied
+
+    def test_evidence_for_safest_next_step(self, result):
+        """Evidence should include safest_next_step source."""
+        fields = [e.field for e in result.evidence]
+        assert "safest_next_step" in fields
+
+        # Find the evidence for safest_next_step
+        for e in result.evidence:
+            if e.field == "safest_next_step":
+                assert e.source.startswith("cli.error.")
+                break
+
+    def test_evidence_for_plan_steps(self, result):
+        """Evidence should include plan step sources."""
+        plan_evidence = [e for e in result.evidence if e.field.startswith("plan[")]
+        # Should have evidence for each plan step
+        assert len(plan_evidence) == len(result.plan)
+
+        # Each should reference cli.error.fix
+        for e in plan_evidence:
+            assert "cli.error.fix" in e.source
+
+    def test_evidence_for_safe_commands(self, result):
+        """Evidence should include safe command sources."""
+        if result.next_safe_commands:
+            cmd_evidence = [
+                e for e in result.evidence if e.field.startswith("next_safe_commands[")
+            ]
+            assert len(cmd_evidence) == len(result.next_safe_commands)
+
+
+class TestMetadataDoesNotAffectRendering:
+    """Verify that metadata doesn't change rendered output."""
+
+    @pytest.fixture
+    def result(self):
+        """Load fixture and get AssistResult."""
+        obj = load_cli_error(str(FIXTURES_DIR / "base_inputs" / "cli_error_high.json"))
+        return assist_from_cli_error(obj)
+
+    def test_lowvision_render_unchanged_with_metadata(self, result):
+        """Lowvision render should be same with or without metadata."""
+        # Render with metadata
+        output_with = render_assist(result)
+
+        # Create same result without metadata
+        result_without = AssistResult(
+            anchored_id=result.anchored_id,
+            confidence=result.confidence,
+            safest_next_step=result.safest_next_step,
+            plan=result.plan,
+            next_safe_commands=result.next_safe_commands,
+            notes=result.notes,
+            methods_applied=(),
+            evidence=(),
+        )
+        output_without = render_assist(result_without)
+
+        assert output_with == output_without
+
+    def test_cognitive_load_render_unchanged_with_metadata(self, result):
+        """Cognitive load render should be same with or without metadata."""
+        transformed = apply_cognitive_load(result)
+        output_with = render_cognitive_load(transformed)
+
+        # Same transform but clear metadata
+        transformed_clean = AssistResult(
+            anchored_id=transformed.anchored_id,
+            confidence=transformed.confidence,
+            safest_next_step=transformed.safest_next_step,
+            plan=transformed.plan,
+            next_safe_commands=transformed.next_safe_commands,
+            notes=transformed.notes,
+            methods_applied=(),
+            evidence=(),
+        )
+        output_without = render_cognitive_load(transformed_clean)
+
+        assert output_with == output_without
+
+
+class TestMetadataDeterminism:
+    """Verify metadata is deterministic."""
+
+    def test_same_input_same_methods(self):
+        """Same input should produce same methods_applied."""
+        obj = load_cli_error(str(FIXTURES_DIR / "base_inputs" / "cli_error_high.json"))
+
+        results = []
+        for _ in range(3):
+            result = assist_from_cli_error(obj)
+            results.append(result.methods_applied)
+
+        # All should be identical
+        assert all(r == results[0] for r in results)
+
+    def test_same_input_same_evidence(self):
+        """Same input should produce same evidence."""
+        obj = load_cli_error(str(FIXTURES_DIR / "base_inputs" / "cli_error_high.json"))
+
+        results = []
+        for _ in range(3):
+            result = assist_from_cli_error(obj)
+            results.append(result.evidence)
+
+        # All should be identical
+        assert all(r == results[0] for r in results)
+
+
+class TestWithMethodsHelpers:
+    """Tests for the methods.py helper functions."""
+
+    @pytest.fixture
+    def base_result(self):
+        """Create a minimal AssistResult."""
+        return AssistResult(
+            anchored_id="TEST.001",
+            confidence="High",
+            safest_next_step="Do something.",
+            plan=["Step 1", "Step 2"],
+            next_safe_commands=[],
+            notes=[],
+            methods_applied=(),
+            evidence=(),
+        )
+
+    def test_with_method_adds_single(self, base_result):
+        """with_method should add a single method ID."""
+        updated = with_method(base_result, "test.method")
+        assert "test.method" in updated.methods_applied
+
+    def test_with_method_preserves_existing(self, base_result):
+        """with_method should preserve existing methods."""
+        updated = with_method(base_result, "first.method")
+        updated = with_method(updated, "second.method")
+        assert "first.method" in updated.methods_applied
+        assert "second.method" in updated.methods_applied
+
+    def test_with_method_deduplicates(self, base_result):
+        """with_method should not add duplicates."""
+        updated = with_method(base_result, "test.method")
+        updated = with_method(updated, "test.method")
+        assert updated.methods_applied.count("test.method") == 1
+
+    def test_with_methods_adds_multiple(self, base_result):
+        """with_methods should add multiple method IDs."""
+        updated = with_methods(base_result, ["method.a", "method.b", "method.c"])
+        assert "method.a" in updated.methods_applied
+        assert "method.b" in updated.methods_applied
+        assert "method.c" in updated.methods_applied
+
+
+class TestMethodIDConstants:
+    """Verify method ID constants are stable and well-formed."""
+
+    def test_normalize_methods_exist(self):
+        """Normalization method IDs should exist."""
+        assert METHOD_NORMALIZE_CLI_ERROR == "engine.normalize.from_cli_error_v0_1"
+        assert METHOD_NORMALIZE_RAW_TEXT == "engine.normalize.from_raw_text"
+
+    def test_profile_methods_exist(self):
+        """Profile method IDs should exist."""
+        assert METHOD_PROFILE_LOWVISION == "profile.lowvision.apply"
+        assert METHOD_PROFILE_COGNITIVE_LOAD == "profile.cognitive_load.apply"
+        assert METHOD_PROFILE_SCREEN_READER == "profile.screen_reader.apply"
+        assert METHOD_PROFILE_DYSLEXIA == "profile.dyslexia.apply"
+        assert METHOD_PROFILE_PLAIN_LANGUAGE == "profile.plain_language.apply"
+
+    def test_guard_methods_exist(self):
+        """Guard method IDs should exist."""
+        assert METHOD_GUARD_VALIDATE == "guard.validate_profile_transform"
+
+    def test_method_ids_are_dotted_namespace(self):
+        """All method IDs should follow dotted namespace convention."""
+        all_methods = [
+            METHOD_NORMALIZE_CLI_ERROR,
+            METHOD_NORMALIZE_RAW_TEXT,
+            METHOD_PROFILE_LOWVISION,
+            METHOD_PROFILE_COGNITIVE_LOAD,
+            METHOD_PROFILE_SCREEN_READER,
+            METHOD_PROFILE_DYSLEXIA,
+            METHOD_PROFILE_PLAIN_LANGUAGE,
+            METHOD_GUARD_VALIDATE,
+        ]
+        for method in all_methods:
+            # Should have at least one dot (namespace.name)
+            assert "." in method, f"{method} should be dotted"
+            # Should not have uppercase
+            assert method == method.lower(), f"{method} should be lowercase"