@mcptoolshop/accessibility-suite 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.github/workflows/ci.yml +63 -0
- package/LICENSE +21 -0
- package/README.md +37 -0
- package/docs/prov-spec/.github/workflows/ci.yml +68 -0
- package/docs/prov-spec/CHANGELOG.md +69 -0
- package/docs/prov-spec/CODE_OF_CONDUCT.md +129 -0
- package/docs/prov-spec/CONFORMANCE_LEVELS.md +223 -0
- package/docs/prov-spec/CONTRIBUTING.md +145 -0
- package/docs/prov-spec/IMPLEMENTER_CHECKLIST.md +137 -0
- package/docs/prov-spec/LICENSE +21 -0
- package/docs/prov-spec/PRESS_RELEASE.md +74 -0
- package/docs/prov-spec/README.md +182 -0
- package/docs/prov-spec/SETUP.md +135 -0
- package/docs/prov-spec/WHY.md +86 -0
- package/docs/prov-spec/examples/artifact.example.json +14 -0
- package/docs/prov-spec/examples/artifact.ref.example.json +9 -0
- package/docs/prov-spec/examples/evidence.example.json +6 -0
- package/docs/prov-spec/examples/mcp.envelope.example.json +97 -0
- package/docs/prov-spec/examples/mcp.request.example.json +28 -0
- package/docs/prov-spec/examples/prov.record.example.json +35 -0
- package/docs/prov-spec/interop/PROOF_NODE_ENGINE.md +114 -0
- package/docs/prov-spec/spec/MCP_COMPATIBILITY.md +241 -0
- package/docs/prov-spec/spec/PROV_METHODS_CATALOG.md +142 -0
- package/docs/prov-spec/spec/PROV_METHODS_SPEC.md +397 -0
- package/docs/prov-spec/spec/methods.json +213 -0
- package/docs/prov-spec/spec/schemas/artifact.ref.schema.v0.1.json +58 -0
- package/docs/prov-spec/spec/schemas/artifact.schema.v0.1.json +61 -0
- package/docs/prov-spec/spec/schemas/assist.request.schema.v0.1.json +52 -0
- package/docs/prov-spec/spec/schemas/assist.response.schema.v0.1.json +70 -0
- package/docs/prov-spec/spec/schemas/cli.error.schema.v0.1.json +78 -0
- package/docs/prov-spec/spec/schemas/evidence.schema.v0.1.json +37 -0
- package/docs/prov-spec/spec/schemas/mcp.envelope.schema.v0.1.json +141 -0
- package/docs/prov-spec/spec/schemas/mcp.request.schema.v0.1.json +79 -0
- package/docs/prov-spec/spec/schemas/methods.schema.json +93 -0
- package/docs/prov-spec/spec/schemas/prov-capabilities.schema.json +122 -0
- package/docs/prov-spec/spec/schemas/prov.record.schema.v0.1.json +133 -0
- package/docs/prov-spec/spec/vectors/adapter.wrap.envelope_v0_1/expected.json +4 -0
- package/docs/prov-spec/spec/vectors/adapter.wrap.envelope_v0_1/input.json +1 -0
- package/docs/prov-spec/spec/vectors/adapter.wrap.envelope_v0_1/negative/double_wrapped.json +14 -0
- package/docs/prov-spec/spec/vectors/adapter.wrap.envelope_v0_1/negative/wrong_schema_version.json +11 -0
- package/docs/prov-spec/spec/vectors/engine.extract.evidence.json_pointer/expected.json +24 -0
- package/docs/prov-spec/spec/vectors/engine.extract.evidence.json_pointer/input.json +8 -0
- package/docs/prov-spec/spec/vectors/integrity.digest.sha256/expected.json +7 -0
- package/docs/prov-spec/spec/vectors/integrity.digest.sha256/input.json +1 -0
- package/docs/prov-spec/spec/vectors/integrity.digest.sha256/negative/non_hex_chars.json +16 -0
- package/docs/prov-spec/spec/vectors/integrity.digest.sha256/negative/uppercase_hex.json +16 -0
- package/docs/prov-spec/spec/vectors/integrity.digest.sha256/negative/wrong_length.json +16 -0
- package/docs/prov-spec/spec/vectors/method_id_syntax/negative/hyphen_separator.json +8 -0
- package/docs/prov-spec/spec/vectors/method_id_syntax/negative/reserved_namespace.json +8 -0
- package/docs/prov-spec/spec/vectors/method_id_syntax/negative/starts_with_digit.json +8 -0
- package/docs/prov-spec/spec/vectors/method_id_syntax/negative/uppercase.json +8 -0
- package/docs/prov-spec/spec/vectors/method_id_syntax/positive/valid_ids.json +18 -0
- package/docs/prov-spec/tools/python/prov_validator.py +428 -0
- package/examples/a11y-demo-site/.github/workflows/a11y-artifacts.yml +81 -0
- package/examples/a11y-demo-site/.github/workflows/a11y.yml +34 -0
- package/examples/a11y-demo-site/CODE_OF_CONDUCT.md +129 -0
- package/examples/a11y-demo-site/CONTRIBUTING.md +83 -0
- package/examples/a11y-demo-site/LICENSE +21 -0
- package/examples/a11y-demo-site/README.md +155 -0
- package/examples/a11y-demo-site/html/contact.html +15 -0
- package/examples/a11y-demo-site/html/index.html +20 -0
- package/examples/a11y-demo-site/scripts/a11y.sh +20 -0
- package/package.json +26 -0
- package/src/a11y-assist/.github/workflows/publish.yml +52 -0
- package/src/a11y-assist/.github/workflows/test.yml +30 -0
- package/src/a11y-assist/A11Y_ASSIST_TEST_COVERAGE_REQUIREMENTS.md +104 -0
- package/src/a11y-assist/CODE_OF_CONDUCT.md +129 -0
- package/src/a11y-assist/CONTRIBUTING.md +98 -0
- package/src/a11y-assist/ENGINE.md +363 -0
- package/src/a11y-assist/LICENSE +21 -0
- package/src/a11y-assist/PRESS_RELEASE.md +71 -0
- package/src/a11y-assist/QUICKSTART.md +101 -0
- package/src/a11y-assist/README.md +192 -0
- package/src/a11y-assist/RELEASE_NOTES.md +319 -0
- package/src/a11y-assist/a11y_assist/__init__.py +3 -0
- package/src/a11y-assist/a11y_assist/cli.py +599 -0
- package/src/a11y-assist/a11y_assist/from_cli_error.py +149 -0
- package/src/a11y-assist/a11y_assist/guard.py +444 -0
- package/src/a11y-assist/a11y_assist/ingest.py +407 -0
- package/src/a11y-assist/a11y_assist/methods.py +137 -0
- package/src/a11y-assist/a11y_assist/parse_raw.py +71 -0
- package/src/a11y-assist/a11y_assist/profiles/__init__.py +29 -0
- package/src/a11y-assist/a11y_assist/profiles/cognitive_load.py +245 -0
- package/src/a11y-assist/a11y_assist/profiles/cognitive_load_render.py +86 -0
- package/src/a11y-assist/a11y_assist/profiles/dyslexia.py +144 -0
- package/src/a11y-assist/a11y_assist/profiles/dyslexia_render.py +77 -0
- package/src/a11y-assist/a11y_assist/profiles/plain_language.py +119 -0
- package/src/a11y-assist/a11y_assist/profiles/plain_language_render.py +66 -0
- package/src/a11y-assist/a11y_assist/profiles/screen_reader.py +348 -0
- package/src/a11y-assist/a11y_assist/profiles/screen_reader_render.py +89 -0
- package/src/a11y-assist/a11y_assist/render.py +95 -0
- package/src/a11y-assist/a11y_assist/schemas/assist.request.schema.v0.1.json +52 -0
- package/src/a11y-assist/a11y_assist/schemas/assist.response.schema.v0.1.json +70 -0
- package/src/a11y-assist/a11y_assist/schemas/cli.error.schema.v0.1.json +78 -0
- package/src/a11y-assist/a11y_assist/storage.py +31 -0
- package/src/a11y-assist/pyproject.toml +60 -0
- package/src/a11y-assist/tests/__init__.py +1 -0
- package/src/a11y-assist/tests/fixtures/base_inputs/cli_error_high.json +18 -0
- package/src/a11y-assist/tests/fixtures/base_inputs/cli_error_medium.json +16 -0
- package/src/a11y-assist/tests/fixtures/base_inputs/raw_text_low.txt +3 -0
- package/src/a11y-assist/tests/fixtures/cli_error_good.json +9 -0
- package/src/a11y-assist/tests/fixtures/cli_error_missing_id.json +7 -0
- package/src/a11y-assist/tests/fixtures/cli_error_string_format.json +7 -0
- package/src/a11y-assist/tests/fixtures/expected/cognitive_load_high.txt +20 -0
- package/src/a11y-assist/tests/fixtures/expected/dyslexia_high.txt +20 -0
- package/src/a11y-assist/tests/fixtures/expected/lowvision_high.txt +18 -0
- package/src/a11y-assist/tests/fixtures/expected/plain_language_high.txt +14 -0
- package/src/a11y-assist/tests/fixtures/expected/screen_reader_high.txt +19 -0
- package/src/a11y-assist/tests/fixtures/golden_screen_reader_cli_error.txt +16 -0
- package/src/a11y-assist/tests/fixtures/golden_screen_reader_raw_no_id.txt +14 -0
- package/src/a11y-assist/tests/fixtures/golden_screen_reader_raw_with_id.txt +14 -0
- package/src/a11y-assist/tests/fixtures/raw_good.txt +11 -0
- package/src/a11y-assist/tests/fixtures/raw_no_id.txt +2 -0
- package/src/a11y-assist/tests/test_cognitive_load.py +469 -0
- package/src/a11y-assist/tests/test_dyslexia.py +337 -0
- package/src/a11y-assist/tests/test_explain.py +74 -0
- package/src/a11y-assist/tests/test_golden.py +127 -0
- package/src/a11y-assist/tests/test_guard.py +819 -0
- package/src/a11y-assist/tests/test_guard_integration.py +457 -0
- package/src/a11y-assist/tests/test_ingest.py +311 -0
- package/src/a11y-assist/tests/test_methods_metadata.py +236 -0
- package/src/a11y-assist/tests/test_plain_language.py +348 -0
- package/src/a11y-assist/tests/test_render.py +117 -0
- package/src/a11y-assist/tests/test_screen_reader.py +703 -0
- package/src/a11y-assist/tests/test_storage_last.py +61 -0
- package/src/a11y-assist/tests/test_triage.py +86 -0
- package/src/a11y-ci/.github/workflows/ci.yml +43 -0
- package/src/a11y-ci/.github/workflows/test.yml +30 -0
- package/src/a11y-ci/A11Y_CI_TEST_COVERAGE_REQUIREMENTS.md +94 -0
- package/src/a11y-ci/CODE_OF_CONDUCT.md +129 -0
- package/src/a11y-ci/CONTRIBUTING.md +142 -0
- package/src/a11y-ci/LICENSE +21 -0
- package/src/a11y-ci/README.md +105 -0
- package/src/a11y-ci/a11y_ci/__init__.py +3 -0
- package/src/a11y-ci/a11y_ci/allowlist.py +83 -0
- package/src/a11y-ci/a11y_ci/cli.py +145 -0
- package/src/a11y-ci/a11y_ci/gate.py +131 -0
- package/src/a11y-ci/a11y_ci/render.py +48 -0
- package/src/a11y-ci/a11y_ci/schemas/allowlist.schema.json +24 -0
- package/src/a11y-ci/a11y_ci/scorecard.py +99 -0
- package/src/a11y-ci/npm/package.json +35 -0
- package/src/a11y-ci/pyproject.toml +64 -0
- package/src/a11y-ci/tests/__init__.py +1 -0
- package/src/a11y-ci/tests/fixtures/allowlist_expired.json +10 -0
- package/src/a11y-ci/tests/fixtures/allowlist_ok.json +10 -0
- package/src/a11y-ci/tests/fixtures/baseline_ok.json +7 -0
- package/src/a11y-ci/tests/fixtures/current_fail.json +6 -0
- package/src/a11y-ci/tests/fixtures/current_ok.json +6 -0
- package/src/a11y-ci/tests/fixtures/current_regresses.json +7 -0
- package/src/a11y-ci/tests/test_gate.py +134 -0
- package/src/a11y-evidence-engine/.github/workflows/ci.yml +53 -0
- package/src/a11y-evidence-engine/CODE_OF_CONDUCT.md +129 -0
- package/src/a11y-evidence-engine/CONTRIBUTING.md +128 -0
- package/src/a11y-evidence-engine/LICENSE +21 -0
- package/src/a11y-evidence-engine/README.md +71 -0
- package/src/a11y-evidence-engine/bin/a11y-engine.js +11 -0
- package/src/a11y-evidence-engine/fixtures/bad/button-no-name.html +30 -0
- package/src/a11y-evidence-engine/fixtures/bad/img-missing-alt.html +19 -0
- package/src/a11y-evidence-engine/fixtures/bad/input-missing-label.html +26 -0
- package/src/a11y-evidence-engine/fixtures/bad/missing-lang.html +11 -0
- package/src/a11y-evidence-engine/fixtures/good/index.html +29 -0
- package/src/a11y-evidence-engine/package-lock.json +109 -0
- package/src/a11y-evidence-engine/package.json +45 -0
- package/src/a11y-evidence-engine/src/cli.js +74 -0
- package/src/a11y-evidence-engine/src/evidence/canonicalize.js +52 -0
- package/src/a11y-evidence-engine/src/evidence/json_pointer.js +34 -0
- package/src/a11y-evidence-engine/src/evidence/prov_emit.js +153 -0
- package/src/a11y-evidence-engine/src/fswalk.js +56 -0
- package/src/a11y-evidence-engine/src/html_parse.js +117 -0
- package/src/a11y-evidence-engine/src/ids.js +53 -0
- package/src/a11y-evidence-engine/src/rules/document_missing_lang.js +50 -0
- package/src/a11y-evidence-engine/src/rules/form_control_missing_label.js +105 -0
- package/src/a11y-evidence-engine/src/rules/img_missing_alt.js +77 -0
- package/src/a11y-evidence-engine/src/rules/index.js +37 -0
- package/src/a11y-evidence-engine/src/rules/interactive_missing_name.js +129 -0
- package/src/a11y-evidence-engine/src/scan.js +128 -0
- package/src/a11y-evidence-engine/test/scan.test.js +149 -0
- package/src/a11y-evidence-engine/test/vectors.test.js +200 -0
- package/src/a11y-lint/.github/workflows/ci.yml +46 -0
- package/src/a11y-lint/.github/workflows/test.yml +34 -0
- package/src/a11y-lint/CODE_OF_CONDUCT.md +129 -0
- package/src/a11y-lint/CONTRIBUTING.md +70 -0
- package/src/a11y-lint/GOVERNANCE.md +57 -0
- package/src/a11y-lint/LICENSE +21 -0
- package/src/a11y-lint/PRESS_RELEASE.md +50 -0
- package/src/a11y-lint/README.md +276 -0
- package/src/a11y-lint/RELEASE_NOTES.md +57 -0
- package/src/a11y-lint/RELEASING.md +57 -0
- package/src/a11y-lint/a11y_lint/__init__.py +64 -0
- package/src/a11y-lint/a11y_lint/cli.py +319 -0
- package/src/a11y-lint/a11y_lint/errors.py +252 -0
- package/src/a11y-lint/a11y_lint/render.py +293 -0
- package/src/a11y-lint/a11y_lint/report_md.py +289 -0
- package/src/a11y-lint/a11y_lint/scan_cli_text.py +434 -0
- package/src/a11y-lint/a11y_lint/schemas/cli.error.schema.v0.1.json +83 -0
- package/src/a11y-lint/a11y_lint/scorecard.py +244 -0
- package/src/a11y-lint/a11y_lint/validate.py +225 -0
- package/src/a11y-lint/pyproject.toml +75 -0
- package/src/a11y-lint/tests/__init__.py +1 -0
- package/src/a11y-lint/tests/test_cli.py +200 -0
- package/src/a11y-lint/tests/test_errors.py +188 -0
- package/src/a11y-lint/tests/test_render.py +202 -0
- package/src/a11y-lint/tests/test_report_md.py +188 -0
- package/src/a11y-lint/tests/test_scan_cli_text.py +290 -0
- package/src/a11y-lint/tests/test_scorecard.py +195 -0
- package/src/a11y-lint/tests/test_validate.py +257 -0
- package/src/a11y-mcp-tools/.github/workflows/ci.yml +53 -0
- package/src/a11y-mcp-tools/CODE_OF_CONDUCT.md +129 -0
- package/src/a11y-mcp-tools/CONTRIBUTING.md +136 -0
- package/src/a11y-mcp-tools/LICENSE +21 -0
- package/src/a11y-mcp-tools/PROV_METHODS_CATALOG.md +104 -0
- package/src/a11y-mcp-tools/README.md +168 -0
- package/src/a11y-mcp-tools/bin/cli.js +452 -0
- package/src/a11y-mcp-tools/bin/server.js +244 -0
- package/src/a11y-mcp-tools/fixtures/requests/a11y.diagnose.ok.json +27 -0
- package/src/a11y-mcp-tools/fixtures/requests/a11y.evidence.ok.json +25 -0
- package/src/a11y-mcp-tools/fixtures/responses/a11y.diagnose.ok.json +139 -0
- package/src/a11y-mcp-tools/fixtures/responses/a11y.diagnose.provenance_fail.json +13 -0
- package/src/a11y-mcp-tools/fixtures/responses/a11y.evidence.ok.json +88 -0
- package/src/a11y-mcp-tools/package-lock.json +189 -0
- package/src/a11y-mcp-tools/package.json +49 -0
- package/src/a11y-mcp-tools/src/envelope.js +197 -0
- package/src/a11y-mcp-tools/src/index.js +9 -0
- package/src/a11y-mcp-tools/src/schemas/artifact.js +85 -0
- package/src/a11y-mcp-tools/src/schemas/diagnosis.schema.v0.1.json +137 -0
- package/src/a11y-mcp-tools/src/schemas/envelope.schema.v0.1.json +108 -0
- package/src/a11y-mcp-tools/src/schemas/evidence.bundle.schema.v0.1.json +129 -0
- package/src/a11y-mcp-tools/src/schemas/evidence.js +97 -0
- package/src/a11y-mcp-tools/src/schemas/index.js +11 -0
- package/src/a11y-mcp-tools/src/schemas/provenance.js +140 -0
- package/src/a11y-mcp-tools/src/schemas/tools/a11y.diagnose.request.schema.v0.1.json +77 -0
- package/src/a11y-mcp-tools/src/schemas/tools/a11y.diagnose.response.schema.v0.1.json +50 -0
- package/src/a11y-mcp-tools/src/schemas/tools/a11y.evidence.request.schema.v0.1.json +120 -0
- package/src/a11y-mcp-tools/src/schemas/tools/a11y.evidence.response.schema.v0.1.json +50 -0
- package/src/a11y-mcp-tools/src/tools/diagnose.js +597 -0
- package/src/a11y-mcp-tools/src/tools/evidence.js +481 -0
- package/src/a11y-mcp-tools/src/tools/index.js +10 -0
- package/src/a11y-mcp-tools/test/contract.test.mjs +154 -0
- package/src/a11y-mcp-tools/test/diagnose.test.js +485 -0
- package/src/a11y-mcp-tools/test/evidence.test.js +183 -0
- package/src/a11y-mcp-tools/test/schema.test.js +327 -0
|
@@ -0,0 +1,819 @@
|
|
|
1
|
+
"""Unit tests for Profile Guard invariants.
|
|
2
|
+
|
|
3
|
+
Tests each guard check individually to ensure violations are detected.
|
|
4
|
+
"""
|
|
5
|
+
|
|
6
|
+
import pytest
|
|
7
|
+
|
|
8
|
+
from a11y_assist.guard import (
|
|
9
|
+
GuardContext,
|
|
10
|
+
GuardIssue,
|
|
11
|
+
GuardViolation,
|
|
12
|
+
get_guard_context,
|
|
13
|
+
validate_profile_transform,
|
|
14
|
+
)
|
|
15
|
+
from a11y_assist.render import AssistResult
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
# Fixtures
|
|
19
|
+
|
|
20
|
+
@pytest.fixture
|
|
21
|
+
def base_result_high() -> AssistResult:
|
|
22
|
+
"""High confidence base result with commands."""
|
|
23
|
+
return AssistResult(
|
|
24
|
+
anchored_id="TEST.ERROR.001",
|
|
25
|
+
confidence="High",
|
|
26
|
+
safest_next_step="Run the fix command.",
|
|
27
|
+
plan=["Step 1: Check config.", "Step 2: Run fix."],
|
|
28
|
+
next_safe_commands=["fix --dry-run"],
|
|
29
|
+
notes=["Original error message."],
|
|
30
|
+
)
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
@pytest.fixture
|
|
34
|
+
def base_result_low() -> AssistResult:
|
|
35
|
+
"""Low confidence base result."""
|
|
36
|
+
return AssistResult(
|
|
37
|
+
anchored_id=None,
|
|
38
|
+
confidence="Low",
|
|
39
|
+
safest_next_step="Check the output.",
|
|
40
|
+
plan=["Re-run with verbosity."],
|
|
41
|
+
next_safe_commands=[],
|
|
42
|
+
notes=["No ID found."],
|
|
43
|
+
)
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
@pytest.fixture
|
|
47
|
+
def base_text() -> str:
|
|
48
|
+
"""Sample base text for content support checking."""
|
|
49
|
+
return "Error: Config file not found. Run fix --dry-run to repair."
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
# Test: ID Invariant
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def test_guard_id_invented_fails(base_result_low: AssistResult, base_text: str):
|
|
56
|
+
"""Guard should fail when profile invents an ID that didn't exist."""
|
|
57
|
+
# Base has no ID
|
|
58
|
+
profiled = AssistResult(
|
|
59
|
+
anchored_id="INVENTED.ID.001", # Invented!
|
|
60
|
+
confidence="Low",
|
|
61
|
+
safest_next_step="Check the output.",
|
|
62
|
+
plan=["Re-run with verbosity."],
|
|
63
|
+
next_safe_commands=[],
|
|
64
|
+
notes=["No ID found."],
|
|
65
|
+
)
|
|
66
|
+
|
|
67
|
+
ctx = get_guard_context(
|
|
68
|
+
profile="lowvision",
|
|
69
|
+
confidence="Low",
|
|
70
|
+
input_kind="raw_text",
|
|
71
|
+
allowed_commands=set(),
|
|
72
|
+
)
|
|
73
|
+
|
|
74
|
+
with pytest.raises(GuardViolation) as exc_info:
|
|
75
|
+
validate_profile_transform(base_text, base_result_low, profiled, ctx)
|
|
76
|
+
|
|
77
|
+
assert any(
|
|
78
|
+
issue.code == "A11Y.ASSIST.GUARD.ID.INVENTED"
|
|
79
|
+
for issue in exc_info.value.issues
|
|
80
|
+
)
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
def test_guard_id_changed_fails(base_result_high: AssistResult, base_text: str):
|
|
84
|
+
"""Guard should fail when profile changes an existing ID."""
|
|
85
|
+
profiled = AssistResult(
|
|
86
|
+
anchored_id="DIFFERENT.ID.002", # Changed!
|
|
87
|
+
confidence="High",
|
|
88
|
+
safest_next_step="Run the fix command.",
|
|
89
|
+
plan=["Step 1: Check config.", "Step 2: Run fix."],
|
|
90
|
+
next_safe_commands=["fix --dry-run"],
|
|
91
|
+
notes=["Original error message."],
|
|
92
|
+
)
|
|
93
|
+
|
|
94
|
+
ctx = get_guard_context(
|
|
95
|
+
profile="lowvision",
|
|
96
|
+
confidence="High",
|
|
97
|
+
input_kind="cli_error_json",
|
|
98
|
+
allowed_commands={"fix --dry-run"},
|
|
99
|
+
)
|
|
100
|
+
|
|
101
|
+
with pytest.raises(GuardViolation) as exc_info:
|
|
102
|
+
validate_profile_transform(base_text, base_result_high, profiled, ctx)
|
|
103
|
+
|
|
104
|
+
assert any(
|
|
105
|
+
issue.code == "A11Y.ASSIST.GUARD.ID.CHANGED"
|
|
106
|
+
for issue in exc_info.value.issues
|
|
107
|
+
)
|
|
108
|
+
|
|
109
|
+
|
|
110
|
+
def test_guard_id_preserved_passes(base_result_high: AssistResult, base_text: str):
|
|
111
|
+
"""Guard should pass when ID is preserved correctly."""
|
|
112
|
+
profiled = AssistResult(
|
|
113
|
+
anchored_id="TEST.ERROR.001", # Same as base
|
|
114
|
+
confidence="High",
|
|
115
|
+
safest_next_step="Run the fix command.",
|
|
116
|
+
plan=["Step 1: Check config.", "Step 2: Run fix."],
|
|
117
|
+
next_safe_commands=["fix --dry-run"],
|
|
118
|
+
notes=["Original error message."],
|
|
119
|
+
)
|
|
120
|
+
|
|
121
|
+
ctx = get_guard_context(
|
|
122
|
+
profile="lowvision",
|
|
123
|
+
confidence="High",
|
|
124
|
+
input_kind="cli_error_json",
|
|
125
|
+
allowed_commands={"fix --dry-run"},
|
|
126
|
+
)
|
|
127
|
+
|
|
128
|
+
# Should not raise
|
|
129
|
+
validate_profile_transform(base_text, base_result_high, profiled, ctx)
|
|
130
|
+
|
|
131
|
+
|
|
132
|
+
# Test: Confidence Invariant
|
|
133
|
+
|
|
134
|
+
|
|
135
|
+
def test_guard_confidence_increase_fails(base_result_low: AssistResult, base_text: str):
|
|
136
|
+
"""Guard should fail when profile increases confidence level."""
|
|
137
|
+
profiled = AssistResult(
|
|
138
|
+
anchored_id=None,
|
|
139
|
+
confidence="High", # Increased from Low!
|
|
140
|
+
safest_next_step="Check the output.",
|
|
141
|
+
plan=["Re-run with verbosity."],
|
|
142
|
+
next_safe_commands=[],
|
|
143
|
+
notes=["No ID found."],
|
|
144
|
+
)
|
|
145
|
+
|
|
146
|
+
ctx = get_guard_context(
|
|
147
|
+
profile="lowvision",
|
|
148
|
+
confidence="Low",
|
|
149
|
+
input_kind="raw_text",
|
|
150
|
+
allowed_commands=set(),
|
|
151
|
+
)
|
|
152
|
+
|
|
153
|
+
with pytest.raises(GuardViolation) as exc_info:
|
|
154
|
+
validate_profile_transform(base_text, base_result_low, profiled, ctx)
|
|
155
|
+
|
|
156
|
+
assert any(
|
|
157
|
+
issue.code == "A11Y.ASSIST.GUARD.CONFIDENCE.INCREASED"
|
|
158
|
+
for issue in exc_info.value.issues
|
|
159
|
+
)
|
|
160
|
+
|
|
161
|
+
|
|
162
|
+
def test_guard_confidence_decrease_passes(base_result_high: AssistResult, base_text: str):
|
|
163
|
+
"""Guard should allow confidence to decrease (conservative)."""
|
|
164
|
+
profiled = AssistResult(
|
|
165
|
+
anchored_id="TEST.ERROR.001",
|
|
166
|
+
confidence="Medium", # Decreased from High - OK
|
|
167
|
+
safest_next_step="Run the fix command.",
|
|
168
|
+
plan=["Step 1: Check config.", "Step 2: Run fix."],
|
|
169
|
+
next_safe_commands=["fix --dry-run"],
|
|
170
|
+
notes=["Original error message."],
|
|
171
|
+
)
|
|
172
|
+
|
|
173
|
+
ctx = get_guard_context(
|
|
174
|
+
profile="lowvision",
|
|
175
|
+
confidence="High",
|
|
176
|
+
input_kind="cli_error_json",
|
|
177
|
+
allowed_commands={"fix --dry-run"},
|
|
178
|
+
)
|
|
179
|
+
|
|
180
|
+
# Should not raise
|
|
181
|
+
validate_profile_transform(base_text, base_result_high, profiled, ctx)
|
|
182
|
+
|
|
183
|
+
|
|
184
|
+
def test_guard_confidence_same_passes(base_result_high: AssistResult, base_text: str):
|
|
185
|
+
"""Guard should allow same confidence level."""
|
|
186
|
+
profiled = AssistResult(
|
|
187
|
+
anchored_id="TEST.ERROR.001",
|
|
188
|
+
confidence="High", # Same as base
|
|
189
|
+
safest_next_step="Run the fix command.",
|
|
190
|
+
plan=["Step 1: Check config.", "Step 2: Run fix."],
|
|
191
|
+
next_safe_commands=["fix --dry-run"],
|
|
192
|
+
notes=["Original error message."],
|
|
193
|
+
)
|
|
194
|
+
|
|
195
|
+
ctx = get_guard_context(
|
|
196
|
+
profile="lowvision",
|
|
197
|
+
confidence="High",
|
|
198
|
+
input_kind="cli_error_json",
|
|
199
|
+
allowed_commands={"fix --dry-run"},
|
|
200
|
+
)
|
|
201
|
+
|
|
202
|
+
# Should not raise
|
|
203
|
+
validate_profile_transform(base_text, base_result_high, profiled, ctx)
|
|
204
|
+
|
|
205
|
+
|
|
206
|
+
# Test: Commands Invariant
|
|
207
|
+
|
|
208
|
+
|
|
209
|
+
def test_guard_command_invented_fails(base_result_high: AssistResult, base_text: str):
|
|
210
|
+
"""Guard should fail when profile invents a command not in allowed set."""
|
|
211
|
+
profiled = AssistResult(
|
|
212
|
+
anchored_id="TEST.ERROR.001",
|
|
213
|
+
confidence="High",
|
|
214
|
+
safest_next_step="Run the fix command.",
|
|
215
|
+
plan=["Step 1: Check config.", "Step 2: Run fix."],
|
|
216
|
+
next_safe_commands=["rm -rf /"], # Invented dangerous command!
|
|
217
|
+
notes=["Original error message."],
|
|
218
|
+
)
|
|
219
|
+
|
|
220
|
+
ctx = get_guard_context(
|
|
221
|
+
profile="lowvision",
|
|
222
|
+
confidence="High",
|
|
223
|
+
input_kind="cli_error_json",
|
|
224
|
+
allowed_commands={"fix --dry-run"}, # Only this is allowed
|
|
225
|
+
)
|
|
226
|
+
|
|
227
|
+
with pytest.raises(GuardViolation) as exc_info:
|
|
228
|
+
validate_profile_transform(base_text, base_result_high, profiled, ctx)
|
|
229
|
+
|
|
230
|
+
assert any(
|
|
231
|
+
issue.code == "A11Y.ASSIST.GUARD.COMMANDS.INVENTED"
|
|
232
|
+
for issue in exc_info.value.issues
|
|
233
|
+
)
|
|
234
|
+
|
|
235
|
+
|
|
236
|
+
def test_guard_command_not_in_allowed_set_fails(base_result_high: AssistResult, base_text: str):
|
|
237
|
+
"""Guard should fail when command is not in the allowed set."""
|
|
238
|
+
profiled = AssistResult(
|
|
239
|
+
anchored_id="TEST.ERROR.001",
|
|
240
|
+
confidence="High",
|
|
241
|
+
safest_next_step="Run the fix command.",
|
|
242
|
+
plan=["Step 1: Check config.", "Step 2: Run fix."],
|
|
243
|
+
next_safe_commands=["fix --dry-run", "another-command"],
|
|
244
|
+
notes=["Original error message."],
|
|
245
|
+
)
|
|
246
|
+
|
|
247
|
+
ctx = get_guard_context(
|
|
248
|
+
profile="lowvision",
|
|
249
|
+
confidence="High",
|
|
250
|
+
input_kind="cli_error_json",
|
|
251
|
+
allowed_commands={"fix --dry-run"}, # Only one allowed
|
|
252
|
+
)
|
|
253
|
+
|
|
254
|
+
with pytest.raises(GuardViolation) as exc_info:
|
|
255
|
+
validate_profile_transform(base_text, base_result_high, profiled, ctx)
|
|
256
|
+
|
|
257
|
+
assert any(
|
|
258
|
+
issue.code == "A11Y.ASSIST.GUARD.COMMANDS.INVENTED"
|
|
259
|
+
for issue in exc_info.value.issues
|
|
260
|
+
)
|
|
261
|
+
|
|
262
|
+
|
|
263
|
+
def test_guard_low_conf_disallows_commands(base_result_low: AssistResult, base_text: str):
|
|
264
|
+
"""Guard should fail when Low confidence result has commands."""
|
|
265
|
+
profiled = AssistResult(
|
|
266
|
+
anchored_id=None,
|
|
267
|
+
confidence="Low",
|
|
268
|
+
safest_next_step="Check the output.",
|
|
269
|
+
plan=["Re-run with verbosity."],
|
|
270
|
+
next_safe_commands=["some-command"], # Commands on Low!
|
|
271
|
+
notes=["No ID found."],
|
|
272
|
+
)
|
|
273
|
+
|
|
274
|
+
ctx = get_guard_context(
|
|
275
|
+
profile="lowvision",
|
|
276
|
+
confidence="Low",
|
|
277
|
+
input_kind="raw_text",
|
|
278
|
+
allowed_commands={"some-command"}, # Even if allowed, still fails
|
|
279
|
+
)
|
|
280
|
+
|
|
281
|
+
with pytest.raises(GuardViolation) as exc_info:
|
|
282
|
+
validate_profile_transform(base_text, base_result_low, profiled, ctx)
|
|
283
|
+
|
|
284
|
+
assert any(
|
|
285
|
+
issue.code == "A11Y.ASSIST.GUARD.COMMANDS.DISALLOWED_LOW_CONF"
|
|
286
|
+
for issue in exc_info.value.issues
|
|
287
|
+
)
|
|
288
|
+
|
|
289
|
+
|
|
290
|
+
def test_guard_command_with_dollar_prefix_passes(base_result_high: AssistResult, base_text: str):
|
|
291
|
+
"""Guard should handle $ prefix normalization correctly."""
|
|
292
|
+
profiled = AssistResult(
|
|
293
|
+
anchored_id="TEST.ERROR.001",
|
|
294
|
+
confidence="High",
|
|
295
|
+
safest_next_step="Run the fix command.",
|
|
296
|
+
plan=["Step 1: Check config.", "Step 2: Run fix."],
|
|
297
|
+
next_safe_commands=["$ fix --dry-run"], # With $ prefix
|
|
298
|
+
notes=["Original error message."],
|
|
299
|
+
)
|
|
300
|
+
|
|
301
|
+
ctx = get_guard_context(
|
|
302
|
+
profile="lowvision",
|
|
303
|
+
confidence="High",
|
|
304
|
+
input_kind="cli_error_json",
|
|
305
|
+
allowed_commands={"fix --dry-run"}, # Without prefix
|
|
306
|
+
)
|
|
307
|
+
|
|
308
|
+
# Should not raise - $ prefix is normalized
|
|
309
|
+
validate_profile_transform(base_text, base_result_high, profiled, ctx)
|
|
310
|
+
|
|
311
|
+
|
|
312
|
+
# Test: Step Count Invariant
|
|
313
|
+
|
|
314
|
+
|
|
315
|
+
def test_guard_plan_too_many_steps_fails():
|
|
316
|
+
"""Guard should fail when plan exceeds max steps."""
|
|
317
|
+
base = AssistResult(
|
|
318
|
+
anchored_id="TEST.ERROR.001",
|
|
319
|
+
confidence="High",
|
|
320
|
+
safest_next_step="Follow the steps.",
|
|
321
|
+
plan=["Step 1", "Step 2", "Step 3", "Step 4", "Step 5", "Step 6"],
|
|
322
|
+
next_safe_commands=[],
|
|
323
|
+
notes=[],
|
|
324
|
+
)
|
|
325
|
+
|
|
326
|
+
profiled = AssistResult(
|
|
327
|
+
anchored_id="TEST.ERROR.001",
|
|
328
|
+
confidence="High",
|
|
329
|
+
safest_next_step="Follow the steps.",
|
|
330
|
+
plan=["Step 1", "Step 2", "Step 3", "Step 4", "Step 5", "Step 6"], # 6 steps
|
|
331
|
+
next_safe_commands=[],
|
|
332
|
+
notes=[],
|
|
333
|
+
)
|
|
334
|
+
|
|
335
|
+
# lowvision profile has max_steps=5
|
|
336
|
+
ctx = get_guard_context(
|
|
337
|
+
profile="lowvision",
|
|
338
|
+
confidence="High",
|
|
339
|
+
input_kind="cli_error_json",
|
|
340
|
+
allowed_commands=set(),
|
|
341
|
+
)
|
|
342
|
+
|
|
343
|
+
with pytest.raises(GuardViolation) as exc_info:
|
|
344
|
+
validate_profile_transform("test content", base, profiled, ctx)
|
|
345
|
+
|
|
346
|
+
assert any(
|
|
347
|
+
issue.code == "A11Y.ASSIST.GUARD.PLAN.TOO_MANY_STEPS"
|
|
348
|
+
for issue in exc_info.value.issues
|
|
349
|
+
)
|
|
350
|
+
|
|
351
|
+
|
|
352
|
+
def test_guard_cognitive_load_max_3_steps():
|
|
353
|
+
"""Cognitive-load profile should enforce max 3 steps."""
|
|
354
|
+
base = AssistResult(
|
|
355
|
+
anchored_id="TEST.ERROR.001",
|
|
356
|
+
confidence="High",
|
|
357
|
+
safest_next_step="Follow the steps.",
|
|
358
|
+
plan=["Step 1", "Step 2", "Step 3", "Step 4"], # 4 steps
|
|
359
|
+
next_safe_commands=[],
|
|
360
|
+
notes=[],
|
|
361
|
+
)
|
|
362
|
+
|
|
363
|
+
profiled = AssistResult(
|
|
364
|
+
anchored_id="TEST.ERROR.001",
|
|
365
|
+
confidence="High",
|
|
366
|
+
safest_next_step="Follow the steps.",
|
|
367
|
+
plan=["Step 1", "Step 2", "Step 3", "Step 4"], # 4 steps > 3 max
|
|
368
|
+
next_safe_commands=[],
|
|
369
|
+
notes=[],
|
|
370
|
+
)
|
|
371
|
+
|
|
372
|
+
ctx = get_guard_context(
|
|
373
|
+
profile="cognitive-load",
|
|
374
|
+
confidence="High",
|
|
375
|
+
input_kind="cli_error_json",
|
|
376
|
+
allowed_commands=set(),
|
|
377
|
+
)
|
|
378
|
+
|
|
379
|
+
with pytest.raises(GuardViolation) as exc_info:
|
|
380
|
+
validate_profile_transform("test content", base, profiled, ctx)
|
|
381
|
+
|
|
382
|
+
assert any(
|
|
383
|
+
issue.code == "A11Y.ASSIST.GUARD.PLAN.TOO_MANY_STEPS"
|
|
384
|
+
for issue in exc_info.value.issues
|
|
385
|
+
)
|
|
386
|
+
|
|
387
|
+
|
|
388
|
+
def test_guard_plan_within_limit_passes():
|
|
389
|
+
"""Guard should pass when plan is within step limit."""
|
|
390
|
+
base = AssistResult(
|
|
391
|
+
anchored_id="TEST.ERROR.001",
|
|
392
|
+
confidence="High",
|
|
393
|
+
safest_next_step="Follow the steps.",
|
|
394
|
+
plan=["Step 1", "Step 2", "Step 3"], # 3 steps
|
|
395
|
+
next_safe_commands=[],
|
|
396
|
+
notes=[],
|
|
397
|
+
)
|
|
398
|
+
|
|
399
|
+
profiled = AssistResult(
|
|
400
|
+
anchored_id="TEST.ERROR.001",
|
|
401
|
+
confidence="High",
|
|
402
|
+
safest_next_step="Follow the steps.",
|
|
403
|
+
plan=["Step 1", "Step 2", "Step 3"],
|
|
404
|
+
next_safe_commands=[],
|
|
405
|
+
notes=[],
|
|
406
|
+
)
|
|
407
|
+
|
|
408
|
+
# cognitive-load has max_steps=3
|
|
409
|
+
ctx = get_guard_context(
|
|
410
|
+
profile="cognitive-load",
|
|
411
|
+
confidence="High",
|
|
412
|
+
input_kind="cli_error_json",
|
|
413
|
+
allowed_commands=set(),
|
|
414
|
+
)
|
|
415
|
+
|
|
416
|
+
# Should not raise
|
|
417
|
+
validate_profile_transform("test content", base, profiled, ctx)
|
|
418
|
+
|
|
419
|
+
|
|
420
|
+
# Test: Screen-reader Specific Constraints
|
|
421
|
+
|
|
422
|
+
|
|
423
|
+
def test_guard_parentheticals_forbidden():
|
|
424
|
+
"""Screen-reader profile should forbid parentheticals."""
|
|
425
|
+
base = AssistResult(
|
|
426
|
+
anchored_id="TEST.ERROR.001",
|
|
427
|
+
confidence="High",
|
|
428
|
+
safest_next_step="Run the command",
|
|
429
|
+
plan=["Step 1"],
|
|
430
|
+
next_safe_commands=[],
|
|
431
|
+
notes=[],
|
|
432
|
+
)
|
|
433
|
+
|
|
434
|
+
profiled = AssistResult(
|
|
435
|
+
anchored_id="TEST.ERROR.001",
|
|
436
|
+
confidence="High",
|
|
437
|
+
safest_next_step="Run the command (see docs)", # Parenthetical!
|
|
438
|
+
plan=["Step 1"],
|
|
439
|
+
next_safe_commands=[],
|
|
440
|
+
notes=[],
|
|
441
|
+
)
|
|
442
|
+
|
|
443
|
+
ctx = get_guard_context(
|
|
444
|
+
profile="screen-reader",
|
|
445
|
+
confidence="High",
|
|
446
|
+
input_kind="cli_error_json",
|
|
447
|
+
allowed_commands=set(),
|
|
448
|
+
)
|
|
449
|
+
|
|
450
|
+
with pytest.raises(GuardViolation) as exc_info:
|
|
451
|
+
validate_profile_transform("test content docs command", base, profiled, ctx)
|
|
452
|
+
|
|
453
|
+
assert any(
|
|
454
|
+
issue.code == "A11Y.ASSIST.GUARD.TEXT.PARENTHETICALS_FORBIDDEN"
|
|
455
|
+
for issue in exc_info.value.issues
|
|
456
|
+
)
|
|
457
|
+
|
|
458
|
+
|
|
459
|
+
def test_guard_brackets_also_forbidden():
|
|
460
|
+
"""Screen-reader profile should forbid square brackets too."""
|
|
461
|
+
base = AssistResult(
|
|
462
|
+
anchored_id="TEST.ERROR.001",
|
|
463
|
+
confidence="High",
|
|
464
|
+
safest_next_step="Run the command",
|
|
465
|
+
plan=["Step 1 [optional]"], # Square brackets!
|
|
466
|
+
next_safe_commands=[],
|
|
467
|
+
notes=[],
|
|
468
|
+
)
|
|
469
|
+
|
|
470
|
+
profiled = AssistResult(
|
|
471
|
+
anchored_id="TEST.ERROR.001",
|
|
472
|
+
confidence="High",
|
|
473
|
+
safest_next_step="Run the command",
|
|
474
|
+
plan=["Step 1 [optional]"],
|
|
475
|
+
next_safe_commands=[],
|
|
476
|
+
notes=[],
|
|
477
|
+
)
|
|
478
|
+
|
|
479
|
+
ctx = get_guard_context(
|
|
480
|
+
profile="screen-reader",
|
|
481
|
+
confidence="High",
|
|
482
|
+
input_kind="cli_error_json",
|
|
483
|
+
allowed_commands=set(),
|
|
484
|
+
)
|
|
485
|
+
|
|
486
|
+
with pytest.raises(GuardViolation) as exc_info:
|
|
487
|
+
validate_profile_transform("test content optional command", base, profiled, ctx)
|
|
488
|
+
|
|
489
|
+
assert any(
|
|
490
|
+
issue.code == "A11Y.ASSIST.GUARD.TEXT.PARENTHETICALS_FORBIDDEN"
|
|
491
|
+
for issue in exc_info.value.issues
|
|
492
|
+
)
|
|
493
|
+
|
|
494
|
+
|
|
495
|
+
def test_guard_visual_refs_forbidden():
|
|
496
|
+
"""Screen-reader profile should forbid visual navigation references."""
|
|
497
|
+
base = AssistResult(
|
|
498
|
+
anchored_id="TEST.ERROR.001",
|
|
499
|
+
confidence="High",
|
|
500
|
+
safest_next_step="Run the command",
|
|
501
|
+
plan=["Step 1"],
|
|
502
|
+
next_safe_commands=[],
|
|
503
|
+
notes=[],
|
|
504
|
+
)
|
|
505
|
+
|
|
506
|
+
profiled = AssistResult(
|
|
507
|
+
anchored_id="TEST.ERROR.001",
|
|
508
|
+
confidence="High",
|
|
509
|
+
safest_next_step="See above for details", # Visual ref!
|
|
510
|
+
plan=["Step 1"],
|
|
511
|
+
next_safe_commands=[],
|
|
512
|
+
notes=[],
|
|
513
|
+
)
|
|
514
|
+
|
|
515
|
+
ctx = get_guard_context(
|
|
516
|
+
profile="screen-reader",
|
|
517
|
+
confidence="High",
|
|
518
|
+
input_kind="cli_error_json",
|
|
519
|
+
allowed_commands=set(),
|
|
520
|
+
)
|
|
521
|
+
|
|
522
|
+
with pytest.raises(GuardViolation) as exc_info:
|
|
523
|
+
validate_profile_transform("test content details command", base, profiled, ctx)
|
|
524
|
+
|
|
525
|
+
assert any(
|
|
526
|
+
issue.code == "A11Y.ASSIST.GUARD.TEXT.VISUAL_REFS_FORBIDDEN"
|
|
527
|
+
for issue in exc_info.value.issues
|
|
528
|
+
)
|
|
529
|
+
|
|
530
|
+
|
|
531
|
+
def test_guard_visual_refs_below_forbidden():
|
|
532
|
+
"""Screen-reader should forbid 'below' visual reference."""
|
|
533
|
+
base = AssistResult(
|
|
534
|
+
anchored_id="TEST.ERROR.001",
|
|
535
|
+
confidence="High",
|
|
536
|
+
safest_next_step="Check the output",
|
|
537
|
+
plan=["See below for instructions"], # Visual ref!
|
|
538
|
+
next_safe_commands=[],
|
|
539
|
+
notes=[],
|
|
540
|
+
)
|
|
541
|
+
|
|
542
|
+
profiled = AssistResult(
|
|
543
|
+
anchored_id="TEST.ERROR.001",
|
|
544
|
+
confidence="High",
|
|
545
|
+
safest_next_step="Check the output",
|
|
546
|
+
plan=["See below for instructions"],
|
|
547
|
+
next_safe_commands=[],
|
|
548
|
+
notes=[],
|
|
549
|
+
)
|
|
550
|
+
|
|
551
|
+
ctx = get_guard_context(
|
|
552
|
+
profile="screen-reader",
|
|
553
|
+
confidence="High",
|
|
554
|
+
input_kind="cli_error_json",
|
|
555
|
+
allowed_commands=set(),
|
|
556
|
+
)
|
|
557
|
+
|
|
558
|
+
with pytest.raises(GuardViolation) as exc_info:
|
|
559
|
+
validate_profile_transform("test content instructions output", base, profiled, ctx)
|
|
560
|
+
|
|
561
|
+
assert any(
|
|
562
|
+
issue.code == "A11Y.ASSIST.GUARD.TEXT.VISUAL_REFS_FORBIDDEN"
|
|
563
|
+
for issue in exc_info.value.issues
|
|
564
|
+
)
|
|
565
|
+
|
|
566
|
+
|
|
567
|
+
def test_guard_lowvision_allows_parentheticals():
|
|
568
|
+
"""Lowvision profile should allow parentheticals."""
|
|
569
|
+
base = AssistResult(
|
|
570
|
+
anchored_id="TEST.ERROR.001",
|
|
571
|
+
confidence="High",
|
|
572
|
+
safest_next_step="Run the command (see docs)", # Parenthetical OK
|
|
573
|
+
plan=["Step 1"],
|
|
574
|
+
next_safe_commands=[],
|
|
575
|
+
notes=[],
|
|
576
|
+
)
|
|
577
|
+
|
|
578
|
+
profiled = AssistResult(
|
|
579
|
+
anchored_id="TEST.ERROR.001",
|
|
580
|
+
confidence="High",
|
|
581
|
+
safest_next_step="Run the command (see docs)",
|
|
582
|
+
plan=["Step 1"],
|
|
583
|
+
next_safe_commands=[],
|
|
584
|
+
notes=[],
|
|
585
|
+
)
|
|
586
|
+
|
|
587
|
+
ctx = get_guard_context(
|
|
588
|
+
profile="lowvision",
|
|
589
|
+
confidence="High",
|
|
590
|
+
input_kind="cli_error_json",
|
|
591
|
+
allowed_commands=set(),
|
|
592
|
+
)
|
|
593
|
+
|
|
594
|
+
# Should not raise - lowvision allows parentheticals
|
|
595
|
+
validate_profile_transform("test content docs command", base, profiled, ctx)
|
|
596
|
+
|
|
597
|
+
|
|
598
|
+
# Test: Content Support (WARN level - doesn't fail but is tracked)
|
|
599
|
+
|
|
600
|
+
|
|
601
|
+
def test_guard_unsupported_content_is_warn_not_error():
|
|
602
|
+
"""Unsupported content should produce WARN, not fail."""
|
|
603
|
+
base = AssistResult(
|
|
604
|
+
anchored_id="TEST.ERROR.001",
|
|
605
|
+
confidence="High",
|
|
606
|
+
safest_next_step="Check configuration.",
|
|
607
|
+
plan=["Step 1"],
|
|
608
|
+
next_safe_commands=[],
|
|
609
|
+
notes=[],
|
|
610
|
+
)
|
|
611
|
+
|
|
612
|
+
profiled = AssistResult(
|
|
613
|
+
anchored_id="TEST.ERROR.001",
|
|
614
|
+
confidence="High",
|
|
615
|
+
safest_next_step="Contact the administrator immediately.", # Not in base
|
|
616
|
+
plan=["Step 1"],
|
|
617
|
+
next_safe_commands=[],
|
|
618
|
+
notes=[],
|
|
619
|
+
)
|
|
620
|
+
|
|
621
|
+
ctx = get_guard_context(
|
|
622
|
+
profile="lowvision",
|
|
623
|
+
confidence="High",
|
|
624
|
+
input_kind="cli_error_json",
|
|
625
|
+
allowed_commands=set(),
|
|
626
|
+
)
|
|
627
|
+
|
|
628
|
+
# Should NOT raise - WARN level doesn't fail
|
|
629
|
+
# Content support is a WARN, not an ERROR
|
|
630
|
+
validate_profile_transform("Check configuration and retry.", base, profiled, ctx)
|
|
631
|
+
|
|
632
|
+
|
|
633
|
+
# Test: GuardContext Factory
|
|
634
|
+
|
|
635
|
+
|
|
636
|
+
def test_get_guard_context_lowvision():
|
|
637
|
+
"""Lowvision profile should have correct constraints."""
|
|
638
|
+
ctx = get_guard_context(
|
|
639
|
+
profile="lowvision",
|
|
640
|
+
confidence="High",
|
|
641
|
+
input_kind="cli_error_json",
|
|
642
|
+
allowed_commands={"cmd1", "cmd2"},
|
|
643
|
+
)
|
|
644
|
+
|
|
645
|
+
assert ctx.profile == "lowvision"
|
|
646
|
+
assert ctx.max_steps == 5
|
|
647
|
+
assert ctx.forbid_parentheticals is False
|
|
648
|
+
assert ctx.forbid_visual_refs is False
|
|
649
|
+
assert ctx.allow_commands_on_low is False
|
|
650
|
+
|
|
651
|
+
|
|
652
|
+
def test_get_guard_context_cognitive_load():
|
|
653
|
+
"""Cognitive-load profile should have correct constraints."""
|
|
654
|
+
ctx = get_guard_context(
|
|
655
|
+
profile="cognitive-load",
|
|
656
|
+
confidence="Medium",
|
|
657
|
+
input_kind="raw_text",
|
|
658
|
+
allowed_commands=set(),
|
|
659
|
+
)
|
|
660
|
+
|
|
661
|
+
assert ctx.profile == "cognitive-load"
|
|
662
|
+
assert ctx.max_steps == 3
|
|
663
|
+
assert ctx.forbid_parentheticals is False
|
|
664
|
+
assert ctx.forbid_visual_refs is False
|
|
665
|
+
|
|
666
|
+
|
|
667
|
+
def test_get_guard_context_screen_reader_high():
|
|
668
|
+
"""Screen-reader profile on High confidence should allow 5 steps."""
|
|
669
|
+
ctx = get_guard_context(
|
|
670
|
+
profile="screen-reader",
|
|
671
|
+
confidence="High",
|
|
672
|
+
input_kind="cli_error_json",
|
|
673
|
+
allowed_commands=set(),
|
|
674
|
+
)
|
|
675
|
+
|
|
676
|
+
assert ctx.profile == "screen-reader"
|
|
677
|
+
assert ctx.max_steps == 5
|
|
678
|
+
assert ctx.forbid_parentheticals is True
|
|
679
|
+
assert ctx.forbid_visual_refs is True
|
|
680
|
+
|
|
681
|
+
|
|
682
|
+
def test_get_guard_context_screen_reader_low():
|
|
683
|
+
"""Screen-reader profile on Low confidence should allow only 3 steps."""
|
|
684
|
+
ctx = get_guard_context(
|
|
685
|
+
profile="screen-reader",
|
|
686
|
+
confidence="Low",
|
|
687
|
+
input_kind="raw_text",
|
|
688
|
+
allowed_commands=set(),
|
|
689
|
+
)
|
|
690
|
+
|
|
691
|
+
assert ctx.profile == "screen-reader"
|
|
692
|
+
assert ctx.max_steps == 3
|
|
693
|
+
assert ctx.forbid_parentheticals is True
|
|
694
|
+
assert ctx.forbid_visual_refs is True
|
|
695
|
+
|
|
696
|
+
|
|
697
|
+
# Test: Multiple Violations
|
|
698
|
+
|
|
699
|
+
|
|
700
|
+
def test_guard_multiple_violations_reported():
|
|
701
|
+
"""Guard should report all violations, not just the first."""
|
|
702
|
+
base = AssistResult(
|
|
703
|
+
anchored_id="TEST.ERROR.001",
|
|
704
|
+
confidence="High",
|
|
705
|
+
safest_next_step="Run command.",
|
|
706
|
+
plan=["Step 1"],
|
|
707
|
+
next_safe_commands=["safe-cmd"],
|
|
708
|
+
notes=[],
|
|
709
|
+
)
|
|
710
|
+
|
|
711
|
+
profiled = AssistResult(
|
|
712
|
+
anchored_id="CHANGED.ID", # Violation 1: ID changed
|
|
713
|
+
confidence="High",
|
|
714
|
+
safest_next_step="Run command.",
|
|
715
|
+
plan=["Step 1"],
|
|
716
|
+
next_safe_commands=["invented-cmd"], # Violation 2: Command invented
|
|
717
|
+
notes=[],
|
|
718
|
+
)
|
|
719
|
+
|
|
720
|
+
ctx = get_guard_context(
|
|
721
|
+
profile="lowvision",
|
|
722
|
+
confidence="High",
|
|
723
|
+
input_kind="cli_error_json",
|
|
724
|
+
allowed_commands={"safe-cmd"},
|
|
725
|
+
)
|
|
726
|
+
|
|
727
|
+
with pytest.raises(GuardViolation) as exc_info:
|
|
728
|
+
validate_profile_transform("test content", base, profiled, ctx)
|
|
729
|
+
|
|
730
|
+
codes = [issue.code for issue in exc_info.value.issues]
|
|
731
|
+
assert "A11Y.ASSIST.GUARD.ID.CHANGED" in codes
|
|
732
|
+
assert "A11Y.ASSIST.GUARD.COMMANDS.INVENTED" in codes
|
|
733
|
+
|
|
734
|
+
|
|
735
|
+
# Test: GuardViolation String Representation
|
|
736
|
+
|
|
737
|
+
|
|
738
|
+
def test_guard_violation_str():
|
|
739
|
+
"""GuardViolation should have a readable string representation."""
|
|
740
|
+
issues = [
|
|
741
|
+
GuardIssue(
|
|
742
|
+
severity="ERROR",
|
|
743
|
+
code="A11Y.ASSIST.GUARD.ID.INVENTED",
|
|
744
|
+
message="Profile invented an ID",
|
|
745
|
+
details={"base_id": "None", "profiled_id": "FAKE.ID"},
|
|
746
|
+
),
|
|
747
|
+
]
|
|
748
|
+
violation = GuardViolation(issues)
|
|
749
|
+
s = str(violation)
|
|
750
|
+
|
|
751
|
+
assert "Profile guard violation" in s
|
|
752
|
+
assert "A11Y.ASSIST.GUARD.ID.INVENTED" in s
|
|
753
|
+
assert "Profile invented an ID" in s
|
|
754
|
+
assert "FAKE.ID" in s
|
|
755
|
+
|
|
756
|
+
|
|
757
|
+
# Test: Edge Cases
|
|
758
|
+
|
|
759
|
+
|
|
760
|
+
def test_guard_empty_commands_passes():
|
|
761
|
+
"""Guard should pass when both base and profiled have no commands."""
|
|
762
|
+
base = AssistResult(
|
|
763
|
+
anchored_id="TEST.ERROR.001",
|
|
764
|
+
confidence="High",
|
|
765
|
+
safest_next_step="Check manually.",
|
|
766
|
+
plan=["Step 1"],
|
|
767
|
+
next_safe_commands=[],
|
|
768
|
+
notes=[],
|
|
769
|
+
)
|
|
770
|
+
|
|
771
|
+
profiled = AssistResult(
|
|
772
|
+
anchored_id="TEST.ERROR.001",
|
|
773
|
+
confidence="High",
|
|
774
|
+
safest_next_step="Check manually.",
|
|
775
|
+
plan=["Step 1"],
|
|
776
|
+
next_safe_commands=[],
|
|
777
|
+
notes=[],
|
|
778
|
+
)
|
|
779
|
+
|
|
780
|
+
ctx = get_guard_context(
|
|
781
|
+
profile="lowvision",
|
|
782
|
+
confidence="High",
|
|
783
|
+
input_kind="cli_error_json",
|
|
784
|
+
allowed_commands=set(),
|
|
785
|
+
)
|
|
786
|
+
|
|
787
|
+
# Should not raise
|
|
788
|
+
validate_profile_transform("Check manually", base, profiled, ctx)
|
|
789
|
+
|
|
790
|
+
|
|
791
|
+
def test_guard_none_id_preserved():
|
|
792
|
+
"""Guard should pass when None ID is preserved as None."""
|
|
793
|
+
base = AssistResult(
|
|
794
|
+
anchored_id=None,
|
|
795
|
+
confidence="Low",
|
|
796
|
+
safest_next_step="Check output.",
|
|
797
|
+
plan=["Step 1"],
|
|
798
|
+
next_safe_commands=[],
|
|
799
|
+
notes=[],
|
|
800
|
+
)
|
|
801
|
+
|
|
802
|
+
profiled = AssistResult(
|
|
803
|
+
anchored_id=None, # Still None - correct
|
|
804
|
+
confidence="Low",
|
|
805
|
+
safest_next_step="Check output.",
|
|
806
|
+
plan=["Step 1"],
|
|
807
|
+
next_safe_commands=[],
|
|
808
|
+
notes=[],
|
|
809
|
+
)
|
|
810
|
+
|
|
811
|
+
ctx = get_guard_context(
|
|
812
|
+
profile="lowvision",
|
|
813
|
+
confidence="Low",
|
|
814
|
+
input_kind="raw_text",
|
|
815
|
+
allowed_commands=set(),
|
|
816
|
+
)
|
|
817
|
+
|
|
818
|
+
# Should not raise
|
|
819
|
+
validate_profile_transform("Check output", base, profiled, ctx)
|