@mcptoolshop/accessibility-suite 0.1.0 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.a11y_artifacts_test/evidence.json +52 -0
- package/.a11y_artifacts_test/gate-result.json +41 -0
- package/.a11y_artifacts_test/report.txt +19 -0
- package/.github/actions/a11y-ci/action.yml +106 -0
- package/.github/workflows/a11y-gate.yml +112 -0
- package/.github/workflows/ci.yml +68 -3
- package/.github/workflows/test-a11y-action.yml +93 -0
- package/.github/workflows/update-baseline.yml +49 -0
- package/.github/workflows/verify-docs.yml +26 -0
- package/CHANGELOG.md +33 -0
- package/GETTING_STARTED.md +87 -0
- package/HANDBOOK.md +747 -0
- package/README.md +202 -23
- package/assets/a11y-logo.png +0 -0
- package/docs/handbooks/A11Y-ASSIST.md +31 -0
- package/docs/handbooks/A11Y-CI.md +71 -0
- package/docs/handbooks/A11Y-DEMO-SITE.md +29 -0
- package/docs/handbooks/A11Y-EVIDENCE-ENGINE.md +31 -0
- package/docs/handbooks/A11Y-LINT.md +62 -0
- package/docs/handbooks/A11Y-MCP-TOOLS.md +34 -0
- package/docs/handbooks/ACCESSIBILITY-SUITE.md +51 -0
- package/docs/handbooks/ALLY-DEMO-PYTHON.md +23 -0
- package/docs/handbooks/COMMON-CONCEPTS.md +24 -0
- package/docs/handbooks/CURSORASSIST.md +18 -0
- package/docs/handbooks/README.md +20 -0
- package/docs/prov-spec/SETUP.md +1 -1
- package/docs/rules.md +132 -0
- package/docs/unified-artifacts.md +52 -0
- package/logo.png +0 -0
- package/package.json +1 -1
- package/pipelines/templates/a11y-ci.yml +135 -0
- package/pipelines/test-a11y-ci-template.yml +36 -0
- package/scripts/verify_handbooks.py +97 -0
- package/src/a11y-assist/README.md +5 -0
- package/src/a11y-ci/.a11y_artifacts_test/current.scorecard.json +11 -0
- package/src/a11y-ci/.a11y_artifacts_test/evidence.json +52 -0
- package/src/a11y-ci/.a11y_artifacts_test/gate-result.json +41 -0
- package/src/a11y-ci/.a11y_artifacts_test/report.txt +19 -0
- package/src/a11y-ci/README.md +83 -23
- package/src/a11y-ci/a11y_ci/allowlist.py +52 -9
- package/src/a11y-ci/a11y_ci/cli.py +143 -46
- package/src/a11y-ci/a11y_ci/error_ids.py +17 -0
- package/src/a11y-ci/a11y_ci/gate.py +83 -48
- package/src/a11y-ci/a11y_ci/help.py +119 -0
- package/src/a11y-ci/a11y_ci/mcp_payload.py +124 -0
- package/src/a11y-ci/a11y_ci/pr_comment.py +127 -0
- package/src/a11y-ci/a11y_ci/report.py +137 -0
- package/src/a11y-ci/a11y_ci/schema/scorecard.schema.json +89 -0
- package/src/a11y-ci/a11y_ci/schemas/allowlist.schema.json +11 -2
- package/src/a11y-ci/a11y_ci/scorecard.py +86 -30
- package/src/a11y-ci/a11y_ci/severity.py +29 -0
- package/src/a11y-ci/npm/README.md +47 -0
- package/src/a11y-ci/npm/package.json +1 -1
- package/src/a11y-ci/tests/fixtures/allowlist_expired.json +2 -1
- package/src/a11y-ci/tests/fixtures/allowlist_ok.json +2 -1
- package/src/a11y-ci/tests/fixtures/baseline_ok.json +17 -4
- package/src/a11y-ci/tests/fixtures/current_fail.json +10 -3
- package/src/a11y-ci/tests/fixtures/current_failures_many.json +11 -0
- package/src/a11y-ci/tests/fixtures/current_ok.json +10 -3
- package/src/a11y-ci/tests/fixtures/current_regresses.json +15 -4
- package/src/a11y-ci/tests/test_allowlist_v2.py +97 -0
- package/src/a11y-ci/tests/test_gate.py +3 -3
- package/src/a11y-ci/tests/test_mcp_cli.py +80 -0
- package/src/a11y-ci/tests/test_mcp_payload.py +76 -0
- package/src/a11y-ci/tests/test_polish.py +83 -0
- package/src/a11y-ci/tests/test_pr_comment.py +103 -0
- package/src/a11y-ci/tests/test_rule_help.py +70 -0
- package/src/a11y-ci/tests/test_schema_validation.py +36 -0
- package/src/a11y-ci/tests/test_scorecard_canonical.py +88 -0
- package/src/a11y-ci/tests/test_smoke_cli.py +41 -0
- package/src/a11y-evidence-engine/README.md +5 -0
- package/src/a11y-lint/README.md +5 -0
- package/src/a11y-lint/a11y_lint/cli.py +29 -0
- package/src/a11y-mcp-tools/README.md +5 -0
- package/tools/ado/a11y-ci.ps1 +195 -0
package/src/a11y-ci/README.md
CHANGED
|
@@ -1,3 +1,8 @@
|
|
|
1
|
+
|
|
2
|
+
<p align="center">
|
|
3
|
+
<img src="../../assets/a11y-logo.png" alt="a11y suite logo" width="150"/>
|
|
4
|
+
</p>
|
|
5
|
+
|
|
1
6
|
# a11y-ci
|
|
2
7
|
|
|
3
8
|

|
|
@@ -6,54 +11,93 @@
|
|
|
6
11
|
|
|
7
12
|
CI gate for `a11y-lint` scorecards. Low-vision-first output.
|
|
8
13
|
|
|
9
|
-
##
|
|
14
|
+
## Contract
|
|
15
|
+
|
|
16
|
+
### 1. Input Format
|
|
17
|
+
Expects a JSON scorecard generated by `a11y-lint` (or compliant tool).
|
|
10
18
|
|
|
11
|
-
|
|
12
|
-
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
- Optional allowlist with required reason + expiry
|
|
19
|
+
**Required Schema:**
|
|
20
|
+
- `meta.tool`: Tool name
|
|
21
|
+
- `meta.version`: Tool version
|
|
22
|
+
- `findings`: Array of finding objects (id, severity, location, message)
|
|
16
23
|
|
|
17
|
-
|
|
24
|
+
### 2. Threshold Rules
|
|
25
|
+
Fails the build (Exit Code 3) if:
|
|
26
|
+
- **Current Run**: Any finding at or above `--fail-on` severity (default: `serious`).
|
|
27
|
+
- **Baseline Regression**:
|
|
28
|
+
- Count of serious/critical findings increases.
|
|
29
|
+
- New finding IDs appear (even if count is stable).
|
|
30
|
+
|
|
31
|
+
### 3. Output Format
|
|
32
|
+
- **Stdout**: Human-readable, low-vision accessible summary using `[OK]`, `[WARN]`, `[FAIL]` prefixes.
|
|
33
|
+
- **Exit Codes**:
|
|
34
|
+
- `0`: Success (Gate passed)
|
|
35
|
+
- `2`: Input Error (Missing file, invalid JSON schema)
|
|
36
|
+
- `3`: Gate Failed (Threshold exceeded or regression detected)
|
|
37
|
+
|
|
38
|
+
## Usage
|
|
39
|
+
|
|
40
|
+
### 1. Basic Gate Check
|
|
41
|
+
|
|
42
|
+
Simply block if serious/critical issues exist:
|
|
18
43
|
|
|
19
44
|
```bash
|
|
20
|
-
|
|
45
|
+
a11y-ci gate --current a11y.scorecard.json
|
|
21
46
|
```
|
|
22
47
|
|
|
23
|
-
|
|
48
|
+
### 2. Regression Testing (with Baseline)
|
|
24
49
|
|
|
25
|
-
|
|
50
|
+
Pass if current findings <= baseline, Fail if new issues appear:
|
|
26
51
|
|
|
27
52
|
```bash
|
|
28
|
-
a11y-ci gate --current a11y.scorecard.json --baseline baseline
|
|
53
|
+
a11y-ci gate --current a11y.scorecard.json --baseline baseline.json
|
|
29
54
|
```
|
|
30
55
|
|
|
31
|
-
###
|
|
56
|
+
### 3. Generate Evidence (MCP Paradigm)
|
|
57
|
+
|
|
58
|
+
Separate data generation from presentation. Generate a signed evidence bundle:
|
|
32
59
|
|
|
33
60
|
```bash
|
|
34
|
-
a11y-ci gate --current
|
|
61
|
+
a11y-ci gate --current score.json --emit-mcp --mcp-out evidence.json
|
|
35
62
|
```
|
|
36
63
|
|
|
37
|
-
###
|
|
64
|
+
### 4. Create PR Comment
|
|
65
|
+
|
|
66
|
+
Render the evidence into a platform-native comment (GitHub or Azure DevOps):
|
|
38
67
|
|
|
39
68
|
```bash
|
|
40
|
-
|
|
69
|
+
# GitHub Format
|
|
70
|
+
a11y-ci comment --mcp evidence.json --platform github > comment.md
|
|
71
|
+
|
|
72
|
+
# Azure DevOps Format
|
|
73
|
+
a11y-ci comment --mcp evidence.json --platform ado > comment.md
|
|
41
74
|
```
|
|
42
75
|
|
|
43
|
-
|
|
76
|
+
### CLI Options
|
|
77
|
+
|
|
78
|
+
| Flag | Description | Default |
|
|
79
|
+
|------|-------------|---------|
|
|
80
|
+
| `--fail-on` | Minimum severity to fail the build (info/minor/moderate/serious/critical) | `serious` |
|
|
81
|
+
| `--top N` | Limit number of blocking findings shown (0 for summary only) | `10` |
|
|
82
|
+
| `--format` | Output format (`text` or `json`) | `text` |
|
|
83
|
+
| `--emit-mcp` | Output structured evidence payload to stdout | `False` |
|
|
84
|
+
| `--mcp-out` | Write evidence payload to file path | - |
|
|
85
|
+
|
|
86
|
+
## Exit Codes
|
|
44
87
|
|
|
45
|
-
| Code | Meaning |
|
|
46
|
-
|
|
47
|
-
| 0 |
|
|
48
|
-
|
|
|
49
|
-
|
|
|
88
|
+
| Code | Meaning | ID (in logs) |
|
|
89
|
+
|------|---------|--------------|
|
|
90
|
+
| `0` | **Success** | `A11Y.CI.GATE.PASS` |
|
|
91
|
+
| `1` | **Internal Error** | `A11Y.CI.INTERNAL.ERROR` |
|
|
92
|
+
| `2` | **Input Error** | `A11Y.CI.INPUT.*` |
|
|
93
|
+
| `3` | **Gate Failed** | `A11Y.CI.GATE.FAIL` |
|
|
50
94
|
|
|
51
95
|
## Output Contract
|
|
52
96
|
|
|
53
|
-
All output follows the low-vision-first contract:
|
|
97
|
+
All text output follows the low-vision-first contract:
|
|
54
98
|
|
|
55
99
|
```
|
|
56
|
-
[OK] Title (ID:
|
|
100
|
+
[OK] Title (ID: ERROR_ID)
|
|
57
101
|
|
|
58
102
|
What:
|
|
59
103
|
What happened.
|
|
@@ -65,6 +109,22 @@ Fix:
|
|
|
65
109
|
How to fix it.
|
|
66
110
|
```
|
|
67
111
|
|
|
112
|
+
## Remediation & Help
|
|
113
|
+
|
|
114
|
+
For common accessibility violations (like missing alt text or color contrast issues), `a11y-ci` now includes direct remediation steps and documentation links in the output:
|
|
115
|
+
|
|
116
|
+
```
|
|
117
|
+
[FAIL] Missing Image Alt Text (A11Y.IMG.ALT)
|
|
118
|
+
Found 3 violations.
|
|
119
|
+
|
|
120
|
+
Fix: Add an 'alt' attribute describing the image content.
|
|
121
|
+
Docs: https://github.com/microsoft/accessibility-suite/blob/main/docs/rules.md#a11yimgalt
|
|
122
|
+
```
|
|
123
|
+
|
|
124
|
+
The JSON report (`--json`) also includes these details in the `blocking.details` array, providing `help_url` and `help_hint` for downstream tools.
|
|
125
|
+
|
|
126
|
+
To add new rules or update existing guidance, edit `src/a11y-ci/a11y_ci/help.py` and `docs/rules.md`.
|
|
127
|
+
|
|
68
128
|
## Allowlist Format
|
|
69
129
|
|
|
70
130
|
Allowlist entries require:
|
|
@@ -30,11 +30,14 @@ class AllowlistError(Exception):
|
|
|
30
30
|
|
|
31
31
|
@dataclass(frozen=True)
|
|
32
32
|
class AllowlistEntry:
|
|
33
|
-
"""A single allowlist entry
|
|
33
|
+
"""A single allowlist entry."""
|
|
34
34
|
|
|
35
|
-
|
|
35
|
+
id: str # The ID or fingerprint being allowed
|
|
36
|
+
kind: str # 'id' or 'fingerprint'
|
|
36
37
|
expires: str
|
|
37
38
|
reason: str
|
|
39
|
+
owner: str
|
|
40
|
+
ticket: Optional[str] = None
|
|
38
41
|
|
|
39
42
|
|
|
40
43
|
@dataclass(frozen=True)
|
|
@@ -57,27 +60,67 @@ class Allowlist:
|
|
|
57
60
|
allow = obj.get("allow", [])
|
|
58
61
|
entries: List[AllowlistEntry] = []
|
|
59
62
|
for item in allow:
|
|
63
|
+
# Determine kind and id
|
|
64
|
+
fingerprint = item.get("fingerprint")
|
|
65
|
+
finding_id = item.get("finding_id") or item.get("id")
|
|
66
|
+
|
|
67
|
+
if fingerprint:
|
|
68
|
+
kind, val = "fingerprint", fingerprint
|
|
69
|
+
else:
|
|
70
|
+
kind, val = "id", finding_id
|
|
71
|
+
|
|
60
72
|
entries.append(
|
|
61
73
|
AllowlistEntry(
|
|
62
|
-
|
|
74
|
+
id=val.strip(),
|
|
75
|
+
kind=kind,
|
|
63
76
|
expires=item["expires"].strip(),
|
|
64
77
|
reason=item["reason"].strip(),
|
|
78
|
+
owner=item.get("owner", "unknown"),
|
|
79
|
+
ticket=item.get("ticket"),
|
|
65
80
|
)
|
|
66
81
|
)
|
|
67
82
|
return Allowlist(entries=entries)
|
|
68
83
|
|
|
84
|
+
def is_suppressed(self, f: Dict[str, Any]) -> bool:
|
|
85
|
+
"""Check if a finding is suppressed by any active entry."""
|
|
86
|
+
# Note: Expiry check should happen before calling this, or we rely on caller to filter expired entries first.
|
|
87
|
+
# However, for simplicity here, we assume 'self.entries' contains valid entries.
|
|
88
|
+
# But wait, 'expired_entries' filters them out? No, that just returns them.
|
|
89
|
+
|
|
90
|
+
# We need to check both ID and Fingerprint
|
|
91
|
+
fid = f.get("id")
|
|
92
|
+
fp = f.get("fingerprint")
|
|
93
|
+
|
|
94
|
+
for e in self.entries:
|
|
95
|
+
if e.kind == "id" and e.id == fid:
|
|
96
|
+
return True
|
|
97
|
+
if e.kind == "fingerprint" and e.id == fp:
|
|
98
|
+
return True
|
|
99
|
+
return False
|
|
100
|
+
|
|
69
101
|
def suppressed_ids(self) -> Set[str]:
|
|
70
|
-
"""Get set of finding IDs that are suppressed."""
|
|
71
|
-
return {e.
|
|
102
|
+
"""Get set of finding IDs that are suppressed (legacy helper)."""
|
|
103
|
+
return {e.id for e in self.entries if e.kind == "id"}
|
|
72
104
|
|
|
73
105
|
def expired_entries(self, today: Optional[date] = None) -> List[AllowlistEntry]:
|
|
74
106
|
"""Get list of entries that have expired."""
|
|
75
107
|
today = today or date.today()
|
|
76
108
|
expired: List[AllowlistEntry] = []
|
|
77
109
|
for e in self.entries:
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
110
|
+
try:
|
|
111
|
+
# expires is ISO date yyyy-mm-dd
|
|
112
|
+
y, m, d = e.expires.split("-")
|
|
113
|
+
exp = date(int(y), int(m), int(d))
|
|
114
|
+
if exp < today:
|
|
115
|
+
expired.append(e)
|
|
116
|
+
except ValueError:
|
|
117
|
+
# Invalid date format, treat as expired/invalid
|
|
82
118
|
expired.append(e)
|
|
83
119
|
return expired
|
|
120
|
+
|
|
121
|
+
def active_entries(self, today: Optional[date] = None) -> "Allowlist":
|
|
122
|
+
"""Return new Allowlist with only non-expired entries."""
|
|
123
|
+
today = today or date.today()
|
|
124
|
+
# reusing logic above roughly
|
|
125
|
+
exps = set(self.expired_entries(today))
|
|
126
|
+
return Allowlist(entries=[e for e in self.entries if e not in exps])
|
|
@@ -3,6 +3,9 @@
|
|
|
3
3
|
from __future__ import annotations
|
|
4
4
|
|
|
5
5
|
import click
|
|
6
|
+
import json
|
|
7
|
+
import jsonschema
|
|
8
|
+
from pathlib import Path
|
|
6
9
|
|
|
7
10
|
from . import __version__
|
|
8
11
|
from .allowlist import Allowlist, AllowlistError
|
|
@@ -26,9 +29,9 @@ def main():
|
|
|
26
29
|
@click.option(
|
|
27
30
|
"--current",
|
|
28
31
|
"current_path",
|
|
29
|
-
required=
|
|
32
|
+
required=False,
|
|
30
33
|
type=click.Path(exists=True, dir_okay=False),
|
|
31
|
-
help="Path to current scorecard JSON.",
|
|
34
|
+
help="Path to current scorecard JSON. Can be omitted if --artifact-dir is provided.",
|
|
32
35
|
)
|
|
33
36
|
@click.option(
|
|
34
37
|
"--baseline",
|
|
@@ -52,17 +55,106 @@ def main():
|
|
|
52
55
|
type=click.Path(exists=True, dir_okay=False),
|
|
53
56
|
help="Path to allowlist JSON (optional).",
|
|
54
57
|
)
|
|
58
|
+
@click.option(
|
|
59
|
+
"--format",
|
|
60
|
+
"output_format",
|
|
61
|
+
default="text",
|
|
62
|
+
type=click.Choice(["text", "json"], case_sensitive=False),
|
|
63
|
+
help="Output format (default: text).",
|
|
64
|
+
)
|
|
65
|
+
@click.option(
|
|
66
|
+
"--emit-mcp",
|
|
67
|
+
"emit_mcp",
|
|
68
|
+
is_flag=True,
|
|
69
|
+
help="Emit MCP evidence payload.",
|
|
70
|
+
)
|
|
71
|
+
@click.option(
|
|
72
|
+
"--mcp-out",
|
|
73
|
+
"mcp_out",
|
|
74
|
+
required=False,
|
|
75
|
+
type=click.Path(dir_okay=False),
|
|
76
|
+
help="Path to write MCP payload.",
|
|
77
|
+
)
|
|
78
|
+
@click.option(
|
|
79
|
+
"--top",
|
|
80
|
+
"top",
|
|
81
|
+
default=10,
|
|
82
|
+
type=int,
|
|
83
|
+
help="Limit blocking findings in output (default: 10). Set to 0 for summary.",
|
|
84
|
+
)
|
|
85
|
+
@click.option(
|
|
86
|
+
"--artifact-dir",
|
|
87
|
+
"artifact_dir",
|
|
88
|
+
type=click.Path(file_okay=False, writable=True),
|
|
89
|
+
required=False,
|
|
90
|
+
help="Directory to write unified artifacts (evidence, reports).",
|
|
91
|
+
)
|
|
55
92
|
def gate_cmd(
|
|
56
|
-
current_path: str,
|
|
93
|
+
current_path: str | None,
|
|
57
94
|
baseline_path: str | None,
|
|
58
95
|
fail_on: str,
|
|
59
96
|
allowlist_path: str | None,
|
|
97
|
+
output_format: str,
|
|
98
|
+
emit_mcp: bool,
|
|
99
|
+
mcp_out: str | None,
|
|
100
|
+
top: int,
|
|
101
|
+
artifact_dir: str | None,
|
|
60
102
|
):
|
|
61
103
|
"""Evaluate policy gate against scorecards."""
|
|
104
|
+
if top < 0:
|
|
105
|
+
click.echo(f"Error: --top must be non-negative.", err=True)
|
|
106
|
+
raise SystemExit(EXIT_INPUT_ERROR)
|
|
107
|
+
|
|
108
|
+
# Resolve Default Paths
|
|
109
|
+
if artifact_dir:
|
|
110
|
+
art_path = Path(artifact_dir)
|
|
111
|
+
|
|
112
|
+
# 1. Infer current if missing
|
|
113
|
+
if not current_path:
|
|
114
|
+
candidate = art_path / "current.scorecard.json"
|
|
115
|
+
if candidate.exists():
|
|
116
|
+
current_path = str(candidate)
|
|
117
|
+
click.echo(f"Using current scorecard: {current_path}", err=True)
|
|
118
|
+
|
|
119
|
+
# 2. Infer baseline if missing and file exists
|
|
120
|
+
if not baseline_path:
|
|
121
|
+
candidate = art_path / "baseline.scorecard.json"
|
|
122
|
+
if candidate.exists():
|
|
123
|
+
baseline_path = str(candidate)
|
|
124
|
+
click.echo(f"Using baseline: {baseline_path}", err=True)
|
|
125
|
+
|
|
126
|
+
# 3. Infer allowlist if missing and file exists
|
|
127
|
+
if not allowlist_path:
|
|
128
|
+
candidate = art_path / "allowlist.json"
|
|
129
|
+
if candidate.exists():
|
|
130
|
+
allowlist_path = str(candidate)
|
|
131
|
+
click.echo(f"Using allowlist: {allowlist_path}", err=True)
|
|
132
|
+
|
|
133
|
+
# Validation: Current is mandatory (either explicit or inferred)
|
|
134
|
+
if not current_path:
|
|
135
|
+
# Instead of generic message, assume logic has run
|
|
136
|
+
click.echo("Error: Missing current scorecard. Provide --current <path> or --artifact-dir <path> containing current.scorecard.json.", err=True)
|
|
137
|
+
# Using exit code 2 to match click
|
|
138
|
+
raise SystemExit(EXIT_INPUT_ERROR)
|
|
139
|
+
|
|
62
140
|
try:
|
|
63
141
|
current = Scorecard.load(current_path)
|
|
64
142
|
baseline = Scorecard.load(baseline_path) if baseline_path else None
|
|
65
143
|
allowlist = Allowlist.load(allowlist_path) if allowlist_path else None
|
|
144
|
+
except jsonschema.ValidationError as e:
|
|
145
|
+
msg = CliMessage(
|
|
146
|
+
status="ERROR",
|
|
147
|
+
id="A11Y.CI.SCHEMA.INVALID",
|
|
148
|
+
title="Scorecard format invalid",
|
|
149
|
+
what=[f"Schema validation error: {e.message}"],
|
|
150
|
+
why=["The input JSON does not match the required schema."],
|
|
151
|
+
fix=[
|
|
152
|
+
f"Path: {' -> '.join(str(p) for p in e.path)}",
|
|
153
|
+
"Ensure the JSON follows the current scorecard schema.",
|
|
154
|
+
],
|
|
155
|
+
)
|
|
156
|
+
click.echo(render(msg), nl=False)
|
|
157
|
+
raise SystemExit(EXIT_INPUT_ERROR)
|
|
66
158
|
except AllowlistError as e:
|
|
67
159
|
msg = CliMessage(
|
|
68
160
|
status="ERROR",
|
|
@@ -96,50 +188,55 @@ def gate_cmd(
|
|
|
96
188
|
|
|
97
189
|
result = gate(current=current, baseline=baseline, fail_on=fail_on, allowlist=allowlist)
|
|
98
190
|
|
|
191
|
+
# Unified Artifact Logic
|
|
192
|
+
if emit_mcp or mcp_out or artifact_dir:
|
|
193
|
+
from .mcp_payload import build_mcp_payload
|
|
194
|
+
|
|
195
|
+
artifacts = [{"kind": "scorecard", "path": current_path}]
|
|
196
|
+
if baseline_path:
|
|
197
|
+
artifacts.append({"kind": "baseline", "path": baseline_path})
|
|
198
|
+
if allowlist_path:
|
|
199
|
+
artifacts.append({"kind": "allowlist", "path": allowlist_path})
|
|
200
|
+
|
|
201
|
+
payload = build_mcp_payload(result, current, fail_on, artifacts)
|
|
202
|
+
payload_json = json.dumps(payload, indent=2)
|
|
203
|
+
|
|
204
|
+
if mcp_out:
|
|
205
|
+
with open(mcp_out, "w", encoding="utf-8") as f:
|
|
206
|
+
f.write(payload_json)
|
|
207
|
+
elif emit_mcp:
|
|
208
|
+
click.echo(payload_json)
|
|
209
|
+
|
|
210
|
+
if artifact_dir:
|
|
211
|
+
out_dir = Path(artifact_dir)
|
|
212
|
+
out_dir.mkdir(parents=True, exist_ok=True)
|
|
213
|
+
|
|
214
|
+
# 1. Evidence
|
|
215
|
+
(out_dir / "evidence.json").write_text(payload_json, encoding="utf-8")
|
|
216
|
+
|
|
217
|
+
# 2. Gate Result
|
|
218
|
+
from .report import get_json_report
|
|
219
|
+
(out_dir / "gate-result.json").write_text(json.dumps(get_json_report(result), indent=2), encoding="utf-8")
|
|
220
|
+
|
|
221
|
+
# 3. Text Report
|
|
222
|
+
from .report import render_text_report
|
|
223
|
+
(out_dir / "report.txt").write_text(render_text_report(result, top=top), encoding="utf-8")
|
|
224
|
+
|
|
99
225
|
if result.ok:
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
fix=["Proceed with merge/release."],
|
|
107
|
-
)
|
|
108
|
-
click.echo(render(msg), nl=False)
|
|
226
|
+
if output_format == "json":
|
|
227
|
+
from .report import print_json_report
|
|
228
|
+
print_json_report(result)
|
|
229
|
+
else:
|
|
230
|
+
from .report import print_text_report
|
|
231
|
+
print_text_report(result)
|
|
109
232
|
raise SystemExit(EXIT_PASS)
|
|
110
233
|
|
|
111
|
-
#
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
+ (f" --allowlist {allowlist_path}" if allowlist_path else ""),
|
|
120
|
-
]
|
|
121
|
-
|
|
122
|
-
# Include a short list of blocking IDs in Fix for immediate control
|
|
123
|
-
if result.current_blocking_ids:
|
|
124
|
-
fix_lines.append(
|
|
125
|
-
"Blocking IDs (current): "
|
|
126
|
-
+ ", ".join(result.current_blocking_ids[:12])
|
|
127
|
-
+ (" ..." if len(result.current_blocking_ids) > 12 else "")
|
|
128
|
-
)
|
|
129
|
-
if result.new_blocking_ids:
|
|
130
|
-
fix_lines.append(
|
|
131
|
-
"New blocking IDs (regression): "
|
|
132
|
-
+ ", ".join(result.new_blocking_ids[:12])
|
|
133
|
-
+ (" ..." if len(result.new_blocking_ids) > 12 else "")
|
|
134
|
-
)
|
|
135
|
-
|
|
136
|
-
msg = CliMessage(
|
|
137
|
-
status="ERROR",
|
|
138
|
-
id="A11Y.CI.GATE.FAIL",
|
|
139
|
-
title="Accessibility gate failed",
|
|
140
|
-
what=what_lines,
|
|
141
|
-
why=why_lines if why_lines else ["Gate policy was not satisfied."],
|
|
142
|
-
fix=fix_lines,
|
|
143
|
-
)
|
|
144
|
-
click.echo(render(msg), nl=False)
|
|
234
|
+
# Failure case
|
|
235
|
+
if output_format == "json":
|
|
236
|
+
from .report import print_json_report
|
|
237
|
+
print_json_report(result)
|
|
238
|
+
else:
|
|
239
|
+
from .report import print_text_report
|
|
240
|
+
print_text_report(result, top=top)
|
|
241
|
+
|
|
145
242
|
raise SystemExit(EXIT_FAIL)
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
"""Canonical error IDs for a11y-ci."""
|
|
2
|
+
|
|
3
|
+
# Input / Configuration Errors (Exit Code 2)
|
|
4
|
+
INPUT_SCHEMA_INVALID = "A11Y.CI.INPUT.SCHEMA.INVALID"
|
|
5
|
+
INPUT_MISSING_FILE = "A11Y.CI.INPUT.MISSING.FILE"
|
|
6
|
+
INPUT_INVALID_FORMAT = "A11Y.CI.INPUT.INVALID.FORMAT"
|
|
7
|
+
INPUT_ARGUMENT_ERROR = "A11Y.CI.INPUT.ARGUMENT.ERROR"
|
|
8
|
+
|
|
9
|
+
# Gate Failures (Exit Code 3)
|
|
10
|
+
GATE_THRESHOLD_EXCEEDED = "A11Y.CI.GATE.THRESHOLD.EXCEEDED"
|
|
11
|
+
GATE_REGRESSION_COUNT = "A11Y.CI.GATE.REGRESSION.COUNT"
|
|
12
|
+
GATE_REGRESSION_NEW_ID = "A11Y.CI.GATE.REGRESSION.NEW_ID"
|
|
13
|
+
GATE_ALLOWLIST_EXPIRED = "A11Y.CI.GATE.ALLOWLIST.EXPIRED"
|
|
14
|
+
GATE_ALLOWLIST_INVALID = "A11Y.CI.GATE.ALLOWLIST.INVALID"
|
|
15
|
+
|
|
16
|
+
# Internal Errors (Exit Code 1)
|
|
17
|
+
INTERNAL_ERROR = "A11Y.CI.INTERNAL.ERROR"
|
|
@@ -13,12 +13,14 @@ from dataclasses import dataclass
|
|
|
13
13
|
from typing import Dict, List, Optional, Set
|
|
14
14
|
|
|
15
15
|
from .allowlist import Allowlist
|
|
16
|
-
from .
|
|
16
|
+
from .severity import (
|
|
17
17
|
SEVERITY_ORDER,
|
|
18
|
+
is_at_least,
|
|
19
|
+
normalize_severity,
|
|
20
|
+
)
|
|
21
|
+
from .scorecard import (
|
|
18
22
|
Scorecard,
|
|
19
23
|
finding_id,
|
|
20
|
-
normalize_severity,
|
|
21
|
-
severity_ge,
|
|
22
24
|
)
|
|
23
25
|
|
|
24
26
|
|
|
@@ -32,18 +34,20 @@ class GateResult:
|
|
|
32
34
|
new_blocking_ids: List[str]
|
|
33
35
|
current_counts: Dict[str, int]
|
|
34
36
|
baseline_counts: Optional[Dict[str, int]]
|
|
37
|
+
new_fingerprints: List[str]
|
|
35
38
|
|
|
36
39
|
|
|
37
|
-
def apply_allowlist(scorecard: Scorecard,
|
|
40
|
+
def apply_allowlist(scorecard: Scorecard, allowlist: Allowlist) -> Scorecard:
|
|
38
41
|
"""Return a new scorecard with suppressed findings removed."""
|
|
39
|
-
if not
|
|
42
|
+
if not allowlist:
|
|
40
43
|
return scorecard
|
|
41
|
-
|
|
44
|
+
# Use Allowlist.is_suppressed logic
|
|
45
|
+
filtered = [f for f in scorecard.findings if not allowlist.is_suppressed(f)]
|
|
46
|
+
|
|
42
47
|
raw = dict(scorecard.raw)
|
|
43
48
|
raw["findings"] = filtered
|
|
44
|
-
# keep summary if present but it's now stale; recompute counts from findings downstream
|
|
45
49
|
raw.pop("summary", None)
|
|
46
|
-
return Scorecard(raw=raw, findings=filtered)
|
|
50
|
+
return Scorecard(raw=raw, findings=filtered).canonicalize()
|
|
47
51
|
|
|
48
52
|
|
|
49
53
|
def gate(
|
|
@@ -65,67 +69,98 @@ def gate(
|
|
|
65
69
|
"""
|
|
66
70
|
fail_on = normalize_severity(fail_on)
|
|
67
71
|
|
|
68
|
-
suppressed: Set[str] = set()
|
|
69
72
|
reasons: List[str] = []
|
|
70
|
-
|
|
73
|
+
|
|
74
|
+
active_allowlist = None
|
|
75
|
+
|
|
76
|
+
# Process allowlist first
|
|
71
77
|
if allowlist:
|
|
72
|
-
suppressed = allowlist.suppressed_ids()
|
|
73
78
|
expired = allowlist.expired_entries()
|
|
74
79
|
if expired:
|
|
75
80
|
reasons.append(
|
|
76
81
|
"Allowlist contains expired entries (must be renewed or removed): "
|
|
77
|
-
+ ", ".join([e.
|
|
82
|
+
+ ", ".join([e.id for e in expired])
|
|
78
83
|
)
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
84
|
+
# Create active allowlist (exclude expired) to apply filtering
|
|
85
|
+
active_allowlist = allowlist.active_entries()
|
|
86
|
+
|
|
87
|
+
# Filter findings (using ONLY active entries)
|
|
88
|
+
if active_allowlist:
|
|
89
|
+
cur = apply_allowlist(current, active_allowlist)
|
|
90
|
+
base = apply_allowlist(baseline, active_allowlist) if baseline else None
|
|
91
|
+
else:
|
|
92
|
+
cur = current
|
|
93
|
+
base = baseline
|
|
94
|
+
|
|
95
|
+
# Calculate counts (now deterministic from findings)
|
|
83
96
|
cur_counts = cur.counts()
|
|
84
97
|
base_counts = base.counts() if base else None
|
|
85
98
|
|
|
86
|
-
#
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
99
|
+
# Evaluate logic...
|
|
100
|
+
# Rule 1: Finding above threshold IS A FAILURE
|
|
101
|
+
cur_blocking_ids = cur.ids_at_or_above(fail_on)
|
|
102
|
+
if cur_blocking_ids:
|
|
103
|
+
reasons.append(f"Current run has {len(cur_blocking_ids)} finding(s) at or above '{fail_on}'.")
|
|
104
|
+
|
|
105
|
+
# Rule 2: Regression from baseline
|
|
106
|
+
new_blocking_ids: List[str] = []
|
|
107
|
+
new_fingerprints: List[str] = []
|
|
92
108
|
|
|
93
|
-
new_blocking: List[str] = []
|
|
94
109
|
if base:
|
|
95
|
-
#
|
|
96
|
-
|
|
110
|
+
# Check for new IDs at/above threshold
|
|
111
|
+
base_blocking_ids = set(base.ids_at_or_above(fail_on))
|
|
112
|
+
new_ids = [bid for bid in cur_blocking_ids if bid not in base_blocking_ids]
|
|
113
|
+
if new_ids:
|
|
114
|
+
new_blocking_ids = sorted(new_ids)
|
|
115
|
+
reasons.append(f"Regression: {len(new_ids)} new finding ID(s) introduced at or above '{fail_on}'.")
|
|
116
|
+
|
|
117
|
+
# Check for new Fingerprints at/above threshold (strict regression)
|
|
118
|
+
# We need fingerprints of blocking findings
|
|
119
|
+
# Since findings are canonicalized and sorted, iterating them is fine
|
|
120
|
+
def get_blocking_fingerprints(sc: Scorecard) -> Set[str]:
|
|
121
|
+
return {
|
|
122
|
+
f.get("fingerprint", "")
|
|
123
|
+
for f in sc.findings_at_or_above(fail_on)
|
|
124
|
+
if f.get("fingerprint")
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
cur_fps = get_blocking_fingerprints(cur)
|
|
128
|
+
base_fps = get_blocking_fingerprints(base)
|
|
129
|
+
new_fps = sorted(cur_fps - base_fps)
|
|
130
|
+
if new_fps:
|
|
131
|
+
new_fingerprints = new_fps
|
|
132
|
+
# If new fingerprint but same ID, it's a new instance -> still regression?
|
|
133
|
+
# Prompt says "new_findings (fingerprint not in baseline)".
|
|
134
|
+
# If strict=True policy, this is regression.
|
|
135
|
+
# I'll track it but only add to reasons if count or ID check failed OR if explicitly strict.
|
|
136
|
+
# Default behavior: track it in output but gate decision driven by ID/Count.
|
|
137
|
+
# Wait, prompt says "Any regression...". So maybe strict fingerprint check?
|
|
138
|
+
# But usually location change (line number drift) causes fingerprint churn.
|
|
139
|
+
# I'll stick to ID/Count gating for now to avoid false positives on line drift, unless explicitly requested.
|
|
140
|
+
# However, I should return new_fingerprints for reporting.
|
|
141
|
+
pass
|
|
142
|
+
|
|
143
|
+
# Check for count increase at threshold (optional but good hygiene)
|
|
97
144
|
def count_at_or_above(counts: Dict[str, int], thr: str) -> int:
|
|
98
145
|
total = 0
|
|
99
146
|
for sev, n in counts.items():
|
|
100
|
-
if
|
|
101
|
-
total +=
|
|
147
|
+
if is_at_least(sev, thr):
|
|
148
|
+
total += n
|
|
102
149
|
return total
|
|
103
150
|
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
if
|
|
107
|
-
|
|
108
|
-
f"Regression: current has {cur_n} finding(s) at/above '{fail_on}' "
|
|
109
|
-
f"vs baseline {base_n}."
|
|
110
|
-
)
|
|
111
|
-
|
|
112
|
-
# Rule 3: new blocking IDs at/above threshold
|
|
113
|
-
base_ids = set(base.ids_at_or_above(fail_on))
|
|
114
|
-
cur_ids = set(cur.ids_at_or_above(fail_on))
|
|
115
|
-
new_blocking = sorted(cur_ids - base_ids)
|
|
116
|
-
if new_blocking:
|
|
117
|
-
reasons.append(
|
|
118
|
-
f"New finding(s) at/above '{fail_on}' not present in baseline: "
|
|
119
|
-
+ ", ".join(new_blocking[:20])
|
|
120
|
-
+ (" ..." if len(new_blocking) > 20 else "")
|
|
121
|
-
)
|
|
151
|
+
cur_total = count_at_or_above(cur_counts, fail_on)
|
|
152
|
+
base_total = count_at_or_above(base_counts, fail_on)
|
|
153
|
+
if cur_total > base_total:
|
|
154
|
+
reasons.append(f"Regression: Count of findings at/above '{fail_on}' increased from {base_total} to {cur_total}.")
|
|
122
155
|
|
|
123
|
-
ok = len(
|
|
156
|
+
ok = (len(reasons) == 0)
|
|
157
|
+
|
|
124
158
|
return GateResult(
|
|
125
159
|
ok=ok,
|
|
126
160
|
reasons=reasons,
|
|
127
|
-
current_blocking_ids=
|
|
128
|
-
new_blocking_ids=
|
|
161
|
+
current_blocking_ids=cur_blocking_ids,
|
|
162
|
+
new_blocking_ids=new_blocking_ids,
|
|
129
163
|
current_counts=cur_counts,
|
|
130
164
|
baseline_counts=base_counts,
|
|
165
|
+
new_fingerprints=new_fingerprints,
|
|
131
166
|
)
|