agentic-scorecard 0.3.0 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENT_PROMPT.md +68 -52
- package/README.md +50 -37
- package/benchmark/v0.4/adapter-contract.md +129 -0
- package/benchmark/v0.4/adapters/known-agent-harnesses.yaml +266 -0
- package/benchmark/v0.4/adapters/known-ci-providers.yaml +57 -0
- package/benchmark/v0.4/adapters/known-command-wrappers.yaml +37 -0
- package/benchmark/v0.4/adapters/known-environment-tooling.yaml +17 -0
- package/benchmark/v0.4/adapters/known-github-conventions.yaml +27 -0
- package/benchmark/v0.4/adapters/known-python-tooling.yaml +15 -0
- package/benchmark/v0.4/adapters/known-security-tools.yaml +45 -0
- package/benchmark/v0.4/adapters/known-verification-tools.yaml +65 -0
- package/benchmark/v0.4/agent-evidence-schema.json +74 -0
- package/benchmark/v0.4/attestation-schema.json +35 -0
- package/benchmark/v0.4/benchmark.yaml +127 -0
- package/benchmark/v0.4/calibration-matrix.md +60 -0
- package/benchmark/v0.4/controls/context.yaml +75 -0
- package/benchmark/v0.4/controls/environment.yaml +93 -0
- package/benchmark/v0.4/controls/governance.yaml +132 -0
- package/benchmark/v0.4/controls/learning.yaml +78 -0
- package/benchmark/v0.4/controls/observability.yaml +91 -0
- package/benchmark/v0.4/controls/resilience.yaml +170 -0
- package/benchmark/v0.4/controls/security.yaml +143 -0
- package/benchmark/v0.4/controls/specification.yaml +65 -0
- package/benchmark/v0.4/controls/testing.yaml +82 -0
- package/benchmark/v0.4/controls/tooling.yaml +69 -0
- package/benchmark/v0.4/report-schema.json +89 -0
- package/benchmark/v0.4/scoring-policy.md +175 -0
- package/dist/cli.js +2315 -162
- package/package.json +3 -2
- package/templates/agent-evidence.yaml +2 -2
- package/templates/attestations.yaml +3 -1
|
@@ -0,0 +1,266 @@
|
|
|
1
|
+
id: known-agent-harnesses
|
|
2
|
+
benchmark_version: 0.4.0
|
|
3
|
+
extensions:
|
|
4
|
+
- control_id: ADRB-CTX-001
|
|
5
|
+
evidence_index: 0
|
|
6
|
+
patterns:
|
|
7
|
+
[
|
|
8
|
+
CLAUDE.md,
|
|
9
|
+
.github/copilot-instructions.md,
|
|
10
|
+
.claude/**,
|
|
11
|
+
.codex/**,
|
|
12
|
+
.cursor/rules/**,
|
|
13
|
+
.cursor/agents/**,
|
|
14
|
+
.cursor/skills/**,
|
|
15
|
+
.opencode/**,
|
|
16
|
+
]
|
|
17
|
+
- control_id: ADRB-CTX-002
|
|
18
|
+
evidence_index: 0
|
|
19
|
+
files:
|
|
20
|
+
[
|
|
21
|
+
CLAUDE.md,
|
|
22
|
+
.github/copilot-instructions.md,
|
|
23
|
+
.claude/**/*.md,
|
|
24
|
+
.codex/**/*.md,
|
|
25
|
+
.cursor/rules/**/*.md,
|
|
26
|
+
.cursor/agents/**/*.md,
|
|
27
|
+
.opencode/**/*.md,
|
|
28
|
+
]
|
|
29
|
+
- control_id: ADRB-CTX-003
|
|
30
|
+
evidence_index: 0
|
|
31
|
+
ci_tools:
|
|
32
|
+
- id: agent-guidance-validation
|
|
33
|
+
commands:
|
|
34
|
+
- executables: [node, tsx]
|
|
35
|
+
argument_groups:
|
|
36
|
+
- [
|
|
37
|
+
scripts/check-agent-docs.js,
|
|
38
|
+
scripts/check-agent-docs.ts,
|
|
39
|
+
scripts/validate-agent-docs.js,
|
|
40
|
+
scripts/validate-agent-docs.ts,
|
|
41
|
+
]
|
|
42
|
+
source_pattern_groups:
|
|
43
|
+
- [
|
|
44
|
+
'\b(?:readfilesync|readfile)\s*\([^\n]*(?:agents\.md|agent[-_ ](?:guidance|instructions))',
|
|
45
|
+
]
|
|
46
|
+
- [
|
|
47
|
+
'\bthrow\s+new\s+error\s*\(',
|
|
48
|
+
'\bprocess\.exit\s*\(\s*[1-9]',
|
|
49
|
+
'\bprocess\.exitcode\s*=\s*[1-9]',
|
|
50
|
+
]
|
|
51
|
+
- executables: [bash, sh]
|
|
52
|
+
argument_groups:
|
|
53
|
+
- [scripts/check-agent-docs.sh, scripts/validate-agent-docs.sh]
|
|
54
|
+
source_pattern_groups:
|
|
55
|
+
- ['\b(?:grep|test)\b[^\n]*(?:agents\.md|agent[-_ ](?:guidance|instructions))']
|
|
56
|
+
- ['\b(?:exit|return)\s+[1-9]']
|
|
57
|
+
- executables: [python, python3]
|
|
58
|
+
argument_groups:
|
|
59
|
+
- [scripts/check_agent_docs.py, scripts/validate_agent_docs.py]
|
|
60
|
+
source_pattern_groups:
|
|
61
|
+
- [
|
|
62
|
+
'\bopen\s*\([^\n]*(?:agents\.md|agent[-_ ](?:guidance|instructions))',
|
|
63
|
+
'\bpath\s*\([^\n]*(?:agents\.md|agent[-_ ](?:guidance|instructions))[^\n]*\)\s*\.\s*read_text\s*\(',
|
|
64
|
+
'\bread_text\s*\([^\n]*(?:agents\.md|agent[-_ ](?:guidance|instructions))',
|
|
65
|
+
]
|
|
66
|
+
- ['\braise\b', '\bsys\.exit\s*\(\s*[1-9]']
|
|
67
|
+
- control_id: ADRB-SPC-003
|
|
68
|
+
evidence_index: 0
|
|
69
|
+
ci_tools:
|
|
70
|
+
- id: specification-traceability
|
|
71
|
+
commands:
|
|
72
|
+
- executables: [node, tsx]
|
|
73
|
+
argument_groups:
|
|
74
|
+
- [
|
|
75
|
+
scripts/check-specs.js,
|
|
76
|
+
scripts/check-specs.ts,
|
|
77
|
+
scripts/validate-specs.js,
|
|
78
|
+
scripts/validate-specs.ts,
|
|
79
|
+
scripts/check-traceability.js,
|
|
80
|
+
scripts/check-traceability.ts,
|
|
81
|
+
]
|
|
82
|
+
source_pattern_groups:
|
|
83
|
+
- ['\b(?:readfilesync|readfile)\s*\([^\n]*(?:specs?/|requirements?/|\.ai/specs?)']
|
|
84
|
+
- ['\b(?:task|work\s*item|issue)[^\n]*\.(?:exec|match)\s*\(']
|
|
85
|
+
- ['\b(?:implementation|change|source|code)[^\n]*\.(?:exec|match)\s*\(']
|
|
86
|
+
- ['\b(?:verification|tests?)[^\n]*\.(?:exec|match)\s*\(']
|
|
87
|
+
- ['\b(?:readfilesync|readfile)\s*\([^\n]*(?:implementation|change|source|code)\w*']
|
|
88
|
+
- ['\b(?:readfilesync|readfile)\s*\([^\n]*(?:verification|tests?)\w*']
|
|
89
|
+
- [
|
|
90
|
+
'\b(?:implementation|change|source|code)\w*\.includes\s*\([^\n]*(?:work|task|issue)',
|
|
91
|
+
]
|
|
92
|
+
- ['\b(?:verification|tests?)\w*\.includes\s*\([^\n]*(?:work|task|issue)']
|
|
93
|
+
- [
|
|
94
|
+
'\bthrow\s+new\s+error\s*\(',
|
|
95
|
+
'\bprocess\.exit\s*\(\s*[1-9]',
|
|
96
|
+
'\bprocess\.exitcode\s*=\s*[1-9]',
|
|
97
|
+
]
|
|
98
|
+
- control_id: ADRB-LRN-003
|
|
99
|
+
evidence_index: 0
|
|
100
|
+
ci_tools:
|
|
101
|
+
- id: knowledge-curation
|
|
102
|
+
commands:
|
|
103
|
+
- executables: [node, tsx]
|
|
104
|
+
argument_groups:
|
|
105
|
+
- [
|
|
106
|
+
scripts/check-knowledge.js,
|
|
107
|
+
scripts/check-knowledge.ts,
|
|
108
|
+
scripts/validate-knowledge.js,
|
|
109
|
+
scripts/validate-knowledge.ts,
|
|
110
|
+
scripts/curate-knowledge.js,
|
|
111
|
+
scripts/curate-knowledge.ts,
|
|
112
|
+
]
|
|
113
|
+
source_pattern_groups:
|
|
114
|
+
- ['\b(?:readfilesync|readfile)\s*\([^\n]*(?:lessons|memory|agents\.md|knowledge)']
|
|
115
|
+
- ['\bif\s*\([^\n]*(?:duplicate|stale|contradict|unpromoted)']
|
|
116
|
+
- [
|
|
117
|
+
'\bthrow\s+new\s+error\s*\(',
|
|
118
|
+
'\bprocess\.exit\s*\(\s*[1-9]',
|
|
119
|
+
'\bprocess\.exitcode\s*=\s*[1-9]',
|
|
120
|
+
]
|
|
121
|
+
- control_id: ADRB-ENV-001
|
|
122
|
+
evidence_index: 0
|
|
123
|
+
files:
|
|
124
|
+
[
|
|
125
|
+
CLAUDE.md,
|
|
126
|
+
.claude/**/*.md,
|
|
127
|
+
.codex/**/*.md,
|
|
128
|
+
.cursor/agents/**/*.md,
|
|
129
|
+
.cursor/rules/**/*.md,
|
|
130
|
+
.cursor/skills/**/*.md,
|
|
131
|
+
.opencode/**/*.md,
|
|
132
|
+
]
|
|
133
|
+
- control_id: ADRB-SPC-001
|
|
134
|
+
evidence_index: 0
|
|
135
|
+
patterns: [.kiro/specs/**, .cursor/plans/**, .codex/plans/**]
|
|
136
|
+
- control_id: ADRB-SPC-002
|
|
137
|
+
evidence_index: 0
|
|
138
|
+
files: [.kiro/specs/**, .cursor/plans/**, .codex/plans/**]
|
|
139
|
+
- control_id: ADRB-TOL-001
|
|
140
|
+
evidence_index: 0
|
|
141
|
+
patterns:
|
|
142
|
+
[
|
|
143
|
+
.claude/**,
|
|
144
|
+
.codex/**,
|
|
145
|
+
.cursor/**,
|
|
146
|
+
.opencode/**,
|
|
147
|
+
.github/copilot-instructions.md,
|
|
148
|
+
opencode.json,
|
|
149
|
+
]
|
|
150
|
+
- control_id: ADRB-TOL-002
|
|
151
|
+
evidence_index: 0
|
|
152
|
+
files:
|
|
153
|
+
[
|
|
154
|
+
CLAUDE.md,
|
|
155
|
+
.claude/**/*.md,
|
|
156
|
+
.codex/**/*.md,
|
|
157
|
+
.cursor/agents/**/*.md,
|
|
158
|
+
.cursor/rules/**/*.md,
|
|
159
|
+
.cursor/skills/**/*.md,
|
|
160
|
+
.opencode/**/*.md,
|
|
161
|
+
]
|
|
162
|
+
- control_id: ADRB-SEC-001
|
|
163
|
+
evidence_index: 0
|
|
164
|
+
files:
|
|
165
|
+
[
|
|
166
|
+
CLAUDE.md,
|
|
167
|
+
.claude/**/*.md,
|
|
168
|
+
.codex/**/*.md,
|
|
169
|
+
.cursor/agents/**/*.md,
|
|
170
|
+
.cursor/rules/**/*.md,
|
|
171
|
+
.cursor/skills/**/*.md,
|
|
172
|
+
.opencode/**/*.md,
|
|
173
|
+
]
|
|
174
|
+
- control_id: ADRB-SEC-002
|
|
175
|
+
evidence_index: 0
|
|
176
|
+
files:
|
|
177
|
+
[
|
|
178
|
+
CLAUDE.md,
|
|
179
|
+
.claude/**/*.md,
|
|
180
|
+
.codex/**/*.md,
|
|
181
|
+
.cursor/agents/**/*.md,
|
|
182
|
+
.cursor/rules/**/*.md,
|
|
183
|
+
.cursor/skills/**/*.md,
|
|
184
|
+
.opencode/**/*.md,
|
|
185
|
+
]
|
|
186
|
+
- control_id: ADRB-TST-002
|
|
187
|
+
evidence_index: 0
|
|
188
|
+
files:
|
|
189
|
+
[
|
|
190
|
+
CLAUDE.md,
|
|
191
|
+
.claude/**/*.md,
|
|
192
|
+
.codex/**/*.md,
|
|
193
|
+
.cursor/agents/**/*.md,
|
|
194
|
+
.cursor/rules/**/*.md,
|
|
195
|
+
.cursor/skills/**/*.md,
|
|
196
|
+
.opencode/**/*.md,
|
|
197
|
+
]
|
|
198
|
+
- control_id: ADRB-GOV-001
|
|
199
|
+
evidence_index: 0
|
|
200
|
+
files:
|
|
201
|
+
[
|
|
202
|
+
CLAUDE.md,
|
|
203
|
+
.claude/**/*.md,
|
|
204
|
+
.codex/**/*.md,
|
|
205
|
+
.cursor/agents/**/*.md,
|
|
206
|
+
.cursor/rules/**/*.md,
|
|
207
|
+
.cursor/skills/**/*.md,
|
|
208
|
+
.opencode/**/*.md,
|
|
209
|
+
]
|
|
210
|
+
- control_id: ADRB-GOV-002
|
|
211
|
+
evidence_index: 1
|
|
212
|
+
files:
|
|
213
|
+
[
|
|
214
|
+
CLAUDE.md,
|
|
215
|
+
.claude/**/*.md,
|
|
216
|
+
.codex/**/*.md,
|
|
217
|
+
.cursor/agents/**/*.md,
|
|
218
|
+
.cursor/rules/**/*.md,
|
|
219
|
+
.cursor/skills/**/*.md,
|
|
220
|
+
.opencode/**/*.md,
|
|
221
|
+
]
|
|
222
|
+
- control_id: ADRB-LRN-001
|
|
223
|
+
evidence_index: 0
|
|
224
|
+
patterns: [.claude/memory/**, .codex/memory/**, .cursor/memory/**, .opencode/memory/**]
|
|
225
|
+
- control_id: ADRB-LRN-002
|
|
226
|
+
evidence_index: 0
|
|
227
|
+
files:
|
|
228
|
+
[
|
|
229
|
+
CLAUDE.md,
|
|
230
|
+
.claude/**/*.md,
|
|
231
|
+
.codex/**/*.md,
|
|
232
|
+
.cursor/agents/**/*.md,
|
|
233
|
+
.cursor/rules/**/*.md,
|
|
234
|
+
.cursor/skills/**/*.md,
|
|
235
|
+
.opencode/**/*.md,
|
|
236
|
+
]
|
|
237
|
+
- control_id: ADRB-OBS-001
|
|
238
|
+
evidence_index: 0
|
|
239
|
+
files: [.cursor/runs/README*, .codex/runs/README*, .claude/runs/README*]
|
|
240
|
+
- control_id: ADRB-OBS-002
|
|
241
|
+
evidence_index: 0
|
|
242
|
+
files: [.cursor/runs/README*, .codex/runs/README*, .claude/runs/README*]
|
|
243
|
+
- control_id: ADRB-RES-001
|
|
244
|
+
evidence_index: 0
|
|
245
|
+
files:
|
|
246
|
+
[
|
|
247
|
+
CLAUDE.md,
|
|
248
|
+
.claude/**/*.md,
|
|
249
|
+
.codex/**/*.md,
|
|
250
|
+
.cursor/agents/**/*.md,
|
|
251
|
+
.cursor/rules/**/*.md,
|
|
252
|
+
.cursor/skills/**/*.md,
|
|
253
|
+
.opencode/**/*.md,
|
|
254
|
+
]
|
|
255
|
+
- control_id: ADRB-RES-002
|
|
256
|
+
evidence_index: 0
|
|
257
|
+
files:
|
|
258
|
+
[
|
|
259
|
+
CLAUDE.md,
|
|
260
|
+
.claude/**/*.md,
|
|
261
|
+
.codex/**/*.md,
|
|
262
|
+
.cursor/agents/**/*.md,
|
|
263
|
+
.cursor/rules/**/*.md,
|
|
264
|
+
.cursor/skills/**/*.md,
|
|
265
|
+
.opencode/**/*.md,
|
|
266
|
+
]
|
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
id: known-ci-providers
|
|
2
|
+
benchmark_version: 0.4.0
|
|
3
|
+
extensions:
|
|
4
|
+
- control_id: ADRB-ENV-003
|
|
5
|
+
evidence_index: 0
|
|
6
|
+
ci_providers:
|
|
7
|
+
- id: github-actions
|
|
8
|
+
files: [.github/workflows/*.yml, .github/workflows/*.yaml]
|
|
9
|
+
- id: gitlab-ci
|
|
10
|
+
files: [.gitlab-ci.yml]
|
|
11
|
+
- id: azure-pipelines
|
|
12
|
+
files: [azure-pipelines.yml, azure-pipelines.yaml]
|
|
13
|
+
- control_id: ADRB-LRN-003
|
|
14
|
+
evidence_index: 0
|
|
15
|
+
ci_providers:
|
|
16
|
+
- id: github-actions
|
|
17
|
+
files: [.github/workflows/*.yml, .github/workflows/*.yaml]
|
|
18
|
+
- id: gitlab-ci
|
|
19
|
+
files: [.gitlab-ci.yml]
|
|
20
|
+
- id: azure-pipelines
|
|
21
|
+
files: [azure-pipelines.yml, azure-pipelines.yaml]
|
|
22
|
+
- control_id: ADRB-SPC-003
|
|
23
|
+
evidence_index: 0
|
|
24
|
+
ci_providers:
|
|
25
|
+
- id: github-actions
|
|
26
|
+
files: [.github/workflows/*.yml, .github/workflows/*.yaml]
|
|
27
|
+
- id: gitlab-ci
|
|
28
|
+
files: [.gitlab-ci.yml]
|
|
29
|
+
- id: azure-pipelines
|
|
30
|
+
files: [azure-pipelines.yml, azure-pipelines.yaml]
|
|
31
|
+
- control_id: ADRB-CTX-003
|
|
32
|
+
evidence_index: 0
|
|
33
|
+
ci_providers:
|
|
34
|
+
- id: github-actions
|
|
35
|
+
files: [.github/workflows/*.yml, .github/workflows/*.yaml]
|
|
36
|
+
- id: gitlab-ci
|
|
37
|
+
files: [.gitlab-ci.yml]
|
|
38
|
+
- id: azure-pipelines
|
|
39
|
+
files: [azure-pipelines.yml, azure-pipelines.yaml]
|
|
40
|
+
- control_id: ADRB-SEC-003
|
|
41
|
+
evidence_index: 0
|
|
42
|
+
ci_providers:
|
|
43
|
+
- id: github-actions
|
|
44
|
+
files: [.github/workflows/*.yml, .github/workflows/*.yaml]
|
|
45
|
+
- id: gitlab-ci
|
|
46
|
+
files: [.gitlab-ci.yml]
|
|
47
|
+
- id: azure-pipelines
|
|
48
|
+
files: [azure-pipelines.yml, azure-pipelines.yaml]
|
|
49
|
+
- control_id: ADRB-TST-003
|
|
50
|
+
evidence_index: 0
|
|
51
|
+
ci_providers:
|
|
52
|
+
- id: github-actions
|
|
53
|
+
files: [.github/workflows/*.yml, .github/workflows/*.yaml]
|
|
54
|
+
- id: gitlab-ci
|
|
55
|
+
files: [.gitlab-ci.yml]
|
|
56
|
+
- id: azure-pipelines
|
|
57
|
+
files: [azure-pipelines.yml, azure-pipelines.yaml]
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
id: known-command-wrappers
|
|
2
|
+
benchmark_version: 0.4.0
|
|
3
|
+
ci_invocation_grammar:
|
|
4
|
+
wrapper_options_with_values:
|
|
5
|
+
- --cache
|
|
6
|
+
- --call
|
|
7
|
+
- --chdir
|
|
8
|
+
- --chroot
|
|
9
|
+
- --close-from
|
|
10
|
+
- --command-timeout
|
|
11
|
+
- --from
|
|
12
|
+
- --group
|
|
13
|
+
- --host
|
|
14
|
+
- --node-options
|
|
15
|
+
- --package
|
|
16
|
+
- --prompt
|
|
17
|
+
- --python
|
|
18
|
+
- --user
|
|
19
|
+
- --with
|
|
20
|
+
- -c
|
|
21
|
+
- -d
|
|
22
|
+
- -g
|
|
23
|
+
- -h
|
|
24
|
+
- -p
|
|
25
|
+
- -r
|
|
26
|
+
- -t
|
|
27
|
+
- -u
|
|
28
|
+
wrappers:
|
|
29
|
+
- executable_patterns: ['^(?:bunx|npx|sudo|uvx)$']
|
|
30
|
+
command_prefixes: [[]]
|
|
31
|
+
- executable_patterns: ['^(?:uv|pipx)$']
|
|
32
|
+
command_prefixes: [[run]]
|
|
33
|
+
- executable_patterns: ['^(?:bun|npm|pnpm|yarn)$']
|
|
34
|
+
command_prefixes: [[dlx], [exec], [x]]
|
|
35
|
+
- executable_patterns: ['^python(?:3(?:\.\d+)?)?$']
|
|
36
|
+
command_prefixes: [[-m]]
|
|
37
|
+
extensions: []
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
id: known-environment-tooling
|
|
2
|
+
benchmark_version: 0.4.0
|
|
3
|
+
extensions:
|
|
4
|
+
- control_id: ADRB-ENV-003
|
|
5
|
+
evidence_index: 0
|
|
6
|
+
ci_tools:
|
|
7
|
+
- id: locked-install
|
|
8
|
+
prohibited_arguments: [--dry-run]
|
|
9
|
+
commands:
|
|
10
|
+
- executables: [npm]
|
|
11
|
+
argument_groups: [[ci]]
|
|
12
|
+
- executables: [pnpm, yarn, bun]
|
|
13
|
+
argument_groups: [[install], [--frozen-lockfile, --immutable]]
|
|
14
|
+
- executables: [uv]
|
|
15
|
+
argument_groups: [[sync], [--frozen, --locked]]
|
|
16
|
+
- executables: [cargo]
|
|
17
|
+
argument_groups: [[fetch], [--locked]]
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
id: known-github-conventions
|
|
2
|
+
benchmark_version: 0.4.0
|
|
3
|
+
extensions:
|
|
4
|
+
- control_id: ADRB-GOV-001
|
|
5
|
+
evidence_index: 0
|
|
6
|
+
files: [.github/pull_request_template.md, .github/PULL_REQUEST_TEMPLATE/**]
|
|
7
|
+
- control_id: ADRB-GOV-002
|
|
8
|
+
evidence_index: 0
|
|
9
|
+
patterns: [.github/CODEOWNERS, .github/OWNERS, docs/CODEOWNERS]
|
|
10
|
+
- control_id: ADRB-GOV-002
|
|
11
|
+
evidence_index: 1
|
|
12
|
+
files: [.github/pull_request_template.md]
|
|
13
|
+
- control_id: ADRB-RES-001
|
|
14
|
+
evidence_index: 0
|
|
15
|
+
files: [.github/SECURITY.md]
|
|
16
|
+
- control_id: ADRB-SEC-001
|
|
17
|
+
evidence_index: 0
|
|
18
|
+
files: [.github/SECURITY.md]
|
|
19
|
+
- control_id: ADRB-SEC-006
|
|
20
|
+
evidence_index: 0
|
|
21
|
+
files: [.github/SECURITY.md]
|
|
22
|
+
- control_id: ADRB-SPC-001
|
|
23
|
+
evidence_index: 0
|
|
24
|
+
patterns: [.github/ISSUE_TEMPLATE/**]
|
|
25
|
+
- control_id: ADRB-SPC-002
|
|
26
|
+
evidence_index: 0
|
|
27
|
+
files: [.github/ISSUE_TEMPLATE/**]
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
id: known-python-tooling
|
|
2
|
+
benchmark_version: 0.4.0
|
|
3
|
+
extensions:
|
|
4
|
+
- control_id: ADRB-ENV-001
|
|
5
|
+
evidence_index: 0
|
|
6
|
+
files: [pyproject.toml]
|
|
7
|
+
- control_id: ADRB-ENV-001
|
|
8
|
+
evidence_index: 0
|
|
9
|
+
terms: [requires-python, uv sync, python]
|
|
10
|
+
- control_id: ADRB-TST-002
|
|
11
|
+
evidence_index: 0
|
|
12
|
+
files: [run_quality_checks.sh, scripts/**/quality*.sh, tox.ini, noxfile.py]
|
|
13
|
+
- control_id: ADRB-TST-002
|
|
14
|
+
evidence_index: 0
|
|
15
|
+
terms: [pytest, mypy, flake8, ruff, pyright, tox, nox]
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
id: known-security-tools
|
|
2
|
+
benchmark_version: 0.4.0
|
|
3
|
+
extensions:
|
|
4
|
+
- control_id: ADRB-SEC-003
|
|
5
|
+
evidence_index: 0
|
|
6
|
+
ci_tools:
|
|
7
|
+
- id: gitleaks
|
|
8
|
+
requires_final_exit_status: true
|
|
9
|
+
commands:
|
|
10
|
+
- executables: [gitleaks]
|
|
11
|
+
argument_groups: [[detect]]
|
|
12
|
+
prohibited_arguments: [--exit-code, --log-opts, --source]
|
|
13
|
+
actions: [gitleaks/gitleaks-action]
|
|
14
|
+
- id: trufflehog
|
|
15
|
+
requires_final_exit_status: true
|
|
16
|
+
commands:
|
|
17
|
+
- executables: [trufflehog]
|
|
18
|
+
argument_groups:
|
|
19
|
+
- [git, filesystem]
|
|
20
|
+
- [--fail]
|
|
21
|
+
required_argument_prefixes:
|
|
22
|
+
- [git, '.']
|
|
23
|
+
- [git, ./]
|
|
24
|
+
- [git, 'file://.']
|
|
25
|
+
- [git, 'file://./']
|
|
26
|
+
- [git, '$GITHUB_WORKSPACE']
|
|
27
|
+
- [git, '${GITHUB_WORKSPACE}']
|
|
28
|
+
- [git, '$CI_PROJECT_DIR']
|
|
29
|
+
- [git, '${CI_PROJECT_DIR}']
|
|
30
|
+
- [git, '$(Build.SourcesDirectory)']
|
|
31
|
+
- [filesystem, '.']
|
|
32
|
+
- [filesystem, ./]
|
|
33
|
+
- [filesystem, '$GITHUB_WORKSPACE']
|
|
34
|
+
- [filesystem, '${GITHUB_WORKSPACE}']
|
|
35
|
+
- [filesystem, '$CI_PROJECT_DIR']
|
|
36
|
+
- [filesystem, '${CI_PROJECT_DIR}']
|
|
37
|
+
- [filesystem, '$(Build.SourcesDirectory)']
|
|
38
|
+
prohibited_arguments: [--fail=false]
|
|
39
|
+
prohibited_argument_sequences: [[--fail, 'false']]
|
|
40
|
+
- id: git-secrets
|
|
41
|
+
requires_final_exit_status: true
|
|
42
|
+
commands:
|
|
43
|
+
- executables: [git-secrets]
|
|
44
|
+
argument_groups: [[--scan, --scan-history]]
|
|
45
|
+
max_arguments: 1
|
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
id: known-verification-tools
|
|
2
|
+
benchmark_version: 0.4.0
|
|
3
|
+
extensions:
|
|
4
|
+
- control_id: ADRB-ENV-003
|
|
5
|
+
evidence_index: 0
|
|
6
|
+
ci_tools:
|
|
7
|
+
- id: tests
|
|
8
|
+
prohibited_arguments:
|
|
9
|
+
[--dry-run, --fixtures, --fixtures-per-test, --list-sessions, --markers, --no-run]
|
|
10
|
+
commands:
|
|
11
|
+
- executables: [npm, pnpm, yarn, bun, turbo, nx]
|
|
12
|
+
argument_groups:
|
|
13
|
+
- [test, test:unit, test:integration, test:e2e, unit-test, integration-test, e2e]
|
|
14
|
+
- executables: [cargo, go, dotnet, mvn, gradle, bazel]
|
|
15
|
+
repository_executables: [mvnw, gradlew]
|
|
16
|
+
argument_groups: [[test]]
|
|
17
|
+
standalone_executables: [pytest, pytest-cov, jest, vitest]
|
|
18
|
+
- id: static-analysis
|
|
19
|
+
commands:
|
|
20
|
+
- executables: [npm, pnpm, yarn, bun, turbo, nx]
|
|
21
|
+
argument_groups: [[lint, typecheck, type-check, check:types, check-types, check]]
|
|
22
|
+
- executables: [cargo]
|
|
23
|
+
argument_groups: [[clippy, check]]
|
|
24
|
+
- executables: [go]
|
|
25
|
+
argument_groups: [[vet]]
|
|
26
|
+
- executables: [mvn]
|
|
27
|
+
repository_executables: [mvnw]
|
|
28
|
+
argument_groups: [[checkstyle:check, spotbugs:check, pmd:check]]
|
|
29
|
+
- executables: [gradle]
|
|
30
|
+
repository_executables: [gradlew]
|
|
31
|
+
argument_groups: [[check]]
|
|
32
|
+
- executables: [ruff]
|
|
33
|
+
argument_groups: [[check]]
|
|
34
|
+
standalone_executables: [eslint, tsc, mypy, flake8, pyright, golangci-lint]
|
|
35
|
+
- control_id: ADRB-TST-003
|
|
36
|
+
evidence_index: 0
|
|
37
|
+
ci_tools:
|
|
38
|
+
- id: tests
|
|
39
|
+
prohibited_arguments:
|
|
40
|
+
[--dry-run, --fixtures, --fixtures-per-test, --list-sessions, --markers, --no-run]
|
|
41
|
+
commands:
|
|
42
|
+
- executables: [npm, pnpm, yarn, bun, turbo, nx]
|
|
43
|
+
argument_groups:
|
|
44
|
+
- [test, test:unit, test:integration, test:e2e, unit-test, integration-test, e2e]
|
|
45
|
+
- executables: [cargo, go, dotnet, mvn, gradle, bazel]
|
|
46
|
+
repository_executables: [mvnw, gradlew]
|
|
47
|
+
argument_groups: [[test]]
|
|
48
|
+
standalone_executables: [pytest, pytest-cov, jest, vitest]
|
|
49
|
+
- id: static-analysis
|
|
50
|
+
commands:
|
|
51
|
+
- executables: [npm, pnpm, yarn, bun, turbo, nx]
|
|
52
|
+
argument_groups: [[lint, typecheck, type-check, check:types, check-types, check]]
|
|
53
|
+
- executables: [cargo]
|
|
54
|
+
argument_groups: [[clippy, check]]
|
|
55
|
+
- executables: [go]
|
|
56
|
+
argument_groups: [[vet]]
|
|
57
|
+
- executables: [mvn]
|
|
58
|
+
repository_executables: [mvnw]
|
|
59
|
+
argument_groups: [[checkstyle:check, spotbugs:check, pmd:check]]
|
|
60
|
+
- executables: [gradle]
|
|
61
|
+
repository_executables: [gradlew]
|
|
62
|
+
argument_groups: [[check]]
|
|
63
|
+
- executables: [ruff]
|
|
64
|
+
argument_groups: [[check]]
|
|
65
|
+
standalone_executables: [eslint, tsc, mypy, flake8, pyright, golangci-lint]
|
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
{
|
|
2
|
+
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
|
3
|
+
"$id": "https://agentic-readiness.dev/schema/v0.4/agent-evidence.json",
|
|
4
|
+
"title": "ADRB v0.4 agent-collected evidence bundle",
|
|
5
|
+
"type": "object",
|
|
6
|
+
"additionalProperties": false,
|
|
7
|
+
"required": ["schema_version", "benchmark_version", "target", "collector", "claims"],
|
|
8
|
+
"properties": {
|
|
9
|
+
"schema_version": { "const": "0.4.0" },
|
|
10
|
+
"benchmark_version": { "const": "0.4.0" },
|
|
11
|
+
"target": {
|
|
12
|
+
"type": "object",
|
|
13
|
+
"additionalProperties": false,
|
|
14
|
+
"required": ["repository", "git_head"],
|
|
15
|
+
"properties": {
|
|
16
|
+
"repository": { "type": "string", "minLength": 1 },
|
|
17
|
+
"git_head": { "type": "string", "minLength": 1 }
|
|
18
|
+
}
|
|
19
|
+
},
|
|
20
|
+
"collector": {
|
|
21
|
+
"type": "object",
|
|
22
|
+
"additionalProperties": false,
|
|
23
|
+
"required": ["name", "version"],
|
|
24
|
+
"properties": {
|
|
25
|
+
"name": { "type": "string", "minLength": 1 },
|
|
26
|
+
"version": { "type": "string", "minLength": 1 }
|
|
27
|
+
}
|
|
28
|
+
},
|
|
29
|
+
"claims": {
|
|
30
|
+
"type": "object",
|
|
31
|
+
"propertyNames": { "pattern": "^ADRB-[A-Z]{3}-[0-9]{3}$" },
|
|
32
|
+
"additionalProperties": {
|
|
33
|
+
"type": "object",
|
|
34
|
+
"additionalProperties": false,
|
|
35
|
+
"required": ["status", "scope", "summary", "references", "collected_at", "expires_at"],
|
|
36
|
+
"properties": {
|
|
37
|
+
"status": { "enum": ["met", "not_met", "unknown"] },
|
|
38
|
+
"scope": { "enum": ["repository", "platform", "organization", "outcome"] },
|
|
39
|
+
"summary": { "type": "string", "minLength": 1 },
|
|
40
|
+
"references": {
|
|
41
|
+
"type": "array",
|
|
42
|
+
"minItems": 1,
|
|
43
|
+
"items": { "type": "string", "minLength": 1 }
|
|
44
|
+
},
|
|
45
|
+
"collected_at": { "type": "string", "format": "date-time" },
|
|
46
|
+
"expires_at": { "type": "string", "format": "date-time" },
|
|
47
|
+
"error": { "type": ["string", "null"], "minLength": 1 }
|
|
48
|
+
},
|
|
49
|
+
"allOf": [
|
|
50
|
+
{
|
|
51
|
+
"if": {
|
|
52
|
+
"required": ["error"],
|
|
53
|
+
"properties": { "error": { "type": "string" } }
|
|
54
|
+
},
|
|
55
|
+
"then": { "properties": { "status": { "const": "unknown" } } }
|
|
56
|
+
},
|
|
57
|
+
{
|
|
58
|
+
"if": { "properties": { "scope": { "const": "repository" } } },
|
|
59
|
+
"then": {
|
|
60
|
+
"properties": {
|
|
61
|
+
"references": {
|
|
62
|
+
"items": {
|
|
63
|
+
"type": "string",
|
|
64
|
+
"pattern": "^repo:(?!/)(?!\\.{1,2}(?:/|#|$))(?!.*\\/\\.{1,2}(?:/|#|$))(?!.*\\\\)[^#]+(?:#L[1-9][0-9]*(?:-L?[1-9][0-9]*)?)?$"
|
|
65
|
+
}
|
|
66
|
+
}
|
|
67
|
+
}
|
|
68
|
+
}
|
|
69
|
+
}
|
|
70
|
+
]
|
|
71
|
+
}
|
|
72
|
+
}
|
|
73
|
+
}
|
|
74
|
+
}
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
{
|
|
2
|
+
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
|
3
|
+
"$id": "https://agentic-readiness.dev/schema/v0.4/attestations.json",
|
|
4
|
+
"title": "ADRB v0.4 human attestation file",
|
|
5
|
+
"type": "object",
|
|
6
|
+
"additionalProperties": false,
|
|
7
|
+
"required": ["benchmark_version", "target", "attestations"],
|
|
8
|
+
"properties": {
|
|
9
|
+
"benchmark_version": { "const": "0.4.0" },
|
|
10
|
+
"target": {
|
|
11
|
+
"type": "object",
|
|
12
|
+
"additionalProperties": false,
|
|
13
|
+
"required": ["repository"],
|
|
14
|
+
"properties": {
|
|
15
|
+
"repository": { "type": "string", "minLength": 1 }
|
|
16
|
+
}
|
|
17
|
+
},
|
|
18
|
+
"attestations": {
|
|
19
|
+
"type": "object",
|
|
20
|
+
"propertyNames": { "pattern": "^ADRB-[A-Z]{3}-[0-9]{3}$" },
|
|
21
|
+
"additionalProperties": {
|
|
22
|
+
"type": "object",
|
|
23
|
+
"additionalProperties": false,
|
|
24
|
+
"required": ["status", "evidence", "owner", "reviewed_at", "expires_at"],
|
|
25
|
+
"properties": {
|
|
26
|
+
"status": { "enum": ["met", "not_met", "not_applicable", "unknown"] },
|
|
27
|
+
"evidence": { "type": "string", "minLength": 1 },
|
|
28
|
+
"owner": { "type": "string", "minLength": 1 },
|
|
29
|
+
"reviewed_at": { "type": "string", "format": "date" },
|
|
30
|
+
"expires_at": { "type": "string", "format": "date" }
|
|
31
|
+
}
|
|
32
|
+
}
|
|
33
|
+
}
|
|
34
|
+
}
|
|
35
|
+
}
|