agentic-scorecard 0.2.0 → 0.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENT_PROMPT.md +72 -51
- package/README.md +62 -33
- package/benchmark/v0.3/adapter-contract.md +44 -0
- package/benchmark/v0.3/adapters/known-agent-harnesses.yaml +177 -0
- package/benchmark/v0.3/adapters/known-python-tooling.yaml +18 -0
- package/benchmark/v0.3/adapters/known-security-tools.yaml +6 -0
- package/benchmark/v0.3/agent-evidence-schema.json +67 -0
- package/benchmark/v0.3/attestation-schema.json +26 -0
- package/benchmark/v0.3/benchmark.yaml +127 -0
- package/benchmark/v0.3/controls/context.yaml +79 -0
- package/benchmark/v0.3/controls/environment.yaml +93 -0
- package/benchmark/v0.3/controls/governance.yaml +97 -0
- package/benchmark/v0.3/controls/learning.yaml +80 -0
- package/benchmark/v0.3/controls/observability.yaml +91 -0
- package/benchmark/v0.3/controls/resilience.yaml +107 -0
- package/benchmark/v0.3/controls/security.yaml +129 -0
- package/benchmark/v0.3/controls/specification.yaml +83 -0
- package/benchmark/v0.3/controls/testing.yaml +85 -0
- package/benchmark/v0.3/controls/tooling.yaml +69 -0
- package/benchmark/v0.3/report-schema.json +89 -0
- package/benchmark/v0.3/scoring-policy.md +100 -0
- package/dist/cli.js +508 -53
- package/package.json +1 -1
- package/templates/agent-evidence.yaml +3 -3
- package/templates/attestations.yaml +1 -1
- package/templates/baseline.csv +1 -1
package/AGENT_PROMPT.md
CHANGED
|
@@ -1,65 +1,85 @@
|
|
|
1
|
-
# Ask an AI coding agent to run ADRB v0.
|
|
1
|
+
# Ask an AI coding agent to run ADRB v0.3
|
|
2
2
|
|
|
3
3
|
These prompts are vendor-neutral. The agent needs terminal access to the repository and permission
|
|
4
4
|
to download the published npm package. The default benchmark runs locally, reads only tracked paths,
|
|
5
5
|
invokes no model, uploads no source, and makes no network calls after installation.
|
|
6
6
|
|
|
7
|
-
## Recommended:
|
|
7
|
+
## Recommended: complete agent-assisted assessment
|
|
8
8
|
|
|
9
9
|
Copy and paste:
|
|
10
10
|
|
|
11
11
|
```text
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
12
|
+
Perform the complete ADRB v0.3 assessment of this repository for AI-agent pull-request work.
|
|
13
|
+
|
|
14
|
+
Do not change product source, tests, configuration, policies, Git history, or the user's current
|
|
15
|
+
checkout. First inspect the repository's commit, tracked-file status, upstream tracking branch, and
|
|
16
|
+
ahead/behind state. If read-only network access is already authorized, fetch the upstream before
|
|
17
|
+
comparing it. Select the commit before creating an assessment worktree. Default to the current HEAD;
|
|
18
|
+
a dirty checkout does not change that selection, so assess HEAD from an isolated clean worktree. Use
|
|
19
|
+
the fetched upstream commit only when I asked for the latest/current upstream state and the branch is
|
|
20
|
+
behind. If the checkout is both dirty and behind, state both facts and preserve HEAD by default unless
|
|
21
|
+
I explicitly asked for upstream or another commit. State the exact commit you selected and why.
|
|
22
|
+
|
|
23
|
+
Before running the scorecard, choose and state an absolute durable artifact directory outside any
|
|
24
|
+
temporary worktree. Do not delete that directory during worktree cleanup. In the commands below,
|
|
25
|
+
replace `<artifact-dir>` with that path. The npm package version is 0.3.1, while report filenames use
|
|
26
|
+
the immutable ADRB benchmark version 0.3.0.
|
|
27
|
+
|
|
28
|
+
From that clean target, run the repository-only baseline:
|
|
29
|
+
npx --yes agentic-scorecard@0.3.1 assess . --profile pr-creation --scope tracked --format markdown --output <artifact-dir>/agentic-readiness-v0.3.0-baseline.md
|
|
30
|
+
|
|
31
|
+
Then prepare unresolved evidence:
|
|
32
|
+
npx --yes agentic-scorecard@0.3.1 init-evidence . --output <artifact-dir>/agent-evidence.yaml --request-output <artifact-dir>/evidence-request.md
|
|
33
|
+
|
|
34
|
+
Read the baseline, `<artifact-dir>/evidence-request.md`, and the empty target-bound
|
|
35
|
+
`<artifact-dir>/agent-evidence.yaml`. Investigate nuanced repository-scoped requests only from
|
|
36
|
+
tracked files at the selected commit. Before accessing Git hosting settings, CI, dashboards, logs,
|
|
37
|
+
ticketing, or any other connected system, tell me exactly which least-privileged read-only tools,
|
|
38
|
+
accounts, repositories, branches, dashboards, and time ranges you need, and why. Wait for my
|
|
39
|
+
authorization.
|
|
40
|
+
|
|
41
|
+
After authorization, attempt only eligible requests. For every attempted claim, record its status,
|
|
42
|
+
scope, concise derivation, collection time, 30-day-or-shorter expiry, and durable privacy-safe
|
|
43
|
+
references. Repository claims must use `repo:<path>[#Lx-Ly]`; do not merely restate a keyword match.
|
|
44
|
+
A permission error, stale source, or inconclusive result remains unknown and records the error.
|
|
45
|
+
Never infer a pass, invent an attestation, or treat unavailable evidence as a failure.
|
|
46
|
+
|
|
47
|
+
Do not place credentials, prompts, source excerpts, raw logs, personal data, customer data, private
|
|
48
|
+
URLs that reveal secrets, or sensitive dashboard contents in generated files.
|
|
49
|
+
|
|
50
|
+
When authorized evidence collection is complete, run:
|
|
51
|
+
npx --yes agentic-scorecard@0.3.1 assess . --profile pr-creation --scope tracked --agent-evidence <artifact-dir>/agent-evidence.yaml --format markdown --output <artifact-dir>/agentic-readiness-v0.3.0-assisted.md
|
|
52
|
+
|
|
53
|
+
Tell me the repository-detected progress and normative score, the highest passed profile, whether
|
|
54
|
+
pr-creation passes, every target blocker, and the five highest-value improvements. Compare baseline
|
|
55
|
+
and assisted reports control by control. Keep repository-detected, agent-collected, and
|
|
56
|
+
human-attested evidence separate; treat UNKNOWN as unresolved, not failed; and flag likely false
|
|
57
|
+
positives or negatives. If I decline external access, stop after the baseline and label it clearly as
|
|
58
|
+
a repository-only baseline. Before removing a temporary worktree, verify that every report, evidence
|
|
59
|
+
request, and evidence bundle you cite exists in the durable artifact directory, then tell me that
|
|
60
|
+
directory's absolute path. Leave generated artifacts uncommitted unless I explicitly ask otherwise.
|
|
61
|
+
Do not describe any result as certified, compliant, safe, secure, or independently verified.
|
|
29
62
|
```
|
|
30
63
|
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
Use this only after reviewing the repository baseline:
|
|
34
|
-
|
|
35
|
-
```text
|
|
36
|
-
Collect the external evidence that ADRB v0.2 could not establish from tracked repository files.
|
|
37
|
-
|
|
38
|
-
First run:
|
|
39
|
-
npx --yes agentic-scorecard@0.2.0 init-evidence .
|
|
40
|
-
|
|
41
|
-
Read `.agentic/evidence-request.md` and the empty target-bound
|
|
42
|
-
`.agentic/agent-evidence.yaml` bundle. Before accessing any connected system, tell me which
|
|
43
|
-
read-only tools, accounts, repositories, branches, dashboards, and time ranges you need. Wait for
|
|
44
|
-
my authorization.
|
|
64
|
+
The agent is an evidence investigator. The scorecard remains the scoring authority.
|
|
45
65
|
|
|
46
|
-
|
|
47
|
-
record the source-backed status, scope, concise derivation, collection time, 30-day-or-shorter
|
|
48
|
-
expiry, and durable privacy-safe references. Add only claims you actually attempted. Do not paste
|
|
49
|
-
credentials, prompts, source excerpts, raw logs, personal data, or sensitive dashboard contents. A
|
|
50
|
-
permission error or inconclusive result must remain unknown and include the error; never infer a
|
|
51
|
-
pass.
|
|
66
|
+
## Fast repository-only baseline
|
|
52
67
|
|
|
53
|
-
|
|
54
|
-
|
|
68
|
+
Use this when you want an offline first look and accept that platform, organization, and outcome
|
|
69
|
+
controls will remain unresolved:
|
|
55
70
|
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
71
|
+
```text
|
|
72
|
+
Run a fast repository-only ADRB v0.3 baseline for AI-agent pull-request work. Verify and state the
|
|
73
|
+
selected commit and whether tracked files match HEAD, then run:
|
|
74
|
+
npx --yes agentic-scorecard@0.3.1 assess . --profile pr-creation --scope tracked --format markdown --output .agentic/reports/agentic-readiness-v0.3.0-baseline.md
|
|
75
|
+
|
|
76
|
+
Do not change repository files or invent evidence. Summarize repository-detected progress before
|
|
77
|
+
the normative score, target-profile blockers, and the five highest-value repository improvements.
|
|
78
|
+
Separate true repository gaps from platform, organization, and outcome evidence that this offline
|
|
79
|
+
scan cannot establish. Treat UNKNOWN as unresolved, not failed, and call the result a repository-only
|
|
80
|
+
baseline rather than a complete assessment. Leave generated artifacts uncommitted.
|
|
59
81
|
```
|
|
60
82
|
|
|
61
|
-
The agent is an evidence investigator. The scorecard remains the scoring authority.
|
|
62
|
-
|
|
63
83
|
## Choose a different authority level
|
|
64
84
|
|
|
65
85
|
Replace `pr-creation` with one of:
|
|
@@ -75,7 +95,7 @@ No benchmark profile grants production deployment authority.
|
|
|
75
95
|
## Guided remediation
|
|
76
96
|
|
|
77
97
|
```text
|
|
78
|
-
Read the latest ADRB v0.
|
|
98
|
+
Read the latest ADRB v0.3 report and create a remediation proposal for the failed target-profile
|
|
79
99
|
controls. Do not implement changes yet. Group recommendations into repository changes, platform or
|
|
80
100
|
organizational controls, and outcome measurement. For each include the control ID, risk reduced,
|
|
81
101
|
systems affected, accountable role, verification method, dependencies, and effort estimate. Never
|
|
@@ -86,8 +106,9 @@ authorize.
|
|
|
86
106
|
## Reassessment
|
|
87
107
|
|
|
88
108
|
```text
|
|
89
|
-
Re-run the pinned ADRB v0.
|
|
90
|
-
control by control. Explain changes using evidence and identify changed commit state, expired
|
|
91
|
-
scope differences, regressions, or newly established controls.
|
|
92
|
-
overwrite the previous
|
|
109
|
+
Re-run the pinned ADRB v0.3.0 benchmark with agentic-scorecard@0.3.1 for the same profile and tracked scope. Compare reports
|
|
110
|
+
control by control. Explain changes using evidence and identify changed commit state, expired
|
|
111
|
+
claims, scope differences, regressions, or newly established controls. Compare normative and
|
|
112
|
+
repository-detected progress separately. Write a new dated report; do not overwrite the previous
|
|
113
|
+
one.
|
|
93
114
|
```
|
package/README.md
CHANGED
|
@@ -16,26 +16,33 @@ The benchmark measures the harness, not the model brand. It assesses ten dimensi
|
|
|
16
16
|
maturity levels and applies non-compensating floors to five autonomy profiles. A high total score
|
|
17
17
|
cannot hide a critical security, testing, governance, or recovery gap.
|
|
18
18
|
|
|
19
|
-
This repository is a **v0.
|
|
20
|
-
is not a certification standard. The immutable v0.1
|
|
21
|
-
reproduction;
|
|
19
|
+
This repository is a **v0.3 reference implementation** intended for public review and piloting. It
|
|
20
|
+
is not a certification standard. The immutable v0.1 and v0.2 benchmarks remain available for
|
|
21
|
+
historical reproduction; scores from different benchmark versions are not directly comparable.
|
|
22
22
|
|
|
23
23
|
## Ask your coding agent
|
|
24
24
|
|
|
25
|
-
You do not need to install or learn the CLI yourself. Paste this
|
|
25
|
+
You do not need to install or learn the CLI yourself. Paste this instruction into Codex, Claude Code,
|
|
26
26
|
GitHub Copilot, Cursor, or another coding agent that has terminal access to your repository:
|
|
27
27
|
|
|
28
|
-
>
|
|
29
|
-
> `
|
|
30
|
-
>
|
|
31
|
-
>
|
|
32
|
-
>
|
|
33
|
-
>
|
|
34
|
-
>
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
28
|
+
> Perform the complete ADRB v0.3 assessment of this repository for AI-agent pull-request work using
|
|
29
|
+
> `agentic-scorecard@0.3.1`. First select and state the commit without modifying my checkout; preserve
|
|
30
|
+
> HEAD when the checkout is dirty, and use upstream only if I requested the latest upstream state.
|
|
31
|
+
> Assess it from a clean worktree and keep all generated artifacts in a durable directory outside any
|
|
32
|
+
> temporary worktree. Run the tracked repository baseline, then run `init-evidence`, read its evidence
|
|
33
|
+
> request, and tell me which least-privileged read-only connected systems you need. Wait for my
|
|
34
|
+
> authorization before accessing them. After authorized collection, rerun the assessment and compare
|
|
35
|
+
> the baseline and assisted reports control by control. Keep repository-detected, agent-collected, and
|
|
36
|
+
> human-attested evidence separate; never invent evidence; treat UNKNOWN as unresolved; and leave
|
|
37
|
+
> generated artifacts uncommitted.
|
|
38
|
+
|
|
39
|
+
The complete copy-and-paste prompt, including privacy limits and exact commands, is in
|
|
40
|
+
[AGENT_PROMPT.md](AGENT_PROMPT.md). It makes the evidence-assisted workflow the recommendation. A
|
|
41
|
+
fast filesystem scan remains available, but it is explicitly a **repository-only baseline** and
|
|
42
|
+
cannot establish platform, organization, or outcome controls by itself.
|
|
43
|
+
|
|
44
|
+
The reference CLI/package is version 0.3.1. Generated report names use benchmark version 0.3.0
|
|
45
|
+
because the patch release changed workflow guidance, not the immutable scoring benchmark.
|
|
39
46
|
|
|
40
47
|
## Feedback and community
|
|
41
48
|
|
|
@@ -70,7 +77,7 @@ A supported Node.js LTS release (20.19+, 22.13+, or 24+) is required. Assessment
|
|
|
70
77
|
read-only, and offline by default.
|
|
71
78
|
|
|
72
79
|
```bash
|
|
73
|
-
npx agentic-scorecard@0.
|
|
80
|
+
npx agentic-scorecard@0.3.1 assess /path/to/repository \
|
|
74
81
|
--profile pr-creation \
|
|
75
82
|
--scope tracked \
|
|
76
83
|
--format markdown \
|
|
@@ -80,7 +87,7 @@ npx agentic-scorecard@0.2.0 assess /path/to/repository \
|
|
|
80
87
|
To record controls that repository inspection cannot prove:
|
|
81
88
|
|
|
82
89
|
```bash
|
|
83
|
-
npx agentic-scorecard@0.
|
|
90
|
+
npx agentic-scorecard@0.3.1 init /path/to/repository
|
|
84
91
|
```
|
|
85
92
|
|
|
86
93
|
Complete `.agentic/attestations.yaml` with owners and durable evidence links, then assess again.
|
|
@@ -91,7 +98,7 @@ external systems, first generate a target-bound template. The repository must be
|
|
|
91
98
|
with at least one commit so the bundle can bind to the exact assessed state:
|
|
92
99
|
|
|
93
100
|
```bash
|
|
94
|
-
npx agentic-scorecard@0.
|
|
101
|
+
npx agentic-scorecard@0.3.1 init-evidence /path/to/repository
|
|
95
102
|
```
|
|
96
103
|
|
|
97
104
|
Ask the agent to review `.agentic/evidence-request.md`, obtain approval before using
|
|
@@ -99,19 +106,27 @@ least-privileged read-only connectors, add attempted claims to `.agentic/agent-e
|
|
|
99
106
|
rerun with `--agent-evidence`. The default bundle path is loaded automatically. See
|
|
100
107
|
[AGENT_PROMPT.md](AGENT_PROMPT.md) for the complete copy-and-paste workflow.
|
|
101
108
|
|
|
102
|
-
### Migrating from v0.1
|
|
109
|
+
### Migrating from v0.1 or v0.2
|
|
103
110
|
|
|
104
111
|
Run a new tracked-scope baseline and retain the old report as historical evidence. Do not present the
|
|
105
|
-
score change as improvement or regression because v0.
|
|
106
|
-
attestations before recreating them for v0.
|
|
107
|
-
|
|
112
|
+
score change as improvement or regression because v0.3 changes evidence and control semantics.
|
|
113
|
+
Re-review prior attestations and agent evidence before recreating them for v0.3. All claims must
|
|
114
|
+
expire, and repository-scoped semantic claims require tracked files to match the commit-bound target
|
|
115
|
+
plus `repo:<path>[#Lx-Ly]` references whose optional line ranges exist in the cited tracked file.
|
|
116
|
+
Untracked generated reports do not block this workflow.
|
|
117
|
+
|
|
118
|
+
The default `.agentic/attestations.yaml` and `.agentic/agent-evidence.yaml` paths are migration-safe:
|
|
119
|
+
if either contains an older benchmark version, v0.3 ignores that auto-loaded file and places a
|
|
120
|
+
prominent warning in the report. Regenerate it with `init --force` or `init-evidence --force` before
|
|
121
|
+
relying on those claims. An explicitly supplied `--attestations` or `--agent-evidence` file still
|
|
122
|
+
fails closed on a version mismatch.
|
|
108
123
|
|
|
109
124
|
For development from this checkout:
|
|
110
125
|
|
|
111
126
|
```bash
|
|
112
127
|
npm ci
|
|
113
128
|
npm run check
|
|
114
|
-
npm run dev -- assess tests/fixtures/mature --profile pr-creation
|
|
129
|
+
npm run dev -- assess tests/fixtures/mature --profile pr-creation --attestations tests/fixtures/mature/.agentic/attestations-v0.3.yaml
|
|
115
130
|
```
|
|
116
131
|
|
|
117
132
|
## What it assesses
|
|
@@ -157,22 +172,32 @@ No profile grants production deployment authority. Organizations should evaluate
|
|
|
157
172
|
through a separate, system-specific safety case.
|
|
158
173
|
|
|
159
174
|
The exact floors live in
|
|
160
|
-
[`benchmark/v0.
|
|
161
|
-
[`benchmark/v0.
|
|
175
|
+
[`benchmark/v0.3/benchmark.yaml`](benchmark/v0.3/benchmark.yaml) and their rationale in
|
|
176
|
+
[`benchmark/v0.3/scoring-policy.md`](benchmark/v0.3/scoring-policy.md).
|
|
162
177
|
|
|
163
178
|
## Evidence scopes and trust labels
|
|
164
179
|
|
|
165
180
|
- **Repository-detected:** the local collector found qualifying evidence in the selected path scope.
|
|
166
181
|
This proves an artifact match, not consistent practice or external enforcement.
|
|
167
|
-
- **Agent-collected:** an authorized agent supplied a target-bound, expiring, source-backed
|
|
168
|
-
claim. It is not independently
|
|
182
|
+
- **Agent-collected:** an authorized agent supplied a target-bound, expiring, source-backed semantic
|
|
183
|
+
repository or external claim for an explicitly eligible control. It is not independently
|
|
184
|
+
verified and is never relabelled as repository-detected.
|
|
169
185
|
- **Human-attested:** an accountable owner supplied a dated evidence link or explanation.
|
|
170
186
|
- **Unknown:** evidence is unavailable, expired, unauthorized, mismatched, or inconclusive.
|
|
171
187
|
|
|
172
188
|
Controls also identify whether their evidence belongs in the repository, hosting platform,
|
|
173
189
|
organization, or outcome systems. This prevents expected external unknowns from masquerading as
|
|
174
|
-
missing files.
|
|
175
|
-
|
|
190
|
+
missing files. Declarative detector adapters recognize common harness layouts without changing
|
|
191
|
+
portable control outcomes; future independently conformant external adapters require a separate
|
|
192
|
+
protocol. v0.3 does not issue certification or independent-verification claims.
|
|
193
|
+
|
|
194
|
+
### Interpreting the two score views
|
|
195
|
+
|
|
196
|
+
The normative score remains `N/40` and drives readiness profiles. v0.3 also reports
|
|
197
|
+
`repository-detected progress: A/C`, where `C` is the maximum consecutive maturity the offline
|
|
198
|
+
collector can establish without platform, organization, outcome, agent-collected, or human-attested
|
|
199
|
+
evidence. This second view explains how complete the visible repository harness is; it does not
|
|
200
|
+
normalize away UNKNOWNs, change readiness floors, or certify the project.
|
|
176
201
|
|
|
177
202
|
## Reports and CI
|
|
178
203
|
|
|
@@ -200,7 +225,8 @@ The default collector:
|
|
|
200
225
|
- reads only Git-tracked paths and records commit and dirty-worktree metadata;
|
|
201
226
|
- ignores `.git`, dependencies, build output, coverage, generated reports, attestations, and imported
|
|
202
227
|
evidence bundles;
|
|
203
|
-
- caps content
|
|
228
|
+
- caps content scanning at 512 KB per file, 5 MB total, and 250 candidates per check, with v0.3
|
|
229
|
+
per-pattern balancing where configured;
|
|
204
230
|
- reports file paths and match counts, never matching source snippets;
|
|
205
231
|
- writes nothing unless `--output`, `init`, or `init-evidence` is explicitly requested.
|
|
206
232
|
|
|
@@ -211,7 +237,9 @@ access-controlled evidence instead. Report suspected vulnerabilities through [SE
|
|
|
211
237
|
|
|
212
238
|
```text
|
|
213
239
|
benchmark/v0.1/ immutable historical v0.1 definition
|
|
214
|
-
benchmark/v0.2/
|
|
240
|
+
benchmark/v0.2/ immutable historical v0.2 definition
|
|
241
|
+
benchmark/v0.3/ current normative benchmark, schemas, and controls
|
|
242
|
+
adapters/ declarative path/term aliases for recognized agent harness layouts
|
|
215
243
|
benchmark/mappings/ informative mappings to external frameworks
|
|
216
244
|
src/ reference CLI and local evidence collectors
|
|
217
245
|
templates/ adoption, preflight, attestation, and remediation templates
|
|
@@ -240,8 +268,9 @@ system, and keep exceptions narrow, owned, expiring, and visible.
|
|
|
240
268
|
|
|
241
269
|
## Status and roadmap
|
|
242
270
|
|
|
243
|
-
v0.
|
|
244
|
-
|
|
271
|
+
v0.3 adds portable harness discovery with declarative detector adapters, proximity-bounded content evidence, prioritized candidate
|
|
272
|
+
selection for large repositories, source-backed semantic repository claims, and an explanatory
|
|
273
|
+
repository-only score and ceiling. Candidate next steps
|
|
245
274
|
include conformant signed adapters, SARIF/HTML reports, organization-level aggregation,
|
|
246
275
|
statistically designed benchmark tasks, and an independent-review protocol. These require public
|
|
247
276
|
design review before becoming normative.
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
# v0.3 adapter contracts
|
|
2
|
+
|
|
3
|
+
## Repository detector adapters
|
|
4
|
+
|
|
5
|
+
Portable controls define outcomes and generic repository conventions. Files in `adapters/*.yaml`
|
|
6
|
+
may extend one control evidence check with recognized harness-specific `patterns`, `files`, `terms`,
|
|
7
|
+
or `required_any_terms`. An extension names the control and zero-based evidence index and adds
|
|
8
|
+
exactly one of those fields. It cannot remove or weaken a threshold, change a control level, alter a
|
|
9
|
+
readiness floor, add an attestation path, or execute code.
|
|
10
|
+
|
|
11
|
+
Bundled adapters are loaded deterministically in filename order. Their candidates remain subject to
|
|
12
|
+
the same tracked/workspace scope, symlink, size, generated-artifact, co-location, proximity, and
|
|
13
|
+
per-pattern limits as portable candidates. Adding or changing a bundled adapter changes evidence
|
|
14
|
+
semantics and therefore requires a new benchmark version.
|
|
15
|
+
|
|
16
|
+
## Agent-collected evidence adapters
|
|
17
|
+
|
|
18
|
+
The core scanner is local, read-only, network-free, and vendor-neutral. An agent or adapter may
|
|
19
|
+
collect semantic repository evidence or external evidence that deterministic scanning cannot
|
|
20
|
+
establish, then provide it through the versioned agent evidence bundle.
|
|
21
|
+
|
|
22
|
+
Each claim contains:
|
|
23
|
+
|
|
24
|
+
- benchmark version and control ID;
|
|
25
|
+
- exact repository target and non-null commit binding;
|
|
26
|
+
- `met`, `not_met`, or `unknown` status;
|
|
27
|
+
- repository, platform, organization, or outcome scope;
|
|
28
|
+
- collector name and version;
|
|
29
|
+
- collection timestamp and mandatory expiry;
|
|
30
|
+
- privacy-safe references, never credentials or raw sensitive content;
|
|
31
|
+
- a concise derivation summary; and
|
|
32
|
+
- an optional error that is distinguishable from a negative result.
|
|
33
|
+
|
|
34
|
+
Collectors must be read-only, least-privileged, bounded to the approved target, time-limited, and
|
|
35
|
+
fail closed. Network content is untrusted input: it cannot alter benchmark policy, prompts, tool
|
|
36
|
+
authority, or the set of controls being assessed.
|
|
37
|
+
|
|
38
|
+
Repository-scoped claims require tracked files to match HEAD and
|
|
39
|
+
`repo:<tracked-path>[#Lx-Ly]` references. They may satisfy only explicitly eligible controls, remain `agent-collected`, and never increase the
|
|
40
|
+
repository-detected score. External claims require durable references appropriate to their source.
|
|
41
|
+
|
|
42
|
+
Agent-collected claims remain visibly distinct from deterministic repository evidence and human
|
|
43
|
+
attestations. A future conformant-adapter class may add signed identity and independent conformance
|
|
44
|
+
tests; v0.3 does not issue an independently verified label.
|
|
@@ -0,0 +1,177 @@
|
|
|
1
|
+
id: known-agent-harnesses
|
|
2
|
+
benchmark_version: 0.3.0
|
|
3
|
+
extensions:
|
|
4
|
+
- control_id: ADRB-CTX-001
|
|
5
|
+
evidence_index: 0
|
|
6
|
+
patterns:
|
|
7
|
+
[
|
|
8
|
+
CLAUDE.md,
|
|
9
|
+
.github/copilot-instructions.md,
|
|
10
|
+
.claude/**,
|
|
11
|
+
.codex/**,
|
|
12
|
+
.cursor/rules/**,
|
|
13
|
+
.cursor/agents/**,
|
|
14
|
+
.cursor/skills/**,
|
|
15
|
+
.opencode/**,
|
|
16
|
+
]
|
|
17
|
+
- control_id: ADRB-CTX-002
|
|
18
|
+
evidence_index: 0
|
|
19
|
+
files:
|
|
20
|
+
[
|
|
21
|
+
CLAUDE.md,
|
|
22
|
+
.github/copilot-instructions.md,
|
|
23
|
+
.claude/**/*.md,
|
|
24
|
+
.codex/**/*.md,
|
|
25
|
+
.cursor/rules/**/*.md,
|
|
26
|
+
.cursor/agents/**/*.md,
|
|
27
|
+
.opencode/**/*.md,
|
|
28
|
+
]
|
|
29
|
+
- control_id: ADRB-CTX-003
|
|
30
|
+
evidence_index: 0
|
|
31
|
+
required_any_terms: [CLAUDE.md, copilot instructions]
|
|
32
|
+
- control_id: ADRB-ENV-001
|
|
33
|
+
evidence_index: 0
|
|
34
|
+
files:
|
|
35
|
+
[
|
|
36
|
+
CLAUDE.md,
|
|
37
|
+
.claude/**/*.md,
|
|
38
|
+
.codex/**/*.md,
|
|
39
|
+
.cursor/agents/**/*.md,
|
|
40
|
+
.cursor/rules/**/*.md,
|
|
41
|
+
.cursor/skills/**/*.md,
|
|
42
|
+
.opencode/**/*.md,
|
|
43
|
+
]
|
|
44
|
+
- control_id: ADRB-SPC-001
|
|
45
|
+
evidence_index: 0
|
|
46
|
+
patterns: [.kiro/specs/**, .cursor/plans/**, .codex/plans/**]
|
|
47
|
+
- control_id: ADRB-SPC-002
|
|
48
|
+
evidence_index: 0
|
|
49
|
+
files: [.kiro/specs/**, .cursor/plans/**, .codex/plans/**]
|
|
50
|
+
- control_id: ADRB-TOL-001
|
|
51
|
+
evidence_index: 0
|
|
52
|
+
patterns:
|
|
53
|
+
[
|
|
54
|
+
.claude/**,
|
|
55
|
+
.codex/**,
|
|
56
|
+
.cursor/**,
|
|
57
|
+
.opencode/**,
|
|
58
|
+
.github/copilot-instructions.md,
|
|
59
|
+
opencode.json,
|
|
60
|
+
]
|
|
61
|
+
- control_id: ADRB-TOL-002
|
|
62
|
+
evidence_index: 0
|
|
63
|
+
files:
|
|
64
|
+
[
|
|
65
|
+
CLAUDE.md,
|
|
66
|
+
.claude/**/*.md,
|
|
67
|
+
.codex/**/*.md,
|
|
68
|
+
.cursor/agents/**/*.md,
|
|
69
|
+
.cursor/rules/**/*.md,
|
|
70
|
+
.cursor/skills/**/*.md,
|
|
71
|
+
.opencode/**/*.md,
|
|
72
|
+
]
|
|
73
|
+
- control_id: ADRB-SEC-001
|
|
74
|
+
evidence_index: 0
|
|
75
|
+
files:
|
|
76
|
+
[
|
|
77
|
+
CLAUDE.md,
|
|
78
|
+
.claude/**/*.md,
|
|
79
|
+
.codex/**/*.md,
|
|
80
|
+
.cursor/agents/**/*.md,
|
|
81
|
+
.cursor/rules/**/*.md,
|
|
82
|
+
.cursor/skills/**/*.md,
|
|
83
|
+
.opencode/**/*.md,
|
|
84
|
+
]
|
|
85
|
+
- control_id: ADRB-SEC-002
|
|
86
|
+
evidence_index: 0
|
|
87
|
+
files:
|
|
88
|
+
[
|
|
89
|
+
CLAUDE.md,
|
|
90
|
+
.claude/**/*.md,
|
|
91
|
+
.codex/**/*.md,
|
|
92
|
+
.cursor/agents/**/*.md,
|
|
93
|
+
.cursor/rules/**/*.md,
|
|
94
|
+
.cursor/skills/**/*.md,
|
|
95
|
+
.opencode/**/*.md,
|
|
96
|
+
]
|
|
97
|
+
- control_id: ADRB-TST-002
|
|
98
|
+
evidence_index: 0
|
|
99
|
+
files:
|
|
100
|
+
[
|
|
101
|
+
CLAUDE.md,
|
|
102
|
+
.claude/**/*.md,
|
|
103
|
+
.codex/**/*.md,
|
|
104
|
+
.cursor/agents/**/*.md,
|
|
105
|
+
.cursor/rules/**/*.md,
|
|
106
|
+
.cursor/skills/**/*.md,
|
|
107
|
+
.opencode/**/*.md,
|
|
108
|
+
]
|
|
109
|
+
- control_id: ADRB-GOV-001
|
|
110
|
+
evidence_index: 0
|
|
111
|
+
files:
|
|
112
|
+
[
|
|
113
|
+
CLAUDE.md,
|
|
114
|
+
.claude/**/*.md,
|
|
115
|
+
.codex/**/*.md,
|
|
116
|
+
.cursor/agents/**/*.md,
|
|
117
|
+
.cursor/rules/**/*.md,
|
|
118
|
+
.cursor/skills/**/*.md,
|
|
119
|
+
.opencode/**/*.md,
|
|
120
|
+
]
|
|
121
|
+
- control_id: ADRB-GOV-002
|
|
122
|
+
evidence_index: 1
|
|
123
|
+
files:
|
|
124
|
+
[
|
|
125
|
+
CLAUDE.md,
|
|
126
|
+
.claude/**/*.md,
|
|
127
|
+
.codex/**/*.md,
|
|
128
|
+
.cursor/agents/**/*.md,
|
|
129
|
+
.cursor/rules/**/*.md,
|
|
130
|
+
.cursor/skills/**/*.md,
|
|
131
|
+
.opencode/**/*.md,
|
|
132
|
+
]
|
|
133
|
+
- control_id: ADRB-LRN-001
|
|
134
|
+
evidence_index: 0
|
|
135
|
+
patterns: [.claude/memory/**, .codex/memory/**, .cursor/memory/**, .opencode/memory/**]
|
|
136
|
+
- control_id: ADRB-LRN-002
|
|
137
|
+
evidence_index: 0
|
|
138
|
+
files:
|
|
139
|
+
[
|
|
140
|
+
CLAUDE.md,
|
|
141
|
+
.claude/**/*.md,
|
|
142
|
+
.codex/**/*.md,
|
|
143
|
+
.cursor/agents/**/*.md,
|
|
144
|
+
.cursor/rules/**/*.md,
|
|
145
|
+
.cursor/skills/**/*.md,
|
|
146
|
+
.opencode/**/*.md,
|
|
147
|
+
]
|
|
148
|
+
- control_id: ADRB-OBS-001
|
|
149
|
+
evidence_index: 0
|
|
150
|
+
files: [.cursor/runs/README*, .codex/runs/README*, .claude/runs/README*]
|
|
151
|
+
- control_id: ADRB-OBS-002
|
|
152
|
+
evidence_index: 0
|
|
153
|
+
files: [.cursor/runs/README*, .codex/runs/README*, .claude/runs/README*]
|
|
154
|
+
- control_id: ADRB-RES-001
|
|
155
|
+
evidence_index: 0
|
|
156
|
+
files:
|
|
157
|
+
[
|
|
158
|
+
CLAUDE.md,
|
|
159
|
+
.claude/**/*.md,
|
|
160
|
+
.codex/**/*.md,
|
|
161
|
+
.cursor/agents/**/*.md,
|
|
162
|
+
.cursor/rules/**/*.md,
|
|
163
|
+
.cursor/skills/**/*.md,
|
|
164
|
+
.opencode/**/*.md,
|
|
165
|
+
]
|
|
166
|
+
- control_id: ADRB-RES-002
|
|
167
|
+
evidence_index: 0
|
|
168
|
+
files:
|
|
169
|
+
[
|
|
170
|
+
CLAUDE.md,
|
|
171
|
+
.claude/**/*.md,
|
|
172
|
+
.codex/**/*.md,
|
|
173
|
+
.cursor/agents/**/*.md,
|
|
174
|
+
.cursor/rules/**/*.md,
|
|
175
|
+
.cursor/skills/**/*.md,
|
|
176
|
+
.opencode/**/*.md,
|
|
177
|
+
]
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
id: known-python-tooling
|
|
2
|
+
benchmark_version: 0.3.0
|
|
3
|
+
extensions:
|
|
4
|
+
- control_id: ADRB-ENV-001
|
|
5
|
+
evidence_index: 0
|
|
6
|
+
files: [pyproject.toml]
|
|
7
|
+
- control_id: ADRB-ENV-001
|
|
8
|
+
evidence_index: 0
|
|
9
|
+
terms: [requires-python, uv sync, python]
|
|
10
|
+
- control_id: ADRB-TST-002
|
|
11
|
+
evidence_index: 0
|
|
12
|
+
files: [run_quality_checks.sh, scripts/**/quality*.sh, tox.ini, noxfile.py]
|
|
13
|
+
- control_id: ADRB-TST-002
|
|
14
|
+
evidence_index: 0
|
|
15
|
+
terms: [pytest, mypy, flake8, ruff, pyright, tox, nox]
|
|
16
|
+
- control_id: ADRB-TST-003
|
|
17
|
+
evidence_index: 0
|
|
18
|
+
terms: [pytest, mypy, flake8, ruff, pyright, tox, nox, pytest-cov]
|
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
{
|
|
2
|
+
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
|
3
|
+
"$id": "https://agentic-readiness.dev/schema/v0.3/agent-evidence.json",
|
|
4
|
+
"title": "ADRB v0.3 agent-collected evidence bundle",
|
|
5
|
+
"type": "object",
|
|
6
|
+
"additionalProperties": false,
|
|
7
|
+
"required": ["schema_version", "benchmark_version", "target", "collector", "claims"],
|
|
8
|
+
"properties": {
|
|
9
|
+
"schema_version": { "const": "0.3.0" },
|
|
10
|
+
"benchmark_version": { "const": "0.3.0" },
|
|
11
|
+
"target": {
|
|
12
|
+
"type": "object",
|
|
13
|
+
"additionalProperties": false,
|
|
14
|
+
"required": ["repository", "git_head"],
|
|
15
|
+
"properties": {
|
|
16
|
+
"repository": { "type": "string", "minLength": 1 },
|
|
17
|
+
"git_head": { "type": "string", "minLength": 1 }
|
|
18
|
+
}
|
|
19
|
+
},
|
|
20
|
+
"collector": {
|
|
21
|
+
"type": "object",
|
|
22
|
+
"additionalProperties": false,
|
|
23
|
+
"required": ["name", "version"],
|
|
24
|
+
"properties": {
|
|
25
|
+
"name": { "type": "string", "minLength": 1 },
|
|
26
|
+
"version": { "type": "string", "minLength": 1 }
|
|
27
|
+
}
|
|
28
|
+
},
|
|
29
|
+
"claims": {
|
|
30
|
+
"type": "object",
|
|
31
|
+
"propertyNames": { "pattern": "^ADRB-[A-Z]{3}-[0-9]{3}$" },
|
|
32
|
+
"additionalProperties": {
|
|
33
|
+
"type": "object",
|
|
34
|
+
"additionalProperties": false,
|
|
35
|
+
"required": ["status", "scope", "summary", "references", "collected_at", "expires_at"],
|
|
36
|
+
"properties": {
|
|
37
|
+
"status": { "enum": ["met", "not_met", "unknown"] },
|
|
38
|
+
"scope": { "enum": ["repository", "platform", "organization", "outcome"] },
|
|
39
|
+
"summary": { "type": "string", "minLength": 1 },
|
|
40
|
+
"references": {
|
|
41
|
+
"type": "array",
|
|
42
|
+
"minItems": 1,
|
|
43
|
+
"items": { "type": "string", "minLength": 1 }
|
|
44
|
+
},
|
|
45
|
+
"collected_at": { "type": "string", "format": "date-time" },
|
|
46
|
+
"expires_at": { "type": "string", "format": "date-time" },
|
|
47
|
+
"error": { "type": ["string", "null"], "minLength": 1 }
|
|
48
|
+
},
|
|
49
|
+
"allOf": [
|
|
50
|
+
{
|
|
51
|
+
"if": { "properties": { "scope": { "const": "repository" } } },
|
|
52
|
+
"then": {
|
|
53
|
+
"properties": {
|
|
54
|
+
"references": {
|
|
55
|
+
"items": {
|
|
56
|
+
"type": "string",
|
|
57
|
+
"pattern": "^repo:(?!/)(?!\\.{1,2}(?:/|#|$))(?!.*\\/\\.{1,2}(?:/|#|$))(?!.*\\\\)[^#]+(?:#L[1-9][0-9]*(?:-L?[1-9][0-9]*)?)?$"
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
}
|
|
63
|
+
]
|
|
64
|
+
}
|
|
65
|
+
}
|
|
66
|
+
}
|
|
67
|
+
}
|