agentic-scorecard 0.1.0 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENT_PROMPT.md +68 -34
- package/FEEDBACK.md +53 -0
- package/README.md +120 -39
- package/benchmark/v0.2/adapter-contract.md +25 -0
- package/benchmark/v0.2/agent-evidence-schema.json +52 -0
- package/benchmark/v0.2/attestation-schema.json +26 -0
- package/benchmark/v0.2/benchmark.yaml +127 -0
- package/benchmark/v0.2/controls/context.yaml +51 -0
- package/benchmark/v0.2/controls/environment.yaml +70 -0
- package/benchmark/v0.2/controls/governance.yaml +60 -0
- package/benchmark/v0.2/controls/learning.yaml +48 -0
- package/benchmark/v0.2/controls/observability.yaml +76 -0
- package/benchmark/v0.2/controls/resilience.yaml +60 -0
- package/benchmark/v0.2/controls/security.yaml +71 -0
- package/benchmark/v0.2/controls/specification.yaml +62 -0
- package/benchmark/v0.2/controls/testing.yaml +60 -0
- package/benchmark/v0.2/controls/tooling.yaml +58 -0
- package/benchmark/v0.2/report-schema.json +47 -0
- package/benchmark/v0.2/scoring-policy.md +71 -0
- package/benchmark/v0.3/adapter-contract.md +44 -0
- package/benchmark/v0.3/adapters/known-agent-harnesses.yaml +177 -0
- package/benchmark/v0.3/adapters/known-python-tooling.yaml +18 -0
- package/benchmark/v0.3/adapters/known-security-tools.yaml +6 -0
- package/benchmark/v0.3/agent-evidence-schema.json +67 -0
- package/benchmark/v0.3/attestation-schema.json +26 -0
- package/benchmark/v0.3/benchmark.yaml +127 -0
- package/benchmark/v0.3/controls/context.yaml +79 -0
- package/benchmark/v0.3/controls/environment.yaml +93 -0
- package/benchmark/v0.3/controls/governance.yaml +97 -0
- package/benchmark/v0.3/controls/learning.yaml +80 -0
- package/benchmark/v0.3/controls/observability.yaml +91 -0
- package/benchmark/v0.3/controls/resilience.yaml +107 -0
- package/benchmark/v0.3/controls/security.yaml +129 -0
- package/benchmark/v0.3/controls/specification.yaml +83 -0
- package/benchmark/v0.3/controls/testing.yaml +85 -0
- package/benchmark/v0.3/controls/tooling.yaml +69 -0
- package/benchmark/v0.3/report-schema.json +89 -0
- package/benchmark/v0.3/scoring-policy.md +100 -0
- package/dist/cli.js +1043 -111
- package/package.json +5 -3
- package/templates/agent-evidence.yaml +11 -0
- package/templates/attestations.yaml +4 -4
- package/templates/baseline.csv +1 -1
package/AGENT_PROMPT.md
CHANGED
|
@@ -1,10 +1,10 @@
|
|
|
1
|
-
# Ask an AI coding agent to run
|
|
1
|
+
# Ask an AI coding agent to run ADRB v0.3
|
|
2
2
|
|
|
3
|
-
These prompts are
|
|
4
|
-
|
|
5
|
-
model, uploads no source, and makes no network calls after
|
|
3
|
+
These prompts are vendor-neutral. The agent needs terminal access to the repository and permission
|
|
4
|
+
to download the published npm package. The default benchmark runs locally, reads only tracked paths,
|
|
5
|
+
invokes no model, uploads no source, and makes no network calls after installation.
|
|
6
6
|
|
|
7
|
-
## Recommended:
|
|
7
|
+
## Recommended: repository baseline
|
|
8
8
|
|
|
9
9
|
Copy and paste:
|
|
10
10
|
|
|
@@ -12,25 +12,61 @@ Copy and paste:
|
|
|
12
12
|
Assess this repository's readiness for AI-agent pull-request work.
|
|
13
13
|
|
|
14
14
|
From the repository root, run:
|
|
15
|
-
npx --yes agentic-scorecard@0.
|
|
16
|
-
|
|
17
|
-
Do not change product source
|
|
18
|
-
|
|
19
|
-
1. the score and
|
|
20
|
-
2. whether
|
|
21
|
-
3. each target-profile blocker and
|
|
22
|
-
4. the five highest-value improvements in dependency order;
|
|
23
|
-
5.
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
Keep
|
|
27
|
-
uncommitted unless I explicitly ask
|
|
28
|
-
blocker without
|
|
15
|
+
npx --yes agentic-scorecard@0.3.0 assess . --profile pr-creation --scope tracked --format markdown --output .agentic/reports/agentic-readiness-v0.3.0.md
|
|
16
|
+
|
|
17
|
+
Do not change product source, configuration, policies, or tests. Do not invent evidence or mark an
|
|
18
|
+
attestation as met. Read the report and tell me:
|
|
19
|
+
1. the normative score out of 40 and the repository-detected progress against its offline ceiling;
|
|
20
|
+
2. the highest passed readiness profile and whether pr-creation passes;
|
|
21
|
+
3. each target-profile blocker and its evidence scope;
|
|
22
|
+
4. the five highest-value improvements in dependency order;
|
|
23
|
+
5. which findings are repository gaps versus external or outcome evidence not yet established; and
|
|
24
|
+
6. any likely false positive or false negative.
|
|
25
|
+
|
|
26
|
+
Keep repository-detected, agent-collected, and human-attested evidence separate. Treat UNKNOWN as
|
|
27
|
+
unresolved, not failed. Leave generated artifacts uncommitted unless I explicitly ask otherwise. If
|
|
28
|
+
the command cannot run, explain the exact blocker without modifying the repository to work around
|
|
29
|
+
it.
|
|
29
30
|
```
|
|
30
31
|
|
|
32
|
+
## Agent-assisted unresolved evidence
|
|
33
|
+
|
|
34
|
+
Use this only after reviewing the repository baseline:
|
|
35
|
+
|
|
36
|
+
```text
|
|
37
|
+
Investigate the controls that the ADRB v0.3 baseline could not establish.
|
|
38
|
+
|
|
39
|
+
First ensure tracked files match HEAD and the repository has a commit, then run:
|
|
40
|
+
npx --yes agentic-scorecard@0.3.0 init-evidence .
|
|
41
|
+
|
|
42
|
+
Read `.agentic/evidence-request.md` and the empty target-bound
|
|
43
|
+
`.agentic/agent-evidence.yaml` bundle. The request includes only unresolved controls that permit
|
|
44
|
+
agent-collected evidence. Before accessing any connected system, tell me which read-only tools,
|
|
45
|
+
accounts, repositories, branches, dashboards, and time ranges you need. Wait for my authorization.
|
|
46
|
+
|
|
47
|
+
After authorization, use only least-privileged read-only operations. For each attempted control,
|
|
48
|
+
record the source-backed status, scope, concise derivation, collection time, 30-day-or-shorter
|
|
49
|
+
expiry, and durable privacy-safe references. For repository-scoped claims, cite only tracked files
|
|
50
|
+
at the bound commit using `repo:<path>[#Lx-Ly]`; do not use a repository claim to restate a keyword
|
|
51
|
+
match. Add only claims you actually investigated. A permission error or inconclusive result must
|
|
52
|
+
remain unknown and include the error; never infer a pass.
|
|
53
|
+
|
|
54
|
+
Do not paste credentials, prompts, source excerpts, raw logs, personal data, customer data, or
|
|
55
|
+
sensitive dashboard contents into the bundle.
|
|
56
|
+
|
|
57
|
+
Then run:
|
|
58
|
+
npx --yes agentic-scorecard@0.3.0 assess . --profile pr-creation --scope tracked --agent-evidence .agentic/agent-evidence.yaml --format markdown --output .agentic/reports/agentic-readiness-v0.3.0-assisted.md
|
|
59
|
+
|
|
60
|
+
Summarize every score change and keep agent-collected evidence distinct from deterministic
|
|
61
|
+
repository evidence and human attestations. Do not describe the result as certified, compliant,
|
|
62
|
+
safe, secure, or independently verified.
|
|
63
|
+
```
|
|
64
|
+
|
|
65
|
+
The agent is an evidence investigator. The scorecard remains the scoring authority.
|
|
66
|
+
|
|
31
67
|
## Choose a different authority level
|
|
32
68
|
|
|
33
|
-
Replace `pr-creation` with
|
|
69
|
+
Replace `pr-creation` with one of:
|
|
34
70
|
|
|
35
71
|
- `read-only-analysis` — inspect approved source and return advice;
|
|
36
72
|
- `planning` — draft plans and specifications;
|
|
@@ -42,23 +78,21 @@ No benchmark profile grants production deployment authority.
|
|
|
42
78
|
|
|
43
79
|
## Guided remediation
|
|
44
80
|
|
|
45
|
-
Use this only after reviewing the first report:
|
|
46
|
-
|
|
47
81
|
```text
|
|
48
|
-
Read
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
implement.
|
|
82
|
+
Read the latest ADRB v0.3 report and create a remediation proposal for the failed target-profile
|
|
83
|
+
controls. Do not implement changes yet. Group recommendations into repository changes, platform or
|
|
84
|
+
organizational controls, and outcome measurement. For each include the control ID, risk reduced,
|
|
85
|
+
systems affected, accountable role, verification method, dependencies, and effort estimate. Never
|
|
86
|
+
invent an attestation or convert unavailable evidence into a failure. End by asking which changes I
|
|
87
|
+
authorize.
|
|
55
88
|
```
|
|
56
89
|
|
|
57
|
-
## Reassessment
|
|
90
|
+
## Reassessment
|
|
58
91
|
|
|
59
92
|
```text
|
|
60
|
-
Re-run the pinned
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
93
|
+
Re-run the pinned ADRB v0.3.0 benchmark for the same profile and tracked scope. Compare reports
|
|
94
|
+
control by control. Explain changes using evidence and identify changed commit state, expired
|
|
95
|
+
claims, scope differences, regressions, or newly established controls. Compare normative and
|
|
96
|
+
repository-detected progress separately. Write a new dated report; do not overwrite the previous
|
|
97
|
+
one.
|
|
64
98
|
```
|
package/FEEDBACK.md
ADDED
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
# Feedback guide
|
|
2
|
+
|
|
3
|
+
Feedback from real assessments is essential to improving the Agentic Development Readiness
|
|
4
|
+
Benchmark without weakening its evidence standards or vendor neutrality.
|
|
5
|
+
|
|
6
|
+
## Choose the right channel
|
|
7
|
+
|
|
8
|
+
| Feedback | Channel |
|
|
9
|
+
| ----------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------ |
|
|
10
|
+
| False positive, false negative, unclear result, usability problem, or adoption experience | [Assessment feedback issue](https://github.com/Planet-B2B/agentic-readiness/issues/new?template=assessment-feedback.yml) |
|
|
11
|
+
| Proposed control, scoring, evidence, or readiness-profile change | [Benchmark change issue](https://github.com/Planet-B2B/agentic-readiness/issues/new?template=benchmark-change.yml) |
|
|
12
|
+
| Private adoption question or feedback that cannot be public | [benchmark@planetb2b.com](mailto:benchmark@planetb2b.com) |
|
|
13
|
+
| Suspected security vulnerability | Follow [SECURITY.md](SECURITY.md); do not open a public issue or email the benchmark contact |
|
|
14
|
+
|
|
15
|
+
Public issues are preferred when the information can be safely shared because they let other users
|
|
16
|
+
compare experiences and participate in the design review.
|
|
17
|
+
|
|
18
|
+
## What useful feedback includes
|
|
19
|
+
|
|
20
|
+
- benchmark and package version;
|
|
21
|
+
- requested readiness profile and assessment scope;
|
|
22
|
+
- relevant control IDs;
|
|
23
|
+
- the observed result and the expected result;
|
|
24
|
+
- whether the concern is scanner behavior, control wording, evidence semantics, scoring policy, or
|
|
25
|
+
documentation;
|
|
26
|
+
- a minimal sanitized reproduction or description; and
|
|
27
|
+
- the material impact on adoption or decision-making.
|
|
28
|
+
|
|
29
|
+
Do not attach complete private reports. Never include credentials, proprietary source, private
|
|
30
|
+
URLs, personal data, customer data, or confidential dashboard content. A path name or summarized
|
|
31
|
+
evidence characteristic is usually enough to begin triage.
|
|
32
|
+
|
|
33
|
+
## Ask a coding agent to prepare feedback
|
|
34
|
+
|
|
35
|
+
Paste this prompt into an agent that can read your assessment report:
|
|
36
|
+
|
|
37
|
+
> Review my Agentic Development Readiness report and draft feedback for the
|
|
38
|
+
> `Planet-B2B/agentic-readiness` repository. Include the benchmark version, target profile,
|
|
39
|
+
> assessment scope, relevant control IDs, observed result, expected result, and a minimal sanitized
|
|
40
|
+
> explanation. Classify the feedback as a likely scanner defect, documentation problem, adoption
|
|
41
|
+
> experience, or proposed normative benchmark change. Do not include credentials, proprietary
|
|
42
|
+
> source, private URLs, personal data, customer data, or full report contents. Show me the complete
|
|
43
|
+
> draft and ask for my approval before opening a GitHub issue. If I approve and you have GitHub
|
|
44
|
+
> access, use the assessment feedback issue form; otherwise give me the final text to submit.
|
|
45
|
+
|
|
46
|
+
Agents must treat opening an issue as an external write and obtain accountable human approval first.
|
|
47
|
+
Security concerns must follow `SECURITY.md` rather than this workflow.
|
|
48
|
+
|
|
49
|
+
## Help others discover the benchmark
|
|
50
|
+
|
|
51
|
+
If this project helps your team, [star the repository](https://github.com/Planet-B2B/agentic-readiness).
|
|
52
|
+
Stars make the public benchmark easier for other engineering teams to discover; substantive feedback
|
|
53
|
+
and reproducible examples are even more valuable.
|
package/README.md
CHANGED
|
@@ -1,3 +1,9 @@
|
|
|
1
|
+

|
|
2
|
+
|
|
3
|
+
<p align="center">
|
|
4
|
+
<img src="assets/brand/avatar.png" width="96" height="96" alt="Agentic Development Readiness project mark">
|
|
5
|
+
</p>
|
|
6
|
+
|
|
1
7
|
# Agentic Development Readiness Benchmark
|
|
2
8
|
|
|
3
9
|
A vendor-neutral, evidence-backed benchmark for the engineering harness around AI coding agents.
|
|
@@ -10,8 +16,9 @@ The benchmark measures the harness, not the model brand. It assesses ten dimensi
|
|
|
10
16
|
maturity levels and applies non-compensating floors to five autonomy profiles. A high total score
|
|
11
17
|
cannot hide a critical security, testing, governance, or recovery gap.
|
|
12
18
|
|
|
13
|
-
This repository is
|
|
14
|
-
|
|
19
|
+
This repository is a **v0.3 reference implementation** intended for public review and piloting. It
|
|
20
|
+
is not a certification standard. The immutable v0.1 and v0.2 benchmarks remain available for
|
|
21
|
+
historical reproduction; scores from different benchmark versions are not directly comparable.
|
|
15
22
|
|
|
16
23
|
## Ask your coding agent
|
|
17
24
|
|
|
@@ -19,17 +26,42 @@ You do not need to install or learn the CLI yourself. Paste this prompt into Cod
|
|
|
19
26
|
GitHub Copilot, Cursor, or another coding agent that has terminal access to your repository:
|
|
20
27
|
|
|
21
28
|
> Assess this repository's readiness for AI-agent pull-request work. From the repository root, run
|
|
22
|
-
> `npx --yes agentic-scorecard@0.
|
|
29
|
+
> `npx --yes agentic-scorecard@0.3.0 assess . --profile pr-creation --scope tracked --format markdown --output .agentic/reports/agentic-readiness-v0.3.0.md`.
|
|
23
30
|
> Do not change product source code or invent attestations. Read the resulting report and summarize
|
|
24
31
|
> the score, highest passed profile, whether `pr-creation` passes, target-profile blockers, and the
|
|
25
|
-
> five highest-value improvements.
|
|
26
|
-
>
|
|
27
|
-
> uncommitted unless I ask you to commit
|
|
32
|
+
> five highest-value improvements. Separate repository gaps from external and outcome evidence, and
|
|
33
|
+
> keep repository-detected, agent-collected, and human-attested evidence distinct. Flag likely false
|
|
34
|
+
> positives or negatives. Leave generated artifacts uncommitted unless I ask you to commit them.
|
|
28
35
|
|
|
29
36
|
That is the recommended first assessment. The agent downloads the pinned benchmark package, runs the
|
|
30
37
|
read-only local collector, and explains the results. For alternate authority levels or a guided
|
|
31
38
|
remediation session, use the prompts in [AGENT_PROMPT.md](AGENT_PROMPT.md).
|
|
32
39
|
|
|
40
|
+
## Feedback and community
|
|
41
|
+
|
|
42
|
+
Real-world assessment feedback helps make the benchmark more accurate and useful across different
|
|
43
|
+
engineering organizations and agent harnesses.
|
|
44
|
+
|
|
45
|
+
- Share false positives, false negatives, unclear controls, and adoption experience through the
|
|
46
|
+
[assessment feedback form](https://github.com/Planet-B2B/agentic-readiness/issues/new?template=assessment-feedback.yml).
|
|
47
|
+
- Propose scoring, control, evidence, or readiness-profile changes through the
|
|
48
|
+
[benchmark change form](https://github.com/Planet-B2B/agentic-readiness/issues/new?template=benchmark-change.yml).
|
|
49
|
+
- Send private adoption questions or feedback that cannot be shared publicly to
|
|
50
|
+
[benchmark@planetb2b.com](mailto:benchmark@planetb2b.com). Do not email vulnerability reports;
|
|
51
|
+
follow [SECURITY.md](SECURITY.md) instead.
|
|
52
|
+
|
|
53
|
+
Ask a coding agent to prepare privacy-safe feedback with this prompt:
|
|
54
|
+
|
|
55
|
+
> Review my Agentic Development Readiness report and draft feedback for the
|
|
56
|
+
> `Planet-B2B/agentic-readiness` repository. Include the benchmark version, target profile, relevant
|
|
57
|
+
> control IDs, observed result, expected result, and a minimal sanitized explanation. Distinguish a
|
|
58
|
+
> likely scanner defect from a proposed benchmark-policy change. Do not include credentials,
|
|
59
|
+
> proprietary source, private URLs, personal data, customer data, or full report contents. Show me
|
|
60
|
+
> the draft and ask for my approval before opening a GitHub issue.
|
|
61
|
+
|
|
62
|
+
See [FEEDBACK.md](FEEDBACK.md) for the full feedback guide. If the project is useful to you, please
|
|
63
|
+
[star the repository](https://github.com/Planet-B2B/agentic-readiness) so others can discover it.
|
|
64
|
+
|
|
33
65
|
## Quick start
|
|
34
66
|
|
|
35
67
|
### Installation and prerequisites
|
|
@@ -38,8 +70,9 @@ A supported Node.js LTS release (20.19+, 22.13+, or 24+) is required. Assessment
|
|
|
38
70
|
read-only, and offline by default.
|
|
39
71
|
|
|
40
72
|
```bash
|
|
41
|
-
npx agentic-scorecard@0.
|
|
73
|
+
npx agentic-scorecard@0.3.0 assess /path/to/repository \
|
|
42
74
|
--profile pr-creation \
|
|
75
|
+
--scope tracked \
|
|
43
76
|
--format markdown \
|
|
44
77
|
--output agentic-readiness.md
|
|
45
78
|
```
|
|
@@ -47,18 +80,46 @@ npx agentic-scorecard@0.1.0 assess /path/to/repository \
|
|
|
47
80
|
To record controls that repository inspection cannot prove:
|
|
48
81
|
|
|
49
82
|
```bash
|
|
50
|
-
npx agentic-scorecard@0.
|
|
83
|
+
npx agentic-scorecard@0.3.0 init /path/to/repository
|
|
84
|
+
```
|
|
85
|
+
|
|
86
|
+
Complete `.agentic/attestations.yaml` with owners and durable evidence links, then assess again.
|
|
87
|
+
Human-attested evidence remains visibly distinct in every report.
|
|
88
|
+
|
|
89
|
+
To let an authorized coding agent collect evidence from Git hosting, CI, dashboards, or other
|
|
90
|
+
external systems, first generate a target-bound template. The repository must be a Git worktree
|
|
91
|
+
with at least one commit so the bundle can bind to the exact assessed state:
|
|
92
|
+
|
|
93
|
+
```bash
|
|
94
|
+
npx agentic-scorecard@0.3.0 init-evidence /path/to/repository
|
|
51
95
|
```
|
|
52
96
|
|
|
53
|
-
|
|
54
|
-
|
|
97
|
+
Ask the agent to review `.agentic/evidence-request.md`, obtain approval before using
|
|
98
|
+
least-privileged read-only connectors, add attempted claims to `.agentic/agent-evidence.yaml`, and
|
|
99
|
+
rerun with `--agent-evidence`. The default bundle path is loaded automatically. See
|
|
100
|
+
[AGENT_PROMPT.md](AGENT_PROMPT.md) for the complete copy-and-paste workflow.
|
|
101
|
+
|
|
102
|
+
### Migrating from v0.1 or v0.2
|
|
103
|
+
|
|
104
|
+
Run a new tracked-scope baseline and retain the old report as historical evidence. Do not present the
|
|
105
|
+
score change as improvement or regression because v0.3 changes evidence and control semantics.
|
|
106
|
+
Re-review prior attestations and agent evidence before recreating them for v0.3. All claims must
|
|
107
|
+
expire, and repository-scoped semantic claims require tracked files to match the commit-bound target
|
|
108
|
+
plus `repo:<path>[#Lx-Ly]` references whose optional line ranges exist in the cited tracked file.
|
|
109
|
+
Untracked generated reports do not block this workflow.
|
|
110
|
+
|
|
111
|
+
The default `.agentic/attestations.yaml` and `.agentic/agent-evidence.yaml` paths are migration-safe:
|
|
112
|
+
if either contains an older benchmark version, v0.3 ignores that auto-loaded file and places a
|
|
113
|
+
prominent warning in the report. Regenerate it with `init --force` or `init-evidence --force` before
|
|
114
|
+
relying on those claims. An explicitly supplied `--attestations` or `--agent-evidence` file still
|
|
115
|
+
fails closed on a version mismatch.
|
|
55
116
|
|
|
56
117
|
For development from this checkout:
|
|
57
118
|
|
|
58
119
|
```bash
|
|
59
120
|
npm ci
|
|
60
121
|
npm run check
|
|
61
|
-
npm run dev -- assess tests/fixtures/mature --profile pr-creation
|
|
122
|
+
npm run dev -- assess tests/fixtures/mature --profile pr-creation --attestations tests/fixtures/mature/.agentic/attestations-v0.3.yaml
|
|
62
123
|
```
|
|
63
124
|
|
|
64
125
|
## What it assesses
|
|
@@ -104,30 +165,42 @@ No profile grants production deployment authority. Organizations should evaluate
|
|
|
104
165
|
through a separate, system-specific safety case.
|
|
105
166
|
|
|
106
167
|
The exact floors live in
|
|
107
|
-
[`benchmark/v0.
|
|
108
|
-
[`benchmark/v0.
|
|
168
|
+
[`benchmark/v0.3/benchmark.yaml`](benchmark/v0.3/benchmark.yaml) and their rationale in
|
|
169
|
+
[`benchmark/v0.3/scoring-policy.md`](benchmark/v0.3/scoring-policy.md).
|
|
170
|
+
|
|
171
|
+
## Evidence scopes and trust labels
|
|
172
|
+
|
|
173
|
+
- **Repository-detected:** the local collector found qualifying evidence in the selected path scope.
|
|
174
|
+
This proves an artifact match, not consistent practice or external enforcement.
|
|
175
|
+
- **Agent-collected:** an authorized agent supplied a target-bound, expiring, source-backed semantic
|
|
176
|
+
repository or external claim for an explicitly eligible control. It is not independently
|
|
177
|
+
verified and is never relabelled as repository-detected.
|
|
178
|
+
- **Human-attested:** an accountable owner supplied a dated evidence link or explanation.
|
|
179
|
+
- **Unknown:** evidence is unavailable, expired, unauthorized, mismatched, or inconclusive.
|
|
109
180
|
|
|
110
|
-
|
|
181
|
+
Controls also identify whether their evidence belongs in the repository, hosting platform,
|
|
182
|
+
organization, or outcome systems. This prevents expected external unknowns from masquerading as
|
|
183
|
+
missing files. Declarative detector adapters recognize common harness layouts without changing
|
|
184
|
+
portable control outcomes; future independently conformant external adapters require a separate
|
|
185
|
+
protocol. v0.3 does not issue certification or independent-verification claims.
|
|
111
186
|
|
|
112
|
-
|
|
113
|
-
- **Attested:** an accountable owner supplied an evidence link or explanation. This may satisfy a
|
|
114
|
-
control, but it never becomes verified evidence.
|
|
115
|
-
- **Unmet/unknown:** evidence is absent, expired, contradictory, or still requires review.
|
|
116
|
-
- **Independently reviewed:** reserved for a future external review protocol; the CLI does not issue
|
|
117
|
-
this label.
|
|
187
|
+
### Interpreting the two score views
|
|
118
188
|
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
189
|
+
The normative score remains `N/40` and drives readiness profiles. v0.3 also reports
|
|
190
|
+
`repository-detected progress: A/C`, where `C` is the maximum consecutive maturity the offline
|
|
191
|
+
collector can establish without platform, organization, outcome, agent-collected, or human-attested
|
|
192
|
+
evidence. This second view explains how complete the visible repository harness is; it does not
|
|
193
|
+
normalize away UNKNOWNs, change readiness floors, or certify the project.
|
|
123
194
|
|
|
124
195
|
## Reports and CI
|
|
125
196
|
|
|
126
197
|
The CLI emits Markdown for people and stable JSON for automation:
|
|
127
198
|
|
|
128
199
|
```bash
|
|
129
|
-
agentic-scorecard assess . --format json --output .agentic/report.json
|
|
200
|
+
agentic-scorecard assess . --scope tracked --format json --output .agentic/report.json
|
|
201
|
+
agentic-scorecard assess . --profile pr-creation --agent-evidence .agentic/agent-evidence.yaml
|
|
130
202
|
agentic-scorecard assess . --profile pr-creation --enforce
|
|
203
|
+
agentic-scorecard init-evidence .
|
|
131
204
|
agentic-scorecard explain ADRB-SEC-003
|
|
132
205
|
agentic-scorecard validate
|
|
133
206
|
```
|
|
@@ -142,11 +215,13 @@ funded the remediation plan. See
|
|
|
142
215
|
The default collector:
|
|
143
216
|
|
|
144
217
|
- makes no network calls and invokes no model;
|
|
145
|
-
- reads only
|
|
146
|
-
- ignores `.git`,
|
|
147
|
-
|
|
218
|
+
- reads only Git-tracked paths and records commit and dirty-worktree metadata;
|
|
219
|
+
- ignores `.git`, dependencies, build output, coverage, generated reports, attestations, and imported
|
|
220
|
+
evidence bundles;
|
|
221
|
+
- caps content scanning at 512 KB per file, 5 MB total, and 250 candidates per check, with v0.3
|
|
222
|
+
per-pattern balancing where configured;
|
|
148
223
|
- reports file paths and match counts, never matching source snippets;
|
|
149
|
-
- writes nothing unless `--output` or `init` is explicitly requested.
|
|
224
|
+
- writes nothing unless `--output`, `init`, or `init-evidence` is explicitly requested.
|
|
150
225
|
|
|
151
226
|
Do not place credentials, private prompts, source excerpts, or personal data in attestations. Link to
|
|
152
227
|
access-controlled evidence instead. Report suspected vulnerabilities through [SECURITY.md](SECURITY.md).
|
|
@@ -154,7 +229,10 @@ access-controlled evidence instead. Report suspected vulnerabilities through [SE
|
|
|
154
229
|
## Repository structure
|
|
155
230
|
|
|
156
231
|
```text
|
|
157
|
-
benchmark/v0.1/ immutable
|
|
232
|
+
benchmark/v0.1/ immutable historical v0.1 definition
|
|
233
|
+
benchmark/v0.2/ immutable historical v0.2 definition
|
|
234
|
+
benchmark/v0.3/ current normative benchmark, schemas, and controls
|
|
235
|
+
adapters/ declarative path/term aliases for recognized agent harness layouts
|
|
158
236
|
benchmark/mappings/ informative mappings to external frameworks
|
|
159
237
|
src/ reference CLI and local evidence collectors
|
|
160
238
|
templates/ adoption, preflight, attestation, and remediation templates
|
|
@@ -171,21 +249,24 @@ OIDC-provenance release process.
|
|
|
171
249
|
|
|
172
250
|
1. Run a local baseline for the least-authoritative profile you actually need.
|
|
173
251
|
2. Review every result with security, platform, and representative delivery teams.
|
|
174
|
-
3.
|
|
175
|
-
4.
|
|
176
|
-
5.
|
|
177
|
-
6.
|
|
178
|
-
7.
|
|
252
|
+
3. Use an authorized agent to collect source-backed external evidence where appropriate.
|
|
253
|
+
4. Supply owned human attestations only where deterministic collection is unavailable.
|
|
254
|
+
5. Publish the report internally with explicit limitations, scope, commit, and benchmark version.
|
|
255
|
+
6. Fund the smallest improvements that close target-profile blockers.
|
|
256
|
+
7. Reassess on material harness changes and at least quarterly.
|
|
257
|
+
8. Add a non-blocking CI report; enforce only the agreed target profile after a pilot.
|
|
179
258
|
|
|
180
259
|
Never optimize to the number alone. Use the control evidence and outcome metrics to improve the
|
|
181
260
|
system, and keep exceptions narrow, owned, expiring, and visible.
|
|
182
261
|
|
|
183
262
|
## Status and roadmap
|
|
184
263
|
|
|
185
|
-
v0.
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
264
|
+
v0.3 adds portable harness discovery with declarative detector adapters, proximity-bounded content evidence, prioritized candidate
|
|
265
|
+
selection for large repositories, source-backed semantic repository claims, and an explanatory
|
|
266
|
+
repository-only score and ceiling. Candidate next steps
|
|
267
|
+
include conformant signed adapters, SARIF/HTML reports, organization-level aggregation,
|
|
268
|
+
statistically designed benchmark tasks, and an independent-review protocol. These require public
|
|
269
|
+
design review before becoming normative.
|
|
189
270
|
|
|
190
271
|
Apache-2.0 licensed. The benchmark is a community engineering tool, not legal, compliance, or
|
|
191
272
|
security advice.
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
# v0.2 external evidence contract
|
|
2
|
+
|
|
3
|
+
The core scanner is local, read-only, network-free, and vendor-neutral. An agent or adapter may
|
|
4
|
+
collect evidence that the repository cannot establish, then provide it through the versioned agent
|
|
5
|
+
evidence bundle.
|
|
6
|
+
|
|
7
|
+
Each claim contains:
|
|
8
|
+
|
|
9
|
+
- benchmark version and control ID;
|
|
10
|
+
- repository target and optional commit binding;
|
|
11
|
+
- `met`, `not_met`, or `unknown` status;
|
|
12
|
+
- platform, organization, or outcome scope;
|
|
13
|
+
- collector name and version;
|
|
14
|
+
- collection timestamp and mandatory expiry;
|
|
15
|
+
- privacy-safe references, never credentials or raw sensitive content;
|
|
16
|
+
- a concise derivation summary; and
|
|
17
|
+
- an optional error that is distinguishable from a negative result.
|
|
18
|
+
|
|
19
|
+
Collectors must be read-only, least-privileged, bounded to the approved target, time-limited, and
|
|
20
|
+
fail closed. Network content is untrusted input: it cannot alter benchmark policy, prompts, tool
|
|
21
|
+
authority, or the set of controls being assessed.
|
|
22
|
+
|
|
23
|
+
Agent-collected claims remain visibly distinct from repository evidence and human attestations. A
|
|
24
|
+
future conformant-adapter class may add signed identity and independent conformance tests; v0.2 does
|
|
25
|
+
not issue an independently verified label.
|
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
{
|
|
2
|
+
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
|
3
|
+
"$id": "https://agentic-readiness.dev/schema/v0.2/agent-evidence.json",
|
|
4
|
+
"title": "ADRB v0.2 agent-collected evidence bundle",
|
|
5
|
+
"type": "object",
|
|
6
|
+
"additionalProperties": false,
|
|
7
|
+
"required": ["schema_version", "benchmark_version", "target", "collector", "claims"],
|
|
8
|
+
"properties": {
|
|
9
|
+
"schema_version": { "const": "0.2.0" },
|
|
10
|
+
"benchmark_version": { "const": "0.2.0" },
|
|
11
|
+
"target": {
|
|
12
|
+
"type": "object",
|
|
13
|
+
"additionalProperties": false,
|
|
14
|
+
"required": ["repository", "git_head"],
|
|
15
|
+
"properties": {
|
|
16
|
+
"repository": { "type": "string", "minLength": 1 },
|
|
17
|
+
"git_head": { "type": ["string", "null"], "minLength": 1 }
|
|
18
|
+
}
|
|
19
|
+
},
|
|
20
|
+
"collector": {
|
|
21
|
+
"type": "object",
|
|
22
|
+
"additionalProperties": false,
|
|
23
|
+
"required": ["name", "version"],
|
|
24
|
+
"properties": {
|
|
25
|
+
"name": { "type": "string", "minLength": 1 },
|
|
26
|
+
"version": { "type": "string", "minLength": 1 }
|
|
27
|
+
}
|
|
28
|
+
},
|
|
29
|
+
"claims": {
|
|
30
|
+
"type": "object",
|
|
31
|
+
"propertyNames": { "pattern": "^ADRB-[A-Z]{3}-[0-9]{3}$" },
|
|
32
|
+
"additionalProperties": {
|
|
33
|
+
"type": "object",
|
|
34
|
+
"additionalProperties": false,
|
|
35
|
+
"required": ["status", "scope", "summary", "references", "collected_at", "expires_at"],
|
|
36
|
+
"properties": {
|
|
37
|
+
"status": { "enum": ["met", "not_met", "unknown"] },
|
|
38
|
+
"scope": { "enum": ["platform", "organization", "outcome"] },
|
|
39
|
+
"summary": { "type": "string", "minLength": 1 },
|
|
40
|
+
"references": {
|
|
41
|
+
"type": "array",
|
|
42
|
+
"minItems": 1,
|
|
43
|
+
"items": { "type": "string", "minLength": 1 }
|
|
44
|
+
},
|
|
45
|
+
"collected_at": { "type": "string", "format": "date-time" },
|
|
46
|
+
"expires_at": { "type": "string", "format": "date-time" },
|
|
47
|
+
"error": { "type": ["string", "null"], "minLength": 1 }
|
|
48
|
+
}
|
|
49
|
+
}
|
|
50
|
+
}
|
|
51
|
+
}
|
|
52
|
+
}
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
{
|
|
2
|
+
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
|
3
|
+
"$id": "https://agentic-readiness.dev/schema/v0.2/attestations.json",
|
|
4
|
+
"title": "ADRB v0.2 human attestation file",
|
|
5
|
+
"type": "object",
|
|
6
|
+
"additionalProperties": false,
|
|
7
|
+
"required": ["benchmark_version", "attestations"],
|
|
8
|
+
"properties": {
|
|
9
|
+
"benchmark_version": { "const": "0.2.0" },
|
|
10
|
+
"attestations": {
|
|
11
|
+
"type": "object",
|
|
12
|
+
"additionalProperties": {
|
|
13
|
+
"type": "object",
|
|
14
|
+
"additionalProperties": false,
|
|
15
|
+
"required": ["status", "evidence", "owner", "reviewed_at", "expires_at"],
|
|
16
|
+
"properties": {
|
|
17
|
+
"status": { "enum": ["met", "not_met", "not_applicable", "unknown"] },
|
|
18
|
+
"evidence": { "type": "string", "minLength": 1 },
|
|
19
|
+
"owner": { "type": "string", "minLength": 1 },
|
|
20
|
+
"reviewed_at": { "type": "string", "format": "date" },
|
|
21
|
+
"expires_at": { "type": "string", "format": "date" }
|
|
22
|
+
}
|
|
23
|
+
}
|
|
24
|
+
}
|
|
25
|
+
}
|
|
26
|
+
}
|