agentic-scorecard 0.1.0 → 0.3.0

package/benchmark/v0.2/controls/testing.yaml

@@ -0,0 +1,60 @@
+dimension: testing
+controls:
+  - id: ADRB-TST-001
+    level: 1
+    title: Automated tests exist
+    outcome: The tracked repository contains executable tests for at least one important behavior.
+    risk: Agent changes rely on plausibility or manual inspection alone.
+    evidence:
+      - type: path_any
+        scope: repository
+        patterns: ['**/*.test.*', '**/*.spec.*', tests/**, test/**]
+    remediation: Add a fast automated test for a critical behavior and document how to run it.
+  - id: ADRB-TST-002
+    level: 2
+    title: Verification layers and canonical commands are documented
+    outcome: Contributors can identify applicable test layers and the canonical verification path.
+    risk: Agents run a convenient subset and overstate confidence in the result.
+    evidence:
+      - type: content_terms
+        scope: repository
+        files: [README.md, CONTRIBUTING.md, AGENTS.md, docs/testing/**, package.json, Makefile]
+        terms: [unit test, integration test, end-to-end, e2e, typecheck, lint, test]
+        min_terms: 2
+    remediation: Define the test taxonomy and one canonical command that matches CI.
+  - id: ADRB-TST-003
+    level: 3
+    title: CI executes the required verification layers
+    outcome: Repository automation runs appropriate tests and static checks from a clean state.
+    risk: Documented verification can be omitted from automation or run in an unrepresentative environment.
+    evidence:
+      - type: content_terms
+        scope: repository
+        files: [.github/workflows/**, .gitlab-ci.yml, Jenkinsfile, azure-pipelines.yml]
+        terms: [test, typecheck, lint, coverage, e2e, integration]
+        min_terms: 2
+    remediation: Configure CI to run the applicable verification layers from a clean environment.
+  - id: ADRB-TST-005
+    level: 3
+    title: Verification is enforced before merge
+    outcome: Merge policy requires the applicable CI checks and restricts bypasses.
+    risk: Healthy CI configuration remains optional under delivery pressure.
+    allow_attestation: true
+    allow_agent_evidence: true
+    evidence:
+      - type: manual
+        scope: platform
+        prompt: Show required status checks or equivalent merge gates, covered branches, bypass actors, and exception controls.
+    remediation: Make appropriate CI checks required merge gates and restrict, record, and review bypasses.
+  - id: ADRB-TST-004
+    level: 4
+    title: Verification effectiveness drives improvement
+    outcome: Escaped defects, flakiness, and other signal-quality outcomes are trended and improve testing.
+    risk: Test count and coverage rise while meaningful defects still escape.
+    allow_attestation: true
+    allow_agent_evidence: true
+    evidence:
+      - type: manual
+        scope: outcome
+        prompt: Provide verification outcome trends by task class and a recent test-strategy improvement driven by them.
+    remediation: Trend escaped defects and signal quality, then invest in the weakest verification layers.

package/benchmark/v0.2/controls/tooling.yaml

@@ -0,0 +1,58 @@
+dimension: tooling
+controls:
+  - id: ADRB-TOL-001
+    level: 1
+    title: Supported agent tools are declared
+    outcome: The tracked repository identifies supported agent interfaces or tool configuration.
+    risk: Agents discover tools opportunistically and use unsupported or unsafe paths.
+    evidence:
+      - type: path_any
+        scope: repository
+        patterns:
+          [
+            .codex/**,
+            .claude/**,
+            .agents/**,
+            .cursor/**,
+            .github/copilot-instructions.md,
+            mcp.json,
+            .mcp.json,
+          ]
+    remediation: Commit a non-empty declaration of supported agent surfaces and their intended use.
+  - id: ADRB-TOL-002
+    level: 2
+    title: Tool scope and failure behavior are documented
+    outcome: Tool guidance defines authority, validation, success signals, failure handling, and approved fallback.
+    risk: Agents bypass failures, widen scope, or mistake partial results for success.
+    evidence:
+      - type: content_terms
+        scope: repository
+        files:
+          [AGENTS.md, CLAUDE.md, .agents/**/*.md, .claude/**/*.md, docs/agentic/**, docs/tools/**]
+        terms: [authority, permission, read-only, validation, failure, fallback, approval]
+        min_terms: 3
+    remediation: Document each high-impact tool's authority, inputs, validation, failure mode, and fallback.
+  - id: ADRB-TOL-003
+    level: 3
+    title: High-impact tool authority is technically bounded and audited
+    outcome: High-impact tools enforce least privilege, authorization, structured validation, and audit records.
+    risk: Prompt instructions become the only barrier between an agent and an irreversible action.
+    allow_attestation: true
+    allow_agent_evidence: true
+    evidence:
+      - type: manual
+        scope: platform
+        prompt: Show technical least-privilege enforcement, authorization tests, validation, audit records, and fail-closed behavior for high-impact tools.
+    remediation: Enforce scoped credentials, authorization, schema validation, audit logging, and deny-by-default behavior.
+  - id: ADRB-TOL-004
+    level: 4
+    title: Tool reliability drives improvement
+    outcome: Tool success, partial failure, denial, latency, and unsafe-fallback attempts are trended and improved.
+    risk: Tooling quality and risky workarounds remain anecdotal until an incident occurs.
+    allow_attestation: true
+    allow_agent_evidence: true
+    evidence:
+      - type: manual
+        scope: outcome
+        prompt: Provide tool reliability trends and a recent tool or fallback improvement driven by them.
+    remediation: Trend tool outcomes by task and risk class and improve weak or bypassed interfaces.

package/benchmark/v0.2/report-schema.json

@@ -0,0 +1,47 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "$id": "https://agentic-readiness.dev/schema/v0.2/report.json",
+  "title": "ADRB v0.2 assessment report",
+  "type": "object",
+  "required": [
+    "schema_version",
+    "benchmark",
+    "target",
+    "assessed_at",
+    "score",
+    "evidence_summary",
+    "dimensions",
+    "controls",
+    "profiles",
+    "readiness",
+    "limitations"
+  ],
+  "properties": {
+    "schema_version": { "const": "0.2.0" },
+    "benchmark": {
+      "type": "object",
+      "required": ["id", "version"],
+      "properties": { "version": { "const": "0.2.0" } }
+    },
+    "target": {
+      "type": "object",
+      "required": [
+        "repository",
+        "profile",
+        "scope",
+        "git_head",
+        "git_remote",
+        "working_tree_dirty"
+      ],
+      "properties": { "scope": { "enum": ["tracked", "workspace"] } }
+    },
+    "assessed_at": { "type": "string", "format": "date-time" },
+    "score": { "type": "object" },
+    "evidence_summary": { "type": "object" },
+    "dimensions": { "type": "array", "minItems": 10, "maxItems": 10 },
+    "controls": { "type": "array" },
+    "profiles": { "type": "array" },
+    "readiness": { "type": "object" },
+    "limitations": { "type": "array", "items": { "type": "string" } }
+  }
+}

package/benchmark/v0.2/scoring-policy.md

@@ -0,0 +1,71 @@
+# v0.2 scoring policy
+
+This document is normative for benchmark version 0.2.0.
+
+## Unit and assessment scope
+
+The unit is one repository plus explicitly supplied external and organizational evidence bound to
+that repository. The default `tracked` mode considers only Git-tracked paths, using their current
+working-tree contents. Reports record the commit, remote, and dirty state. `workspace` mode is
+available for provisional assessment and is not directly comparable with tracked-mode reports.
+
+Generated reports, attestations, evidence requests, and imported evidence bundles never qualify as
+repository evidence. An assessment must be idempotent: adding its own generated artifacts cannot
+change its score.
+
+## Dimension scores
+
+Each dimension earns level `n` only when every applicable control at that level and every lower
+level is met. More than one control may exist at a level when separate observable outcomes are
+required. Controls met above the first gap are reported but do not increase the dimension score.
+
+The overall score is the sum of ten dimension scores, with a maximum of 40. The percentage is
+display-only and does not determine readiness.
+
+## Readiness decisions
+
+Each autonomy profile defines a floor for every dimension. A profile passes only if every floor is
+met. Strength in one dimension cannot compensate for a security, testing, governance, or recovery
+gap elsewhere. No profile grants deployment or production authority.
+
+## Evidence scopes and provenance
+
+- `repository-detected`: a deterministic collector found qualifying evidence in the selected path
+  scope. It proves the reported artifact match, not consistent human behavior or external
+  enforcement.
+- `agent-collected`: an authorized agent supplied a structured, source-backed external claim. It may
+  satisfy controls explicitly eligible for external evidence but is never independently verified.
+- `attested`: an accountable person supplied a dated, expiring claim and durable reference.
+- `unknown`: evidence is unavailable, expired, mismatched, unauthorized, or inconclusive.
+
+Repository controls use non-empty files and control-specific, co-located term thresholds. External
+platform, organization, and outcome controls cannot be inferred from repository keywords.
+Human attestations and agent evidence may satisfy only controls explicitly eligible for those
+classes; v0.2 repository-artifact controls are not overridable by declaration.
+
+Level 4 requires time-series or outcome evidence and an improvement decision influenced by it. The
+existence of a dashboard is insufficient.
+
+## Agent evidence bundles
+
+An agent evidence bundle is version-bound and target-bound. Claims record scope, status, collector,
+collection and expiry times, a privacy-safe derivation summary, and durable references. The core
+scanner makes no network calls; the agent obtains explicit authorization and uses read-only tools to
+create the bundle. An error is distinct from a negative result and cannot pass a control.
+
+Agent evidence can satisfy only controls with `allow_agent_evidence: true`. It cannot override a
+failed repository artifact control.
+
+## Changes and comparability
+
+v0.1 remains immutable. v0.1 and v0.2 scores are not directly comparable because evidence and
+control semantics changed. Reports are comparable only when benchmark version, target profile,
+repository scope, evidence policy, and assessment mode match.
+
+## Claims
+
+Allowed: “We self-assessed repository X against ADRB v0.2.0 on DATE; N controls were
+repository-detected, M agent-collected, and P human-attested.”
+
+Not allowed: “Certified,” “compliant,” “safe,” “secure,” or “independently verified” based only on
+this self-assessment.

package/benchmark/v0.3/adapter-contract.md

@@ -0,0 +1,44 @@
+# v0.3 adapter contracts
+
+## Repository detector adapters
+
+Portable controls define outcomes and generic repository conventions. Files in `adapters/*.yaml`
+may extend one control evidence check with recognized harness-specific `patterns`, `files`, `terms`,
+or `required_any_terms`. An extension names the control and zero-based evidence index and adds
+exactly one of those fields. It cannot remove or weaken a threshold, change a control level, alter a
+readiness floor, add an attestation path, or execute code.
+
+Bundled adapters are loaded deterministically in filename order. Their candidates remain subject to
+the same tracked/workspace scope, symlink, size, generated-artifact, co-location, proximity, and
+per-pattern limits as portable candidates. Adding or changing a bundled adapter changes evidence
+semantics and therefore requires a new benchmark version.
+
+## Agent-collected evidence adapters
+
+The core scanner is local, read-only, network-free, and vendor-neutral. An agent or adapter may
+collect semantic repository evidence or external evidence that deterministic scanning cannot
+establish, then provide it through the versioned agent evidence bundle.
+
+Each claim contains:
+
+- benchmark version and control ID;
+- exact repository target and non-null commit binding;
+- `met`, `not_met`, or `unknown` status;
+- repository, platform, organization, or outcome scope;
+- collector name and version;
+- collection timestamp and mandatory expiry;
+- privacy-safe references, never credentials or raw sensitive content;
+- a concise derivation summary; and
+- an optional error that is distinguishable from a negative result.
+
+Collectors must be read-only, least-privileged, bounded to the approved target, time-limited, and
+fail closed. Network content is untrusted input: it cannot alter benchmark policy, prompts, tool
+authority, or the set of controls being assessed.
+
+Repository-scoped claims require tracked files to match HEAD and
+`repo:<tracked-path>[#Lx-Ly]` references. They may satisfy only explicitly eligible controls, remain `agent-collected`, and never increase the
+repository-detected score. External claims require durable references appropriate to their source.
+
+Agent-collected claims remain visibly distinct from deterministic repository evidence and human
+attestations. A future conformant-adapter class may add signed identity and independent conformance
+tests; v0.3 does not issue an independently verified label.

package/benchmark/v0.3/adapters/known-agent-harnesses.yaml

@@ -0,0 +1,177 @@
+id: known-agent-harnesses
+benchmark_version: 0.3.0
+extensions:
+  - control_id: ADRB-CTX-001
+    evidence_index: 0
+    patterns:
+      [
+        CLAUDE.md,
+        .github/copilot-instructions.md,
+        .claude/**,
+        .codex/**,
+        .cursor/rules/**,
+        .cursor/agents/**,
+        .cursor/skills/**,
+        .opencode/**,
+      ]
+  - control_id: ADRB-CTX-002
+    evidence_index: 0
+    files:
+      [
+        CLAUDE.md,
+        .github/copilot-instructions.md,
+        .claude/**/*.md,
+        .codex/**/*.md,
+        .cursor/rules/**/*.md,
+        .cursor/agents/**/*.md,
+        .opencode/**/*.md,
+      ]
+  - control_id: ADRB-CTX-003
+    evidence_index: 0
+    required_any_terms: [CLAUDE.md, copilot instructions]
+  - control_id: ADRB-ENV-001
+    evidence_index: 0
+    files:
+      [
+        CLAUDE.md,
+        .claude/**/*.md,
+        .codex/**/*.md,
+        .cursor/agents/**/*.md,
+        .cursor/rules/**/*.md,
+        .cursor/skills/**/*.md,
+        .opencode/**/*.md,
+      ]
+  - control_id: ADRB-SPC-001
+    evidence_index: 0
+    patterns: [.kiro/specs/**, .cursor/plans/**, .codex/plans/**]
+  - control_id: ADRB-SPC-002
+    evidence_index: 0
+    files: [.kiro/specs/**, .cursor/plans/**, .codex/plans/**]
+  - control_id: ADRB-TOL-001
+    evidence_index: 0
+    patterns:
+      [
+        .claude/**,
+        .codex/**,
+        .cursor/**,
+        .opencode/**,
+        .github/copilot-instructions.md,
+        opencode.json,
+      ]
+  - control_id: ADRB-TOL-002
+    evidence_index: 0
+    files:
+      [
+        CLAUDE.md,
+        .claude/**/*.md,
+        .codex/**/*.md,
+        .cursor/agents/**/*.md,
+        .cursor/rules/**/*.md,
+        .cursor/skills/**/*.md,
+        .opencode/**/*.md,
+      ]
+  - control_id: ADRB-SEC-001
+    evidence_index: 0
+    files:
+      [
+        CLAUDE.md,
+        .claude/**/*.md,
+        .codex/**/*.md,
+        .cursor/agents/**/*.md,
+        .cursor/rules/**/*.md,
+        .cursor/skills/**/*.md,
+        .opencode/**/*.md,
+      ]
+  - control_id: ADRB-SEC-002
+    evidence_index: 0
+    files:
+      [
+        CLAUDE.md,
+        .claude/**/*.md,
+        .codex/**/*.md,
+        .cursor/agents/**/*.md,
+        .cursor/rules/**/*.md,
+        .cursor/skills/**/*.md,
+        .opencode/**/*.md,
+      ]
+  - control_id: ADRB-TST-002
+    evidence_index: 0
+    files:
+      [
+        CLAUDE.md,
+        .claude/**/*.md,
+        .codex/**/*.md,
+        .cursor/agents/**/*.md,
+        .cursor/rules/**/*.md,
+        .cursor/skills/**/*.md,
+        .opencode/**/*.md,
+      ]
+  - control_id: ADRB-GOV-001
+    evidence_index: 0
+    files:
+      [
+        CLAUDE.md,
+        .claude/**/*.md,
+        .codex/**/*.md,
+        .cursor/agents/**/*.md,
+        .cursor/rules/**/*.md,
+        .cursor/skills/**/*.md,
+        .opencode/**/*.md,
+      ]
+  - control_id: ADRB-GOV-002
+    evidence_index: 1
+    files:
+      [
+        CLAUDE.md,
+        .claude/**/*.md,
+        .codex/**/*.md,
+        .cursor/agents/**/*.md,
+        .cursor/rules/**/*.md,
+        .cursor/skills/**/*.md,
+        .opencode/**/*.md,
+      ]
+  - control_id: ADRB-LRN-001
+    evidence_index: 0
+    patterns: [.claude/memory/**, .codex/memory/**, .cursor/memory/**, .opencode/memory/**]
+  - control_id: ADRB-LRN-002
+    evidence_index: 0
+    files:
+      [
+        CLAUDE.md,
+        .claude/**/*.md,
+        .codex/**/*.md,
+        .cursor/agents/**/*.md,
+        .cursor/rules/**/*.md,
+        .cursor/skills/**/*.md,
+        .opencode/**/*.md,
+      ]
+  - control_id: ADRB-OBS-001
+    evidence_index: 0
+    files: [.cursor/runs/README*, .codex/runs/README*, .claude/runs/README*]
+  - control_id: ADRB-OBS-002
+    evidence_index: 0
+    files: [.cursor/runs/README*, .codex/runs/README*, .claude/runs/README*]
+  - control_id: ADRB-RES-001
+    evidence_index: 0
+    files:
+      [
+        CLAUDE.md,
+        .claude/**/*.md,
+        .codex/**/*.md,
+        .cursor/agents/**/*.md,
+        .cursor/rules/**/*.md,
+        .cursor/skills/**/*.md,
+        .opencode/**/*.md,
+      ]
+  - control_id: ADRB-RES-002
+    evidence_index: 0
+    files:
+      [
+        CLAUDE.md,
+        .claude/**/*.md,
+        .codex/**/*.md,
+        .cursor/agents/**/*.md,
+        .cursor/rules/**/*.md,
+        .cursor/skills/**/*.md,
+        .opencode/**/*.md,
+      ]

package/benchmark/v0.3/adapters/known-python-tooling.yaml

@@ -0,0 +1,18 @@
+id: known-python-tooling
+benchmark_version: 0.3.0
+extensions:
+  - control_id: ADRB-ENV-001
+    evidence_index: 0
+    files: [pyproject.toml]
+  - control_id: ADRB-ENV-001
+    evidence_index: 0
+    terms: [requires-python, uv sync, python]
+  - control_id: ADRB-TST-002
+    evidence_index: 0
+    files: [run_quality_checks.sh, scripts/**/quality*.sh, tox.ini, noxfile.py]
+  - control_id: ADRB-TST-002
+    evidence_index: 0
+    terms: [pytest, mypy, flake8, ruff, pyright, tox, nox]
+  - control_id: ADRB-TST-003
+    evidence_index: 0
+    terms: [pytest, mypy, flake8, ruff, pyright, tox, nox, pytest-cov]

package/benchmark/v0.3/adapters/known-security-tools.yaml

@@ -0,0 +1,6 @@
+id: known-security-tools
+benchmark_version: 0.3.0
+extensions:
+  - control_id: ADRB-SEC-003
+    evidence_index: 0
+    terms: [gitleaks, trufflehog]

package/benchmark/v0.3/agent-evidence-schema.json

@@ -0,0 +1,67 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "$id": "https://agentic-readiness.dev/schema/v0.3/agent-evidence.json",
+  "title": "ADRB v0.3 agent-collected evidence bundle",
+  "type": "object",
+  "additionalProperties": false,
+  "required": ["schema_version", "benchmark_version", "target", "collector", "claims"],
+  "properties": {
+    "schema_version": { "const": "0.3.0" },
+    "benchmark_version": { "const": "0.3.0" },
+    "target": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": ["repository", "git_head"],
+      "properties": {
+        "repository": { "type": "string", "minLength": 1 },
+        "git_head": { "type": "string", "minLength": 1 }
+      }
+    },
+    "collector": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": ["name", "version"],
+      "properties": {
+        "name": { "type": "string", "minLength": 1 },
+        "version": { "type": "string", "minLength": 1 }
+      }
+    },
+    "claims": {
+      "type": "object",
+      "propertyNames": { "pattern": "^ADRB-[A-Z]{3}-[0-9]{3}$" },
+      "additionalProperties": {
+        "type": "object",
+        "additionalProperties": false,
+        "required": ["status", "scope", "summary", "references", "collected_at", "expires_at"],
+        "properties": {
+          "status": { "enum": ["met", "not_met", "unknown"] },
+          "scope": { "enum": ["repository", "platform", "organization", "outcome"] },
+          "summary": { "type": "string", "minLength": 1 },
+          "references": {
+            "type": "array",
+            "minItems": 1,
+            "items": { "type": "string", "minLength": 1 }
+          },
+          "collected_at": { "type": "string", "format": "date-time" },
+          "expires_at": { "type": "string", "format": "date-time" },
+          "error": { "type": ["string", "null"], "minLength": 1 }
+        },
+        "allOf": [
+          {
+            "if": { "properties": { "scope": { "const": "repository" } } },
+            "then": {
+              "properties": {
+                "references": {
+                  "items": {
+                    "type": "string",
+                    "pattern": "^repo:(?!/)(?!\\.{1,2}(?:/|#|$))(?!.*\\/\\.{1,2}(?:/|#|$))(?!.*\\\\)[^#]+(?:#L[1-9][0-9]*(?:-L?[1-9][0-9]*)?)?$"
+                  }
+                }
+              }
+            }
+          }
+        ]
+      }
+    }
+  }
+}

package/benchmark/v0.3/attestation-schema.json

@@ -0,0 +1,26 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "$id": "https://agentic-readiness.dev/schema/v0.3/attestations.json",
+  "title": "ADRB v0.3 human attestation file",
+  "type": "object",
+  "additionalProperties": false,
+  "required": ["benchmark_version", "attestations"],
+  "properties": {
+    "benchmark_version": { "const": "0.3.0" },
+    "attestations": {
+      "type": "object",
+      "additionalProperties": {
+        "type": "object",
+        "additionalProperties": false,
+        "required": ["status", "evidence", "owner", "reviewed_at", "expires_at"],
+        "properties": {
+          "status": { "enum": ["met", "not_met", "not_applicable", "unknown"] },
+          "evidence": { "type": "string", "minLength": 1 },
+          "owner": { "type": "string", "minLength": 1 },
+          "reviewed_at": { "type": "string", "format": "date" },
+          "expires_at": { "type": "string", "format": "date" }
+        }
+      }
+    }
+  }
+}